pvcalls-front.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * (c) 2017 Stefano Stabellini <stefano@aporeto.com>
   4 */
   5
   6#include <linux/module.h>
   7#include <linux/net.h>
   8#include <linux/socket.h>
   9
  10#include <net/sock.h>
  11
  12#include <xen/events.h>
  13#include <xen/grant_table.h>
  14#include <xen/xen.h>
  15#include <xen/xenbus.h>
  16#include <xen/interface/io/pvcalls.h>
  17
  18#include "pvcalls-front.h"
  19
  20#define PVCALLS_INVALID_ID UINT_MAX
  21#define PVCALLS_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER
  22#define PVCALLS_NR_RSP_PER_RING __CONST_RING_SIZE(xen_pvcalls, XEN_PAGE_SIZE)
  23#define PVCALLS_FRONT_MAX_SPIN 5000
  24
  25static struct proto pvcalls_proto = {
  26	.name	= "PVCalls",
  27	.owner	= THIS_MODULE,
  28	.obj_size = sizeof(struct sock),
  29};
  30
  31struct pvcalls_bedata {
  32	struct xen_pvcalls_front_ring ring;
  33	grant_ref_t ref;
  34	int irq;
  35
  36	struct list_head socket_mappings;
  37	spinlock_t socket_lock;
  38
  39	wait_queue_head_t inflight_req;
  40	struct xen_pvcalls_response rsp[PVCALLS_NR_RSP_PER_RING];
  41};
  42/* Only one front/back connection supported. */
  43static struct xenbus_device *pvcalls_front_dev;
  44static atomic_t pvcalls_refcount;
  45
  46/* first increment refcount, then proceed */
  47#define pvcalls_enter() {               \
  48	atomic_inc(&pvcalls_refcount);      \
  49}
  50
  51/* first complete other operations, then decrement refcount */
  52#define pvcalls_exit() {                \
  53	atomic_dec(&pvcalls_refcount);      \
  54}
  55
  56struct sock_mapping {
  57	bool active_socket;
  58	struct list_head list;
  59	struct socket *sock;
  60	atomic_t refcount;
  61	union {
  62		struct {
  63			int irq;
  64			grant_ref_t ref;
  65			struct pvcalls_data_intf *ring;
  66			struct pvcalls_data data;
  67			struct mutex in_mutex;
  68			struct mutex out_mutex;
  69
  70			wait_queue_head_t inflight_conn_req;
  71		} active;
  72		struct {
  73		/*
  74		 * Socket status, needs to be 64-bit aligned due to the
  75		 * test_and_* functions which have this requirement on arm64.
  76		 */
  77#define PVCALLS_STATUS_UNINITALIZED  0
  78#define PVCALLS_STATUS_BIND          1
  79#define PVCALLS_STATUS_LISTEN        2
  80			uint8_t status __attribute__((aligned(8)));
  81		/*
  82		 * Internal state-machine flags.
  83		 * Only one accept operation can be inflight for a socket.
  84		 * Only one poll operation can be inflight for a given socket.
  85		 * flags needs to be 64-bit aligned due to the test_and_*
  86		 * functions which have this requirement on arm64.
  87		 */
  88#define PVCALLS_FLAG_ACCEPT_INFLIGHT 0
  89#define PVCALLS_FLAG_POLL_INFLIGHT   1
  90#define PVCALLS_FLAG_POLL_RET        2
  91			uint8_t flags __attribute__((aligned(8)));
  92			uint32_t inflight_req_id;
  93			struct sock_mapping *accept_map;
  94			wait_queue_head_t inflight_accept_req;
  95		} passive;
  96	};
  97};
  98
  99static inline struct sock_mapping *pvcalls_enter_sock(struct socket *sock)
 100{
 101	struct sock_mapping *map;
 102
 103	if (!pvcalls_front_dev ||
 104		dev_get_drvdata(&pvcalls_front_dev->dev) == NULL)
 105		return ERR_PTR(-ENOTCONN);
 106
 107	map = (struct sock_mapping *)sock->sk->sk_send_head;
 108	if (map == NULL)
 109		return ERR_PTR(-ENOTSOCK);
 110
 111	pvcalls_enter();
 112	atomic_inc(&map->refcount);
 113	return map;
 114}
 115
 116static inline void pvcalls_exit_sock(struct socket *sock)
 117{
 118	struct sock_mapping *map;
 119
 120	map = (struct sock_mapping *)sock->sk->sk_send_head;
 121	atomic_dec(&map->refcount);
 122	pvcalls_exit();
 123}
 124
 125static inline int get_request(struct pvcalls_bedata *bedata, int *req_id)
 126{
 127	*req_id = bedata->ring.req_prod_pvt & (RING_SIZE(&bedata->ring) - 1);
 128	if (RING_FULL(&bedata->ring) ||
 129	    bedata->rsp[*req_id].req_id != PVCALLS_INVALID_ID)
 130		return -EAGAIN;
 131	return 0;
 132}
 133
 134static bool pvcalls_front_write_todo(struct sock_mapping *map)
 135{
 136	struct pvcalls_data_intf *intf = map->active.ring;
 137	RING_IDX cons, prod, size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 138	int32_t error;
 139
 140	error = intf->out_error;
 141	if (error == -ENOTCONN)
 142		return false;
 143	if (error != 0)
 144		return true;
 145
 146	cons = intf->out_cons;
 147	prod = intf->out_prod;
 148	return !!(size - pvcalls_queued(prod, cons, size));
 149}
 150
 151static bool pvcalls_front_read_todo(struct sock_mapping *map)
 152{
 153	struct pvcalls_data_intf *intf = map->active.ring;
 154	RING_IDX cons, prod;
 155	int32_t error;
 156
 157	cons = intf->in_cons;
 158	prod = intf->in_prod;
 159	error = intf->in_error;
 160	return (error != 0 ||
 161		pvcalls_queued(prod, cons,
 162			       XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER)) != 0);
 163}
 164
 165static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id)
 166{
 167	struct xenbus_device *dev = dev_id;
 168	struct pvcalls_bedata *bedata;
 169	struct xen_pvcalls_response *rsp;
 170	uint8_t *src, *dst;
 171	int req_id = 0, more = 0, done = 0;
 172
 173	if (dev == NULL)
 174		return IRQ_HANDLED;
 175
 176	pvcalls_enter();
 177	bedata = dev_get_drvdata(&dev->dev);
 178	if (bedata == NULL) {
 179		pvcalls_exit();
 180		return IRQ_HANDLED;
 181	}
 182
 183again:
 184	while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) {
 185		rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons);
 186
 187		req_id = rsp->req_id;
 188		if (rsp->cmd == PVCALLS_POLL) {
 189			struct sock_mapping *map = (struct sock_mapping *)(uintptr_t)
 190						   rsp->u.poll.id;
 191
 192			clear_bit(PVCALLS_FLAG_POLL_INFLIGHT,
 193				  (void *)&map->passive.flags);
 194			/*
 195			 * clear INFLIGHT, then set RET. It pairs with
 196			 * the checks at the beginning of
 197			 * pvcalls_front_poll_passive.
 198			 */
 199			smp_wmb();
 200			set_bit(PVCALLS_FLAG_POLL_RET,
 201				(void *)&map->passive.flags);
 202		} else {
 203			dst = (uint8_t *)&bedata->rsp[req_id] +
 204			      sizeof(rsp->req_id);
 205			src = (uint8_t *)rsp + sizeof(rsp->req_id);
 206			memcpy(dst, src, sizeof(*rsp) - sizeof(rsp->req_id));
 207			/*
 208			 * First copy the rest of the data, then req_id. It is
 209			 * paired with the barrier when accessing bedata->rsp.
 210			 */
 211			smp_wmb();
 212			bedata->rsp[req_id].req_id = req_id;
 213		}
 214
 215		done = 1;
 216		bedata->ring.rsp_cons++;
 217	}
 218
 219	RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more);
 220	if (more)
 221		goto again;
 222	if (done)
 223		wake_up(&bedata->inflight_req);
 224	pvcalls_exit();
 225	return IRQ_HANDLED;
 226}
 227
 228static void free_active_ring(struct sock_mapping *map);
 229
 230static void pvcalls_front_destroy_active(struct pvcalls_bedata *bedata,
 231					 struct sock_mapping *map)
 232{
 233	int i;
 234
 235	unbind_from_irqhandler(map->active.irq, map);
 236
 237	if (bedata) {
 238		spin_lock(&bedata->socket_lock);
 239		if (!list_empty(&map->list))
 240			list_del_init(&map->list);
 241		spin_unlock(&bedata->socket_lock);
 242	}
 243
 244	for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
 245		gnttab_end_foreign_access(map->active.ring->ref[i], NULL);
 246	gnttab_end_foreign_access(map->active.ref, NULL);
 247	free_active_ring(map);
 248}
 249
 250static void pvcalls_front_free_map(struct pvcalls_bedata *bedata,
 251				   struct sock_mapping *map)
 252{
 253	pvcalls_front_destroy_active(bedata, map);
 254
 255	kfree(map);
 256}
 257
 258static irqreturn_t pvcalls_front_conn_handler(int irq, void *sock_map)
 259{
 260	struct sock_mapping *map = sock_map;
 261
 262	if (map == NULL)
 263		return IRQ_HANDLED;
 264
 265	wake_up_interruptible(&map->active.inflight_conn_req);
 266
 267	return IRQ_HANDLED;
 268}
 269
 270int pvcalls_front_socket(struct socket *sock)
 271{
 272	struct pvcalls_bedata *bedata;
 273	struct sock_mapping *map = NULL;
 274	struct xen_pvcalls_request *req;
 275	int notify, req_id, ret;
 276
 277	/*
 278	 * PVCalls only supports domain AF_INET,
 279	 * type SOCK_STREAM and protocol 0 sockets for now.
 280	 *
 281	 * Check socket type here, AF_INET and protocol checks are done
 282	 * by the caller.
 283	 */
 284	if (sock->type != SOCK_STREAM)
 285		return -EOPNOTSUPP;
 286
 287	pvcalls_enter();
 288	if (!pvcalls_front_dev) {
 289		pvcalls_exit();
 290		return -EACCES;
 291	}
 292	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 293
 294	map = kzalloc(sizeof(*map), GFP_KERNEL);
 295	if (map == NULL) {
 296		pvcalls_exit();
 297		return -ENOMEM;
 298	}
 299
 300	spin_lock(&bedata->socket_lock);
 301
 302	ret = get_request(bedata, &req_id);
 303	if (ret < 0) {
 304		kfree(map);
 305		spin_unlock(&bedata->socket_lock);
 306		pvcalls_exit();
 307		return ret;
 308	}
 309
 310	/*
 311	 * sock->sk->sk_send_head is not used for ip sockets: reuse the
 312	 * field to store a pointer to the struct sock_mapping
 313	 * corresponding to the socket. This way, we can easily get the
 314	 * struct sock_mapping from the struct socket.
 315	 */
 316	sock->sk->sk_send_head = (void *)map;
 317	list_add_tail(&map->list, &bedata->socket_mappings);
 318
 319	req = RING_GET_REQUEST(&bedata->ring, req_id);
 320	req->req_id = req_id;
 321	req->cmd = PVCALLS_SOCKET;
 322	req->u.socket.id = (uintptr_t) map;
 323	req->u.socket.domain = AF_INET;
 324	req->u.socket.type = SOCK_STREAM;
 325	req->u.socket.protocol = IPPROTO_IP;
 326
 327	bedata->ring.req_prod_pvt++;
 328	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 329	spin_unlock(&bedata->socket_lock);
 330	if (notify)
 331		notify_remote_via_irq(bedata->irq);
 332
 333	wait_event(bedata->inflight_req,
 334		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 335
 336	/* read req_id, then the content */
 337	smp_rmb();
 338	ret = bedata->rsp[req_id].ret;
 339	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 340
 341	pvcalls_exit();
 342	return ret;
 343}
 344EXPORT_SYMBOL_GPL(pvcalls_front_socket);
 345
 346static void free_active_ring(struct sock_mapping *map)
 347{
 348	if (!map->active.ring)
 349		return;
 350
 351	free_pages_exact(map->active.data.in,
 352			 PAGE_SIZE << map->active.ring->ring_order);
 353	free_page((unsigned long)map->active.ring);
 354}
 355
 356static int alloc_active_ring(struct sock_mapping *map)
 357{
 358	void *bytes;
 359
 360	map->active.ring = (struct pvcalls_data_intf *)
 361		get_zeroed_page(GFP_KERNEL);
 362	if (!map->active.ring)
 363		goto out;
 364
 365	map->active.ring->ring_order = PVCALLS_RING_ORDER;
 366	bytes = alloc_pages_exact(PAGE_SIZE << PVCALLS_RING_ORDER,
 367				  GFP_KERNEL | __GFP_ZERO);
 368	if (!bytes)
 369		goto out;
 370
 371	map->active.data.in = bytes;
 372	map->active.data.out = bytes +
 373		XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 374
 375	return 0;
 376
 377out:
 378	free_active_ring(map);
 379	return -ENOMEM;
 380}
 381
 382static int create_active(struct sock_mapping *map, evtchn_port_t *evtchn)
 383{
 384	void *bytes;
 385	int ret, irq = -1, i;
 386
 387	*evtchn = 0;
 388	init_waitqueue_head(&map->active.inflight_conn_req);
 389
 390	bytes = map->active.data.in;
 391	for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
 392		map->active.ring->ref[i] = gnttab_grant_foreign_access(
 393			pvcalls_front_dev->otherend_id,
 394			pfn_to_gfn(virt_to_pfn(bytes) + i), 0);
 395
 396	map->active.ref = gnttab_grant_foreign_access(
 397		pvcalls_front_dev->otherend_id,
 398		pfn_to_gfn(virt_to_pfn((void *)map->active.ring)), 0);
 399
 400	ret = xenbus_alloc_evtchn(pvcalls_front_dev, evtchn);
 401	if (ret)
 402		goto out_error;
 403	irq = bind_evtchn_to_irqhandler(*evtchn, pvcalls_front_conn_handler,
 404					0, "pvcalls-frontend", map);
 405	if (irq < 0) {
 406		ret = irq;
 407		goto out_error;
 408	}
 409
 410	map->active.irq = irq;
 411	map->active_socket = true;
 412	mutex_init(&map->active.in_mutex);
 413	mutex_init(&map->active.out_mutex);
 414
 415	return 0;
 416
 417out_error:
 418	if (*evtchn > 0)
 419		xenbus_free_evtchn(pvcalls_front_dev, *evtchn);
 420	return ret;
 421}
 422
 423int pvcalls_front_connect(struct socket *sock, struct sockaddr *addr,
 424				int addr_len, int flags)
 425{
 426	struct pvcalls_bedata *bedata;
 427	struct sock_mapping *map = NULL;
 428	struct xen_pvcalls_request *req;
 429	int notify, req_id, ret;
 430	evtchn_port_t evtchn;
 431
 432	if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
 433		return -EOPNOTSUPP;
 434
 435	map = pvcalls_enter_sock(sock);
 436	if (IS_ERR(map))
 437		return PTR_ERR(map);
 438
 439	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 440	ret = alloc_active_ring(map);
 441	if (ret < 0) {
 442		pvcalls_exit_sock(sock);
 443		return ret;
 444	}
 445	ret = create_active(map, &evtchn);
 446	if (ret < 0) {
 447		free_active_ring(map);
 448		pvcalls_exit_sock(sock);
 449		return ret;
 450	}
 451
 452	spin_lock(&bedata->socket_lock);
 453	ret = get_request(bedata, &req_id);
 454	if (ret < 0) {
 455		spin_unlock(&bedata->socket_lock);
 456		pvcalls_front_destroy_active(NULL, map);
 457		pvcalls_exit_sock(sock);
 458		return ret;
 459	}
 460
 461	req = RING_GET_REQUEST(&bedata->ring, req_id);
 462	req->req_id = req_id;
 463	req->cmd = PVCALLS_CONNECT;
 464	req->u.connect.id = (uintptr_t)map;
 465	req->u.connect.len = addr_len;
 466	req->u.connect.flags = flags;
 467	req->u.connect.ref = map->active.ref;
 468	req->u.connect.evtchn = evtchn;
 469	memcpy(req->u.connect.addr, addr, sizeof(*addr));
 470
 471	map->sock = sock;
 472
 473	bedata->ring.req_prod_pvt++;
 474	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 475	spin_unlock(&bedata->socket_lock);
 476
 477	if (notify)
 478		notify_remote_via_irq(bedata->irq);
 479
 480	wait_event(bedata->inflight_req,
 481		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 482
 483	/* read req_id, then the content */
 484	smp_rmb();
 485	ret = bedata->rsp[req_id].ret;
 486	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 487	pvcalls_exit_sock(sock);
 488	return ret;
 489}
 490EXPORT_SYMBOL_GPL(pvcalls_front_connect);
 491
 492static int __write_ring(struct pvcalls_data_intf *intf,
 493			struct pvcalls_data *data,
 494			struct iov_iter *msg_iter,
 495			int len)
 496{
 497	RING_IDX cons, prod, size, masked_prod, masked_cons;
 498	RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 499	int32_t error;
 500
 501	error = intf->out_error;
 502	if (error < 0)
 503		return error;
 504	cons = intf->out_cons;
 505	prod = intf->out_prod;
 506	/* read indexes before continuing */
 507	virt_mb();
 508
 509	size = pvcalls_queued(prod, cons, array_size);
 510	if (size > array_size)
 511		return -EINVAL;
 512	if (size == array_size)
 513		return 0;
 514	if (len > array_size - size)
 515		len = array_size - size;
 516
 517	masked_prod = pvcalls_mask(prod, array_size);
 518	masked_cons = pvcalls_mask(cons, array_size);
 519
 520	if (masked_prod < masked_cons) {
 521		len = copy_from_iter(data->out + masked_prod, len, msg_iter);
 522	} else {
 523		if (len > array_size - masked_prod) {
 524			int ret = copy_from_iter(data->out + masked_prod,
 525				       array_size - masked_prod, msg_iter);
 526			if (ret != array_size - masked_prod) {
 527				len = ret;
 528				goto out;
 529			}
 530			len = ret + copy_from_iter(data->out, len - ret, msg_iter);
 531		} else {
 532			len = copy_from_iter(data->out + masked_prod, len, msg_iter);
 533		}
 534	}
 535out:
 536	/* write to ring before updating pointer */
 537	virt_wmb();
 538	intf->out_prod += len;
 539
 540	return len;
 541}
 542
 543int pvcalls_front_sendmsg(struct socket *sock, struct msghdr *msg,
 544			  size_t len)
 545{
 546	struct sock_mapping *map;
 547	int sent, tot_sent = 0;
 548	int count = 0, flags;
 549
 550	flags = msg->msg_flags;
 551	if (flags & (MSG_CONFIRM|MSG_DONTROUTE|MSG_EOR|MSG_OOB))
 552		return -EOPNOTSUPP;
 553
 554	map = pvcalls_enter_sock(sock);
 555	if (IS_ERR(map))
 556		return PTR_ERR(map);
 557
 558	mutex_lock(&map->active.out_mutex);
 559	if ((flags & MSG_DONTWAIT) && !pvcalls_front_write_todo(map)) {
 560		mutex_unlock(&map->active.out_mutex);
 561		pvcalls_exit_sock(sock);
 562		return -EAGAIN;
 563	}
 564	if (len > INT_MAX)
 565		len = INT_MAX;
 566
 567again:
 568	count++;
 569	sent = __write_ring(map->active.ring,
 570			    &map->active.data, &msg->msg_iter,
 571			    len);
 572	if (sent > 0) {
 573		len -= sent;
 574		tot_sent += sent;
 575		notify_remote_via_irq(map->active.irq);
 576	}
 577	if (sent >= 0 && len > 0 && count < PVCALLS_FRONT_MAX_SPIN)
 578		goto again;
 579	if (sent < 0)
 580		tot_sent = sent;
 581
 582	mutex_unlock(&map->active.out_mutex);
 583	pvcalls_exit_sock(sock);
 584	return tot_sent;
 585}
 586EXPORT_SYMBOL_GPL(pvcalls_front_sendmsg);
 587
 588static int __read_ring(struct pvcalls_data_intf *intf,
 589		       struct pvcalls_data *data,
 590		       struct iov_iter *msg_iter,
 591		       size_t len, int flags)
 592{
 593	RING_IDX cons, prod, size, masked_prod, masked_cons;
 594	RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 595	int32_t error;
 596
 597	cons = intf->in_cons;
 598	prod = intf->in_prod;
 599	error = intf->in_error;
 600	/* get pointers before reading from the ring */
 601	virt_rmb();
 602
 603	size = pvcalls_queued(prod, cons, array_size);
 604	masked_prod = pvcalls_mask(prod, array_size);
 605	masked_cons = pvcalls_mask(cons, array_size);
 606
 607	if (size == 0)
 608		return error ?: size;
 609
 610	if (len > size)
 611		len = size;
 612
 613	if (masked_prod > masked_cons) {
 614		len = copy_to_iter(data->in + masked_cons, len, msg_iter);
 615	} else {
 616		if (len > (array_size - masked_cons)) {
 617			int ret = copy_to_iter(data->in + masked_cons,
 618				     array_size - masked_cons, msg_iter);
 619			if (ret != array_size - masked_cons) {
 620				len = ret;
 621				goto out;
 622			}
 623			len = ret + copy_to_iter(data->in, len - ret, msg_iter);
 624		} else {
 625			len = copy_to_iter(data->in + masked_cons, len, msg_iter);
 626		}
 627	}
 628out:
 629	/* read data from the ring before increasing the index */
 630	virt_mb();
 631	if (!(flags & MSG_PEEK))
 632		intf->in_cons += len;
 633
 634	return len;
 635}
 636
 637int pvcalls_front_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
 638		     int flags)
 639{
 640	int ret;
 641	struct sock_mapping *map;
 642
 643	if (flags & (MSG_CMSG_CLOEXEC|MSG_ERRQUEUE|MSG_OOB|MSG_TRUNC))
 644		return -EOPNOTSUPP;
 645
 646	map = pvcalls_enter_sock(sock);
 647	if (IS_ERR(map))
 648		return PTR_ERR(map);
 649
 650	mutex_lock(&map->active.in_mutex);
 651	if (len > XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER))
 652		len = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 653
 654	while (!(flags & MSG_DONTWAIT) && !pvcalls_front_read_todo(map)) {
 655		wait_event_interruptible(map->active.inflight_conn_req,
 656					 pvcalls_front_read_todo(map));
 657	}
 658	ret = __read_ring(map->active.ring, &map->active.data,
 659			  &msg->msg_iter, len, flags);
 660
 661	if (ret > 0)
 662		notify_remote_via_irq(map->active.irq);
 663	if (ret == 0)
 664		ret = (flags & MSG_DONTWAIT) ? -EAGAIN : 0;
 665	if (ret == -ENOTCONN)
 666		ret = 0;
 667
 668	mutex_unlock(&map->active.in_mutex);
 669	pvcalls_exit_sock(sock);
 670	return ret;
 671}
 672EXPORT_SYMBOL_GPL(pvcalls_front_recvmsg);
 673
 674int pvcalls_front_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
 675{
 676	struct pvcalls_bedata *bedata;
 677	struct sock_mapping *map = NULL;
 678	struct xen_pvcalls_request *req;
 679	int notify, req_id, ret;
 680
 681	if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
 682		return -EOPNOTSUPP;
 683
 684	map = pvcalls_enter_sock(sock);
 685	if (IS_ERR(map))
 686		return PTR_ERR(map);
 687	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 688
 689	spin_lock(&bedata->socket_lock);
 690	ret = get_request(bedata, &req_id);
 691	if (ret < 0) {
 692		spin_unlock(&bedata->socket_lock);
 693		pvcalls_exit_sock(sock);
 694		return ret;
 695	}
 696	req = RING_GET_REQUEST(&bedata->ring, req_id);
 697	req->req_id = req_id;
 698	map->sock = sock;
 699	req->cmd = PVCALLS_BIND;
 700	req->u.bind.id = (uintptr_t)map;
 701	memcpy(req->u.bind.addr, addr, sizeof(*addr));
 702	req->u.bind.len = addr_len;
 703
 704	init_waitqueue_head(&map->passive.inflight_accept_req);
 705
 706	map->active_socket = false;
 707
 708	bedata->ring.req_prod_pvt++;
 709	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 710	spin_unlock(&bedata->socket_lock);
 711	if (notify)
 712		notify_remote_via_irq(bedata->irq);
 713
 714	wait_event(bedata->inflight_req,
 715		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 716
 717	/* read req_id, then the content */
 718	smp_rmb();
 719	ret = bedata->rsp[req_id].ret;
 720	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 721
 722	map->passive.status = PVCALLS_STATUS_BIND;
 723	pvcalls_exit_sock(sock);
 724	return 0;
 725}
 726EXPORT_SYMBOL_GPL(pvcalls_front_bind);
 727
 728int pvcalls_front_listen(struct socket *sock, int backlog)
 729{
 730	struct pvcalls_bedata *bedata;
 731	struct sock_mapping *map;
 732	struct xen_pvcalls_request *req;
 733	int notify, req_id, ret;
 734
 735	map = pvcalls_enter_sock(sock);
 736	if (IS_ERR(map))
 737		return PTR_ERR(map);
 738	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 739
 740	if (map->passive.status != PVCALLS_STATUS_BIND) {
 741		pvcalls_exit_sock(sock);
 742		return -EOPNOTSUPP;
 743	}
 744
 745	spin_lock(&bedata->socket_lock);
 746	ret = get_request(bedata, &req_id);
 747	if (ret < 0) {
 748		spin_unlock(&bedata->socket_lock);
 749		pvcalls_exit_sock(sock);
 750		return ret;
 751	}
 752	req = RING_GET_REQUEST(&bedata->ring, req_id);
 753	req->req_id = req_id;
 754	req->cmd = PVCALLS_LISTEN;
 755	req->u.listen.id = (uintptr_t) map;
 756	req->u.listen.backlog = backlog;
 757
 758	bedata->ring.req_prod_pvt++;
 759	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 760	spin_unlock(&bedata->socket_lock);
 761	if (notify)
 762		notify_remote_via_irq(bedata->irq);
 763
 764	wait_event(bedata->inflight_req,
 765		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 766
 767	/* read req_id, then the content */
 768	smp_rmb();
 769	ret = bedata->rsp[req_id].ret;
 770	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 771
 772	map->passive.status = PVCALLS_STATUS_LISTEN;
 773	pvcalls_exit_sock(sock);
 774	return ret;
 775}
 776EXPORT_SYMBOL_GPL(pvcalls_front_listen);
 777
 778int pvcalls_front_accept(struct socket *sock, struct socket *newsock,
 779			 struct proto_accept_arg *arg)
 780{
 781	struct pvcalls_bedata *bedata;
 782	struct sock_mapping *map;
 783	struct sock_mapping *map2 = NULL;
 784	struct xen_pvcalls_request *req;
 785	int notify, req_id, ret, nonblock;
 786	evtchn_port_t evtchn;
 787
 788	map = pvcalls_enter_sock(sock);
 789	if (IS_ERR(map))
 790		return PTR_ERR(map);
 791	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 792
 793	if (map->passive.status != PVCALLS_STATUS_LISTEN) {
 794		pvcalls_exit_sock(sock);
 795		return -EINVAL;
 796	}
 797
 798	nonblock = arg->flags & SOCK_NONBLOCK;
 799	/*
 800	 * Backend only supports 1 inflight accept request, will return
 801	 * errors for the others
 802	 */
 803	if (test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 804			     (void *)&map->passive.flags)) {
 805		req_id = READ_ONCE(map->passive.inflight_req_id);
 806		if (req_id != PVCALLS_INVALID_ID &&
 807		    READ_ONCE(bedata->rsp[req_id].req_id) == req_id) {
 808			map2 = map->passive.accept_map;
 809			goto received;
 810		}
 811		if (nonblock) {
 812			pvcalls_exit_sock(sock);
 813			return -EAGAIN;
 814		}
 815		if (wait_event_interruptible(map->passive.inflight_accept_req,
 816			!test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 817					  (void *)&map->passive.flags))) {
 818			pvcalls_exit_sock(sock);
 819			return -EINTR;
 820		}
 821	}
 822
 823	map2 = kzalloc(sizeof(*map2), GFP_KERNEL);
 824	if (map2 == NULL) {
 825		clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 826			  (void *)&map->passive.flags);
 827		pvcalls_exit_sock(sock);
 828		return -ENOMEM;
 829	}
 830	ret = alloc_active_ring(map2);
 831	if (ret < 0) {
 832		clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 833				(void *)&map->passive.flags);
 834		kfree(map2);
 835		pvcalls_exit_sock(sock);
 836		return ret;
 837	}
 838	ret = create_active(map2, &evtchn);
 839	if (ret < 0) {
 840		free_active_ring(map2);
 841		kfree(map2);
 842		clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 843			  (void *)&map->passive.flags);
 844		pvcalls_exit_sock(sock);
 845		return ret;
 846	}
 847
 848	spin_lock(&bedata->socket_lock);
 849	ret = get_request(bedata, &req_id);
 850	if (ret < 0) {
 851		clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 852			  (void *)&map->passive.flags);
 853		spin_unlock(&bedata->socket_lock);
 854		pvcalls_front_free_map(bedata, map2);
 855		pvcalls_exit_sock(sock);
 856		return ret;
 857	}
 858
 859	list_add_tail(&map2->list, &bedata->socket_mappings);
 860
 861	req = RING_GET_REQUEST(&bedata->ring, req_id);
 862	req->req_id = req_id;
 863	req->cmd = PVCALLS_ACCEPT;
 864	req->u.accept.id = (uintptr_t) map;
 865	req->u.accept.ref = map2->active.ref;
 866	req->u.accept.id_new = (uintptr_t) map2;
 867	req->u.accept.evtchn = evtchn;
 868	map->passive.accept_map = map2;
 869
 870	bedata->ring.req_prod_pvt++;
 871	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 872	spin_unlock(&bedata->socket_lock);
 873	if (notify)
 874		notify_remote_via_irq(bedata->irq);
 875	/* We could check if we have received a response before returning. */
 876	if (nonblock) {
 877		WRITE_ONCE(map->passive.inflight_req_id, req_id);
 878		pvcalls_exit_sock(sock);
 879		return -EAGAIN;
 880	}
 881
 882	if (wait_event_interruptible(bedata->inflight_req,
 883		READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) {
 884		pvcalls_exit_sock(sock);
 885		return -EINTR;
 886	}
 887	/* read req_id, then the content */
 888	smp_rmb();
 889
 890received:
 891	map2->sock = newsock;
 892	newsock->sk = sk_alloc(sock_net(sock->sk), PF_INET, GFP_KERNEL, &pvcalls_proto, false);
 893	if (!newsock->sk) {
 894		bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 895		map->passive.inflight_req_id = PVCALLS_INVALID_ID;
 896		clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 897			  (void *)&map->passive.flags);
 898		pvcalls_front_free_map(bedata, map2);
 899		pvcalls_exit_sock(sock);
 900		return -ENOMEM;
 901	}
 902	newsock->sk->sk_send_head = (void *)map2;
 903
 904	ret = bedata->rsp[req_id].ret;
 905	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 906	map->passive.inflight_req_id = PVCALLS_INVALID_ID;
 907
 908	clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, (void *)&map->passive.flags);
 909	wake_up(&map->passive.inflight_accept_req);
 910
 911	pvcalls_exit_sock(sock);
 912	return ret;
 913}
 914EXPORT_SYMBOL_GPL(pvcalls_front_accept);
 915
 916static __poll_t pvcalls_front_poll_passive(struct file *file,
 917					       struct pvcalls_bedata *bedata,
 918					       struct sock_mapping *map,
 919					       poll_table *wait)
 920{
 921	int notify, req_id, ret;
 922	struct xen_pvcalls_request *req;
 923
 924	if (test_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 925		     (void *)&map->passive.flags)) {
 926		uint32_t req_id = READ_ONCE(map->passive.inflight_req_id);
 927
 928		if (req_id != PVCALLS_INVALID_ID &&
 929		    READ_ONCE(bedata->rsp[req_id].req_id) == req_id)
 930			return EPOLLIN | EPOLLRDNORM;
 931
 932		poll_wait(file, &map->passive.inflight_accept_req, wait);
 933		return 0;
 934	}
 935
 936	if (test_and_clear_bit(PVCALLS_FLAG_POLL_RET,
 937			       (void *)&map->passive.flags))
 938		return EPOLLIN | EPOLLRDNORM;
 939
 940	/*
 941	 * First check RET, then INFLIGHT. No barriers necessary to
 942	 * ensure execution ordering because of the conditional
 943	 * instructions creating control dependencies.
 944	 */
 945
 946	if (test_and_set_bit(PVCALLS_FLAG_POLL_INFLIGHT,
 947			     (void *)&map->passive.flags)) {
 948		poll_wait(file, &bedata->inflight_req, wait);
 949		return 0;
 950	}
 951
 952	spin_lock(&bedata->socket_lock);
 953	ret = get_request(bedata, &req_id);
 954	if (ret < 0) {
 955		spin_unlock(&bedata->socket_lock);
 956		return ret;
 957	}
 958	req = RING_GET_REQUEST(&bedata->ring, req_id);
 959	req->req_id = req_id;
 960	req->cmd = PVCALLS_POLL;
 961	req->u.poll.id = (uintptr_t) map;
 962
 963	bedata->ring.req_prod_pvt++;
 964	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 965	spin_unlock(&bedata->socket_lock);
 966	if (notify)
 967		notify_remote_via_irq(bedata->irq);
 968
 969	poll_wait(file, &bedata->inflight_req, wait);
 970	return 0;
 971}
 972
 973static __poll_t pvcalls_front_poll_active(struct file *file,
 974					      struct pvcalls_bedata *bedata,
 975					      struct sock_mapping *map,
 976					      poll_table *wait)
 977{
 978	__poll_t mask = 0;
 979	int32_t in_error, out_error;
 980	struct pvcalls_data_intf *intf = map->active.ring;
 981
 982	out_error = intf->out_error;
 983	in_error = intf->in_error;
 984
 985	poll_wait(file, &map->active.inflight_conn_req, wait);
 986	if (pvcalls_front_write_todo(map))
 987		mask |= EPOLLOUT | EPOLLWRNORM;
 988	if (pvcalls_front_read_todo(map))
 989		mask |= EPOLLIN | EPOLLRDNORM;
 990	if (in_error != 0 || out_error != 0)
 991		mask |= EPOLLERR;
 992
 993	return mask;
 994}
 995
 996__poll_t pvcalls_front_poll(struct file *file, struct socket *sock,
 997			       poll_table *wait)
 998{
 999	struct pvcalls_bedata *bedata;
1000	struct sock_mapping *map;
1001	__poll_t ret;
1002
1003	map = pvcalls_enter_sock(sock);
1004	if (IS_ERR(map))
1005		return EPOLLNVAL;
1006	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1007
1008	if (map->active_socket)
1009		ret = pvcalls_front_poll_active(file, bedata, map, wait);
1010	else
1011		ret = pvcalls_front_poll_passive(file, bedata, map, wait);
1012	pvcalls_exit_sock(sock);
1013	return ret;
1014}
1015EXPORT_SYMBOL_GPL(pvcalls_front_poll);
1016
1017int pvcalls_front_release(struct socket *sock)
1018{
1019	struct pvcalls_bedata *bedata;
1020	struct sock_mapping *map;
1021	int req_id, notify, ret;
1022	struct xen_pvcalls_request *req;
1023
1024	if (sock->sk == NULL)
1025		return 0;
1026
1027	map = pvcalls_enter_sock(sock);
1028	if (IS_ERR(map)) {
1029		if (PTR_ERR(map) == -ENOTCONN)
1030			return -EIO;
1031		else
1032			return 0;
1033	}
1034	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1035
1036	spin_lock(&bedata->socket_lock);
1037	ret = get_request(bedata, &req_id);
1038	if (ret < 0) {
1039		spin_unlock(&bedata->socket_lock);
1040		pvcalls_exit_sock(sock);
1041		return ret;
1042	}
1043	sock->sk->sk_send_head = NULL;
1044
1045	req = RING_GET_REQUEST(&bedata->ring, req_id);
1046	req->req_id = req_id;
1047	req->cmd = PVCALLS_RELEASE;
1048	req->u.release.id = (uintptr_t)map;
1049
1050	bedata->ring.req_prod_pvt++;
1051	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
1052	spin_unlock(&bedata->socket_lock);
1053	if (notify)
1054		notify_remote_via_irq(bedata->irq);
1055
1056	wait_event(bedata->inflight_req,
1057		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
1058
1059	if (map->active_socket) {
1060		/*
1061		 * Set in_error and wake up inflight_conn_req to force
1062		 * recvmsg waiters to exit.
1063		 */
1064		map->active.ring->in_error = -EBADF;
1065		wake_up_interruptible(&map->active.inflight_conn_req);
1066
1067		/*
1068		 * We need to make sure that sendmsg/recvmsg on this socket have
1069		 * not started before we've cleared sk_send_head here. The
1070		 * easiest way to guarantee this is to see that no pvcalls
1071		 * (other than us) is in progress on this socket.
1072		 */
1073		while (atomic_read(&map->refcount) > 1)
1074			cpu_relax();
1075
1076		pvcalls_front_free_map(bedata, map);
1077	} else {
1078		wake_up(&bedata->inflight_req);
1079		wake_up(&map->passive.inflight_accept_req);
1080
1081		while (atomic_read(&map->refcount) > 1)
1082			cpu_relax();
1083
1084		spin_lock(&bedata->socket_lock);
1085		list_del(&map->list);
1086		spin_unlock(&bedata->socket_lock);
1087		if (READ_ONCE(map->passive.inflight_req_id) != PVCALLS_INVALID_ID &&
1088			READ_ONCE(map->passive.inflight_req_id) != 0) {
1089			pvcalls_front_free_map(bedata,
1090					       map->passive.accept_map);
1091		}
1092		kfree(map);
1093	}
1094	WRITE_ONCE(bedata->rsp[req_id].req_id, PVCALLS_INVALID_ID);
1095
1096	pvcalls_exit();
1097	return 0;
1098}
1099EXPORT_SYMBOL_GPL(pvcalls_front_release);
1100
1101static const struct xenbus_device_id pvcalls_front_ids[] = {
1102	{ "pvcalls" },
1103	{ "" }
1104};
1105
1106static void pvcalls_front_remove(struct xenbus_device *dev)
1107{
1108	struct pvcalls_bedata *bedata;
1109	struct sock_mapping *map = NULL, *n;
1110
1111	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1112	dev_set_drvdata(&dev->dev, NULL);
1113	pvcalls_front_dev = NULL;
1114	if (bedata->irq >= 0)
1115		unbind_from_irqhandler(bedata->irq, dev);
1116
1117	list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
1118		map->sock->sk->sk_send_head = NULL;
1119		if (map->active_socket) {
1120			map->active.ring->in_error = -EBADF;
1121			wake_up_interruptible(&map->active.inflight_conn_req);
1122		}
1123	}
1124
1125	smp_mb();
1126	while (atomic_read(&pvcalls_refcount) > 0)
1127		cpu_relax();
1128	list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
1129		if (map->active_socket) {
1130			/* No need to lock, refcount is 0 */
1131			pvcalls_front_free_map(bedata, map);
1132		} else {
1133			list_del(&map->list);
1134			kfree(map);
1135		}
1136	}
1137	if (bedata->ref != -1)
1138		gnttab_end_foreign_access(bedata->ref, NULL);
1139	kfree(bedata->ring.sring);
1140	kfree(bedata);
1141	xenbus_switch_state(dev, XenbusStateClosed);
1142}
1143
1144static int pvcalls_front_probe(struct xenbus_device *dev,
1145			  const struct xenbus_device_id *id)
1146{
1147	int ret = -ENOMEM, i;
1148	evtchn_port_t evtchn;
1149	unsigned int max_page_order, function_calls, len;
1150	char *versions;
1151	grant_ref_t gref_head = 0;
1152	struct xenbus_transaction xbt;
1153	struct pvcalls_bedata *bedata = NULL;
1154	struct xen_pvcalls_sring *sring;
1155
1156	if (pvcalls_front_dev != NULL) {
1157		dev_err(&dev->dev, "only one PV Calls connection supported\n");
1158		return -EINVAL;
1159	}
1160
1161	versions = xenbus_read(XBT_NIL, dev->otherend, "versions", &len);
1162	if (IS_ERR(versions))
1163		return PTR_ERR(versions);
1164	if (!len)
1165		return -EINVAL;
1166	if (strcmp(versions, "1")) {
1167		kfree(versions);
1168		return -EINVAL;
1169	}
1170	kfree(versions);
1171	max_page_order = xenbus_read_unsigned(dev->otherend,
1172					      "max-page-order", 0);
1173	if (max_page_order < PVCALLS_RING_ORDER)
1174		return -ENODEV;
1175	function_calls = xenbus_read_unsigned(dev->otherend,
1176					      "function-calls", 0);
1177	/* See XENBUS_FUNCTIONS_CALLS in pvcalls.h */
1178	if (function_calls != 1)
1179		return -ENODEV;
1180	pr_info("%s max-page-order is %u\n", __func__, max_page_order);
1181
1182	bedata = kzalloc(sizeof(struct pvcalls_bedata), GFP_KERNEL);
1183	if (!bedata)
1184		return -ENOMEM;
1185
1186	dev_set_drvdata(&dev->dev, bedata);
1187	pvcalls_front_dev = dev;
1188	init_waitqueue_head(&bedata->inflight_req);
1189	INIT_LIST_HEAD(&bedata->socket_mappings);
1190	spin_lock_init(&bedata->socket_lock);
1191	bedata->irq = -1;
1192	bedata->ref = -1;
1193
1194	for (i = 0; i < PVCALLS_NR_RSP_PER_RING; i++)
1195		bedata->rsp[i].req_id = PVCALLS_INVALID_ID;
1196
1197	sring = (struct xen_pvcalls_sring *) __get_free_page(GFP_KERNEL |
1198							     __GFP_ZERO);
1199	if (!sring)
1200		goto error;
1201	SHARED_RING_INIT(sring);
1202	FRONT_RING_INIT(&bedata->ring, sring, XEN_PAGE_SIZE);
1203
1204	ret = xenbus_alloc_evtchn(dev, &evtchn);
1205	if (ret)
1206		goto error;
1207
1208	bedata->irq = bind_evtchn_to_irqhandler(evtchn,
1209						pvcalls_front_event_handler,
1210						0, "pvcalls-frontend", dev);
1211	if (bedata->irq < 0) {
1212		ret = bedata->irq;
1213		goto error;
1214	}
1215
1216	ret = gnttab_alloc_grant_references(1, &gref_head);
1217	if (ret < 0)
1218		goto error;
1219	ret = gnttab_claim_grant_reference(&gref_head);
1220	if (ret < 0)
1221		goto error;
1222	bedata->ref = ret;
1223	gnttab_grant_foreign_access_ref(bedata->ref, dev->otherend_id,
1224					virt_to_gfn((void *)sring), 0);
1225
1226 again:
1227	ret = xenbus_transaction_start(&xbt);
1228	if (ret) {
1229		xenbus_dev_fatal(dev, ret, "starting transaction");
1230		goto error;
1231	}
1232	ret = xenbus_printf(xbt, dev->nodename, "version", "%u", 1);
1233	if (ret)
1234		goto error_xenbus;
1235	ret = xenbus_printf(xbt, dev->nodename, "ring-ref", "%d", bedata->ref);
1236	if (ret)
1237		goto error_xenbus;
1238	ret = xenbus_printf(xbt, dev->nodename, "port", "%u",
1239			    evtchn);
1240	if (ret)
1241		goto error_xenbus;
1242	ret = xenbus_transaction_end(xbt, 0);
1243	if (ret) {
1244		if (ret == -EAGAIN)
1245			goto again;
1246		xenbus_dev_fatal(dev, ret, "completing transaction");
1247		goto error;
1248	}
1249	xenbus_switch_state(dev, XenbusStateInitialised);
1250
1251	return 0;
1252
1253 error_xenbus:
1254	xenbus_transaction_end(xbt, 1);
1255	xenbus_dev_fatal(dev, ret, "writing xenstore");
1256 error:
1257	pvcalls_front_remove(dev);
1258	return ret;
1259}
1260
1261static void pvcalls_front_changed(struct xenbus_device *dev,
1262			    enum xenbus_state backend_state)
1263{
1264	switch (backend_state) {
1265	case XenbusStateReconfiguring:
1266	case XenbusStateReconfigured:
1267	case XenbusStateInitialising:
1268	case XenbusStateInitialised:
1269	case XenbusStateUnknown:
1270		break;
1271
1272	case XenbusStateInitWait:
1273		break;
1274
1275	case XenbusStateConnected:
1276		xenbus_switch_state(dev, XenbusStateConnected);
1277		break;
1278
1279	case XenbusStateClosed:
1280		if (dev->state == XenbusStateClosed)
1281			break;
1282		/* Missed the backend's CLOSING state */
1283		fallthrough;
1284	case XenbusStateClosing:
1285		xenbus_frontend_closed(dev);
1286		break;
1287	}
1288}
1289
1290static struct xenbus_driver pvcalls_front_driver = {
1291	.ids = pvcalls_front_ids,
1292	.probe = pvcalls_front_probe,
1293	.remove = pvcalls_front_remove,
1294	.otherend_changed = pvcalls_front_changed,
1295	.not_essential = true,
1296};
1297
1298static int __init pvcalls_frontend_init(void)
1299{
1300	if (!xen_domain())
1301		return -ENODEV;
1302
1303	pr_info("Initialising Xen pvcalls frontend driver\n");
1304
1305	return xenbus_register_frontend(&pvcalls_front_driver);
1306}
1307
1308module_init(pvcalls_frontend_init);
1309
1310MODULE_DESCRIPTION("Xen PV Calls frontend driver");
1311MODULE_AUTHOR("Stefano Stabellini <sstabellini@kernel.org>");
1312MODULE_LICENSE("GPL");