xen-netfront.c

   1/*
   2 * Virtual network driver for conversing with remote driver backends.
   3 *
   4 * Copyright (c) 2002-2005, K A Fraser
   5 * Copyright (c) 2005, XenSource Ltd
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License version 2
   9 * as published by the Free Software Foundation; or, when distributed
  10 * separately from the Linux kernel or incorporated into other
  11 * software packages, subject to the following license:
  12 *
  13 * Permission is hereby granted, free of charge, to any person obtaining a copy
  14 * of this source file (the "Software"), to deal in the Software without
  15 * restriction, including without limitation the rights to use, copy, modify,
  16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  17 * and to permit persons to whom the Software is furnished to do so, subject to
  18 * the following conditions:
  19 *
  20 * The above copyright notice and this permission notice shall be included in
  21 * all copies or substantial portions of the Software.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  29 * IN THE SOFTWARE.
  30 */
  31
  32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34#include <linux/module.h>
  35#include <linux/kernel.h>
  36#include <linux/netdevice.h>
  37#include <linux/etherdevice.h>
  38#include <linux/skbuff.h>
  39#include <linux/ethtool.h>
  40#include <linux/if_ether.h>
  41#include <net/tcp.h>
  42#include <linux/udp.h>
  43#include <linux/moduleparam.h>
  44#include <linux/mm.h>
  45#include <linux/slab.h>
  46#include <net/ip.h>
  47#include <linux/bpf.h>
  48#include <net/page_pool/types.h>
  49#include <linux/bpf_trace.h>
  50
  51#include <xen/xen.h>
  52#include <xen/xenbus.h>
  53#include <xen/events.h>
  54#include <xen/page.h>
  55#include <xen/platform_pci.h>
  56#include <xen/grant_table.h>
  57
  58#include <xen/interface/io/netif.h>
  59#include <xen/interface/memory.h>
  60#include <xen/interface/grant_table.h>
  61
  62/* Module parameters */
  63#define MAX_QUEUES_DEFAULT 8
  64static unsigned int xennet_max_queues;
  65module_param_named(max_queues, xennet_max_queues, uint, 0644);
  66MODULE_PARM_DESC(max_queues,
  67		 "Maximum number of queues per virtual interface");
  68
  69static bool __read_mostly xennet_trusted = true;
  70module_param_named(trusted, xennet_trusted, bool, 0644);
  71MODULE_PARM_DESC(trusted, "Is the backend trusted");
  72
  73#define XENNET_TIMEOUT  (5 * HZ)
  74
  75static const struct ethtool_ops xennet_ethtool_ops;
  76
  77struct netfront_cb {
  78	int pull_to;
  79};
  80
  81#define NETFRONT_SKB_CB(skb)	((struct netfront_cb *)((skb)->cb))
  82
  83#define RX_COPY_THRESHOLD 256
  84
  85#define NET_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE)
  86#define NET_RX_RING_SIZE __CONST_RING_SIZE(xen_netif_rx, XEN_PAGE_SIZE)
  87
  88/* Minimum number of Rx slots (includes slot for GSO metadata). */
  89#define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1)
  90
  91/* Queue name is interface name with "-qNNN" appended */
  92#define QUEUE_NAME_SIZE (IFNAMSIZ + 6)
  93
  94/* IRQ name is queue name with "-tx" or "-rx" appended */
  95#define IRQ_NAME_SIZE (QUEUE_NAME_SIZE + 3)
  96
  97static DECLARE_WAIT_QUEUE_HEAD(module_wq);
  98
  99struct netfront_stats {
 100	u64			packets;
 101	u64			bytes;
 102	struct u64_stats_sync	syncp;
 103};
 104
 105struct netfront_info;
 106
 107struct netfront_queue {
 108	unsigned int id; /* Queue ID, 0-based */
 109	char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
 110	struct netfront_info *info;
 111
 112	struct bpf_prog __rcu *xdp_prog;
 113
 114	struct napi_struct napi;
 115
 116	/* Split event channels support, tx_* == rx_* when using
 117	 * single event channel.
 118	 */
 119	unsigned int tx_evtchn, rx_evtchn;
 120	unsigned int tx_irq, rx_irq;
 121	/* Only used when split event channels support is enabled */
 122	char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
 123	char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
 124
 125	spinlock_t   tx_lock;
 126	struct xen_netif_tx_front_ring tx;
 127	int tx_ring_ref;
 128
 129	/*
 130	 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
 131	 * are linked from tx_skb_freelist through tx_link.
 132	 */
 133	struct sk_buff *tx_skbs[NET_TX_RING_SIZE];
 134	unsigned short tx_link[NET_TX_RING_SIZE];
 135#define TX_LINK_NONE 0xffff
 136#define TX_PENDING   0xfffe
 137	grant_ref_t gref_tx_head;
 138	grant_ref_t grant_tx_ref[NET_TX_RING_SIZE];
 139	struct page *grant_tx_page[NET_TX_RING_SIZE];
 140	unsigned tx_skb_freelist;
 141	unsigned int tx_pend_queue;
 142
 143	spinlock_t   rx_lock ____cacheline_aligned_in_smp;
 144	struct xen_netif_rx_front_ring rx;
 145	int rx_ring_ref;
 146
 147	struct timer_list rx_refill_timer;
 148
 149	struct sk_buff *rx_skbs[NET_RX_RING_SIZE];
 150	grant_ref_t gref_rx_head;
 151	grant_ref_t grant_rx_ref[NET_RX_RING_SIZE];
 152
 153	unsigned int rx_rsp_unconsumed;
 154	spinlock_t rx_cons_lock;
 155
 156	struct page_pool *page_pool;
 157	struct xdp_rxq_info xdp_rxq;
 158};
 159
 160struct netfront_info {
 161	struct list_head list;
 162	struct net_device *netdev;
 163
 164	struct xenbus_device *xbdev;
 165
 166	/* Multi-queue support */
 167	struct netfront_queue *queues;
 168
 169	/* Statistics */
 170	struct netfront_stats __percpu *rx_stats;
 171	struct netfront_stats __percpu *tx_stats;
 172
 173	/* XDP state */
 174	bool netback_has_xdp_headroom;
 175	bool netfront_xdp_enabled;
 176
 177	/* Is device behaving sane? */
 178	bool broken;
 179
 180	/* Should skbs be bounced into a zeroed buffer? */
 181	bool bounce;
 182
 183	atomic_t rx_gso_checksum_fixup;
 184};
 185
 186struct netfront_rx_info {
 187	struct xen_netif_rx_response rx;
 188	struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
 189};
 190
 191/*
 192 * Access macros for acquiring freeing slots in tx_skbs[].
 193 */
 194
 195static void add_id_to_list(unsigned *head, unsigned short *list,
 196			   unsigned short id)
 197{
 198	list[id] = *head;
 199	*head = id;
 200}
 201
 202static unsigned short get_id_from_list(unsigned *head, unsigned short *list)
 203{
 204	unsigned int id = *head;
 205
 206	if (id != TX_LINK_NONE) {
 207		*head = list[id];
 208		list[id] = TX_LINK_NONE;
 209	}
 210	return id;
 211}
 212
 213static int xennet_rxidx(RING_IDX idx)
 214{
 215	return idx & (NET_RX_RING_SIZE - 1);
 216}
 217
 218static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue,
 219					 RING_IDX ri)
 220{
 221	int i = xennet_rxidx(ri);
 222	struct sk_buff *skb = queue->rx_skbs[i];
 223	queue->rx_skbs[i] = NULL;
 224	return skb;
 225}
 226
 227static grant_ref_t xennet_get_rx_ref(struct netfront_queue *queue,
 228					    RING_IDX ri)
 229{
 230	int i = xennet_rxidx(ri);
 231	grant_ref_t ref = queue->grant_rx_ref[i];
 232	queue->grant_rx_ref[i] = INVALID_GRANT_REF;
 233	return ref;
 234}
 235
 236#ifdef CONFIG_SYSFS
 237static const struct attribute_group xennet_dev_group;
 238#endif
 239
 240static bool xennet_can_sg(struct net_device *dev)
 241{
 242	return dev->features & NETIF_F_SG;
 243}
 244
 245
 246static void rx_refill_timeout(struct timer_list *t)
 247{
 248	struct netfront_queue *queue = timer_container_of(queue, t,
 249							  rx_refill_timer);
 250	napi_schedule(&queue->napi);
 251}
 252
 253static int netfront_tx_slot_available(struct netfront_queue *queue)
 254{
 255	return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) <
 256		(NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1);
 257}
 258
 259static void xennet_maybe_wake_tx(struct netfront_queue *queue)
 260{
 261	struct net_device *dev = queue->info->netdev;
 262	struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id);
 263
 264	if (unlikely(netif_tx_queue_stopped(dev_queue)) &&
 265	    netfront_tx_slot_available(queue) &&
 266	    likely(netif_running(dev)))
 267		netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id));
 268}
 269
 270
 271static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)
 272{
 273	struct sk_buff *skb;
 274	struct page *page;
 275
 276	skb = __netdev_alloc_skb(queue->info->netdev,
 277				 RX_COPY_THRESHOLD + NET_IP_ALIGN,
 278				 GFP_ATOMIC | __GFP_NOWARN);
 279	if (unlikely(!skb))
 280		return NULL;
 281
 282	page = page_pool_alloc_pages(queue->page_pool,
 283				     GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO);
 284	if (unlikely(!page)) {
 285		kfree_skb(skb);
 286		return NULL;
 287	}
 288	skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE);
 289	skb_mark_for_recycle(skb);
 290
 291	/* Align ip header to a 16 bytes boundary */
 292	skb_reserve(skb, NET_IP_ALIGN);
 293	skb->dev = queue->info->netdev;
 294
 295	return skb;
 296}
 297
 298
 299static void xennet_alloc_rx_buffers(struct netfront_queue *queue)
 300{
 301	RING_IDX req_prod = queue->rx.req_prod_pvt;
 302	int notify;
 303	int err = 0;
 304
 305	if (unlikely(!netif_carrier_ok(queue->info->netdev)))
 306		return;
 307
 308	for (req_prod = queue->rx.req_prod_pvt;
 309	     req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE;
 310	     req_prod++) {
 311		struct sk_buff *skb;
 312		unsigned short id;
 313		grant_ref_t ref;
 314		struct page *page;
 315		struct xen_netif_rx_request *req;
 316
 317		skb = xennet_alloc_one_rx_buffer(queue);
 318		if (!skb) {
 319			err = -ENOMEM;
 320			break;
 321		}
 322
 323		id = xennet_rxidx(req_prod);
 324
 325		BUG_ON(queue->rx_skbs[id]);
 326		queue->rx_skbs[id] = skb;
 327
 328		ref = gnttab_claim_grant_reference(&queue->gref_rx_head);
 329		WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
 330		queue->grant_rx_ref[id] = ref;
 331
 332		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
 333
 334		req = RING_GET_REQUEST(&queue->rx, req_prod);
 335		gnttab_page_grant_foreign_access_ref_one(ref,
 336							 queue->info->xbdev->otherend_id,
 337							 page,
 338							 0);
 339		req->id = id;
 340		req->gref = ref;
 341	}
 342
 343	queue->rx.req_prod_pvt = req_prod;
 344
 345	/* Try again later if there are not enough requests or skb allocation
 346	 * failed.
 347	 * Enough requests is quantified as the sum of newly created slots and
 348	 * the unconsumed slots at the backend.
 349	 */
 350	if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN ||
 351	    unlikely(err)) {
 352		mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10));
 353		return;
 354	}
 355
 356	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify);
 357	if (notify)
 358		notify_remote_via_irq(queue->rx_irq);
 359}
 360
 361static int xennet_open(struct net_device *dev)
 362{
 363	struct netfront_info *np = netdev_priv(dev);
 364	unsigned int num_queues = dev->real_num_tx_queues;
 365	unsigned int i = 0;
 366	struct netfront_queue *queue = NULL;
 367
 368	if (!np->queues || np->broken)
 369		return -ENODEV;
 370
 371	for (i = 0; i < num_queues; ++i) {
 372		queue = &np->queues[i];
 373		napi_enable(&queue->napi);
 374
 375		spin_lock_bh(&queue->rx_lock);
 376		if (netif_carrier_ok(dev)) {
 377			xennet_alloc_rx_buffers(queue);
 378			queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1;
 379			if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))
 380				napi_schedule(&queue->napi);
 381		}
 382		spin_unlock_bh(&queue->rx_lock);
 383	}
 384
 385	netif_tx_start_all_queues(dev);
 386
 387	return 0;
 388}
 389
 390static bool xennet_tx_buf_gc(struct netfront_queue *queue)
 391{
 392	RING_IDX cons, prod;
 393	unsigned short id;
 394	struct sk_buff *skb;
 395	bool more_to_do;
 396	bool work_done = false;
 397	const struct device *dev = &queue->info->netdev->dev;
 398
 399	BUG_ON(!netif_carrier_ok(queue->info->netdev));
 400
 401	do {
 402		prod = queue->tx.sring->rsp_prod;
 403		if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) {
 404			dev_alert(dev, "Illegal number of responses %u\n",
 405				  prod - queue->tx.rsp_cons);
 406			goto err;
 407		}
 408		rmb(); /* Ensure we see responses up to 'rp'. */
 409
 410		for (cons = queue->tx.rsp_cons; cons != prod; cons++) {
 411			struct xen_netif_tx_response txrsp;
 412
 413			work_done = true;
 414
 415			RING_COPY_RESPONSE(&queue->tx, cons, &txrsp);
 416			if (txrsp.status == XEN_NETIF_RSP_NULL)
 417				continue;
 418
 419			id = txrsp.id;
 420			if (id >= RING_SIZE(&queue->tx)) {
 421				dev_alert(dev,
 422					  "Response has incorrect id (%u)\n",
 423					  id);
 424				goto err;
 425			}
 426			if (queue->tx_link[id] != TX_PENDING) {
 427				dev_alert(dev,
 428					  "Response for inactive request\n");
 429				goto err;
 430			}
 431
 432			queue->tx_link[id] = TX_LINK_NONE;
 433			skb = queue->tx_skbs[id];
 434			queue->tx_skbs[id] = NULL;
 435			if (unlikely(!gnttab_end_foreign_access_ref(
 436				queue->grant_tx_ref[id]))) {
 437				dev_alert(dev,
 438					  "Grant still in use by backend domain\n");
 439				goto err;
 440			}
 441			gnttab_release_grant_reference(
 442				&queue->gref_tx_head, queue->grant_tx_ref[id]);
 443			queue->grant_tx_ref[id] = INVALID_GRANT_REF;
 444			queue->grant_tx_page[id] = NULL;
 445			add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id);
 446			dev_kfree_skb_irq(skb);
 447		}
 448
 449		queue->tx.rsp_cons = prod;
 450
 451		RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do);
 452	} while (more_to_do);
 453
 454	xennet_maybe_wake_tx(queue);
 455
 456	return work_done;
 457
 458 err:
 459	queue->info->broken = true;
 460	dev_alert(dev, "Disabled for further use\n");
 461
 462	return work_done;
 463}
 464
 465struct xennet_gnttab_make_txreq {
 466	struct netfront_queue *queue;
 467	struct sk_buff *skb;
 468	struct page *page;
 469	struct xen_netif_tx_request *tx;      /* Last request on ring page */
 470	struct xen_netif_tx_request tx_local; /* Last request local copy*/
 471	unsigned int size;
 472};
 473
 474static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset,
 475				  unsigned int len, void *data)
 476{
 477	struct xennet_gnttab_make_txreq *info = data;
 478	unsigned int id;
 479	struct xen_netif_tx_request *tx;
 480	grant_ref_t ref;
 481	/* convenient aliases */
 482	struct page *page = info->page;
 483	struct netfront_queue *queue = info->queue;
 484	struct sk_buff *skb = info->skb;
 485
 486	id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link);
 487	tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
 488	ref = gnttab_claim_grant_reference(&queue->gref_tx_head);
 489	WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
 490
 491	gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id,
 492					gfn, GNTMAP_readonly);
 493
 494	queue->tx_skbs[id] = skb;
 495	queue->grant_tx_page[id] = page;
 496	queue->grant_tx_ref[id] = ref;
 497
 498	info->tx_local.id = id;
 499	info->tx_local.gref = ref;
 500	info->tx_local.offset = offset;
 501	info->tx_local.size = len;
 502	info->tx_local.flags = 0;
 503
 504	*tx = info->tx_local;
 505
 506	/*
 507	 * Put the request in the pending queue, it will be set to be pending
 508	 * when the producer index is about to be raised.
 509	 */
 510	add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id);
 511
 512	info->tx = tx;
 513	info->size += info->tx_local.size;
 514}
 515
 516static struct xen_netif_tx_request *xennet_make_first_txreq(
 517	struct xennet_gnttab_make_txreq *info,
 518	unsigned int offset, unsigned int len)
 519{
 520	info->size = 0;
 521
 522	gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info);
 523
 524	return info->tx;
 525}
 526
 527static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset,
 528				  unsigned int len, void *data)
 529{
 530	struct xennet_gnttab_make_txreq *info = data;
 531
 532	info->tx->flags |= XEN_NETTXF_more_data;
 533	skb_get(info->skb);
 534	xennet_tx_setup_grant(gfn, offset, len, data);
 535}
 536
 537static void xennet_make_txreqs(
 538	struct xennet_gnttab_make_txreq *info,
 539	struct page *page,
 540	unsigned int offset, unsigned int len)
 541{
 542	/* Skip unused frames from start of page */
 543	page += offset >> PAGE_SHIFT;
 544	offset &= ~PAGE_MASK;
 545
 546	while (len) {
 547		info->page = page;
 548		info->size = 0;
 549
 550		gnttab_foreach_grant_in_range(page, offset, len,
 551					      xennet_make_one_txreq,
 552					      info);
 553
 554		page++;
 555		offset = 0;
 556		len -= info->size;
 557	}
 558}
 559
 560/*
 561 * Count how many ring slots are required to send this skb. Each frag
 562 * might be a compound page.
 563 */
 564static int xennet_count_skb_slots(struct sk_buff *skb)
 565{
 566	int i, frags = skb_shinfo(skb)->nr_frags;
 567	int slots;
 568
 569	slots = gnttab_count_grant(offset_in_page(skb->data),
 570				   skb_headlen(skb));
 571
 572	for (i = 0; i < frags; i++) {
 573		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
 574		unsigned long size = skb_frag_size(frag);
 575		unsigned long offset = skb_frag_off(frag);
 576
 577		/* Skip unused frames from start of page */
 578		offset &= ~PAGE_MASK;
 579
 580		slots += gnttab_count_grant(offset, size);
 581	}
 582
 583	return slots;
 584}
 585
 586static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb,
 587			       struct net_device *sb_dev)
 588{
 589	unsigned int num_queues = dev->real_num_tx_queues;
 590	u32 hash;
 591	u16 queue_idx;
 592
 593	/* First, check if there is only one queue */
 594	if (num_queues == 1) {
 595		queue_idx = 0;
 596	} else {
 597		hash = skb_get_hash(skb);
 598		queue_idx = hash % num_queues;
 599	}
 600
 601	return queue_idx;
 602}
 603
 604static void xennet_mark_tx_pending(struct netfront_queue *queue)
 605{
 606	unsigned int i;
 607
 608	while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) !=
 609	       TX_LINK_NONE)
 610		queue->tx_link[i] = TX_PENDING;
 611}
 612
 613static int xennet_xdp_xmit_one(struct net_device *dev,
 614			       struct netfront_queue *queue,
 615			       struct xdp_frame *xdpf)
 616{
 617	struct netfront_info *np = netdev_priv(dev);
 618	struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
 619	struct xennet_gnttab_make_txreq info = {
 620		.queue = queue,
 621		.skb = NULL,
 622		.page = virt_to_page(xdpf->data),
 623	};
 624	int notify;
 625
 626	xennet_make_first_txreq(&info,
 627				offset_in_page(xdpf->data),
 628				xdpf->len);
 629
 630	xennet_mark_tx_pending(queue);
 631
 632	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
 633	if (notify)
 634		notify_remote_via_irq(queue->tx_irq);
 635
 636	u64_stats_update_begin(&tx_stats->syncp);
 637	tx_stats->bytes += xdpf->len;
 638	tx_stats->packets++;
 639	u64_stats_update_end(&tx_stats->syncp);
 640
 641	return 0;
 642}
 643
 644static int xennet_xdp_xmit(struct net_device *dev, int n,
 645			   struct xdp_frame **frames, u32 flags)
 646{
 647	unsigned int num_queues = dev->real_num_tx_queues;
 648	struct netfront_info *np = netdev_priv(dev);
 649	struct netfront_queue *queue = NULL;
 650	unsigned long irq_flags;
 651	int nxmit = 0;
 652	int i;
 653
 654	if (unlikely(np->broken))
 655		return -ENODEV;
 656	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 657		return -EINVAL;
 658
 659	queue = &np->queues[smp_processor_id() % num_queues];
 660
 661	spin_lock_irqsave(&queue->tx_lock, irq_flags);
 662	for (i = 0; i < n; i++) {
 663		struct xdp_frame *xdpf = frames[i];
 664
 665		if (!xdpf)
 666			continue;
 667		if (xennet_xdp_xmit_one(dev, queue, xdpf))
 668			break;
 669		nxmit++;
 670	}
 671	spin_unlock_irqrestore(&queue->tx_lock, irq_flags);
 672
 673	return nxmit;
 674}
 675
 676static struct sk_buff *bounce_skb(const struct sk_buff *skb)
 677{
 678	unsigned int headerlen = skb_headroom(skb);
 679	/* Align size to allocate full pages and avoid contiguous data leaks */
 680	unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len,
 681				  XEN_PAGE_SIZE);
 682	struct sk_buff *n = alloc_skb(size, GFP_ATOMIC | __GFP_ZERO);
 683
 684	if (!n)
 685		return NULL;
 686
 687	if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) {
 688		WARN_ONCE(1, "misaligned skb allocated\n");
 689		kfree_skb(n);
 690		return NULL;
 691	}
 692
 693	/* Set the data pointer */
 694	skb_reserve(n, headerlen);
 695	/* Set the tail pointer and length */
 696	skb_put(n, skb->len);
 697
 698	BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
 699
 700	skb_copy_header(n, skb);
 701	return n;
 702}
 703
 704#define MAX_XEN_SKB_FRAGS (65536 / XEN_PAGE_SIZE + 1)
 705
 706static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 707{
 708	struct netfront_info *np = netdev_priv(dev);
 709	struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
 710	struct xen_netif_tx_request *first_tx;
 711	unsigned int i;
 712	int notify;
 713	int slots;
 714	struct page *page;
 715	unsigned int offset;
 716	unsigned int len;
 717	unsigned long flags;
 718	struct netfront_queue *queue = NULL;
 719	struct xennet_gnttab_make_txreq info = { };
 720	unsigned int num_queues = dev->real_num_tx_queues;
 721	u16 queue_index;
 722	struct sk_buff *nskb;
 723
 724	/* Drop the packet if no queues are set up */
 725	if (num_queues < 1)
 726		goto drop;
 727	if (unlikely(np->broken))
 728		goto drop;
 729	/* Determine which queue to transmit this SKB on */
 730	queue_index = skb_get_queue_mapping(skb);
 731	queue = &np->queues[queue_index];
 732
 733	/* If skb->len is too big for wire format, drop skb and alert
 734	 * user about misconfiguration.
 735	 */
 736	if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) {
 737		net_alert_ratelimited(
 738			"xennet: skb->len = %u, too big for wire format\n",
 739			skb->len);
 740		goto drop;
 741	}
 742
 743	slots = xennet_count_skb_slots(skb);
 744	if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) {
 745		net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n",
 746				    slots, skb->len);
 747		if (skb_linearize(skb))
 748			goto drop;
 749	}
 750
 751	page = virt_to_page(skb->data);
 752	offset = offset_in_page(skb->data);
 753
 754	/* The first req should be at least ETH_HLEN size or the packet will be
 755	 * dropped by netback.
 756	 *
 757	 * If the backend is not trusted bounce all data to zeroed pages to
 758	 * avoid exposing contiguous data on the granted page not belonging to
 759	 * the skb.
 760	 */
 761	if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
 762		nskb = bounce_skb(skb);
 763		if (!nskb)
 764			goto drop;
 765		dev_consume_skb_any(skb);
 766		skb = nskb;
 767		page = virt_to_page(skb->data);
 768		offset = offset_in_page(skb->data);
 769	}
 770
 771	len = skb_headlen(skb);
 772
 773	spin_lock_irqsave(&queue->tx_lock, flags);
 774
 775	if (unlikely(!netif_carrier_ok(dev) ||
 776		     (slots > 1 && !xennet_can_sg(dev)) ||
 777		     netif_needs_gso(skb, netif_skb_features(skb)))) {
 778		spin_unlock_irqrestore(&queue->tx_lock, flags);
 779		goto drop;
 780	}
 781
 782	/* First request for the linear area. */
 783	info.queue = queue;
 784	info.skb = skb;
 785	info.page = page;
 786	first_tx = xennet_make_first_txreq(&info, offset, len);
 787	offset += info.tx_local.size;
 788	if (offset == PAGE_SIZE) {
 789		page++;
 790		offset = 0;
 791	}
 792	len -= info.tx_local.size;
 793
 794	if (skb->ip_summed == CHECKSUM_PARTIAL)
 795		/* local packet? */
 796		first_tx->flags |= XEN_NETTXF_csum_blank |
 797				   XEN_NETTXF_data_validated;
 798	else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
 799		/* remote but checksummed. */
 800		first_tx->flags |= XEN_NETTXF_data_validated;
 801
 802	/* Optional extra info after the first request. */
 803	if (skb_shinfo(skb)->gso_size) {
 804		struct xen_netif_extra_info *gso;
 805
 806		gso = (struct xen_netif_extra_info *)
 807			RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
 808
 809		first_tx->flags |= XEN_NETTXF_extra_info;
 810
 811		gso->u.gso.size = skb_shinfo(skb)->gso_size;
 812		gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ?
 813			XEN_NETIF_GSO_TYPE_TCPV6 :
 814			XEN_NETIF_GSO_TYPE_TCPV4;
 815		gso->u.gso.pad = 0;
 816		gso->u.gso.features = 0;
 817
 818		gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
 819		gso->flags = 0;
 820	}
 821
 822	/* Requests for the rest of the linear area. */
 823	xennet_make_txreqs(&info, page, offset, len);
 824
 825	/* Requests for all the frags. */
 826	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 827		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 828		xennet_make_txreqs(&info, skb_frag_page(frag),
 829					skb_frag_off(frag),
 830					skb_frag_size(frag));
 831	}
 832
 833	/* First request has the packet length. */
 834	first_tx->size = skb->len;
 835
 836	/* timestamp packet in software */
 837	skb_tx_timestamp(skb);
 838
 839	xennet_mark_tx_pending(queue);
 840
 841	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
 842	if (notify)
 843		notify_remote_via_irq(queue->tx_irq);
 844
 845	u64_stats_update_begin(&tx_stats->syncp);
 846	tx_stats->bytes += skb->len;
 847	tx_stats->packets++;
 848	u64_stats_update_end(&tx_stats->syncp);
 849
 850	if (!netfront_tx_slot_available(queue))
 851		netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));
 852
 853	spin_unlock_irqrestore(&queue->tx_lock, flags);
 854
 855	return NETDEV_TX_OK;
 856
 857 drop:
 858	dev->stats.tx_dropped++;
 859	dev_kfree_skb_any(skb);
 860	return NETDEV_TX_OK;
 861}
 862
 863static int xennet_close(struct net_device *dev)
 864{
 865	struct netfront_info *np = netdev_priv(dev);
 866	unsigned int num_queues = np->queues ? dev->real_num_tx_queues : 0;
 867	unsigned int i;
 868	struct netfront_queue *queue;
 869	netif_tx_stop_all_queues(np->netdev);
 870	for (i = 0; i < num_queues; ++i) {
 871		queue = &np->queues[i];
 872		napi_disable(&queue->napi);
 873	}
 874	return 0;
 875}
 876
 877static void xennet_destroy_queues(struct netfront_info *info)
 878{
 879	unsigned int i;
 880
 881	if (!info->queues)
 882		return;
 883
 884	for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
 885		struct netfront_queue *queue = &info->queues[i];
 886
 887		if (netif_running(info->netdev))
 888			napi_disable(&queue->napi);
 889		netif_napi_del(&queue->napi);
 890	}
 891
 892	kfree(info->queues);
 893	info->queues = NULL;
 894}
 895
 896static void xennet_uninit(struct net_device *dev)
 897{
 898	struct netfront_info *np = netdev_priv(dev);
 899	xennet_destroy_queues(np);
 900}
 901
 902static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val)
 903{
 904	unsigned long flags;
 905
 906	spin_lock_irqsave(&queue->rx_cons_lock, flags);
 907	queue->rx.rsp_cons = val;
 908	queue->rx_rsp_unconsumed = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx);
 909	spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
 910}
 911
 912static void xennet_move_rx_slot(struct netfront_queue *queue, struct sk_buff *skb,
 913				grant_ref_t ref)
 914{
 915	int new = xennet_rxidx(queue->rx.req_prod_pvt);
 916
 917	BUG_ON(queue->rx_skbs[new]);
 918	queue->rx_skbs[new] = skb;
 919	queue->grant_rx_ref[new] = ref;
 920	RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new;
 921	RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref;
 922	queue->rx.req_prod_pvt++;
 923}
 924
 925static int xennet_get_extras(struct netfront_queue *queue,
 926			     struct xen_netif_extra_info *extras,
 927			     RING_IDX rp)
 928
 929{
 930	struct xen_netif_extra_info extra;
 931	struct device *dev = &queue->info->netdev->dev;
 932	RING_IDX cons = queue->rx.rsp_cons;
 933	int err = 0;
 934
 935	do {
 936		struct sk_buff *skb;
 937		grant_ref_t ref;
 938
 939		if (unlikely(cons + 1 == rp)) {
 940			if (net_ratelimit())
 941				dev_warn(dev, "Missing extra info\n");
 942			err = -EBADR;
 943			break;
 944		}
 945
 946		RING_COPY_RESPONSE(&queue->rx, ++cons, &extra);
 947
 948		if (unlikely(!extra.type ||
 949			     extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
 950			if (net_ratelimit())
 951				dev_warn(dev, "Invalid extra type: %d\n",
 952					 extra.type);
 953			err = -EINVAL;
 954		} else {
 955			extras[extra.type - 1] = extra;
 956		}
 957
 958		skb = xennet_get_rx_skb(queue, cons);
 959		ref = xennet_get_rx_ref(queue, cons);
 960		xennet_move_rx_slot(queue, skb, ref);
 961	} while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
 962
 963	xennet_set_rx_rsp_cons(queue, cons);
 964	return err;
 965}
 966
 967static u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata,
 968		   struct xen_netif_rx_response *rx, struct bpf_prog *prog,
 969		   struct xdp_buff *xdp, bool *need_xdp_flush)
 970{
 971	struct xdp_frame *xdpf;
 972	u32 len = rx->status;
 973	u32 act;
 974	int err;
 975
 976	xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
 977		      &queue->xdp_rxq);
 978	xdp_prepare_buff(xdp, page_address(pdata), XDP_PACKET_HEADROOM,
 979			 len, false);
 980
 981	act = bpf_prog_run_xdp(prog, xdp);
 982	switch (act) {
 983	case XDP_TX:
 984		xdpf = xdp_convert_buff_to_frame(xdp);
 985		if (unlikely(!xdpf)) {
 986			trace_xdp_exception(queue->info->netdev, prog, act);
 987			break;
 988		}
 989		get_page(pdata);
 990		err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0);
 991		if (unlikely(err <= 0)) {
 992			if (err < 0)
 993				trace_xdp_exception(queue->info->netdev, prog, act);
 994			xdp_return_frame_rx_napi(xdpf);
 995		}
 996		break;
 997	case XDP_REDIRECT:
 998		get_page(pdata);
 999		err = xdp_do_redirect(queue->info->netdev, xdp, prog);
1000		*need_xdp_flush = true;
1001		if (unlikely(err)) {
1002			trace_xdp_exception(queue->info->netdev, prog, act);
1003			xdp_return_buff(xdp);
1004		}
1005		break;
1006	case XDP_PASS:
1007	case XDP_DROP:
1008		break;
1009
1010	case XDP_ABORTED:
1011		trace_xdp_exception(queue->info->netdev, prog, act);
1012		break;
1013
1014	default:
1015		bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act);
1016	}
1017
1018	return act;
1019}
1020
1021static int xennet_get_responses(struct netfront_queue *queue,
1022				struct netfront_rx_info *rinfo, RING_IDX rp,
1023				struct sk_buff_head *list,
1024				bool *need_xdp_flush)
1025{
1026	struct xen_netif_rx_response *rx = &rinfo->rx, rx_local;
1027	int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD);
1028	RING_IDX cons = queue->rx.rsp_cons;
1029	struct sk_buff *skb = xennet_get_rx_skb(queue, cons);
1030	struct xen_netif_extra_info *extras = rinfo->extras;
1031	grant_ref_t ref = xennet_get_rx_ref(queue, cons);
1032	struct device *dev = &queue->info->netdev->dev;
1033	struct bpf_prog *xdp_prog;
1034	struct xdp_buff xdp;
1035	int slots = 1;
1036	int err = 0;
1037	u32 verdict;
1038
1039	if (rx->flags & XEN_NETRXF_extra_info) {
1040		err = xennet_get_extras(queue, extras, rp);
1041		if (!err) {
1042			if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) {
1043				struct xen_netif_extra_info *xdp;
1044
1045				xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1];
1046				rx->offset = xdp->u.xdp.headroom;
1047			}
1048		}
1049		cons = queue->rx.rsp_cons;
1050	}
1051
1052	for (;;) {
1053		/*
1054		 * This definitely indicates a bug, either in this driver or in
1055		 * the backend driver. In future this should flag the bad
1056		 * situation to the system controller to reboot the backend.
1057		 */
1058		if (ref == INVALID_GRANT_REF) {
1059			if (net_ratelimit())
1060				dev_warn(dev, "Bad rx response id %d.\n",
1061					 rx->id);
1062			err = -EINVAL;
1063			goto next;
1064		}
1065
1066		if (unlikely(rx->status < 0 ||
1067			     rx->offset + rx->status > XEN_PAGE_SIZE)) {
1068			if (net_ratelimit())
1069				dev_warn(dev, "rx->offset: %u, size: %d\n",
1070					 rx->offset, rx->status);
1071			xennet_move_rx_slot(queue, skb, ref);
1072			err = -EINVAL;
1073			goto next;
1074		}
1075
1076		if (!gnttab_end_foreign_access_ref(ref)) {
1077			dev_alert(dev,
1078				  "Grant still in use by backend domain\n");
1079			queue->info->broken = true;
1080			dev_alert(dev, "Disabled for further use\n");
1081			return -EINVAL;
1082		}
1083
1084		gnttab_release_grant_reference(&queue->gref_rx_head, ref);
1085
1086		rcu_read_lock();
1087		xdp_prog = rcu_dereference(queue->xdp_prog);
1088		if (xdp_prog) {
1089			if (!(rx->flags & XEN_NETRXF_more_data)) {
1090				/* currently only a single page contains data */
1091				verdict = xennet_run_xdp(queue,
1092							 skb_frag_page(&skb_shinfo(skb)->frags[0]),
1093							 rx, xdp_prog, &xdp, need_xdp_flush);
1094				if (verdict != XDP_PASS)
1095					err = -EINVAL;
1096			} else {
1097				/* drop the frame */
1098				err = -EINVAL;
1099			}
1100		}
1101		rcu_read_unlock();
1102
1103		__skb_queue_tail(list, skb);
1104
1105next:
1106		if (!(rx->flags & XEN_NETRXF_more_data))
1107			break;
1108
1109		if (cons + slots == rp) {
1110			if (net_ratelimit())
1111				dev_warn(dev, "Need more slots\n");
1112			err = -ENOENT;
1113			break;
1114		}
1115
1116		RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local);
1117		rx = &rx_local;
1118		skb = xennet_get_rx_skb(queue, cons + slots);
1119		ref = xennet_get_rx_ref(queue, cons + slots);
1120		slots++;
1121	}
1122
1123	if (unlikely(slots > max)) {
1124		if (net_ratelimit())
1125			dev_warn(dev, "Too many slots\n");
1126		err = -E2BIG;
1127	}
1128
1129	if (unlikely(err))
1130		xennet_set_rx_rsp_cons(queue, cons + slots);
1131
1132	return err;
1133}
1134
1135static int xennet_set_skb_gso(struct sk_buff *skb,
1136			      struct xen_netif_extra_info *gso)
1137{
1138	if (!gso->u.gso.size) {
1139		if (net_ratelimit())
1140			pr_warn("GSO size must not be zero\n");
1141		return -EINVAL;
1142	}
1143
1144	if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 &&
1145	    gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) {
1146		if (net_ratelimit())
1147			pr_warn("Bad GSO type %d\n", gso->u.gso.type);
1148		return -EINVAL;
1149	}
1150
1151	skb_shinfo(skb)->gso_size = gso->u.gso.size;
1152	skb_shinfo(skb)->gso_type =
1153		(gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ?
1154		SKB_GSO_TCPV4 :
1155		SKB_GSO_TCPV6;
1156
1157	/* Header must be checked, and gso_segs computed. */
1158	skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1159	skb_shinfo(skb)->gso_segs = 0;
1160
1161	return 0;
1162}
1163
1164static int xennet_fill_frags(struct netfront_queue *queue,
1165			     struct sk_buff *skb,
1166			     struct sk_buff_head *list)
1167{
1168	RING_IDX cons = queue->rx.rsp_cons;
1169	struct sk_buff *nskb;
1170
1171	while ((nskb = __skb_dequeue(list))) {
1172		struct xen_netif_rx_response rx;
1173		skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0];
1174
1175		RING_COPY_RESPONSE(&queue->rx, ++cons, &rx);
1176
1177		if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) {
1178			unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
1179
1180			BUG_ON(pull_to < skb_headlen(skb));
1181			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
1182		}
1183		if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) {
1184			xennet_set_rx_rsp_cons(queue,
1185					       ++cons + skb_queue_len(list));
1186			kfree_skb(nskb);
1187			return -ENOENT;
1188		}
1189
1190		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
1191				skb_frag_page(nfrag),
1192				rx.offset, rx.status, PAGE_SIZE);
1193
1194		skb_shinfo(nskb)->nr_frags = 0;
1195		kfree_skb(nskb);
1196	}
1197
1198	xennet_set_rx_rsp_cons(queue, cons);
1199
1200	return 0;
1201}
1202
1203static int checksum_setup(struct net_device *dev, struct sk_buff *skb)
1204{
1205	bool recalculate_partial_csum = false;
1206
1207	/*
1208	 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
1209	 * peers can fail to set NETRXF_csum_blank when sending a GSO
1210	 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
1211	 * recalculate the partial checksum.
1212	 */
1213	if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
1214		struct netfront_info *np = netdev_priv(dev);
1215		atomic_inc(&np->rx_gso_checksum_fixup);
1216		skb->ip_summed = CHECKSUM_PARTIAL;
1217		recalculate_partial_csum = true;
1218	}
1219
1220	/* A non-CHECKSUM_PARTIAL SKB does not require setup. */
1221	if (skb->ip_summed != CHECKSUM_PARTIAL)
1222		return 0;
1223
1224	return skb_checksum_setup(skb, recalculate_partial_csum);
1225}
1226
1227static int handle_incoming_queue(struct netfront_queue *queue,
1228				 struct sk_buff_head *rxq)
1229{
1230	struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats);
1231	int packets_dropped = 0;
1232	struct sk_buff *skb;
1233
1234	while ((skb = __skb_dequeue(rxq)) != NULL) {
1235		int pull_to = NETFRONT_SKB_CB(skb)->pull_to;
1236
1237		if (pull_to > skb_headlen(skb))
1238			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
1239
1240		/* Ethernet work: Delayed to here as it peeks the header. */
1241		skb->protocol = eth_type_trans(skb, queue->info->netdev);
1242		skb_reset_network_header(skb);
1243
1244		if (checksum_setup(queue->info->netdev, skb)) {
1245			kfree_skb(skb);
1246			packets_dropped++;
1247			queue->info->netdev->stats.rx_errors++;
1248			continue;
1249		}
1250
1251		u64_stats_update_begin(&rx_stats->syncp);
1252		rx_stats->packets++;
1253		rx_stats->bytes += skb->len;
1254		u64_stats_update_end(&rx_stats->syncp);
1255
1256		/* Pass it up. */
1257		napi_gro_receive(&queue->napi, skb);
1258	}
1259
1260	return packets_dropped;
1261}
1262
1263static int xennet_poll(struct napi_struct *napi, int budget)
1264{
1265	struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi);
1266	struct net_device *dev = queue->info->netdev;
1267	struct sk_buff *skb;
1268	struct netfront_rx_info rinfo;
1269	struct xen_netif_rx_response *rx = &rinfo.rx;
1270	struct xen_netif_extra_info *extras = rinfo.extras;
1271	RING_IDX i, rp;
1272	int work_done;
1273	struct sk_buff_head rxq;
1274	struct sk_buff_head errq;
1275	struct sk_buff_head tmpq;
1276	int err;
1277	bool need_xdp_flush = false;
1278
1279	spin_lock(&queue->rx_lock);
1280
1281	skb_queue_head_init(&rxq);
1282	skb_queue_head_init(&errq);
1283	skb_queue_head_init(&tmpq);
1284
1285	rp = queue->rx.sring->rsp_prod;
1286	if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) {
1287		dev_alert(&dev->dev, "Illegal number of responses %u\n",
1288			  rp - queue->rx.rsp_cons);
1289		queue->info->broken = true;
1290		spin_unlock(&queue->rx_lock);
1291		return 0;
1292	}
1293	rmb(); /* Ensure we see queued responses up to 'rp'. */
1294
1295	i = queue->rx.rsp_cons;
1296	work_done = 0;
1297	while ((i != rp) && (work_done < budget)) {
1298		RING_COPY_RESPONSE(&queue->rx, i, rx);
1299		memset(extras, 0, sizeof(rinfo.extras));
1300
1301		err = xennet_get_responses(queue, &rinfo, rp, &tmpq,
1302					   &need_xdp_flush);
1303
1304		if (unlikely(err)) {
1305			if (queue->info->broken) {
1306				spin_unlock(&queue->rx_lock);
1307				return 0;
1308			}
1309err:
1310			while ((skb = __skb_dequeue(&tmpq)))
1311				__skb_queue_tail(&errq, skb);
1312			dev->stats.rx_errors++;
1313			i = queue->rx.rsp_cons;
1314			continue;
1315		}
1316
1317		skb = __skb_dequeue(&tmpq);
1318
1319		if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1320			struct xen_netif_extra_info *gso;
1321			gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1322
1323			if (unlikely(xennet_set_skb_gso(skb, gso))) {
1324				__skb_queue_head(&tmpq, skb);
1325				xennet_set_rx_rsp_cons(queue,
1326						       queue->rx.rsp_cons +
1327						       skb_queue_len(&tmpq));
1328				goto err;
1329			}
1330		}
1331
1332		NETFRONT_SKB_CB(skb)->pull_to = rx->status;
1333		if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD)
1334			NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD;
1335
1336		skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset);
1337		skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status);
1338		skb->data_len = rx->status;
1339		skb->len += rx->status;
1340
1341		if (unlikely(xennet_fill_frags(queue, skb, &tmpq)))
1342			goto err;
1343
1344		if (rx->flags & XEN_NETRXF_csum_blank)
1345			skb->ip_summed = CHECKSUM_PARTIAL;
1346		else if (rx->flags & XEN_NETRXF_data_validated)
1347			skb->ip_summed = CHECKSUM_UNNECESSARY;
1348
1349		__skb_queue_tail(&rxq, skb);
1350
1351		i = queue->rx.rsp_cons + 1;
1352		xennet_set_rx_rsp_cons(queue, i);
1353		work_done++;
1354	}
1355	if (need_xdp_flush)
1356		xdp_do_flush();
1357
1358	__skb_queue_purge(&errq);
1359
1360	work_done -= handle_incoming_queue(queue, &rxq);
1361
1362	xennet_alloc_rx_buffers(queue);
1363
1364	if (work_done < budget) {
1365		int more_to_do = 0;
1366
1367		napi_complete_done(napi, work_done);
1368
1369		RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do);
1370		if (more_to_do)
1371			napi_schedule(napi);
1372	}
1373
1374	spin_unlock(&queue->rx_lock);
1375
1376	return work_done;
1377}
1378
1379static int xennet_change_mtu(struct net_device *dev, int mtu)
1380{
1381	int max = xennet_can_sg(dev) ? XEN_NETIF_MAX_TX_SIZE : ETH_DATA_LEN;
1382
1383	if (mtu > max)
1384		return -EINVAL;
1385	WRITE_ONCE(dev->mtu, mtu);
1386	return 0;
1387}
1388
1389static void xennet_get_stats64(struct net_device *dev,
1390			       struct rtnl_link_stats64 *tot)
1391{
1392	struct netfront_info *np = netdev_priv(dev);
1393	int cpu;
1394
1395	for_each_possible_cpu(cpu) {
1396		struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu);
1397		struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu);
1398		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1399		unsigned int start;
1400
1401		do {
1402			start = u64_stats_fetch_begin(&tx_stats->syncp);
1403			tx_packets = tx_stats->packets;
1404			tx_bytes = tx_stats->bytes;
1405		} while (u64_stats_fetch_retry(&tx_stats->syncp, start));
1406
1407		do {
1408			start = u64_stats_fetch_begin(&rx_stats->syncp);
1409			rx_packets = rx_stats->packets;
1410			rx_bytes = rx_stats->bytes;
1411		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
1412
1413		tot->rx_packets += rx_packets;
1414		tot->tx_packets += tx_packets;
1415		tot->rx_bytes   += rx_bytes;
1416		tot->tx_bytes   += tx_bytes;
1417	}
1418
1419	tot->rx_errors  = dev->stats.rx_errors;
1420	tot->tx_dropped = dev->stats.tx_dropped;
1421}
1422
1423static void xennet_release_tx_bufs(struct netfront_queue *queue)
1424{
1425	struct sk_buff *skb;
1426	int i;
1427
1428	for (i = 0; i < NET_TX_RING_SIZE; i++) {
1429		/* Skip over entries which are actually freelist references */
1430		if (!queue->tx_skbs[i])
1431			continue;
1432
1433		skb = queue->tx_skbs[i];
1434		queue->tx_skbs[i] = NULL;
1435		get_page(queue->grant_tx_page[i]);
1436		gnttab_end_foreign_access(queue->grant_tx_ref[i],
1437					  queue->grant_tx_page[i]);
1438		queue->grant_tx_page[i] = NULL;
1439		queue->grant_tx_ref[i] = INVALID_GRANT_REF;
1440		add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i);
1441		dev_kfree_skb_irq(skb);
1442	}
1443}
1444
1445static void xennet_release_rx_bufs(struct netfront_queue *queue)
1446{
1447	int id, ref;
1448
1449	spin_lock_bh(&queue->rx_lock);
1450
1451	for (id = 0; id < NET_RX_RING_SIZE; id++) {
1452		struct sk_buff *skb;
1453		struct page *page;
1454
1455		skb = queue->rx_skbs[id];
1456		if (!skb)
1457			continue;
1458
1459		ref = queue->grant_rx_ref[id];
1460		if (ref == INVALID_GRANT_REF)
1461			continue;
1462
1463		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);
1464
1465		/* gnttab_end_foreign_access() needs a page ref until
1466		 * foreign access is ended (which may be deferred).
1467		 */
1468		get_page(page);
1469		gnttab_end_foreign_access(ref, page);
1470		queue->grant_rx_ref[id] = INVALID_GRANT_REF;
1471
1472		kfree_skb(skb);
1473	}
1474
1475	spin_unlock_bh(&queue->rx_lock);
1476}
1477
1478static netdev_features_t xennet_fix_features(struct net_device *dev,
1479	netdev_features_t features)
1480{
1481	struct netfront_info *np = netdev_priv(dev);
1482
1483	if (features & NETIF_F_SG &&
1484	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0))
1485		features &= ~NETIF_F_SG;
1486
1487	if (features & NETIF_F_IPV6_CSUM &&
1488	    !xenbus_read_unsigned(np->xbdev->otherend,
1489				  "feature-ipv6-csum-offload", 0))
1490		features &= ~NETIF_F_IPV6_CSUM;
1491
1492	if (features & NETIF_F_TSO &&
1493	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0))
1494		features &= ~NETIF_F_TSO;
1495
1496	if (features & NETIF_F_TSO6 &&
1497	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0))
1498		features &= ~NETIF_F_TSO6;
1499
1500	return features;
1501}
1502
1503static int xennet_set_features(struct net_device *dev,
1504	netdev_features_t features)
1505{
1506	if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) {
1507		netdev_info(dev, "Reducing MTU because no SG offload");
1508		dev->mtu = ETH_DATA_LEN;
1509	}
1510
1511	return 0;
1512}
1513
1514static bool xennet_handle_tx(struct netfront_queue *queue, unsigned int *eoi)
1515{
1516	unsigned long flags;
1517
1518	if (unlikely(queue->info->broken))
1519		return false;
1520
1521	spin_lock_irqsave(&queue->tx_lock, flags);
1522	if (xennet_tx_buf_gc(queue))
1523		*eoi = 0;
1524	spin_unlock_irqrestore(&queue->tx_lock, flags);
1525
1526	return true;
1527}
1528
1529static irqreturn_t xennet_tx_interrupt(int irq, void *dev_id)
1530{
1531	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
1532
1533	if (likely(xennet_handle_tx(dev_id, &eoiflag)))
1534		xen_irq_lateeoi(irq, eoiflag);
1535
1536	return IRQ_HANDLED;
1537}
1538
1539static bool xennet_handle_rx(struct netfront_queue *queue, unsigned int *eoi)
1540{
1541	unsigned int work_queued;
1542	unsigned long flags;
1543
1544	if (unlikely(queue->info->broken))
1545		return false;
1546
1547	spin_lock_irqsave(&queue->rx_cons_lock, flags);
1548	work_queued = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx);
1549	if (work_queued > queue->rx_rsp_unconsumed) {
1550		queue->rx_rsp_unconsumed = work_queued;
1551		*eoi = 0;
1552	} else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) {
1553		const struct device *dev = &queue->info->netdev->dev;
1554
1555		spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
1556		dev_alert(dev, "RX producer index going backwards\n");
1557		dev_alert(dev, "Disabled for further use\n");
1558		queue->info->broken = true;
1559		return false;
1560	}
1561	spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
1562
1563	if (likely(netif_carrier_ok(queue->info->netdev) && work_queued))
1564		napi_schedule(&queue->napi);
1565
1566	return true;
1567}
1568
1569static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
1570{
1571	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
1572
1573	if (likely(xennet_handle_rx(dev_id, &eoiflag)))
1574		xen_irq_lateeoi(irq, eoiflag);
1575
1576	return IRQ_HANDLED;
1577}
1578
1579static irqreturn_t xennet_interrupt(int irq, void *dev_id)
1580{
1581	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;
1582
1583	if (xennet_handle_tx(dev_id, &eoiflag) &&
1584	    xennet_handle_rx(dev_id, &eoiflag))
1585		xen_irq_lateeoi(irq, eoiflag);
1586
1587	return IRQ_HANDLED;
1588}
1589
1590#ifdef CONFIG_NET_POLL_CONTROLLER
1591static void xennet_poll_controller(struct net_device *dev)
1592{
1593	/* Poll each queue */
1594	struct netfront_info *info = netdev_priv(dev);
1595	unsigned int num_queues = dev->real_num_tx_queues;
1596	unsigned int i;
1597
1598	if (info->broken)
1599		return;
1600
1601	for (i = 0; i < num_queues; ++i)
1602		xennet_interrupt(0, &info->queues[i]);
1603}
1604#endif
1605
1606#define NETBACK_XDP_HEADROOM_DISABLE	0
1607#define NETBACK_XDP_HEADROOM_ENABLE	1
1608
1609static int talk_to_netback_xdp(struct netfront_info *np, int xdp)
1610{
1611	int err;
1612	unsigned short headroom;
1613
1614	headroom = xdp ? XDP_PACKET_HEADROOM : 0;
1615	err = xenbus_printf(XBT_NIL, np->xbdev->nodename,
1616			    "xdp-headroom", "%hu",
1617			    headroom);
1618	if (err)
1619		pr_warn("Error writing xdp-headroom\n");
1620
1621	return err;
1622}
1623
1624static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1625			  struct netlink_ext_ack *extack)
1626{
1627	unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM;
1628	struct netfront_info *np = netdev_priv(dev);
1629	struct bpf_prog *old_prog;
1630	unsigned int i, err;
1631
1632	if (dev->mtu > max_mtu) {
1633		netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu);
1634		return -EINVAL;
1635	}
1636
1637	if (!np->netback_has_xdp_headroom)
1638		return 0;
1639
1640	xenbus_switch_state(np->xbdev, XenbusStateReconfiguring);
1641
1642	err = talk_to_netback_xdp(np, prog ? NETBACK_XDP_HEADROOM_ENABLE :
1643				  NETBACK_XDP_HEADROOM_DISABLE);
1644	if (err)
1645		return err;
1646
1647	/* avoid the race with XDP headroom adjustment */
1648	wait_event(module_wq,
1649		   xenbus_read_driver_state(np->xbdev->otherend) ==
1650		   XenbusStateReconfigured);
1651	np->netfront_xdp_enabled = true;
1652
1653	old_prog = rtnl_dereference(np->queues[0].xdp_prog);
1654
1655	if (prog)
1656		bpf_prog_add(prog, dev->real_num_tx_queues);
1657
1658	for (i = 0; i < dev->real_num_tx_queues; ++i)
1659		rcu_assign_pointer(np->queues[i].xdp_prog, prog);
1660
1661	if (old_prog)
1662		for (i = 0; i < dev->real_num_tx_queues; ++i)
1663			bpf_prog_put(old_prog);
1664
1665	xenbus_switch_state(np->xbdev, XenbusStateConnected);
1666
1667	return 0;
1668}
1669
1670static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1671{
1672	struct netfront_info *np = netdev_priv(dev);
1673
1674	if (np->broken)
1675		return -ENODEV;
1676
1677	switch (xdp->command) {
1678	case XDP_SETUP_PROG:
1679		return xennet_xdp_set(dev, xdp->prog, xdp->extack);
1680	default:
1681		return -EINVAL;
1682	}
1683}
1684
1685static const struct net_device_ops xennet_netdev_ops = {
1686	.ndo_uninit          = xennet_uninit,
1687	.ndo_open            = xennet_open,
1688	.ndo_stop            = xennet_close,
1689	.ndo_start_xmit      = xennet_start_xmit,
1690	.ndo_change_mtu	     = xennet_change_mtu,
1691	.ndo_get_stats64     = xennet_get_stats64,
1692	.ndo_set_mac_address = eth_mac_addr,
1693	.ndo_validate_addr   = eth_validate_addr,
1694	.ndo_fix_features    = xennet_fix_features,
1695	.ndo_set_features    = xennet_set_features,
1696	.ndo_select_queue    = xennet_select_queue,
1697	.ndo_bpf            = xennet_xdp,
1698	.ndo_xdp_xmit	    = xennet_xdp_xmit,
1699#ifdef CONFIG_NET_POLL_CONTROLLER
1700	.ndo_poll_controller = xennet_poll_controller,
1701#endif
1702};
1703
1704static void xennet_free_netdev(struct net_device *netdev)
1705{
1706	struct netfront_info *np = netdev_priv(netdev);
1707
1708	free_percpu(np->rx_stats);
1709	free_percpu(np->tx_stats);
1710	free_netdev(netdev);
1711}
1712
1713static struct net_device *xennet_create_dev(struct xenbus_device *dev)
1714{
1715	int err;
1716	struct net_device *netdev;
1717	struct netfront_info *np;
1718
1719	netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues);
1720	if (!netdev)
1721		return ERR_PTR(-ENOMEM);
1722
1723	np                   = netdev_priv(netdev);
1724	np->xbdev            = dev;
1725
1726	np->queues = NULL;
1727
1728	err = -ENOMEM;
1729	np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1730	if (np->rx_stats == NULL)
1731		goto exit;
1732	np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
1733	if (np->tx_stats == NULL)
1734		goto exit;
1735
1736	netdev->netdev_ops	= &xennet_netdev_ops;
1737
1738	netdev->features        = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
1739				  NETIF_F_GSO_ROBUST;
1740	netdev->hw_features	= NETIF_F_SG |
1741				  NETIF_F_IPV6_CSUM |
1742				  NETIF_F_TSO | NETIF_F_TSO6;
1743
1744	/*
1745         * Assume that all hw features are available for now. This set
1746         * will be adjusted by the call to netdev_update_features() in
1747         * xennet_connect() which is the earliest point where we can
1748         * negotiate with the backend regarding supported features.
1749         */
1750	netdev->features |= netdev->hw_features;
1751	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
1752			       NETDEV_XDP_ACT_NDO_XMIT;
1753
1754	netdev->ethtool_ops = &xennet_ethtool_ops;
1755	netdev->min_mtu = ETH_MIN_MTU;
1756	netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE;
1757	SET_NETDEV_DEV(netdev, &dev->dev);
1758
1759	np->netdev = netdev;
1760	np->netfront_xdp_enabled = false;
1761
1762	netif_carrier_off(netdev);
1763
1764	do {
1765		xenbus_switch_state(dev, XenbusStateInitialising);
1766		err = wait_event_timeout(module_wq,
1767				 xenbus_read_driver_state(dev->otherend) !=
1768				 XenbusStateClosed &&
1769				 xenbus_read_driver_state(dev->otherend) !=
1770				 XenbusStateUnknown, XENNET_TIMEOUT);
1771	} while (!err);
1772
1773	return netdev;
1774
1775 exit:
1776	xennet_free_netdev(netdev);
1777	return ERR_PTR(err);
1778}
1779
1780/*
1781 * Entry point to this code when a new device is created.  Allocate the basic
1782 * structures and the ring buffers for communication with the backend, and
1783 * inform the backend of the appropriate details for those.
1784 */
1785static int netfront_probe(struct xenbus_device *dev,
1786			  const struct xenbus_device_id *id)
1787{
1788	int err;
1789	struct net_device *netdev;
1790	struct netfront_info *info;
1791
1792	netdev = xennet_create_dev(dev);
1793	if (IS_ERR(netdev)) {
1794		err = PTR_ERR(netdev);
1795		xenbus_dev_fatal(dev, err, "creating netdev");
1796		return err;
1797	}
1798
1799	info = netdev_priv(netdev);
1800	dev_set_drvdata(&dev->dev, info);
1801#ifdef CONFIG_SYSFS
1802	info->netdev->sysfs_groups[0] = &xennet_dev_group;
1803#endif
1804
1805	return 0;
1806}
1807
1808static void xennet_end_access(int ref, void *page)
1809{
1810	/* This frees the page as a side-effect */
1811	if (ref != INVALID_GRANT_REF)
1812		gnttab_end_foreign_access(ref, virt_to_page(page));
1813}
1814
1815static void xennet_disconnect_backend(struct netfront_info *info)
1816{
1817	unsigned int i = 0;
1818	unsigned int num_queues = info->netdev->real_num_tx_queues;
1819
1820	netif_carrier_off(info->netdev);
1821
1822	for (i = 0; i < num_queues && info->queues; ++i) {
1823		struct netfront_queue *queue = &info->queues[i];
1824
1825		timer_delete_sync(&queue->rx_refill_timer);
1826
1827		if (queue->tx_irq && (queue->tx_irq == queue->rx_irq))
1828			unbind_from_irqhandler(queue->tx_irq, queue);
1829		if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) {
1830			unbind_from_irqhandler(queue->tx_irq, queue);
1831			unbind_from_irqhandler(queue->rx_irq, queue);
1832		}
1833		queue->tx_evtchn = queue->rx_evtchn = 0;
1834		queue->tx_irq = queue->rx_irq = 0;
1835
1836		if (netif_running(info->netdev))
1837			napi_synchronize(&queue->napi);
1838
1839		xennet_release_tx_bufs(queue);
1840		xennet_release_rx_bufs(queue);
1841		gnttab_free_grant_references(queue->gref_tx_head);
1842		gnttab_free_grant_references(queue->gref_rx_head);
1843
1844		/* End access and free the pages */
1845		xennet_end_access(queue->tx_ring_ref, queue->tx.sring);
1846		xennet_end_access(queue->rx_ring_ref, queue->rx.sring);
1847
1848		queue->tx_ring_ref = INVALID_GRANT_REF;
1849		queue->rx_ring_ref = INVALID_GRANT_REF;
1850		queue->tx.sring = NULL;
1851		queue->rx.sring = NULL;
1852
1853		page_pool_destroy(queue->page_pool);
1854	}
1855}
1856
1857/*
1858 * We are reconnecting to the backend, due to a suspend/resume, or a backend
1859 * driver restart.  We tear down our netif structure and recreate it, but
1860 * leave the device-layer structures intact so that this is transparent to the
1861 * rest of the kernel.
1862 */
1863static int netfront_resume(struct xenbus_device *dev)
1864{
1865	struct netfront_info *info = dev_get_drvdata(&dev->dev);
1866
1867	dev_dbg(&dev->dev, "%s\n", dev->nodename);
1868
1869	netif_tx_lock_bh(info->netdev);
1870	netif_device_detach(info->netdev);
1871	netif_tx_unlock_bh(info->netdev);
1872
1873	xennet_disconnect_backend(info);
1874
1875	rtnl_lock();
1876	if (info->queues)
1877		xennet_destroy_queues(info);
1878	rtnl_unlock();
1879
1880	return 0;
1881}
1882
1883static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[])
1884{
1885	char *s, *e, *macstr;
1886	int i;
1887
1888	macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL);
1889	if (IS_ERR(macstr))
1890		return PTR_ERR(macstr);
1891
1892	for (i = 0; i < ETH_ALEN; i++) {
1893		mac[i] = simple_strtoul(s, &e, 16);
1894		if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) {
1895			kfree(macstr);
1896			return -ENOENT;
1897		}
1898		s = e+1;
1899	}
1900
1901	kfree(macstr);
1902	return 0;
1903}
1904
1905static int setup_netfront_single(struct netfront_queue *queue)
1906{
1907	int err;
1908
1909	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
1910	if (err < 0)
1911		goto fail;
1912
1913	err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
1914						xennet_interrupt, 0,
1915						queue->info->netdev->name,
1916						queue);
1917	if (err < 0)
1918		goto bind_fail;
1919	queue->rx_evtchn = queue->tx_evtchn;
1920	queue->rx_irq = queue->tx_irq = err;
1921
1922	return 0;
1923
1924bind_fail:
1925	xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
1926	queue->tx_evtchn = 0;
1927fail:
1928	return err;
1929}
1930
1931static int setup_netfront_split(struct netfront_queue *queue)
1932{
1933	int err;
1934
1935	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
1936	if (err < 0)
1937		goto fail;
1938	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn);
1939	if (err < 0)
1940		goto alloc_rx_evtchn_fail;
1941
1942	snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),
1943		 "%s-tx", queue->name);
1944	err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
1945						xennet_tx_interrupt, 0,
1946						queue->tx_irq_name, queue);
1947	if (err < 0)
1948		goto bind_tx_fail;
1949	queue->tx_irq = err;
1950
1951	snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),
1952		 "%s-rx", queue->name);
1953	err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn,
1954						xennet_rx_interrupt, 0,
1955						queue->rx_irq_name, queue);
1956	if (err < 0)
1957		goto bind_rx_fail;
1958	queue->rx_irq = err;
1959
1960	return 0;
1961
1962bind_rx_fail:
1963	unbind_from_irqhandler(queue->tx_irq, queue);
1964	queue->tx_irq = 0;
1965bind_tx_fail:
1966	xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn);
1967	queue->rx_evtchn = 0;
1968alloc_rx_evtchn_fail:
1969	xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
1970	queue->tx_evtchn = 0;
1971fail:
1972	return err;
1973}
1974
1975static int setup_netfront(struct xenbus_device *dev,
1976			struct netfront_queue *queue, unsigned int feature_split_evtchn)
1977{
1978	struct xen_netif_tx_sring *txs;
1979	struct xen_netif_rx_sring *rxs;
1980	int err;
1981
1982	queue->tx_ring_ref = INVALID_GRANT_REF;
1983	queue->rx_ring_ref = INVALID_GRANT_REF;
1984	queue->rx.sring = NULL;
1985	queue->tx.sring = NULL;
1986
1987	err = xenbus_setup_ring(dev, GFP_NOIO | __GFP_HIGH, (void **)&txs,
1988				1, &queue->tx_ring_ref);
1989	if (err)
1990		goto fail;
1991
1992	XEN_FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE);
1993
1994	err = xenbus_setup_ring(dev, GFP_NOIO | __GFP_HIGH, (void **)&rxs,
1995				1, &queue->rx_ring_ref);
1996	if (err)
1997		goto fail;
1998
1999	XEN_FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE);
2000
2001	if (feature_split_evtchn)
2002		err = setup_netfront_split(queue);
2003	/* setup single event channel if
2004	 *  a) feature-split-event-channels == 0
2005	 *  b) feature-split-event-channels == 1 but failed to setup
2006	 */
2007	if (!feature_split_evtchn || err)
2008		err = setup_netfront_single(queue);
2009
2010	if (err)
2011		goto fail;
2012
2013	return 0;
2014
2015 fail:
2016	xenbus_teardown_ring((void **)&queue->rx.sring, 1, &queue->rx_ring_ref);
2017	xenbus_teardown_ring((void **)&queue->tx.sring, 1, &queue->tx_ring_ref);
2018
2019	return err;
2020}
2021
2022/* Queue-specific initialisation
2023 * This used to be done in xennet_create_dev() but must now
2024 * be run per-queue.
2025 */
2026static int xennet_init_queue(struct netfront_queue *queue)
2027{
2028	unsigned short i;
2029	int err = 0;
2030	char *devid;
2031
2032	spin_lock_init(&queue->tx_lock);
2033	spin_lock_init(&queue->rx_lock);
2034	spin_lock_init(&queue->rx_cons_lock);
2035
2036	timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0);
2037
2038	devid = strrchr(queue->info->xbdev->nodename, '/') + 1;
2039	snprintf(queue->name, sizeof(queue->name), "vif%s-q%u",
2040		 devid, queue->id);
2041
2042	/* Initialise tx_skb_freelist as a free chain containing every entry. */
2043	queue->tx_skb_freelist = 0;
2044	queue->tx_pend_queue = TX_LINK_NONE;
2045	for (i = 0; i < NET_TX_RING_SIZE; i++) {
2046		queue->tx_link[i] = i + 1;
2047		queue->grant_tx_ref[i] = INVALID_GRANT_REF;
2048		queue->grant_tx_page[i] = NULL;
2049	}
2050	queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE;
2051
2052	/* Clear out rx_skbs */
2053	for (i = 0; i < NET_RX_RING_SIZE; i++) {
2054		queue->rx_skbs[i] = NULL;
2055		queue->grant_rx_ref[i] = INVALID_GRANT_REF;
2056	}
2057
2058	/* A grant for every tx ring slot */
2059	if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
2060					  &queue->gref_tx_head) < 0) {
2061		pr_alert("can't alloc tx grant refs\n");
2062		err = -ENOMEM;
2063		goto exit;
2064	}
2065
2066	/* A grant for every rx ring slot */
2067	if (gnttab_alloc_grant_references(NET_RX_RING_SIZE,
2068					  &queue->gref_rx_head) < 0) {
2069		pr_alert("can't alloc rx grant refs\n");
2070		err = -ENOMEM;
2071		goto exit_free_tx;
2072	}
2073
2074	return 0;
2075
2076 exit_free_tx:
2077	gnttab_free_grant_references(queue->gref_tx_head);
2078 exit:
2079	return err;
2080}
2081
2082static int write_queue_xenstore_keys(struct netfront_queue *queue,
2083			   struct xenbus_transaction *xbt, int write_hierarchical)
2084{
2085	/* Write the queue-specific keys into XenStore in the traditional
2086	 * way for a single queue, or in a queue subkeys for multiple
2087	 * queues.
2088	 */
2089	struct xenbus_device *dev = queue->info->xbdev;
2090	int err;
2091	const char *message;
2092	char *path;
2093	size_t pathsize;
2094
2095	/* Choose the correct place to write the keys */
2096	if (write_hierarchical) {
2097		pathsize = strlen(dev->nodename) + 10;
2098		path = kzalloc(pathsize, GFP_KERNEL);
2099		if (!path) {
2100			err = -ENOMEM;
2101			message = "out of memory while writing ring references";
2102			goto error;
2103		}
2104		snprintf(path, pathsize, "%s/queue-%u",
2105				dev->nodename, queue->id);
2106	} else {
2107		path = (char *)dev->nodename;
2108	}
2109
2110	/* Write ring references */
2111	err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u",
2112			queue->tx_ring_ref);
2113	if (err) {
2114		message = "writing tx-ring-ref";
2115		goto error;
2116	}
2117
2118	err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u",
2119			queue->rx_ring_ref);
2120	if (err) {
2121		message = "writing rx-ring-ref";
2122		goto error;
2123	}
2124
2125	/* Write event channels; taking into account both shared
2126	 * and split event channel scenarios.
2127	 */
2128	if (queue->tx_evtchn == queue->rx_evtchn) {
2129		/* Shared event channel */
2130		err = xenbus_printf(*xbt, path,
2131				"event-channel", "%u", queue->tx_evtchn);
2132		if (err) {
2133			message = "writing event-channel";
2134			goto error;
2135		}
2136	} else {
2137		/* Split event channels */
2138		err = xenbus_printf(*xbt, path,
2139				"event-channel-tx", "%u", queue->tx_evtchn);
2140		if (err) {
2141			message = "writing event-channel-tx";
2142			goto error;
2143		}
2144
2145		err = xenbus_printf(*xbt, path,
2146				"event-channel-rx", "%u", queue->rx_evtchn);
2147		if (err) {
2148			message = "writing event-channel-rx";
2149			goto error;
2150		}
2151	}
2152
2153	if (write_hierarchical)
2154		kfree(path);
2155	return 0;
2156
2157error:
2158	if (write_hierarchical)
2159		kfree(path);
2160	xenbus_dev_fatal(dev, err, "%s", message);
2161	return err;
2162}
2163
2164
2165
2166static int xennet_create_page_pool(struct netfront_queue *queue)
2167{
2168	int err;
2169	struct page_pool_params pp_params = {
2170		.order = 0,
2171		.flags = 0,
2172		.pool_size = NET_RX_RING_SIZE,
2173		.nid = NUMA_NO_NODE,
2174		.dev = &queue->info->netdev->dev,
2175		.offset = XDP_PACKET_HEADROOM,
2176		.max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
2177	};
2178
2179	queue->page_pool = page_pool_create(&pp_params);
2180	if (IS_ERR(queue->page_pool)) {
2181		err = PTR_ERR(queue->page_pool);
2182		queue->page_pool = NULL;
2183		return err;
2184	}
2185
2186	err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev,
2187			       queue->id, 0);
2188	if (err) {
2189		netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n");
2190		goto err_free_pp;
2191	}
2192
2193	err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq,
2194					 MEM_TYPE_PAGE_POOL, queue->page_pool);
2195	if (err) {
2196		netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n");
2197		goto err_unregister_rxq;
2198	}
2199	return 0;
2200
2201err_unregister_rxq:
2202	xdp_rxq_info_unreg(&queue->xdp_rxq);
2203err_free_pp:
2204	page_pool_destroy(queue->page_pool);
2205	queue->page_pool = NULL;
2206	return err;
2207}
2208
2209static int xennet_create_queues(struct netfront_info *info,
2210				unsigned int *num_queues)
2211{
2212	unsigned int i;
2213	int ret;
2214
2215	info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue),
2216			       GFP_KERNEL);
2217	if (!info->queues)
2218		return -ENOMEM;
2219
2220	for (i = 0; i < *num_queues; i++) {
2221		struct netfront_queue *queue = &info->queues[i];
2222
2223		queue->id = i;
2224		queue->info = info;
2225
2226		ret = xennet_init_queue(queue);
2227		if (ret < 0) {
2228			dev_warn(&info->xbdev->dev,
2229				 "only created %d queues\n", i);
2230			*num_queues = i;
2231			break;
2232		}
2233
2234		/* use page pool recycling instead of buddy allocator */
2235		ret = xennet_create_page_pool(queue);
2236		if (ret < 0) {
2237			dev_err(&info->xbdev->dev, "can't allocate page pool\n");
2238			*num_queues = i;
2239			return ret;
2240		}
2241
2242		netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll);
2243		if (netif_running(info->netdev))
2244			napi_enable(&queue->napi);
2245	}
2246
2247	netif_set_real_num_tx_queues(info->netdev, *num_queues);
2248
2249	if (*num_queues == 0) {
2250		dev_err(&info->xbdev->dev, "no queues\n");
2251		return -EINVAL;
2252	}
2253	return 0;
2254}
2255
2256/* Common code used when first setting up, and when resuming. */
2257static int talk_to_netback(struct xenbus_device *dev,
2258			   struct netfront_info *info)
2259{
2260	const char *message;
2261	struct xenbus_transaction xbt;
2262	int err;
2263	unsigned int feature_split_evtchn;
2264	unsigned int i = 0;
2265	unsigned int max_queues = 0;
2266	struct netfront_queue *queue = NULL;
2267	unsigned int num_queues = 1;
2268	u8 addr[ETH_ALEN];
2269
2270	info->netdev->irq = 0;
2271
2272	/* Check if backend is trusted. */
2273	info->bounce = !xennet_trusted ||
2274		       !xenbus_read_unsigned(dev->nodename, "trusted", 1);
2275
2276	/* Check if backend supports multiple queues */
2277	max_queues = xenbus_read_unsigned(info->xbdev->otherend,
2278					  "multi-queue-max-queues", 1);
2279	num_queues = min(max_queues, xennet_max_queues);
2280
2281	/* Check feature-split-event-channels */
2282	feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend,
2283					"feature-split-event-channels", 0);
2284
2285	/* Read mac addr. */
2286	err = xen_net_read_mac(dev, addr);
2287	if (err) {
2288		xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename);
2289		goto out_unlocked;
2290	}
2291	eth_hw_addr_set(info->netdev, addr);
2292
2293	info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend,
2294							      "feature-xdp-headroom", 0);
2295	if (info->netback_has_xdp_headroom) {
2296		/* set the current xen-netfront xdp state */
2297		err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ?
2298					  NETBACK_XDP_HEADROOM_ENABLE :
2299					  NETBACK_XDP_HEADROOM_DISABLE);
2300		if (err)
2301			goto out_unlocked;
2302	}
2303
2304	rtnl_lock();
2305	if (info->queues)
2306		xennet_destroy_queues(info);
2307
2308	/* For the case of a reconnect reset the "broken" indicator. */
2309	info->broken = false;
2310
2311	err = xennet_create_queues(info, &num_queues);
2312	if (err < 0) {
2313		xenbus_dev_fatal(dev, err, "creating queues");
2314		kfree(info->queues);
2315		info->queues = NULL;
2316		goto out;
2317	}
2318	rtnl_unlock();
2319
2320	/* Create shared ring, alloc event channel -- for each queue */
2321	for (i = 0; i < num_queues; ++i) {
2322		queue = &info->queues[i];
2323		err = setup_netfront(dev, queue, feature_split_evtchn);
2324		if (err)
2325			goto destroy_ring;
2326	}
2327
2328again:
2329	err = xenbus_transaction_start(&xbt);
2330	if (err) {
2331		xenbus_dev_fatal(dev, err, "starting transaction");
2332		goto destroy_ring;
2333	}
2334
2335	if (xenbus_exists(XBT_NIL,
2336			  info->xbdev->otherend, "multi-queue-max-queues")) {
2337		/* Write the number of queues */
2338		err = xenbus_printf(xbt, dev->nodename,
2339				    "multi-queue-num-queues", "%u", num_queues);
2340		if (err) {
2341			message = "writing multi-queue-num-queues";
2342			goto abort_transaction_no_dev_fatal;
2343		}
2344	}
2345
2346	if (num_queues == 1) {
2347		err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */
2348		if (err)
2349			goto abort_transaction_no_dev_fatal;
2350	} else {
2351		/* Write the keys for each queue */
2352		for (i = 0; i < num_queues; ++i) {
2353			queue = &info->queues[i];
2354			err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */
2355			if (err)
2356				goto abort_transaction_no_dev_fatal;
2357		}
2358	}
2359
2360	/* The remaining keys are not queue-specific */
2361	err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u",
2362			    1);
2363	if (err) {
2364		message = "writing request-rx-copy";
2365		goto abort_transaction;
2366	}
2367
2368	err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1);
2369	if (err) {
2370		message = "writing feature-rx-notify";
2371		goto abort_transaction;
2372	}
2373
2374	err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1);
2375	if (err) {
2376		message = "writing feature-sg";
2377		goto abort_transaction;
2378	}
2379
2380	err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1);
2381	if (err) {
2382		message = "writing feature-gso-tcpv4";
2383		goto abort_transaction;
2384	}
2385
2386	err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1");
2387	if (err) {
2388		message = "writing feature-gso-tcpv6";
2389		goto abort_transaction;
2390	}
2391
2392	err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload",
2393			   "1");
2394	if (err) {
2395		message = "writing feature-ipv6-csum-offload";
2396		goto abort_transaction;
2397	}
2398
2399	err = xenbus_transaction_end(xbt, 0);
2400	if (err) {
2401		if (err == -EAGAIN)
2402			goto again;
2403		xenbus_dev_fatal(dev, err, "completing transaction");
2404		goto destroy_ring;
2405	}
2406
2407	return 0;
2408
2409 abort_transaction:
2410	xenbus_dev_fatal(dev, err, "%s", message);
2411abort_transaction_no_dev_fatal:
2412	xenbus_transaction_end(xbt, 1);
2413 destroy_ring:
2414	xennet_disconnect_backend(info);
2415	rtnl_lock();
2416	xennet_destroy_queues(info);
2417 out:
2418	rtnl_unlock();
2419out_unlocked:
2420	device_unregister(&dev->dev);
2421	return err;
2422}
2423
2424static int xennet_connect(struct net_device *dev)
2425{
2426	struct netfront_info *np = netdev_priv(dev);
2427	unsigned int num_queues = 0;
2428	int err;
2429	unsigned int j = 0;
2430	struct netfront_queue *queue = NULL;
2431
2432	if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) {
2433		dev_info(&dev->dev,
2434			 "backend does not support copying receive path\n");
2435		return -ENODEV;
2436	}
2437
2438	err = talk_to_netback(np->xbdev, np);
2439	if (err)
2440		return err;
2441	if (np->netback_has_xdp_headroom)
2442		pr_info("backend supports XDP headroom\n");
2443	if (np->bounce)
2444		dev_info(&np->xbdev->dev,
2445			 "bouncing transmitted data to zeroed pages\n");
2446
2447	/* talk_to_netback() sets the correct number of queues */
2448	num_queues = dev->real_num_tx_queues;
2449
2450	if (dev->reg_state == NETREG_UNINITIALIZED) {
2451		err = register_netdev(dev);
2452		if (err) {
2453			pr_warn("%s: register_netdev err=%d\n", __func__, err);
2454			device_unregister(&np->xbdev->dev);
2455			return err;
2456		}
2457	}
2458
2459	rtnl_lock();
2460	netdev_update_features(dev);
2461	rtnl_unlock();
2462
2463	/*
2464	 * All public and private state should now be sane.  Get
2465	 * ready to start sending and receiving packets and give the driver
2466	 * domain a kick because we've probably just requeued some
2467	 * packets.
2468	 */
2469	netif_tx_lock_bh(np->netdev);
2470	netif_device_attach(np->netdev);
2471	netif_tx_unlock_bh(np->netdev);
2472
2473	netif_carrier_on(np->netdev);
2474	for (j = 0; j < num_queues; ++j) {
2475		queue = &np->queues[j];
2476
2477		notify_remote_via_irq(queue->tx_irq);
2478		if (queue->tx_irq != queue->rx_irq)
2479			notify_remote_via_irq(queue->rx_irq);
2480
2481		spin_lock_bh(&queue->rx_lock);
2482		xennet_alloc_rx_buffers(queue);
2483		spin_unlock_bh(&queue->rx_lock);
2484	}
2485
2486	return 0;
2487}
2488
2489/*
2490 * Callback received when the backend's state changes.
2491 */
2492static void netback_changed(struct xenbus_device *dev,
2493			    enum xenbus_state backend_state)
2494{
2495	struct netfront_info *np = dev_get_drvdata(&dev->dev);
2496	struct net_device *netdev = np->netdev;
2497
2498	dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state));
2499
2500	wake_up_all(&module_wq);
2501
2502	switch (backend_state) {
2503	case XenbusStateInitialising:
2504	case XenbusStateInitialised:
2505	case XenbusStateReconfiguring:
2506	case XenbusStateReconfigured:
2507	case XenbusStateUnknown:
2508		break;
2509
2510	case XenbusStateInitWait:
2511		if (dev->state != XenbusStateInitialising)
2512			break;
2513		if (xennet_connect(netdev) != 0)
2514			break;
2515		xenbus_switch_state(dev, XenbusStateConnected);
2516		break;
2517
2518	case XenbusStateConnected:
2519		netdev_notify_peers(netdev);
2520		break;
2521
2522	case XenbusStateClosed:
2523		if (dev->state == XenbusStateClosed)
2524			break;
2525		fallthrough;	/* Missed the backend's CLOSING state */
2526	case XenbusStateClosing:
2527		xenbus_frontend_closed(dev);
2528		break;
2529	}
2530}
2531
2532static const struct xennet_stat {
2533	char name[ETH_GSTRING_LEN];
2534	u16 offset;
2535} xennet_stats[] = {
2536	{
2537		"rx_gso_checksum_fixup",
2538		offsetof(struct netfront_info, rx_gso_checksum_fixup)
2539	},
2540};
2541
2542static int xennet_get_sset_count(struct net_device *dev, int string_set)
2543{
2544	switch (string_set) {
2545	case ETH_SS_STATS:
2546		return ARRAY_SIZE(xennet_stats);
2547	default:
2548		return -EINVAL;
2549	}
2550}
2551
2552static void xennet_get_ethtool_stats(struct net_device *dev,
2553				     struct ethtool_stats *stats, u64 * data)
2554{
2555	void *np = netdev_priv(dev);
2556	int i;
2557
2558	for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
2559		data[i] = atomic_read((atomic_t *)(np + xennet_stats[i].offset));
2560}
2561
2562static void xennet_get_strings(struct net_device *dev, u32 stringset, u8 * data)
2563{
2564	int i;
2565
2566	switch (stringset) {
2567	case ETH_SS_STATS:
2568		for (i = 0; i < ARRAY_SIZE(xennet_stats); i++)
2569			memcpy(data + i * ETH_GSTRING_LEN,
2570			       xennet_stats[i].name, ETH_GSTRING_LEN);
2571		break;
2572	}
2573}
2574
2575static const struct ethtool_ops xennet_ethtool_ops =
2576{
2577	.get_link = ethtool_op_get_link,
2578
2579	.get_sset_count = xennet_get_sset_count,
2580	.get_ethtool_stats = xennet_get_ethtool_stats,
2581	.get_strings = xennet_get_strings,
2582	.get_ts_info = ethtool_op_get_ts_info,
2583};
2584
2585#ifdef CONFIG_SYSFS
2586static ssize_t show_rxbuf(struct device *dev,
2587			  struct device_attribute *attr, char *buf)
2588{
2589	return sprintf(buf, "%lu\n", NET_RX_RING_SIZE);
2590}
2591
2592static ssize_t store_rxbuf(struct device *dev,
2593			   struct device_attribute *attr,
2594			   const char *buf, size_t len)
2595{
2596	char *endp;
2597
2598	if (!capable(CAP_NET_ADMIN))
2599		return -EPERM;
2600
2601	simple_strtoul(buf, &endp, 0);
2602	if (endp == buf)
2603		return -EBADMSG;
2604
2605	/* rxbuf_min and rxbuf_max are no longer configurable. */
2606
2607	return len;
2608}
2609
2610static DEVICE_ATTR(rxbuf_min, 0644, show_rxbuf, store_rxbuf);
2611static DEVICE_ATTR(rxbuf_max, 0644, show_rxbuf, store_rxbuf);
2612static DEVICE_ATTR(rxbuf_cur, 0444, show_rxbuf, NULL);
2613
2614static struct attribute *xennet_dev_attrs[] = {
2615	&dev_attr_rxbuf_min.attr,
2616	&dev_attr_rxbuf_max.attr,
2617	&dev_attr_rxbuf_cur.attr,
2618	NULL
2619};
2620
2621static const struct attribute_group xennet_dev_group = {
2622	.attrs = xennet_dev_attrs
2623};
2624#endif /* CONFIG_SYSFS */
2625
2626static void xennet_bus_close(struct xenbus_device *dev)
2627{
2628	int ret;
2629
2630	if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
2631		return;
2632	do {
2633		xenbus_switch_state(dev, XenbusStateClosing);
2634		ret = wait_event_timeout(module_wq,
2635				   xenbus_read_driver_state(dev->otherend) ==
2636				   XenbusStateClosing ||
2637				   xenbus_read_driver_state(dev->otherend) ==
2638				   XenbusStateClosed ||
2639				   xenbus_read_driver_state(dev->otherend) ==
2640				   XenbusStateUnknown,
2641				   XENNET_TIMEOUT);
2642	} while (!ret);
2643
2644	if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
2645		return;
2646
2647	do {
2648		xenbus_switch_state(dev, XenbusStateClosed);
2649		ret = wait_event_timeout(module_wq,
2650				   xenbus_read_driver_state(dev->otherend) ==
2651				   XenbusStateClosed ||
2652				   xenbus_read_driver_state(dev->otherend) ==
2653				   XenbusStateUnknown,
2654				   XENNET_TIMEOUT);
2655	} while (!ret);
2656}
2657
2658static void xennet_remove(struct xenbus_device *dev)
2659{
2660	struct netfront_info *info = dev_get_drvdata(&dev->dev);
2661
2662	xennet_bus_close(dev);
2663	xennet_disconnect_backend(info);
2664
2665	if (info->netdev->reg_state == NETREG_REGISTERED)
2666		unregister_netdev(info->netdev);
2667
2668	if (info->queues) {
2669		rtnl_lock();
2670		xennet_destroy_queues(info);
2671		rtnl_unlock();
2672	}
2673	xennet_free_netdev(info->netdev);
2674}
2675
2676static const struct xenbus_device_id netfront_ids[] = {
2677	{ "vif" },
2678	{ "" }
2679};
2680
2681static struct xenbus_driver netfront_driver = {
2682	.ids = netfront_ids,
2683	.probe = netfront_probe,
2684	.remove = xennet_remove,
2685	.resume = netfront_resume,
2686	.otherend_changed = netback_changed,
2687};
2688
2689static int __init netif_init(void)
2690{
2691	if (!xen_domain())
2692		return -ENODEV;
2693
2694	if (!xen_has_pv_nic_devices())
2695		return -ENODEV;
2696
2697	pr_info("Initialising Xen virtual ethernet driver\n");
2698
2699	/* Allow the number of queues to match the number of CPUs, but not exceed
2700	 * the maximum limit. If the user has not specified a value, the default
2701	 * maximum limit is 8.
2702	 */
2703	if (xennet_max_queues == 0)
2704		xennet_max_queues = min_t(unsigned int, MAX_QUEUES_DEFAULT,
2705					  num_online_cpus());
2706
2707	return xenbus_register_frontend(&netfront_driver);
2708}
2709module_init(netif_init);
2710
2711
2712static void __exit netif_exit(void)
2713{
2714	xenbus_unregister_driver(&netfront_driver);
2715}
2716module_exit(netif_exit);
2717
2718MODULE_DESCRIPTION("Xen virtual network device frontend");
2719MODULE_LICENSE("GPL");
2720MODULE_ALIAS("xen:vif");
2721MODULE_ALIAS("xennet");