netdev.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2009 - 2018 Intel Corporation. */
   3
   4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6#include <linux/bitfield.h>
   7#include <linux/delay.h>
   8#include <linux/ethtool.h>
   9#include <linux/if_vlan.h>
  10#include <linux/init.h>
  11#include <linux/ipv6.h>
  12#include <linux/mii.h>
  13#include <linux/module.h>
  14#include <linux/netdevice.h>
  15#include <linux/pagemap.h>
  16#include <linux/pci.h>
  17#include <linux/prefetch.h>
  18#include <linux/sctp.h>
  19#include <linux/slab.h>
  20#include <linux/tcp.h>
  21#include <linux/types.h>
  22#include <linux/vmalloc.h>
  23#include <net/checksum.h>
  24#include <net/ip6_checksum.h>
  25#include "igbvf.h"
  26
  27char igbvf_driver_name[] = "igbvf";
  28static const char igbvf_driver_string[] =
  29		  "Intel(R) Gigabit Virtual Function Network Driver";
  30static const char igbvf_copyright[] =
  31		  "Copyright (c) 2009 - 2012 Intel Corporation.";
  32
  33#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  34static int debug = -1;
  35module_param(debug, int, 0);
  36MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  37
  38static int igbvf_poll(struct napi_struct *napi, int budget);
  39static void igbvf_reset(struct igbvf_adapter *);
  40static void igbvf_set_interrupt_capability(struct igbvf_adapter *);
  41static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);
  42
  43static struct igbvf_info igbvf_vf_info = {
  44	.mac		= e1000_vfadapt,
  45	.flags		= 0,
  46	.pba		= 10,
  47	.init_ops	= e1000_init_function_pointers_vf,
  48};
  49
  50static struct igbvf_info igbvf_i350_vf_info = {
  51	.mac		= e1000_vfadapt_i350,
  52	.flags		= 0,
  53	.pba		= 10,
  54	.init_ops	= e1000_init_function_pointers_vf,
  55};
  56
  57static const struct igbvf_info *igbvf_info_tbl[] = {
  58	[board_vf]	= &igbvf_vf_info,
  59	[board_i350_vf]	= &igbvf_i350_vf_info,
  60};
  61
  62/**
  63 * igbvf_desc_unused - calculate if we have unused descriptors
  64 * @ring: address of receive ring structure
  65 **/
  66static int igbvf_desc_unused(struct igbvf_ring *ring)
  67{
  68	if (ring->next_to_clean > ring->next_to_use)
  69		return ring->next_to_clean - ring->next_to_use - 1;
  70
  71	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
  72}
  73
  74/**
  75 * igbvf_receive_skb - helper function to handle Rx indications
  76 * @adapter: board private structure
  77 * @netdev: pointer to netdev struct
  78 * @skb: skb to indicate to stack
  79 * @status: descriptor status field as written by hardware
  80 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
  81 * @skb: pointer to sk_buff to be indicated to stack
  82 **/
  83static void igbvf_receive_skb(struct igbvf_adapter *adapter,
  84			      struct net_device *netdev,
  85			      struct sk_buff *skb,
  86			      u32 status, __le16 vlan)
  87{
  88	u16 vid;
  89
  90	if (status & E1000_RXD_STAT_VP) {
  91		if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&
  92		    (status & E1000_RXDEXT_STATERR_LB))
  93			vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;
  94		else
  95			vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
  96		if (test_bit(vid, adapter->active_vlans))
  97			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
  98	}
  99
 100	napi_gro_receive(&adapter->rx_ring->napi, skb);
 101}
 102
 103static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,
 104					 u32 status_err, struct sk_buff *skb)
 105{
 106	skb_checksum_none_assert(skb);
 107
 108	/* Ignore Checksum bit is set or checksum is disabled through ethtool */
 109	if ((status_err & E1000_RXD_STAT_IXSM) ||
 110	    (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))
 111		return;
 112
 113	/* TCP/UDP checksum error bit is set */
 114	if (status_err &
 115	    (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
 116		/* let the stack verify checksum errors */
 117		adapter->hw_csum_err++;
 118		return;
 119	}
 120
 121	/* It must be a TCP or UDP packet with a valid checksum */
 122	if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
 123		skb->ip_summed = CHECKSUM_UNNECESSARY;
 124
 125	adapter->hw_csum_good++;
 126}
 127
 128/**
 129 * igbvf_alloc_rx_buffers - Replace used receive buffers; packet split
 130 * @rx_ring: address of ring structure to repopulate
 131 * @cleaned_count: number of buffers to repopulate
 132 **/
 133static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,
 134				   int cleaned_count)
 135{
 136	struct igbvf_adapter *adapter = rx_ring->adapter;
 137	struct net_device *netdev = adapter->netdev;
 138	struct pci_dev *pdev = adapter->pdev;
 139	union e1000_adv_rx_desc *rx_desc;
 140	struct igbvf_buffer *buffer_info;
 141	struct sk_buff *skb;
 142	unsigned int i;
 143	int bufsz;
 144
 145	i = rx_ring->next_to_use;
 146	buffer_info = &rx_ring->buffer_info[i];
 147
 148	if (adapter->rx_ps_hdr_size)
 149		bufsz = adapter->rx_ps_hdr_size;
 150	else
 151		bufsz = adapter->rx_buffer_len;
 152
 153	while (cleaned_count--) {
 154		rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
 155
 156		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
 157			if (!buffer_info->page) {
 158				buffer_info->page = alloc_page(GFP_ATOMIC);
 159				if (!buffer_info->page) {
 160					adapter->alloc_rx_buff_failed++;
 161					goto no_buffers;
 162				}
 163				buffer_info->page_offset = 0;
 164			} else {
 165				buffer_info->page_offset ^= PAGE_SIZE / 2;
 166			}
 167			buffer_info->page_dma =
 168				dma_map_page(&pdev->dev, buffer_info->page,
 169					     buffer_info->page_offset,
 170					     PAGE_SIZE / 2,
 171					     DMA_FROM_DEVICE);
 172			if (dma_mapping_error(&pdev->dev,
 173					      buffer_info->page_dma)) {
 174				__free_page(buffer_info->page);
 175				buffer_info->page = NULL;
 176				dev_err(&pdev->dev, "RX DMA map failed\n");
 177				break;
 178			}
 179		}
 180
 181		if (!buffer_info->skb) {
 182			skb = netdev_alloc_skb_ip_align(netdev, bufsz);
 183			if (!skb) {
 184				adapter->alloc_rx_buff_failed++;
 185				goto no_buffers;
 186			}
 187
 188			buffer_info->skb = skb;
 189			buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 190							  bufsz,
 191							  DMA_FROM_DEVICE);
 192			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 193				dev_kfree_skb(buffer_info->skb);
 194				buffer_info->skb = NULL;
 195				dev_err(&pdev->dev, "RX DMA map failed\n");
 196				goto no_buffers;
 197			}
 198		}
 199		/* Refresh the desc even if buffer_addrs didn't change because
 200		 * each write-back erases this info.
 201		 */
 202		if (adapter->rx_ps_hdr_size) {
 203			rx_desc->read.pkt_addr =
 204			     cpu_to_le64(buffer_info->page_dma);
 205			rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
 206		} else {
 207			rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
 208			rx_desc->read.hdr_addr = 0;
 209		}
 210
 211		i++;
 212		if (i == rx_ring->count)
 213			i = 0;
 214		buffer_info = &rx_ring->buffer_info[i];
 215	}
 216
 217no_buffers:
 218	if (rx_ring->next_to_use != i) {
 219		rx_ring->next_to_use = i;
 220		if (i == 0)
 221			i = (rx_ring->count - 1);
 222		else
 223			i--;
 224
 225		/* Force memory writes to complete before letting h/w
 226		 * know there are new descriptors to fetch.  (Only
 227		 * applicable for weak-ordered memory model archs,
 228		 * such as IA-64).
 229		*/
 230		wmb();
 231		writel(i, adapter->hw.hw_addr + rx_ring->tail);
 232	}
 233}
 234
 235/**
 236 * igbvf_clean_rx_irq - Send received data up the network stack; legacy
 237 * @adapter: board private structure
 238 * @work_done: output parameter used to indicate completed work
 239 * @work_to_do: input parameter setting limit of work
 240 *
 241 * the return value indicates whether actual cleaning was done, there
 242 * is no guarantee that everything was cleaned
 243 **/
 244static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
 245			       int *work_done, int work_to_do)
 246{
 247	struct igbvf_ring *rx_ring = adapter->rx_ring;
 248	struct net_device *netdev = adapter->netdev;
 249	struct pci_dev *pdev = adapter->pdev;
 250	union e1000_adv_rx_desc *rx_desc, *next_rxd;
 251	struct igbvf_buffer *buffer_info, *next_buffer;
 252	struct sk_buff *skb;
 253	bool cleaned = false;
 254	int cleaned_count = 0;
 255	unsigned int total_bytes = 0, total_packets = 0;
 256	unsigned int i;
 257	u32 length, hlen, staterr;
 258
 259	i = rx_ring->next_to_clean;
 260	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
 261	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 262
 263	while (staterr & E1000_RXD_STAT_DD) {
 264		if (*work_done >= work_to_do)
 265			break;
 266		(*work_done)++;
 267		rmb(); /* read descriptor and rx_buffer_info after status DD */
 268
 269		buffer_info = &rx_ring->buffer_info[i];
 270
 271		/* HW will not DMA in data larger than the given buffer, even
 272		 * if it parses the (NFS, of course) header to be larger.  In
 273		 * that case, it fills the header buffer and spills the rest
 274		 * into the page.
 275		 */
 276		hlen = le16_get_bits(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info,
 277				     E1000_RXDADV_HDRBUFLEN_MASK);
 278		if (hlen > adapter->rx_ps_hdr_size)
 279			hlen = adapter->rx_ps_hdr_size;
 280
 281		length = le16_to_cpu(rx_desc->wb.upper.length);
 282		cleaned = true;
 283		cleaned_count++;
 284
 285		skb = buffer_info->skb;
 286		prefetch(skb->data - NET_IP_ALIGN);
 287		buffer_info->skb = NULL;
 288		if (!adapter->rx_ps_hdr_size) {
 289			dma_unmap_single(&pdev->dev, buffer_info->dma,
 290					 adapter->rx_buffer_len,
 291					 DMA_FROM_DEVICE);
 292			buffer_info->dma = 0;
 293			skb_put(skb, length);
 294			goto send_up;
 295		}
 296
 297		if (!skb_shinfo(skb)->nr_frags) {
 298			dma_unmap_single(&pdev->dev, buffer_info->dma,
 299					 adapter->rx_ps_hdr_size,
 300					 DMA_FROM_DEVICE);
 301			buffer_info->dma = 0;
 302			skb_put(skb, hlen);
 303		}
 304
 305		if (length) {
 306			dma_unmap_page(&pdev->dev, buffer_info->page_dma,
 307				       PAGE_SIZE / 2,
 308				       DMA_FROM_DEVICE);
 309			buffer_info->page_dma = 0;
 310
 311			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 312					   buffer_info->page,
 313					   buffer_info->page_offset,
 314					   length);
 315
 316			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
 317			    (page_count(buffer_info->page) != 1))
 318				buffer_info->page = NULL;
 319			else
 320				get_page(buffer_info->page);
 321
 322			skb->len += length;
 323			skb->data_len += length;
 324			skb->truesize += PAGE_SIZE / 2;
 325		}
 326send_up:
 327		i++;
 328		if (i == rx_ring->count)
 329			i = 0;
 330		next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);
 331		prefetch(next_rxd);
 332		next_buffer = &rx_ring->buffer_info[i];
 333
 334		if (!(staterr & E1000_RXD_STAT_EOP)) {
 335			buffer_info->skb = next_buffer->skb;
 336			buffer_info->dma = next_buffer->dma;
 337			next_buffer->skb = skb;
 338			next_buffer->dma = 0;
 339			goto next_desc;
 340		}
 341
 342		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
 343			dev_kfree_skb_irq(skb);
 344			goto next_desc;
 345		}
 346
 347		total_bytes += skb->len;
 348		total_packets++;
 349
 350		igbvf_rx_checksum_adv(adapter, staterr, skb);
 351
 352		skb->protocol = eth_type_trans(skb, netdev);
 353
 354		igbvf_receive_skb(adapter, netdev, skb, staterr,
 355				  rx_desc->wb.upper.vlan);
 356
 357next_desc:
 358		rx_desc->wb.upper.status_error = 0;
 359
 360		/* return some buffers to hardware, one at a time is too slow */
 361		if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {
 362			igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
 363			cleaned_count = 0;
 364		}
 365
 366		/* use prefetched values */
 367		rx_desc = next_rxd;
 368		buffer_info = next_buffer;
 369
 370		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 371	}
 372
 373	rx_ring->next_to_clean = i;
 374	cleaned_count = igbvf_desc_unused(rx_ring);
 375
 376	if (cleaned_count)
 377		igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
 378
 379	adapter->total_rx_packets += total_packets;
 380	adapter->total_rx_bytes += total_bytes;
 381	netdev->stats.rx_bytes += total_bytes;
 382	netdev->stats.rx_packets += total_packets;
 383	return cleaned;
 384}
 385
 386static void igbvf_put_txbuf(struct igbvf_adapter *adapter,
 387			    struct igbvf_buffer *buffer_info)
 388{
 389	if (buffer_info->dma) {
 390		if (buffer_info->mapped_as_page)
 391			dma_unmap_page(&adapter->pdev->dev,
 392				       buffer_info->dma,
 393				       buffer_info->length,
 394				       DMA_TO_DEVICE);
 395		else
 396			dma_unmap_single(&adapter->pdev->dev,
 397					 buffer_info->dma,
 398					 buffer_info->length,
 399					 DMA_TO_DEVICE);
 400		buffer_info->dma = 0;
 401	}
 402	if (buffer_info->skb) {
 403		dev_kfree_skb_any(buffer_info->skb);
 404		buffer_info->skb = NULL;
 405	}
 406	buffer_info->time_stamp = 0;
 407}
 408
 409/**
 410 * igbvf_setup_tx_resources - allocate Tx resources (Descriptors)
 411 * @adapter: board private structure
 412 * @tx_ring: ring being initialized
 413 *
 414 * Return 0 on success, negative on failure
 415 **/
 416int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,
 417			     struct igbvf_ring *tx_ring)
 418{
 419	struct pci_dev *pdev = adapter->pdev;
 420	int size;
 421
 422	size = sizeof(struct igbvf_buffer) * tx_ring->count;
 423	tx_ring->buffer_info = vzalloc(size);
 424	if (!tx_ring->buffer_info)
 425		goto err;
 426
 427	/* round up to nearest 4K */
 428	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
 429	tx_ring->size = ALIGN(tx_ring->size, 4096);
 430
 431	tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
 432					   &tx_ring->dma, GFP_KERNEL);
 433	if (!tx_ring->desc)
 434		goto err;
 435
 436	tx_ring->adapter = adapter;
 437	tx_ring->next_to_use = 0;
 438	tx_ring->next_to_clean = 0;
 439
 440	return 0;
 441err:
 442	vfree(tx_ring->buffer_info);
 443	dev_err(&adapter->pdev->dev,
 444		"Unable to allocate memory for the transmit descriptor ring\n");
 445	return -ENOMEM;
 446}
 447
 448/**
 449 * igbvf_setup_rx_resources - allocate Rx resources (Descriptors)
 450 * @adapter: board private structure
 451 * @rx_ring: ring being initialized
 452 *
 453 * Returns 0 on success, negative on failure
 454 **/
 455int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,
 456			     struct igbvf_ring *rx_ring)
 457{
 458	struct pci_dev *pdev = adapter->pdev;
 459	int size, desc_len;
 460
 461	size = sizeof(struct igbvf_buffer) * rx_ring->count;
 462	rx_ring->buffer_info = vzalloc(size);
 463	if (!rx_ring->buffer_info)
 464		goto err;
 465
 466	desc_len = sizeof(union e1000_adv_rx_desc);
 467
 468	/* Round up to nearest 4K */
 469	rx_ring->size = rx_ring->count * desc_len;
 470	rx_ring->size = ALIGN(rx_ring->size, 4096);
 471
 472	rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
 473					   &rx_ring->dma, GFP_KERNEL);
 474	if (!rx_ring->desc)
 475		goto err;
 476
 477	rx_ring->next_to_clean = 0;
 478	rx_ring->next_to_use = 0;
 479
 480	rx_ring->adapter = adapter;
 481
 482	return 0;
 483
 484err:
 485	vfree(rx_ring->buffer_info);
 486	rx_ring->buffer_info = NULL;
 487	dev_err(&adapter->pdev->dev,
 488		"Unable to allocate memory for the receive descriptor ring\n");
 489	return -ENOMEM;
 490}
 491
 492/**
 493 * igbvf_clean_tx_ring - Free Tx Buffers
 494 * @tx_ring: ring to be cleaned
 495 **/
 496static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)
 497{
 498	struct igbvf_adapter *adapter = tx_ring->adapter;
 499	struct igbvf_buffer *buffer_info;
 500	unsigned long size;
 501	unsigned int i;
 502
 503	if (!tx_ring->buffer_info)
 504		return;
 505
 506	/* Free all the Tx ring sk_buffs */
 507	for (i = 0; i < tx_ring->count; i++) {
 508		buffer_info = &tx_ring->buffer_info[i];
 509		igbvf_put_txbuf(adapter, buffer_info);
 510	}
 511
 512	size = sizeof(struct igbvf_buffer) * tx_ring->count;
 513	memset(tx_ring->buffer_info, 0, size);
 514
 515	/* Zero out the descriptor ring */
 516	memset(tx_ring->desc, 0, tx_ring->size);
 517
 518	tx_ring->next_to_use = 0;
 519	tx_ring->next_to_clean = 0;
 520
 521	writel(0, adapter->hw.hw_addr + tx_ring->head);
 522	writel(0, adapter->hw.hw_addr + tx_ring->tail);
 523}
 524
 525/**
 526 * igbvf_free_tx_resources - Free Tx Resources per Queue
 527 * @tx_ring: ring to free resources from
 528 *
 529 * Free all transmit software resources
 530 **/
 531void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)
 532{
 533	struct pci_dev *pdev = tx_ring->adapter->pdev;
 534
 535	igbvf_clean_tx_ring(tx_ring);
 536
 537	vfree(tx_ring->buffer_info);
 538	tx_ring->buffer_info = NULL;
 539
 540	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
 541			  tx_ring->dma);
 542
 543	tx_ring->desc = NULL;
 544}
 545
 546/**
 547 * igbvf_clean_rx_ring - Free Rx Buffers per Queue
 548 * @rx_ring: ring structure pointer to free buffers from
 549 **/
 550static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)
 551{
 552	struct igbvf_adapter *adapter = rx_ring->adapter;
 553	struct igbvf_buffer *buffer_info;
 554	struct pci_dev *pdev = adapter->pdev;
 555	unsigned long size;
 556	unsigned int i;
 557
 558	if (!rx_ring->buffer_info)
 559		return;
 560
 561	/* Free all the Rx ring sk_buffs */
 562	for (i = 0; i < rx_ring->count; i++) {
 563		buffer_info = &rx_ring->buffer_info[i];
 564		if (buffer_info->dma) {
 565			if (adapter->rx_ps_hdr_size) {
 566				dma_unmap_single(&pdev->dev, buffer_info->dma,
 567						 adapter->rx_ps_hdr_size,
 568						 DMA_FROM_DEVICE);
 569			} else {
 570				dma_unmap_single(&pdev->dev, buffer_info->dma,
 571						 adapter->rx_buffer_len,
 572						 DMA_FROM_DEVICE);
 573			}
 574			buffer_info->dma = 0;
 575		}
 576
 577		if (buffer_info->skb) {
 578			dev_kfree_skb(buffer_info->skb);
 579			buffer_info->skb = NULL;
 580		}
 581
 582		if (buffer_info->page) {
 583			if (buffer_info->page_dma)
 584				dma_unmap_page(&pdev->dev,
 585					       buffer_info->page_dma,
 586					       PAGE_SIZE / 2,
 587					       DMA_FROM_DEVICE);
 588			put_page(buffer_info->page);
 589			buffer_info->page = NULL;
 590			buffer_info->page_dma = 0;
 591			buffer_info->page_offset = 0;
 592		}
 593	}
 594
 595	size = sizeof(struct igbvf_buffer) * rx_ring->count;
 596	memset(rx_ring->buffer_info, 0, size);
 597
 598	/* Zero out the descriptor ring */
 599	memset(rx_ring->desc, 0, rx_ring->size);
 600
 601	rx_ring->next_to_clean = 0;
 602	rx_ring->next_to_use = 0;
 603
 604	writel(0, adapter->hw.hw_addr + rx_ring->head);
 605	writel(0, adapter->hw.hw_addr + rx_ring->tail);
 606}
 607
 608/**
 609 * igbvf_free_rx_resources - Free Rx Resources
 610 * @rx_ring: ring to clean the resources from
 611 *
 612 * Free all receive software resources
 613 **/
 614
 615void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
 616{
 617	struct pci_dev *pdev = rx_ring->adapter->pdev;
 618
 619	igbvf_clean_rx_ring(rx_ring);
 620
 621	vfree(rx_ring->buffer_info);
 622	rx_ring->buffer_info = NULL;
 623
 624	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
 625			  rx_ring->dma);
 626	rx_ring->desc = NULL;
 627}
 628
 629/**
 630 * igbvf_update_itr - update the dynamic ITR value based on statistics
 631 * @adapter: pointer to adapter
 632 * @itr_setting: current adapter->itr
 633 * @packets: the number of packets during this measurement interval
 634 * @bytes: the number of bytes during this measurement interval
 635 *
 636 * Stores a new ITR value based on packets and byte counts during the last
 637 * interrupt.  The advantage of per interrupt computation is faster updates
 638 * and more accurate ITR for the current traffic pattern.  Constants in this
 639 * function were computed based on theoretical maximum wire speed and thresholds
 640 * were set based on testing data as well as attempting to minimize response
 641 * time while increasing bulk throughput.
 642 **/
 643static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
 644					   enum latency_range itr_setting,
 645					   int packets, int bytes)
 646{
 647	enum latency_range retval = itr_setting;
 648
 649	if (packets == 0)
 650		goto update_itr_done;
 651
 652	switch (itr_setting) {
 653	case lowest_latency:
 654		/* handle TSO and jumbo frames */
 655		if (bytes/packets > 8000)
 656			retval = bulk_latency;
 657		else if ((packets < 5) && (bytes > 512))
 658			retval = low_latency;
 659		break;
 660	case low_latency:  /* 50 usec aka 20000 ints/s */
 661		if (bytes > 10000) {
 662			/* this if handles the TSO accounting */
 663			if (bytes/packets > 8000)
 664				retval = bulk_latency;
 665			else if ((packets < 10) || ((bytes/packets) > 1200))
 666				retval = bulk_latency;
 667			else if ((packets > 35))
 668				retval = lowest_latency;
 669		} else if (bytes/packets > 2000) {
 670			retval = bulk_latency;
 671		} else if (packets <= 2 && bytes < 512) {
 672			retval = lowest_latency;
 673		}
 674		break;
 675	case bulk_latency: /* 250 usec aka 4000 ints/s */
 676		if (bytes > 25000) {
 677			if (packets > 35)
 678				retval = low_latency;
 679		} else if (bytes < 6000) {
 680			retval = low_latency;
 681		}
 682		break;
 683	default:
 684		break;
 685	}
 686
 687update_itr_done:
 688	return retval;
 689}
 690
 691static int igbvf_range_to_itr(enum latency_range current_range)
 692{
 693	int new_itr;
 694
 695	switch (current_range) {
 696	/* counts and packets in update_itr are dependent on these numbers */
 697	case lowest_latency:
 698		new_itr = IGBVF_70K_ITR;
 699		break;
 700	case low_latency:
 701		new_itr = IGBVF_20K_ITR;
 702		break;
 703	case bulk_latency:
 704		new_itr = IGBVF_4K_ITR;
 705		break;
 706	default:
 707		new_itr = IGBVF_START_ITR;
 708		break;
 709	}
 710	return new_itr;
 711}
 712
 713static void igbvf_set_itr(struct igbvf_adapter *adapter)
 714{
 715	u32 new_itr;
 716
 717	adapter->tx_ring->itr_range =
 718			igbvf_update_itr(adapter,
 719					 adapter->tx_ring->itr_val,
 720					 adapter->total_tx_packets,
 721					 adapter->total_tx_bytes);
 722
 723	/* conservative mode (itr 3) eliminates the lowest_latency setting */
 724	if (adapter->requested_itr == 3 &&
 725	    adapter->tx_ring->itr_range == lowest_latency)
 726		adapter->tx_ring->itr_range = low_latency;
 727
 728	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);
 729
 730	if (new_itr != adapter->tx_ring->itr_val) {
 731		u32 current_itr = adapter->tx_ring->itr_val;
 732		/* this attempts to bias the interrupt rate towards Bulk
 733		 * by adding intermediate steps when interrupt rate is
 734		 * increasing
 735		 */
 736		new_itr = new_itr > current_itr ?
 737			  min(current_itr + (new_itr >> 2), new_itr) :
 738			  new_itr;
 739		adapter->tx_ring->itr_val = new_itr;
 740
 741		adapter->tx_ring->set_itr = 1;
 742	}
 743
 744	adapter->rx_ring->itr_range =
 745			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,
 746					 adapter->total_rx_packets,
 747					 adapter->total_rx_bytes);
 748	if (adapter->requested_itr == 3 &&
 749	    adapter->rx_ring->itr_range == lowest_latency)
 750		adapter->rx_ring->itr_range = low_latency;
 751
 752	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);
 753
 754	if (new_itr != adapter->rx_ring->itr_val) {
 755		u32 current_itr = adapter->rx_ring->itr_val;
 756
 757		new_itr = new_itr > current_itr ?
 758			  min(current_itr + (new_itr >> 2), new_itr) :
 759			  new_itr;
 760		adapter->rx_ring->itr_val = new_itr;
 761
 762		adapter->rx_ring->set_itr = 1;
 763	}
 764}
 765
 766/**
 767 * igbvf_clean_tx_irq - Reclaim resources after transmit completes
 768 * @tx_ring: ring structure to clean descriptors from
 769 *
 770 * returns true if ring is completely cleaned
 771 **/
 772static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
 773{
 774	struct igbvf_adapter *adapter = tx_ring->adapter;
 775	struct net_device *netdev = adapter->netdev;
 776	struct igbvf_buffer *buffer_info;
 777	struct sk_buff *skb;
 778	union e1000_adv_tx_desc *tx_desc, *eop_desc;
 779	unsigned int total_bytes = 0, total_packets = 0;
 780	unsigned int i, count = 0;
 781	bool cleaned = false;
 782
 783	i = tx_ring->next_to_clean;
 784	buffer_info = &tx_ring->buffer_info[i];
 785	eop_desc = buffer_info->next_to_watch;
 786
 787	do {
 788		/* if next_to_watch is not set then there is no work pending */
 789		if (!eop_desc)
 790			break;
 791
 792		/* prevent any other reads prior to eop_desc */
 793		smp_rmb();
 794
 795		/* if DD is not set pending work has not been completed */
 796		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
 797			break;
 798
 799		/* clear next_to_watch to prevent false hangs */
 800		buffer_info->next_to_watch = NULL;
 801
 802		for (cleaned = false; !cleaned; count++) {
 803			tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
 804			cleaned = (tx_desc == eop_desc);
 805			skb = buffer_info->skb;
 806
 807			if (skb) {
 808				unsigned int segs, bytecount;
 809
 810				/* gso_segs is currently only valid for tcp */
 811				segs = skb_shinfo(skb)->gso_segs ?: 1;
 812				/* multiply data chunks by size of headers */
 813				bytecount = ((segs - 1) * skb_headlen(skb)) +
 814					    skb->len;
 815				total_packets += segs;
 816				total_bytes += bytecount;
 817			}
 818
 819			igbvf_put_txbuf(adapter, buffer_info);
 820			tx_desc->wb.status = 0;
 821
 822			i++;
 823			if (i == tx_ring->count)
 824				i = 0;
 825
 826			buffer_info = &tx_ring->buffer_info[i];
 827		}
 828
 829		eop_desc = buffer_info->next_to_watch;
 830	} while (count < tx_ring->count);
 831
 832	tx_ring->next_to_clean = i;
 833
 834	if (unlikely(count && netif_carrier_ok(netdev) &&
 835	    igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {
 836		/* Make sure that anybody stopping the queue after this
 837		 * sees the new next_to_clean.
 838		 */
 839		smp_mb();
 840		if (netif_queue_stopped(netdev) &&
 841		    !(test_bit(__IGBVF_DOWN, &adapter->state))) {
 842			netif_wake_queue(netdev);
 843			++adapter->restart_queue;
 844		}
 845	}
 846
 847	netdev->stats.tx_bytes += total_bytes;
 848	netdev->stats.tx_packets += total_packets;
 849	return count < tx_ring->count;
 850}
 851
 852static irqreturn_t igbvf_msix_other(int irq, void *data)
 853{
 854	struct net_device *netdev = data;
 855	struct igbvf_adapter *adapter = netdev_priv(netdev);
 856	struct e1000_hw *hw = &adapter->hw;
 857
 858	hw->mac.get_link_status = 1;
 859	if (!test_bit(__IGBVF_DOWN, &adapter->state))
 860		mod_timer(&adapter->watchdog_timer, jiffies + 1);
 861
 862	ew32(EIMS, adapter->eims_other);
 863
 864	return IRQ_HANDLED;
 865}
 866
 867static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
 868{
 869	struct net_device *netdev = data;
 870	struct igbvf_adapter *adapter = netdev_priv(netdev);
 871	struct e1000_hw *hw = &adapter->hw;
 872	struct igbvf_ring *tx_ring = adapter->tx_ring;
 873
 874	if (tx_ring->set_itr) {
 875		writel(tx_ring->itr_val,
 876		       adapter->hw.hw_addr + tx_ring->itr_register);
 877		adapter->tx_ring->set_itr = 0;
 878	}
 879
 880	adapter->total_tx_bytes = 0;
 881	adapter->total_tx_packets = 0;
 882
 883	/* auto mask will automatically re-enable the interrupt when we write
 884	 * EICS
 885	 */
 886	if (!igbvf_clean_tx_irq(tx_ring))
 887		/* Ring was not completely cleaned, so fire another interrupt */
 888		ew32(EICS, tx_ring->eims_value);
 889	else
 890		ew32(EIMS, tx_ring->eims_value);
 891
 892	return IRQ_HANDLED;
 893}
 894
 895static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)
 896{
 897	struct net_device *netdev = data;
 898	struct igbvf_adapter *adapter = netdev_priv(netdev);
 899
 900	/* Write the ITR value calculated at the end of the
 901	 * previous interrupt.
 902	 */
 903	if (adapter->rx_ring->set_itr) {
 904		writel(adapter->rx_ring->itr_val,
 905		       adapter->hw.hw_addr + adapter->rx_ring->itr_register);
 906		adapter->rx_ring->set_itr = 0;
 907	}
 908
 909	if (napi_schedule_prep(&adapter->rx_ring->napi)) {
 910		adapter->total_rx_bytes = 0;
 911		adapter->total_rx_packets = 0;
 912		__napi_schedule(&adapter->rx_ring->napi);
 913	}
 914
 915	return IRQ_HANDLED;
 916}
 917
 918#define IGBVF_NO_QUEUE -1
 919
 920static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,
 921				int tx_queue, int msix_vector)
 922{
 923	struct e1000_hw *hw = &adapter->hw;
 924	u32 ivar, index;
 925
 926	/* 82576 uses a table-based method for assigning vectors.
 927	 * Each queue has a single entry in the table to which we write
 928	 * a vector number along with a "valid" bit.  Sadly, the layout
 929	 * of the table is somewhat counterintuitive.
 930	 */
 931	if (rx_queue > IGBVF_NO_QUEUE) {
 932		index = (rx_queue >> 1);
 933		ivar = array_er32(IVAR0, index);
 934		if (rx_queue & 0x1) {
 935			/* vector goes into third byte of register */
 936			ivar = ivar & 0xFF00FFFF;
 937			ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 938		} else {
 939			/* vector goes into low byte of register */
 940			ivar = ivar & 0xFFFFFF00;
 941			ivar |= msix_vector | E1000_IVAR_VALID;
 942		}
 943		adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);
 944		array_ew32(IVAR0, index, ivar);
 945	}
 946	if (tx_queue > IGBVF_NO_QUEUE) {
 947		index = (tx_queue >> 1);
 948		ivar = array_er32(IVAR0, index);
 949		if (tx_queue & 0x1) {
 950			/* vector goes into high byte of register */
 951			ivar = ivar & 0x00FFFFFF;
 952			ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 953		} else {
 954			/* vector goes into second byte of register */
 955			ivar = ivar & 0xFFFF00FF;
 956			ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 957		}
 958		adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);
 959		array_ew32(IVAR0, index, ivar);
 960	}
 961}
 962
 963/**
 964 * igbvf_configure_msix - Configure MSI-X hardware
 965 * @adapter: board private structure
 966 *
 967 * igbvf_configure_msix sets up the hardware to properly
 968 * generate MSI-X interrupts.
 969 **/
 970static void igbvf_configure_msix(struct igbvf_adapter *adapter)
 971{
 972	u32 tmp;
 973	struct e1000_hw *hw = &adapter->hw;
 974	struct igbvf_ring *tx_ring = adapter->tx_ring;
 975	struct igbvf_ring *rx_ring = adapter->rx_ring;
 976	int vector = 0;
 977
 978	adapter->eims_enable_mask = 0;
 979
 980	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);
 981	adapter->eims_enable_mask |= tx_ring->eims_value;
 982	writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);
 983	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);
 984	adapter->eims_enable_mask |= rx_ring->eims_value;
 985	writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);
 986
 987	/* set vector for other causes, i.e. link changes */
 988
 989	tmp = (vector++ | E1000_IVAR_VALID);
 990
 991	ew32(IVAR_MISC, tmp);
 992
 993	adapter->eims_enable_mask = GENMASK(vector - 1, 0);
 994	adapter->eims_other = BIT(vector - 1);
 995	e1e_flush();
 996}
 997
 998static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)
 999{
1000	if (adapter->msix_entries) {
1001		pci_disable_msix(adapter->pdev);
1002		kfree(adapter->msix_entries);
1003		adapter->msix_entries = NULL;
1004	}
1005}
1006
1007/**
1008 * igbvf_set_interrupt_capability - set MSI or MSI-X if supported
1009 * @adapter: board private structure
1010 *
1011 * Attempt to configure interrupts using the best available
1012 * capabilities of the hardware and kernel.
1013 **/
1014static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)
1015{
1016	int err = -ENOMEM;
1017	int i;
1018
1019	/* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */
1020	adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),
1021					GFP_KERNEL);
1022	if (adapter->msix_entries) {
1023		for (i = 0; i < 3; i++)
1024			adapter->msix_entries[i].entry = i;
1025
1026		err = pci_enable_msix_range(adapter->pdev,
1027					    adapter->msix_entries, 3, 3);
1028	}
1029
1030	if (err < 0) {
1031		/* MSI-X failed */
1032		dev_err(&adapter->pdev->dev,
1033			"Failed to initialize MSI-X interrupts.\n");
1034		igbvf_reset_interrupt_capability(adapter);
1035	}
1036}
1037
1038/**
1039 * igbvf_request_msix - Initialize MSI-X interrupts
1040 * @adapter: board private structure
1041 *
1042 * igbvf_request_msix allocates MSI-X vectors and requests interrupts from the
1043 * kernel.
1044 **/
1045static int igbvf_request_msix(struct igbvf_adapter *adapter)
1046{
1047	struct net_device *netdev = adapter->netdev;
1048	int err = 0, vector = 0;
1049
1050	if (strlen(netdev->name) < (IFNAMSIZ - 5)) {
1051		sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);
1052		sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);
1053	} else {
1054		memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1055		memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1056	}
1057
1058	err = request_irq(adapter->msix_entries[vector].vector,
1059			  igbvf_intr_msix_tx, 0, adapter->tx_ring->name,
1060			  netdev);
1061	if (err)
1062		goto out;
1063
1064	adapter->tx_ring->itr_register = E1000_EITR(vector);
1065	adapter->tx_ring->itr_val = adapter->current_itr;
1066	vector++;
1067
1068	err = request_irq(adapter->msix_entries[vector].vector,
1069			  igbvf_intr_msix_rx, 0, adapter->rx_ring->name,
1070			  netdev);
1071	if (err)
1072		goto free_irq_tx;
1073
1074	adapter->rx_ring->itr_register = E1000_EITR(vector);
1075	adapter->rx_ring->itr_val = adapter->current_itr;
1076	vector++;
1077
1078	err = request_irq(adapter->msix_entries[vector].vector,
1079			  igbvf_msix_other, 0, netdev->name, netdev);
1080	if (err)
1081		goto free_irq_rx;
1082
1083	igbvf_configure_msix(adapter);
1084	return 0;
1085free_irq_rx:
1086	free_irq(adapter->msix_entries[--vector].vector, netdev);
1087free_irq_tx:
1088	free_irq(adapter->msix_entries[--vector].vector, netdev);
1089out:
1090	return err;
1091}
1092
1093/**
1094 * igbvf_alloc_queues - Allocate memory for all rings
1095 * @adapter: board private structure to initialize
1096 **/
1097static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
1098{
1099	struct net_device *netdev = adapter->netdev;
1100
1101	adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1102	if (!adapter->tx_ring)
1103		return -ENOMEM;
1104
1105	adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1106	if (!adapter->rx_ring) {
1107		kfree(adapter->tx_ring);
1108		return -ENOMEM;
1109	}
1110
1111	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);
1112
1113	return 0;
1114}
1115
1116/**
1117 * igbvf_request_irq - initialize interrupts
1118 * @adapter: board private structure
1119 *
1120 * Attempts to configure interrupts using the best available
1121 * capabilities of the hardware and kernel.
1122 **/
1123static int igbvf_request_irq(struct igbvf_adapter *adapter)
1124{
1125	int err = -1;
1126
1127	/* igbvf supports msi-x only */
1128	if (adapter->msix_entries)
1129		err = igbvf_request_msix(adapter);
1130
1131	if (!err)
1132		return err;
1133
1134	dev_err(&adapter->pdev->dev,
1135		"Unable to allocate interrupt, Error: %d\n", err);
1136
1137	return err;
1138}
1139
1140static void igbvf_free_irq(struct igbvf_adapter *adapter)
1141{
1142	struct net_device *netdev = adapter->netdev;
1143	int vector;
1144
1145	if (adapter->msix_entries) {
1146		for (vector = 0; vector < 3; vector++)
1147			free_irq(adapter->msix_entries[vector].vector, netdev);
1148	}
1149}
1150
1151/**
1152 * igbvf_irq_disable - Mask off interrupt generation on the NIC
1153 * @adapter: board private structure
1154 **/
1155static void igbvf_irq_disable(struct igbvf_adapter *adapter)
1156{
1157	struct e1000_hw *hw = &adapter->hw;
1158
1159	ew32(EIMC, ~0);
1160
1161	if (adapter->msix_entries)
1162		ew32(EIAC, 0);
1163}
1164
1165/**
1166 * igbvf_irq_enable - Enable default interrupt generation settings
1167 * @adapter: board private structure
1168 **/
1169static void igbvf_irq_enable(struct igbvf_adapter *adapter)
1170{
1171	struct e1000_hw *hw = &adapter->hw;
1172
1173	ew32(EIAC, adapter->eims_enable_mask);
1174	ew32(EIAM, adapter->eims_enable_mask);
1175	ew32(EIMS, adapter->eims_enable_mask);
1176}
1177
1178/**
1179 * igbvf_poll - NAPI Rx polling callback
1180 * @napi: struct associated with this polling callback
1181 * @budget: amount of packets driver is allowed to process this poll
1182 **/
1183static int igbvf_poll(struct napi_struct *napi, int budget)
1184{
1185	struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);
1186	struct igbvf_adapter *adapter = rx_ring->adapter;
1187	struct e1000_hw *hw = &adapter->hw;
1188	int work_done = 0;
1189
1190	igbvf_clean_rx_irq(adapter, &work_done, budget);
1191
1192	if (work_done == budget)
1193		return budget;
1194
1195	/* Exit the polling mode, but don't re-enable interrupts if stack might
1196	 * poll us due to busy-polling
1197	 */
1198	if (likely(napi_complete_done(napi, work_done))) {
1199		if (adapter->requested_itr & 3)
1200			igbvf_set_itr(adapter);
1201
1202		if (!test_bit(__IGBVF_DOWN, &adapter->state))
1203			ew32(EIMS, adapter->rx_ring->eims_value);
1204	}
1205
1206	return work_done;
1207}
1208
1209/**
1210 * igbvf_set_rlpml - set receive large packet maximum length
1211 * @adapter: board private structure
1212 *
1213 * Configure the maximum size of packets that will be received
1214 */
1215static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
1216{
1217	int max_frame_size;
1218	struct e1000_hw *hw = &adapter->hw;
1219
1220	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
1221
1222	spin_lock_bh(&hw->mbx_lock);
1223
1224	e1000_rlpml_set_vf(hw, max_frame_size);
1225
1226	spin_unlock_bh(&hw->mbx_lock);
1227}
1228
1229static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
1230				 __be16 proto, u16 vid)
1231{
1232	struct igbvf_adapter *adapter = netdev_priv(netdev);
1233	struct e1000_hw *hw = &adapter->hw;
1234
1235	spin_lock_bh(&hw->mbx_lock);
1236
1237	if (hw->mac.ops.set_vfta(hw, vid, true)) {
1238		dev_warn(&adapter->pdev->dev, "Vlan id %d is not added\n", vid);
1239		spin_unlock_bh(&hw->mbx_lock);
1240		return -EINVAL;
1241	}
1242
1243	spin_unlock_bh(&hw->mbx_lock);
1244
1245	set_bit(vid, adapter->active_vlans);
1246	return 0;
1247}
1248
1249static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
1250				  __be16 proto, u16 vid)
1251{
1252	struct igbvf_adapter *adapter = netdev_priv(netdev);
1253	struct e1000_hw *hw = &adapter->hw;
1254
1255	spin_lock_bh(&hw->mbx_lock);
1256
1257	if (hw->mac.ops.set_vfta(hw, vid, false)) {
1258		dev_err(&adapter->pdev->dev,
1259			"Failed to remove vlan id %d\n", vid);
1260		spin_unlock_bh(&hw->mbx_lock);
1261		return -EINVAL;
1262	}
1263
1264	spin_unlock_bh(&hw->mbx_lock);
1265
1266	clear_bit(vid, adapter->active_vlans);
1267	return 0;
1268}
1269
1270static void igbvf_restore_vlan(struct igbvf_adapter *adapter)
1271{
1272	u16 vid;
1273
1274	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1275		igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
1276}
1277
1278/**
1279 * igbvf_configure_tx - Configure Transmit Unit after Reset
1280 * @adapter: board private structure
1281 *
1282 * Configure the Tx unit of the MAC after a reset.
1283 **/
1284static void igbvf_configure_tx(struct igbvf_adapter *adapter)
1285{
1286	struct e1000_hw *hw = &adapter->hw;
1287	struct igbvf_ring *tx_ring = adapter->tx_ring;
1288	u64 tdba;
1289	u32 txdctl, dca_txctrl;
1290
1291	/* disable transmits */
1292	txdctl = er32(TXDCTL(0));
1293	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1294	e1e_flush();
1295	msleep(10);
1296
1297	/* Setup the HW Tx Head and Tail descriptor pointers */
1298	ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));
1299	tdba = tx_ring->dma;
1300	ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
1301	ew32(TDBAH(0), (tdba >> 32));
1302	ew32(TDH(0), 0);
1303	ew32(TDT(0), 0);
1304	tx_ring->head = E1000_TDH(0);
1305	tx_ring->tail = E1000_TDT(0);
1306
1307	/* Turn off Relaxed Ordering on head write-backs.  The writebacks
1308	 * MUST be delivered in order or it will completely screw up
1309	 * our bookkeeping.
1310	 */
1311	dca_txctrl = er32(DCA_TXCTRL(0));
1312	dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1313	ew32(DCA_TXCTRL(0), dca_txctrl);
1314
1315	/* enable transmits */
1316	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1317	ew32(TXDCTL(0), txdctl);
1318
1319	/* Setup Transmit Descriptor Settings for eop descriptor */
1320	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;
1321
1322	/* enable Report Status bit */
1323	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;
1324}
1325
1326/**
1327 * igbvf_setup_srrctl - configure the receive control registers
1328 * @adapter: Board private structure
1329 **/
1330static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)
1331{
1332	struct e1000_hw *hw = &adapter->hw;
1333	u32 srrctl = 0;
1334
1335	srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |
1336		    E1000_SRRCTL_BSIZEHDR_MASK |
1337		    E1000_SRRCTL_BSIZEPKT_MASK);
1338
1339	/* Enable queue drop to avoid head of line blocking */
1340	srrctl |= E1000_SRRCTL_DROP_EN;
1341
1342	/* Setup buffer sizes */
1343	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>
1344		  E1000_SRRCTL_BSIZEPKT_SHIFT;
1345
1346	if (adapter->rx_buffer_len < 2048) {
1347		adapter->rx_ps_hdr_size = 0;
1348		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1349	} else {
1350		adapter->rx_ps_hdr_size = 128;
1351		srrctl |= adapter->rx_ps_hdr_size <<
1352			  E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1353		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1354	}
1355
1356	ew32(SRRCTL(0), srrctl);
1357}
1358
1359/**
1360 * igbvf_configure_rx - Configure Receive Unit after Reset
1361 * @adapter: board private structure
1362 *
1363 * Configure the Rx unit of the MAC after a reset.
1364 **/
1365static void igbvf_configure_rx(struct igbvf_adapter *adapter)
1366{
1367	struct e1000_hw *hw = &adapter->hw;
1368	struct igbvf_ring *rx_ring = adapter->rx_ring;
1369	u64 rdba;
1370	u32 rxdctl;
1371
1372	/* disable receives */
1373	rxdctl = er32(RXDCTL(0));
1374	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1375	e1e_flush();
1376	msleep(10);
1377
1378	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1379	 * the Base and Length of the Rx Descriptor Ring
1380	 */
1381	rdba = rx_ring->dma;
1382	ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
1383	ew32(RDBAH(0), (rdba >> 32));
1384	ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));
1385	rx_ring->head = E1000_RDH(0);
1386	rx_ring->tail = E1000_RDT(0);
1387	ew32(RDH(0), 0);
1388	ew32(RDT(0), 0);
1389
1390	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1391	rxdctl &= 0xFFF00000;
1392	rxdctl |= IGBVF_RX_PTHRESH;
1393	rxdctl |= IGBVF_RX_HTHRESH << 8;
1394	rxdctl |= IGBVF_RX_WTHRESH << 16;
1395
1396	igbvf_set_rlpml(adapter);
1397
1398	/* enable receives */
1399	ew32(RXDCTL(0), rxdctl);
1400}
1401
1402/**
1403 * igbvf_set_multi - Multicast and Promiscuous mode set
1404 * @netdev: network interface device structure
1405 *
1406 * The set_multi entry point is called whenever the multicast address
1407 * list or the network interface flags are updated.  This routine is
1408 * responsible for configuring the hardware for proper multicast,
1409 * promiscuous mode, and all-multi behavior.
1410 **/
1411static void igbvf_set_multi(struct net_device *netdev)
1412{
1413	struct igbvf_adapter *adapter = netdev_priv(netdev);
1414	struct e1000_hw *hw = &adapter->hw;
1415	struct netdev_hw_addr *ha;
1416	u8  *mta_list = NULL;
1417	int i;
1418
1419	if (!netdev_mc_empty(netdev)) {
1420		mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,
1421					 GFP_ATOMIC);
1422		if (!mta_list)
1423			return;
1424	}
1425
1426	/* prepare a packed array of only addresses. */
1427	i = 0;
1428	netdev_for_each_mc_addr(ha, netdev)
1429		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1430
1431	spin_lock_bh(&hw->mbx_lock);
1432
1433	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
1434
1435	spin_unlock_bh(&hw->mbx_lock);
1436	kfree(mta_list);
1437}
1438
1439/**
1440 * igbvf_set_uni - Configure unicast MAC filters
1441 * @netdev: network interface device structure
1442 *
1443 * This routine is responsible for configuring the hardware for proper
1444 * unicast filters.
1445 **/
1446static int igbvf_set_uni(struct net_device *netdev)
1447{
1448	struct igbvf_adapter *adapter = netdev_priv(netdev);
1449	struct e1000_hw *hw = &adapter->hw;
1450
1451	if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {
1452		pr_err("Too many unicast filters - No Space\n");
1453		return -ENOSPC;
1454	}
1455
1456	spin_lock_bh(&hw->mbx_lock);
1457
1458	/* Clear all unicast MAC filters */
1459	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
1460
1461	spin_unlock_bh(&hw->mbx_lock);
1462
1463	if (!netdev_uc_empty(netdev)) {
1464		struct netdev_hw_addr *ha;
1465
1466		/* Add MAC filters one by one */
1467		netdev_for_each_uc_addr(ha, netdev) {
1468			spin_lock_bh(&hw->mbx_lock);
1469
1470			hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
1471						ha->addr);
1472
1473			spin_unlock_bh(&hw->mbx_lock);
1474			udelay(200);
1475		}
1476	}
1477
1478	return 0;
1479}
1480
1481static void igbvf_set_rx_mode(struct net_device *netdev)
1482{
1483	igbvf_set_multi(netdev);
1484	igbvf_set_uni(netdev);
1485}
1486
1487/**
1488 * igbvf_configure - configure the hardware for Rx and Tx
1489 * @adapter: private board structure
1490 **/
1491static void igbvf_configure(struct igbvf_adapter *adapter)
1492{
1493	igbvf_set_rx_mode(adapter->netdev);
1494
1495	igbvf_restore_vlan(adapter);
1496
1497	igbvf_configure_tx(adapter);
1498	igbvf_setup_srrctl(adapter);
1499	igbvf_configure_rx(adapter);
1500	igbvf_alloc_rx_buffers(adapter->rx_ring,
1501			       igbvf_desc_unused(adapter->rx_ring));
1502}
1503
1504/* igbvf_reset - bring the hardware into a known good state
1505 * @adapter: private board structure
1506 *
1507 * This function boots the hardware and enables some settings that
1508 * require a configuration cycle of the hardware - those cannot be
1509 * set/changed during runtime. After reset the device needs to be
1510 * properly configured for Rx, Tx etc.
1511 */
1512static void igbvf_reset(struct igbvf_adapter *adapter)
1513{
1514	struct e1000_mac_info *mac = &adapter->hw.mac;
1515	struct net_device *netdev = adapter->netdev;
1516	struct e1000_hw *hw = &adapter->hw;
1517
1518	spin_lock_bh(&hw->mbx_lock);
1519
1520	/* Allow time for pending master requests to run */
1521	if (mac->ops.reset_hw(hw))
1522		dev_info(&adapter->pdev->dev, "PF still resetting\n");
1523
1524	mac->ops.init_hw(hw);
1525
1526	spin_unlock_bh(&hw->mbx_lock);
1527
1528	if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1529		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
1530		memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1531		       netdev->addr_len);
1532	}
1533
1534	adapter->last_reset = jiffies;
1535}
1536
1537int igbvf_up(struct igbvf_adapter *adapter)
1538{
1539	struct e1000_hw *hw = &adapter->hw;
1540
1541	/* hardware has been reset, we need to reload some things */
1542	igbvf_configure(adapter);
1543
1544	clear_bit(__IGBVF_DOWN, &adapter->state);
1545
1546	napi_enable(&adapter->rx_ring->napi);
1547	if (adapter->msix_entries)
1548		igbvf_configure_msix(adapter);
1549
1550	/* Clear any pending interrupts. */
1551	er32(EICR);
1552	igbvf_irq_enable(adapter);
1553
1554	/* start the watchdog */
1555	hw->mac.get_link_status = 1;
1556	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1557
1558	return 0;
1559}
1560
1561void igbvf_down(struct igbvf_adapter *adapter)
1562{
1563	struct net_device *netdev = adapter->netdev;
1564	struct e1000_hw *hw = &adapter->hw;
1565	u32 rxdctl, txdctl;
1566
1567	/* signal that we're down so the interrupt handler does not
1568	 * reschedule our watchdog timer
1569	 */
1570	set_bit(__IGBVF_DOWN, &adapter->state);
1571
1572	/* disable receives in the hardware */
1573	rxdctl = er32(RXDCTL(0));
1574	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1575
1576	netif_carrier_off(netdev);
1577	netif_stop_queue(netdev);
1578
1579	/* disable transmits in the hardware */
1580	txdctl = er32(TXDCTL(0));
1581	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1582
1583	/* flush both disables and wait for them to finish */
1584	e1e_flush();
1585	msleep(10);
1586
1587	napi_disable(&adapter->rx_ring->napi);
1588
1589	igbvf_irq_disable(adapter);
1590
1591	timer_delete_sync(&adapter->watchdog_timer);
1592
1593	/* record the stats before reset*/
1594	igbvf_update_stats(adapter);
1595
1596	adapter->link_speed = 0;
1597	adapter->link_duplex = 0;
1598
1599	igbvf_reset(adapter);
1600	igbvf_clean_tx_ring(adapter->tx_ring);
1601	igbvf_clean_rx_ring(adapter->rx_ring);
1602}
1603
1604void igbvf_reinit_locked(struct igbvf_adapter *adapter)
1605{
1606	might_sleep();
1607	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
1608		usleep_range(1000, 2000);
1609	igbvf_down(adapter);
1610	igbvf_up(adapter);
1611	clear_bit(__IGBVF_RESETTING, &adapter->state);
1612}
1613
1614/**
1615 * igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)
1616 * @adapter: board private structure to initialize
1617 *
1618 * igbvf_sw_init initializes the Adapter private data structure.
1619 * Fields are initialized based on PCI device information and
1620 * OS network device settings (MTU size).
1621 **/
1622static int igbvf_sw_init(struct igbvf_adapter *adapter)
1623{
1624	struct net_device *netdev = adapter->netdev;
1625	s32 rc;
1626
1627	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
1628	adapter->rx_ps_hdr_size = 0;
1629	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1630	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1631
1632	adapter->requested_itr = 3;
1633	adapter->current_itr = IGBVF_START_ITR;
1634
1635	/* Set various function pointers */
1636	adapter->ei->init_ops(&adapter->hw);
1637
1638	rc = adapter->hw.mac.ops.init_params(&adapter->hw);
1639	if (rc)
1640		return rc;
1641
1642	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);
1643	if (rc)
1644		return rc;
1645
1646	igbvf_set_interrupt_capability(adapter);
1647
1648	if (igbvf_alloc_queues(adapter))
1649		return -ENOMEM;
1650
1651	/* Explicitly disable IRQ since the NIC can be in any state. */
1652	igbvf_irq_disable(adapter);
1653
1654	spin_lock_init(&adapter->hw.mbx_lock);
1655
1656	set_bit(__IGBVF_DOWN, &adapter->state);
1657	return 0;
1658}
1659
1660static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)
1661{
1662	struct e1000_hw *hw = &adapter->hw;
1663
1664	adapter->stats.last_gprc = er32(VFGPRC);
1665	adapter->stats.last_gorc = er32(VFGORC);
1666	adapter->stats.last_gptc = er32(VFGPTC);
1667	adapter->stats.last_gotc = er32(VFGOTC);
1668	adapter->stats.last_mprc = er32(VFMPRC);
1669	adapter->stats.last_gotlbc = er32(VFGOTLBC);
1670	adapter->stats.last_gptlbc = er32(VFGPTLBC);
1671	adapter->stats.last_gorlbc = er32(VFGORLBC);
1672	adapter->stats.last_gprlbc = er32(VFGPRLBC);
1673
1674	adapter->stats.base_gprc = er32(VFGPRC);
1675	adapter->stats.base_gorc = er32(VFGORC);
1676	adapter->stats.base_gptc = er32(VFGPTC);
1677	adapter->stats.base_gotc = er32(VFGOTC);
1678	adapter->stats.base_mprc = er32(VFMPRC);
1679	adapter->stats.base_gotlbc = er32(VFGOTLBC);
1680	adapter->stats.base_gptlbc = er32(VFGPTLBC);
1681	adapter->stats.base_gorlbc = er32(VFGORLBC);
1682	adapter->stats.base_gprlbc = er32(VFGPRLBC);
1683}
1684
1685/**
1686 * igbvf_open - Called when a network interface is made active
1687 * @netdev: network interface device structure
1688 *
1689 * Returns 0 on success, negative value on failure
1690 *
1691 * The open entry point is called when a network interface is made
1692 * active by the system (IFF_UP).  At this point all resources needed
1693 * for transmit and receive operations are allocated, the interrupt
1694 * handler is registered with the OS, the watchdog timer is started,
1695 * and the stack is notified that the interface is ready.
1696 **/
1697static int igbvf_open(struct net_device *netdev)
1698{
1699	struct igbvf_adapter *adapter = netdev_priv(netdev);
1700	struct e1000_hw *hw = &adapter->hw;
1701	int err;
1702
1703	/* disallow open during test */
1704	if (test_bit(__IGBVF_TESTING, &adapter->state))
1705		return -EBUSY;
1706
1707	/* allocate transmit descriptors */
1708	err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);
1709	if (err)
1710		goto err_setup_tx;
1711
1712	/* allocate receive descriptors */
1713	err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);
1714	if (err)
1715		goto err_setup_rx;
1716
1717	/* before we allocate an interrupt, we must be ready to handle it.
1718	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1719	 * as soon as we call pci_request_irq, so we have to setup our
1720	 * clean_rx handler before we do so.
1721	 */
1722	igbvf_configure(adapter);
1723
1724	err = igbvf_request_irq(adapter);
1725	if (err)
1726		goto err_req_irq;
1727
1728	/* From here on the code is the same as igbvf_up() */
1729	clear_bit(__IGBVF_DOWN, &adapter->state);
1730
1731	napi_enable(&adapter->rx_ring->napi);
1732
1733	/* clear any pending interrupts */
1734	er32(EICR);
1735
1736	igbvf_irq_enable(adapter);
1737
1738	/* start the watchdog */
1739	hw->mac.get_link_status = 1;
1740	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1741
1742	return 0;
1743
1744err_req_irq:
1745	igbvf_free_rx_resources(adapter->rx_ring);
1746err_setup_rx:
1747	igbvf_free_tx_resources(adapter->tx_ring);
1748err_setup_tx:
1749	igbvf_reset(adapter);
1750
1751	return err;
1752}
1753
1754/**
1755 * igbvf_close - Disables a network interface
1756 * @netdev: network interface device structure
1757 *
1758 * Returns 0, this is not allowed to fail
1759 *
1760 * The close entry point is called when an interface is de-activated
1761 * by the OS.  The hardware is still under the drivers control, but
1762 * needs to be disabled.  A global MAC reset is issued to stop the
1763 * hardware, and all transmit and receive resources are freed.
1764 **/
1765static int igbvf_close(struct net_device *netdev)
1766{
1767	struct igbvf_adapter *adapter = netdev_priv(netdev);
1768
1769	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
1770	igbvf_down(adapter);
1771
1772	igbvf_free_irq(adapter);
1773
1774	igbvf_free_tx_resources(adapter->tx_ring);
1775	igbvf_free_rx_resources(adapter->rx_ring);
1776
1777	return 0;
1778}
1779
1780/**
1781 * igbvf_set_mac - Change the Ethernet Address of the NIC
1782 * @netdev: network interface device structure
1783 * @p: pointer to an address structure
1784 *
1785 * Returns 0 on success, negative on failure
1786 **/
1787static int igbvf_set_mac(struct net_device *netdev, void *p)
1788{
1789	struct igbvf_adapter *adapter = netdev_priv(netdev);
1790	struct e1000_hw *hw = &adapter->hw;
1791	struct sockaddr *addr = p;
1792
1793	if (!is_valid_ether_addr(addr->sa_data))
1794		return -EADDRNOTAVAIL;
1795
1796	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
1797
1798	spin_lock_bh(&hw->mbx_lock);
1799
1800	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
1801
1802	spin_unlock_bh(&hw->mbx_lock);
1803
1804	if (!ether_addr_equal(addr->sa_data, hw->mac.addr))
1805		return -EADDRNOTAVAIL;
1806
1807	eth_hw_addr_set(netdev, addr->sa_data);
1808
1809	return 0;
1810}
1811
1812#define UPDATE_VF_COUNTER(reg, name) \
1813{ \
1814	u32 current_counter = er32(reg); \
1815	if (current_counter < adapter->stats.last_##name) \
1816		adapter->stats.name += 0x100000000LL; \
1817	adapter->stats.last_##name = current_counter; \
1818	adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1819	adapter->stats.name |= current_counter; \
1820}
1821
1822/**
1823 * igbvf_update_stats - Update the board statistics counters
1824 * @adapter: board private structure
1825**/
1826void igbvf_update_stats(struct igbvf_adapter *adapter)
1827{
1828	struct e1000_hw *hw = &adapter->hw;
1829	struct pci_dev *pdev = adapter->pdev;
1830
1831	/* Prevent stats update while adapter is being reset, link is down
1832	 * or if the pci connection is down.
1833	 */
1834	if (adapter->link_speed == 0)
1835		return;
1836
1837	if (test_bit(__IGBVF_RESETTING, &adapter->state))
1838		return;
1839
1840	if (pci_channel_offline(pdev))
1841		return;
1842
1843	UPDATE_VF_COUNTER(VFGPRC, gprc);
1844	UPDATE_VF_COUNTER(VFGORC, gorc);
1845	UPDATE_VF_COUNTER(VFGPTC, gptc);
1846	UPDATE_VF_COUNTER(VFGOTC, gotc);
1847	UPDATE_VF_COUNTER(VFMPRC, mprc);
1848	UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);
1849	UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);
1850	UPDATE_VF_COUNTER(VFGORLBC, gorlbc);
1851	UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);
1852
1853	/* Fill out the OS statistics structure */
1854	adapter->netdev->stats.multicast = adapter->stats.mprc;
1855}
1856
1857static void igbvf_print_link_info(struct igbvf_adapter *adapter)
1858{
1859	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",
1860		 adapter->link_speed,
1861		 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
1862}
1863
1864static bool igbvf_has_link(struct igbvf_adapter *adapter)
1865{
1866	struct e1000_hw *hw = &adapter->hw;
1867	s32 ret_val = E1000_SUCCESS;
1868	bool link_active;
1869
1870	/* If interface is down, stay link down */
1871	if (test_bit(__IGBVF_DOWN, &adapter->state))
1872		return false;
1873
1874	spin_lock_bh(&hw->mbx_lock);
1875
1876	ret_val = hw->mac.ops.check_for_link(hw);
1877
1878	spin_unlock_bh(&hw->mbx_lock);
1879
1880	link_active = !hw->mac.get_link_status;
1881
1882	/* if check for link returns error we will need to reset */
1883	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))
1884		schedule_work(&adapter->reset_task);
1885
1886	return link_active;
1887}
1888
1889/**
1890 * igbvf_watchdog - Timer Call-back
1891 * @t: timer list pointer containing private struct
1892 **/
1893static void igbvf_watchdog(struct timer_list *t)
1894{
1895	struct igbvf_adapter *adapter = timer_container_of(adapter, t,
1896							   watchdog_timer);
1897
1898	/* Do the rest outside of interrupt context */
1899	schedule_work(&adapter->watchdog_task);
1900}
1901
1902static void igbvf_watchdog_task(struct work_struct *work)
1903{
1904	struct igbvf_adapter *adapter = container_of(work,
1905						     struct igbvf_adapter,
1906						     watchdog_task);
1907	struct net_device *netdev = adapter->netdev;
1908	struct e1000_mac_info *mac = &adapter->hw.mac;
1909	struct igbvf_ring *tx_ring = adapter->tx_ring;
1910	struct e1000_hw *hw = &adapter->hw;
1911	u32 link;
1912	int tx_pending = 0;
1913
1914	link = igbvf_has_link(adapter);
1915
1916	if (link) {
1917		if (!netif_carrier_ok(netdev)) {
1918			mac->ops.get_link_up_info(&adapter->hw,
1919						  &adapter->link_speed,
1920						  &adapter->link_duplex);
1921			igbvf_print_link_info(adapter);
1922
1923			netif_carrier_on(netdev);
1924			netif_wake_queue(netdev);
1925		}
1926	} else {
1927		if (netif_carrier_ok(netdev)) {
1928			adapter->link_speed = 0;
1929			adapter->link_duplex = 0;
1930			dev_info(&adapter->pdev->dev, "Link is Down\n");
1931			netif_carrier_off(netdev);
1932			netif_stop_queue(netdev);
1933		}
1934	}
1935
1936	if (netif_carrier_ok(netdev)) {
1937		igbvf_update_stats(adapter);
1938	} else {
1939		tx_pending = (igbvf_desc_unused(tx_ring) + 1 <
1940			      tx_ring->count);
1941		if (tx_pending) {
1942			/* We've lost link, so the controller stops DMA,
1943			 * but we've got queued Tx work that's never going
1944			 * to get done, so reset controller to flush Tx.
1945			 * (Do the reset outside of interrupt context).
1946			 */
1947			adapter->tx_timeout_count++;
1948			schedule_work(&adapter->reset_task);
1949		}
1950	}
1951
1952	/* Cause software interrupt to ensure Rx ring is cleaned */
1953	ew32(EICS, adapter->rx_ring->eims_value);
1954
1955	/* Reset the timer */
1956	if (!test_bit(__IGBVF_DOWN, &adapter->state))
1957		mod_timer(&adapter->watchdog_timer,
1958			  round_jiffies(jiffies + (2 * HZ)));
1959}
1960
1961#define IGBVF_TX_FLAGS_CSUM		0x00000001
1962#define IGBVF_TX_FLAGS_VLAN		0x00000002
1963#define IGBVF_TX_FLAGS_TSO		0x00000004
1964#define IGBVF_TX_FLAGS_IPV4		0x00000008
1965#define IGBVF_TX_FLAGS_VLAN_MASK	0xffff0000
1966#define IGBVF_TX_FLAGS_VLAN_SHIFT	16
1967
1968static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,
1969			      u32 type_tucmd, u32 mss_l4len_idx)
1970{
1971	struct e1000_adv_tx_context_desc *context_desc;
1972	struct igbvf_buffer *buffer_info;
1973	u16 i = tx_ring->next_to_use;
1974
1975	context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);
1976	buffer_info = &tx_ring->buffer_info[i];
1977
1978	i++;
1979	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1980
1981	/* set bits to identify this as an advanced context descriptor */
1982	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
1983
1984	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1985	context_desc->seqnum_seed	= 0;
1986	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1987	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1988
1989	buffer_info->time_stamp = jiffies;
1990	buffer_info->dma = 0;
1991}
1992
1993static int igbvf_tso(struct igbvf_ring *tx_ring,
1994		     struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
1995{
1996	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1997	union {
1998		struct iphdr *v4;
1999		struct ipv6hdr *v6;
2000		unsigned char *hdr;
2001	} ip;
2002	union {
2003		struct tcphdr *tcp;
2004		unsigned char *hdr;
2005	} l4;
2006	u32 paylen, l4_offset;
2007	int err;
2008
2009	if (skb->ip_summed != CHECKSUM_PARTIAL)
2010		return 0;
2011
2012	if (!skb_is_gso(skb))
2013		return 0;
2014
2015	err = skb_cow_head(skb, 0);
2016	if (err < 0)
2017		return err;
2018
2019	ip.hdr = skb_network_header(skb);
2020	l4.hdr = skb_checksum_start(skb);
2021
2022	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2023	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2024
2025	/* initialize outer IP header fields */
2026	if (ip.v4->version == 4) {
2027		unsigned char *csum_start = skb_checksum_start(skb);
2028		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
2029
2030		/* IP header will have to cancel out any data that
2031		 * is not a part of the outer IP header
2032		 */
2033		ip.v4->check = csum_fold(csum_partial(trans_start,
2034						      csum_start - trans_start,
2035						      0));
2036		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
2037
2038		ip.v4->tot_len = 0;
2039	} else {
2040		ip.v6->payload_len = 0;
2041	}
2042
2043	/* determine offset of inner transport header */
2044	l4_offset = l4.hdr - skb->data;
2045
2046	/* compute length of segmentation header */
2047	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
2048
2049	/* remove payload length from inner checksum */
2050	paylen = skb->len - l4_offset;
2051	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
2052
2053	/* MSS L4LEN IDX */
2054	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
2055	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
2056
2057	/* VLAN MACLEN IPLEN */
2058	vlan_macip_lens = l4.hdr - ip.hdr;
2059	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
2060	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2061
2062	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
2063
2064	return 1;
2065}
2066
2067static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,
2068			  u32 tx_flags, __be16 protocol)
2069{
2070	u32 vlan_macip_lens = 0;
2071	u32 type_tucmd = 0;
2072
2073	if (skb->ip_summed != CHECKSUM_PARTIAL) {
2074csum_failed:
2075		if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))
2076			return false;
2077		goto no_csum;
2078	}
2079
2080	switch (skb->csum_offset) {
2081	case offsetof(struct tcphdr, check):
2082		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2083		fallthrough;
2084	case offsetof(struct udphdr, check):
2085		break;
2086	case offsetof(struct sctphdr, checksum):
2087		/* validate that this is actually an SCTP request */
2088		if (skb_csum_is_sctp(skb)) {
2089			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
2090			break;
2091		}
2092		fallthrough;
2093	default:
2094		skb_checksum_help(skb);
2095		goto csum_failed;
2096	}
2097
2098	vlan_macip_lens = skb_checksum_start_offset(skb) -
2099			  skb_network_offset(skb);
2100no_csum:
2101	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
2102	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2103
2104	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
2105	return true;
2106}
2107
2108static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)
2109{
2110	struct igbvf_adapter *adapter = netdev_priv(netdev);
2111
2112	/* there is enough descriptors then we don't need to worry  */
2113	if (igbvf_desc_unused(adapter->tx_ring) >= size)
2114		return 0;
2115
2116	netif_stop_queue(netdev);
2117
2118	/* Herbert's original patch had:
2119	 *  smp_mb__after_netif_stop_queue();
2120	 * but since that doesn't exist yet, just open code it.
2121	 */
2122	smp_mb();
2123
2124	/* We need to check again just in case room has been made available */
2125	if (igbvf_desc_unused(adapter->tx_ring) < size)
2126		return -EBUSY;
2127
2128	netif_wake_queue(netdev);
2129
2130	++adapter->restart_queue;
2131	return 0;
2132}
2133
2134#define IGBVF_MAX_TXD_PWR	16
2135#define IGBVF_MAX_DATA_PER_TXD	(1u << IGBVF_MAX_TXD_PWR)
2136
2137static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,
2138				   struct igbvf_ring *tx_ring,
2139				   struct sk_buff *skb)
2140{
2141	struct igbvf_buffer *buffer_info;
2142	struct pci_dev *pdev = adapter->pdev;
2143	unsigned int len = skb_headlen(skb);
2144	unsigned int count = 0, i;
2145	unsigned int f;
2146
2147	i = tx_ring->next_to_use;
2148
2149	buffer_info = &tx_ring->buffer_info[i];
2150	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2151	buffer_info->length = len;
2152	/* set time_stamp *before* dma to help avoid a possible race */
2153	buffer_info->time_stamp = jiffies;
2154	buffer_info->mapped_as_page = false;
2155	buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,
2156					  DMA_TO_DEVICE);
2157	if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2158		goto dma_error;
2159
2160	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2161		const skb_frag_t *frag;
2162
2163		count++;
2164		i++;
2165		if (i == tx_ring->count)
2166			i = 0;
2167
2168		frag = &skb_shinfo(skb)->frags[f];
2169		len = skb_frag_size(frag);
2170
2171		buffer_info = &tx_ring->buffer_info[i];
2172		BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2173		buffer_info->length = len;
2174		buffer_info->time_stamp = jiffies;
2175		buffer_info->mapped_as_page = true;
2176		buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 0, len,
2177						    DMA_TO_DEVICE);
2178		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2179			goto dma_error;
2180	}
2181
2182	tx_ring->buffer_info[i].skb = skb;
2183
2184	return ++count;
2185
2186dma_error:
2187	dev_err(&pdev->dev, "TX DMA map failed\n");
2188
2189	/* clear timestamp and dma mappings for failed buffer_info mapping */
2190	buffer_info->dma = 0;
2191	buffer_info->time_stamp = 0;
2192	buffer_info->length = 0;
2193	buffer_info->mapped_as_page = false;
2194	if (count)
2195		count--;
2196
2197	/* clear timestamp and dma mappings for remaining portion of packet */
2198	while (count--) {
2199		if (i == 0)
2200			i += tx_ring->count;
2201		i--;
2202		buffer_info = &tx_ring->buffer_info[i];
2203		igbvf_put_txbuf(adapter, buffer_info);
2204	}
2205
2206	return 0;
2207}
2208
2209static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
2210				      struct igbvf_ring *tx_ring,
2211				      int tx_flags, int count,
2212				      unsigned int first, u32 paylen,
2213				      u8 hdr_len)
2214{
2215	union e1000_adv_tx_desc *tx_desc = NULL;
2216	struct igbvf_buffer *buffer_info;
2217	u32 olinfo_status = 0, cmd_type_len;
2218	unsigned int i;
2219
2220	cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2221			E1000_ADVTXD_DCMD_DEXT);
2222
2223	if (tx_flags & IGBVF_TX_FLAGS_VLAN)
2224		cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2225
2226	if (tx_flags & IGBVF_TX_FLAGS_TSO) {
2227		cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2228
2229		/* insert tcp checksum */
2230		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2231
2232		/* insert ip checksum */
2233		if (tx_flags & IGBVF_TX_FLAGS_IPV4)
2234			olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2235
2236	} else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {
2237		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2238	}
2239
2240	olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2241
2242	i = tx_ring->next_to_use;
2243	while (count--) {
2244		buffer_info = &tx_ring->buffer_info[i];
2245		tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
2246		tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2247		tx_desc->read.cmd_type_len =
2248			 cpu_to_le32(cmd_type_len | buffer_info->length);
2249		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2250		i++;
2251		if (i == tx_ring->count)
2252			i = 0;
2253	}
2254
2255	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2256	/* Force memory writes to complete before letting h/w
2257	 * know there are new descriptors to fetch.  (Only
2258	 * applicable for weak-ordered memory model archs,
2259	 * such as IA-64).
2260	 */
2261	wmb();
2262
2263	tx_ring->buffer_info[first].next_to_watch = tx_desc;
2264	tx_ring->next_to_use = i;
2265	writel(i, adapter->hw.hw_addr + tx_ring->tail);
2266}
2267
2268static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,
2269					     struct net_device *netdev,
2270					     struct igbvf_ring *tx_ring)
2271{
2272	struct igbvf_adapter *adapter = netdev_priv(netdev);
2273	unsigned int first, tx_flags = 0;
2274	u8 hdr_len = 0;
2275	int count = 0;
2276	int tso = 0;
2277	__be16 protocol = vlan_get_protocol(skb);
2278
2279	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2280		dev_kfree_skb_any(skb);
2281		return NETDEV_TX_OK;
2282	}
2283
2284	if (skb->len <= 0) {
2285		dev_kfree_skb_any(skb);
2286		return NETDEV_TX_OK;
2287	}
2288
2289	/* need: count + 4 desc gap to keep tail from touching
2290	 *       + 2 desc gap to keep tail from touching head,
2291	 *       + 1 desc for skb->data,
2292	 *       + 1 desc for context descriptor,
2293	 * head, otherwise try next time
2294	 */
2295	if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {
2296		/* this is a hard error */
2297		return NETDEV_TX_BUSY;
2298	}
2299
2300	if (skb_vlan_tag_present(skb)) {
2301		tx_flags |= IGBVF_TX_FLAGS_VLAN;
2302		tx_flags |= (skb_vlan_tag_get(skb) <<
2303			     IGBVF_TX_FLAGS_VLAN_SHIFT);
2304	}
2305
2306	if (protocol == htons(ETH_P_IP))
2307		tx_flags |= IGBVF_TX_FLAGS_IPV4;
2308
2309	first = tx_ring->next_to_use;
2310
2311	tso = igbvf_tso(tx_ring, skb, tx_flags, &hdr_len);
2312	if (unlikely(tso < 0)) {
2313		dev_kfree_skb_any(skb);
2314		return NETDEV_TX_OK;
2315	}
2316
2317	if (tso)
2318		tx_flags |= IGBVF_TX_FLAGS_TSO;
2319	else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&
2320		 (skb->ip_summed == CHECKSUM_PARTIAL))
2321		tx_flags |= IGBVF_TX_FLAGS_CSUM;
2322
2323	/* count reflects descriptors mapped, if 0 then mapping error
2324	 * has occurred and we need to rewind the descriptor queue
2325	 */
2326	count = igbvf_tx_map_adv(adapter, tx_ring, skb);
2327
2328	if (count) {
2329		igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,
2330				   first, skb->len, hdr_len);
2331		/* Make sure there is space in the ring for the next send. */
2332		igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);
2333	} else {
2334		dev_kfree_skb_any(skb);
2335		tx_ring->buffer_info[first].time_stamp = 0;
2336		tx_ring->next_to_use = first;
2337	}
2338
2339	return NETDEV_TX_OK;
2340}
2341
2342static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,
2343				    struct net_device *netdev)
2344{
2345	struct igbvf_adapter *adapter = netdev_priv(netdev);
2346	struct igbvf_ring *tx_ring;
2347
2348	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2349		dev_kfree_skb_any(skb);
2350		return NETDEV_TX_OK;
2351	}
2352
2353	tx_ring = &adapter->tx_ring[0];
2354
2355	return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);
2356}
2357
2358/**
2359 * igbvf_tx_timeout - Respond to a Tx Hang
2360 * @netdev: network interface device structure
2361 * @txqueue: queue timing out (unused)
2362 **/
2363static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
2364{
2365	struct igbvf_adapter *adapter = netdev_priv(netdev);
2366
2367	/* Do the reset outside of interrupt context */
2368	adapter->tx_timeout_count++;
2369	schedule_work(&adapter->reset_task);
2370}
2371
2372static void igbvf_reset_task(struct work_struct *work)
2373{
2374	struct igbvf_adapter *adapter;
2375
2376	adapter = container_of(work, struct igbvf_adapter, reset_task);
2377
2378	igbvf_reinit_locked(adapter);
2379}
2380
2381/**
2382 * igbvf_change_mtu - Change the Maximum Transfer Unit
2383 * @netdev: network interface device structure
2384 * @new_mtu: new value for maximum frame size
2385 *
2386 * Returns 0 on success, negative on failure
2387 **/
2388static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)
2389{
2390	struct igbvf_adapter *adapter = netdev_priv(netdev);
2391	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2392
2393	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2394		usleep_range(1000, 2000);
2395	/* igbvf_down has a dependency on max_frame_size */
2396	adapter->max_frame_size = max_frame;
2397	if (netif_running(netdev))
2398		igbvf_down(adapter);
2399
2400	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2401	 * means we reserve 2 more, this pushes us to allocate from the next
2402	 * larger slab size.
2403	 * i.e. RXBUFFER_2048 --> size-4096 slab
2404	 * However with the new *_jumbo_rx* routines, jumbo receives will use
2405	 * fragmented skbs
2406	 */
2407
2408	if (max_frame <= 1024)
2409		adapter->rx_buffer_len = 1024;
2410	else if (max_frame <= 2048)
2411		adapter->rx_buffer_len = 2048;
2412	else
2413#if (PAGE_SIZE / 2) > 16384
2414		adapter->rx_buffer_len = 16384;
2415#else
2416		adapter->rx_buffer_len = PAGE_SIZE / 2;
2417#endif
2418
2419	/* adjust allocation if LPE protects us, and we aren't using SBP */
2420	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2421	    (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
2422		adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +
2423					 ETH_FCS_LEN;
2424
2425	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2426		   netdev->mtu, new_mtu);
2427	WRITE_ONCE(netdev->mtu, new_mtu);
2428
2429	if (netif_running(netdev))
2430		igbvf_up(adapter);
2431	else
2432		igbvf_reset(adapter);
2433
2434	clear_bit(__IGBVF_RESETTING, &adapter->state);
2435
2436	return 0;
2437}
2438
2439static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2440{
2441	switch (cmd) {
2442	default:
2443		return -EOPNOTSUPP;
2444	}
2445}
2446
2447static int igbvf_suspend(struct device *dev_d)
2448{
2449	struct net_device *netdev = dev_get_drvdata(dev_d);
2450	struct igbvf_adapter *adapter = netdev_priv(netdev);
2451
2452	netif_device_detach(netdev);
2453
2454	if (netif_running(netdev)) {
2455		WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
2456		igbvf_down(adapter);
2457		igbvf_free_irq(adapter);
2458	}
2459
2460	return 0;
2461}
2462
2463static int igbvf_resume(struct device *dev_d)
2464{
2465	struct pci_dev *pdev = to_pci_dev(dev_d);
2466	struct net_device *netdev = pci_get_drvdata(pdev);
2467	struct igbvf_adapter *adapter = netdev_priv(netdev);
2468	u32 err;
2469
2470	pci_set_master(pdev);
2471
2472	if (netif_running(netdev)) {
2473		err = igbvf_request_irq(adapter);
2474		if (err)
2475			return err;
2476	}
2477
2478	igbvf_reset(adapter);
2479
2480	if (netif_running(netdev))
2481		igbvf_up(adapter);
2482
2483	netif_device_attach(netdev);
2484
2485	return 0;
2486}
2487
2488static void igbvf_shutdown(struct pci_dev *pdev)
2489{
2490	igbvf_suspend(&pdev->dev);
2491}
2492
2493#ifdef CONFIG_NET_POLL_CONTROLLER
2494/* Polling 'interrupt' - used by things like netconsole to send skbs
2495 * without having to re-enable interrupts. It's not called while
2496 * the interrupt routine is executing.
2497 */
2498static void igbvf_netpoll(struct net_device *netdev)
2499{
2500	struct igbvf_adapter *adapter = netdev_priv(netdev);
2501
2502	disable_irq(adapter->pdev->irq);
2503
2504	igbvf_clean_tx_irq(adapter->tx_ring);
2505
2506	enable_irq(adapter->pdev->irq);
2507}
2508#endif
2509
2510/**
2511 * igbvf_io_error_detected - called when PCI error is detected
2512 * @pdev: Pointer to PCI device
2513 * @state: The current pci connection state
2514 *
2515 * This function is called after a PCI bus error affecting
2516 * this device has been detected.
2517 */
2518static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,
2519						pci_channel_state_t state)
2520{
2521	struct net_device *netdev = pci_get_drvdata(pdev);
2522	struct igbvf_adapter *adapter = netdev_priv(netdev);
2523
2524	netif_device_detach(netdev);
2525
2526	if (state == pci_channel_io_perm_failure)
2527		return PCI_ERS_RESULT_DISCONNECT;
2528
2529	if (netif_running(netdev))
2530		igbvf_down(adapter);
2531	pci_disable_device(pdev);
2532
2533	/* Request a slot reset. */
2534	return PCI_ERS_RESULT_NEED_RESET;
2535}
2536
2537/**
2538 * igbvf_io_slot_reset - called after the pci bus has been reset.
2539 * @pdev: Pointer to PCI device
2540 *
2541 * Restart the card from scratch, as if from a cold-boot. Implementation
2542 * resembles the first-half of the igbvf_resume routine.
2543 */
2544static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)
2545{
2546	struct net_device *netdev = pci_get_drvdata(pdev);
2547	struct igbvf_adapter *adapter = netdev_priv(netdev);
2548
2549	if (pci_enable_device_mem(pdev)) {
2550		dev_err(&pdev->dev,
2551			"Cannot re-enable PCI device after reset.\n");
2552		return PCI_ERS_RESULT_DISCONNECT;
2553	}
2554	pci_set_master(pdev);
2555
2556	igbvf_reset(adapter);
2557
2558	return PCI_ERS_RESULT_RECOVERED;
2559}
2560
2561/**
2562 * igbvf_io_resume - called when traffic can start flowing again.
2563 * @pdev: Pointer to PCI device
2564 *
2565 * This callback is called when the error recovery driver tells us that
2566 * its OK to resume normal operation. Implementation resembles the
2567 * second-half of the igbvf_resume routine.
2568 */
2569static void igbvf_io_resume(struct pci_dev *pdev)
2570{
2571	struct net_device *netdev = pci_get_drvdata(pdev);
2572	struct igbvf_adapter *adapter = netdev_priv(netdev);
2573
2574	if (netif_running(netdev)) {
2575		if (igbvf_up(adapter)) {
2576			dev_err(&pdev->dev,
2577				"can't bring device back up after reset\n");
2578			return;
2579		}
2580	}
2581
2582	netif_device_attach(netdev);
2583}
2584
2585/**
2586 * igbvf_io_prepare - prepare device driver for PCI reset
2587 * @pdev: PCI device information struct
2588 */
2589static void igbvf_io_prepare(struct pci_dev *pdev)
2590{
2591	struct net_device *netdev = pci_get_drvdata(pdev);
2592	struct igbvf_adapter *adapter = netdev_priv(netdev);
2593
2594	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2595		usleep_range(1000, 2000);
2596	igbvf_down(adapter);
2597}
2598
2599/**
2600 * igbvf_io_reset_done - PCI reset done, device driver reset can begin
2601 * @pdev: PCI device information struct
2602 */
2603static void igbvf_io_reset_done(struct pci_dev *pdev)
2604{
2605	struct net_device *netdev = pci_get_drvdata(pdev);
2606	struct igbvf_adapter *adapter = netdev_priv(netdev);
2607
2608	igbvf_up(adapter);
2609	clear_bit(__IGBVF_RESETTING, &adapter->state);
2610}
2611
2612static void igbvf_print_device_info(struct igbvf_adapter *adapter)
2613{
2614	struct e1000_hw *hw = &adapter->hw;
2615	struct net_device *netdev = adapter->netdev;
2616	struct pci_dev *pdev = adapter->pdev;
2617
2618	if (hw->mac.type == e1000_vfadapt_i350)
2619		dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");
2620	else
2621		dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");
2622	dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);
2623}
2624
2625static int igbvf_set_features(struct net_device *netdev,
2626			      netdev_features_t features)
2627{
2628	struct igbvf_adapter *adapter = netdev_priv(netdev);
2629
2630	if (features & NETIF_F_RXCSUM)
2631		adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;
2632	else
2633		adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;
2634
2635	return 0;
2636}
2637
2638#define IGBVF_MAX_MAC_HDR_LEN		127
2639#define IGBVF_MAX_NETWORK_HDR_LEN	511
2640
2641static netdev_features_t
2642igbvf_features_check(struct sk_buff *skb, struct net_device *dev,
2643		     netdev_features_t features)
2644{
2645	unsigned int network_hdr_len, mac_hdr_len;
2646
2647	/* Make certain the headers can be described by a context descriptor */
2648	mac_hdr_len = skb_network_offset(skb);
2649	if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))
2650		return features & ~(NETIF_F_HW_CSUM |
2651				    NETIF_F_SCTP_CRC |
2652				    NETIF_F_HW_VLAN_CTAG_TX |
2653				    NETIF_F_TSO |
2654				    NETIF_F_TSO6);
2655
2656	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2657	if (unlikely(network_hdr_len >  IGBVF_MAX_NETWORK_HDR_LEN))
2658		return features & ~(NETIF_F_HW_CSUM |
2659				    NETIF_F_SCTP_CRC |
2660				    NETIF_F_TSO |
2661				    NETIF_F_TSO6);
2662
2663	/* We can only support IPV4 TSO in tunnels if we can mangle the
2664	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2665	 */
2666	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2667		features &= ~NETIF_F_TSO;
2668
2669	return features;
2670}
2671
2672static const struct net_device_ops igbvf_netdev_ops = {
2673	.ndo_open		= igbvf_open,
2674	.ndo_stop		= igbvf_close,
2675	.ndo_start_xmit		= igbvf_xmit_frame,
2676	.ndo_set_rx_mode	= igbvf_set_rx_mode,
2677	.ndo_set_mac_address	= igbvf_set_mac,
2678	.ndo_change_mtu		= igbvf_change_mtu,
2679	.ndo_eth_ioctl		= igbvf_ioctl,
2680	.ndo_tx_timeout		= igbvf_tx_timeout,
2681	.ndo_vlan_rx_add_vid	= igbvf_vlan_rx_add_vid,
2682	.ndo_vlan_rx_kill_vid	= igbvf_vlan_rx_kill_vid,
2683#ifdef CONFIG_NET_POLL_CONTROLLER
2684	.ndo_poll_controller	= igbvf_netpoll,
2685#endif
2686	.ndo_set_features	= igbvf_set_features,
2687	.ndo_features_check	= igbvf_features_check,
2688};
2689
2690/**
2691 * igbvf_probe - Device Initialization Routine
2692 * @pdev: PCI device information struct
2693 * @ent: entry in igbvf_pci_tbl
2694 *
2695 * Returns 0 on success, negative on failure
2696 *
2697 * igbvf_probe initializes an adapter identified by a pci_dev structure.
2698 * The OS initialization, configuring of the adapter private structure,
2699 * and a hardware reset occur.
2700 **/
2701static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2702{
2703	struct net_device *netdev;
2704	struct igbvf_adapter *adapter;
2705	struct e1000_hw *hw;
2706	const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];
2707	int err;
2708
2709	err = pci_enable_device_mem(pdev);
2710	if (err)
2711		return err;
2712
2713	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2714	if (err) {
2715		dev_err(&pdev->dev,
2716			"No usable DMA configuration, aborting\n");
2717		goto err_dma;
2718	}
2719
2720	err = pci_request_regions(pdev, igbvf_driver_name);
2721	if (err)
2722		goto err_pci_reg;
2723
2724	pci_set_master(pdev);
2725
2726	err = -ENOMEM;
2727	netdev = alloc_etherdev(sizeof(struct igbvf_adapter));
2728	if (!netdev)
2729		goto err_alloc_etherdev;
2730
2731	SET_NETDEV_DEV(netdev, &pdev->dev);
2732
2733	pci_set_drvdata(pdev, netdev);
2734	adapter = netdev_priv(netdev);
2735	hw = &adapter->hw;
2736	adapter->netdev = netdev;
2737	adapter->pdev = pdev;
2738	adapter->ei = ei;
2739	adapter->pba = ei->pba;
2740	adapter->flags = ei->flags;
2741	adapter->hw.back = adapter;
2742	adapter->hw.mac.type = ei->mac;
2743	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2744
2745	/* PCI config space info */
2746
2747	hw->vendor_id = pdev->vendor;
2748	hw->device_id = pdev->device;
2749	hw->subsystem_vendor_id = pdev->subsystem_vendor;
2750	hw->subsystem_device_id = pdev->subsystem_device;
2751	hw->revision_id = pdev->revision;
2752
2753	err = -EIO;
2754	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),
2755				      pci_resource_len(pdev, 0));
2756
2757	if (!adapter->hw.hw_addr)
2758		goto err_ioremap;
2759
2760	if (ei->get_variants) {
2761		err = ei->get_variants(adapter);
2762		if (err)
2763			goto err_get_variants;
2764	}
2765
2766	/* setup adapter struct */
2767	err = igbvf_sw_init(adapter);
2768	if (err)
2769		goto err_sw_init;
2770
2771	/* construct the net_device struct */
2772	netdev->netdev_ops = &igbvf_netdev_ops;
2773
2774	igbvf_set_ethtool_ops(netdev);
2775	netdev->watchdog_timeo = 5 * HZ;
2776	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
2777
2778	netdev->hw_features = NETIF_F_SG |
2779			      NETIF_F_TSO |
2780			      NETIF_F_TSO6 |
2781			      NETIF_F_RXCSUM |
2782			      NETIF_F_HW_CSUM |
2783			      NETIF_F_SCTP_CRC;
2784
2785#define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2786				    NETIF_F_GSO_GRE_CSUM | \
2787				    NETIF_F_GSO_IPXIP4 | \
2788				    NETIF_F_GSO_IPXIP6 | \
2789				    NETIF_F_GSO_UDP_TUNNEL | \
2790				    NETIF_F_GSO_UDP_TUNNEL_CSUM)
2791
2792	netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;
2793	netdev->hw_features |= NETIF_F_GSO_PARTIAL |
2794			       IGBVF_GSO_PARTIAL_FEATURES;
2795
2796	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
2797
2798	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2799	netdev->mpls_features |= NETIF_F_HW_CSUM;
2800	netdev->hw_enc_features |= netdev->vlan_features;
2801
2802	/* set this bit last since it cannot be part of vlan_features */
2803	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2804			    NETIF_F_HW_VLAN_CTAG_RX |
2805			    NETIF_F_HW_VLAN_CTAG_TX;
2806
2807	/* MTU range: 68 - 9216 */
2808	netdev->min_mtu = ETH_MIN_MTU;
2809	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2810
2811	spin_lock_bh(&hw->mbx_lock);
2812
2813	/*reset the controller to put the device in a known good state */
2814	err = hw->mac.ops.reset_hw(hw);
2815	if (err) {
2816		dev_info(&pdev->dev,
2817			 "PF still in reset state. Is the PF interface up?\n");
2818	} else {
2819		err = hw->mac.ops.read_mac_addr(hw);
2820		if (err)
2821			dev_info(&pdev->dev, "Error reading MAC address.\n");
2822		else if (is_zero_ether_addr(adapter->hw.mac.addr))
2823			dev_info(&pdev->dev,
2824				 "MAC address not assigned by administrator.\n");
2825		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2826	}
2827
2828	spin_unlock_bh(&hw->mbx_lock);
2829
2830	if (!is_valid_ether_addr(netdev->dev_addr)) {
2831		dev_info(&pdev->dev, "Assigning random MAC address.\n");
2832		eth_hw_addr_random(netdev);
2833		memcpy(adapter->hw.mac.addr, netdev->dev_addr,
2834		       netdev->addr_len);
2835	}
2836
2837	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);
2838
2839	INIT_WORK(&adapter->reset_task, igbvf_reset_task);
2840	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);
2841
2842	/* ring size defaults */
2843	adapter->rx_ring->count = 1024;
2844	adapter->tx_ring->count = 1024;
2845
2846	/* reset the hardware with the new settings */
2847	igbvf_reset(adapter);
2848
2849	/* set hardware-specific flags */
2850	if (adapter->hw.mac.type == e1000_vfadapt_i350)
2851		adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;
2852
2853	strcpy(netdev->name, "eth%d");
2854	err = register_netdev(netdev);
2855	if (err)
2856		goto err_hw_init;
2857
2858	/* tell the stack to leave us alone until igbvf_open() is called */
2859	netif_carrier_off(netdev);
2860	netif_stop_queue(netdev);
2861
2862	igbvf_print_device_info(adapter);
2863
2864	igbvf_initialize_last_counter_stats(adapter);
2865
2866	return 0;
2867
2868err_hw_init:
2869	netif_napi_del(&adapter->rx_ring->napi);
2870	kfree(adapter->tx_ring);
2871	kfree(adapter->rx_ring);
2872err_sw_init:
2873	igbvf_reset_interrupt_capability(adapter);
2874err_get_variants:
2875	iounmap(adapter->hw.hw_addr);
2876err_ioremap:
2877	free_netdev(netdev);
2878err_alloc_etherdev:
2879	pci_release_regions(pdev);
2880err_pci_reg:
2881err_dma:
2882	pci_disable_device(pdev);
2883	return err;
2884}
2885
2886/**
2887 * igbvf_remove - Device Removal Routine
2888 * @pdev: PCI device information struct
2889 *
2890 * igbvf_remove is called by the PCI subsystem to alert the driver
2891 * that it should release a PCI device.  The could be caused by a
2892 * Hot-Plug event, or because the driver is going to be removed from
2893 * memory.
2894 **/
2895static void igbvf_remove(struct pci_dev *pdev)
2896{
2897	struct net_device *netdev = pci_get_drvdata(pdev);
2898	struct igbvf_adapter *adapter = netdev_priv(netdev);
2899	struct e1000_hw *hw = &adapter->hw;
2900
2901	/* The watchdog timer may be rescheduled, so explicitly
2902	 * disable it from being rescheduled.
2903	 */
2904	set_bit(__IGBVF_DOWN, &adapter->state);
2905	timer_delete_sync(&adapter->watchdog_timer);
2906
2907	cancel_work_sync(&adapter->reset_task);
2908	cancel_work_sync(&adapter->watchdog_task);
2909
2910	unregister_netdev(netdev);
2911
2912	igbvf_reset_interrupt_capability(adapter);
2913
2914	/* it is important to delete the NAPI struct prior to freeing the
2915	 * Rx ring so that you do not end up with null pointer refs
2916	 */
2917	netif_napi_del(&adapter->rx_ring->napi);
2918	kfree(adapter->tx_ring);
2919	kfree(adapter->rx_ring);
2920
2921	iounmap(hw->hw_addr);
2922	if (hw->flash_address)
2923		iounmap(hw->flash_address);
2924	pci_release_regions(pdev);
2925
2926	free_netdev(netdev);
2927
2928	pci_disable_device(pdev);
2929}
2930
2931/* PCI Error Recovery (ERS) */
2932static const struct pci_error_handlers igbvf_err_handler = {
2933	.error_detected = igbvf_io_error_detected,
2934	.slot_reset = igbvf_io_slot_reset,
2935	.resume = igbvf_io_resume,
2936	.reset_prepare = igbvf_io_prepare,
2937	.reset_done = igbvf_io_reset_done,
2938};
2939
2940static const struct pci_device_id igbvf_pci_tbl[] = {
2941	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
2942	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
2943	{ } /* terminate list */
2944};
2945MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
2946
2947static DEFINE_SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);
2948
2949/* PCI Device API Driver */
2950static struct pci_driver igbvf_driver = {
2951	.name		= igbvf_driver_name,
2952	.id_table	= igbvf_pci_tbl,
2953	.probe		= igbvf_probe,
2954	.remove		= igbvf_remove,
2955	.driver.pm	= pm_sleep_ptr(&igbvf_pm_ops),
2956	.shutdown	= igbvf_shutdown,
2957	.err_handler	= &igbvf_err_handler
2958};
2959
2960/**
2961 * igbvf_init_module - Driver Registration Routine
2962 *
2963 * igbvf_init_module is the first routine called when the driver is
2964 * loaded. All it does is register with the PCI subsystem.
2965 **/
2966static int __init igbvf_init_module(void)
2967{
2968	int ret;
2969
2970	pr_info("%s\n", igbvf_driver_string);
2971	pr_info("%s\n", igbvf_copyright);
2972
2973	ret = pci_register_driver(&igbvf_driver);
2974
2975	return ret;
2976}
2977module_init(igbvf_init_module);
2978
2979/**
2980 * igbvf_exit_module - Driver Exit Cleanup Routine
2981 *
2982 * igbvf_exit_module is called just before the driver is removed
2983 * from memory.
2984 **/
2985static void __exit igbvf_exit_module(void)
2986{
2987	pci_unregister_driver(&igbvf_driver);
2988}
2989module_exit(igbvf_exit_module);
2990
2991MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");
2992MODULE_LICENSE("GPL v2");
2993
2994/* netdev.c */