qlcnic_io.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * QLogic qlcnic NIC Driver
   4 * Copyright (c) 2009-2013 QLogic Corporation
   5 */
   6
   7#include <linux/netdevice.h>
   8#include <linux/if_vlan.h>
   9#include <net/ip.h>
  10#include <linux/ipv6.h>
  11#include <net/checksum.h>
  12#include <linux/printk.h>
  13#include <linux/jiffies.h>
  14
  15#include "qlcnic.h"
  16
  17#define QLCNIC_TX_ETHER_PKT		0x01
  18#define QLCNIC_TX_TCP_PKT		0x02
  19#define QLCNIC_TX_UDP_PKT		0x03
  20#define QLCNIC_TX_IP_PKT		0x04
  21#define QLCNIC_TX_TCP_LSO		0x05
  22#define QLCNIC_TX_TCP_LSO6		0x06
  23#define QLCNIC_TX_ENCAP_PKT		0x07
  24#define QLCNIC_TX_ENCAP_LSO		0x08
  25#define QLCNIC_TX_TCPV6_PKT		0x0b
  26#define QLCNIC_TX_UDPV6_PKT		0x0c
  27
  28#define QLCNIC_FLAGS_VLAN_TAGGED	0x10
  29#define QLCNIC_FLAGS_VLAN_OOB		0x40
  30
  31#define qlcnic_set_tx_vlan_tci(cmd_desc, v)	\
  32	(cmd_desc)->vlan_TCI = cpu_to_le16(v);
  33#define qlcnic_set_cmd_desc_port(cmd_desc, var)	\
  34	((cmd_desc)->port_ctxid |= ((var) & 0x0F))
  35#define qlcnic_set_cmd_desc_ctxid(cmd_desc, var)	\
  36	((cmd_desc)->port_ctxid |= ((var) << 4 & 0xF0))
  37
  38#define qlcnic_set_tx_port(_desc, _port) \
  39	((_desc)->port_ctxid = ((_port) & 0xf) | (((_port) << 4) & 0xf0))
  40
  41#define qlcnic_set_tx_flags_opcode(_desc, _flags, _opcode) \
  42	((_desc)->flags_opcode |= \
  43	cpu_to_le16(((_flags) & 0x7f) | (((_opcode) & 0x3f) << 7)))
  44
  45#define qlcnic_set_tx_frags_len(_desc, _frags, _len) \
  46	((_desc)->nfrags__length = \
  47	cpu_to_le32(((_frags) & 0xff) | (((_len) & 0xffffff) << 8)))
  48
  49/* owner bits of status_desc */
  50#define STATUS_OWNER_HOST	(0x1ULL << 56)
  51#define STATUS_OWNER_PHANTOM	(0x2ULL << 56)
  52
  53/* Status descriptor:
  54   0-3 port, 4-7 status, 8-11 type, 12-27 total_length
  55   28-43 reference_handle, 44-47 protocol, 48-52 pkt_offset
  56   53-55 desc_cnt, 56-57 owner, 58-63 opcode
  57 */
  58#define qlcnic_get_sts_port(sts_data)	\
  59	((sts_data) & 0x0F)
  60#define qlcnic_get_sts_status(sts_data)	\
  61	(((sts_data) >> 4) & 0x0F)
  62#define qlcnic_get_sts_type(sts_data)	\
  63	(((sts_data) >> 8) & 0x0F)
  64#define qlcnic_get_sts_totallength(sts_data)	\
  65	(((sts_data) >> 12) & 0xFFFF)
  66#define qlcnic_get_sts_refhandle(sts_data)	\
  67	(((sts_data) >> 28) & 0xFFFF)
  68#define qlcnic_get_sts_prot(sts_data)	\
  69	(((sts_data) >> 44) & 0x0F)
  70#define qlcnic_get_sts_pkt_offset(sts_data)	\
  71	(((sts_data) >> 48) & 0x1F)
  72#define qlcnic_get_sts_desc_cnt(sts_data)	\
  73	(((sts_data) >> 53) & 0x7)
  74#define qlcnic_get_sts_opcode(sts_data)	\
  75	(((sts_data) >> 58) & 0x03F)
  76
  77#define qlcnic_get_lro_sts_refhandle(sts_data) 	\
  78	((sts_data) & 0x07FFF)
  79#define qlcnic_get_lro_sts_length(sts_data)	\
  80	(((sts_data) >> 16) & 0x0FFFF)
  81#define qlcnic_get_lro_sts_l2_hdr_offset(sts_data)	\
  82	(((sts_data) >> 32) & 0x0FF)
  83#define qlcnic_get_lro_sts_l4_hdr_offset(sts_data)	\
  84	(((sts_data) >> 40) & 0x0FF)
  85#define qlcnic_get_lro_sts_timestamp(sts_data)	\
  86	(((sts_data) >> 48) & 0x1)
  87#define qlcnic_get_lro_sts_type(sts_data)	\
  88	(((sts_data) >> 49) & 0x7)
  89#define qlcnic_get_lro_sts_push_flag(sts_data)		\
  90	(((sts_data) >> 52) & 0x1)
  91#define qlcnic_get_lro_sts_seq_number(sts_data)		\
  92	((sts_data) & 0x0FFFFFFFF)
  93#define qlcnic_get_lro_sts_mss(sts_data1)		\
  94	((sts_data1 >> 32) & 0x0FFFF)
  95
  96#define qlcnic_83xx_get_lro_sts_mss(sts) ((sts) & 0xffff)
  97
  98/* opcode field in status_desc */
  99#define QLCNIC_SYN_OFFLOAD	0x03
 100#define QLCNIC_RXPKT_DESC  	0x04
 101#define QLCNIC_OLD_RXPKT_DESC	0x3f
 102#define QLCNIC_RESPONSE_DESC	0x05
 103#define QLCNIC_LRO_DESC  	0x12
 104
 105#define QLCNIC_TCP_HDR_SIZE		20
 106#define QLCNIC_TCP_TS_OPTION_SIZE	12
 107#define QLCNIC_FETCH_RING_ID(handle)	((handle) >> 63)
 108#define QLCNIC_DESC_OWNER_FW		cpu_to_le64(STATUS_OWNER_PHANTOM)
 109
 110#define QLCNIC_TCP_TS_HDR_SIZE (QLCNIC_TCP_HDR_SIZE + QLCNIC_TCP_TS_OPTION_SIZE)
 111
 112/* for status field in status_desc */
 113#define STATUS_CKSUM_LOOP	0
 114#define STATUS_CKSUM_OK		2
 115
 116#define qlcnic_83xx_pktln(sts)		((sts >> 32) & 0x3FFF)
 117#define qlcnic_83xx_hndl(sts)		((sts >> 48) & 0x7FFF)
 118#define qlcnic_83xx_csum_status(sts)	((sts >> 39) & 7)
 119#define qlcnic_83xx_opcode(sts)	((sts >> 42) & 0xF)
 120#define qlcnic_83xx_vlan_tag(sts)	(((sts) >> 48) & 0xFFFF)
 121#define qlcnic_83xx_lro_pktln(sts)	(((sts) >> 32) & 0x3FFF)
 122#define qlcnic_83xx_l2_hdr_off(sts)	(((sts) >> 16) & 0xFF)
 123#define qlcnic_83xx_l4_hdr_off(sts)	(((sts) >> 24) & 0xFF)
 124#define qlcnic_83xx_pkt_cnt(sts)	(((sts) >> 16) & 0x7)
 125#define qlcnic_83xx_is_tstamp(sts)	(((sts) >> 40) & 1)
 126#define qlcnic_83xx_is_psh_bit(sts)	(((sts) >> 41) & 1)
 127#define qlcnic_83xx_is_ip_align(sts)	(((sts) >> 46) & 1)
 128#define qlcnic_83xx_has_vlan_tag(sts)	(((sts) >> 47) & 1)
 129
 130static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring,
 131				   int max);
 132
 133static struct sk_buff *qlcnic_process_rxbuf(struct qlcnic_adapter *,
 134					    struct qlcnic_host_rds_ring *,
 135					    u16, u16);
 136
 137static inline u8 qlcnic_mac_hash(u64 mac, u16 vlan)
 138{
 139	return (u8)((mac & 0xff) ^ ((mac >> 40) & 0xff) ^ (vlan & 0xff));
 140}
 141
 142static inline u32 qlcnic_get_ref_handle(struct qlcnic_adapter *adapter,
 143					u16 handle, u8 ring_id)
 144{
 145	if (qlcnic_83xx_check(adapter))
 146		return handle | (ring_id << 15);
 147	else
 148		return handle;
 149}
 150
 151static inline int qlcnic_82xx_is_lb_pkt(u64 sts_data)
 152{
 153	return (qlcnic_get_sts_status(sts_data) == STATUS_CKSUM_LOOP) ? 1 : 0;
 154}
 155
 156static void qlcnic_delete_rx_list_mac(struct qlcnic_adapter *adapter,
 157				      struct qlcnic_filter *fil,
 158				      void *addr, u16 vlan_id)
 159{
 160	int ret;
 161	u8 op;
 162
 163	op = vlan_id ? QLCNIC_MAC_VLAN_ADD : QLCNIC_MAC_ADD;
 164	ret = qlcnic_sre_macaddr_change(adapter, addr, vlan_id, op);
 165	if (ret)
 166		return;
 167
 168	op = vlan_id ? QLCNIC_MAC_VLAN_DEL : QLCNIC_MAC_DEL;
 169	ret = qlcnic_sre_macaddr_change(adapter, addr, vlan_id, op);
 170	if (!ret) {
 171		hlist_del(&fil->fnode);
 172		adapter->rx_fhash.fnum--;
 173	}
 174}
 175
 176static struct qlcnic_filter *qlcnic_find_mac_filter(struct hlist_head *head,
 177						    void *addr, u16 vlan_id)
 178{
 179	struct qlcnic_filter *tmp_fil = NULL;
 180	struct hlist_node *n;
 181
 182	hlist_for_each_entry_safe(tmp_fil, n, head, fnode) {
 183		if (ether_addr_equal(tmp_fil->faddr, addr) &&
 184		    tmp_fil->vlan_id == vlan_id)
 185			return tmp_fil;
 186	}
 187
 188	return NULL;
 189}
 190
 191static void qlcnic_add_lb_filter(struct qlcnic_adapter *adapter,
 192				 struct sk_buff *skb, int loopback_pkt, u16 vlan_id)
 193{
 194	struct ethhdr *phdr = (struct ethhdr *)(skb->data);
 195	struct qlcnic_filter *fil, *tmp_fil;
 196	struct hlist_head *head;
 197	unsigned long time;
 198	u64 src_addr = 0;
 199	u8 hindex, op;
 200	int ret;
 201
 202	if (!qlcnic_sriov_pf_check(adapter) || (vlan_id == 0xffff))
 203		vlan_id = 0;
 204
 205	memcpy(&src_addr, phdr->h_source, ETH_ALEN);
 206	hindex = qlcnic_mac_hash(src_addr, vlan_id) &
 207		 (adapter->fhash.fbucket_size - 1);
 208
 209	if (loopback_pkt) {
 210		if (adapter->rx_fhash.fnum >= adapter->rx_fhash.fmax)
 211			return;
 212
 213		head = &(adapter->rx_fhash.fhead[hindex]);
 214
 215		tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id);
 216		if (tmp_fil) {
 217			time = tmp_fil->ftime;
 218			if (time_after(jiffies, QLCNIC_READD_AGE * HZ + time))
 219				tmp_fil->ftime = jiffies;
 220			return;
 221		}
 222
 223		fil = kzalloc(sizeof(struct qlcnic_filter), GFP_ATOMIC);
 224		if (!fil)
 225			return;
 226
 227		fil->ftime = jiffies;
 228		memcpy(fil->faddr, &src_addr, ETH_ALEN);
 229		fil->vlan_id = vlan_id;
 230		spin_lock(&adapter->rx_mac_learn_lock);
 231		hlist_add_head(&(fil->fnode), head);
 232		adapter->rx_fhash.fnum++;
 233		spin_unlock(&adapter->rx_mac_learn_lock);
 234	} else {
 235		head = &adapter->fhash.fhead[hindex];
 236
 237		spin_lock(&adapter->mac_learn_lock);
 238
 239		tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id);
 240		if (tmp_fil) {
 241			op = vlan_id ? QLCNIC_MAC_VLAN_DEL : QLCNIC_MAC_DEL;
 242			ret = qlcnic_sre_macaddr_change(adapter,
 243							(u8 *)&src_addr,
 244							vlan_id, op);
 245			if (!ret) {
 246				hlist_del(&tmp_fil->fnode);
 247				adapter->fhash.fnum--;
 248			}
 249
 250			spin_unlock(&adapter->mac_learn_lock);
 251
 252			return;
 253		}
 254
 255		spin_unlock(&adapter->mac_learn_lock);
 256
 257		head = &adapter->rx_fhash.fhead[hindex];
 258
 259		spin_lock(&adapter->rx_mac_learn_lock);
 260
 261		tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id);
 262		if (tmp_fil)
 263			qlcnic_delete_rx_list_mac(adapter, tmp_fil, &src_addr,
 264						  vlan_id);
 265
 266		spin_unlock(&adapter->rx_mac_learn_lock);
 267	}
 268}
 269
 270void qlcnic_82xx_change_filter(struct qlcnic_adapter *adapter, u64 *uaddr,
 271			       u16 vlan_id, struct qlcnic_host_tx_ring *tx_ring)
 272{
 273	struct cmd_desc_type0 *hwdesc;
 274	struct qlcnic_nic_req *req;
 275	struct qlcnic_mac_req *mac_req;
 276	struct qlcnic_vlan_req *vlan_req;
 277	u32 producer;
 278	u64 word;
 279
 280	producer = tx_ring->producer;
 281	hwdesc = &tx_ring->desc_head[tx_ring->producer];
 282
 283	req = (struct qlcnic_nic_req *)hwdesc;
 284	memset(req, 0, sizeof(struct qlcnic_nic_req));
 285	req->qhdr = cpu_to_le64(QLCNIC_REQUEST << 23);
 286
 287	word = QLCNIC_MAC_EVENT | ((u64)(adapter->portnum) << 16);
 288	req->req_hdr = cpu_to_le64(word);
 289
 290	mac_req = (struct qlcnic_mac_req *)&(req->words[0]);
 291	mac_req->op = vlan_id ? QLCNIC_MAC_VLAN_ADD : QLCNIC_MAC_ADD;
 292	memcpy(mac_req->mac_addr, uaddr, ETH_ALEN);
 293
 294	vlan_req = (struct qlcnic_vlan_req *)&req->words[1];
 295	vlan_req->vlan_id = cpu_to_le16(vlan_id);
 296
 297	tx_ring->producer = get_next_index(producer, tx_ring->num_desc);
 298	smp_mb();
 299}
 300
 301static void qlcnic_send_filter(struct qlcnic_adapter *adapter,
 302			       struct cmd_desc_type0 *first_desc,
 303			       struct sk_buff *skb,
 304			       struct qlcnic_host_tx_ring *tx_ring)
 305{
 306	struct vlan_ethhdr *vh = (struct vlan_ethhdr *)(skb->data);
 307	struct ethhdr *phdr = (struct ethhdr *)(skb->data);
 308	u16 protocol = ntohs(skb->protocol);
 309	struct qlcnic_filter *fil, *tmp_fil;
 310	struct hlist_head *head;
 311	struct hlist_node *n;
 312	u64 src_addr = 0;
 313	u16 vlan_id = 0;
 314	u8 hindex, hval;
 315
 316	if (ether_addr_equal(phdr->h_source, adapter->mac_addr))
 317		return;
 318
 319	if (adapter->flags & QLCNIC_VLAN_FILTERING) {
 320		if (protocol == ETH_P_8021Q) {
 321			vh = skb_vlan_eth_hdr(skb);
 322			vlan_id = ntohs(vh->h_vlan_TCI);
 323		} else if (skb_vlan_tag_present(skb)) {
 324			vlan_id = skb_vlan_tag_get(skb);
 325		}
 326	}
 327
 328	memcpy(&src_addr, phdr->h_source, ETH_ALEN);
 329	hval = qlcnic_mac_hash(src_addr, vlan_id);
 330	hindex = hval & (adapter->fhash.fbucket_size - 1);
 331	head = &(adapter->fhash.fhead[hindex]);
 332
 333	hlist_for_each_entry_safe(tmp_fil, n, head, fnode) {
 334		if (ether_addr_equal(tmp_fil->faddr, (u8 *)&src_addr) &&
 335		    tmp_fil->vlan_id == vlan_id) {
 336			if (time_is_before_jiffies(QLCNIC_READD_AGE * HZ + tmp_fil->ftime))
 337				qlcnic_change_filter(adapter, &src_addr,
 338						     vlan_id, tx_ring);
 339			tmp_fil->ftime = jiffies;
 340			return;
 341		}
 342	}
 343
 344	if (unlikely(adapter->fhash.fnum >= adapter->fhash.fmax)) {
 345		adapter->stats.mac_filter_limit_overrun++;
 346		return;
 347	}
 348
 349	fil = kzalloc(sizeof(struct qlcnic_filter), GFP_ATOMIC);
 350	if (!fil)
 351		return;
 352
 353	qlcnic_change_filter(adapter, &src_addr, vlan_id, tx_ring);
 354	fil->ftime = jiffies;
 355	fil->vlan_id = vlan_id;
 356	memcpy(fil->faddr, &src_addr, ETH_ALEN);
 357	spin_lock(&adapter->mac_learn_lock);
 358	hlist_add_head(&(fil->fnode), head);
 359	adapter->fhash.fnum++;
 360	spin_unlock(&adapter->mac_learn_lock);
 361}
 362
 363#define QLCNIC_ENCAP_VXLAN_PKT		BIT_0
 364#define QLCNIC_ENCAP_OUTER_L3_IP6	BIT_1
 365#define QLCNIC_ENCAP_INNER_L3_IP6	BIT_2
 366#define QLCNIC_ENCAP_INNER_L4_UDP	BIT_3
 367#define QLCNIC_ENCAP_DO_L3_CSUM		BIT_4
 368#define QLCNIC_ENCAP_DO_L4_CSUM		BIT_5
 369
 370static int qlcnic_tx_encap_pkt(struct qlcnic_adapter *adapter,
 371			       struct cmd_desc_type0 *first_desc,
 372			       struct sk_buff *skb,
 373			       struct qlcnic_host_tx_ring *tx_ring)
 374{
 375	u8 opcode = 0, inner_hdr_len = 0, outer_hdr_len = 0, total_hdr_len = 0;
 376	int copied, copy_len, descr_size;
 377	u32 producer = tx_ring->producer;
 378	struct cmd_desc_type0 *hwdesc;
 379	u16 flags = 0, encap_descr = 0;
 380
 381	opcode = QLCNIC_TX_ETHER_PKT;
 382	encap_descr = QLCNIC_ENCAP_VXLAN_PKT;
 383
 384	if (skb_is_gso(skb)) {
 385		inner_hdr_len = skb_inner_transport_header(skb) +
 386				inner_tcp_hdrlen(skb) -
 387				skb_inner_mac_header(skb);
 388
 389		/* VXLAN header size = 8 */
 390		outer_hdr_len = skb_transport_offset(skb) + 8 +
 391				sizeof(struct udphdr);
 392		first_desc->outer_hdr_length = outer_hdr_len;
 393		total_hdr_len = inner_hdr_len + outer_hdr_len;
 394		encap_descr |= QLCNIC_ENCAP_DO_L3_CSUM |
 395			       QLCNIC_ENCAP_DO_L4_CSUM;
 396		first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
 397		first_desc->hdr_length = inner_hdr_len;
 398
 399		/* Copy inner and outer headers in Tx descriptor(s)
 400		 * If total_hdr_len > cmd_desc_type0, use multiple
 401		 * descriptors
 402		 */
 403		copied = 0;
 404		descr_size = (int)sizeof(struct cmd_desc_type0);
 405		while (copied < total_hdr_len) {
 406			copy_len = min(descr_size, (total_hdr_len - copied));
 407			hwdesc = &tx_ring->desc_head[producer];
 408			tx_ring->cmd_buf_arr[producer].skb = NULL;
 409			skb_copy_from_linear_data_offset(skb, copied,
 410							 (char *)hwdesc,
 411							 copy_len);
 412			copied += copy_len;
 413			producer = get_next_index(producer, tx_ring->num_desc);
 414		}
 415
 416		tx_ring->producer = producer;
 417
 418		/* Make sure updated tx_ring->producer is visible
 419		 * for qlcnic_tx_avail()
 420		 */
 421		smp_mb();
 422		adapter->stats.encap_lso_frames++;
 423
 424		opcode = QLCNIC_TX_ENCAP_LSO;
 425	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 426		if (inner_ip_hdr(skb)->version == 6) {
 427			if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
 428				encap_descr |= QLCNIC_ENCAP_INNER_L4_UDP;
 429		} else {
 430			if (inner_ip_hdr(skb)->protocol == IPPROTO_UDP)
 431				encap_descr |= QLCNIC_ENCAP_INNER_L4_UDP;
 432		}
 433
 434		adapter->stats.encap_tx_csummed++;
 435		opcode = QLCNIC_TX_ENCAP_PKT;
 436	}
 437
 438	/* Prepare first 16 bits of byte offset 16 of Tx descriptor */
 439	if (ip_hdr(skb)->version == 6)
 440		encap_descr |= QLCNIC_ENCAP_OUTER_L3_IP6;
 441
 442	/* outer IP header's size in 32bit words size*/
 443	encap_descr |= (skb_network_header_len(skb) >> 2) << 6;
 444
 445	/* outer IP header offset */
 446	encap_descr |= skb_network_offset(skb) << 10;
 447	first_desc->encap_descr = cpu_to_le16(encap_descr);
 448
 449	first_desc->tcp_hdr_offset = skb_inner_transport_offset(skb);
 450	first_desc->ip_hdr_offset = skb_inner_network_offset(skb);
 451
 452	qlcnic_set_tx_flags_opcode(first_desc, flags, opcode);
 453
 454	return 0;
 455}
 456
 457static int qlcnic_tx_pkt(struct qlcnic_adapter *adapter,
 458			 struct cmd_desc_type0 *first_desc, struct sk_buff *skb,
 459			 struct qlcnic_host_tx_ring *tx_ring)
 460{
 461	u8 l4proto, opcode = 0, hdr_len = 0, tag_vlan = 0;
 462	u16 flags = 0, vlan_tci = 0;
 463	int copied, offset, copy_len, size;
 464	struct cmd_desc_type0 *hwdesc;
 465	struct vlan_ethhdr *vh;
 466	u16 protocol = ntohs(skb->protocol);
 467	u32 producer = tx_ring->producer;
 468
 469	if (protocol == ETH_P_8021Q) {
 470		vh = skb_vlan_eth_hdr(skb);
 471		flags = QLCNIC_FLAGS_VLAN_TAGGED;
 472		vlan_tci = ntohs(vh->h_vlan_TCI);
 473		protocol = ntohs(vh->h_vlan_encapsulated_proto);
 474		tag_vlan = 1;
 475	} else if (skb_vlan_tag_present(skb)) {
 476		flags = QLCNIC_FLAGS_VLAN_OOB;
 477		vlan_tci = skb_vlan_tag_get(skb);
 478		tag_vlan = 1;
 479	}
 480	if (unlikely(adapter->tx_pvid)) {
 481		if (tag_vlan && !(adapter->flags & QLCNIC_TAGGING_ENABLED))
 482			return -EIO;
 483		if (tag_vlan && (adapter->flags & QLCNIC_TAGGING_ENABLED))
 484			goto set_flags;
 485
 486		flags = QLCNIC_FLAGS_VLAN_OOB;
 487		vlan_tci = adapter->tx_pvid;
 488	}
 489set_flags:
 490	qlcnic_set_tx_vlan_tci(first_desc, vlan_tci);
 491	qlcnic_set_tx_flags_opcode(first_desc, flags, opcode);
 492
 493	if (*(skb->data) & BIT_0) {
 494		flags |= BIT_0;
 495		memcpy(&first_desc->eth_addr, skb->data, ETH_ALEN);
 496	}
 497	opcode = QLCNIC_TX_ETHER_PKT;
 498	if (skb_is_gso(skb)) {
 499		hdr_len = skb_tcp_all_headers(skb);
 500		first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
 501		first_desc->hdr_length = hdr_len;
 502		opcode = (protocol == ETH_P_IPV6) ? QLCNIC_TX_TCP_LSO6 :
 503						    QLCNIC_TX_TCP_LSO;
 504
 505		/* For LSO, we need to copy the MAC/IP/TCP headers into
 506		* the descriptor ring */
 507		copied = 0;
 508		offset = 2;
 509
 510		if (flags & QLCNIC_FLAGS_VLAN_OOB) {
 511			first_desc->hdr_length += VLAN_HLEN;
 512			first_desc->tcp_hdr_offset = VLAN_HLEN;
 513			first_desc->ip_hdr_offset = VLAN_HLEN;
 514
 515			/* Only in case of TSO on vlan device */
 516			flags |= QLCNIC_FLAGS_VLAN_TAGGED;
 517
 518			/* Create a TSO vlan header template for firmware */
 519			hwdesc = &tx_ring->desc_head[producer];
 520			tx_ring->cmd_buf_arr[producer].skb = NULL;
 521
 522			copy_len = min((int)sizeof(struct cmd_desc_type0) -
 523				       offset, hdr_len + VLAN_HLEN);
 524
 525			vh = (struct vlan_ethhdr *)((char *) hwdesc + 2);
 526			skb_copy_from_linear_data(skb, vh, 12);
 527			vh->h_vlan_proto = htons(ETH_P_8021Q);
 528			vh->h_vlan_TCI = htons(vlan_tci);
 529
 530			skb_copy_from_linear_data_offset(skb, 12,
 531							 (char *)vh + 16,
 532							 copy_len - 16);
 533			copied = copy_len - VLAN_HLEN;
 534			offset = 0;
 535			producer = get_next_index(producer, tx_ring->num_desc);
 536		}
 537
 538		while (copied < hdr_len) {
 539			size = (int)sizeof(struct cmd_desc_type0) - offset;
 540			copy_len = min(size, (hdr_len - copied));
 541			hwdesc = &tx_ring->desc_head[producer];
 542			tx_ring->cmd_buf_arr[producer].skb = NULL;
 543			skb_copy_from_linear_data_offset(skb, copied,
 544							 (char *)hwdesc +
 545							 offset, copy_len);
 546			copied += copy_len;
 547			offset = 0;
 548			producer = get_next_index(producer, tx_ring->num_desc);
 549		}
 550
 551		tx_ring->producer = producer;
 552		smp_mb();
 553		adapter->stats.lso_frames++;
 554
 555	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 556		if (protocol == ETH_P_IP) {
 557			l4proto = ip_hdr(skb)->protocol;
 558
 559			if (l4proto == IPPROTO_TCP)
 560				opcode = QLCNIC_TX_TCP_PKT;
 561			else if (l4proto == IPPROTO_UDP)
 562				opcode = QLCNIC_TX_UDP_PKT;
 563		} else if (protocol == ETH_P_IPV6) {
 564			l4proto = ipv6_hdr(skb)->nexthdr;
 565
 566			if (l4proto == IPPROTO_TCP)
 567				opcode = QLCNIC_TX_TCPV6_PKT;
 568			else if (l4proto == IPPROTO_UDP)
 569				opcode = QLCNIC_TX_UDPV6_PKT;
 570		}
 571	}
 572	first_desc->tcp_hdr_offset += skb_transport_offset(skb);
 573	first_desc->ip_hdr_offset += skb_network_offset(skb);
 574	qlcnic_set_tx_flags_opcode(first_desc, flags, opcode);
 575
 576	return 0;
 577}
 578
 579static int qlcnic_map_tx_skb(struct pci_dev *pdev, struct sk_buff *skb,
 580			     struct qlcnic_cmd_buffer *pbuf)
 581{
 582	struct qlcnic_skb_frag *nf;
 583	skb_frag_t *frag;
 584	int i, nr_frags;
 585	dma_addr_t map;
 586
 587	nr_frags = skb_shinfo(skb)->nr_frags;
 588	nf = &pbuf->frag_array[0];
 589
 590	map = dma_map_single(&pdev->dev, skb->data, skb_headlen(skb),
 591			     DMA_TO_DEVICE);
 592	if (dma_mapping_error(&pdev->dev, map))
 593		goto out_err;
 594
 595	nf->dma = map;
 596	nf->length = skb_headlen(skb);
 597
 598	for (i = 0; i < nr_frags; i++) {
 599		frag = &skb_shinfo(skb)->frags[i];
 600		nf = &pbuf->frag_array[i+1];
 601		map = skb_frag_dma_map(&pdev->dev, frag, 0, skb_frag_size(frag),
 602				       DMA_TO_DEVICE);
 603		if (dma_mapping_error(&pdev->dev, map))
 604			goto unwind;
 605
 606		nf->dma = map;
 607		nf->length = skb_frag_size(frag);
 608	}
 609
 610	return 0;
 611
 612unwind:
 613	while (--i >= 0) {
 614		nf = &pbuf->frag_array[i+1];
 615		dma_unmap_page(&pdev->dev, nf->dma, nf->length, DMA_TO_DEVICE);
 616	}
 617
 618	nf = &pbuf->frag_array[0];
 619	dma_unmap_single(&pdev->dev, nf->dma, skb_headlen(skb), DMA_TO_DEVICE);
 620
 621out_err:
 622	return -ENOMEM;
 623}
 624
 625static void qlcnic_unmap_buffers(struct pci_dev *pdev, struct sk_buff *skb,
 626				 struct qlcnic_cmd_buffer *pbuf)
 627{
 628	struct qlcnic_skb_frag *nf = &pbuf->frag_array[0];
 629	int i, nr_frags = skb_shinfo(skb)->nr_frags;
 630
 631	for (i = 0; i < nr_frags; i++) {
 632		nf = &pbuf->frag_array[i+1];
 633		dma_unmap_page(&pdev->dev, nf->dma, nf->length, DMA_TO_DEVICE);
 634	}
 635
 636	nf = &pbuf->frag_array[0];
 637	dma_unmap_single(&pdev->dev, nf->dma, skb_headlen(skb), DMA_TO_DEVICE);
 638	pbuf->skb = NULL;
 639}
 640
 641static inline void qlcnic_clear_cmddesc(u64 *desc)
 642{
 643	desc[0] = 0ULL;
 644	desc[2] = 0ULL;
 645	desc[7] = 0ULL;
 646}
 647
 648netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 649{
 650	struct qlcnic_adapter *adapter = netdev_priv(netdev);
 651	struct qlcnic_host_tx_ring *tx_ring;
 652	struct qlcnic_cmd_buffer *pbuf;
 653	struct qlcnic_skb_frag *buffrag;
 654	struct cmd_desc_type0 *hwdesc, *first_desc;
 655	struct pci_dev *pdev;
 656	struct ethhdr *phdr;
 657	int i, k, frag_count, delta = 0;
 658	u32 producer, num_txd;
 659	u16 protocol;
 660	bool l4_is_udp = false;
 661
 662	if (!test_bit(__QLCNIC_DEV_UP, &adapter->state)) {
 663		netif_tx_stop_all_queues(netdev);
 664		return NETDEV_TX_BUSY;
 665	}
 666
 667	if (adapter->flags & QLCNIC_MACSPOOF) {
 668		phdr = (struct ethhdr *)skb->data;
 669		if (!ether_addr_equal(phdr->h_source, adapter->mac_addr))
 670			goto drop_packet;
 671	}
 672
 673	tx_ring = &adapter->tx_ring[skb_get_queue_mapping(skb)];
 674	num_txd = tx_ring->num_desc;
 675
 676	frag_count = skb_shinfo(skb)->nr_frags + 1;
 677
 678	/* 14 frags supported for normal packet and
 679	 * 32 frags supported for TSO packet
 680	 */
 681	if (!skb_is_gso(skb) && frag_count > QLCNIC_MAX_FRAGS_PER_TX) {
 682		for (i = 0; i < (frag_count - QLCNIC_MAX_FRAGS_PER_TX); i++)
 683			delta += skb_frag_size(&skb_shinfo(skb)->frags[i]);
 684
 685		if (!__pskb_pull_tail(skb, delta))
 686			goto drop_packet;
 687
 688		frag_count = 1 + skb_shinfo(skb)->nr_frags;
 689	}
 690
 691	if (unlikely(qlcnic_tx_avail(tx_ring) <= TX_STOP_THRESH)) {
 692		netif_tx_stop_queue(tx_ring->txq);
 693		if (qlcnic_tx_avail(tx_ring) > TX_STOP_THRESH) {
 694			netif_tx_start_queue(tx_ring->txq);
 695		} else {
 696			tx_ring->tx_stats.xmit_off++;
 697			return NETDEV_TX_BUSY;
 698		}
 699	}
 700
 701	producer = tx_ring->producer;
 702	pbuf = &tx_ring->cmd_buf_arr[producer];
 703	pdev = adapter->pdev;
 704	first_desc = &tx_ring->desc_head[producer];
 705	hwdesc = &tx_ring->desc_head[producer];
 706	qlcnic_clear_cmddesc((u64 *)hwdesc);
 707
 708	if (qlcnic_map_tx_skb(pdev, skb, pbuf)) {
 709		adapter->stats.tx_dma_map_error++;
 710		goto drop_packet;
 711	}
 712
 713	pbuf->skb = skb;
 714	pbuf->frag_count = frag_count;
 715
 716	qlcnic_set_tx_frags_len(first_desc, frag_count, skb->len);
 717	qlcnic_set_tx_port(first_desc, adapter->portnum);
 718
 719	for (i = 0; i < frag_count; i++) {
 720		k = i % 4;
 721
 722		if ((k == 0) && (i > 0)) {
 723			/* move to next desc.*/
 724			producer = get_next_index(producer, num_txd);
 725			hwdesc = &tx_ring->desc_head[producer];
 726			qlcnic_clear_cmddesc((u64 *)hwdesc);
 727			tx_ring->cmd_buf_arr[producer].skb = NULL;
 728		}
 729
 730		buffrag = &pbuf->frag_array[i];
 731		hwdesc->buffer_length[k] = cpu_to_le16(buffrag->length);
 732		switch (k) {
 733		case 0:
 734			hwdesc->addr_buffer1 = cpu_to_le64(buffrag->dma);
 735			break;
 736		case 1:
 737			hwdesc->addr_buffer2 = cpu_to_le64(buffrag->dma);
 738			break;
 739		case 2:
 740			hwdesc->addr_buffer3 = cpu_to_le64(buffrag->dma);
 741			break;
 742		case 3:
 743			hwdesc->addr_buffer4 = cpu_to_le64(buffrag->dma);
 744			break;
 745		}
 746	}
 747
 748	tx_ring->producer = get_next_index(producer, num_txd);
 749	smp_mb();
 750
 751	protocol = ntohs(skb->protocol);
 752	if (protocol == ETH_P_IP)
 753		l4_is_udp = ip_hdr(skb)->protocol == IPPROTO_UDP;
 754	else if (protocol == ETH_P_IPV6)
 755		l4_is_udp = ipv6_hdr(skb)->nexthdr == IPPROTO_UDP;
 756
 757	/* Check if it is a VXLAN packet */
 758	if (!skb->encapsulation || !l4_is_udp ||
 759	    !qlcnic_encap_tx_offload(adapter)) {
 760		if (unlikely(qlcnic_tx_pkt(adapter, first_desc, skb,
 761					   tx_ring)))
 762			goto unwind_buff;
 763	} else {
 764		if (unlikely(qlcnic_tx_encap_pkt(adapter, first_desc,
 765						 skb, tx_ring)))
 766			goto unwind_buff;
 767	}
 768
 769	if (adapter->drv_mac_learn)
 770		qlcnic_send_filter(adapter, first_desc, skb, tx_ring);
 771
 772	tx_ring->tx_stats.tx_bytes += skb->len;
 773	tx_ring->tx_stats.xmit_called++;
 774
 775	/* Ensure writes are complete before HW fetches Tx descriptors */
 776	wmb();
 777	qlcnic_update_cmd_producer(tx_ring);
 778
 779	return NETDEV_TX_OK;
 780
 781unwind_buff:
 782	qlcnic_unmap_buffers(pdev, skb, pbuf);
 783drop_packet:
 784	adapter->stats.txdropped++;
 785	dev_kfree_skb_any(skb);
 786	return NETDEV_TX_OK;
 787}
 788
 789void qlcnic_advert_link_change(struct qlcnic_adapter *adapter, int linkup)
 790{
 791	struct net_device *netdev = adapter->netdev;
 792
 793	if (adapter->ahw->linkup && !linkup) {
 794		netdev_info(netdev, "NIC Link is down\n");
 795		adapter->ahw->linkup = 0;
 796		netif_carrier_off(netdev);
 797	} else if (!adapter->ahw->linkup && linkup) {
 798		adapter->ahw->linkup = 1;
 799
 800		/* Do not advertise Link up to the stack if device
 801		 * is in loopback mode
 802		 */
 803		if (qlcnic_83xx_check(adapter) && adapter->ahw->lb_mode) {
 804			netdev_info(netdev, "NIC Link is up for loopback test\n");
 805			return;
 806		}
 807
 808		netdev_info(netdev, "NIC Link is up\n");
 809		netif_carrier_on(netdev);
 810	}
 811}
 812
 813static int qlcnic_alloc_rx_skb(struct qlcnic_adapter *adapter,
 814			       struct qlcnic_host_rds_ring *rds_ring,
 815			       struct qlcnic_rx_buffer *buffer)
 816{
 817	struct sk_buff *skb;
 818	dma_addr_t dma;
 819	struct pci_dev *pdev = adapter->pdev;
 820
 821	skb = netdev_alloc_skb(adapter->netdev, rds_ring->skb_size);
 822	if (!skb) {
 823		adapter->stats.skb_alloc_failure++;
 824		return -ENOMEM;
 825	}
 826
 827	skb_reserve(skb, NET_IP_ALIGN);
 828	dma = dma_map_single(&pdev->dev, skb->data, rds_ring->dma_size,
 829			     DMA_FROM_DEVICE);
 830
 831	if (dma_mapping_error(&pdev->dev, dma)) {
 832		adapter->stats.rx_dma_map_error++;
 833		dev_kfree_skb_any(skb);
 834		return -ENOMEM;
 835	}
 836
 837	buffer->skb = skb;
 838	buffer->dma = dma;
 839
 840	return 0;
 841}
 842
 843static void qlcnic_post_rx_buffers_nodb(struct qlcnic_adapter *adapter,
 844					struct qlcnic_host_rds_ring *rds_ring,
 845					u8 ring_id)
 846{
 847	struct rcv_desc *pdesc;
 848	struct qlcnic_rx_buffer *buffer;
 849	int  count = 0;
 850	uint32_t producer, handle;
 851	struct list_head *head;
 852
 853	if (!spin_trylock(&rds_ring->lock))
 854		return;
 855
 856	producer = rds_ring->producer;
 857	head = &rds_ring->free_list;
 858	while (!list_empty(head)) {
 859		buffer = list_entry(head->next, struct qlcnic_rx_buffer, list);
 860
 861		if (!buffer->skb) {
 862			if (qlcnic_alloc_rx_skb(adapter, rds_ring, buffer))
 863				break;
 864		}
 865		count++;
 866		list_del(&buffer->list);
 867
 868		/* make a rcv descriptor  */
 869		pdesc = &rds_ring->desc_head[producer];
 870		handle = qlcnic_get_ref_handle(adapter,
 871					       buffer->ref_handle, ring_id);
 872		pdesc->reference_handle = cpu_to_le16(handle);
 873		pdesc->buffer_length = cpu_to_le32(rds_ring->dma_size);
 874		pdesc->addr_buffer = cpu_to_le64(buffer->dma);
 875		producer = get_next_index(producer, rds_ring->num_desc);
 876	}
 877	if (count) {
 878		rds_ring->producer = producer;
 879		writel((producer - 1) & (rds_ring->num_desc - 1),
 880		       rds_ring->crb_rcv_producer);
 881	}
 882	spin_unlock(&rds_ring->lock);
 883}
 884
 885static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter,
 886				   struct qlcnic_host_tx_ring *tx_ring,
 887				   int budget)
 888{
 889	u32 sw_consumer, hw_consumer;
 890	int i, done, count = 0;
 891	struct qlcnic_cmd_buffer *buffer;
 892	struct pci_dev *pdev = adapter->pdev;
 893	struct net_device *netdev = adapter->netdev;
 894	struct qlcnic_skb_frag *frag;
 895
 896	if (!spin_trylock(&tx_ring->tx_clean_lock))
 897		return 1;
 898
 899	sw_consumer = tx_ring->sw_consumer;
 900	hw_consumer = le32_to_cpu(*(tx_ring->hw_consumer));
 901
 902	while (sw_consumer != hw_consumer) {
 903		buffer = &tx_ring->cmd_buf_arr[sw_consumer];
 904		if (buffer->skb) {
 905			frag = &buffer->frag_array[0];
 906			dma_unmap_single(&pdev->dev, frag->dma, frag->length,
 907					 DMA_TO_DEVICE);
 908			frag->dma = 0ULL;
 909			for (i = 1; i < buffer->frag_count; i++) {
 910				frag++;
 911				dma_unmap_page(&pdev->dev, frag->dma,
 912					       frag->length, DMA_TO_DEVICE);
 913				frag->dma = 0ULL;
 914			}
 915			tx_ring->tx_stats.xmit_finished++;
 916			dev_kfree_skb_any(buffer->skb);
 917			buffer->skb = NULL;
 918		}
 919
 920		sw_consumer = get_next_index(sw_consumer, tx_ring->num_desc);
 921		if (++count >= budget)
 922			break;
 923	}
 924
 925	tx_ring->sw_consumer = sw_consumer;
 926
 927	if (count && netif_running(netdev)) {
 928		smp_mb();
 929		if (netif_tx_queue_stopped(tx_ring->txq) &&
 930		    netif_carrier_ok(netdev)) {
 931			if (qlcnic_tx_avail(tx_ring) > TX_STOP_THRESH) {
 932				netif_tx_wake_queue(tx_ring->txq);
 933				tx_ring->tx_stats.xmit_on++;
 934			}
 935		}
 936		adapter->tx_timeo_cnt = 0;
 937	}
 938	/*
 939	 * If everything is freed up to consumer then check if the ring is full
 940	 * If the ring is full then check if more needs to be freed and
 941	 * schedule the call back again.
 942	 *
 943	 * This happens when there are 2 CPUs. One could be freeing and the
 944	 * other filling it. If the ring is full when we get out of here and
 945	 * the card has already interrupted the host then the host can miss the
 946	 * interrupt.
 947	 *
 948	 * There is still a possible race condition and the host could miss an
 949	 * interrupt. The card has to take care of this.
 950	 */
 951	hw_consumer = le32_to_cpu(*(tx_ring->hw_consumer));
 952	done = (sw_consumer == hw_consumer);
 953
 954	spin_unlock(&tx_ring->tx_clean_lock);
 955
 956	return done;
 957}
 958
 959static int qlcnic_poll(struct napi_struct *napi, int budget)
 960{
 961	int tx_complete, work_done;
 962	struct qlcnic_host_sds_ring *sds_ring;
 963	struct qlcnic_adapter *adapter;
 964	struct qlcnic_host_tx_ring *tx_ring;
 965
 966	sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
 967	adapter = sds_ring->adapter;
 968	tx_ring = sds_ring->tx_ring;
 969
 970	tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring,
 971					      budget);
 972	work_done = qlcnic_process_rcv_ring(sds_ring, budget);
 973
 974	/* Check if we need a repoll */
 975	if (!tx_complete)
 976		work_done = budget;
 977
 978	if (work_done < budget) {
 979		napi_complete_done(&sds_ring->napi, work_done);
 980		if (test_bit(__QLCNIC_DEV_UP, &adapter->state)) {
 981			qlcnic_enable_sds_intr(adapter, sds_ring);
 982			qlcnic_enable_tx_intr(adapter, tx_ring);
 983		}
 984	}
 985
 986	return work_done;
 987}
 988
 989static int qlcnic_tx_poll(struct napi_struct *napi, int budget)
 990{
 991	struct qlcnic_host_tx_ring *tx_ring;
 992	struct qlcnic_adapter *adapter;
 993	int work_done;
 994
 995	tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi);
 996	adapter = tx_ring->adapter;
 997
 998	work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
 999	if (work_done) {
1000		napi_complete(&tx_ring->napi);
1001		if (test_bit(__QLCNIC_DEV_UP, &adapter->state))
1002			qlcnic_enable_tx_intr(adapter, tx_ring);
1003	} else {
1004		/* As qlcnic_process_cmd_ring() returned 0, we need a repoll*/
1005		work_done = budget;
1006	}
1007
1008	return work_done;
1009}
1010
1011static int qlcnic_rx_poll(struct napi_struct *napi, int budget)
1012{
1013	struct qlcnic_host_sds_ring *sds_ring;
1014	struct qlcnic_adapter *adapter;
1015	int work_done;
1016
1017	sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
1018	adapter = sds_ring->adapter;
1019
1020	work_done = qlcnic_process_rcv_ring(sds_ring, budget);
1021
1022	if (work_done < budget) {
1023		napi_complete_done(&sds_ring->napi, work_done);
1024		if (test_bit(__QLCNIC_DEV_UP, &adapter->state))
1025			qlcnic_enable_sds_intr(adapter, sds_ring);
1026	}
1027
1028	return work_done;
1029}
1030
1031static void qlcnic_handle_linkevent(struct qlcnic_adapter *adapter,
1032				    struct qlcnic_fw_msg *msg)
1033{
1034	u32 cable_OUI;
1035	u16 cable_len, link_speed;
1036	u8  link_status, module, duplex, autoneg, lb_status = 0;
1037	struct net_device *netdev = adapter->netdev;
1038
1039	adapter->ahw->has_link_events = 1;
1040
1041	cable_OUI = msg->body[1] & 0xffffffff;
1042	cable_len = (msg->body[1] >> 32) & 0xffff;
1043	link_speed = (msg->body[1] >> 48) & 0xffff;
1044
1045	link_status = msg->body[2] & 0xff;
1046	duplex = (msg->body[2] >> 16) & 0xff;
1047	autoneg = (msg->body[2] >> 24) & 0xff;
1048	lb_status = (msg->body[2] >> 32) & 0x3;
1049
1050	module = (msg->body[2] >> 8) & 0xff;
1051	if (module == LINKEVENT_MODULE_TWINAX_UNSUPPORTED_CABLE)
1052		dev_info(&netdev->dev,
1053			 "unsupported cable: OUI 0x%x, length %d\n",
1054			 cable_OUI, cable_len);
1055	else if (module == LINKEVENT_MODULE_TWINAX_UNSUPPORTED_CABLELEN)
1056		dev_info(&netdev->dev, "unsupported cable length %d\n",
1057			 cable_len);
1058
1059	if (!link_status && (lb_status == QLCNIC_ILB_MODE ||
1060	    lb_status == QLCNIC_ELB_MODE))
1061		adapter->ahw->loopback_state |= QLCNIC_LINKEVENT;
1062
1063	qlcnic_advert_link_change(adapter, link_status);
1064
1065	if (duplex == LINKEVENT_FULL_DUPLEX)
1066		adapter->ahw->link_duplex = DUPLEX_FULL;
1067	else
1068		adapter->ahw->link_duplex = DUPLEX_HALF;
1069
1070	adapter->ahw->module_type = module;
1071	adapter->ahw->link_autoneg = autoneg;
1072
1073	if (link_status) {
1074		adapter->ahw->link_speed = link_speed;
1075	} else {
1076		adapter->ahw->link_speed = SPEED_UNKNOWN;
1077		adapter->ahw->link_duplex = DUPLEX_UNKNOWN;
1078	}
1079}
1080
1081static void qlcnic_handle_fw_message(int desc_cnt, int index,
1082				     struct qlcnic_host_sds_ring *sds_ring)
1083{
1084	struct qlcnic_fw_msg msg;
1085	struct status_desc *desc;
1086	struct qlcnic_adapter *adapter;
1087	struct device *dev;
1088	int i = 0, opcode, ret;
1089
1090	while (desc_cnt > 0 && i < 8) {
1091		desc = &sds_ring->desc_head[index];
1092		msg.words[i++] = le64_to_cpu(desc->status_desc_data[0]);
1093		msg.words[i++] = le64_to_cpu(desc->status_desc_data[1]);
1094
1095		index = get_next_index(index, sds_ring->num_desc);
1096		desc_cnt--;
1097	}
1098
1099	adapter = sds_ring->adapter;
1100	dev = &adapter->pdev->dev;
1101	opcode = qlcnic_get_nic_msg_opcode(msg.body[0]);
1102
1103	switch (opcode) {
1104	case QLCNIC_C2H_OPCODE_GET_LINKEVENT_RESPONSE:
1105		qlcnic_handle_linkevent(adapter, &msg);
1106		break;
1107	case QLCNIC_C2H_OPCODE_CONFIG_LOOPBACK:
1108		ret = (u32)(msg.body[1]);
1109		switch (ret) {
1110		case 0:
1111			adapter->ahw->loopback_state |= QLCNIC_LB_RESPONSE;
1112			break;
1113		case 1:
1114			dev_info(dev, "loopback already in progress\n");
1115			adapter->ahw->diag_cnt = -EINPROGRESS;
1116			break;
1117		case 2:
1118			dev_info(dev, "loopback cable is not connected\n");
1119			adapter->ahw->diag_cnt = -ENODEV;
1120			break;
1121		default:
1122			dev_info(dev,
1123				 "loopback configure request failed, err %x\n",
1124				 ret);
1125			adapter->ahw->diag_cnt = -EIO;
1126			break;
1127		}
1128		break;
1129	case QLCNIC_C2H_OPCODE_GET_DCB_AEN:
1130		qlcnic_dcb_aen_handler(adapter->dcb, (void *)&msg);
1131		break;
1132	default:
1133		break;
1134	}
1135}
1136
1137static struct sk_buff *qlcnic_process_rxbuf(struct qlcnic_adapter *adapter,
1138					    struct qlcnic_host_rds_ring *ring,
1139					    u16 index, u16 cksum)
1140{
1141	struct qlcnic_rx_buffer *buffer;
1142	struct sk_buff *skb;
1143
1144	buffer = &ring->rx_buf_arr[index];
1145	if (unlikely(buffer->skb == NULL)) {
1146		WARN_ON(1);
1147		return NULL;
1148	}
1149
1150	dma_unmap_single(&adapter->pdev->dev, buffer->dma, ring->dma_size,
1151			 DMA_FROM_DEVICE);
1152
1153	skb = buffer->skb;
1154	if (likely((adapter->netdev->features & NETIF_F_RXCSUM) &&
1155		   (cksum == STATUS_CKSUM_OK || cksum == STATUS_CKSUM_LOOP))) {
1156		adapter->stats.csummed++;
1157		skb->ip_summed = CHECKSUM_UNNECESSARY;
1158	} else {
1159		skb_checksum_none_assert(skb);
1160	}
1161
1162
1163	buffer->skb = NULL;
1164
1165	return skb;
1166}
1167
1168static inline int qlcnic_check_rx_tagging(struct qlcnic_adapter *adapter,
1169					  struct sk_buff *skb, u16 *vlan_tag)
1170{
1171	struct ethhdr *eth_hdr;
1172
1173	if (!__vlan_get_tag(skb, vlan_tag)) {
1174		eth_hdr = (struct ethhdr *)skb->data;
1175		memmove(skb->data + VLAN_HLEN, eth_hdr, ETH_ALEN * 2);
1176		skb_pull(skb, VLAN_HLEN);
1177	}
1178	if (!adapter->rx_pvid)
1179		return 0;
1180
1181	if (*vlan_tag == adapter->rx_pvid) {
1182		/* Outer vlan tag. Packet should follow non-vlan path */
1183		*vlan_tag = 0xffff;
1184		return 0;
1185	}
1186	if (adapter->flags & QLCNIC_TAGGING_ENABLED)
1187		return 0;
1188
1189	return -EINVAL;
1190}
1191
1192static struct qlcnic_rx_buffer *
1193qlcnic_process_rcv(struct qlcnic_adapter *adapter,
1194		   struct qlcnic_host_sds_ring *sds_ring, int ring,
1195		   u64 sts_data0)
1196{
1197	struct net_device *netdev = adapter->netdev;
1198	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1199	struct qlcnic_rx_buffer *buffer;
1200	struct sk_buff *skb;
1201	struct qlcnic_host_rds_ring *rds_ring;
1202	int index, length, cksum, pkt_offset, is_lb_pkt;
1203	u16 vid = 0xffff, t_vid;
1204
1205	if (unlikely(ring >= adapter->max_rds_rings))
1206		return NULL;
1207
1208	rds_ring = &recv_ctx->rds_rings[ring];
1209
1210	index = qlcnic_get_sts_refhandle(sts_data0);
1211	if (unlikely(index >= rds_ring->num_desc))
1212		return NULL;
1213
1214	buffer = &rds_ring->rx_buf_arr[index];
1215	length = qlcnic_get_sts_totallength(sts_data0);
1216	cksum  = qlcnic_get_sts_status(sts_data0);
1217	pkt_offset = qlcnic_get_sts_pkt_offset(sts_data0);
1218
1219	skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum);
1220	if (!skb)
1221		return buffer;
1222
1223	if (adapter->rx_mac_learn) {
1224		t_vid = 0;
1225		is_lb_pkt = qlcnic_82xx_is_lb_pkt(sts_data0);
1226		qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, t_vid);
1227	}
1228
1229	if (length > rds_ring->skb_size)
1230		skb_put(skb, rds_ring->skb_size);
1231	else
1232		skb_put(skb, length);
1233
1234	if (pkt_offset)
1235		skb_pull(skb, pkt_offset);
1236
1237	if (unlikely(qlcnic_check_rx_tagging(adapter, skb, &vid))) {
1238		adapter->stats.rxdropped++;
1239		dev_kfree_skb(skb);
1240		return buffer;
1241	}
1242
1243	skb->protocol = eth_type_trans(skb, netdev);
1244
1245	if (vid != 0xffff)
1246		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1247
1248	napi_gro_receive(&sds_ring->napi, skb);
1249
1250	adapter->stats.rx_pkts++;
1251	adapter->stats.rxbytes += length;
1252
1253	return buffer;
1254}
1255
1256#define QLC_TCP_HDR_SIZE            20
1257#define QLC_TCP_TS_OPTION_SIZE      12
1258#define QLC_TCP_TS_HDR_SIZE         (QLC_TCP_HDR_SIZE + QLC_TCP_TS_OPTION_SIZE)
1259
1260static struct qlcnic_rx_buffer *
1261qlcnic_process_lro(struct qlcnic_adapter *adapter,
1262		   int ring, u64 sts_data0, u64 sts_data1)
1263{
1264	struct net_device *netdev = adapter->netdev;
1265	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1266	struct qlcnic_rx_buffer *buffer;
1267	struct sk_buff *skb;
1268	struct qlcnic_host_rds_ring *rds_ring;
1269	struct iphdr *iph;
1270	struct ipv6hdr *ipv6h;
1271	struct tcphdr *th;
1272	bool push, timestamp;
1273	int index, l2_hdr_offset, l4_hdr_offset, is_lb_pkt;
1274	u16 lro_length, length, data_offset, t_vid, vid = 0xffff;
1275	u32 seq_number;
1276
1277	if (unlikely(ring >= adapter->max_rds_rings))
1278		return NULL;
1279
1280	rds_ring = &recv_ctx->rds_rings[ring];
1281
1282	index = qlcnic_get_lro_sts_refhandle(sts_data0);
1283	if (unlikely(index >= rds_ring->num_desc))
1284		return NULL;
1285
1286	buffer = &rds_ring->rx_buf_arr[index];
1287
1288	timestamp = qlcnic_get_lro_sts_timestamp(sts_data0);
1289	lro_length = qlcnic_get_lro_sts_length(sts_data0);
1290	l2_hdr_offset = qlcnic_get_lro_sts_l2_hdr_offset(sts_data0);
1291	l4_hdr_offset = qlcnic_get_lro_sts_l4_hdr_offset(sts_data0);
1292	push = qlcnic_get_lro_sts_push_flag(sts_data0);
1293	seq_number = qlcnic_get_lro_sts_seq_number(sts_data1);
1294
1295	skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK);
1296	if (!skb)
1297		return buffer;
1298
1299	if (adapter->rx_mac_learn) {
1300		t_vid = 0;
1301		is_lb_pkt = qlcnic_82xx_is_lb_pkt(sts_data0);
1302		qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, t_vid);
1303	}
1304
1305	if (timestamp)
1306		data_offset = l4_hdr_offset + QLC_TCP_TS_HDR_SIZE;
1307	else
1308		data_offset = l4_hdr_offset + QLC_TCP_HDR_SIZE;
1309
1310	skb_put(skb, lro_length + data_offset);
1311	skb_pull(skb, l2_hdr_offset);
1312
1313	if (unlikely(qlcnic_check_rx_tagging(adapter, skb, &vid))) {
1314		adapter->stats.rxdropped++;
1315		dev_kfree_skb(skb);
1316		return buffer;
1317	}
1318
1319	skb->protocol = eth_type_trans(skb, netdev);
1320
1321	if (ntohs(skb->protocol) == ETH_P_IPV6) {
1322		ipv6h = (struct ipv6hdr *)skb->data;
1323		th = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr));
1324		length = (th->doff << 2) + lro_length;
1325		ipv6h->payload_len = htons(length);
1326	} else {
1327		iph = (struct iphdr *)skb->data;
1328		th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
1329		length = (iph->ihl << 2) + (th->doff << 2) + lro_length;
1330		csum_replace2(&iph->check, iph->tot_len, htons(length));
1331		iph->tot_len = htons(length);
1332	}
1333
1334	th->psh = push;
1335	th->seq = htonl(seq_number);
1336	length = skb->len;
1337
1338	if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) {
1339		skb_shinfo(skb)->gso_size = qlcnic_get_lro_sts_mss(sts_data1);
1340		if (skb->protocol == htons(ETH_P_IPV6))
1341			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1342		else
1343			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1344	}
1345
1346	if (vid != 0xffff)
1347		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1348	netif_receive_skb(skb);
1349
1350	adapter->stats.lro_pkts++;
1351	adapter->stats.lrobytes += length;
1352
1353	return buffer;
1354}
1355
1356static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int max)
1357{
1358	struct qlcnic_host_rds_ring *rds_ring;
1359	struct qlcnic_adapter *adapter = sds_ring->adapter;
1360	struct list_head *cur;
1361	struct status_desc *desc;
1362	struct qlcnic_rx_buffer *rxbuf;
1363	int opcode, desc_cnt, count = 0;
1364	u64 sts_data0, sts_data1;
1365	u8 ring;
1366	u32 consumer = sds_ring->consumer;
1367
1368	while (count < max) {
1369		desc = &sds_ring->desc_head[consumer];
1370		sts_data0 = le64_to_cpu(desc->status_desc_data[0]);
1371
1372		if (!(sts_data0 & STATUS_OWNER_HOST))
1373			break;
1374
1375		desc_cnt = qlcnic_get_sts_desc_cnt(sts_data0);
1376		opcode = qlcnic_get_sts_opcode(sts_data0);
1377		switch (opcode) {
1378		case QLCNIC_RXPKT_DESC:
1379		case QLCNIC_OLD_RXPKT_DESC:
1380		case QLCNIC_SYN_OFFLOAD:
1381			ring = qlcnic_get_sts_type(sts_data0);
1382			rxbuf = qlcnic_process_rcv(adapter, sds_ring, ring,
1383						   sts_data0);
1384			break;
1385		case QLCNIC_LRO_DESC:
1386			ring = qlcnic_get_lro_sts_type(sts_data0);
1387			sts_data1 = le64_to_cpu(desc->status_desc_data[1]);
1388			rxbuf = qlcnic_process_lro(adapter, ring, sts_data0,
1389						   sts_data1);
1390			break;
1391		case QLCNIC_RESPONSE_DESC:
1392			qlcnic_handle_fw_message(desc_cnt, consumer, sds_ring);
1393			goto skip;
1394		default:
1395			goto skip;
1396		}
1397		WARN_ON(desc_cnt > 1);
1398
1399		if (likely(rxbuf))
1400			list_add_tail(&rxbuf->list, &sds_ring->free_list[ring]);
1401		else
1402			adapter->stats.null_rxbuf++;
1403skip:
1404		for (; desc_cnt > 0; desc_cnt--) {
1405			desc = &sds_ring->desc_head[consumer];
1406			desc->status_desc_data[0] = QLCNIC_DESC_OWNER_FW;
1407			consumer = get_next_index(consumer, sds_ring->num_desc);
1408		}
1409		count++;
1410	}
1411
1412	for (ring = 0; ring < adapter->max_rds_rings; ring++) {
1413		rds_ring = &adapter->recv_ctx->rds_rings[ring];
1414		if (!list_empty(&sds_ring->free_list[ring])) {
1415			list_for_each(cur, &sds_ring->free_list[ring]) {
1416				rxbuf = list_entry(cur, struct qlcnic_rx_buffer,
1417						   list);
1418				qlcnic_alloc_rx_skb(adapter, rds_ring, rxbuf);
1419			}
1420			spin_lock(&rds_ring->lock);
1421			list_splice_tail_init(&sds_ring->free_list[ring],
1422					      &rds_ring->free_list);
1423			spin_unlock(&rds_ring->lock);
1424		}
1425
1426		qlcnic_post_rx_buffers_nodb(adapter, rds_ring, ring);
1427	}
1428
1429	if (count) {
1430		sds_ring->consumer = consumer;
1431		writel(consumer, sds_ring->crb_sts_consumer);
1432	}
1433
1434	return count;
1435}
1436
1437void qlcnic_post_rx_buffers(struct qlcnic_adapter *adapter,
1438			    struct qlcnic_host_rds_ring *rds_ring, u8 ring_id)
1439{
1440	struct rcv_desc *pdesc;
1441	struct qlcnic_rx_buffer *buffer;
1442	int count = 0;
1443	u32 producer, handle;
1444	struct list_head *head;
1445
1446	producer = rds_ring->producer;
1447	head = &rds_ring->free_list;
1448
1449	while (!list_empty(head)) {
1450
1451		buffer = list_entry(head->next, struct qlcnic_rx_buffer, list);
1452
1453		if (!buffer->skb) {
1454			if (qlcnic_alloc_rx_skb(adapter, rds_ring, buffer))
1455				break;
1456		}
1457
1458		count++;
1459		list_del(&buffer->list);
1460
1461		/* make a rcv descriptor  */
1462		pdesc = &rds_ring->desc_head[producer];
1463		pdesc->addr_buffer = cpu_to_le64(buffer->dma);
1464		handle = qlcnic_get_ref_handle(adapter, buffer->ref_handle,
1465					       ring_id);
1466		pdesc->reference_handle = cpu_to_le16(handle);
1467		pdesc->buffer_length = cpu_to_le32(rds_ring->dma_size);
1468		producer = get_next_index(producer, rds_ring->num_desc);
1469	}
1470
1471	if (count) {
1472		rds_ring->producer = producer;
1473		writel((producer-1) & (rds_ring->num_desc-1),
1474		       rds_ring->crb_rcv_producer);
1475	}
1476}
1477
1478static void dump_skb(struct sk_buff *skb, struct qlcnic_adapter *adapter)
1479{
1480	if (adapter->ahw->msg_enable & NETIF_MSG_DRV) {
1481		char prefix[30];
1482
1483		scnprintf(prefix, sizeof(prefix), "%s: %s: ",
1484			  dev_name(&adapter->pdev->dev), __func__);
1485
1486		print_hex_dump_debug(prefix, DUMP_PREFIX_NONE, 16, 1,
1487				     skb->data, skb->len, true);
1488	}
1489}
1490
1491static void qlcnic_process_rcv_diag(struct qlcnic_adapter *adapter, int ring,
1492				    u64 sts_data0)
1493{
1494	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1495	struct sk_buff *skb;
1496	struct qlcnic_host_rds_ring *rds_ring;
1497	int index, length, cksum, pkt_offset;
1498
1499	if (unlikely(ring >= adapter->max_rds_rings))
1500		return;
1501
1502	rds_ring = &recv_ctx->rds_rings[ring];
1503
1504	index = qlcnic_get_sts_refhandle(sts_data0);
1505	length = qlcnic_get_sts_totallength(sts_data0);
1506	if (unlikely(index >= rds_ring->num_desc))
1507		return;
1508
1509	cksum  = qlcnic_get_sts_status(sts_data0);
1510	pkt_offset = qlcnic_get_sts_pkt_offset(sts_data0);
1511
1512	skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum);
1513	if (!skb)
1514		return;
1515
1516	if (length > rds_ring->skb_size)
1517		skb_put(skb, rds_ring->skb_size);
1518	else
1519		skb_put(skb, length);
1520
1521	if (pkt_offset)
1522		skb_pull(skb, pkt_offset);
1523
1524	if (!qlcnic_check_loopback_buff(skb->data, adapter->mac_addr))
1525		adapter->ahw->diag_cnt++;
1526	else
1527		dump_skb(skb, adapter);
1528
1529	dev_kfree_skb_any(skb);
1530	adapter->stats.rx_pkts++;
1531	adapter->stats.rxbytes += length;
1532
1533	return;
1534}
1535
1536void qlcnic_82xx_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring)
1537{
1538	struct qlcnic_adapter *adapter = sds_ring->adapter;
1539	struct status_desc *desc;
1540	u64 sts_data0;
1541	int ring, opcode, desc_cnt;
1542
1543	u32 consumer = sds_ring->consumer;
1544
1545	desc = &sds_ring->desc_head[consumer];
1546	sts_data0 = le64_to_cpu(desc->status_desc_data[0]);
1547
1548	if (!(sts_data0 & STATUS_OWNER_HOST))
1549		return;
1550
1551	desc_cnt = qlcnic_get_sts_desc_cnt(sts_data0);
1552	opcode = qlcnic_get_sts_opcode(sts_data0);
1553	switch (opcode) {
1554	case QLCNIC_RESPONSE_DESC:
1555		qlcnic_handle_fw_message(desc_cnt, consumer, sds_ring);
1556		break;
1557	default:
1558		ring = qlcnic_get_sts_type(sts_data0);
1559		qlcnic_process_rcv_diag(adapter, ring, sts_data0);
1560		break;
1561	}
1562
1563	for (; desc_cnt > 0; desc_cnt--) {
1564		desc = &sds_ring->desc_head[consumer];
1565		desc->status_desc_data[0] = cpu_to_le64(STATUS_OWNER_PHANTOM);
1566		consumer = get_next_index(consumer, sds_ring->num_desc);
1567	}
1568
1569	sds_ring->consumer = consumer;
1570	writel(consumer, sds_ring->crb_sts_consumer);
1571}
1572
1573int qlcnic_82xx_napi_add(struct qlcnic_adapter *adapter,
1574			 struct net_device *netdev)
1575{
1576	int ring;
1577	struct qlcnic_host_sds_ring *sds_ring;
1578	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1579	struct qlcnic_host_tx_ring *tx_ring;
1580
1581	if (qlcnic_alloc_sds_rings(recv_ctx, adapter->drv_sds_rings))
1582		return -ENOMEM;
1583
1584	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1585		sds_ring = &recv_ctx->sds_rings[ring];
1586		if (qlcnic_check_multi_tx(adapter) &&
1587		    !adapter->ahw->diag_test) {
1588			netif_napi_add(netdev, &sds_ring->napi,
1589				       qlcnic_rx_poll);
1590		} else {
1591			if (ring == (adapter->drv_sds_rings - 1))
1592				netif_napi_add(netdev, &sds_ring->napi,
1593					       qlcnic_poll);
1594			else
1595				netif_napi_add(netdev, &sds_ring->napi,
1596					       qlcnic_rx_poll);
1597		}
1598	}
1599
1600	if (qlcnic_alloc_tx_rings(adapter, netdev)) {
1601		qlcnic_free_sds_rings(recv_ctx);
1602		return -ENOMEM;
1603	}
1604
1605	if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) {
1606		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1607			tx_ring = &adapter->tx_ring[ring];
1608			netif_napi_add_tx(netdev, &tx_ring->napi,
1609					  qlcnic_tx_poll);
1610		}
1611	}
1612
1613	return 0;
1614}
1615
1616void qlcnic_82xx_napi_del(struct qlcnic_adapter *adapter)
1617{
1618	int ring;
1619	struct qlcnic_host_sds_ring *sds_ring;
1620	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1621	struct qlcnic_host_tx_ring *tx_ring;
1622
1623	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1624		sds_ring = &recv_ctx->sds_rings[ring];
1625		netif_napi_del(&sds_ring->napi);
1626	}
1627
1628	qlcnic_free_sds_rings(adapter->recv_ctx);
1629
1630	if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) {
1631		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1632			tx_ring = &adapter->tx_ring[ring];
1633			netif_napi_del(&tx_ring->napi);
1634		}
1635	}
1636
1637	qlcnic_free_tx_rings(adapter);
1638}
1639
1640void qlcnic_82xx_napi_enable(struct qlcnic_adapter *adapter)
1641{
1642	int ring;
1643	struct qlcnic_host_sds_ring *sds_ring;
1644	struct qlcnic_host_tx_ring *tx_ring;
1645	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1646
1647	if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
1648		return;
1649
1650	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1651		sds_ring = &recv_ctx->sds_rings[ring];
1652		napi_enable(&sds_ring->napi);
1653		qlcnic_enable_sds_intr(adapter, sds_ring);
1654	}
1655
1656	if (qlcnic_check_multi_tx(adapter) &&
1657	    (adapter->flags & QLCNIC_MSIX_ENABLED) &&
1658	    !adapter->ahw->diag_test) {
1659		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1660			tx_ring = &adapter->tx_ring[ring];
1661			napi_enable(&tx_ring->napi);
1662			qlcnic_enable_tx_intr(adapter, tx_ring);
1663		}
1664	}
1665}
1666
1667void qlcnic_82xx_napi_disable(struct qlcnic_adapter *adapter)
1668{
1669	int ring;
1670	struct qlcnic_host_sds_ring *sds_ring;
1671	struct qlcnic_host_tx_ring *tx_ring;
1672	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1673
1674	if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
1675		return;
1676
1677	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1678		sds_ring = &recv_ctx->sds_rings[ring];
1679		qlcnic_disable_sds_intr(adapter, sds_ring);
1680		napi_synchronize(&sds_ring->napi);
1681		napi_disable(&sds_ring->napi);
1682	}
1683
1684	if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
1685	    !adapter->ahw->diag_test &&
1686	    qlcnic_check_multi_tx(adapter)) {
1687		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1688			tx_ring = &adapter->tx_ring[ring];
1689			qlcnic_disable_tx_intr(adapter, tx_ring);
1690			napi_synchronize(&tx_ring->napi);
1691			napi_disable(&tx_ring->napi);
1692		}
1693	}
1694}
1695
1696#define QLC_83XX_NORMAL_LB_PKT	(1ULL << 36)
1697#define QLC_83XX_LRO_LB_PKT	(1ULL << 46)
1698
1699static inline int qlcnic_83xx_is_lb_pkt(u64 sts_data, int lro_pkt)
1700{
1701	if (lro_pkt)
1702		return (sts_data & QLC_83XX_LRO_LB_PKT) ? 1 : 0;
1703	else
1704		return (sts_data & QLC_83XX_NORMAL_LB_PKT) ? 1 : 0;
1705}
1706
1707#define QLCNIC_ENCAP_LENGTH_MASK	0x7f
1708
1709static inline u8 qlcnic_encap_length(u64 sts_data)
1710{
1711	return sts_data & QLCNIC_ENCAP_LENGTH_MASK;
1712}
1713
1714static struct qlcnic_rx_buffer *
1715qlcnic_83xx_process_rcv(struct qlcnic_adapter *adapter,
1716			struct qlcnic_host_sds_ring *sds_ring,
1717			u8 ring, u64 sts_data[])
1718{
1719	struct net_device *netdev = adapter->netdev;
1720	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1721	struct qlcnic_rx_buffer *buffer;
1722	struct sk_buff *skb;
1723	struct qlcnic_host_rds_ring *rds_ring;
1724	int index, length, cksum, is_lb_pkt;
1725	u16 vid = 0xffff;
1726	int err;
1727
1728	if (unlikely(ring >= adapter->max_rds_rings))
1729		return NULL;
1730
1731	rds_ring = &recv_ctx->rds_rings[ring];
1732
1733	index = qlcnic_83xx_hndl(sts_data[0]);
1734	if (unlikely(index >= rds_ring->num_desc))
1735		return NULL;
1736
1737	buffer = &rds_ring->rx_buf_arr[index];
1738	length = qlcnic_83xx_pktln(sts_data[0]);
1739	cksum  = qlcnic_83xx_csum_status(sts_data[1]);
1740	skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum);
1741	if (!skb)
1742		return buffer;
1743
1744	if (length > rds_ring->skb_size)
1745		skb_put(skb, rds_ring->skb_size);
1746	else
1747		skb_put(skb, length);
1748
1749	err = qlcnic_check_rx_tagging(adapter, skb, &vid);
1750
1751	if (adapter->rx_mac_learn) {
1752		is_lb_pkt = qlcnic_83xx_is_lb_pkt(sts_data[1], 0);
1753		qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, vid);
1754	}
1755
1756	if (unlikely(err)) {
1757		adapter->stats.rxdropped++;
1758		dev_kfree_skb(skb);
1759		return buffer;
1760	}
1761
1762	skb->protocol = eth_type_trans(skb, netdev);
1763
1764	if (qlcnic_encap_length(sts_data[1]) &&
1765	    skb->ip_summed == CHECKSUM_UNNECESSARY) {
1766		skb->csum_level = 1;
1767		adapter->stats.encap_rx_csummed++;
1768	}
1769
1770	if (vid != 0xffff)
1771		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1772
1773	napi_gro_receive(&sds_ring->napi, skb);
1774
1775	adapter->stats.rx_pkts++;
1776	adapter->stats.rxbytes += length;
1777
1778	return buffer;
1779}
1780
1781static struct qlcnic_rx_buffer *
1782qlcnic_83xx_process_lro(struct qlcnic_adapter *adapter,
1783			u8 ring, u64 sts_data[])
1784{
1785	struct net_device *netdev = adapter->netdev;
1786	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1787	struct qlcnic_rx_buffer *buffer;
1788	struct sk_buff *skb;
1789	struct qlcnic_host_rds_ring *rds_ring;
1790	struct iphdr *iph;
1791	struct ipv6hdr *ipv6h;
1792	struct tcphdr *th;
1793	bool push;
1794	int l2_hdr_offset, l4_hdr_offset;
1795	int index, is_lb_pkt;
1796	u16 lro_length, length, data_offset, gso_size;
1797	u16 vid = 0xffff;
1798	int err;
1799
1800	if (unlikely(ring >= adapter->max_rds_rings))
1801		return NULL;
1802
1803	rds_ring = &recv_ctx->rds_rings[ring];
1804
1805	index = qlcnic_83xx_hndl(sts_data[0]);
1806	if (unlikely(index >= rds_ring->num_desc))
1807		return NULL;
1808
1809	buffer = &rds_ring->rx_buf_arr[index];
1810
1811	lro_length = qlcnic_83xx_lro_pktln(sts_data[0]);
1812	l2_hdr_offset = qlcnic_83xx_l2_hdr_off(sts_data[1]);
1813	l4_hdr_offset = qlcnic_83xx_l4_hdr_off(sts_data[1]);
1814	push = qlcnic_83xx_is_psh_bit(sts_data[1]);
1815
1816	skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK);
1817	if (!skb)
1818		return buffer;
1819
1820	if (qlcnic_83xx_is_tstamp(sts_data[1]))
1821		data_offset = l4_hdr_offset + QLCNIC_TCP_TS_HDR_SIZE;
1822	else
1823		data_offset = l4_hdr_offset + QLCNIC_TCP_HDR_SIZE;
1824
1825	skb_put(skb, lro_length + data_offset);
1826	skb_pull(skb, l2_hdr_offset);
1827
1828	err = qlcnic_check_rx_tagging(adapter, skb, &vid);
1829
1830	if (adapter->rx_mac_learn) {
1831		is_lb_pkt = qlcnic_83xx_is_lb_pkt(sts_data[1], 1);
1832		qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, vid);
1833	}
1834
1835	if (unlikely(err)) {
1836		adapter->stats.rxdropped++;
1837		dev_kfree_skb(skb);
1838		return buffer;
1839	}
1840
1841	skb->protocol = eth_type_trans(skb, netdev);
1842	if (ntohs(skb->protocol) == ETH_P_IPV6) {
1843		ipv6h = (struct ipv6hdr *)skb->data;
1844		th = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr));
1845
1846		length = (th->doff << 2) + lro_length;
1847		ipv6h->payload_len = htons(length);
1848	} else {
1849		iph = (struct iphdr *)skb->data;
1850		th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
1851		length = (iph->ihl << 2) + (th->doff << 2) + lro_length;
1852		csum_replace2(&iph->check, iph->tot_len, htons(length));
1853		iph->tot_len = htons(length);
1854	}
1855
1856	th->psh = push;
1857	length = skb->len;
1858
1859	if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) {
1860		gso_size = qlcnic_83xx_get_lro_sts_mss(sts_data[0]);
1861		skb_shinfo(skb)->gso_size = gso_size;
1862		if (skb->protocol == htons(ETH_P_IPV6))
1863			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1864		else
1865			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1866	}
1867
1868	if (vid != 0xffff)
1869		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1870
1871	netif_receive_skb(skb);
1872
1873	adapter->stats.lro_pkts++;
1874	adapter->stats.lrobytes += length;
1875	return buffer;
1876}
1877
1878static int qlcnic_83xx_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring,
1879					int max)
1880{
1881	struct qlcnic_host_rds_ring *rds_ring;
1882	struct qlcnic_adapter *adapter = sds_ring->adapter;
1883	struct list_head *cur;
1884	struct status_desc *desc;
1885	struct qlcnic_rx_buffer *rxbuf = NULL;
1886	u8 ring;
1887	u64 sts_data[2];
1888	int count = 0, opcode;
1889	u32 consumer = sds_ring->consumer;
1890
1891	while (count < max) {
1892		desc = &sds_ring->desc_head[consumer];
1893		sts_data[1] = le64_to_cpu(desc->status_desc_data[1]);
1894		opcode = qlcnic_83xx_opcode(sts_data[1]);
1895		if (!opcode)
1896			break;
1897		sts_data[0] = le64_to_cpu(desc->status_desc_data[0]);
1898		ring = QLCNIC_FETCH_RING_ID(sts_data[0]);
1899
1900		switch (opcode) {
1901		case QLC_83XX_REG_DESC:
1902			rxbuf = qlcnic_83xx_process_rcv(adapter, sds_ring,
1903							ring, sts_data);
1904			break;
1905		case QLC_83XX_LRO_DESC:
1906			rxbuf = qlcnic_83xx_process_lro(adapter, ring,
1907							sts_data);
1908			break;
1909		default:
1910			dev_info(&adapter->pdev->dev,
1911				 "Unknown opcode: 0x%x\n", opcode);
1912			goto skip;
1913		}
1914
1915		if (likely(rxbuf))
1916			list_add_tail(&rxbuf->list, &sds_ring->free_list[ring]);
1917		else
1918			adapter->stats.null_rxbuf++;
1919skip:
1920		desc = &sds_ring->desc_head[consumer];
1921		/* Reset the descriptor */
1922		desc->status_desc_data[1] = 0;
1923		consumer = get_next_index(consumer, sds_ring->num_desc);
1924		count++;
1925	}
1926	for (ring = 0; ring < adapter->max_rds_rings; ring++) {
1927		rds_ring = &adapter->recv_ctx->rds_rings[ring];
1928		if (!list_empty(&sds_ring->free_list[ring])) {
1929			list_for_each(cur, &sds_ring->free_list[ring]) {
1930				rxbuf = list_entry(cur, struct qlcnic_rx_buffer,
1931						   list);
1932				qlcnic_alloc_rx_skb(adapter, rds_ring, rxbuf);
1933			}
1934			spin_lock(&rds_ring->lock);
1935			list_splice_tail_init(&sds_ring->free_list[ring],
1936					      &rds_ring->free_list);
1937			spin_unlock(&rds_ring->lock);
1938		}
1939		qlcnic_post_rx_buffers_nodb(adapter, rds_ring, ring);
1940	}
1941	if (count) {
1942		sds_ring->consumer = consumer;
1943		writel(consumer, sds_ring->crb_sts_consumer);
1944	}
1945	return count;
1946}
1947
1948static int qlcnic_83xx_msix_sriov_vf_poll(struct napi_struct *napi, int budget)
1949{
1950	int tx_complete;
1951	int work_done;
1952	struct qlcnic_host_sds_ring *sds_ring;
1953	struct qlcnic_adapter *adapter;
1954	struct qlcnic_host_tx_ring *tx_ring;
1955
1956	sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
1957	adapter = sds_ring->adapter;
1958	/* tx ring count = 1 */
1959	tx_ring = adapter->tx_ring;
1960
1961	tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
1962	work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget);
1963
1964	/* Check if we need a repoll */
1965	if (!tx_complete)
1966		work_done = budget;
1967
1968	if (work_done < budget) {
1969		napi_complete_done(&sds_ring->napi, work_done);
1970		qlcnic_enable_sds_intr(adapter, sds_ring);
1971	}
1972
1973	return work_done;
1974}
1975
1976static int qlcnic_83xx_poll(struct napi_struct *napi, int budget)
1977{
1978	int tx_complete;
1979	int work_done;
1980	struct qlcnic_host_sds_ring *sds_ring;
1981	struct qlcnic_adapter *adapter;
1982	struct qlcnic_host_tx_ring *tx_ring;
1983
1984	sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
1985	adapter = sds_ring->adapter;
1986	/* tx ring count = 1 */
1987	tx_ring = adapter->tx_ring;
1988
1989	tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
1990	work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget);
1991
1992	/* Check if we need a repoll */
1993	if (!tx_complete)
1994		work_done = budget;
1995
1996	if (work_done < budget) {
1997		napi_complete_done(&sds_ring->napi, work_done);
1998		qlcnic_enable_sds_intr(adapter, sds_ring);
1999	}
2000
2001	return work_done;
2002}
2003
2004static int qlcnic_83xx_msix_tx_poll(struct napi_struct *napi, int budget)
2005{
2006	int work_done;
2007	struct qlcnic_host_tx_ring *tx_ring;
2008	struct qlcnic_adapter *adapter;
2009
2010	tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi);
2011	adapter = tx_ring->adapter;
2012	work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
2013	if (work_done) {
2014		napi_complete(&tx_ring->napi);
2015		if (test_bit(__QLCNIC_DEV_UP , &adapter->state))
2016			qlcnic_enable_tx_intr(adapter, tx_ring);
2017	} else {
2018		/* need a repoll */
2019		work_done = budget;
2020	}
2021
2022	return work_done;
2023}
2024
2025static int qlcnic_83xx_rx_poll(struct napi_struct *napi, int budget)
2026{
2027	int work_done;
2028	struct qlcnic_host_sds_ring *sds_ring;
2029	struct qlcnic_adapter *adapter;
2030
2031	sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
2032	adapter = sds_ring->adapter;
2033	work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget);
2034	if (work_done < budget) {
2035		napi_complete_done(&sds_ring->napi, work_done);
2036		if (test_bit(__QLCNIC_DEV_UP, &adapter->state))
2037			qlcnic_enable_sds_intr(adapter, sds_ring);
2038	}
2039
2040	return work_done;
2041}
2042
2043void qlcnic_83xx_napi_enable(struct qlcnic_adapter *adapter)
2044{
2045	int ring;
2046	struct qlcnic_host_sds_ring *sds_ring;
2047	struct qlcnic_host_tx_ring *tx_ring;
2048	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2049
2050	if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
2051		return;
2052
2053	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2054		sds_ring = &recv_ctx->sds_rings[ring];
2055		napi_enable(&sds_ring->napi);
2056		if (adapter->flags & QLCNIC_MSIX_ENABLED)
2057			qlcnic_enable_sds_intr(adapter, sds_ring);
2058	}
2059
2060	if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2061	    !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2062		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2063			tx_ring = &adapter->tx_ring[ring];
2064			napi_enable(&tx_ring->napi);
2065			qlcnic_enable_tx_intr(adapter, tx_ring);
2066		}
2067	}
2068}
2069
2070void qlcnic_83xx_napi_disable(struct qlcnic_adapter *adapter)
2071{
2072	int ring;
2073	struct qlcnic_host_sds_ring *sds_ring;
2074	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2075	struct qlcnic_host_tx_ring *tx_ring;
2076
2077	if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
2078		return;
2079
2080	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2081		sds_ring = &recv_ctx->sds_rings[ring];
2082		if (adapter->flags & QLCNIC_MSIX_ENABLED)
2083			qlcnic_disable_sds_intr(adapter, sds_ring);
2084		napi_synchronize(&sds_ring->napi);
2085		napi_disable(&sds_ring->napi);
2086	}
2087
2088	if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2089	    !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2090		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2091			tx_ring = &adapter->tx_ring[ring];
2092			qlcnic_disable_tx_intr(adapter, tx_ring);
2093			napi_synchronize(&tx_ring->napi);
2094			napi_disable(&tx_ring->napi);
2095		}
2096	}
2097}
2098
2099int qlcnic_83xx_napi_add(struct qlcnic_adapter *adapter,
2100			 struct net_device *netdev)
2101{
2102	int ring;
2103	struct qlcnic_host_sds_ring *sds_ring;
2104	struct qlcnic_host_tx_ring *tx_ring;
2105	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2106
2107	if (qlcnic_alloc_sds_rings(recv_ctx, adapter->drv_sds_rings))
2108		return -ENOMEM;
2109
2110	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2111		sds_ring = &recv_ctx->sds_rings[ring];
2112		if (adapter->flags & QLCNIC_MSIX_ENABLED) {
2113			if (!(adapter->flags & QLCNIC_TX_INTR_SHARED))
2114				netif_napi_add(netdev, &sds_ring->napi,
2115					       qlcnic_83xx_rx_poll);
2116			else
2117				netif_napi_add(netdev, &sds_ring->napi,
2118					       qlcnic_83xx_msix_sriov_vf_poll);
2119
2120		} else {
2121			netif_napi_add(netdev, &sds_ring->napi,
2122				       qlcnic_83xx_poll);
2123		}
2124	}
2125
2126	if (qlcnic_alloc_tx_rings(adapter, netdev)) {
2127		qlcnic_free_sds_rings(recv_ctx);
2128		return -ENOMEM;
2129	}
2130
2131	if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2132	    !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2133		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2134			tx_ring = &adapter->tx_ring[ring];
2135			netif_napi_add_tx(netdev, &tx_ring->napi,
2136					  qlcnic_83xx_msix_tx_poll);
2137		}
2138	}
2139
2140	return 0;
2141}
2142
2143void qlcnic_83xx_napi_del(struct qlcnic_adapter *adapter)
2144{
2145	int ring;
2146	struct qlcnic_host_sds_ring *sds_ring;
2147	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2148	struct qlcnic_host_tx_ring *tx_ring;
2149
2150	for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2151		sds_ring = &recv_ctx->sds_rings[ring];
2152		netif_napi_del(&sds_ring->napi);
2153	}
2154
2155	qlcnic_free_sds_rings(adapter->recv_ctx);
2156
2157	if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2158	    !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2159		for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2160			tx_ring = &adapter->tx_ring[ring];
2161			netif_napi_del(&tx_ring->napi);
2162		}
2163	}
2164
2165	qlcnic_free_tx_rings(adapter);
2166}
2167
2168static void qlcnic_83xx_process_rcv_diag(struct qlcnic_adapter *adapter,
2169					 int ring, u64 sts_data[])
2170{
2171	struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2172	struct sk_buff *skb;
2173	struct qlcnic_host_rds_ring *rds_ring;
2174	int index, length;
2175
2176	if (unlikely(ring >= adapter->max_rds_rings))
2177		return;
2178
2179	rds_ring = &recv_ctx->rds_rings[ring];
2180	index = qlcnic_83xx_hndl(sts_data[0]);
2181	if (unlikely(index >= rds_ring->num_desc))
2182		return;
2183
2184	length = qlcnic_83xx_pktln(sts_data[0]);
2185
2186	skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK);
2187	if (!skb)
2188		return;
2189
2190	if (length > rds_ring->skb_size)
2191		skb_put(skb, rds_ring->skb_size);
2192	else
2193		skb_put(skb, length);
2194
2195	if (!qlcnic_check_loopback_buff(skb->data, adapter->mac_addr))
2196		adapter->ahw->diag_cnt++;
2197	else
2198		dump_skb(skb, adapter);
2199
2200	dev_kfree_skb_any(skb);
2201	return;
2202}
2203
2204void qlcnic_83xx_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring)
2205{
2206	struct qlcnic_adapter *adapter = sds_ring->adapter;
2207	struct status_desc *desc;
2208	u64 sts_data[2];
2209	int ring, opcode;
2210	u32 consumer = sds_ring->consumer;
2211
2212	desc = &sds_ring->desc_head[consumer];
2213	sts_data[0] = le64_to_cpu(desc->status_desc_data[0]);
2214	sts_data[1] = le64_to_cpu(desc->status_desc_data[1]);
2215	opcode = qlcnic_83xx_opcode(sts_data[1]);
2216	if (!opcode)
2217		return;
2218
2219	ring = QLCNIC_FETCH_RING_ID(sts_data[0]);
2220	qlcnic_83xx_process_rcv_diag(adapter, ring, sts_data);
2221	desc = &sds_ring->desc_head[consumer];
2222	desc->status_desc_data[0] = cpu_to_le64(STATUS_OWNER_PHANTOM);
2223	consumer = get_next_index(consumer, sds_ring->num_desc);
2224	sds_ring->consumer = consumer;
2225	writel(consumer, sds_ring->crb_sts_consumer);
2226}