nicvf_main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2015 Cavium, Inc.
   4 */
   5
   6#include <linux/module.h>
   7#include <linux/interrupt.h>
   8#include <linux/pci.h>
   9#include <linux/netdevice.h>
  10#include <linux/if_vlan.h>
  11#include <linux/etherdevice.h>
  12#include <linux/ethtool.h>
  13#include <linux/log2.h>
  14#include <linux/prefetch.h>
  15#include <linux/irq.h>
  16#include <linux/iommu.h>
  17#include <linux/bpf.h>
  18#include <linux/bpf_trace.h>
  19#include <linux/filter.h>
  20#include <linux/net_tstamp.h>
  21#include <linux/workqueue.h>
  22
  23#include "nic_reg.h"
  24#include "nic.h"
  25#include "nicvf_queues.h"
  26#include "thunder_bgx.h"
  27#include "../common/cavium_ptp.h"
  28
  29#define DRV_NAME	"nicvf"
  30#define DRV_VERSION	"1.0"
  31
  32/* NOTE: Packets bigger than 1530 are split across multiple pages and XDP needs
  33 * the buffer to be contiguous. Allow XDP to be set up only if we don't exceed
  34 * this value, keeping headroom for the 14 byte Ethernet header and two
  35 * VLAN tags (for QinQ)
  36 */
  37#define MAX_XDP_MTU	(1530 - ETH_HLEN - VLAN_HLEN * 2)
  38
  39/* Supported devices */
  40static const struct pci_device_id nicvf_id_table[] = {
  41	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
  42			 PCI_DEVICE_ID_THUNDER_NIC_VF,
  43			 PCI_VENDOR_ID_CAVIUM,
  44			 PCI_SUBSYS_DEVID_88XX_NIC_VF) },
  45	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
  46			 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
  47			 PCI_VENDOR_ID_CAVIUM,
  48			 PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF) },
  49	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
  50			 PCI_DEVICE_ID_THUNDER_NIC_VF,
  51			 PCI_VENDOR_ID_CAVIUM,
  52			 PCI_SUBSYS_DEVID_81XX_NIC_VF) },
  53	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
  54			 PCI_DEVICE_ID_THUNDER_NIC_VF,
  55			 PCI_VENDOR_ID_CAVIUM,
  56			 PCI_SUBSYS_DEVID_83XX_NIC_VF) },
  57	{ 0, }  /* end of table */
  58};
  59
  60MODULE_AUTHOR("Sunil Goutham");
  61MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver");
  62MODULE_LICENSE("GPL v2");
  63MODULE_VERSION(DRV_VERSION);
  64MODULE_DEVICE_TABLE(pci, nicvf_id_table);
  65
  66static int debug = 0x00;
  67module_param(debug, int, 0644);
  68MODULE_PARM_DESC(debug, "Debug message level bitmap");
  69
  70static int cpi_alg = CPI_ALG_NONE;
  71module_param(cpi_alg, int, 0444);
  72MODULE_PARM_DESC(cpi_alg,
  73		 "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)");
  74
  75static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx)
  76{
  77	if (nic->sqs_mode)
  78		return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS);
  79	else
  80		return qidx;
  81}
  82
  83/* The Cavium ThunderX network controller can *only* be found in SoCs
  84 * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
  85 * registers on this platform are implicitly strongly ordered with respect
  86 * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
  87 * with no memory barriers in this driver.  The readq()/writeq() functions add
  88 * explicit ordering operation which in this case are redundant, and only
  89 * add overhead.
  90 */
  91
  92/* Register read/write APIs */
  93void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val)
  94{
  95	writeq_relaxed(val, nic->reg_base + offset);
  96}
  97
  98u64 nicvf_reg_read(struct nicvf *nic, u64 offset)
  99{
 100	return readq_relaxed(nic->reg_base + offset);
 101}
 102
 103void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
 104			   u64 qidx, u64 val)
 105{
 106	void __iomem *addr = nic->reg_base + offset;
 107
 108	writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT));
 109}
 110
 111u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx)
 112{
 113	void __iomem *addr = nic->reg_base + offset;
 114
 115	return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT));
 116}
 117
 118/* VF -> PF mailbox communication */
 119static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
 120{
 121	u64 *msg = (u64 *)mbx;
 122
 123	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
 124	nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
 125}
 126
 127int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
 128{
 129	unsigned long timeout;
 130	int ret = 0;
 131
 132	mutex_lock(&nic->rx_mode_mtx);
 133
 134	nic->pf_acked = false;
 135	nic->pf_nacked = false;
 136
 137	nicvf_write_to_mbx(nic, mbx);
 138
 139	timeout = jiffies + msecs_to_jiffies(NIC_MBOX_MSG_TIMEOUT);
 140	/* Wait for previous message to be acked, timeout 2sec */
 141	while (!nic->pf_acked) {
 142		if (nic->pf_nacked) {
 143			netdev_err(nic->netdev,
 144				   "PF NACK to mbox msg 0x%02x from VF%d\n",
 145				   (mbx->msg.msg & 0xFF), nic->vf_id);
 146			ret = -EINVAL;
 147			break;
 148		}
 149		usleep_range(8000, 10000);
 150		if (nic->pf_acked)
 151			break;
 152		if (time_after(jiffies, timeout)) {
 153			netdev_err(nic->netdev,
 154				   "PF didn't ACK to mbox msg 0x%02x from VF%d\n",
 155				   (mbx->msg.msg & 0xFF), nic->vf_id);
 156			ret = -EBUSY;
 157			break;
 158		}
 159	}
 160	mutex_unlock(&nic->rx_mode_mtx);
 161	return ret;
 162}
 163
 164/* Checks if VF is able to comminicate with PF
 165* and also gets the VNIC number this VF is associated to.
 166*/
 167static int nicvf_check_pf_ready(struct nicvf *nic)
 168{
 169	union nic_mbx mbx = {};
 170
 171	mbx.msg.msg = NIC_MBOX_MSG_READY;
 172	if (nicvf_send_msg_to_pf(nic, &mbx)) {
 173		netdev_err(nic->netdev,
 174			   "PF didn't respond to READY msg\n");
 175		return 0;
 176	}
 177
 178	return 1;
 179}
 180
 181static void nicvf_send_cfg_done(struct nicvf *nic)
 182{
 183	union nic_mbx mbx = {};
 184
 185	mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
 186	if (nicvf_send_msg_to_pf(nic, &mbx)) {
 187		netdev_err(nic->netdev,
 188			   "PF didn't respond to CFG DONE msg\n");
 189	}
 190}
 191
 192static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
 193{
 194	if (bgx->rx)
 195		nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
 196	else
 197		nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
 198}
 199
 200static void  nicvf_handle_mbx_intr(struct nicvf *nic)
 201{
 202	union nic_mbx mbx = {};
 203	u64 *mbx_data;
 204	u64 mbx_addr;
 205	int i;
 206
 207	mbx_addr = NIC_VF_PF_MAILBOX_0_1;
 208	mbx_data = (u64 *)&mbx;
 209
 210	for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
 211		*mbx_data = nicvf_reg_read(nic, mbx_addr);
 212		mbx_data++;
 213		mbx_addr += sizeof(u64);
 214	}
 215
 216	netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg);
 217	switch (mbx.msg.msg) {
 218	case NIC_MBOX_MSG_READY:
 219		nic->pf_acked = true;
 220		nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
 221		nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
 222		nic->node = mbx.nic_cfg.node_id;
 223		if (!nic->set_mac_pending)
 224			eth_hw_addr_set(nic->netdev, mbx.nic_cfg.mac_addr);
 225		nic->sqs_mode = mbx.nic_cfg.sqs_mode;
 226		nic->loopback_supported = mbx.nic_cfg.loopback_supported;
 227		nic->link_up = false;
 228		nic->duplex = 0;
 229		nic->speed = 0;
 230		break;
 231	case NIC_MBOX_MSG_ACK:
 232		nic->pf_acked = true;
 233		break;
 234	case NIC_MBOX_MSG_NACK:
 235		nic->pf_nacked = true;
 236		break;
 237	case NIC_MBOX_MSG_RSS_SIZE:
 238		nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
 239		nic->pf_acked = true;
 240		break;
 241	case NIC_MBOX_MSG_BGX_STATS:
 242		nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
 243		nic->pf_acked = true;
 244		break;
 245	case NIC_MBOX_MSG_BGX_LINK_CHANGE:
 246		nic->pf_acked = true;
 247		if (nic->link_up != mbx.link_status.link_up) {
 248			nic->link_up = mbx.link_status.link_up;
 249			nic->duplex = mbx.link_status.duplex;
 250			nic->speed = mbx.link_status.speed;
 251			nic->mac_type = mbx.link_status.mac_type;
 252			if (nic->link_up) {
 253				netdev_info(nic->netdev,
 254					    "Link is Up %d Mbps %s duplex\n",
 255					    nic->speed,
 256					    nic->duplex == DUPLEX_FULL ?
 257					    "Full" : "Half");
 258				netif_carrier_on(nic->netdev);
 259				netif_tx_start_all_queues(nic->netdev);
 260			} else {
 261				netdev_info(nic->netdev, "Link is Down\n");
 262				netif_carrier_off(nic->netdev);
 263				netif_tx_stop_all_queues(nic->netdev);
 264			}
 265		}
 266		break;
 267	case NIC_MBOX_MSG_ALLOC_SQS:
 268		nic->sqs_count = mbx.sqs_alloc.qs_count;
 269		nic->pf_acked = true;
 270		break;
 271	case NIC_MBOX_MSG_SNICVF_PTR:
 272		/* Primary VF: make note of secondary VF's pointer
 273		 * to be used while packet transmission.
 274		 */
 275		nic->snicvf[mbx.nicvf.sqs_id] =
 276			(struct nicvf *)mbx.nicvf.nicvf;
 277		nic->pf_acked = true;
 278		break;
 279	case NIC_MBOX_MSG_PNICVF_PTR:
 280		/* Secondary VF/Qset: make note of primary VF's pointer
 281		 * to be used while packet reception, to handover packet
 282		 * to primary VF's netdev.
 283		 */
 284		nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf;
 285		nic->pf_acked = true;
 286		break;
 287	case NIC_MBOX_MSG_PFC:
 288		nic->pfc.autoneg = mbx.pfc.autoneg;
 289		nic->pfc.fc_rx = mbx.pfc.fc_rx;
 290		nic->pfc.fc_tx = mbx.pfc.fc_tx;
 291		nic->pf_acked = true;
 292		break;
 293	default:
 294		netdev_err(nic->netdev,
 295			   "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
 296		break;
 297	}
 298	nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
 299}
 300
 301static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev)
 302{
 303	union nic_mbx mbx = {};
 304
 305	mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
 306	mbx.mac.vf_id = nic->vf_id;
 307	ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr);
 308
 309	return nicvf_send_msg_to_pf(nic, &mbx);
 310}
 311
 312static void nicvf_config_cpi(struct nicvf *nic)
 313{
 314	union nic_mbx mbx = {};
 315
 316	mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
 317	mbx.cpi_cfg.vf_id = nic->vf_id;
 318	mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
 319	mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
 320
 321	nicvf_send_msg_to_pf(nic, &mbx);
 322}
 323
 324static void nicvf_get_rss_size(struct nicvf *nic)
 325{
 326	union nic_mbx mbx = {};
 327
 328	mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
 329	mbx.rss_size.vf_id = nic->vf_id;
 330	nicvf_send_msg_to_pf(nic, &mbx);
 331}
 332
 333void nicvf_config_rss(struct nicvf *nic)
 334{
 335	union nic_mbx mbx = {};
 336	struct nicvf_rss_info *rss = &nic->rss_info;
 337	int ind_tbl_len = rss->rss_size;
 338	int i, nextq = 0;
 339
 340	mbx.rss_cfg.vf_id = nic->vf_id;
 341	mbx.rss_cfg.hash_bits = rss->hash_bits;
 342	while (ind_tbl_len) {
 343		mbx.rss_cfg.tbl_offset = nextq;
 344		mbx.rss_cfg.tbl_len = min(ind_tbl_len,
 345					       RSS_IND_TBL_LEN_PER_MBX_MSG);
 346		mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
 347			  NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
 348
 349		for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
 350			mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
 351
 352		nicvf_send_msg_to_pf(nic, &mbx);
 353
 354		ind_tbl_len -= mbx.rss_cfg.tbl_len;
 355	}
 356}
 357
 358void nicvf_set_rss_key(struct nicvf *nic)
 359{
 360	struct nicvf_rss_info *rss = &nic->rss_info;
 361	u64 key_addr = NIC_VNIC_RSS_KEY_0_4;
 362	int idx;
 363
 364	for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
 365		nicvf_reg_write(nic, key_addr, rss->key[idx]);
 366		key_addr += sizeof(u64);
 367	}
 368}
 369
 370static int nicvf_rss_init(struct nicvf *nic)
 371{
 372	struct nicvf_rss_info *rss = &nic->rss_info;
 373	int idx;
 374
 375	nicvf_get_rss_size(nic);
 376
 377	if (cpi_alg != CPI_ALG_NONE) {
 378		rss->enable = false;
 379		rss->hash_bits = 0;
 380		return 0;
 381	}
 382
 383	rss->enable = true;
 384
 385	netdev_rss_key_fill(rss->key, RSS_HASH_KEY_SIZE * sizeof(u64));
 386	nicvf_set_rss_key(nic);
 387
 388	rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
 389	nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
 390
 391	rss->hash_bits =  ilog2(rounddown_pow_of_two(rss->rss_size));
 392
 393	for (idx = 0; idx < rss->rss_size; idx++)
 394		rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx,
 395							       nic->rx_queues);
 396	nicvf_config_rss(nic);
 397	return 1;
 398}
 399
 400/* Request PF to allocate additional Qsets */
 401static void nicvf_request_sqs(struct nicvf *nic)
 402{
 403	union nic_mbx mbx = {};
 404	int sqs;
 405	int sqs_count = nic->sqs_count;
 406	int rx_queues = 0, tx_queues = 0;
 407
 408	/* Only primary VF should request */
 409	if (nic->sqs_mode ||  !nic->sqs_count)
 410		return;
 411
 412	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
 413	mbx.sqs_alloc.vf_id = nic->vf_id;
 414	mbx.sqs_alloc.qs_count = nic->sqs_count;
 415	if (nicvf_send_msg_to_pf(nic, &mbx)) {
 416		/* No response from PF */
 417		nic->sqs_count = 0;
 418		return;
 419	}
 420
 421	/* Return if no Secondary Qsets available */
 422	if (!nic->sqs_count)
 423		return;
 424
 425	if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
 426		rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
 427
 428	tx_queues = nic->tx_queues + nic->xdp_tx_queues;
 429	if (tx_queues > MAX_SND_QUEUES_PER_QS)
 430		tx_queues = tx_queues - MAX_SND_QUEUES_PER_QS;
 431
 432	/* Set no of Rx/Tx queues in each of the SQsets */
 433	for (sqs = 0; sqs < nic->sqs_count; sqs++) {
 434		mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
 435		mbx.nicvf.vf_id = nic->vf_id;
 436		mbx.nicvf.sqs_id = sqs;
 437		nicvf_send_msg_to_pf(nic, &mbx);
 438
 439		nic->snicvf[sqs]->sqs_id = sqs;
 440		if (rx_queues > MAX_RCV_QUEUES_PER_QS) {
 441			nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS;
 442			rx_queues -= MAX_RCV_QUEUES_PER_QS;
 443		} else {
 444			nic->snicvf[sqs]->qs->rq_cnt = rx_queues;
 445			rx_queues = 0;
 446		}
 447
 448		if (tx_queues > MAX_SND_QUEUES_PER_QS) {
 449			nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS;
 450			tx_queues -= MAX_SND_QUEUES_PER_QS;
 451		} else {
 452			nic->snicvf[sqs]->qs->sq_cnt = tx_queues;
 453			tx_queues = 0;
 454		}
 455
 456		nic->snicvf[sqs]->qs->cq_cnt =
 457		max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt);
 458
 459		/* Initialize secondary Qset's queues and its interrupts */
 460		nicvf_open(nic->snicvf[sqs]->netdev);
 461	}
 462
 463	/* Update stack with actual Rx/Tx queue count allocated */
 464	if (sqs_count != nic->sqs_count)
 465		nicvf_set_real_num_queues(nic->netdev,
 466					  nic->tx_queues, nic->rx_queues);
 467}
 468
 469/* Send this Qset's nicvf pointer to PF.
 470 * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs
 471 * so that packets received by these Qsets can use primary VF's netdev
 472 */
 473static void nicvf_send_vf_struct(struct nicvf *nic)
 474{
 475	union nic_mbx mbx = {};
 476
 477	mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR;
 478	mbx.nicvf.sqs_mode = nic->sqs_mode;
 479	mbx.nicvf.nicvf = (u64)nic;
 480	nicvf_send_msg_to_pf(nic, &mbx);
 481}
 482
 483static void nicvf_get_primary_vf_struct(struct nicvf *nic)
 484{
 485	union nic_mbx mbx = {};
 486
 487	mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
 488	nicvf_send_msg_to_pf(nic, &mbx);
 489}
 490
 491int nicvf_set_real_num_queues(struct net_device *netdev,
 492			      int tx_queues, int rx_queues)
 493{
 494	int err = 0;
 495
 496	err = netif_set_real_num_tx_queues(netdev, tx_queues);
 497	if (err) {
 498		netdev_err(netdev,
 499			   "Failed to set no of Tx queues: %d\n", tx_queues);
 500		return err;
 501	}
 502
 503	err = netif_set_real_num_rx_queues(netdev, rx_queues);
 504	if (err)
 505		netdev_err(netdev,
 506			   "Failed to set no of Rx queues: %d\n", rx_queues);
 507	return err;
 508}
 509
 510static int nicvf_init_resources(struct nicvf *nic)
 511{
 512	int err;
 513
 514	/* Enable Qset */
 515	nicvf_qset_config(nic, true);
 516
 517	/* Initialize queues and HW for data transfer */
 518	err = nicvf_config_data_transfer(nic, true);
 519	if (err) {
 520		netdev_err(nic->netdev,
 521			   "Failed to alloc/config VF's QSet resources\n");
 522		return err;
 523	}
 524
 525	return 0;
 526}
 527
 528static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
 529				struct cqe_rx_t *cqe_rx, struct snd_queue *sq,
 530				struct rcv_queue *rq, struct sk_buff **skb)
 531{
 532	unsigned char *hard_start, *data;
 533	struct xdp_buff xdp;
 534	struct page *page;
 535	u32 action;
 536	u16 len, offset = 0;
 537	u64 dma_addr, cpu_addr;
 538	void *orig_data;
 539
 540	/* Retrieve packet buffer's DMA address and length */
 541	len = *((u16 *)((void *)cqe_rx + (3 * sizeof(u64))));
 542	dma_addr = *((u64 *)((void *)cqe_rx + (7 * sizeof(u64))));
 543
 544	cpu_addr = nicvf_iova_to_phys(nic, dma_addr);
 545	if (!cpu_addr)
 546		return false;
 547	cpu_addr = (u64)phys_to_virt(cpu_addr);
 548	page = virt_to_page((void *)cpu_addr);
 549
 550	xdp_init_buff(&xdp, RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
 551		      &rq->xdp_rxq);
 552	hard_start = page_address(page);
 553	data = (unsigned char *)cpu_addr;
 554	xdp_prepare_buff(&xdp, hard_start, data - hard_start, len, false);
 555	orig_data = xdp.data;
 556
 557	action = bpf_prog_run_xdp(prog, &xdp);
 558
 559	len = xdp.data_end - xdp.data;
 560	/* Check if XDP program has changed headers */
 561	if (orig_data != xdp.data) {
 562		offset = orig_data - xdp.data;
 563		dma_addr -= offset;
 564	}
 565
 566	switch (action) {
 567	case XDP_PASS:
 568		/* Check if it's a recycled page, if not
 569		 * unmap the DMA mapping.
 570		 *
 571		 * Recycled page holds an extra reference.
 572		 */
 573		if (page_ref_count(page) == 1) {
 574			dma_addr &= PAGE_MASK;
 575			dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
 576					     RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
 577					     DMA_FROM_DEVICE,
 578					     DMA_ATTR_SKIP_CPU_SYNC);
 579		}
 580
 581		/* Build SKB and pass on packet to network stack */
 582		*skb = build_skb(xdp.data,
 583				 RCV_FRAG_LEN - cqe_rx->align_pad + offset);
 584		if (!*skb)
 585			put_page(page);
 586		else
 587			skb_put(*skb, len);
 588		return false;
 589	case XDP_TX:
 590		nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len);
 591		return true;
 592	default:
 593		bpf_warn_invalid_xdp_action(nic->netdev, prog, action);
 594		fallthrough;
 595	case XDP_ABORTED:
 596		trace_xdp_exception(nic->netdev, prog, action);
 597		fallthrough;
 598	case XDP_DROP:
 599		/* Check if it's a recycled page, if not
 600		 * unmap the DMA mapping.
 601		 *
 602		 * Recycled page holds an extra reference.
 603		 */
 604		if (page_ref_count(page) == 1) {
 605			dma_addr &= PAGE_MASK;
 606			dma_unmap_page_attrs(&nic->pdev->dev, dma_addr,
 607					     RCV_FRAG_LEN + XDP_PACKET_HEADROOM,
 608					     DMA_FROM_DEVICE,
 609					     DMA_ATTR_SKIP_CPU_SYNC);
 610		}
 611		put_page(page);
 612		return true;
 613	}
 614	return false;
 615}
 616
 617static void nicvf_snd_ptp_handler(struct net_device *netdev,
 618				  struct cqe_send_t *cqe_tx)
 619{
 620	struct nicvf *nic = netdev_priv(netdev);
 621	struct skb_shared_hwtstamps ts;
 622	u64 ns;
 623
 624	nic = nic->pnicvf;
 625
 626	/* Sync for 'ptp_skb' */
 627	smp_rmb();
 628
 629	/* New timestamp request can be queued now */
 630	atomic_set(&nic->tx_ptp_skbs, 0);
 631
 632	/* Check for timestamp requested skb */
 633	if (!nic->ptp_skb)
 634		return;
 635
 636	/* Check if timestamping is timedout, which is set to 10us */
 637	if (cqe_tx->send_status == CQ_TX_ERROP_TSTMP_TIMEOUT ||
 638	    cqe_tx->send_status == CQ_TX_ERROP_TSTMP_CONFLICT)
 639		goto no_tstamp;
 640
 641	/* Get the timestamp */
 642	memset(&ts, 0, sizeof(ts));
 643	ns = cavium_ptp_tstamp2time(nic->ptp_clock, cqe_tx->ptp_timestamp);
 644	ts.hwtstamp = ns_to_ktime(ns);
 645	skb_tstamp_tx(nic->ptp_skb, &ts);
 646
 647no_tstamp:
 648	/* Free the original skb */
 649	dev_kfree_skb_any(nic->ptp_skb);
 650	nic->ptp_skb = NULL;
 651	/* Sync 'ptp_skb' */
 652	smp_wmb();
 653}
 654
 655static void nicvf_snd_pkt_handler(struct net_device *netdev,
 656				  struct cqe_send_t *cqe_tx,
 657				  int budget, int *subdesc_cnt,
 658				  unsigned int *tx_pkts, unsigned int *tx_bytes)
 659{
 660	struct sk_buff *skb = NULL;
 661	struct page *page;
 662	struct nicvf *nic = netdev_priv(netdev);
 663	struct snd_queue *sq;
 664	struct sq_hdr_subdesc *hdr;
 665	struct sq_hdr_subdesc *tso_sqe;
 666
 667	sq = &nic->qs->sq[cqe_tx->sq_idx];
 668
 669	hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
 670	if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
 671		return;
 672
 673	/* Check for errors */
 674	if (cqe_tx->send_status)
 675		nicvf_check_cqe_tx_errs(nic->pnicvf, cqe_tx);
 676
 677	/* Is this a XDP designated Tx queue */
 678	if (sq->is_xdp) {
 679		page = (struct page *)sq->xdp_page[cqe_tx->sqe_ptr];
 680		/* Check if it's recycled page or else unmap DMA mapping */
 681		if (page && (page_ref_count(page) == 1))
 682			nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
 683						 hdr->subdesc_cnt);
 684
 685		/* Release page reference for recycling */
 686		if (page)
 687			put_page(page);
 688		sq->xdp_page[cqe_tx->sqe_ptr] = (u64)NULL;
 689		*subdesc_cnt += hdr->subdesc_cnt + 1;
 690		return;
 691	}
 692
 693	skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
 694	if (skb) {
 695		/* Check for dummy descriptor used for HW TSO offload on 88xx */
 696		if (hdr->dont_send) {
 697			/* Get actual TSO descriptors and free them */
 698			tso_sqe =
 699			 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
 700			nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,
 701						 tso_sqe->subdesc_cnt);
 702			*subdesc_cnt += tso_sqe->subdesc_cnt + 1;
 703		} else {
 704			nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
 705						 hdr->subdesc_cnt);
 706		}
 707		*subdesc_cnt += hdr->subdesc_cnt + 1;
 708		prefetch(skb);
 709		(*tx_pkts)++;
 710		*tx_bytes += skb->len;
 711		/* If timestamp is requested for this skb, don't free it */
 712		if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS &&
 713		    !nic->pnicvf->ptp_skb)
 714			nic->pnicvf->ptp_skb = skb;
 715		else
 716			napi_consume_skb(skb, budget);
 717		sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
 718	} else {
 719		/* In case of SW TSO on 88xx, only last segment will have
 720		 * a SKB attached, so just free SQEs here.
 721		 */
 722		if (!nic->hw_tso)
 723			*subdesc_cnt += hdr->subdesc_cnt + 1;
 724	}
 725}
 726
 727static inline void nicvf_set_rxhash(struct net_device *netdev,
 728				    struct cqe_rx_t *cqe_rx,
 729				    struct sk_buff *skb)
 730{
 731	u8 hash_type;
 732	u32 hash;
 733
 734	if (!(netdev->features & NETIF_F_RXHASH))
 735		return;
 736
 737	switch (cqe_rx->rss_alg) {
 738	case RSS_ALG_TCP_IP:
 739	case RSS_ALG_UDP_IP:
 740		hash_type = PKT_HASH_TYPE_L4;
 741		hash = cqe_rx->rss_tag;
 742		break;
 743	case RSS_ALG_IP:
 744		hash_type = PKT_HASH_TYPE_L3;
 745		hash = cqe_rx->rss_tag;
 746		break;
 747	default:
 748		hash_type = PKT_HASH_TYPE_NONE;
 749		hash = 0;
 750	}
 751
 752	skb_set_hash(skb, hash, hash_type);
 753}
 754
 755static inline void nicvf_set_rxtstamp(struct nicvf *nic, struct sk_buff *skb)
 756{
 757	u64 ns;
 758
 759	if (!nic->ptp_clock || !nic->hw_rx_tstamp)
 760		return;
 761
 762	/* The first 8 bytes is the timestamp */
 763	ns = cavium_ptp_tstamp2time(nic->ptp_clock,
 764				    be64_to_cpu(*(__be64 *)skb->data));
 765	skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
 766
 767	__skb_pull(skb, 8);
 768}
 769
 770static void nicvf_rcv_pkt_handler(struct net_device *netdev,
 771				  struct napi_struct *napi,
 772				  struct cqe_rx_t *cqe_rx,
 773				  struct snd_queue *sq, struct rcv_queue *rq)
 774{
 775	struct sk_buff *skb = NULL;
 776	struct nicvf *nic = netdev_priv(netdev);
 777	struct nicvf *snic = nic;
 778	int err = 0;
 779	int rq_idx;
 780
 781	rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx);
 782
 783	if (nic->sqs_mode) {
 784		/* Use primary VF's 'nicvf' struct */
 785		nic = nic->pnicvf;
 786		netdev = nic->netdev;
 787	}
 788
 789	/* Check for errors */
 790	if (cqe_rx->err_level || cqe_rx->err_opcode) {
 791		err = nicvf_check_cqe_rx_errs(nic, cqe_rx);
 792		if (err && !cqe_rx->rb_cnt)
 793			return;
 794	}
 795
 796	/* For XDP, ignore pkts spanning multiple pages */
 797	if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) {
 798		/* Packet consumed by XDP */
 799		if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, rq, &skb))
 800			return;
 801	} else {
 802		skb = nicvf_get_rcv_skb(snic, cqe_rx,
 803					nic->xdp_prog ? true : false);
 804	}
 805
 806	if (!skb)
 807		return;
 808
 809	if (netif_msg_pktdata(nic)) {
 810		netdev_info(nic->netdev, "skb 0x%p, len=%d\n", skb, skb->len);
 811		print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
 812			       skb->data, skb->len, true);
 813	}
 814
 815	/* If error packet, drop it here */
 816	if (err) {
 817		dev_kfree_skb_any(skb);
 818		return;
 819	}
 820
 821	nicvf_set_rxtstamp(nic, skb);
 822	nicvf_set_rxhash(netdev, cqe_rx, skb);
 823
 824	skb_record_rx_queue(skb, rq_idx);
 825	if (netdev->hw_features & NETIF_F_RXCSUM) {
 826		/* HW by default verifies TCP/UDP/SCTP checksums */
 827		skb->ip_summed = CHECKSUM_UNNECESSARY;
 828	} else {
 829		skb_checksum_none_assert(skb);
 830	}
 831
 832	skb->protocol = eth_type_trans(skb, netdev);
 833
 834	/* Check for stripped VLAN */
 835	if (cqe_rx->vlan_found && cqe_rx->vlan_stripped)
 836		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 837				       ntohs((__force __be16)cqe_rx->vlan_tci));
 838
 839	if (napi && (netdev->features & NETIF_F_GRO))
 840		napi_gro_receive(napi, skb);
 841	else
 842		netif_receive_skb(skb);
 843}
 844
 845static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
 846				 struct napi_struct *napi, int budget)
 847{
 848	int processed_cqe, work_done = 0, tx_done = 0;
 849	int cqe_count, cqe_head;
 850	int subdesc_cnt = 0;
 851	struct nicvf *nic = netdev_priv(netdev);
 852	struct queue_set *qs = nic->qs;
 853	struct cmp_queue *cq = &qs->cq[cq_idx];
 854	struct cqe_rx_t *cq_desc;
 855	struct netdev_queue *txq;
 856	struct snd_queue *sq = &qs->sq[cq_idx];
 857	struct rcv_queue *rq = &qs->rq[cq_idx];
 858	unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx;
 859
 860	spin_lock_bh(&cq->lock);
 861loop:
 862	processed_cqe = 0;
 863	/* Get no of valid CQ entries to process */
 864	cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
 865	cqe_count &= CQ_CQE_COUNT;
 866	if (!cqe_count)
 867		goto done;
 868
 869	/* Get head of the valid CQ entries */
 870	cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
 871	cqe_head &= 0xFFFF;
 872
 873	while (processed_cqe < cqe_count) {
 874		/* Get the CQ descriptor */
 875		cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
 876		cqe_head++;
 877		cqe_head &= (cq->dmem.q_len - 1);
 878		/* Initiate prefetch for next descriptor */
 879		prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
 880
 881		if ((work_done >= budget) && napi &&
 882		    (cq_desc->cqe_type != CQE_TYPE_SEND)) {
 883			break;
 884		}
 885
 886		switch (cq_desc->cqe_type) {
 887		case CQE_TYPE_RX:
 888			nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq, rq);
 889			work_done++;
 890		break;
 891		case CQE_TYPE_SEND:
 892			nicvf_snd_pkt_handler(netdev, (void *)cq_desc,
 893					      budget, &subdesc_cnt,
 894					      &tx_pkts, &tx_bytes);
 895			tx_done++;
 896		break;
 897		case CQE_TYPE_SEND_PTP:
 898			nicvf_snd_ptp_handler(netdev, (void *)cq_desc);
 899		break;
 900		case CQE_TYPE_INVALID:
 901		case CQE_TYPE_RX_SPLIT:
 902		case CQE_TYPE_RX_TCP:
 903			/* Ignore for now */
 904		break;
 905		}
 906		processed_cqe++;
 907	}
 908
 909	/* Ring doorbell to inform H/W to reuse processed CQEs */
 910	nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
 911			      cq_idx, processed_cqe);
 912
 913	if ((work_done < budget) && napi)
 914		goto loop;
 915
 916done:
 917	/* Update SQ's descriptor free count */
 918	if (subdesc_cnt)
 919		nicvf_put_sq_desc(sq, subdesc_cnt);
 920
 921	txq_idx = nicvf_netdev_qidx(nic, cq_idx);
 922	/* Handle XDP TX queues */
 923	if (nic->pnicvf->xdp_prog) {
 924		if (txq_idx < nic->pnicvf->xdp_tx_queues) {
 925			nicvf_xdp_sq_doorbell(nic, sq, cq_idx);
 926			goto out;
 927		}
 928		nic = nic->pnicvf;
 929		txq_idx -= nic->pnicvf->xdp_tx_queues;
 930	}
 931
 932	/* Wakeup TXQ if its stopped earlier due to SQ full */
 933	if (tx_done ||
 934	    (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) {
 935		netdev = nic->pnicvf->netdev;
 936		txq = netdev_get_tx_queue(netdev, txq_idx);
 937		if (tx_pkts)
 938			netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
 939
 940		/* To read updated queue and carrier status */
 941		smp_mb();
 942		if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
 943			netif_tx_wake_queue(txq);
 944			nic = nic->pnicvf;
 945			this_cpu_inc(nic->drv_stats->txq_wake);
 946			netif_warn(nic, tx_err, netdev,
 947				   "Transmit queue wakeup SQ%d\n", txq_idx);
 948		}
 949	}
 950
 951out:
 952	spin_unlock_bh(&cq->lock);
 953	return work_done;
 954}
 955
 956static int nicvf_poll(struct napi_struct *napi, int budget)
 957{
 958	u64  cq_head;
 959	int  work_done = 0;
 960	struct net_device *netdev = napi->dev;
 961	struct nicvf *nic = netdev_priv(netdev);
 962	struct nicvf_cq_poll *cq;
 963
 964	cq = container_of(napi, struct nicvf_cq_poll, napi);
 965	work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget);
 966
 967	if (work_done < budget) {
 968		/* Slow packet rate, exit polling */
 969		napi_complete_done(napi, work_done);
 970		/* Re-enable interrupts */
 971		cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD,
 972					       cq->cq_idx);
 973		nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
 974		nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD,
 975				      cq->cq_idx, cq_head);
 976		nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
 977	}
 978	return work_done;
 979}
 980
 981/* Qset error interrupt handler
 982 *
 983 * As of now only CQ errors are handled
 984 */
 985static void nicvf_handle_qs_err(struct tasklet_struct *t)
 986{
 987	struct nicvf *nic = from_tasklet(nic, t, qs_err_task);
 988	struct queue_set *qs = nic->qs;
 989	int qidx;
 990	u64 status;
 991
 992	netif_tx_disable(nic->netdev);
 993
 994	/* Check if it is CQ err */
 995	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
 996		status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
 997					      qidx);
 998		if (!(status & CQ_ERR_MASK))
 999			continue;
1000		/* Process already queued CQEs and reconfig CQ */
1001		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1002		nicvf_sq_disable(nic, qidx);
1003		nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0);
1004		nicvf_cmp_queue_config(nic, qs, qidx, true);
1005		nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx);
1006		nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
1007
1008		nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
1009	}
1010
1011	netif_tx_start_all_queues(nic->netdev);
1012	/* Re-enable Qset error interrupt */
1013	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
1014}
1015
1016static void nicvf_dump_intr_status(struct nicvf *nic)
1017{
1018	netif_info(nic, intr, nic->netdev, "interrupt status 0x%llx\n",
1019		   nicvf_reg_read(nic, NIC_VF_INT));
1020}
1021
1022static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq)
1023{
1024	struct nicvf *nic = (struct nicvf *)nicvf_irq;
1025	u64 intr;
1026
1027	nicvf_dump_intr_status(nic);
1028
1029	intr = nicvf_reg_read(nic, NIC_VF_INT);
1030	/* Check for spurious interrupt */
1031	if (!(intr & NICVF_INTR_MBOX_MASK))
1032		return IRQ_HANDLED;
1033
1034	nicvf_handle_mbx_intr(nic);
1035
1036	return IRQ_HANDLED;
1037}
1038
1039static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq)
1040{
1041	struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq;
1042	struct nicvf *nic = cq_poll->nicvf;
1043	int qidx = cq_poll->cq_idx;
1044
1045	nicvf_dump_intr_status(nic);
1046
1047	/* Disable interrupts */
1048	nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1049
1050	/* Schedule NAPI */
1051	napi_schedule_irqoff(&cq_poll->napi);
1052
1053	/* Clear interrupt */
1054	nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1055
1056	return IRQ_HANDLED;
1057}
1058
1059static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq)
1060{
1061	struct nicvf *nic = (struct nicvf *)nicvf_irq;
1062	u8 qidx;
1063
1064
1065	nicvf_dump_intr_status(nic);
1066
1067	/* Disable RBDR interrupt and schedule softirq */
1068	for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
1069		if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
1070			continue;
1071		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1072		tasklet_hi_schedule(&nic->rbdr_task);
1073		/* Clear interrupt */
1074		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1075	}
1076
1077	return IRQ_HANDLED;
1078}
1079
1080static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq)
1081{
1082	struct nicvf *nic = (struct nicvf *)nicvf_irq;
1083
1084	nicvf_dump_intr_status(nic);
1085
1086	/* Disable Qset err interrupt and schedule softirq */
1087	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1088	tasklet_hi_schedule(&nic->qs_err_task);
1089	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1090
1091	return IRQ_HANDLED;
1092}
1093
1094static void nicvf_set_irq_affinity(struct nicvf *nic)
1095{
1096	int vec, cpu;
1097
1098	for (vec = 0; vec < nic->num_vec; vec++) {
1099		if (!nic->irq_allocated[vec])
1100			continue;
1101
1102		if (!zalloc_cpumask_var(&nic->affinity_mask[vec], GFP_KERNEL))
1103			return;
1104		 /* CQ interrupts */
1105		if (vec < NICVF_INTR_ID_SQ)
1106			/* Leave CPU0 for RBDR and other interrupts */
1107			cpu = nicvf_netdev_qidx(nic, vec) + 1;
1108		else
1109			cpu = 0;
1110
1111		cpumask_set_cpu(cpumask_local_spread(cpu, nic->node),
1112				nic->affinity_mask[vec]);
1113		irq_set_affinity_hint(pci_irq_vector(nic->pdev, vec),
1114				      nic->affinity_mask[vec]);
1115	}
1116}
1117
1118static int nicvf_register_interrupts(struct nicvf *nic)
1119{
1120	int irq, ret = 0;
1121
1122	for_each_cq_irq(irq)
1123		sprintf(nic->irq_name[irq], "%s-rxtx-%d",
1124			nic->pnicvf->netdev->name,
1125			nicvf_netdev_qidx(nic, irq));
1126
1127	for_each_sq_irq(irq)
1128		sprintf(nic->irq_name[irq], "%s-sq-%d",
1129			nic->pnicvf->netdev->name,
1130			nicvf_netdev_qidx(nic, irq - NICVF_INTR_ID_SQ));
1131
1132	for_each_rbdr_irq(irq)
1133		sprintf(nic->irq_name[irq], "%s-rbdr-%d",
1134			nic->pnicvf->netdev->name,
1135			nic->sqs_mode ? (nic->sqs_id + 1) : 0);
1136
1137	/* Register CQ interrupts */
1138	for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
1139		ret = request_irq(pci_irq_vector(nic->pdev, irq),
1140				  nicvf_intr_handler,
1141				  0, nic->irq_name[irq], nic->napi[irq]);
1142		if (ret)
1143			goto err;
1144		nic->irq_allocated[irq] = true;
1145	}
1146
1147	/* Register RBDR interrupt */
1148	for (irq = NICVF_INTR_ID_RBDR;
1149	     irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) {
1150		ret = request_irq(pci_irq_vector(nic->pdev, irq),
1151				  nicvf_rbdr_intr_handler,
1152				  0, nic->irq_name[irq], nic);
1153		if (ret)
1154			goto err;
1155		nic->irq_allocated[irq] = true;
1156	}
1157
1158	/* Register QS error interrupt */
1159	sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], "%s-qset-err-%d",
1160		nic->pnicvf->netdev->name,
1161		nic->sqs_mode ? (nic->sqs_id + 1) : 0);
1162	irq = NICVF_INTR_ID_QS_ERR;
1163	ret = request_irq(pci_irq_vector(nic->pdev, irq),
1164			  nicvf_qs_err_intr_handler,
1165			  0, nic->irq_name[irq], nic);
1166	if (ret)
1167		goto err;
1168
1169	nic->irq_allocated[irq] = true;
1170
1171	/* Set IRQ affinities */
1172	nicvf_set_irq_affinity(nic);
1173
1174err:
1175	if (ret)
1176		netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq);
1177
1178	return ret;
1179}
1180
1181static void nicvf_unregister_interrupts(struct nicvf *nic)
1182{
1183	struct pci_dev *pdev = nic->pdev;
1184	int irq;
1185
1186	/* Free registered interrupts */
1187	for (irq = 0; irq < nic->num_vec; irq++) {
1188		if (!nic->irq_allocated[irq])
1189			continue;
1190
1191		irq_set_affinity_hint(pci_irq_vector(pdev, irq), NULL);
1192		free_cpumask_var(nic->affinity_mask[irq]);
1193
1194		if (irq < NICVF_INTR_ID_SQ)
1195			free_irq(pci_irq_vector(pdev, irq), nic->napi[irq]);
1196		else
1197			free_irq(pci_irq_vector(pdev, irq), nic);
1198
1199		nic->irq_allocated[irq] = false;
1200	}
1201
1202	/* Disable MSI-X */
1203	pci_free_irq_vectors(pdev);
1204	nic->num_vec = 0;
1205}
1206
1207/* Initialize MSIX vectors and register MISC interrupt.
1208 * Send READY message to PF to check if its alive
1209 */
1210static int nicvf_register_misc_interrupt(struct nicvf *nic)
1211{
1212	int ret = 0;
1213	int irq = NICVF_INTR_ID_MISC;
1214
1215	/* Return if mailbox interrupt is already registered */
1216	if (nic->pdev->msix_enabled)
1217		return 0;
1218
1219	/* Enable MSI-X */
1220	nic->num_vec = pci_msix_vec_count(nic->pdev);
1221	ret = pci_alloc_irq_vectors(nic->pdev, nic->num_vec, nic->num_vec,
1222				    PCI_IRQ_MSIX);
1223	if (ret < 0) {
1224		netdev_err(nic->netdev,
1225			   "Req for #%d msix vectors failed\n", nic->num_vec);
1226		return ret;
1227	}
1228
1229	sprintf(nic->irq_name[irq], "%s Mbox", "NICVF");
1230	/* Register Misc interrupt */
1231	ret = request_irq(pci_irq_vector(nic->pdev, irq),
1232			  nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic);
1233
1234	if (ret)
1235		return ret;
1236	nic->irq_allocated[irq] = true;
1237
1238	/* Enable mailbox interrupt */
1239	nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
1240
1241	/* Check if VF is able to communicate with PF */
1242	if (!nicvf_check_pf_ready(nic)) {
1243		nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1244		nicvf_unregister_interrupts(nic);
1245		return -EIO;
1246	}
1247
1248	return 0;
1249}
1250
1251static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
1252{
1253	struct nicvf *nic = netdev_priv(netdev);
1254	int qid = skb_get_queue_mapping(skb);
1255	struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid);
1256	struct nicvf *snic;
1257	struct snd_queue *sq;
1258	int tmp;
1259
1260	/* Check for minimum packet length */
1261	if (skb->len <= ETH_HLEN) {
1262		dev_kfree_skb(skb);
1263		return NETDEV_TX_OK;
1264	}
1265
1266	/* In XDP case, initial HW tx queues are used for XDP,
1267	 * but stack's queue mapping starts at '0', so skip the
1268	 * Tx queues attached to Rx queues for XDP.
1269	 */
1270	if (nic->xdp_prog)
1271		qid += nic->xdp_tx_queues;
1272
1273	snic = nic;
1274	/* Get secondary Qset's SQ structure */
1275	if (qid >= MAX_SND_QUEUES_PER_QS) {
1276		tmp = qid / MAX_SND_QUEUES_PER_QS;
1277		snic = (struct nicvf *)nic->snicvf[tmp - 1];
1278		if (!snic) {
1279			netdev_warn(nic->netdev,
1280				    "Secondary Qset#%d's ptr not initialized\n",
1281				    tmp - 1);
1282			dev_kfree_skb(skb);
1283			return NETDEV_TX_OK;
1284		}
1285		qid = qid % MAX_SND_QUEUES_PER_QS;
1286	}
1287
1288	sq = &snic->qs->sq[qid];
1289	if (!netif_tx_queue_stopped(txq) &&
1290	    !nicvf_sq_append_skb(snic, sq, skb, qid)) {
1291		netif_tx_stop_queue(txq);
1292
1293		/* Barrier, so that stop_queue visible to other cpus */
1294		smp_mb();
1295
1296		/* Check again, incase another cpu freed descriptors */
1297		if (atomic_read(&sq->free_cnt) > MIN_SQ_DESC_PER_PKT_XMIT) {
1298			netif_tx_wake_queue(txq);
1299		} else {
1300			this_cpu_inc(nic->drv_stats->txq_stop);
1301			netif_warn(nic, tx_err, netdev,
1302				   "Transmit ring full, stopping SQ%d\n", qid);
1303		}
1304		return NETDEV_TX_BUSY;
1305	}
1306
1307	return NETDEV_TX_OK;
1308}
1309
1310static inline void nicvf_free_cq_poll(struct nicvf *nic)
1311{
1312	struct nicvf_cq_poll *cq_poll;
1313	int qidx;
1314
1315	for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1316		cq_poll = nic->napi[qidx];
1317		if (!cq_poll)
1318			continue;
1319		nic->napi[qidx] = NULL;
1320		kfree(cq_poll);
1321	}
1322}
1323
1324int nicvf_stop(struct net_device *netdev)
1325{
1326	int irq, qidx;
1327	struct nicvf *nic = netdev_priv(netdev);
1328	struct queue_set *qs = nic->qs;
1329	struct nicvf_cq_poll *cq_poll = NULL;
1330	union nic_mbx mbx = {};
1331
1332	/* wait till all queued set_rx_mode tasks completes */
1333	if (nic->nicvf_rx_mode_wq) {
1334		cancel_delayed_work_sync(&nic->link_change_work);
1335		drain_workqueue(nic->nicvf_rx_mode_wq);
1336	}
1337
1338	mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
1339	nicvf_send_msg_to_pf(nic, &mbx);
1340
1341	netif_carrier_off(netdev);
1342	netif_tx_stop_all_queues(nic->netdev);
1343	nic->link_up = false;
1344
1345	/* Teardown secondary qsets first */
1346	if (!nic->sqs_mode) {
1347		for (qidx = 0; qidx < nic->sqs_count; qidx++) {
1348			if (!nic->snicvf[qidx])
1349				continue;
1350			nicvf_stop(nic->snicvf[qidx]->netdev);
1351			nic->snicvf[qidx] = NULL;
1352		}
1353	}
1354
1355	/* Disable RBDR & QS error interrupts */
1356	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
1357		nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1358		nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1359	}
1360	nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1361	nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1362
1363	/* Wait for pending IRQ handlers to finish */
1364	for (irq = 0; irq < nic->num_vec; irq++)
1365		synchronize_irq(pci_irq_vector(nic->pdev, irq));
1366
1367	tasklet_kill(&nic->rbdr_task);
1368	tasklet_kill(&nic->qs_err_task);
1369	if (nic->rb_work_scheduled)
1370		cancel_delayed_work_sync(&nic->rbdr_work);
1371
1372	for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1373		cq_poll = nic->napi[qidx];
1374		if (!cq_poll)
1375			continue;
1376		napi_synchronize(&cq_poll->napi);
1377		/* CQ intr is enabled while napi_complete,
1378		 * so disable it now
1379		 */
1380		nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1381		nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1382		napi_disable(&cq_poll->napi);
1383		netif_napi_del(&cq_poll->napi);
1384	}
1385
1386	netif_tx_disable(netdev);
1387
1388	for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
1389		netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
1390
1391	/* Free resources */
1392	nicvf_config_data_transfer(nic, false);
1393
1394	/* Disable HW Qset */
1395	nicvf_qset_config(nic, false);
1396
1397	/* disable mailbox interrupt */
1398	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1399
1400	nicvf_unregister_interrupts(nic);
1401
1402	nicvf_free_cq_poll(nic);
1403
1404	/* Free any pending SKB saved to receive timestamp */
1405	if (nic->ptp_skb) {
1406		dev_kfree_skb_any(nic->ptp_skb);
1407		nic->ptp_skb = NULL;
1408	}
1409
1410	/* Clear multiqset info */
1411	nic->pnicvf = nic;
1412
1413	return 0;
1414}
1415
1416static int nicvf_config_hw_rx_tstamp(struct nicvf *nic, bool enable)
1417{
1418	union nic_mbx mbx = {};
1419
1420	mbx.ptp.msg = NIC_MBOX_MSG_PTP_CFG;
1421	mbx.ptp.enable = enable;
1422
1423	return nicvf_send_msg_to_pf(nic, &mbx);
1424}
1425
1426static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
1427{
1428	union nic_mbx mbx = {};
1429
1430	mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
1431	mbx.frs.max_frs = mtu;
1432	mbx.frs.vf_id = nic->vf_id;
1433
1434	return nicvf_send_msg_to_pf(nic, &mbx);
1435}
1436
1437static void nicvf_link_status_check_task(struct work_struct *work_arg)
1438{
1439	struct nicvf *nic = container_of(work_arg,
1440					 struct nicvf,
1441					 link_change_work.work);
1442	union nic_mbx mbx = {};
1443	mbx.msg.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE;
1444	nicvf_send_msg_to_pf(nic, &mbx);
1445	queue_delayed_work(nic->nicvf_rx_mode_wq,
1446			   &nic->link_change_work, 2 * HZ);
1447}
1448
1449int nicvf_open(struct net_device *netdev)
1450{
1451	int cpu, err, qidx;
1452	struct nicvf *nic = netdev_priv(netdev);
1453	struct queue_set *qs = nic->qs;
1454	struct nicvf_cq_poll *cq_poll = NULL;
1455
1456	/* wait till all queued set_rx_mode tasks completes if any */
1457	if (nic->nicvf_rx_mode_wq)
1458		drain_workqueue(nic->nicvf_rx_mode_wq);
1459
1460	netif_carrier_off(netdev);
1461
1462	err = nicvf_register_misc_interrupt(nic);
1463	if (err)
1464		return err;
1465
1466	/* Register NAPI handler for processing CQEs */
1467	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1468		cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL);
1469		if (!cq_poll) {
1470			err = -ENOMEM;
1471			goto napi_del;
1472		}
1473		cq_poll->cq_idx = qidx;
1474		cq_poll->nicvf = nic;
1475		netif_napi_add(netdev, &cq_poll->napi, nicvf_poll);
1476		napi_enable(&cq_poll->napi);
1477		nic->napi[qidx] = cq_poll;
1478	}
1479
1480	/* Check if we got MAC address from PF or else generate a radom MAC */
1481	if (!nic->sqs_mode && is_zero_ether_addr(netdev->dev_addr)) {
1482		eth_hw_addr_random(netdev);
1483		nicvf_hw_set_mac_addr(nic, netdev);
1484	}
1485
1486	if (nic->set_mac_pending) {
1487		nic->set_mac_pending = false;
1488		nicvf_hw_set_mac_addr(nic, netdev);
1489	}
1490
1491	/* Init tasklet for handling Qset err interrupt */
1492	tasklet_setup(&nic->qs_err_task, nicvf_handle_qs_err);
1493
1494	/* Init RBDR tasklet which will refill RBDR */
1495	tasklet_setup(&nic->rbdr_task, nicvf_rbdr_task);
1496	INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work);
1497
1498	/* Configure CPI alorithm */
1499	nic->cpi_alg = cpi_alg;
1500	if (!nic->sqs_mode)
1501		nicvf_config_cpi(nic);
1502
1503	nicvf_request_sqs(nic);
1504	if (nic->sqs_mode)
1505		nicvf_get_primary_vf_struct(nic);
1506
1507	/* Configure PTP timestamp */
1508	if (nic->ptp_clock)
1509		nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp);
1510	atomic_set(&nic->tx_ptp_skbs, 0);
1511	nic->ptp_skb = NULL;
1512
1513	/* Configure receive side scaling and MTU */
1514	if (!nic->sqs_mode) {
1515		nicvf_rss_init(nic);
1516		err = nicvf_update_hw_max_frs(nic, netdev->mtu);
1517		if (err)
1518			goto cleanup;
1519
1520		/* Clear percpu stats */
1521		for_each_possible_cpu(cpu)
1522			memset(per_cpu_ptr(nic->drv_stats, cpu), 0,
1523			       sizeof(struct nicvf_drv_stats));
1524	}
1525
1526	err = nicvf_register_interrupts(nic);
1527	if (err)
1528		goto cleanup;
1529
1530	/* Initialize the queues */
1531	err = nicvf_init_resources(nic);
1532	if (err)
1533		goto cleanup;
1534
1535	/* Make sure queue initialization is written */
1536	wmb();
1537
1538	nicvf_reg_write(nic, NIC_VF_INT, -1);
1539	/* Enable Qset err interrupt */
1540	nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
1541
1542	/* Enable completion queue interrupt */
1543	for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1544		nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
1545
1546	/* Enable RBDR threshold interrupt */
1547	for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1548		nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
1549
1550	/* Send VF config done msg to PF */
1551	nicvf_send_cfg_done(nic);
1552
1553	if (nic->nicvf_rx_mode_wq) {
1554		INIT_DELAYED_WORK(&nic->link_change_work,
1555				  nicvf_link_status_check_task);
1556		queue_delayed_work(nic->nicvf_rx_mode_wq,
1557				   &nic->link_change_work, 0);
1558	}
1559
1560	return 0;
1561cleanup:
1562	nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1563	nicvf_unregister_interrupts(nic);
1564	tasklet_kill(&nic->qs_err_task);
1565	tasklet_kill(&nic->rbdr_task);
1566napi_del:
1567	for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1568		cq_poll = nic->napi[qidx];
1569		if (!cq_poll)
1570			continue;
1571		napi_disable(&cq_poll->napi);
1572		netif_napi_del(&cq_poll->napi);
1573	}
1574	nicvf_free_cq_poll(nic);
1575	return err;
1576}
1577
1578static int nicvf_change_mtu(struct net_device *netdev, int new_mtu)
1579{
1580	struct nicvf *nic = netdev_priv(netdev);
1581
1582	/* For now just support only the usual MTU sized frames,
1583	 * plus some headroom for VLAN, QinQ.
1584	 */
1585	if (nic->xdp_prog && new_mtu > MAX_XDP_MTU) {
1586		netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
1587			    netdev->mtu);
1588		return -EINVAL;
1589	}
1590
1591	if (netif_running(netdev) && nicvf_update_hw_max_frs(nic, new_mtu))
1592		return -EINVAL;
1593
1594	WRITE_ONCE(netdev->mtu, new_mtu);
1595
1596	return 0;
1597}
1598
1599static int nicvf_set_mac_address(struct net_device *netdev, void *p)
1600{
1601	struct sockaddr *addr = p;
1602	struct nicvf *nic = netdev_priv(netdev);
1603
1604	if (!is_valid_ether_addr(addr->sa_data))
1605		return -EADDRNOTAVAIL;
1606
1607	eth_hw_addr_set(netdev, addr->sa_data);
1608
1609	if (nic->pdev->msix_enabled) {
1610		if (nicvf_hw_set_mac_addr(nic, netdev))
1611			return -EBUSY;
1612	} else {
1613		nic->set_mac_pending = true;
1614	}
1615
1616	return 0;
1617}
1618
1619void nicvf_update_lmac_stats(struct nicvf *nic)
1620{
1621	int stat = 0;
1622	union nic_mbx mbx = {};
1623
1624	if (!netif_running(nic->netdev))
1625		return;
1626
1627	mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
1628	mbx.bgx_stats.vf_id = nic->vf_id;
1629	/* Rx stats */
1630	mbx.bgx_stats.rx = 1;
1631	while (stat < BGX_RX_STATS_COUNT) {
1632		mbx.bgx_stats.idx = stat;
1633		if (nicvf_send_msg_to_pf(nic, &mbx))
1634			return;
1635		stat++;
1636	}
1637
1638	stat = 0;
1639
1640	/* Tx stats */
1641	mbx.bgx_stats.rx = 0;
1642	while (stat < BGX_TX_STATS_COUNT) {
1643		mbx.bgx_stats.idx = stat;
1644		if (nicvf_send_msg_to_pf(nic, &mbx))
1645			return;
1646		stat++;
1647	}
1648}
1649
1650void nicvf_update_stats(struct nicvf *nic)
1651{
1652	int qidx, cpu;
1653	u64 tmp_stats = 0;
1654	struct nicvf_hw_stats *stats = &nic->hw_stats;
1655	struct nicvf_drv_stats *drv_stats;
1656	struct queue_set *qs = nic->qs;
1657
1658#define GET_RX_STATS(reg) \
1659	nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3))
1660#define GET_TX_STATS(reg) \
1661	nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3))
1662
1663	stats->rx_bytes = GET_RX_STATS(RX_OCTS);
1664	stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
1665	stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
1666	stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
1667	stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
1668	stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
1669	stats->rx_drop_red = GET_RX_STATS(RX_RED);
1670	stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
1671	stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
1672	stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
1673	stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
1674	stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
1675	stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
1676	stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
1677
1678	stats->tx_bytes = GET_TX_STATS(TX_OCTS);
1679	stats->tx_ucast_frames = GET_TX_STATS(TX_UCAST);
1680	stats->tx_bcast_frames = GET_TX_STATS(TX_BCAST);
1681	stats->tx_mcast_frames = GET_TX_STATS(TX_MCAST);
1682	stats->tx_drops = GET_TX_STATS(TX_DROP);
1683
1684	/* On T88 pass 2.0, the dummy SQE added for TSO notification
1685	 * via CQE has 'dont_send' set. Hence HW drops the pkt pointed
1686	 * pointed by dummy SQE and results in tx_drops counter being
1687	 * incremented. Subtracting it from tx_tso counter will give
1688	 * exact tx_drops counter.
1689	 */
1690	if (nic->t88 && nic->hw_tso) {
1691		for_each_possible_cpu(cpu) {
1692			drv_stats = per_cpu_ptr(nic->drv_stats, cpu);
1693			tmp_stats += drv_stats->tx_tso;
1694		}
1695		stats->tx_drops = tmp_stats - stats->tx_drops;
1696	}
1697	stats->tx_frames = stats->tx_ucast_frames +
1698			   stats->tx_bcast_frames +
1699			   stats->tx_mcast_frames;
1700	stats->rx_frames = stats->rx_ucast_frames +
1701			   stats->rx_bcast_frames +
1702			   stats->rx_mcast_frames;
1703	stats->rx_drops = stats->rx_drop_red +
1704			  stats->rx_drop_overrun;
1705
1706	/* Update RQ and SQ stats */
1707	for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1708		nicvf_update_rq_stats(nic, qidx);
1709	for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1710		nicvf_update_sq_stats(nic, qidx);
1711}
1712
1713static void nicvf_get_stats64(struct net_device *netdev,
1714			      struct rtnl_link_stats64 *stats)
1715{
1716	struct nicvf *nic = netdev_priv(netdev);
1717	struct nicvf_hw_stats *hw_stats = &nic->hw_stats;
1718
1719	nicvf_update_stats(nic);
1720
1721	stats->rx_bytes = hw_stats->rx_bytes;
1722	stats->rx_packets = hw_stats->rx_frames;
1723	stats->rx_dropped = hw_stats->rx_drops;
1724	stats->multicast = hw_stats->rx_mcast_frames;
1725
1726	stats->tx_bytes = hw_stats->tx_bytes;
1727	stats->tx_packets = hw_stats->tx_frames;
1728	stats->tx_dropped = hw_stats->tx_drops;
1729
1730}
1731
1732static void nicvf_tx_timeout(struct net_device *dev, unsigned int txqueue)
1733{
1734	struct nicvf *nic = netdev_priv(dev);
1735
1736	netif_warn(nic, tx_err, dev, "Transmit timed out, resetting\n");
1737
1738	this_cpu_inc(nic->drv_stats->tx_timeout);
1739	schedule_work(&nic->reset_task);
1740}
1741
1742static void nicvf_reset_task(struct work_struct *work)
1743{
1744	struct nicvf *nic;
1745
1746	nic = container_of(work, struct nicvf, reset_task);
1747
1748	if (!netif_running(nic->netdev))
1749		return;
1750
1751	nicvf_stop(nic->netdev);
1752	nicvf_open(nic->netdev);
1753	netif_trans_update(nic->netdev);
1754}
1755
1756static int nicvf_config_loopback(struct nicvf *nic,
1757				 netdev_features_t features)
1758{
1759	union nic_mbx mbx = {};
1760
1761	mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK;
1762	mbx.lbk.vf_id = nic->vf_id;
1763	mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0;
1764
1765	return nicvf_send_msg_to_pf(nic, &mbx);
1766}
1767
1768static netdev_features_t nicvf_fix_features(struct net_device *netdev,
1769					    netdev_features_t features)
1770{
1771	struct nicvf *nic = netdev_priv(netdev);
1772
1773	if ((features & NETIF_F_LOOPBACK) &&
1774	    netif_running(netdev) && !nic->loopback_supported)
1775		features &= ~NETIF_F_LOOPBACK;
1776
1777	return features;
1778}
1779
1780static int nicvf_set_features(struct net_device *netdev,
1781			      netdev_features_t features)
1782{
1783	struct nicvf *nic = netdev_priv(netdev);
1784	netdev_features_t changed = features ^ netdev->features;
1785
1786	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1787		nicvf_config_vlan_stripping(nic, features);
1788
1789	if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
1790		return nicvf_config_loopback(nic, features);
1791
1792	return 0;
1793}
1794
1795static void nicvf_set_xdp_queues(struct nicvf *nic, bool bpf_attached)
1796{
1797	u8 cq_count, txq_count;
1798
1799	/* Set XDP Tx queue count same as Rx queue count */
1800	if (!bpf_attached)
1801		nic->xdp_tx_queues = 0;
1802	else
1803		nic->xdp_tx_queues = nic->rx_queues;
1804
1805	/* If queue count > MAX_CMP_QUEUES_PER_QS, then additional qsets
1806	 * needs to be allocated, check how many.
1807	 */
1808	txq_count = nic->xdp_tx_queues + nic->tx_queues;
1809	cq_count = max(nic->rx_queues, txq_count);
1810	if (cq_count > MAX_CMP_QUEUES_PER_QS) {
1811		nic->sqs_count = roundup(cq_count, MAX_CMP_QUEUES_PER_QS);
1812		nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1;
1813	} else {
1814		nic->sqs_count = 0;
1815	}
1816
1817	/* Set primary Qset's resources */
1818	nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS);
1819	nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS);
1820	nic->qs->cq_cnt = max_t(u8, nic->qs->rq_cnt, nic->qs->sq_cnt);
1821
1822	/* Update stack */
1823	nicvf_set_real_num_queues(nic->netdev, nic->tx_queues, nic->rx_queues);
1824}
1825
1826static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog)
1827{
1828	struct net_device *dev = nic->netdev;
1829	bool if_up = netif_running(nic->netdev);
1830	struct bpf_prog *old_prog;
1831	bool bpf_attached = false;
1832	int ret = 0;
1833
1834	/* For now just support only the usual MTU sized frames,
1835	 * plus some headroom for VLAN, QinQ.
1836	 */
1837	if (prog && dev->mtu > MAX_XDP_MTU) {
1838		netdev_warn(dev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
1839			    dev->mtu);
1840		return -EOPNOTSUPP;
1841	}
1842
1843	/* ALL SQs attached to CQs i.e same as RQs, are treated as
1844	 * XDP Tx queues and more Tx queues are allocated for
1845	 * network stack to send pkts out.
1846	 *
1847	 * No of Tx queues are either same as Rx queues or whatever
1848	 * is left in max no of queues possible.
1849	 */
1850	if ((nic->rx_queues + nic->tx_queues) > nic->max_queues) {
1851		netdev_warn(dev,
1852			    "Failed to attach BPF prog, RXQs + TXQs > Max %d\n",
1853			    nic->max_queues);
1854		return -ENOMEM;
1855	}
1856
1857	if (if_up)
1858		nicvf_stop(nic->netdev);
1859
1860	old_prog = xchg(&nic->xdp_prog, prog);
1861	/* Detach old prog, if any */
1862	if (old_prog)
1863		bpf_prog_put(old_prog);
1864
1865	if (nic->xdp_prog) {
1866		/* Attach BPF program */
1867		bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
1868		bpf_attached = true;
1869	}
1870
1871	/* Calculate Tx queues needed for XDP and network stack */
1872	nicvf_set_xdp_queues(nic, bpf_attached);
1873
1874	if (if_up) {
1875		/* Reinitialize interface, clean slate */
1876		nicvf_open(nic->netdev);
1877		netif_trans_update(nic->netdev);
1878	}
1879
1880	return ret;
1881}
1882
1883static int nicvf_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
1884{
1885	struct nicvf *nic = netdev_priv(netdev);
1886
1887	/* To avoid checks while retrieving buffer address from CQE_RX,
1888	 * do not support XDP for T88 pass1.x silicons which are anyway
1889	 * not in use widely.
1890	 */
1891	if (pass1_silicon(nic->pdev))
1892		return -EOPNOTSUPP;
1893
1894	switch (xdp->command) {
1895	case XDP_SETUP_PROG:
1896		return nicvf_xdp_setup(nic, xdp->prog);
1897	default:
1898		return -EINVAL;
1899	}
1900}
1901
1902static int nicvf_hwtstamp_set(struct net_device *netdev,
1903			      struct kernel_hwtstamp_config *config,
1904			      struct netlink_ext_ack *extack)
1905{
1906	struct nicvf *nic = netdev_priv(netdev);
1907
1908	if (!nic->ptp_clock) {
1909		NL_SET_ERR_MSG_MOD(extack, "HW timestamping is not supported");
1910		return -ENODEV;
1911	}
1912
1913	switch (config->tx_type) {
1914	case HWTSTAMP_TX_OFF:
1915	case HWTSTAMP_TX_ON:
1916		break;
1917	default:
1918		return -ERANGE;
1919	}
1920
1921	switch (config->rx_filter) {
1922	case HWTSTAMP_FILTER_NONE:
1923		nic->hw_rx_tstamp = false;
1924		break;
1925	case HWTSTAMP_FILTER_ALL:
1926	case HWTSTAMP_FILTER_SOME:
1927	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1928	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1929	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1930	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1931	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1932	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1933	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1934	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1935	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1936	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1937	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1938	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1939		nic->hw_rx_tstamp = true;
1940		config->rx_filter = HWTSTAMP_FILTER_ALL;
1941		break;
1942	default:
1943		return -ERANGE;
1944	}
1945
1946	if (netif_running(netdev))
1947		nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp);
1948
1949	return 0;
1950}
1951
1952static int nicvf_hwtstamp_get(struct net_device *netdev,
1953			      struct kernel_hwtstamp_config *config)
1954{
1955	struct nicvf *nic = netdev_priv(netdev);
1956
1957	if (!nic->ptp_clock)
1958		return -ENODEV;
1959
1960	/* TX timestamping is technically always on */
1961	config->tx_type = HWTSTAMP_TX_ON;
1962	config->rx_filter = nic->hw_rx_tstamp ?
1963			    HWTSTAMP_FILTER_ALL :
1964			    HWTSTAMP_FILTER_NONE;
1965
1966	return 0;
1967}
1968
1969static void __nicvf_set_rx_mode_task(u8 mode, struct xcast_addr_list *mc_addrs,
1970				     struct nicvf *nic)
1971{
1972	union nic_mbx mbx = {};
1973	int idx;
1974
1975	/* From the inside of VM code flow we have only 128 bits memory
1976	 * available to send message to host's PF, so send all mc addrs
1977	 * one by one, starting from flush command in case if kernel
1978	 * requests to configure specific MAC filtering
1979	 */
1980
1981	/* flush DMAC filters and reset RX mode */
1982	mbx.xcast.msg = NIC_MBOX_MSG_RESET_XCAST;
1983	if (nicvf_send_msg_to_pf(nic, &mbx) < 0)
1984		goto free_mc;
1985
1986	if (mode & BGX_XCAST_MCAST_FILTER) {
1987		/* once enabling filtering, we need to signal to PF to add
1988		 * its' own LMAC to the filter to accept packets for it.
1989		 */
1990		mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST;
1991		mbx.xcast.mac = 0;
1992		if (nicvf_send_msg_to_pf(nic, &mbx) < 0)
1993			goto free_mc;
1994	}
1995
1996	/* check if we have any specific MACs to be added to PF DMAC filter */
1997	if (mc_addrs) {
1998		/* now go through kernel list of MACs and add them one by one */
1999		for (idx = 0; idx < mc_addrs->count; idx++) {
2000			mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST;
2001			mbx.xcast.mac = mc_addrs->mc[idx];
2002			if (nicvf_send_msg_to_pf(nic, &mbx) < 0)
2003				goto free_mc;
2004		}
2005	}
2006
2007	/* and finally set rx mode for PF accordingly */
2008	mbx.xcast.msg = NIC_MBOX_MSG_SET_XCAST;
2009	mbx.xcast.mode = mode;
2010
2011	nicvf_send_msg_to_pf(nic, &mbx);
2012free_mc:
2013	kfree(mc_addrs);
2014}
2015
2016static void nicvf_set_rx_mode_task(struct work_struct *work_arg)
2017{
2018	struct nicvf_work *vf_work = container_of(work_arg, struct nicvf_work,
2019						  work);
2020	struct nicvf *nic = container_of(vf_work, struct nicvf, rx_mode_work);
2021	u8 mode;
2022	struct xcast_addr_list *mc;
2023
2024	/* Save message data locally to prevent them from
2025	 * being overwritten by next ndo_set_rx_mode call().
2026	 */
2027	spin_lock_bh(&nic->rx_mode_wq_lock);
2028	mode = vf_work->mode;
2029	mc = vf_work->mc;
2030	vf_work->mc = NULL;
2031	spin_unlock_bh(&nic->rx_mode_wq_lock);
2032
2033	__nicvf_set_rx_mode_task(mode, mc, nic);
2034}
2035
2036static void nicvf_set_rx_mode(struct net_device *netdev)
2037{
2038	struct nicvf *nic = netdev_priv(netdev);
2039	struct netdev_hw_addr *ha;
2040	struct xcast_addr_list *mc_list = NULL;
2041	u8 mode = 0;
2042
2043	if (netdev->flags & IFF_PROMISC) {
2044		mode = BGX_XCAST_BCAST_ACCEPT | BGX_XCAST_MCAST_ACCEPT;
2045	} else {
2046		if (netdev->flags & IFF_BROADCAST)
2047			mode |= BGX_XCAST_BCAST_ACCEPT;
2048
2049		if (netdev->flags & IFF_ALLMULTI) {
2050			mode |= BGX_XCAST_MCAST_ACCEPT;
2051		} else if (netdev->flags & IFF_MULTICAST) {
2052			mode |= BGX_XCAST_MCAST_FILTER;
2053			/* here we need to copy mc addrs */
2054			if (netdev_mc_count(netdev)) {
2055				mc_list = kmalloc(struct_size(mc_list, mc,
2056							      netdev_mc_count(netdev)),
2057						  GFP_ATOMIC);
2058				if (unlikely(!mc_list))
2059					return;
2060				mc_list->count = 0;
2061				netdev_hw_addr_list_for_each(ha, &netdev->mc) {
2062					mc_list->mc[mc_list->count] =
2063						ether_addr_to_u64(ha->addr);
2064					mc_list->count++;
2065				}
2066			}
2067		}
2068	}
2069	spin_lock(&nic->rx_mode_wq_lock);
2070	kfree(nic->rx_mode_work.mc);
2071	nic->rx_mode_work.mc = mc_list;
2072	nic->rx_mode_work.mode = mode;
2073	queue_work(nic->nicvf_rx_mode_wq, &nic->rx_mode_work.work);
2074	spin_unlock(&nic->rx_mode_wq_lock);
2075}
2076
2077static const struct net_device_ops nicvf_netdev_ops = {
2078	.ndo_open		= nicvf_open,
2079	.ndo_stop		= nicvf_stop,
2080	.ndo_start_xmit		= nicvf_xmit,
2081	.ndo_change_mtu		= nicvf_change_mtu,
2082	.ndo_set_mac_address	= nicvf_set_mac_address,
2083	.ndo_get_stats64	= nicvf_get_stats64,
2084	.ndo_tx_timeout         = nicvf_tx_timeout,
2085	.ndo_fix_features       = nicvf_fix_features,
2086	.ndo_set_features       = nicvf_set_features,
2087	.ndo_bpf		= nicvf_xdp,
2088	.ndo_set_rx_mode        = nicvf_set_rx_mode,
2089	.ndo_hwtstamp_get	= nicvf_hwtstamp_get,
2090	.ndo_hwtstamp_set	= nicvf_hwtstamp_set,
2091};
2092
2093static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2094{
2095	struct device *dev = &pdev->dev;
2096	struct net_device *netdev;
2097	struct nicvf *nic;
2098	int    err, qcount;
2099	u16    sdevid;
2100	struct cavium_ptp *ptp_clock;
2101
2102	ptp_clock = cavium_ptp_get();
2103	if (IS_ERR(ptp_clock)) {
2104		if (PTR_ERR(ptp_clock) == -ENODEV)
2105			/* In virtualized environment we proceed without ptp */
2106			ptp_clock = NULL;
2107		else
2108			return PTR_ERR(ptp_clock);
2109	}
2110
2111	err = pci_enable_device(pdev);
2112	if (err)
2113		return dev_err_probe(dev, err, "Failed to enable PCI device\n");
2114
2115	err = pci_request_regions(pdev, DRV_NAME);
2116	if (err) {
2117		dev_err(dev, "PCI request regions failed 0x%x\n", err);
2118		goto err_disable_device;
2119	}
2120
2121	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
2122	if (err) {
2123		dev_err(dev, "Unable to get usable DMA configuration\n");
2124		goto err_release_regions;
2125	}
2126
2127	qcount = netif_get_num_default_rss_queues();
2128
2129	/* Restrict multiqset support only for host bound VFs */
2130	if (pdev->is_virtfn) {
2131		/* Set max number of queues per VF */
2132		qcount = min_t(int, num_online_cpus(),
2133			       (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
2134	}
2135
2136	netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
2137	if (!netdev) {
2138		err = -ENOMEM;
2139		goto err_release_regions;
2140	}
2141
2142	pci_set_drvdata(pdev, netdev);
2143
2144	SET_NETDEV_DEV(netdev, &pdev->dev);
2145
2146	nic = netdev_priv(netdev);
2147	nic->netdev = netdev;
2148	nic->pdev = pdev;
2149	nic->pnicvf = nic;
2150	nic->max_queues = qcount;
2151	/* If no of CPUs are too low, there won't be any queues left
2152	 * for XDP_TX, hence double it.
2153	 */
2154	if (!nic->t88)
2155		nic->max_queues *= 2;
2156	nic->ptp_clock = ptp_clock;
2157
2158	/* Initialize mutex that serializes usage of VF's mailbox */
2159	mutex_init(&nic->rx_mode_mtx);
2160
2161	/* MAP VF's configuration registers */
2162	nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
2163	if (!nic->reg_base) {
2164		dev_err(dev, "Cannot map config register space, aborting\n");
2165		err = -ENOMEM;
2166		goto err_free_netdev;
2167	}
2168
2169	nic->drv_stats = netdev_alloc_pcpu_stats(struct nicvf_drv_stats);
2170	if (!nic->drv_stats) {
2171		err = -ENOMEM;
2172		goto err_free_netdev;
2173	}
2174
2175	err = nicvf_set_qset_resources(nic);
2176	if (err)
2177		goto err_free_netdev;
2178
2179	/* Check if PF is alive and get MAC address for this VF */
2180	err = nicvf_register_misc_interrupt(nic);
2181	if (err)
2182		goto err_free_netdev;
2183
2184	nicvf_send_vf_struct(nic);
2185
2186	if (!pass1_silicon(nic->pdev))
2187		nic->hw_tso = true;
2188
2189	/* Get iommu domain for iova to physical addr conversion */
2190	nic->iommu_domain = iommu_get_domain_for_dev(dev);
2191
2192	pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
2193	if (sdevid == 0xA134)
2194		nic->t88 = true;
2195
2196	/* Check if this VF is in QS only mode */
2197	if (nic->sqs_mode)
2198		return 0;
2199
2200	err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues);
2201	if (err)
2202		goto err_unregister_interrupts;
2203
2204	netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_SG |
2205			       NETIF_F_TSO | NETIF_F_GRO | NETIF_F_TSO6 |
2206			       NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2207			       NETIF_F_HW_VLAN_CTAG_RX);
2208
2209	netdev->hw_features |= NETIF_F_RXHASH;
2210
2211	netdev->features |= netdev->hw_features;
2212	netdev->hw_features |= NETIF_F_LOOPBACK;
2213
2214	netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM |
2215				NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6;
2216
2217	netdev->netdev_ops = &nicvf_netdev_ops;
2218	netdev->watchdog_timeo = NICVF_TX_TIMEOUT;
2219
2220	if (!pass1_silicon(nic->pdev) &&
2221	    nic->rx_queues + nic->tx_queues <= nic->max_queues)
2222		netdev->xdp_features = NETDEV_XDP_ACT_BASIC;
2223
2224	/* MTU range: 64 - 9200 */
2225	netdev->min_mtu = NIC_HW_MIN_FRS;
2226	netdev->max_mtu = NIC_HW_MAX_FRS;
2227
2228	INIT_WORK(&nic->reset_task, nicvf_reset_task);
2229
2230	nic->nicvf_rx_mode_wq = alloc_ordered_workqueue("nicvf_rx_mode_wq_VF%d",
2231							WQ_MEM_RECLAIM,
2232							nic->vf_id);
2233	if (!nic->nicvf_rx_mode_wq) {
2234		err = -ENOMEM;
2235		dev_err(dev, "Failed to allocate work queue\n");
2236		goto err_unregister_interrupts;
2237	}
2238
2239	INIT_WORK(&nic->rx_mode_work.work, nicvf_set_rx_mode_task);
2240	spin_lock_init(&nic->rx_mode_wq_lock);
2241
2242	err = register_netdev(netdev);
2243	if (err) {
2244		dev_err(dev, "Failed to register netdevice\n");
2245		goto err_destroy_workqueue;
2246	}
2247
2248	nic->msg_enable = debug;
2249
2250	nicvf_set_ethtool_ops(netdev);
2251
2252	return 0;
2253
2254err_destroy_workqueue:
2255	destroy_workqueue(nic->nicvf_rx_mode_wq);
2256err_unregister_interrupts:
2257	nicvf_unregister_interrupts(nic);
2258err_free_netdev:
2259	pci_set_drvdata(pdev, NULL);
2260	if (nic->drv_stats)
2261		free_percpu(nic->drv_stats);
2262	free_netdev(netdev);
2263err_release_regions:
2264	pci_release_regions(pdev);
2265err_disable_device:
2266	pci_disable_device(pdev);
2267	return err;
2268}
2269
2270static void nicvf_remove(struct pci_dev *pdev)
2271{
2272	struct net_device *netdev = pci_get_drvdata(pdev);
2273	struct nicvf *nic;
2274	struct net_device *pnetdev;
2275
2276	if (!netdev)
2277		return;
2278
2279	nic = netdev_priv(netdev);
2280	pnetdev = nic->pnicvf->netdev;
2281
2282	/* Check if this Qset is assigned to different VF.
2283	 * If yes, clean primary and all secondary Qsets.
2284	 */
2285	if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED))
2286		unregister_netdev(pnetdev);
2287	if (nic->nicvf_rx_mode_wq) {
2288		destroy_workqueue(nic->nicvf_rx_mode_wq);
2289		nic->nicvf_rx_mode_wq = NULL;
2290	}
2291	nicvf_unregister_interrupts(nic);
2292	pci_set_drvdata(pdev, NULL);
2293	if (nic->drv_stats)
2294		free_percpu(nic->drv_stats);
2295	cavium_ptp_put(nic->ptp_clock);
2296	free_netdev(netdev);
2297	pci_release_regions(pdev);
2298	pci_disable_device(pdev);
2299}
2300
2301static void nicvf_shutdown(struct pci_dev *pdev)
2302{
2303	nicvf_remove(pdev);
2304}
2305
2306static struct pci_driver nicvf_driver = {
2307	.name = DRV_NAME,
2308	.id_table = nicvf_id_table,
2309	.probe = nicvf_probe,
2310	.remove = nicvf_remove,
2311	.shutdown = nicvf_shutdown,
2312};
2313
2314static int __init nicvf_init_module(void)
2315{
2316	pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
2317	return pci_register_driver(&nicvf_driver);
2318}
2319
2320static void __exit nicvf_cleanup_module(void)
2321{
2322	pci_unregister_driver(&nicvf_driver);
2323}
2324
2325module_init(nicvf_init_module);
2326module_exit(nicvf_cleanup_module);