octep_vf_main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Marvell Octeon EP (EndPoint) VF Ethernet Driver
   3 *
   4 * Copyright (C) 2020 Marvell.
   5 *
   6 */
   7
   8#include <linux/types.h>
   9#include <linux/module.h>
  10#include <linux/pci.h>
  11#include <linux/aer.h>
  12#include <linux/netdevice.h>
  13#include <linux/etherdevice.h>
  14#include <linux/rtnetlink.h>
  15#include <linux/vmalloc.h>
  16#include <net/netdev_queues.h>
  17
  18#include "octep_vf_config.h"
  19#include "octep_vf_main.h"
  20
  21/* Supported Devices */
  22static const struct pci_device_id octep_vf_pci_id_tbl[] = {
  23	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_VF)},
  24	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_VF)},
  25	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_VF)},
  26	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_VF)},
  27	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_VF)},
  28	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_VF)},
  29	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_VF)},
  30	{0, },
  31};
  32MODULE_DEVICE_TABLE(pci, octep_vf_pci_id_tbl);
  33
  34MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
  35MODULE_DESCRIPTION(OCTEP_VF_DRV_STRING);
  36MODULE_LICENSE("GPL");
  37
  38/**
  39 * octep_vf_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
  40 *
  41 * @oct: Octeon device private data structure.
  42 *
  43 * Allocate resources to hold per Tx/Rx queue interrupt info.
  44 * This is the information passed to interrupt handler, from which napi poll
  45 * is scheduled and includes quick access to private data of Tx/Rx queue
  46 * corresponding to the interrupt being handled.
  47 *
  48 * Return: 0, on successful allocation of resources for all queue interrupts.
  49 *         -1, if failed to allocate any resource.
  50 */
  51static int octep_vf_alloc_ioq_vectors(struct octep_vf_device *oct)
  52{
  53	struct octep_vf_ioq_vector *ioq_vector;
  54	int i;
  55
  56	for (i = 0; i < oct->num_oqs; i++) {
  57		oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
  58		if (!oct->ioq_vector[i])
  59			goto free_ioq_vector;
  60
  61		ioq_vector = oct->ioq_vector[i];
  62		ioq_vector->iq = oct->iq[i];
  63		ioq_vector->oq = oct->oq[i];
  64		ioq_vector->octep_vf_dev = oct;
  65	}
  66
  67	dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
  68	return 0;
  69
  70free_ioq_vector:
  71	while (i) {
  72		i--;
  73		vfree(oct->ioq_vector[i]);
  74		oct->ioq_vector[i] = NULL;
  75	}
  76	return -1;
  77}
  78
  79/**
  80 * octep_vf_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
  81 *
  82 * @oct: Octeon device private data structure.
  83 */
  84static void octep_vf_free_ioq_vectors(struct octep_vf_device *oct)
  85{
  86	int i;
  87
  88	for (i = 0; i < oct->num_oqs; i++) {
  89		if (oct->ioq_vector[i]) {
  90			vfree(oct->ioq_vector[i]);
  91			oct->ioq_vector[i] = NULL;
  92		}
  93	}
  94	netdev_info(oct->netdev, "Freed IOQ Vectors\n");
  95}
  96
  97/**
  98 * octep_vf_enable_msix_range() - enable MSI-x interrupts.
  99 *
 100 * @oct: Octeon device private data structure.
 101 *
 102 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
 103 * for the Octeon device.
 104 *
 105 * Return: 0, on successfully enabling all MSI-x interrupts.
 106 *         -1, if failed to enable any MSI-x interrupt.
 107 */
 108static int octep_vf_enable_msix_range(struct octep_vf_device *oct)
 109{
 110	int num_msix, msix_allocated;
 111	int i;
 112
 113	/* Generic interrupts apart from input/output queues */
 114	//num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
 115	num_msix = oct->num_oqs;
 116	oct->msix_entries = kcalloc(num_msix, sizeof(struct msix_entry), GFP_KERNEL);
 117	if (!oct->msix_entries)
 118		goto msix_alloc_err;
 119
 120	for (i = 0; i < num_msix; i++)
 121		oct->msix_entries[i].entry = i;
 122
 123	msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
 124					       num_msix, num_msix);
 125	if (msix_allocated != num_msix) {
 126		dev_err(&oct->pdev->dev,
 127			"Failed to enable %d msix irqs; got only %d\n",
 128			num_msix, msix_allocated);
 129		goto enable_msix_err;
 130	}
 131	oct->num_irqs = msix_allocated;
 132	dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
 133
 134	return 0;
 135
 136enable_msix_err:
 137	if (msix_allocated > 0)
 138		pci_disable_msix(oct->pdev);
 139	kfree(oct->msix_entries);
 140	oct->msix_entries = NULL;
 141msix_alloc_err:
 142	return -1;
 143}
 144
 145/**
 146 * octep_vf_disable_msix() - disable MSI-x interrupts.
 147 *
 148 * @oct: Octeon device private data structure.
 149 *
 150 * Disable MSI-x on the Octeon device.
 151 */
 152static void octep_vf_disable_msix(struct octep_vf_device *oct)
 153{
 154	pci_disable_msix(oct->pdev);
 155	kfree(oct->msix_entries);
 156	oct->msix_entries = NULL;
 157	dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
 158}
 159
 160/**
 161 * octep_vf_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
 162 *
 163 * @irq: Interrupt number.
 164 * @data: interrupt data contains pointers to Tx/Rx queue private data
 165 *         and correspong NAPI context.
 166 *
 167 * this is common handler for all non-queue (generic) interrupts.
 168 */
 169static irqreturn_t octep_vf_ioq_intr_handler(int irq, void *data)
 170{
 171	struct octep_vf_ioq_vector *ioq_vector = data;
 172	struct octep_vf_device *oct = ioq_vector->octep_vf_dev;
 173
 174	return oct->hw_ops.ioq_intr_handler(ioq_vector);
 175}
 176
 177/**
 178 * octep_vf_request_irqs() - Register interrupt handlers.
 179 *
 180 * @oct: Octeon device private data structure.
 181 *
 182 * Register handlers for all queue and non-queue interrupts.
 183 *
 184 * Return: 0, on successful registration of all interrupt handlers.
 185 *         -1, on any error.
 186 */
 187static int octep_vf_request_irqs(struct octep_vf_device *oct)
 188{
 189	struct net_device *netdev = oct->netdev;
 190	struct octep_vf_ioq_vector *ioq_vector;
 191	struct msix_entry *msix_entry;
 192	int ret, i;
 193
 194	/* Request IRQs for Tx/Rx queues */
 195	for (i = 0; i < oct->num_oqs; i++) {
 196		ioq_vector = oct->ioq_vector[i];
 197		msix_entry = &oct->msix_entries[i];
 198
 199		snprintf(ioq_vector->name, sizeof(ioq_vector->name),
 200			 "%s-q%d", netdev->name, i);
 201		ret = request_irq(msix_entry->vector,
 202				  octep_vf_ioq_intr_handler, 0,
 203				  ioq_vector->name, ioq_vector);
 204		if (ret) {
 205			netdev_err(netdev,
 206				   "request_irq failed for Q-%d; err=%d",
 207				   i, ret);
 208			goto ioq_irq_err;
 209		}
 210
 211		cpumask_set_cpu(i % num_online_cpus(),
 212				&ioq_vector->affinity_mask);
 213		irq_set_affinity_hint(msix_entry->vector,
 214				      &ioq_vector->affinity_mask);
 215	}
 216
 217	return 0;
 218ioq_irq_err:
 219	while (i) {
 220		--i;
 221		free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]);
 222	}
 223	return -1;
 224}
 225
 226/**
 227 * octep_vf_free_irqs() - free all registered interrupts.
 228 *
 229 * @oct: Octeon device private data structure.
 230 *
 231 * Free all queue and non-queue interrupts of the Octeon device.
 232 */
 233static void octep_vf_free_irqs(struct octep_vf_device *oct)
 234{
 235	int i;
 236
 237	for (i = 0; i < oct->num_irqs; i++) {
 238		irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
 239		free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]);
 240	}
 241	netdev_info(oct->netdev, "IRQs freed\n");
 242}
 243
 244/**
 245 * octep_vf_setup_irqs() - setup interrupts for the Octeon device.
 246 *
 247 * @oct: Octeon device private data structure.
 248 *
 249 * Allocate data structures to hold per interrupt information, allocate/enable
 250 * MSI-x interrupt and register interrupt handlers.
 251 *
 252 * Return: 0, on successful allocation and registration of all interrupts.
 253 *         -1, on any error.
 254 */
 255static int octep_vf_setup_irqs(struct octep_vf_device *oct)
 256{
 257	if (octep_vf_alloc_ioq_vectors(oct))
 258		goto ioq_vector_err;
 259
 260	if (octep_vf_enable_msix_range(oct))
 261		goto enable_msix_err;
 262
 263	if (octep_vf_request_irqs(oct))
 264		goto request_irq_err;
 265
 266	return 0;
 267
 268request_irq_err:
 269	octep_vf_disable_msix(oct);
 270enable_msix_err:
 271	octep_vf_free_ioq_vectors(oct);
 272ioq_vector_err:
 273	return -1;
 274}
 275
 276/**
 277 * octep_vf_clean_irqs() - free all interrupts and its resources.
 278 *
 279 * @oct: Octeon device private data structure.
 280 */
 281static void octep_vf_clean_irqs(struct octep_vf_device *oct)
 282{
 283	octep_vf_free_irqs(oct);
 284	octep_vf_disable_msix(oct);
 285	octep_vf_free_ioq_vectors(oct);
 286}
 287
 288/**
 289 * octep_vf_update_pkt() - Update IQ/OQ IN/OUT_CNT registers.
 290 *
 291 * @iq: Octeon Tx queue data structure.
 292 * @oq: Octeon Rx queue data structure.
 293 */
 294
 295static void octep_vf_update_pkt(struct octep_vf_iq *iq, struct octep_vf_oq *oq)
 296{
 297	u32 pkts_pend = READ_ONCE(oq->pkts_pending);
 298	u32 last_pkt_count = READ_ONCE(oq->last_pkt_count);
 299	u32 pkts_processed = READ_ONCE(iq->pkts_processed);
 300	u32 pkt_in_done = READ_ONCE(iq->pkt_in_done);
 301
 302	netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
 303	if (pkts_processed) {
 304		writel(pkts_processed, iq->inst_cnt_reg);
 305		readl(iq->inst_cnt_reg);
 306		WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed));
 307		WRITE_ONCE(iq->pkts_processed, 0);
 308	}
 309	if (last_pkt_count - pkts_pend) {
 310		writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg);
 311		readl(oq->pkts_sent_reg);
 312		WRITE_ONCE(oq->last_pkt_count, pkts_pend);
 313	}
 314
 315	/* Flush the previous wrties before writing to RESEND bit */
 316	smp_wmb();
 317}
 318
 319/**
 320 * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
 321 *
 322 * @iq: Octeon Tx queue data structure.
 323 * @oq: Octeon Rx queue data structure.
 324 */
 325static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq,
 326				    struct octep_vf_oq *oq)
 327{
 328	writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
 329	writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
 330}
 331
 332/**
 333 * octep_vf_napi_poll() - NAPI poll function for Tx/Rx.
 334 *
 335 * @napi: pointer to napi context.
 336 * @budget: max number of packets to be processed in single invocation.
 337 */
 338static int octep_vf_napi_poll(struct napi_struct *napi, int budget)
 339{
 340	struct octep_vf_ioq_vector *ioq_vector =
 341		container_of(napi, struct octep_vf_ioq_vector, napi);
 342	u32 tx_pending, rx_done;
 343
 344	tx_pending = octep_vf_iq_process_completions(ioq_vector->iq, 64);
 345	rx_done = octep_vf_oq_process_rx(ioq_vector->oq, budget);
 346
 347	/* need more polling if tx completion processing is still pending or
 348	 * processed at least 'budget' number of rx packets.
 349	 */
 350	if (tx_pending || rx_done >= budget)
 351		return budget;
 352
 353	octep_vf_update_pkt(ioq_vector->iq, ioq_vector->oq);
 354	if (likely(napi_complete_done(napi, rx_done)))
 355		octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
 356
 357	return rx_done;
 358}
 359
 360/**
 361 * octep_vf_napi_add() - Add NAPI poll for all Tx/Rx queues.
 362 *
 363 * @oct: Octeon device private data structure.
 364 */
 365static void octep_vf_napi_add(struct octep_vf_device *oct)
 366{
 367	int i;
 368
 369	for (i = 0; i < oct->num_oqs; i++) {
 370		netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
 371		netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, octep_vf_napi_poll);
 372		oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
 373	}
 374}
 375
 376/**
 377 * octep_vf_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
 378 *
 379 * @oct: Octeon device private data structure.
 380 */
 381static void octep_vf_napi_delete(struct octep_vf_device *oct)
 382{
 383	int i;
 384
 385	for (i = 0; i < oct->num_oqs; i++) {
 386		netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
 387		netif_napi_del(&oct->ioq_vector[i]->napi);
 388		oct->oq[i]->napi = NULL;
 389	}
 390}
 391
 392/**
 393 * octep_vf_napi_enable() - enable NAPI for all Tx/Rx queues.
 394 *
 395 * @oct: Octeon device private data structure.
 396 */
 397static void octep_vf_napi_enable(struct octep_vf_device *oct)
 398{
 399	int i;
 400
 401	for (i = 0; i < oct->num_oqs; i++) {
 402		netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
 403		napi_enable(&oct->ioq_vector[i]->napi);
 404	}
 405}
 406
 407/**
 408 * octep_vf_napi_disable() - disable NAPI for all Tx/Rx queues.
 409 *
 410 * @oct: Octeon device private data structure.
 411 */
 412static void octep_vf_napi_disable(struct octep_vf_device *oct)
 413{
 414	int i;
 415
 416	for (i = 0; i < oct->num_oqs; i++) {
 417		netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
 418		napi_disable(&oct->ioq_vector[i]->napi);
 419	}
 420}
 421
 422static void octep_vf_link_up(struct net_device *netdev)
 423{
 424	netif_carrier_on(netdev);
 425	netif_tx_start_all_queues(netdev);
 426}
 427
 428static void octep_vf_set_rx_state(struct octep_vf_device *oct, bool up)
 429{
 430	int err;
 431
 432	err = octep_vf_mbox_set_rx_state(oct, up);
 433	if (err)
 434		netdev_err(oct->netdev, "Set Rx state to %d failed with err:%d\n", up, err);
 435}
 436
 437static int octep_vf_get_link_status(struct octep_vf_device *oct)
 438{
 439	int err;
 440
 441	err = octep_vf_mbox_get_link_status(oct, &oct->link_info.oper_up);
 442	if (err)
 443		netdev_err(oct->netdev, "Get link status failed with err:%d\n", err);
 444	return oct->link_info.oper_up;
 445}
 446
 447static void octep_vf_set_link_status(struct octep_vf_device *oct, bool up)
 448{
 449	int err;
 450
 451	err = octep_vf_mbox_set_link_status(oct, up);
 452	if (err) {
 453		netdev_err(oct->netdev, "Set link status to %d failed with err:%d\n", up, err);
 454		return;
 455	}
 456	oct->link_info.oper_up = up;
 457}
 458
 459/**
 460 * octep_vf_open() - start the octeon network device.
 461 *
 462 * @netdev: pointer to kernel network device.
 463 *
 464 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
 465 * and interrupts..
 466 *
 467 * Return: 0, on successfully setting up device and bring it up.
 468 *         -1, on any error.
 469 */
 470static int octep_vf_open(struct net_device *netdev)
 471{
 472	struct octep_vf_device *oct = netdev_priv(netdev);
 473	int err, ret;
 474
 475	netdev_info(netdev, "Starting netdev ...\n");
 476	netif_carrier_off(netdev);
 477
 478	oct->hw_ops.reset_io_queues(oct);
 479
 480	if (octep_vf_setup_iqs(oct))
 481		goto setup_iq_err;
 482	if (octep_vf_setup_oqs(oct))
 483		goto setup_oq_err;
 484	if (octep_vf_setup_irqs(oct))
 485		goto setup_irq_err;
 486
 487	err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
 488	if (err)
 489		goto set_queues_err;
 490	err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
 491	if (err)
 492		goto set_queues_err;
 493
 494	octep_vf_napi_add(oct);
 495	octep_vf_napi_enable(oct);
 496
 497	oct->link_info.admin_up = 1;
 498	octep_vf_set_rx_state(oct, true);
 499
 500	ret = octep_vf_get_link_status(oct);
 501	if (!ret)
 502		octep_vf_set_link_status(oct, true);
 503
 504	/* Enable the input and output queues for this Octeon device */
 505	oct->hw_ops.enable_io_queues(oct);
 506
 507	/* Enable Octeon device interrupts */
 508	oct->hw_ops.enable_interrupts(oct);
 509
 510	octep_vf_oq_dbell_init(oct);
 511
 512	ret = octep_vf_get_link_status(oct);
 513	if (ret)
 514		octep_vf_link_up(netdev);
 515
 516	return 0;
 517
 518set_queues_err:
 519	octep_vf_napi_disable(oct);
 520	octep_vf_napi_delete(oct);
 521	octep_vf_clean_irqs(oct);
 522setup_irq_err:
 523	octep_vf_free_oqs(oct);
 524setup_oq_err:
 525	octep_vf_free_iqs(oct);
 526setup_iq_err:
 527	return -1;
 528}
 529
 530/**
 531 * octep_vf_stop() - stop the octeon network device.
 532 *
 533 * @netdev: pointer to kernel network device.
 534 *
 535 * stop the device Tx/Rx operations, bring down the link and
 536 * free up all resources allocated for Tx/Rx queues and interrupts.
 537 */
 538static int octep_vf_stop(struct net_device *netdev)
 539{
 540	struct octep_vf_device *oct = netdev_priv(netdev);
 541
 542	netdev_info(netdev, "Stopping the device ...\n");
 543
 544	/* Stop Tx from stack */
 545	netif_carrier_off(netdev);
 546	netif_tx_disable(netdev);
 547
 548	octep_vf_set_link_status(oct, false);
 549	octep_vf_set_rx_state(oct, false);
 550
 551	oct->link_info.admin_up = 0;
 552	oct->link_info.oper_up = 0;
 553
 554	oct->hw_ops.disable_interrupts(oct);
 555	octep_vf_napi_disable(oct);
 556	octep_vf_napi_delete(oct);
 557
 558	octep_vf_clean_irqs(oct);
 559	octep_vf_clean_iqs(oct);
 560
 561	oct->hw_ops.disable_io_queues(oct);
 562	oct->hw_ops.reset_io_queues(oct);
 563	octep_vf_free_oqs(oct);
 564	octep_vf_free_iqs(oct);
 565	netdev_info(netdev, "Device stopped !!\n");
 566	return 0;
 567}
 568
 569/**
 570 * octep_vf_iq_full_check() - check if a Tx queue is full.
 571 *
 572 * @iq: Octeon Tx queue data structure.
 573 *
 574 * Return: 0, if the Tx queue is not full.
 575 *         1, if the Tx queue is full.
 576 */
 577static int octep_vf_iq_full_check(struct octep_vf_iq *iq)
 578{
 579	int ret;
 580
 581	ret = netif_subqueue_maybe_stop(iq->netdev, iq->q_no, IQ_INSTR_SPACE(iq),
 582					OCTEP_VF_WAKE_QUEUE_THRESHOLD,
 583					OCTEP_VF_WAKE_QUEUE_THRESHOLD);
 584	switch (ret) {
 585	case 0: /* Stopped the queue, since IQ is full */
 586		return 1;
 587	case -1: /*
 588		  * Pending updates in write index from
 589		  * iq_process_completion in other cpus
 590		  * caused queues to get re-enabled after
 591		  * being stopped
 592		  */
 593		iq->stats->restart_cnt++;
 594		fallthrough;
 595	case 1: /* Queue left enabled, since IQ is not yet full*/
 596		return 0;
 597	}
 598
 599	return 1;
 600}
 601
 602/**
 603 * octep_vf_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
 604 *
 605 * @skb: packet skbuff pointer.
 606 * @netdev: kernel network device.
 607 *
 608 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
 609 *         NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
 610 */
 611static netdev_tx_t octep_vf_start_xmit(struct sk_buff *skb,
 612				       struct net_device *netdev)
 613{
 614	struct octep_vf_device *oct = netdev_priv(netdev);
 615	netdev_features_t feat  = netdev->features;
 616	struct octep_vf_tx_sglist_desc *sglist;
 617	struct octep_vf_tx_buffer *tx_buffer;
 618	struct octep_vf_tx_desc_hw *hw_desc;
 619	struct skb_shared_info *shinfo;
 620	struct octep_vf_instr_hdr *ih;
 621	struct octep_vf_iq *iq;
 622	skb_frag_t *frag;
 623	u16 nr_frags, si;
 624	int xmit_more;
 625	u16 q_no, wi;
 626
 627	if (skb_put_padto(skb, ETH_ZLEN))
 628		return NETDEV_TX_OK;
 629
 630	q_no = skb_get_queue_mapping(skb);
 631	if (q_no >= oct->num_iqs) {
 632		netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
 633		q_no = q_no % oct->num_iqs;
 634	}
 635
 636	iq = oct->iq[q_no];
 637
 638	shinfo = skb_shinfo(skb);
 639	nr_frags = shinfo->nr_frags;
 640
 641	wi = iq->host_write_index;
 642	hw_desc = &iq->desc_ring[wi];
 643	hw_desc->ih64 = 0;
 644
 645	tx_buffer = iq->buff_info + wi;
 646	tx_buffer->skb = skb;
 647
 648	ih = &hw_desc->ih;
 649	ih->tlen = skb->len;
 650	ih->pkind = oct->fw_info.pkind;
 651	ih->fsz = oct->fw_info.fsz;
 652	ih->tlen = skb->len + ih->fsz;
 653
 654	if (!nr_frags) {
 655		tx_buffer->gather = 0;
 656		tx_buffer->dma = dma_map_single(iq->dev, skb->data,
 657						skb->len, DMA_TO_DEVICE);
 658		if (dma_mapping_error(iq->dev, tx_buffer->dma))
 659			goto dma_map_err;
 660		hw_desc->dptr = tx_buffer->dma;
 661	} else {
 662		/* Scatter/Gather */
 663		dma_addr_t dma;
 664		u16 len;
 665
 666		sglist = tx_buffer->sglist;
 667
 668		ih->gsz = nr_frags + 1;
 669		ih->gather = 1;
 670		tx_buffer->gather = 1;
 671
 672		len = skb_headlen(skb);
 673		dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
 674		if (dma_mapping_error(iq->dev, dma))
 675			goto dma_map_err;
 676
 677		memset(sglist, 0, OCTEP_VF_SGLIST_SIZE_PER_PKT);
 678		sglist[0].len[3] = len;
 679		sglist[0].dma_ptr[0] = dma;
 680
 681		si = 1; /* entry 0 is main skb, mapped above */
 682		frag = &shinfo->frags[0];
 683		while (nr_frags--) {
 684			len = skb_frag_size(frag);
 685			dma = skb_frag_dma_map(iq->dev, frag, 0,
 686					       len, DMA_TO_DEVICE);
 687			if (dma_mapping_error(iq->dev, dma))
 688				goto dma_map_sg_err;
 689
 690			sglist[si >> 2].len[3 - (si & 3)] = len;
 691			sglist[si >> 2].dma_ptr[si & 3] = dma;
 692
 693			frag++;
 694			si++;
 695		}
 696		hw_desc->dptr = tx_buffer->sglist_dma;
 697	}
 698	if (oct->fw_info.tx_ol_flags) {
 699		if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {
 700			hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM;
 701			hw_desc->txm.ol_flags |= OCTEP_VF_TX_OFFLOAD_TSO;
 702			hw_desc->txm.gso_size =  skb_shinfo(skb)->gso_size;
 703			hw_desc->txm.gso_segs =  skb_shinfo(skb)->gso_segs;
 704		} else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
 705			hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM;
 706		}
 707		/* due to ESR txm will be swapped by hw */
 708		hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);
 709	}
 710
 711	xmit_more = netdev_xmit_more();
 712
 713	netdev_tx_sent_queue(iq->netdev_q, skb->len);
 714
 715	skb_tx_timestamp(skb);
 716	iq->fill_cnt++;
 717	wi++;
 718	iq->host_write_index = wi & iq->ring_size_mask;
 719
 720	/* octep_iq_full_check stops the queue and returns
 721	 * true if so, in case the queue has become full
 722	 * by inserting current packet. If so, we can
 723	 * go ahead and ring doorbell.
 724	 */
 725	if (!octep_vf_iq_full_check(iq) && xmit_more &&
 726	    iq->fill_cnt < iq->fill_threshold)
 727		return NETDEV_TX_OK;
 728
 729	goto ring_dbell;
 730
 731dma_map_sg_err:
 732	if (si > 0) {
 733		dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
 734				 sglist[0].len[0], DMA_TO_DEVICE);
 735		sglist[0].len[0] = 0;
 736	}
 737	while (si > 1) {
 738		dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
 739			       sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);
 740		sglist[si >> 2].len[si & 3] = 0;
 741		si--;
 742	}
 743	tx_buffer->gather = 0;
 744dma_map_err:
 745	dev_kfree_skb_any(skb);
 746ring_dbell:
 747	/* Flush the hw descriptors before writing to doorbell */
 748	smp_wmb();
 749	writel(iq->fill_cnt, iq->doorbell_reg);
 750	iq->stats->instr_posted += iq->fill_cnt;
 751	iq->fill_cnt = 0;
 752	return NETDEV_TX_OK;
 753}
 754
 755int octep_vf_get_if_stats(struct octep_vf_device *oct)
 756{
 757	struct octep_vf_iface_rxtx_stats vf_stats;
 758	int ret, size;
 759
 760	memset(&vf_stats, 0, sizeof(struct octep_vf_iface_rxtx_stats));
 761	ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_STATS,
 762				      (u8 *)&vf_stats, &size);
 763
 764	if (ret)
 765		return ret;
 766
 767	memcpy(&oct->iface_rx_stats, &vf_stats.iface_rx_stats,
 768	       sizeof(struct octep_vf_iface_rx_stats));
 769	memcpy(&oct->iface_tx_stats, &vf_stats.iface_tx_stats,
 770	       sizeof(struct octep_vf_iface_tx_stats));
 771
 772	return 0;
 773}
 774
 775int octep_vf_get_link_info(struct octep_vf_device *oct)
 776{
 777	int ret, size;
 778
 779	ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_LINK_INFO,
 780				      (u8 *)&oct->link_info, &size);
 781	if (ret) {
 782		dev_err(&oct->pdev->dev, "Get VF link info failed via VF Mbox\n");
 783		return ret;
 784	}
 785	return 0;
 786}
 787
 788/**
 789 * octep_vf_get_stats64() - Get Octeon network device statistics.
 790 *
 791 * @netdev: kernel network device.
 792 * @stats: pointer to stats structure to be filled in.
 793 */
 794static void octep_vf_get_stats64(struct net_device *netdev,
 795				 struct rtnl_link_stats64 *stats)
 796{
 797	struct octep_vf_device *oct = netdev_priv(netdev);
 798	u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
 799	int q;
 800
 801	tx_packets = 0;
 802	tx_bytes = 0;
 803	rx_packets = 0;
 804	rx_bytes = 0;
 805	for (q = 0; q < OCTEP_VF_MAX_QUEUES; q++) {
 806		tx_packets += oct->stats_iq[q].instr_completed;
 807		tx_bytes += oct->stats_iq[q].bytes_sent;
 808		rx_packets += oct->stats_oq[q].packets;
 809		rx_bytes += oct->stats_oq[q].bytes;
 810	}
 811	stats->tx_packets = tx_packets;
 812	stats->tx_bytes = tx_bytes;
 813	stats->rx_packets = rx_packets;
 814	stats->rx_bytes = rx_bytes;
 815}
 816
 817/**
 818 * octep_vf_tx_timeout_task - work queue task to Handle Tx queue timeout.
 819 *
 820 * @work: pointer to Tx queue timeout work_struct
 821 *
 822 * Stop and start the device so that it frees up all queue resources
 823 * and restarts the queues, that potentially clears a Tx queue timeout
 824 * condition.
 825 **/
 826static void octep_vf_tx_timeout_task(struct work_struct *work)
 827{
 828	struct octep_vf_device *oct = container_of(work, struct octep_vf_device,
 829						tx_timeout_task);
 830	struct net_device *netdev = oct->netdev;
 831
 832	rtnl_lock();
 833	if (netif_running(netdev)) {
 834		octep_vf_stop(netdev);
 835		octep_vf_open(netdev);
 836	}
 837	rtnl_unlock();
 838	netdev_put(netdev, NULL);
 839}
 840
 841/**
 842 * octep_vf_tx_timeout() - Handle Tx Queue timeout.
 843 *
 844 * @netdev: pointer to kernel network device.
 845 * @txqueue: Timed out Tx queue number.
 846 *
 847 * Schedule a work to handle Tx queue timeout.
 848 */
 849static void octep_vf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 850{
 851	struct octep_vf_device *oct = netdev_priv(netdev);
 852
 853	netdev_hold(netdev, NULL, GFP_ATOMIC);
 854	if (!schedule_work(&oct->tx_timeout_task))
 855		netdev_put(netdev, NULL);
 856
 857}
 858
 859static int octep_vf_set_mac(struct net_device *netdev, void *p)
 860{
 861	struct octep_vf_device *oct = netdev_priv(netdev);
 862	struct sockaddr *addr = (struct sockaddr *)p;
 863	int err;
 864
 865	if (!is_valid_ether_addr(addr->sa_data))
 866		return -EADDRNOTAVAIL;
 867
 868	err = octep_vf_mbox_set_mac_addr(oct, addr->sa_data);
 869	if (err)
 870		return err;
 871
 872	memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
 873	eth_hw_addr_set(netdev, addr->sa_data);
 874
 875	return 0;
 876}
 877
 878static int octep_vf_change_mtu(struct net_device *netdev, int new_mtu)
 879{
 880	struct octep_vf_device *oct = netdev_priv(netdev);
 881	struct octep_vf_iface_link_info *link_info;
 882	int err;
 883
 884	link_info = &oct->link_info;
 885	if (link_info->mtu == new_mtu)
 886		return 0;
 887
 888	err = octep_vf_mbox_set_mtu(oct, new_mtu);
 889	if (!err) {
 890		oct->link_info.mtu = new_mtu;
 891		WRITE_ONCE(netdev->mtu, new_mtu);
 892	}
 893	return err;
 894}
 895
 896static int octep_vf_set_features(struct net_device *netdev,
 897				 netdev_features_t features)
 898{
 899	struct octep_vf_device *oct = netdev_priv(netdev);
 900	u16 rx_offloads = 0, tx_offloads = 0;
 901	int err;
 902
 903	/* We only support features received from firmware */
 904	if ((features & netdev->hw_features) != features)
 905		return -EINVAL;
 906
 907	if (features & NETIF_F_TSO)
 908		tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO;
 909
 910	if (features & NETIF_F_TSO6)
 911		tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO;
 912
 913	if (features & NETIF_F_IP_CSUM)
 914		tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM;
 915
 916	if (features & NETIF_F_IPV6_CSUM)
 917		tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM;
 918
 919	if (features & NETIF_F_RXCSUM)
 920		rx_offloads |= OCTEP_VF_RX_OFFLOAD_CKSUM;
 921
 922	err = octep_vf_mbox_set_offloads(oct, tx_offloads, rx_offloads);
 923	if (!err)
 924		netdev->features = features;
 925
 926	return err;
 927}
 928
 929static const struct net_device_ops octep_vf_netdev_ops = {
 930	.ndo_open                = octep_vf_open,
 931	.ndo_stop                = octep_vf_stop,
 932	.ndo_start_xmit          = octep_vf_start_xmit,
 933	.ndo_get_stats64         = octep_vf_get_stats64,
 934	.ndo_tx_timeout          = octep_vf_tx_timeout,
 935	.ndo_set_mac_address     = octep_vf_set_mac,
 936	.ndo_change_mtu          = octep_vf_change_mtu,
 937	.ndo_set_features        = octep_vf_set_features,
 938};
 939
 940static const char *octep_vf_devid_to_str(struct octep_vf_device *oct)
 941{
 942	switch (oct->chip_id) {
 943	case OCTEP_PCI_DEVICE_ID_CN93_VF:
 944		return "CN93XX";
 945	case OCTEP_PCI_DEVICE_ID_CNF95N_VF:
 946		return "CNF95N";
 947	case OCTEP_PCI_DEVICE_ID_CN10KA_VF:
 948		return "CN10KA";
 949	case OCTEP_PCI_DEVICE_ID_CNF10KA_VF:
 950		return "CNF10KA";
 951	case OCTEP_PCI_DEVICE_ID_CNF10KB_VF:
 952		return "CNF10KB";
 953	case OCTEP_PCI_DEVICE_ID_CN10KB_VF:
 954		return "CN10KB";
 955	default:
 956		return "Unsupported";
 957	}
 958}
 959
 960/**
 961 * octep_vf_device_setup() - Setup Octeon Device.
 962 *
 963 * @oct: Octeon device private data structure.
 964 *
 965 * Setup Octeon device hardware operations, configuration, etc ...
 966 */
 967int octep_vf_device_setup(struct octep_vf_device *oct)
 968{
 969	struct pci_dev *pdev = oct->pdev;
 970
 971	/* allocate memory for oct->conf */
 972	oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
 973	if (!oct->conf)
 974		return -ENOMEM;
 975
 976	/* Map BAR region 0 */
 977	oct->mmio.hw_addr = ioremap(pci_resource_start(oct->pdev, 0),
 978				    pci_resource_len(oct->pdev, 0));
 979	if (!oct->mmio.hw_addr) {
 980		dev_err(&pdev->dev,
 981			"Failed to remap BAR0; start=0x%llx len=0x%llx\n",
 982			pci_resource_start(oct->pdev, 0),
 983			pci_resource_len(oct->pdev, 0));
 984		goto ioremap_err;
 985	}
 986	oct->mmio.mapped = 1;
 987
 988	oct->chip_id = pdev->device;
 989	oct->rev_id = pdev->revision;
 990	dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
 991
 992	switch (oct->chip_id) {
 993	case OCTEP_PCI_DEVICE_ID_CN93_VF:
 994	case OCTEP_PCI_DEVICE_ID_CNF95N_VF:
 995	case OCTEP_PCI_DEVICE_ID_CN98_VF:
 996		dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n",
 997			 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct),
 998			 OCTEP_VF_MINOR_REV(oct));
 999		octep_vf_device_setup_cn93(oct);
1000		break;
1001	case OCTEP_PCI_DEVICE_ID_CNF10KA_VF:
1002	case OCTEP_PCI_DEVICE_ID_CN10KA_VF:
1003	case OCTEP_PCI_DEVICE_ID_CNF10KB_VF:
1004	case OCTEP_PCI_DEVICE_ID_CN10KB_VF:
1005		dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n",
1006			 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct),
1007			 OCTEP_VF_MINOR_REV(oct));
1008		octep_vf_device_setup_cnxk(oct);
1009		break;
1010	default:
1011		dev_err(&pdev->dev, "Unsupported device\n");
1012		goto unsupported_dev;
1013	}
1014
1015	return 0;
1016
1017unsupported_dev:
1018	iounmap(oct->mmio.hw_addr);
1019ioremap_err:
1020	kfree(oct->conf);
1021	return -EOPNOTSUPP;
1022}
1023
1024/**
1025 * octep_vf_device_cleanup() - Cleanup Octeon Device.
1026 *
1027 * @oct: Octeon device private data structure.
1028 *
1029 * Cleanup Octeon device allocated resources.
1030 */
1031static void octep_vf_device_cleanup(struct octep_vf_device *oct)
1032{
1033	dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1034
1035	if (oct->mmio.mapped)
1036		iounmap(oct->mmio.hw_addr);
1037
1038	kfree(oct->conf);
1039	oct->conf = NULL;
1040}
1041
1042static int octep_vf_get_mac_addr(struct octep_vf_device *oct, u8 *addr)
1043{
1044	return octep_vf_mbox_get_mac_addr(oct, addr);
1045}
1046
1047/**
1048 * octep_vf_probe() - Octeon PCI device probe handler.
1049 *
1050 * @pdev: PCI device structure.
1051 * @ent: entry in Octeon PCI device ID table.
1052 *
1053 * Initializes and enables the Octeon PCI device for network operations.
1054 * Initializes Octeon private data structure and registers a network device.
1055 */
1056static int octep_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1057{
1058	struct octep_vf_device *octep_vf_dev;
1059	struct net_device *netdev;
1060	int err;
1061
1062	err = pci_enable_device(pdev);
1063	if (err) {
1064		dev_err(&pdev->dev, "Failed to enable PCI device\n");
1065		return  err;
1066	}
1067
1068	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1069	if (err) {
1070		dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1071		goto disable_pci_device;
1072	}
1073
1074	err = pci_request_mem_regions(pdev, OCTEP_VF_DRV_NAME);
1075	if (err) {
1076		dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1077		goto disable_pci_device;
1078	}
1079
1080	pci_set_master(pdev);
1081
1082	netdev = alloc_etherdev_mq(sizeof(struct octep_vf_device),
1083				   OCTEP_VF_MAX_QUEUES);
1084	if (!netdev) {
1085		dev_err(&pdev->dev, "Failed to allocate netdev\n");
1086		err = -ENOMEM;
1087		goto mem_regions_release;
1088	}
1089	SET_NETDEV_DEV(netdev, &pdev->dev);
1090
1091	octep_vf_dev = netdev_priv(netdev);
1092	octep_vf_dev->netdev = netdev;
1093	octep_vf_dev->pdev = pdev;
1094	octep_vf_dev->dev = &pdev->dev;
1095	pci_set_drvdata(pdev, octep_vf_dev);
1096
1097	err = octep_vf_device_setup(octep_vf_dev);
1098	if (err) {
1099		dev_err(&pdev->dev, "Device setup failed\n");
1100		goto netdevice_free;
1101	}
1102	INIT_WORK(&octep_vf_dev->tx_timeout_task, octep_vf_tx_timeout_task);
1103
1104	netdev->netdev_ops = &octep_vf_netdev_ops;
1105	octep_vf_set_ethtool_ops(netdev);
1106	netif_carrier_off(netdev);
1107
1108	if (octep_vf_setup_mbox(octep_vf_dev)) {
1109		dev_err(&pdev->dev, "VF Mailbox setup failed\n");
1110		err = -ENOMEM;
1111		goto device_cleanup;
1112	}
1113
1114	if (octep_vf_mbox_version_check(octep_vf_dev)) {
1115		dev_err(&pdev->dev, "PF VF Mailbox version mismatch\n");
1116		err = -EINVAL;
1117		goto delete_mbox;
1118	}
1119
1120	if (octep_vf_mbox_get_fw_info(octep_vf_dev)) {
1121		dev_err(&pdev->dev, "unable to get fw info\n");
1122		err = -EINVAL;
1123		goto delete_mbox;
1124	}
1125
1126	netdev->hw_features = NETIF_F_SG;
1127	if (OCTEP_VF_TX_IP_CSUM(octep_vf_dev->fw_info.tx_ol_flags))
1128		netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
1129
1130	if (OCTEP_VF_RX_IP_CSUM(octep_vf_dev->fw_info.rx_ol_flags))
1131		netdev->hw_features |= NETIF_F_RXCSUM;
1132
1133	netdev->min_mtu = OCTEP_VF_MIN_MTU;
1134	netdev->max_mtu = OCTEP_VF_MAX_MTU;
1135	netdev->mtu = OCTEP_VF_DEFAULT_MTU;
1136
1137	if (OCTEP_VF_TX_TSO(octep_vf_dev->fw_info.tx_ol_flags)) {
1138		netdev->hw_features |= NETIF_F_TSO;
1139		netif_set_tso_max_size(netdev, netdev->max_mtu);
1140	}
1141
1142	netdev->features |= netdev->hw_features;
1143	octep_vf_get_mac_addr(octep_vf_dev, octep_vf_dev->mac_addr);
1144	eth_hw_addr_set(netdev, octep_vf_dev->mac_addr);
1145	err = register_netdev(netdev);
1146	if (err) {
1147		dev_err(&pdev->dev, "Failed to register netdev\n");
1148		goto delete_mbox;
1149	}
1150	dev_info(&pdev->dev, "Device probe successful\n");
1151	return 0;
1152
1153delete_mbox:
1154	octep_vf_delete_mbox(octep_vf_dev);
1155device_cleanup:
1156	octep_vf_device_cleanup(octep_vf_dev);
1157netdevice_free:
1158	free_netdev(netdev);
1159mem_regions_release:
1160	pci_release_mem_regions(pdev);
1161disable_pci_device:
1162	pci_disable_device(pdev);
1163	dev_err(&pdev->dev, "Device probe failed\n");
1164	return err;
1165}
1166
1167/**
1168 * octep_vf_remove() - Remove Octeon PCI device from driver control.
1169 *
1170 * @pdev: PCI device structure of the Octeon device.
1171 *
1172 * Cleanup all resources allocated for the Octeon device.
1173 * Unregister from network device and disable the PCI device.
1174 */
1175static void octep_vf_remove(struct pci_dev *pdev)
1176{
1177	struct octep_vf_device *oct = pci_get_drvdata(pdev);
1178	struct net_device *netdev;
1179
1180	if (!oct)
1181		return;
1182
1183	octep_vf_mbox_dev_remove(oct);
1184	cancel_work_sync(&oct->tx_timeout_task);
1185	netdev = oct->netdev;
1186	if (netdev->reg_state == NETREG_REGISTERED)
1187		unregister_netdev(netdev);
1188	octep_vf_delete_mbox(oct);
1189	octep_vf_device_cleanup(oct);
1190	pci_release_mem_regions(pdev);
1191	free_netdev(netdev);
1192	pci_disable_device(pdev);
1193}
1194
1195static struct pci_driver octep_vf_driver = {
1196	.name = OCTEP_VF_DRV_NAME,
1197	.id_table = octep_vf_pci_id_tbl,
1198	.probe = octep_vf_probe,
1199	.remove = octep_vf_remove,
1200};
1201
1202/**
1203 * octep_vf_init_module() - Module initialization.
1204 *
1205 * create common resource for the driver and register PCI driver.
1206 */
1207static int __init octep_vf_init_module(void)
1208{
1209	int ret;
1210
1211	pr_info("%s: Loading %s ...\n", OCTEP_VF_DRV_NAME, OCTEP_VF_DRV_STRING);
1212
1213	ret = pci_register_driver(&octep_vf_driver);
1214	if (ret < 0) {
1215		pr_err("%s: Failed to register PCI driver; err=%d\n",
1216		       OCTEP_VF_DRV_NAME, ret);
1217		return ret;
1218	}
1219
1220	return ret;
1221}
1222
1223/**
1224 * octep_vf_exit_module() - Module exit routine.
1225 *
1226 * unregister the driver with PCI subsystem and cleanup common resources.
1227 */
1228static void __exit octep_vf_exit_module(void)
1229{
1230	pr_info("%s: Unloading ...\n", OCTEP_VF_DRV_NAME);
1231
1232	pci_unregister_driver(&octep_vf_driver);
1233
1234	pr_info("%s: Unloading complete\n", OCTEP_VF_DRV_NAME);
1235}
1236
1237module_init(octep_vf_init_module);
1238module_exit(octep_vf_exit_module);