octep_main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Marvell Octeon EP (EndPoint) Ethernet Driver
   3 *
   4 * Copyright (C) 2020 Marvell.
   5 *
   6 */
   7
   8#include <linux/types.h>
   9#include <linux/module.h>
  10#include <linux/pci.h>
  11#include <linux/netdevice.h>
  12#include <linux/etherdevice.h>
  13#include <linux/rtnetlink.h>
  14#include <linux/vmalloc.h>
  15
  16#include "octep_config.h"
  17#include "octep_main.h"
  18#include "octep_ctrl_net.h"
  19#include "octep_pfvf_mbox.h"
  20
  21#define OCTEP_INTR_POLL_TIME_MSECS    100
  22struct workqueue_struct *octep_wq;
  23
  24/* Supported Devices */
  25static const struct pci_device_id octep_pci_id_tbl[] = {
  26	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_PF)},
  27	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
  28	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
  29	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_PF)},
  30	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_PF)},
  31	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_PF)},
  32	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_PF)},
  33	{0, },
  34};
  35MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
  36
  37MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
  38MODULE_DESCRIPTION(OCTEP_DRV_STRING);
  39MODULE_LICENSE("GPL");
  40
  41/**
  42 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
  43 *
  44 * @oct: Octeon device private data structure.
  45 *
  46 * Allocate resources to hold per Tx/Rx queue interrupt info.
  47 * This is the information passed to interrupt handler, from which napi poll
  48 * is scheduled and includes quick access to private data of Tx/Rx queue
  49 * corresponding to the interrupt being handled.
  50 *
  51 * Return: 0, on successful allocation of resources for all queue interrupts.
  52 *         -1, if failed to allocate any resource.
  53 */
  54static int octep_alloc_ioq_vectors(struct octep_device *oct)
  55{
  56	int i;
  57	struct octep_ioq_vector *ioq_vector;
  58
  59	for (i = 0; i < oct->num_oqs; i++) {
  60		oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
  61		if (!oct->ioq_vector[i])
  62			goto free_ioq_vector;
  63
  64		ioq_vector = oct->ioq_vector[i];
  65		ioq_vector->iq = oct->iq[i];
  66		ioq_vector->oq = oct->oq[i];
  67		ioq_vector->octep_dev = oct;
  68	}
  69
  70	dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
  71	return 0;
  72
  73free_ioq_vector:
  74	while (i) {
  75		i--;
  76		vfree(oct->ioq_vector[i]);
  77		oct->ioq_vector[i] = NULL;
  78	}
  79	return -1;
  80}
  81
  82/**
  83 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
  84 *
  85 * @oct: Octeon device private data structure.
  86 */
  87static void octep_free_ioq_vectors(struct octep_device *oct)
  88{
  89	int i;
  90
  91	for (i = 0; i < oct->num_oqs; i++) {
  92		if (oct->ioq_vector[i]) {
  93			vfree(oct->ioq_vector[i]);
  94			oct->ioq_vector[i] = NULL;
  95		}
  96	}
  97	netdev_info(oct->netdev, "Freed IOQ Vectors\n");
  98}
  99
 100/**
 101 * octep_enable_msix_range() - enable MSI-x interrupts.
 102 *
 103 * @oct: Octeon device private data structure.
 104 *
 105 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
 106 * for the Octeon device.
 107 *
 108 * Return: 0, on successfully enabling all MSI-x interrupts.
 109 *         -1, if failed to enable any MSI-x interrupt.
 110 */
 111static int octep_enable_msix_range(struct octep_device *oct)
 112{
 113	int num_msix, msix_allocated;
 114	int i;
 115
 116	/* Generic interrupts apart from input/output queues */
 117	num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
 118	oct->msix_entries = kcalloc(num_msix,
 119				    sizeof(struct msix_entry), GFP_KERNEL);
 120	if (!oct->msix_entries)
 121		goto msix_alloc_err;
 122
 123	for (i = 0; i < num_msix; i++)
 124		oct->msix_entries[i].entry = i;
 125
 126	msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
 127					       num_msix, num_msix);
 128	if (msix_allocated != num_msix) {
 129		dev_err(&oct->pdev->dev,
 130			"Failed to enable %d msix irqs; got only %d\n",
 131			num_msix, msix_allocated);
 132		goto enable_msix_err;
 133	}
 134	oct->num_irqs = msix_allocated;
 135	dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
 136
 137	return 0;
 138
 139enable_msix_err:
 140	if (msix_allocated > 0)
 141		pci_disable_msix(oct->pdev);
 142	kfree(oct->msix_entries);
 143	oct->msix_entries = NULL;
 144msix_alloc_err:
 145	return -1;
 146}
 147
 148/**
 149 * octep_disable_msix() - disable MSI-x interrupts.
 150 *
 151 * @oct: Octeon device private data structure.
 152 *
 153 * Disable MSI-x on the Octeon device.
 154 */
 155static void octep_disable_msix(struct octep_device *oct)
 156{
 157	pci_disable_msix(oct->pdev);
 158	kfree(oct->msix_entries);
 159	oct->msix_entries = NULL;
 160	dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
 161}
 162
 163/**
 164 * octep_mbox_intr_handler() - common handler for pfvf mbox interrupts.
 165 *
 166 * @irq: Interrupt number.
 167 * @data: interrupt data.
 168 *
 169 * this is common handler for pfvf mbox interrupts.
 170 */
 171static irqreturn_t octep_mbox_intr_handler(int irq, void *data)
 172{
 173	struct octep_device *oct = data;
 174
 175	return oct->hw_ops.mbox_intr_handler(oct);
 176}
 177
 178/**
 179 * octep_oei_intr_handler() - common handler for output endpoint interrupts.
 180 *
 181 * @irq: Interrupt number.
 182 * @data: interrupt data.
 183 *
 184 * this is common handler for all output endpoint interrupts.
 185 */
 186static irqreturn_t octep_oei_intr_handler(int irq, void *data)
 187{
 188	struct octep_device *oct = data;
 189
 190	return oct->hw_ops.oei_intr_handler(oct);
 191}
 192
 193/**
 194 * octep_ire_intr_handler() - common handler for input ring error interrupts.
 195 *
 196 * @irq: Interrupt number.
 197 * @data: interrupt data.
 198 *
 199 * this is common handler for input ring error interrupts.
 200 */
 201static irqreturn_t octep_ire_intr_handler(int irq, void *data)
 202{
 203	struct octep_device *oct = data;
 204
 205	return oct->hw_ops.ire_intr_handler(oct);
 206}
 207
 208/**
 209 * octep_ore_intr_handler() - common handler for output ring error interrupts.
 210 *
 211 * @irq: Interrupt number.
 212 * @data: interrupt data.
 213 *
 214 * this is common handler for output ring error interrupts.
 215 */
 216static irqreturn_t octep_ore_intr_handler(int irq, void *data)
 217{
 218	struct octep_device *oct = data;
 219
 220	return oct->hw_ops.ore_intr_handler(oct);
 221}
 222
 223/**
 224 * octep_vfire_intr_handler() - common handler for vf input ring error interrupts.
 225 *
 226 * @irq: Interrupt number.
 227 * @data: interrupt data.
 228 *
 229 * this is common handler for vf input ring error interrupts.
 230 */
 231static irqreturn_t octep_vfire_intr_handler(int irq, void *data)
 232{
 233	struct octep_device *oct = data;
 234
 235	return oct->hw_ops.vfire_intr_handler(oct);
 236}
 237
 238/**
 239 * octep_vfore_intr_handler() - common handler for vf output ring error interrupts.
 240 *
 241 * @irq: Interrupt number.
 242 * @data: interrupt data.
 243 *
 244 * this is common handler for vf output ring error interrupts.
 245 */
 246static irqreturn_t octep_vfore_intr_handler(int irq, void *data)
 247{
 248	struct octep_device *oct = data;
 249
 250	return oct->hw_ops.vfore_intr_handler(oct);
 251}
 252
 253/**
 254 * octep_dma_intr_handler() - common handler for dpi dma related interrupts.
 255 *
 256 * @irq: Interrupt number.
 257 * @data: interrupt data.
 258 *
 259 * this is common handler for dpi dma related interrupts.
 260 */
 261static irqreturn_t octep_dma_intr_handler(int irq, void *data)
 262{
 263	struct octep_device *oct = data;
 264
 265	return oct->hw_ops.dma_intr_handler(oct);
 266}
 267
 268/**
 269 * octep_dma_vf_intr_handler() - common handler for dpi dma transaction error interrupts for VFs.
 270 *
 271 * @irq: Interrupt number.
 272 * @data: interrupt data.
 273 *
 274 * this is common handler for dpi dma transaction error interrupts for VFs.
 275 */
 276static irqreturn_t octep_dma_vf_intr_handler(int irq, void *data)
 277{
 278	struct octep_device *oct = data;
 279
 280	return oct->hw_ops.dma_vf_intr_handler(oct);
 281}
 282
 283/**
 284 * octep_pp_vf_intr_handler() - common handler for pp transaction error interrupts for VFs.
 285 *
 286 * @irq: Interrupt number.
 287 * @data: interrupt data.
 288 *
 289 * this is common handler for pp transaction error interrupts for VFs.
 290 */
 291static irqreturn_t octep_pp_vf_intr_handler(int irq, void *data)
 292{
 293	struct octep_device *oct = data;
 294
 295	return oct->hw_ops.pp_vf_intr_handler(oct);
 296}
 297
 298/**
 299 * octep_misc_intr_handler() - common handler for mac related interrupts.
 300 *
 301 * @irq: Interrupt number.
 302 * @data: interrupt data.
 303 *
 304 * this is common handler for mac related interrupts.
 305 */
 306static irqreturn_t octep_misc_intr_handler(int irq, void *data)
 307{
 308	struct octep_device *oct = data;
 309
 310	return oct->hw_ops.misc_intr_handler(oct);
 311}
 312
 313/**
 314 * octep_rsvd_intr_handler() - common handler for reserved interrupts (future use).
 315 *
 316 * @irq: Interrupt number.
 317 * @data: interrupt data.
 318 *
 319 * this is common handler for all reserved interrupts.
 320 */
 321static irqreturn_t octep_rsvd_intr_handler(int irq, void *data)
 322{
 323	struct octep_device *oct = data;
 324
 325	return oct->hw_ops.rsvd_intr_handler(oct);
 326}
 327
 328/**
 329 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
 330 *
 331 * @irq: Interrupt number.
 332 * @data: interrupt data contains pointers to Tx/Rx queue private data
 333 *         and correspong NAPI context.
 334 *
 335 * this is common handler for all non-queue (generic) interrupts.
 336 */
 337static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
 338{
 339	struct octep_ioq_vector *ioq_vector = data;
 340	struct octep_device *oct = ioq_vector->octep_dev;
 341
 342	return oct->hw_ops.ioq_intr_handler(ioq_vector);
 343}
 344
 345/**
 346 * octep_request_irqs() - Register interrupt handlers.
 347 *
 348 * @oct: Octeon device private data structure.
 349 *
 350 * Register handlers for all queue and non-queue interrupts.
 351 *
 352 * Return: 0, on successful registration of all interrupt handlers.
 353 *         -1, on any error.
 354 */
 355static int octep_request_irqs(struct octep_device *oct)
 356{
 357	struct net_device *netdev = oct->netdev;
 358	struct octep_ioq_vector *ioq_vector;
 359	struct msix_entry *msix_entry;
 360	char **non_ioq_msix_names;
 361	int num_non_ioq_msix;
 362	int ret, i, j;
 363
 364	num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
 365	non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
 366
 367	oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
 368					 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
 369	if (!oct->non_ioq_irq_names)
 370		goto alloc_err;
 371
 372	/* First few MSI-X interrupts are non-queue interrupts */
 373	for (i = 0; i < num_non_ioq_msix; i++) {
 374		char *irq_name;
 375
 376		irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
 377		msix_entry = &oct->msix_entries[i];
 378
 379		snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
 380			 "%s-%s", netdev->name, non_ioq_msix_names[i]);
 381		if (!strncmp(non_ioq_msix_names[i], "epf_mbox_rint", strlen("epf_mbox_rint"))) {
 382			ret = request_irq(msix_entry->vector,
 383					  octep_mbox_intr_handler, 0,
 384					  irq_name, oct);
 385		} else if (!strncmp(non_ioq_msix_names[i], "epf_oei_rint",
 386			   strlen("epf_oei_rint"))) {
 387			ret = request_irq(msix_entry->vector,
 388					  octep_oei_intr_handler, 0,
 389					  irq_name, oct);
 390		} else if (!strncmp(non_ioq_msix_names[i], "epf_ire_rint",
 391			   strlen("epf_ire_rint"))) {
 392			ret = request_irq(msix_entry->vector,
 393					  octep_ire_intr_handler, 0,
 394					  irq_name, oct);
 395		} else if (!strncmp(non_ioq_msix_names[i], "epf_ore_rint",
 396			   strlen("epf_ore_rint"))) {
 397			ret = request_irq(msix_entry->vector,
 398					  octep_ore_intr_handler, 0,
 399					  irq_name, oct);
 400		} else if (!strncmp(non_ioq_msix_names[i], "epf_vfire_rint",
 401			   strlen("epf_vfire_rint"))) {
 402			ret = request_irq(msix_entry->vector,
 403					  octep_vfire_intr_handler, 0,
 404					  irq_name, oct);
 405		} else if (!strncmp(non_ioq_msix_names[i], "epf_vfore_rint",
 406			   strlen("epf_vfore_rint"))) {
 407			ret = request_irq(msix_entry->vector,
 408					  octep_vfore_intr_handler, 0,
 409					  irq_name, oct);
 410		} else if (!strncmp(non_ioq_msix_names[i], "epf_dma_rint",
 411			   strlen("epf_dma_rint"))) {
 412			ret = request_irq(msix_entry->vector,
 413					  octep_dma_intr_handler, 0,
 414					  irq_name, oct);
 415		} else if (!strncmp(non_ioq_msix_names[i], "epf_dma_vf_rint",
 416			   strlen("epf_dma_vf_rint"))) {
 417			ret = request_irq(msix_entry->vector,
 418					  octep_dma_vf_intr_handler, 0,
 419					  irq_name, oct);
 420		} else if (!strncmp(non_ioq_msix_names[i], "epf_pp_vf_rint",
 421			   strlen("epf_pp_vf_rint"))) {
 422			ret = request_irq(msix_entry->vector,
 423					  octep_pp_vf_intr_handler, 0,
 424					  irq_name, oct);
 425		} else if (!strncmp(non_ioq_msix_names[i], "epf_misc_rint",
 426			   strlen("epf_misc_rint"))) {
 427			ret = request_irq(msix_entry->vector,
 428					  octep_misc_intr_handler, 0,
 429					  irq_name, oct);
 430		} else {
 431			ret = request_irq(msix_entry->vector,
 432					  octep_rsvd_intr_handler, 0,
 433					  irq_name, oct);
 434		}
 435
 436		if (ret) {
 437			netdev_err(netdev,
 438				   "request_irq failed for %s; err=%d",
 439				   irq_name, ret);
 440			goto non_ioq_irq_err;
 441		}
 442	}
 443
 444	/* Request IRQs for Tx/Rx queues */
 445	for (j = 0; j < oct->num_oqs; j++) {
 446		ioq_vector = oct->ioq_vector[j];
 447		msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
 448
 449		snprintf(ioq_vector->name, sizeof(ioq_vector->name),
 450			 "%s-q%d", netdev->name, j);
 451		ret = request_irq(msix_entry->vector,
 452				  octep_ioq_intr_handler, 0,
 453				  ioq_vector->name, ioq_vector);
 454		if (ret) {
 455			netdev_err(netdev,
 456				   "request_irq failed for Q-%d; err=%d",
 457				   j, ret);
 458			goto ioq_irq_err;
 459		}
 460
 461		cpumask_set_cpu(j % num_online_cpus(),
 462				&ioq_vector->affinity_mask);
 463		irq_set_affinity_hint(msix_entry->vector,
 464				      &ioq_vector->affinity_mask);
 465	}
 466
 467	return 0;
 468ioq_irq_err:
 469	while (j) {
 470		--j;
 471		ioq_vector = oct->ioq_vector[j];
 472		msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
 473
 474		irq_set_affinity_hint(msix_entry->vector, NULL);
 475		free_irq(msix_entry->vector, ioq_vector);
 476	}
 477non_ioq_irq_err:
 478	while (i) {
 479		--i;
 480		free_irq(oct->msix_entries[i].vector, oct);
 481	}
 482	kfree(oct->non_ioq_irq_names);
 483	oct->non_ioq_irq_names = NULL;
 484alloc_err:
 485	return -1;
 486}
 487
 488/**
 489 * octep_free_irqs() - free all registered interrupts.
 490 *
 491 * @oct: Octeon device private data structure.
 492 *
 493 * Free all queue and non-queue interrupts of the Octeon device.
 494 */
 495static void octep_free_irqs(struct octep_device *oct)
 496{
 497	int i;
 498
 499	/* First few MSI-X interrupts are non queue interrupts; free them */
 500	for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
 501		free_irq(oct->msix_entries[i].vector, oct);
 502	kfree(oct->non_ioq_irq_names);
 503
 504	/* Free IRQs for Input/Output (Tx/Rx) queues */
 505	for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
 506		irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
 507		free_irq(oct->msix_entries[i].vector,
 508			 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
 509	}
 510	netdev_info(oct->netdev, "IRQs freed\n");
 511}
 512
 513/**
 514 * octep_setup_irqs() - setup interrupts for the Octeon device.
 515 *
 516 * @oct: Octeon device private data structure.
 517 *
 518 * Allocate data structures to hold per interrupt information, allocate/enable
 519 * MSI-x interrupt and register interrupt handlers.
 520 *
 521 * Return: 0, on successful allocation and registration of all interrupts.
 522 *         -1, on any error.
 523 */
 524static int octep_setup_irqs(struct octep_device *oct)
 525{
 526	if (octep_alloc_ioq_vectors(oct))
 527		goto ioq_vector_err;
 528
 529	if (octep_enable_msix_range(oct))
 530		goto enable_msix_err;
 531
 532	if (octep_request_irqs(oct))
 533		goto request_irq_err;
 534
 535	return 0;
 536
 537request_irq_err:
 538	octep_disable_msix(oct);
 539enable_msix_err:
 540	octep_free_ioq_vectors(oct);
 541ioq_vector_err:
 542	return -1;
 543}
 544
 545/**
 546 * octep_clean_irqs() - free all interrupts and its resources.
 547 *
 548 * @oct: Octeon device private data structure.
 549 */
 550static void octep_clean_irqs(struct octep_device *oct)
 551{
 552	octep_free_irqs(oct);
 553	octep_disable_msix(oct);
 554	octep_free_ioq_vectors(oct);
 555}
 556
 557/**
 558 * octep_update_pkt() - Update IQ/OQ IN/OUT_CNT registers.
 559 *
 560 * @iq: Octeon Tx queue data structure.
 561 * @oq: Octeon Rx queue data structure.
 562 */
 563static void octep_update_pkt(struct octep_iq *iq, struct octep_oq *oq)
 564{
 565	u32 pkts_pend = READ_ONCE(oq->pkts_pending);
 566	u32 last_pkt_count = READ_ONCE(oq->last_pkt_count);
 567	u32 pkts_processed = READ_ONCE(iq->pkts_processed);
 568	u32 pkt_in_done = READ_ONCE(iq->pkt_in_done);
 569
 570	netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
 571	if (pkts_processed) {
 572		writel(pkts_processed, iq->inst_cnt_reg);
 573		readl(iq->inst_cnt_reg);
 574		WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed));
 575		WRITE_ONCE(iq->pkts_processed, 0);
 576	}
 577	if (last_pkt_count - pkts_pend) {
 578		writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg);
 579		readl(oq->pkts_sent_reg);
 580		WRITE_ONCE(oq->last_pkt_count, pkts_pend);
 581	}
 582
 583	/* Flush the previous wrties before writing to RESEND bit */
 584	smp_wmb();
 585}
 586
 587/**
 588 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
 589 *
 590 * @iq: Octeon Tx queue data structure.
 591 * @oq: Octeon Rx queue data structure.
 592 */
 593static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
 594{
 595	writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
 596	writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
 597}
 598
 599/**
 600 * octep_napi_poll() - NAPI poll function for Tx/Rx.
 601 *
 602 * @napi: pointer to napi context.
 603 * @budget: max number of packets to be processed in single invocation.
 604 */
 605static int octep_napi_poll(struct napi_struct *napi, int budget)
 606{
 607	struct octep_ioq_vector *ioq_vector =
 608		container_of(napi, struct octep_ioq_vector, napi);
 609	u32 tx_pending, rx_done;
 610
 611	tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
 612	rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
 613
 614	/* need more polling if tx completion processing is still pending or
 615	 * processed at least 'budget' number of rx packets.
 616	 */
 617	if (tx_pending || rx_done >= budget)
 618		return budget;
 619
 620	octep_update_pkt(ioq_vector->iq, ioq_vector->oq);
 621	napi_complete_done(napi, rx_done);
 622	octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
 623	return rx_done;
 624}
 625
 626/**
 627 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
 628 *
 629 * @oct: Octeon device private data structure.
 630 */
 631static void octep_napi_add(struct octep_device *oct)
 632{
 633	int i;
 634
 635	for (i = 0; i < oct->num_oqs; i++) {
 636		netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
 637		netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
 638			       octep_napi_poll);
 639		oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
 640	}
 641}
 642
 643/**
 644 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
 645 *
 646 * @oct: Octeon device private data structure.
 647 */
 648static void octep_napi_delete(struct octep_device *oct)
 649{
 650	int i;
 651
 652	for (i = 0; i < oct->num_oqs; i++) {
 653		netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
 654		netif_napi_del(&oct->ioq_vector[i]->napi);
 655		oct->oq[i]->napi = NULL;
 656	}
 657}
 658
 659/**
 660 * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
 661 *
 662 * @oct: Octeon device private data structure.
 663 */
 664static void octep_napi_enable(struct octep_device *oct)
 665{
 666	int i;
 667
 668	for (i = 0; i < oct->num_oqs; i++) {
 669		netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
 670		napi_enable(&oct->ioq_vector[i]->napi);
 671	}
 672}
 673
 674/**
 675 * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
 676 *
 677 * @oct: Octeon device private data structure.
 678 */
 679static void octep_napi_disable(struct octep_device *oct)
 680{
 681	int i;
 682
 683	for (i = 0; i < oct->num_oqs; i++) {
 684		netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
 685		napi_disable(&oct->ioq_vector[i]->napi);
 686	}
 687}
 688
 689static void octep_link_up(struct net_device *netdev)
 690{
 691	netif_carrier_on(netdev);
 692	netif_tx_start_all_queues(netdev);
 693}
 694
 695/**
 696 * octep_open() - start the octeon network device.
 697 *
 698 * @netdev: pointer to kernel network device.
 699 *
 700 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
 701 * and interrupts..
 702 *
 703 * Return: 0, on successfully setting up device and bring it up.
 704 *         -1, on any error.
 705 */
 706static int octep_open(struct net_device *netdev)
 707{
 708	struct octep_device *oct = netdev_priv(netdev);
 709	int err, ret;
 710
 711	netdev_info(netdev, "Starting netdev ...\n");
 712	netif_carrier_off(netdev);
 713
 714	oct->hw_ops.reset_io_queues(oct);
 715
 716	if (octep_setup_iqs(oct))
 717		goto setup_iq_err;
 718	if (octep_setup_oqs(oct))
 719		goto setup_oq_err;
 720	if (octep_setup_irqs(oct))
 721		goto setup_irq_err;
 722
 723	err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
 724	if (err)
 725		goto set_queues_err;
 726	err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
 727	if (err)
 728		goto set_queues_err;
 729
 730	octep_napi_add(oct);
 731	octep_napi_enable(oct);
 732
 733	oct->link_info.admin_up = 1;
 734	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
 735				    false);
 736	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
 737				       false);
 738	oct->poll_non_ioq_intr = false;
 739
 740	/* Enable the input and output queues for this Octeon device */
 741	oct->hw_ops.enable_io_queues(oct);
 742
 743	/* Enable Octeon device interrupts */
 744	oct->hw_ops.enable_interrupts(oct);
 745
 746	octep_oq_dbell_init(oct);
 747
 748	ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID);
 749	if (ret > 0)
 750		octep_link_up(netdev);
 751
 752	return 0;
 753
 754set_queues_err:
 755	octep_clean_irqs(oct);
 756setup_irq_err:
 757	octep_free_oqs(oct);
 758setup_oq_err:
 759	octep_free_iqs(oct);
 760setup_iq_err:
 761	return -1;
 762}
 763
 764/**
 765 * octep_stop() - stop the octeon network device.
 766 *
 767 * @netdev: pointer to kernel network device.
 768 *
 769 * stop the device Tx/Rx operations, bring down the link and
 770 * free up all resources allocated for Tx/Rx queues and interrupts.
 771 */
 772static int octep_stop(struct net_device *netdev)
 773{
 774	struct octep_device *oct = netdev_priv(netdev);
 775
 776	netdev_info(netdev, "Stopping the device ...\n");
 777
 778	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
 779				       false);
 780	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
 781				    false);
 782
 783	/* Stop Tx from stack */
 784	netif_tx_stop_all_queues(netdev);
 785	netif_carrier_off(netdev);
 786	netif_tx_disable(netdev);
 787
 788	oct->link_info.admin_up = 0;
 789	oct->link_info.oper_up = 0;
 790
 791	oct->hw_ops.disable_interrupts(oct);
 792	octep_napi_disable(oct);
 793	octep_napi_delete(oct);
 794
 795	octep_clean_irqs(oct);
 796	octep_clean_iqs(oct);
 797
 798	oct->hw_ops.disable_io_queues(oct);
 799	oct->hw_ops.reset_io_queues(oct);
 800	octep_free_oqs(oct);
 801	octep_free_iqs(oct);
 802
 803	oct->poll_non_ioq_intr = true;
 804	queue_delayed_work(octep_wq, &oct->intr_poll_task,
 805			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
 806
 807	netdev_info(netdev, "Device stopped !!\n");
 808	return 0;
 809}
 810
 811/**
 812 * octep_iq_full_check() - check if a Tx queue is full.
 813 *
 814 * @iq: Octeon Tx queue data structure.
 815 *
 816 * Return: 0, if the Tx queue is not full.
 817 *         1, if the Tx queue is full.
 818 */
 819static inline int octep_iq_full_check(struct octep_iq *iq)
 820{
 821	if (likely((IQ_INSTR_SPACE(iq)) >
 822		   OCTEP_WAKE_QUEUE_THRESHOLD))
 823		return 0;
 824
 825	/* Stop the queue if unable to send */
 826	netif_stop_subqueue(iq->netdev, iq->q_no);
 827
 828	/* Allow for pending updates in write index
 829	 * from iq_process_completion in other cpus
 830	 * to reflect, in case queue gets free
 831	 * entries.
 832	 */
 833	smp_mb();
 834
 835	/* check again and restart the queue, in case NAPI has just freed
 836	 * enough Tx ring entries.
 837	 */
 838	if (unlikely(IQ_INSTR_SPACE(iq) >
 839		     OCTEP_WAKE_QUEUE_THRESHOLD)) {
 840		netif_start_subqueue(iq->netdev, iq->q_no);
 841		iq->stats->restart_cnt++;
 842		return 0;
 843	}
 844
 845	return 1;
 846}
 847
 848/**
 849 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
 850 *
 851 * @skb: packet skbuff pointer.
 852 * @netdev: kernel network device.
 853 *
 854 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
 855 *         NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
 856 */
 857static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
 858				    struct net_device *netdev)
 859{
 860	struct octep_device *oct = netdev_priv(netdev);
 861	netdev_features_t feat  = netdev->features;
 862	struct octep_tx_sglist_desc *sglist;
 863	struct octep_tx_buffer *tx_buffer;
 864	struct octep_tx_desc_hw *hw_desc;
 865	struct skb_shared_info *shinfo;
 866	struct octep_instr_hdr *ih;
 867	struct octep_iq *iq;
 868	skb_frag_t *frag;
 869	u16 nr_frags, si;
 870	int xmit_more;
 871	u16 q_no, wi;
 872
 873	if (skb_put_padto(skb, ETH_ZLEN))
 874		return NETDEV_TX_OK;
 875
 876	q_no = skb_get_queue_mapping(skb);
 877	if (q_no >= oct->num_iqs) {
 878		netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
 879		q_no = q_no % oct->num_iqs;
 880	}
 881
 882	iq = oct->iq[q_no];
 883
 884	shinfo = skb_shinfo(skb);
 885	nr_frags = shinfo->nr_frags;
 886
 887	wi = iq->host_write_index;
 888	hw_desc = &iq->desc_ring[wi];
 889	hw_desc->ih64 = 0;
 890
 891	tx_buffer = iq->buff_info + wi;
 892	tx_buffer->skb = skb;
 893
 894	ih = &hw_desc->ih;
 895	ih->pkind = oct->conf->fw_info.pkind;
 896	ih->fsz = oct->conf->fw_info.fsz;
 897	ih->tlen = skb->len + ih->fsz;
 898
 899	if (!nr_frags) {
 900		tx_buffer->gather = 0;
 901		tx_buffer->dma = dma_map_single(iq->dev, skb->data,
 902						skb->len, DMA_TO_DEVICE);
 903		if (dma_mapping_error(iq->dev, tx_buffer->dma))
 904			goto dma_map_err;
 905		hw_desc->dptr = tx_buffer->dma;
 906	} else {
 907		/* Scatter/Gather */
 908		dma_addr_t dma;
 909		u16 len;
 910
 911		sglist = tx_buffer->sglist;
 912
 913		ih->gsz = nr_frags + 1;
 914		ih->gather = 1;
 915		tx_buffer->gather = 1;
 916
 917		len = skb_headlen(skb);
 918		dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
 919		if (dma_mapping_error(iq->dev, dma))
 920			goto dma_map_err;
 921
 922		memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
 923		sglist[0].len[3] = len;
 924		sglist[0].dma_ptr[0] = dma;
 925
 926		si = 1; /* entry 0 is main skb, mapped above */
 927		frag = &shinfo->frags[0];
 928		while (nr_frags--) {
 929			len = skb_frag_size(frag);
 930			dma = skb_frag_dma_map(iq->dev, frag, 0,
 931					       len, DMA_TO_DEVICE);
 932			if (dma_mapping_error(iq->dev, dma))
 933				goto dma_map_sg_err;
 934
 935			sglist[si >> 2].len[3 - (si & 3)] = len;
 936			sglist[si >> 2].dma_ptr[si & 3] = dma;
 937
 938			frag++;
 939			si++;
 940		}
 941		hw_desc->dptr = tx_buffer->sglist_dma;
 942	}
 943
 944	if (oct->conf->fw_info.tx_ol_flags) {
 945		if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {
 946			hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
 947			hw_desc->txm.ol_flags |= OCTEP_TX_OFFLOAD_TSO;
 948			hw_desc->txm.gso_size =  skb_shinfo(skb)->gso_size;
 949			hw_desc->txm.gso_segs =  skb_shinfo(skb)->gso_segs;
 950		} else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
 951			hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
 952		}
 953		/* due to ESR txm will be swapped by hw */
 954		hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);
 955	}
 956
 957	xmit_more = netdev_xmit_more();
 958
 959	__netdev_tx_sent_queue(iq->netdev_q, skb->len, xmit_more);
 960
 961	skb_tx_timestamp(skb);
 962	iq->fill_cnt++;
 963	wi++;
 964	iq->host_write_index = wi & iq->ring_size_mask;
 965
 966	/* octep_iq_full_check stops the queue and returns
 967	 * true if so, in case the queue has become full
 968	 * by inserting current packet. If so, we can
 969	 * go ahead and ring doorbell.
 970	 */
 971	if (!octep_iq_full_check(iq) && xmit_more &&
 972	    iq->fill_cnt < iq->fill_threshold)
 973		return NETDEV_TX_OK;
 974
 975	/* Flush the hw descriptor before writing to doorbell */
 976	wmb();
 977	/* Ring Doorbell to notify the NIC of new packets */
 978	writel(iq->fill_cnt, iq->doorbell_reg);
 979	iq->stats->instr_posted += iq->fill_cnt;
 980	iq->fill_cnt = 0;
 981	return NETDEV_TX_OK;
 982
 983dma_map_sg_err:
 984	if (si > 0) {
 985		dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
 986				 sglist[0].len[3], DMA_TO_DEVICE);
 987		sglist[0].len[3] = 0;
 988	}
 989	while (si > 1) {
 990		dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
 991			       sglist[si >> 2].len[3 - (si & 3)], DMA_TO_DEVICE);
 992		sglist[si >> 2].len[3 - (si & 3)] = 0;
 993		si--;
 994	}
 995	tx_buffer->gather = 0;
 996dma_map_err:
 997	dev_kfree_skb_any(skb);
 998	return NETDEV_TX_OK;
 999}
1000
1001/**
1002 * octep_get_stats64() - Get Octeon network device statistics.
1003 *
1004 * @netdev: kernel network device.
1005 * @stats: pointer to stats structure to be filled in.
1006 */
1007static void octep_get_stats64(struct net_device *netdev,
1008			      struct rtnl_link_stats64 *stats)
1009{
1010	struct octep_device *oct = netdev_priv(netdev);
1011	u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
1012	int q;
1013
1014	tx_packets = 0;
1015	tx_bytes = 0;
1016	rx_packets = 0;
1017	rx_bytes = 0;
1018	for (q = 0; q < OCTEP_MAX_QUEUES; q++) {
1019		tx_packets += oct->stats_iq[q].instr_completed;
1020		tx_bytes += oct->stats_iq[q].bytes_sent;
1021		rx_packets += oct->stats_oq[q].packets;
1022		rx_bytes += oct->stats_oq[q].bytes;
1023	}
1024	stats->tx_packets = tx_packets;
1025	stats->tx_bytes = tx_bytes;
1026	stats->rx_packets = rx_packets;
1027	stats->rx_bytes = rx_bytes;
1028}
1029
1030/**
1031 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
1032 *
1033 * @work: pointer to Tx queue timeout work_struct
1034 *
1035 * Stop and start the device so that it frees up all queue resources
1036 * and restarts the queues, that potentially clears a Tx queue timeout
1037 * condition.
1038 **/
1039static void octep_tx_timeout_task(struct work_struct *work)
1040{
1041	struct octep_device *oct = container_of(work, struct octep_device,
1042						tx_timeout_task);
1043	struct net_device *netdev = oct->netdev;
1044
1045	rtnl_lock();
1046	if (netif_running(netdev)) {
1047		octep_stop(netdev);
1048		octep_open(netdev);
1049	}
1050	rtnl_unlock();
1051}
1052
1053/**
1054 * octep_tx_timeout() - Handle Tx Queue timeout.
1055 *
1056 * @netdev: pointer to kernel network device.
1057 * @txqueue: Timed out Tx queue number.
1058 *
1059 * Schedule a work to handle Tx queue timeout.
1060 */
1061static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1062{
1063	struct octep_device *oct = netdev_priv(netdev);
1064
1065	queue_work(octep_wq, &oct->tx_timeout_task);
1066}
1067
1068static int octep_set_mac(struct net_device *netdev, void *p)
1069{
1070	struct octep_device *oct = netdev_priv(netdev);
1071	struct sockaddr *addr = (struct sockaddr *)p;
1072	int err;
1073
1074	if (!is_valid_ether_addr(addr->sa_data))
1075		return -EADDRNOTAVAIL;
1076
1077	err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID,
1078					  addr->sa_data, true);
1079	if (err)
1080		return err;
1081
1082	memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
1083	eth_hw_addr_set(netdev, addr->sa_data);
1084
1085	return 0;
1086}
1087
1088static int octep_change_mtu(struct net_device *netdev, int new_mtu)
1089{
1090	struct octep_device *oct = netdev_priv(netdev);
1091	struct octep_iface_link_info *link_info;
1092	int err = 0;
1093
1094	link_info = &oct->link_info;
1095	if (link_info->mtu == new_mtu)
1096		return 0;
1097
1098	err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, new_mtu,
1099				     true);
1100	if (!err) {
1101		oct->link_info.mtu = new_mtu;
1102		WRITE_ONCE(netdev->mtu, new_mtu);
1103	}
1104
1105	return err;
1106}
1107
1108static int octep_set_features(struct net_device *dev, netdev_features_t features)
1109{
1110	struct octep_ctrl_net_offloads offloads = { 0 };
1111	struct octep_device *oct = netdev_priv(dev);
1112	int err;
1113
1114	/* We only support features received from firmware */
1115	if ((features & dev->hw_features) != features)
1116		return -EINVAL;
1117
1118	if (features & NETIF_F_TSO)
1119		offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1120
1121	if (features & NETIF_F_TSO6)
1122		offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1123
1124	if (features & NETIF_F_IP_CSUM)
1125		offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1126
1127	if (features & NETIF_F_IPV6_CSUM)
1128		offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1129
1130	if (features & NETIF_F_RXCSUM)
1131		offloads.rx_offloads |= OCTEP_RX_OFFLOAD_CKSUM;
1132
1133	err = octep_ctrl_net_set_offloads(oct,
1134					  OCTEP_CTRL_NET_INVALID_VFID,
1135					  &offloads,
1136					  true);
1137	if (!err)
1138		dev->features = features;
1139
1140	return err;
1141}
1142
1143static bool octep_is_vf_valid(struct octep_device *oct, int vf)
1144{
1145	if (vf >= CFG_GET_ACTIVE_VFS(oct->conf)) {
1146		netdev_err(oct->netdev, "Invalid VF ID %d\n", vf);
1147		return false;
1148	}
1149
1150	return true;
1151}
1152
1153static int octep_get_vf_config(struct net_device *dev, int vf,
1154			       struct ifla_vf_info *ivi)
1155{
1156	struct octep_device *oct = netdev_priv(dev);
1157
1158	if (!octep_is_vf_valid(oct, vf))
1159		return -EINVAL;
1160
1161	ivi->vf = vf;
1162	ether_addr_copy(ivi->mac, oct->vf_info[vf].mac_addr);
1163	ivi->spoofchk = true;
1164	ivi->linkstate = IFLA_VF_LINK_STATE_ENABLE;
1165	ivi->trusted = false;
1166
1167	return 0;
1168}
1169
1170static int octep_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
1171{
1172	struct octep_device *oct = netdev_priv(dev);
1173	int err;
1174
1175	if (!octep_is_vf_valid(oct, vf))
1176		return -EINVAL;
1177
1178	if (!is_valid_ether_addr(mac)) {
1179		dev_err(&oct->pdev->dev, "Invalid  MAC Address %pM\n", mac);
1180		return -EADDRNOTAVAIL;
1181	}
1182
1183	dev_dbg(&oct->pdev->dev, "set vf-%d mac to %pM\n", vf, mac);
1184	ether_addr_copy(oct->vf_info[vf].mac_addr, mac);
1185	oct->vf_info[vf].flags |= OCTEON_PFVF_FLAG_MAC_SET_BY_PF;
1186
1187	err = octep_ctrl_net_set_mac_addr(oct, vf, mac, true);
1188	if (err)
1189		dev_err(&oct->pdev->dev,
1190			"Set VF%d MAC address failed via host control Mbox\n",
1191			vf);
1192
1193	return err;
1194}
1195
1196static const struct net_device_ops octep_netdev_ops = {
1197	.ndo_open                = octep_open,
1198	.ndo_stop                = octep_stop,
1199	.ndo_start_xmit          = octep_start_xmit,
1200	.ndo_get_stats64         = octep_get_stats64,
1201	.ndo_tx_timeout          = octep_tx_timeout,
1202	.ndo_set_mac_address     = octep_set_mac,
1203	.ndo_change_mtu          = octep_change_mtu,
1204	.ndo_set_features        = octep_set_features,
1205	.ndo_get_vf_config       = octep_get_vf_config,
1206	.ndo_set_vf_mac          = octep_set_vf_mac
1207};
1208
1209/**
1210 * octep_intr_poll_task - work queue task to process non-ioq interrupts.
1211 *
1212 * @work: pointer to mbox work_struct
1213 *
1214 * Process non-ioq interrupts to handle control mailbox, pfvf mailbox.
1215 **/
1216static void octep_intr_poll_task(struct work_struct *work)
1217{
1218	struct octep_device *oct = container_of(work, struct octep_device,
1219						intr_poll_task.work);
1220
1221	if (!oct->poll_non_ioq_intr) {
1222		dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n");
1223		return;
1224	}
1225
1226	oct->hw_ops.poll_non_ioq_interrupts(oct);
1227	queue_delayed_work(octep_wq, &oct->intr_poll_task,
1228			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1229}
1230
1231/**
1232 * octep_hb_timeout_task - work queue task to check firmware heartbeat.
1233 *
1234 * @work: pointer to hb work_struct
1235 *
1236 * Check for heartbeat miss count. Uninitialize oct device if miss count
1237 * exceeds configured max heartbeat miss count.
1238 *
1239 **/
1240static void octep_hb_timeout_task(struct work_struct *work)
1241{
1242	struct octep_device *oct = container_of(work, struct octep_device,
1243						hb_task.work);
1244
1245	int miss_cnt;
1246
1247	miss_cnt = atomic_inc_return(&oct->hb_miss_cnt);
1248	if (miss_cnt < oct->conf->fw_info.hb_miss_count) {
1249		queue_delayed_work(octep_wq, &oct->hb_task,
1250				   msecs_to_jiffies(oct->conf->fw_info.hb_interval));
1251		return;
1252	}
1253
1254	dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n",
1255		miss_cnt);
1256	rtnl_lock();
1257	if (netif_running(oct->netdev))
1258		dev_close(oct->netdev);
1259	rtnl_unlock();
1260}
1261
1262/**
1263 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
1264 *
1265 * @work: pointer to ctrl mbox work_struct
1266 *
1267 * Poll ctrl mbox message queue and handle control messages from firmware.
1268 **/
1269static void octep_ctrl_mbox_task(struct work_struct *work)
1270{
1271	struct octep_device *oct = container_of(work, struct octep_device,
1272						ctrl_mbox_task);
1273
1274	octep_ctrl_net_recv_fw_messages(oct);
1275}
1276
1277static const char *octep_devid_to_str(struct octep_device *oct)
1278{
1279	switch (oct->chip_id) {
1280	case OCTEP_PCI_DEVICE_ID_CN98_PF:
1281		return "CN98XX";
1282	case OCTEP_PCI_DEVICE_ID_CN93_PF:
1283		return "CN93XX";
1284	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1285		return "CNF95N";
1286	case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1287		return "CN10KA";
1288	case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1289		return "CNF10KA";
1290	case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1291		return "CNF10KB";
1292	case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1293		return "CN10KB";
1294	default:
1295		return "Unsupported";
1296	}
1297}
1298
1299/**
1300 * octep_device_setup() - Setup Octeon Device.
1301 *
1302 * @oct: Octeon device private data structure.
1303 *
1304 * Setup Octeon device hardware operations, configuration, etc ...
1305 */
1306int octep_device_setup(struct octep_device *oct)
1307{
1308	struct pci_dev *pdev = oct->pdev;
1309	int i, ret;
1310
1311	/* allocate memory for oct->conf */
1312	oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
1313	if (!oct->conf)
1314		return -ENOMEM;
1315
1316	/* Map BAR regions */
1317	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1318		oct->mmio[i].hw_addr =
1319			ioremap(pci_resource_start(oct->pdev, i * 2),
1320				pci_resource_len(oct->pdev, i * 2));
1321		if (!oct->mmio[i].hw_addr)
1322			goto unmap_prev;
1323
1324		oct->mmio[i].mapped = 1;
1325	}
1326
1327	oct->chip_id = pdev->device;
1328	oct->rev_id = pdev->revision;
1329	dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
1330
1331	switch (oct->chip_id) {
1332	case OCTEP_PCI_DEVICE_ID_CN98_PF:
1333	case OCTEP_PCI_DEVICE_ID_CN93_PF:
1334	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1335		dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1336			 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct),
1337			 OCTEP_MINOR_REV(oct));
1338		octep_device_setup_cn93_pf(oct);
1339		break;
1340	case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1341	case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1342	case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1343	case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1344		dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1345			 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
1346		octep_device_setup_cnxk_pf(oct);
1347		break;
1348	default:
1349		dev_err(&pdev->dev,
1350			"%s: unsupported device\n", __func__);
1351		goto unsupported_dev;
1352	}
1353
1354
1355	ret = octep_ctrl_net_init(oct);
1356	if (ret)
1357		goto unsupported_dev;
1358
1359	INIT_WORK(&oct->tx_timeout_task, octep_tx_timeout_task);
1360	INIT_WORK(&oct->ctrl_mbox_task, octep_ctrl_mbox_task);
1361	INIT_DELAYED_WORK(&oct->intr_poll_task, octep_intr_poll_task);
1362	oct->poll_non_ioq_intr = true;
1363	queue_delayed_work(octep_wq, &oct->intr_poll_task,
1364			   msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1365
1366	atomic_set(&oct->hb_miss_cnt, 0);
1367	INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task);
1368
1369	return 0;
1370
1371unsupported_dev:
1372	i = OCTEP_MMIO_REGIONS;
1373unmap_prev:
1374	while (i--)
1375		iounmap(oct->mmio[i].hw_addr);
1376
1377	kfree(oct->conf);
1378	return -1;
1379}
1380
1381/**
1382 * octep_device_cleanup() - Cleanup Octeon Device.
1383 *
1384 * @oct: Octeon device private data structure.
1385 *
1386 * Cleanup Octeon device allocated resources.
1387 */
1388static void octep_device_cleanup(struct octep_device *oct)
1389{
1390	int i;
1391
1392	oct->poll_non_ioq_intr = false;
1393	cancel_delayed_work_sync(&oct->intr_poll_task);
1394	cancel_work_sync(&oct->ctrl_mbox_task);
1395
1396	dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1397
1398	for (i = 0; i < OCTEP_MAX_VF; i++) {
1399		vfree(oct->mbox[i]);
1400		oct->mbox[i] = NULL;
1401	}
1402
1403	octep_delete_pfvf_mbox(oct);
1404	octep_ctrl_net_uninit(oct);
1405	cancel_delayed_work_sync(&oct->hb_task);
1406
1407	oct->hw_ops.soft_reset(oct);
1408	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1409		if (oct->mmio[i].mapped)
1410			iounmap(oct->mmio[i].hw_addr);
1411	}
1412
1413	kfree(oct->conf);
1414	oct->conf = NULL;
1415}
1416
1417static bool get_fw_ready_status(struct pci_dev *pdev)
1418{
1419	u32 pos = 0;
1420	u16 vsec_id;
1421	u8 status;
1422
1423	while ((pos = pci_find_next_ext_capability(pdev, pos,
1424						   PCI_EXT_CAP_ID_VNDR))) {
1425		pci_read_config_word(pdev, pos + 4, &vsec_id);
1426#define FW_STATUS_VSEC_ID  0xA3
1427		if (vsec_id != FW_STATUS_VSEC_ID)
1428			continue;
1429
1430		pci_read_config_byte(pdev, (pos + 8), &status);
1431		dev_info(&pdev->dev, "Firmware ready status = %u\n", status);
1432#define FW_STATUS_READY 1ULL
1433		return status == FW_STATUS_READY;
1434	}
1435	return false;
1436}
1437
1438/**
1439 * octep_probe() - Octeon PCI device probe handler.
1440 *
1441 * @pdev: PCI device structure.
1442 * @ent: entry in Octeon PCI device ID table.
1443 *
1444 * Initializes and enables the Octeon PCI device for network operations.
1445 * Initializes Octeon private data structure and registers a network device.
1446 */
1447static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1448{
1449	struct octep_device *octep_dev = NULL;
1450	struct net_device *netdev;
1451	int max_rx_pktlen;
1452	int err;
1453
1454	err = pci_enable_device(pdev);
1455	if (err) {
1456		dev_err(&pdev->dev, "Failed to enable PCI device\n");
1457		return  err;
1458	}
1459
1460	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1461	if (err) {
1462		dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1463		goto err_dma_mask;
1464	}
1465
1466	err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1467	if (err) {
1468		dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1469		goto err_pci_regions;
1470	}
1471
1472	pci_set_master(pdev);
1473
1474	if (!get_fw_ready_status(pdev)) {
1475		dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n");
1476		err = -EPROBE_DEFER;
1477		goto err_alloc_netdev;
1478	}
1479
1480	netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1481				   OCTEP_MAX_QUEUES);
1482	if (!netdev) {
1483		dev_err(&pdev->dev, "Failed to allocate netdev\n");
1484		err = -ENOMEM;
1485		goto err_alloc_netdev;
1486	}
1487	SET_NETDEV_DEV(netdev, &pdev->dev);
1488
1489	octep_dev = netdev_priv(netdev);
1490	octep_dev->netdev = netdev;
1491	octep_dev->pdev = pdev;
1492	octep_dev->dev = &pdev->dev;
1493	pci_set_drvdata(pdev, octep_dev);
1494
1495	err = octep_device_setup(octep_dev);
1496	if (err) {
1497		dev_err(&pdev->dev, "Device setup failed\n");
1498		goto err_octep_config;
1499	}
1500
1501	err = octep_setup_pfvf_mbox(octep_dev);
1502	if (err) {
1503		dev_err(&pdev->dev, "PF-VF mailbox setup failed\n");
1504		goto register_dev_err;
1505	}
1506
1507	err = octep_ctrl_net_get_info(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1508				      &octep_dev->conf->fw_info);
1509	if (err) {
1510		dev_err(&pdev->dev, "Failed to get firmware info\n");
1511		goto register_dev_err;
1512	}
1513	dev_info(&octep_dev->pdev->dev, "Heartbeat interval %u msecs Heartbeat miss count %u\n",
1514		 octep_dev->conf->fw_info.hb_interval,
1515		 octep_dev->conf->fw_info.hb_miss_count);
1516	queue_delayed_work(octep_wq, &octep_dev->hb_task,
1517			   msecs_to_jiffies(octep_dev->conf->fw_info.hb_interval));
1518
1519	netdev->netdev_ops = &octep_netdev_ops;
1520	octep_set_ethtool_ops(netdev);
1521	netif_carrier_off(netdev);
1522
1523	netdev->hw_features = NETIF_F_SG;
1524	if (OCTEP_TX_IP_CSUM(octep_dev->conf->fw_info.tx_ol_flags))
1525		netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
1526
1527	if (OCTEP_RX_IP_CSUM(octep_dev->conf->fw_info.rx_ol_flags))
1528		netdev->hw_features |= NETIF_F_RXCSUM;
1529
1530	max_rx_pktlen = octep_ctrl_net_get_mtu(octep_dev, OCTEP_CTRL_NET_INVALID_VFID);
1531	if (max_rx_pktlen < 0) {
1532		dev_err(&octep_dev->pdev->dev,
1533			"Failed to get max receive packet size; err = %d\n", max_rx_pktlen);
1534		err = max_rx_pktlen;
1535		goto register_dev_err;
1536	}
1537	netdev->min_mtu = OCTEP_MIN_MTU;
1538	netdev->max_mtu = max_rx_pktlen - (ETH_HLEN + ETH_FCS_LEN);
1539	netdev->mtu = OCTEP_DEFAULT_MTU;
1540
1541	if (OCTEP_TX_TSO(octep_dev->conf->fw_info.tx_ol_flags)) {
1542		netdev->hw_features |= NETIF_F_TSO;
1543		netif_set_tso_max_size(netdev, netdev->max_mtu);
1544	}
1545
1546	netdev->features |= netdev->hw_features;
1547	err = octep_ctrl_net_get_mac_addr(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1548					  octep_dev->mac_addr);
1549	if (err) {
1550		dev_err(&pdev->dev, "Failed to get mac address\n");
1551		goto register_dev_err;
1552	}
1553	eth_hw_addr_set(netdev, octep_dev->mac_addr);
1554
1555	err = register_netdev(netdev);
1556	if (err) {
1557		dev_err(&pdev->dev, "Failed to register netdev\n");
1558		goto register_dev_err;
1559	}
1560	dev_info(&pdev->dev, "Device probe successful\n");
1561	return 0;
1562
1563register_dev_err:
1564	octep_device_cleanup(octep_dev);
1565err_octep_config:
1566	free_netdev(netdev);
1567err_alloc_netdev:
1568	pci_release_mem_regions(pdev);
1569err_pci_regions:
1570err_dma_mask:
1571	pci_disable_device(pdev);
1572	return err;
1573}
1574
1575static int octep_sriov_disable(struct octep_device *oct)
1576{
1577	struct pci_dev *pdev = oct->pdev;
1578
1579	if (pci_vfs_assigned(oct->pdev)) {
1580		dev_warn(&pdev->dev, "Can't disable SRIOV while VFs are assigned\n");
1581		return -EPERM;
1582	}
1583
1584	pci_disable_sriov(pdev);
1585	CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1586
1587	return 0;
1588}
1589
1590/**
1591 * octep_remove() - Remove Octeon PCI device from driver control.
1592 *
1593 * @pdev: PCI device structure of the Octeon device.
1594 *
1595 * Cleanup all resources allocated for the Octeon device.
1596 * Unregister from network device and disable the PCI device.
1597 */
1598static void octep_remove(struct pci_dev *pdev)
1599{
1600	struct octep_device *oct = pci_get_drvdata(pdev);
1601	struct net_device *netdev;
1602
1603	if (!oct)
1604		return;
1605
1606	netdev = oct->netdev;
1607	octep_sriov_disable(oct);
1608	if (netdev->reg_state == NETREG_REGISTERED)
1609		unregister_netdev(netdev);
1610
1611	cancel_work_sync(&oct->tx_timeout_task);
1612	octep_device_cleanup(oct);
1613	pci_release_mem_regions(pdev);
1614	free_netdev(netdev);
1615	pci_disable_device(pdev);
1616}
1617
1618static int octep_sriov_enable(struct octep_device *oct, int num_vfs)
1619{
1620	struct pci_dev *pdev = oct->pdev;
1621	int err;
1622
1623	CFG_GET_ACTIVE_VFS(oct->conf) = num_vfs;
1624	err = pci_enable_sriov(pdev, num_vfs);
1625	if (err) {
1626		dev_warn(&pdev->dev, "Failed to enable SRIOV err=%d\n", err);
1627		CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1628		return err;
1629	}
1630
1631	return num_vfs;
1632}
1633
1634static int octep_sriov_configure(struct pci_dev *pdev, int num_vfs)
1635{
1636	struct octep_device *oct = pci_get_drvdata(pdev);
1637	int max_nvfs;
1638
1639	if (num_vfs == 0)
1640		return octep_sriov_disable(oct);
1641
1642	max_nvfs = CFG_GET_MAX_VFS(oct->conf);
1643
1644	if (num_vfs > max_nvfs) {
1645		dev_err(&pdev->dev, "Invalid VF count Max supported VFs = %d\n",
1646			max_nvfs);
1647		return -EINVAL;
1648	}
1649
1650	return octep_sriov_enable(oct, num_vfs);
1651}
1652
1653static struct pci_driver octep_driver = {
1654	.name = OCTEP_DRV_NAME,
1655	.id_table = octep_pci_id_tbl,
1656	.probe = octep_probe,
1657	.remove = octep_remove,
1658	.sriov_configure = octep_sriov_configure,
1659};
1660
1661/**
1662 * octep_init_module() - Module initialiation.
1663 *
1664 * create common resource for the driver and register PCI driver.
1665 */
1666static int __init octep_init_module(void)
1667{
1668	int ret;
1669
1670	pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1671
1672	/* work queue for all deferred tasks */
1673	octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1674	if (!octep_wq) {
1675		pr_err("%s: Failed to create common workqueue\n",
1676		       OCTEP_DRV_NAME);
1677		return -ENOMEM;
1678	}
1679
1680	ret = pci_register_driver(&octep_driver);
1681	if (ret < 0) {
1682		pr_err("%s: Failed to register PCI driver; err=%d\n",
1683		       OCTEP_DRV_NAME, ret);
1684		destroy_workqueue(octep_wq);
1685		return ret;
1686	}
1687
1688	pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1689
1690	return ret;
1691}
1692
1693/**
1694 * octep_exit_module() - Module exit routine.
1695 *
1696 * unregister the driver with PCI subsystem and cleanup common resources.
1697 */
1698static void __exit octep_exit_module(void)
1699{
1700	pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1701
1702	pci_unregister_driver(&octep_driver);
1703	destroy_workqueue(octep_wq);
1704
1705	pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1706}
1707
1708module_init(octep_init_module);
1709module_exit(octep_exit_module);