bnge_netdev.c

   1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (c) 2025 Broadcom.
   3
   4#include <asm/byteorder.h>
   5#include <linux/dma-mapping.h>
   6#include <linux/dmapool.h>
   7#include <linux/delay.h>
   8#include <linux/errno.h>
   9#include <linux/kernel.h>
  10#include <linux/list.h>
  11#include <linux/pci.h>
  12#include <linux/netdevice.h>
  13#include <linux/etherdevice.h>
  14#include <linux/if.h>
  15#include <net/ip.h>
  16#include <linux/skbuff.h>
  17#include <net/page_pool/helpers.h>
  18
  19#include "bnge.h"
  20#include "bnge_hwrm_lib.h"
  21#include "bnge_ethtool.h"
  22#include "bnge_rmem.h"
  23
  24#define BNGE_RING_TO_TC_OFF(bd, tx)	\
  25	((tx) % (bd)->tx_nr_rings_per_tc)
  26
  27#define BNGE_RING_TO_TC(bd, tx)		\
  28	((tx) / (bd)->tx_nr_rings_per_tc)
  29
  30#define BNGE_TC_TO_RING_BASE(bd, tc)	\
  31	((tc) * (bd)->tx_nr_rings_per_tc)
  32
  33static void bnge_free_stats_mem(struct bnge_net *bn,
  34				struct bnge_stats_mem *stats)
  35{
  36	struct bnge_dev *bd = bn->bd;
  37
  38	if (stats->hw_stats) {
  39		dma_free_coherent(bd->dev, stats->len, stats->hw_stats,
  40				  stats->hw_stats_map);
  41		stats->hw_stats = NULL;
  42	}
  43}
  44
  45static int bnge_alloc_stats_mem(struct bnge_net *bn,
  46				struct bnge_stats_mem *stats)
  47{
  48	struct bnge_dev *bd = bn->bd;
  49
  50	stats->hw_stats = dma_alloc_coherent(bd->dev, stats->len,
  51					     &stats->hw_stats_map, GFP_KERNEL);
  52	if (!stats->hw_stats)
  53		return -ENOMEM;
  54
  55	return 0;
  56}
  57
  58static void bnge_free_ring_stats(struct bnge_net *bn)
  59{
  60	struct bnge_dev *bd = bn->bd;
  61	int i;
  62
  63	if (!bn->bnapi)
  64		return;
  65
  66	for (i = 0; i < bd->nq_nr_rings; i++) {
  67		struct bnge_napi *bnapi = bn->bnapi[i];
  68		struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
  69
  70		bnge_free_stats_mem(bn, &nqr->stats);
  71	}
  72}
  73
  74static int bnge_alloc_ring_stats(struct bnge_net *bn)
  75{
  76	struct bnge_dev *bd = bn->bd;
  77	u32 size, i;
  78	int rc;
  79
  80	size = bd->hw_ring_stats_size;
  81
  82	for (i = 0; i < bd->nq_nr_rings; i++) {
  83		struct bnge_napi *bnapi = bn->bnapi[i];
  84		struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
  85
  86		nqr->stats.len = size;
  87		rc = bnge_alloc_stats_mem(bn, &nqr->stats);
  88		if (rc)
  89			goto err_free_ring_stats;
  90
  91		nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
  92	}
  93	return 0;
  94
  95err_free_ring_stats:
  96	bnge_free_ring_stats(bn);
  97	return rc;
  98}
  99
 100static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr)
 101{
 102	struct bnge_ring_struct *ring = &nqr->ring_struct;
 103
 104	kfree(nqr->desc_ring);
 105	nqr->desc_ring = NULL;
 106	ring->ring_mem.pg_arr = NULL;
 107	kfree(nqr->desc_mapping);
 108	nqr->desc_mapping = NULL;
 109	ring->ring_mem.dma_arr = NULL;
 110}
 111
 112static void bnge_free_cp_desc_arr(struct bnge_cp_ring_info *cpr)
 113{
 114	struct bnge_ring_struct *ring = &cpr->ring_struct;
 115
 116	kfree(cpr->desc_ring);
 117	cpr->desc_ring = NULL;
 118	ring->ring_mem.pg_arr = NULL;
 119	kfree(cpr->desc_mapping);
 120	cpr->desc_mapping = NULL;
 121	ring->ring_mem.dma_arr = NULL;
 122}
 123
 124static int bnge_alloc_nq_desc_arr(struct bnge_nq_ring_info *nqr, int n)
 125{
 126	nqr->desc_ring = kcalloc(n, sizeof(*nqr->desc_ring), GFP_KERNEL);
 127	if (!nqr->desc_ring)
 128		return -ENOMEM;
 129
 130	nqr->desc_mapping = kcalloc(n, sizeof(*nqr->desc_mapping), GFP_KERNEL);
 131	if (!nqr->desc_mapping)
 132		goto err_free_desc_ring;
 133	return 0;
 134
 135err_free_desc_ring:
 136	kfree(nqr->desc_ring);
 137	nqr->desc_ring = NULL;
 138	return -ENOMEM;
 139}
 140
 141static int bnge_alloc_cp_desc_arr(struct bnge_cp_ring_info *cpr, int n)
 142{
 143	cpr->desc_ring = kcalloc(n, sizeof(*cpr->desc_ring), GFP_KERNEL);
 144	if (!cpr->desc_ring)
 145		return -ENOMEM;
 146
 147	cpr->desc_mapping = kcalloc(n, sizeof(*cpr->desc_mapping), GFP_KERNEL);
 148	if (!cpr->desc_mapping)
 149		goto err_free_desc_ring;
 150	return 0;
 151
 152err_free_desc_ring:
 153	kfree(cpr->desc_ring);
 154	cpr->desc_ring = NULL;
 155	return -ENOMEM;
 156}
 157
 158static void bnge_free_nq_arrays(struct bnge_net *bn)
 159{
 160	struct bnge_dev *bd = bn->bd;
 161	int i;
 162
 163	for (i = 0; i < bd->nq_nr_rings; i++) {
 164		struct bnge_napi *bnapi = bn->bnapi[i];
 165
 166		bnge_free_nq_desc_arr(&bnapi->nq_ring);
 167	}
 168}
 169
 170static int bnge_alloc_nq_arrays(struct bnge_net *bn)
 171{
 172	struct bnge_dev *bd = bn->bd;
 173	int i, rc;
 174
 175	for (i = 0; i < bd->nq_nr_rings; i++) {
 176		struct bnge_napi *bnapi = bn->bnapi[i];
 177
 178		rc = bnge_alloc_nq_desc_arr(&bnapi->nq_ring, bn->cp_nr_pages);
 179		if (rc)
 180			goto err_free_nq_arrays;
 181	}
 182	return 0;
 183
 184err_free_nq_arrays:
 185	bnge_free_nq_arrays(bn);
 186	return rc;
 187}
 188
 189static void bnge_free_nq_tree(struct bnge_net *bn)
 190{
 191	struct bnge_dev *bd = bn->bd;
 192	int i;
 193
 194	for (i = 0; i < bd->nq_nr_rings; i++) {
 195		struct bnge_napi *bnapi = bn->bnapi[i];
 196		struct bnge_nq_ring_info *nqr;
 197		struct bnge_ring_struct *ring;
 198		int j;
 199
 200		nqr = &bnapi->nq_ring;
 201		ring = &nqr->ring_struct;
 202
 203		bnge_free_ring(bd, &ring->ring_mem);
 204
 205		if (!nqr->cp_ring_arr)
 206			continue;
 207
 208		for (j = 0; j < nqr->cp_ring_count; j++) {
 209			struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j];
 210
 211			ring = &cpr->ring_struct;
 212			bnge_free_ring(bd, &ring->ring_mem);
 213			bnge_free_cp_desc_arr(cpr);
 214		}
 215		kfree(nqr->cp_ring_arr);
 216		nqr->cp_ring_arr = NULL;
 217		nqr->cp_ring_count = 0;
 218	}
 219}
 220
 221static int alloc_one_cp_ring(struct bnge_net *bn,
 222			     struct bnge_cp_ring_info *cpr)
 223{
 224	struct bnge_ring_mem_info *rmem;
 225	struct bnge_ring_struct *ring;
 226	struct bnge_dev *bd = bn->bd;
 227	int rc;
 228
 229	rc = bnge_alloc_cp_desc_arr(cpr, bn->cp_nr_pages);
 230	if (rc)
 231		return -ENOMEM;
 232	ring = &cpr->ring_struct;
 233	rmem = &ring->ring_mem;
 234	rmem->nr_pages = bn->cp_nr_pages;
 235	rmem->page_size = HW_CMPD_RING_SIZE;
 236	rmem->pg_arr = (void **)cpr->desc_ring;
 237	rmem->dma_arr = cpr->desc_mapping;
 238	rmem->flags = BNGE_RMEM_RING_PTE_FLAG;
 239	rc = bnge_alloc_ring(bd, rmem);
 240	if (rc)
 241		goto err_free_cp_desc_arr;
 242	return rc;
 243
 244err_free_cp_desc_arr:
 245	bnge_free_cp_desc_arr(cpr);
 246	return rc;
 247}
 248
 249static int bnge_alloc_nq_tree(struct bnge_net *bn)
 250{
 251	struct bnge_dev *bd = bn->bd;
 252	int i, j, ulp_msix, rc;
 253	int tcs = 1;
 254
 255	ulp_msix = bnge_aux_get_msix(bd);
 256	for (i = 0, j = 0; i < bd->nq_nr_rings; i++) {
 257		bool sh = !!(bd->flags & BNGE_EN_SHARED_CHNL);
 258		struct bnge_napi *bnapi = bn->bnapi[i];
 259		struct bnge_nq_ring_info *nqr;
 260		struct bnge_cp_ring_info *cpr;
 261		struct bnge_ring_struct *ring;
 262		int cp_count = 0, k;
 263		int rx = 0, tx = 0;
 264
 265		nqr = &bnapi->nq_ring;
 266		nqr->bnapi = bnapi;
 267		ring = &nqr->ring_struct;
 268
 269		rc = bnge_alloc_ring(bd, &ring->ring_mem);
 270		if (rc)
 271			goto err_free_nq_tree;
 272
 273		ring->map_idx = ulp_msix + i;
 274
 275		if (i < bd->rx_nr_rings) {
 276			cp_count++;
 277			rx = 1;
 278		}
 279
 280		if ((sh && i < bd->tx_nr_rings) ||
 281		    (!sh && i >= bd->rx_nr_rings)) {
 282			cp_count += tcs;
 283			tx = 1;
 284		}
 285
 286		nqr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
 287					   GFP_KERNEL);
 288		if (!nqr->cp_ring_arr) {
 289			rc = -ENOMEM;
 290			goto err_free_nq_tree;
 291		}
 292
 293		nqr->cp_ring_count = cp_count;
 294
 295		for (k = 0; k < cp_count; k++) {
 296			cpr = &nqr->cp_ring_arr[k];
 297			rc = alloc_one_cp_ring(bn, cpr);
 298			if (rc)
 299				goto err_free_nq_tree;
 300
 301			cpr->bnapi = bnapi;
 302			cpr->cp_idx = k;
 303			if (!k && rx) {
 304				bn->rx_ring[i].rx_cpr = cpr;
 305				cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_RX;
 306			} else {
 307				int n, tc = k - rx;
 308
 309				n = BNGE_TC_TO_RING_BASE(bd, tc) + j;
 310				bn->tx_ring[n].tx_cpr = cpr;
 311				cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_TX;
 312			}
 313		}
 314		if (tx)
 315			j++;
 316	}
 317	return 0;
 318
 319err_free_nq_tree:
 320	bnge_free_nq_tree(bn);
 321	return rc;
 322}
 323
 324static bool bnge_separate_head_pool(struct bnge_rx_ring_info *rxr)
 325{
 326	return rxr->need_head_pool || PAGE_SIZE > BNGE_RX_PAGE_SIZE;
 327}
 328
 329static void bnge_free_one_rx_ring_bufs(struct bnge_net *bn,
 330				       struct bnge_rx_ring_info *rxr)
 331{
 332	int i, max_idx;
 333
 334	if (!rxr->rx_buf_ring)
 335		return;
 336
 337	max_idx = bn->rx_nr_pages * RX_DESC_CNT;
 338
 339	for (i = 0; i < max_idx; i++) {
 340		struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
 341		void *data = rx_buf->data;
 342
 343		if (!data)
 344			continue;
 345
 346		rx_buf->data = NULL;
 347		page_pool_free_va(rxr->head_pool, data, true);
 348	}
 349}
 350
 351static void bnge_free_one_agg_ring_bufs(struct bnge_net *bn,
 352					struct bnge_rx_ring_info *rxr)
 353{
 354	int i, max_idx;
 355
 356	if (!rxr->rx_agg_buf_ring)
 357		return;
 358
 359	max_idx = bn->rx_agg_nr_pages * RX_DESC_CNT;
 360
 361	for (i = 0; i < max_idx; i++) {
 362		struct bnge_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_buf_ring[i];
 363		netmem_ref netmem = rx_agg_buf->netmem;
 364
 365		if (!netmem)
 366			continue;
 367
 368		rx_agg_buf->netmem = 0;
 369		__clear_bit(i, rxr->rx_agg_bmap);
 370
 371		page_pool_recycle_direct_netmem(rxr->page_pool, netmem);
 372	}
 373}
 374
 375static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn,
 376					    struct bnge_rx_ring_info *rxr)
 377{
 378	bnge_free_one_rx_ring_bufs(bn, rxr);
 379	bnge_free_one_agg_ring_bufs(bn, rxr);
 380}
 381
 382static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn)
 383{
 384	struct bnge_dev *bd = bn->bd;
 385	int i;
 386
 387	if (!bn->rx_ring)
 388		return;
 389
 390	for (i = 0; i < bd->rx_nr_rings; i++)
 391		bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]);
 392}
 393
 394static void bnge_free_all_rings_bufs(struct bnge_net *bn)
 395{
 396	bnge_free_rx_ring_pair_bufs(bn);
 397}
 398
 399static void bnge_free_rx_rings(struct bnge_net *bn)
 400{
 401	struct bnge_dev *bd = bn->bd;
 402	int i;
 403
 404	for (i = 0; i < bd->rx_nr_rings; i++) {
 405		struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
 406		struct bnge_ring_struct *ring;
 407
 408		page_pool_destroy(rxr->page_pool);
 409		page_pool_destroy(rxr->head_pool);
 410		rxr->page_pool = rxr->head_pool = NULL;
 411
 412		kfree(rxr->rx_agg_bmap);
 413		rxr->rx_agg_bmap = NULL;
 414
 415		ring = &rxr->rx_ring_struct;
 416		bnge_free_ring(bd, &ring->ring_mem);
 417
 418		ring = &rxr->rx_agg_ring_struct;
 419		bnge_free_ring(bd, &ring->ring_mem);
 420	}
 421}
 422
 423static int bnge_alloc_rx_page_pool(struct bnge_net *bn,
 424				   struct bnge_rx_ring_info *rxr,
 425				   int numa_node)
 426{
 427	const unsigned int agg_size_fac = PAGE_SIZE / BNGE_RX_PAGE_SIZE;
 428	const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K;
 429	struct page_pool_params pp = { 0 };
 430	struct bnge_dev *bd = bn->bd;
 431	struct page_pool *pool;
 432
 433	pp.pool_size = bn->rx_agg_ring_size / agg_size_fac;
 434	pp.nid = numa_node;
 435	pp.netdev = bn->netdev;
 436	pp.dev = bd->dev;
 437	pp.dma_dir = bn->rx_dir;
 438	pp.max_len = PAGE_SIZE;
 439	pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV |
 440		   PP_FLAG_ALLOW_UNREADABLE_NETMEM;
 441	pp.queue_idx = rxr->bnapi->index;
 442
 443	pool = page_pool_create(&pp);
 444	if (IS_ERR(pool))
 445		return PTR_ERR(pool);
 446	rxr->page_pool = pool;
 447
 448	rxr->need_head_pool = page_pool_is_unreadable(pool);
 449	if (bnge_separate_head_pool(rxr)) {
 450		pp.pool_size = min(bn->rx_ring_size / rx_size_fac, 1024);
 451		pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
 452		pool = page_pool_create(&pp);
 453		if (IS_ERR(pool))
 454			goto err_destroy_pp;
 455	} else {
 456		page_pool_get(pool);
 457	}
 458	rxr->head_pool = pool;
 459	return 0;
 460
 461err_destroy_pp:
 462	page_pool_destroy(rxr->page_pool);
 463	rxr->page_pool = NULL;
 464	return PTR_ERR(pool);
 465}
 466
 467static void bnge_enable_rx_page_pool(struct bnge_rx_ring_info *rxr)
 468{
 469	page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi);
 470	page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi);
 471}
 472
 473static int bnge_alloc_rx_agg_bmap(struct bnge_net *bn,
 474				  struct bnge_rx_ring_info *rxr)
 475{
 476	u16 mem_size;
 477
 478	rxr->rx_agg_bmap_size = bn->rx_agg_ring_mask + 1;
 479	mem_size = rxr->rx_agg_bmap_size / 8;
 480	rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
 481	if (!rxr->rx_agg_bmap)
 482		return -ENOMEM;
 483
 484	return 0;
 485}
 486
 487static int bnge_alloc_rx_rings(struct bnge_net *bn)
 488{
 489	int i, rc = 0, agg_rings = 0, cpu;
 490	struct bnge_dev *bd = bn->bd;
 491
 492	if (bnge_is_agg_reqd(bd))
 493		agg_rings = 1;
 494
 495	for (i = 0; i < bd->rx_nr_rings; i++) {
 496		struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
 497		struct bnge_ring_struct *ring;
 498		int cpu_node;
 499
 500		ring = &rxr->rx_ring_struct;
 501
 502		cpu = cpumask_local_spread(i, dev_to_node(bd->dev));
 503		cpu_node = cpu_to_node(cpu);
 504		netdev_dbg(bn->netdev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n",
 505			   i, cpu_node);
 506		rc = bnge_alloc_rx_page_pool(bn, rxr, cpu_node);
 507		if (rc)
 508			goto err_free_rx_rings;
 509		bnge_enable_rx_page_pool(rxr);
 510
 511		rc = bnge_alloc_ring(bd, &ring->ring_mem);
 512		if (rc)
 513			goto err_free_rx_rings;
 514
 515		ring->grp_idx = i;
 516		if (agg_rings) {
 517			ring = &rxr->rx_agg_ring_struct;
 518			rc = bnge_alloc_ring(bd, &ring->ring_mem);
 519			if (rc)
 520				goto err_free_rx_rings;
 521
 522			ring->grp_idx = i;
 523			rc = bnge_alloc_rx_agg_bmap(bn, rxr);
 524			if (rc)
 525				goto err_free_rx_rings;
 526		}
 527	}
 528	return rc;
 529
 530err_free_rx_rings:
 531	bnge_free_rx_rings(bn);
 532	return rc;
 533}
 534
 535static void bnge_free_tx_rings(struct bnge_net *bn)
 536{
 537	struct bnge_dev *bd = bn->bd;
 538	int i;
 539
 540	for (i = 0; i < bd->tx_nr_rings; i++) {
 541		struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
 542		struct bnge_ring_struct *ring;
 543
 544		ring = &txr->tx_ring_struct;
 545
 546		bnge_free_ring(bd, &ring->ring_mem);
 547	}
 548}
 549
 550static int bnge_alloc_tx_rings(struct bnge_net *bn)
 551{
 552	struct bnge_dev *bd = bn->bd;
 553	int i, j, rc;
 554
 555	for (i = 0, j = 0; i < bd->tx_nr_rings; i++) {
 556		struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
 557		struct bnge_ring_struct *ring;
 558		u8 qidx;
 559
 560		ring = &txr->tx_ring_struct;
 561
 562		rc = bnge_alloc_ring(bd, &ring->ring_mem);
 563		if (rc)
 564			goto err_free_tx_rings;
 565
 566		ring->grp_idx = txr->bnapi->index;
 567		qidx = bd->tc_to_qidx[j];
 568		ring->queue_id = bd->q_info[qidx].queue_id;
 569		if (BNGE_RING_TO_TC_OFF(bd, i) == (bd->tx_nr_rings_per_tc - 1))
 570			j++;
 571	}
 572	return 0;
 573
 574err_free_tx_rings:
 575	bnge_free_tx_rings(bn);
 576	return rc;
 577}
 578
 579static void bnge_free_vnic_attributes(struct bnge_net *bn)
 580{
 581	struct pci_dev *pdev = bn->bd->pdev;
 582	struct bnge_vnic_info *vnic;
 583	int i;
 584
 585	if (!bn->vnic_info)
 586		return;
 587
 588	for (i = 0; i < bn->nr_vnics; i++) {
 589		vnic = &bn->vnic_info[i];
 590
 591		kfree(vnic->uc_list);
 592		vnic->uc_list = NULL;
 593
 594		if (vnic->mc_list) {
 595			dma_free_coherent(&pdev->dev, vnic->mc_list_size,
 596					  vnic->mc_list, vnic->mc_list_mapping);
 597			vnic->mc_list = NULL;
 598		}
 599
 600		if (vnic->rss_table) {
 601			dma_free_coherent(&pdev->dev, vnic->rss_table_size,
 602					  vnic->rss_table,
 603					  vnic->rss_table_dma_addr);
 604			vnic->rss_table = NULL;
 605		}
 606
 607		vnic->rss_hash_key = NULL;
 608		vnic->flags = 0;
 609	}
 610}
 611
 612static int bnge_alloc_vnic_attributes(struct bnge_net *bn)
 613{
 614	struct bnge_dev *bd = bn->bd;
 615	struct bnge_vnic_info *vnic;
 616	int i, size;
 617
 618	for (i = 0; i < bn->nr_vnics; i++) {
 619		vnic = &bn->vnic_info[i];
 620
 621		if (vnic->flags & BNGE_VNIC_UCAST_FLAG) {
 622			int mem_size = (BNGE_MAX_UC_ADDRS - 1) * ETH_ALEN;
 623
 624			vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
 625			if (!vnic->uc_list)
 626				goto err_free_vnic_attributes;
 627		}
 628
 629		if (vnic->flags & BNGE_VNIC_MCAST_FLAG) {
 630			vnic->mc_list_size = BNGE_MAX_MC_ADDRS * ETH_ALEN;
 631			vnic->mc_list =
 632				dma_alloc_coherent(bd->dev,
 633						   vnic->mc_list_size,
 634						   &vnic->mc_list_mapping,
 635						   GFP_KERNEL);
 636			if (!vnic->mc_list)
 637				goto err_free_vnic_attributes;
 638		}
 639
 640		/* Allocate rss table and hash key */
 641		size = L1_CACHE_ALIGN(BNGE_MAX_RSS_TABLE_SIZE);
 642
 643		vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
 644		vnic->rss_table = dma_alloc_coherent(bd->dev,
 645						     vnic->rss_table_size,
 646						     &vnic->rss_table_dma_addr,
 647						     GFP_KERNEL);
 648		if (!vnic->rss_table)
 649			goto err_free_vnic_attributes;
 650
 651		vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
 652		vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
 653	}
 654	return 0;
 655
 656err_free_vnic_attributes:
 657	bnge_free_vnic_attributes(bn);
 658	return -ENOMEM;
 659}
 660
 661static int bnge_alloc_vnics(struct bnge_net *bn)
 662{
 663	int num_vnics;
 664
 665	/* Allocate only 1 VNIC for now
 666	 * Additional VNICs will be added based on RFS/NTUPLE in future patches
 667	 */
 668	num_vnics = 1;
 669
 670	bn->vnic_info = kcalloc(num_vnics, sizeof(struct bnge_vnic_info),
 671				GFP_KERNEL);
 672	if (!bn->vnic_info)
 673		return -ENOMEM;
 674
 675	bn->nr_vnics = num_vnics;
 676
 677	return 0;
 678}
 679
 680static void bnge_free_vnics(struct bnge_net *bn)
 681{
 682	kfree(bn->vnic_info);
 683	bn->vnic_info = NULL;
 684	bn->nr_vnics = 0;
 685}
 686
 687static void bnge_free_ring_grps(struct bnge_net *bn)
 688{
 689	kfree(bn->grp_info);
 690	bn->grp_info = NULL;
 691}
 692
 693static int bnge_init_ring_grps(struct bnge_net *bn)
 694{
 695	struct bnge_dev *bd = bn->bd;
 696	int i;
 697
 698	bn->grp_info = kcalloc(bd->nq_nr_rings,
 699			       sizeof(struct bnge_ring_grp_info),
 700			       GFP_KERNEL);
 701	if (!bn->grp_info)
 702		return -ENOMEM;
 703	for (i = 0; i < bd->nq_nr_rings; i++) {
 704		bn->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
 705		bn->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
 706		bn->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
 707		bn->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
 708		bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID;
 709	}
 710
 711	return 0;
 712}
 713
 714static void bnge_free_core(struct bnge_net *bn)
 715{
 716	bnge_free_vnic_attributes(bn);
 717	bnge_free_tx_rings(bn);
 718	bnge_free_rx_rings(bn);
 719	bnge_free_nq_tree(bn);
 720	bnge_free_nq_arrays(bn);
 721	bnge_free_ring_stats(bn);
 722	bnge_free_ring_grps(bn);
 723	bnge_free_vnics(bn);
 724	kfree(bn->tx_ring_map);
 725	bn->tx_ring_map = NULL;
 726	kfree(bn->tx_ring);
 727	bn->tx_ring = NULL;
 728	kfree(bn->rx_ring);
 729	bn->rx_ring = NULL;
 730	kfree(bn->bnapi);
 731	bn->bnapi = NULL;
 732}
 733
 734static int bnge_alloc_core(struct bnge_net *bn)
 735{
 736	struct bnge_dev *bd = bn->bd;
 737	int i, j, size, arr_size;
 738	int rc = -ENOMEM;
 739	void *bnapi;
 740
 741	arr_size = L1_CACHE_ALIGN(sizeof(struct bnge_napi *) *
 742			bd->nq_nr_rings);
 743	size = L1_CACHE_ALIGN(sizeof(struct bnge_napi));
 744	bnapi = kzalloc(arr_size + size * bd->nq_nr_rings, GFP_KERNEL);
 745	if (!bnapi)
 746		return rc;
 747
 748	bn->bnapi = bnapi;
 749	bnapi += arr_size;
 750	for (i = 0; i < bd->nq_nr_rings; i++, bnapi += size) {
 751		struct bnge_nq_ring_info *nqr;
 752
 753		bn->bnapi[i] = bnapi;
 754		bn->bnapi[i]->index = i;
 755		bn->bnapi[i]->bn = bn;
 756		nqr = &bn->bnapi[i]->nq_ring;
 757		nqr->ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG;
 758	}
 759
 760	bn->rx_ring = kcalloc(bd->rx_nr_rings,
 761			      sizeof(struct bnge_rx_ring_info),
 762			      GFP_KERNEL);
 763	if (!bn->rx_ring)
 764		goto err_free_core;
 765
 766	for (i = 0; i < bd->rx_nr_rings; i++) {
 767		struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
 768
 769		rxr->rx_ring_struct.ring_mem.flags =
 770			BNGE_RMEM_RING_PTE_FLAG;
 771		rxr->rx_agg_ring_struct.ring_mem.flags =
 772			BNGE_RMEM_RING_PTE_FLAG;
 773		rxr->bnapi = bn->bnapi[i];
 774		bn->bnapi[i]->rx_ring = &bn->rx_ring[i];
 775	}
 776
 777	bn->tx_ring = kcalloc(bd->tx_nr_rings,
 778			      sizeof(struct bnge_tx_ring_info),
 779			      GFP_KERNEL);
 780	if (!bn->tx_ring)
 781		goto err_free_core;
 782
 783	bn->tx_ring_map = kcalloc(bd->tx_nr_rings, sizeof(u16),
 784				  GFP_KERNEL);
 785	if (!bn->tx_ring_map)
 786		goto err_free_core;
 787
 788	if (bd->flags & BNGE_EN_SHARED_CHNL)
 789		j = 0;
 790	else
 791		j = bd->rx_nr_rings;
 792
 793	for (i = 0; i < bd->tx_nr_rings; i++) {
 794		struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
 795		struct bnge_napi *bnapi2;
 796		int k;
 797
 798		txr->tx_ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG;
 799		bn->tx_ring_map[i] = i;
 800		k = j + BNGE_RING_TO_TC_OFF(bd, i);
 801
 802		bnapi2 = bn->bnapi[k];
 803		txr->txq_index = i;
 804		txr->tx_napi_idx =
 805			BNGE_RING_TO_TC(bd, txr->txq_index);
 806		bnapi2->tx_ring[txr->tx_napi_idx] = txr;
 807		txr->bnapi = bnapi2;
 808	}
 809
 810	rc = bnge_alloc_ring_stats(bn);
 811	if (rc)
 812		goto err_free_core;
 813
 814	rc = bnge_alloc_vnics(bn);
 815	if (rc)
 816		goto err_free_core;
 817
 818	rc = bnge_alloc_nq_arrays(bn);
 819	if (rc)
 820		goto err_free_core;
 821
 822	bnge_init_ring_struct(bn);
 823
 824	rc = bnge_alloc_rx_rings(bn);
 825	if (rc)
 826		goto err_free_core;
 827
 828	rc = bnge_alloc_tx_rings(bn);
 829	if (rc)
 830		goto err_free_core;
 831
 832	rc = bnge_alloc_nq_tree(bn);
 833	if (rc)
 834		goto err_free_core;
 835
 836	bn->vnic_info[BNGE_VNIC_DEFAULT].flags |= BNGE_VNIC_RSS_FLAG |
 837						  BNGE_VNIC_MCAST_FLAG |
 838						  BNGE_VNIC_UCAST_FLAG;
 839	rc = bnge_alloc_vnic_attributes(bn);
 840	if (rc)
 841		goto err_free_core;
 842	return 0;
 843
 844err_free_core:
 845	bnge_free_core(bn);
 846	return rc;
 847}
 848
 849u16 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr)
 850{
 851	return rxr->rx_cpr->ring_struct.fw_ring_id;
 852}
 853
 854u16 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr)
 855{
 856	return txr->tx_cpr->ring_struct.fw_ring_id;
 857}
 858
 859static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx)
 860{
 861	bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK |
 862		    DB_RING_IDX(db, idx), db->doorbell);
 863}
 864
 865static void bnge_db_cq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx)
 866{
 867	bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_CQ_ARMALL |
 868		    DB_RING_IDX(db, idx), db->doorbell);
 869}
 870
 871static int bnge_cp_num_to_irq_num(struct bnge_net *bn, int n)
 872{
 873	struct bnge_napi *bnapi = bn->bnapi[n];
 874	struct bnge_nq_ring_info *nqr;
 875
 876	nqr = &bnapi->nq_ring;
 877
 878	return nqr->ring_struct.map_idx;
 879}
 880
 881static irqreturn_t bnge_msix(int irq, void *dev_instance)
 882{
 883	/* NAPI scheduling to be added in a future patch */
 884	return IRQ_HANDLED;
 885}
 886
 887static void bnge_init_nq_tree(struct bnge_net *bn)
 888{
 889	struct bnge_dev *bd = bn->bd;
 890	int i, j;
 891
 892	for (i = 0; i < bd->nq_nr_rings; i++) {
 893		struct bnge_nq_ring_info *nqr = &bn->bnapi[i]->nq_ring;
 894		struct bnge_ring_struct *ring = &nqr->ring_struct;
 895
 896		ring->fw_ring_id = INVALID_HW_RING_ID;
 897		for (j = 0; j < nqr->cp_ring_count; j++) {
 898			struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j];
 899
 900			ring = &cpr->ring_struct;
 901			ring->fw_ring_id = INVALID_HW_RING_ID;
 902		}
 903	}
 904}
 905
 906static netmem_ref __bnge_alloc_rx_netmem(struct bnge_net *bn,
 907					 dma_addr_t *mapping,
 908					 struct bnge_rx_ring_info *rxr,
 909					 unsigned int *offset,
 910					 gfp_t gfp)
 911{
 912	netmem_ref netmem;
 913
 914	if (PAGE_SIZE > BNGE_RX_PAGE_SIZE) {
 915		netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset,
 916						     BNGE_RX_PAGE_SIZE, gfp);
 917	} else {
 918		netmem = page_pool_alloc_netmems(rxr->page_pool, gfp);
 919		*offset = 0;
 920	}
 921	if (!netmem)
 922		return 0;
 923
 924	*mapping = page_pool_get_dma_addr_netmem(netmem) + *offset;
 925	return netmem;
 926}
 927
 928static u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping,
 929				struct bnge_rx_ring_info *rxr,
 930				gfp_t gfp)
 931{
 932	unsigned int offset;
 933	struct page *page;
 934
 935	page = page_pool_alloc_frag(rxr->head_pool, &offset,
 936				    bn->rx_buf_size, gfp);
 937	if (!page)
 938		return NULL;
 939
 940	*mapping = page_pool_get_dma_addr(page) + bn->rx_dma_offset + offset;
 941	return page_address(page) + offset;
 942}
 943
 944static int bnge_alloc_rx_data(struct bnge_net *bn,
 945			      struct bnge_rx_ring_info *rxr,
 946			      u16 prod, gfp_t gfp)
 947{
 948	struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)];
 949	struct rx_bd *rxbd;
 950	dma_addr_t mapping;
 951	u8 *data;
 952
 953	rxbd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)];
 954	data = __bnge_alloc_rx_frag(bn, &mapping, rxr, gfp);
 955	if (!data)
 956		return -ENOMEM;
 957
 958	rx_buf->data = data;
 959	rx_buf->data_ptr = data + bn->rx_offset;
 960	rx_buf->mapping = mapping;
 961
 962	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 963
 964	return 0;
 965}
 966
 967static int bnge_alloc_one_rx_ring_bufs(struct bnge_net *bn,
 968				       struct bnge_rx_ring_info *rxr,
 969				       int ring_nr)
 970{
 971	u32 prod = rxr->rx_prod;
 972	int i, rc = 0;
 973
 974	for (i = 0; i < bn->rx_ring_size; i++) {
 975		rc = bnge_alloc_rx_data(bn, rxr, prod, GFP_KERNEL);
 976		if (rc)
 977			break;
 978		prod = NEXT_RX(prod);
 979	}
 980
 981	/* Abort if not a single buffer can be allocated */
 982	if (rc && !i) {
 983		netdev_err(bn->netdev,
 984			   "RX ring %d: allocated %d/%d buffers, abort\n",
 985			   ring_nr, i, bn->rx_ring_size);
 986		return rc;
 987	}
 988
 989	rxr->rx_prod = prod;
 990
 991	if (i < bn->rx_ring_size)
 992		netdev_warn(bn->netdev,
 993			    "RX ring %d: allocated %d/%d buffers, continuing\n",
 994			    ring_nr, i, bn->rx_ring_size);
 995	return 0;
 996}
 997
 998static u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx)
 999{
1000	u16 next, max = rxr->rx_agg_bmap_size;
1001
1002	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
1003	if (next >= max)
1004		next = find_first_zero_bit(rxr->rx_agg_bmap, max);
1005	return next;
1006}
1007
1008static int bnge_alloc_rx_netmem(struct bnge_net *bn,
1009				struct bnge_rx_ring_info *rxr,
1010				u16 prod, gfp_t gfp)
1011{
1012	struct bnge_sw_rx_agg_bd *rx_agg_buf;
1013	u16 sw_prod = rxr->rx_sw_agg_prod;
1014	unsigned int offset = 0;
1015	struct rx_bd *rxbd;
1016	dma_addr_t mapping;
1017	netmem_ref netmem;
1018
1019	rxbd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)][RX_IDX(prod)];
1020	netmem = __bnge_alloc_rx_netmem(bn, &mapping, rxr, &offset, gfp);
1021	if (!netmem)
1022		return -ENOMEM;
1023
1024	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
1025		sw_prod = bnge_find_next_agg_idx(rxr, sw_prod);
1026
1027	__set_bit(sw_prod, rxr->rx_agg_bmap);
1028	rx_agg_buf = &rxr->rx_agg_buf_ring[sw_prod];
1029	rxr->rx_sw_agg_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod));
1030
1031	rx_agg_buf->netmem = netmem;
1032	rx_agg_buf->offset = offset;
1033	rx_agg_buf->mapping = mapping;
1034	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
1035	rxbd->rx_bd_opaque = sw_prod;
1036	return 0;
1037}
1038
1039static int bnge_alloc_one_agg_ring_bufs(struct bnge_net *bn,
1040					struct bnge_rx_ring_info *rxr,
1041					int ring_nr)
1042{
1043	u32 prod = rxr->rx_agg_prod;
1044	int i, rc = 0;
1045
1046	for (i = 0; i < bn->rx_agg_ring_size; i++) {
1047		rc = bnge_alloc_rx_netmem(bn, rxr, prod, GFP_KERNEL);
1048		if (rc)
1049			break;
1050		prod = NEXT_RX_AGG(prod);
1051	}
1052
1053	if (rc && i < MAX_SKB_FRAGS) {
1054		netdev_err(bn->netdev,
1055			   "Agg ring %d: allocated %d/%d buffers (min %d), abort\n",
1056			   ring_nr, i, bn->rx_agg_ring_size, MAX_SKB_FRAGS);
1057		goto err_free_one_agg_ring_bufs;
1058	}
1059
1060	rxr->rx_agg_prod = prod;
1061
1062	if (i < bn->rx_agg_ring_size)
1063		netdev_warn(bn->netdev,
1064			    "Agg ring %d: allocated %d/%d buffers, continuing\n",
1065			    ring_nr, i, bn->rx_agg_ring_size);
1066	return 0;
1067
1068err_free_one_agg_ring_bufs:
1069	bnge_free_one_agg_ring_bufs(bn, rxr);
1070	return -ENOMEM;
1071}
1072
1073static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr)
1074{
1075	struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr];
1076	int rc;
1077
1078	rc = bnge_alloc_one_rx_ring_bufs(bn, rxr, ring_nr);
1079	if (rc)
1080		return rc;
1081
1082	if (bnge_is_agg_reqd(bn->bd)) {
1083		rc = bnge_alloc_one_agg_ring_bufs(bn, rxr, ring_nr);
1084		if (rc)
1085			goto err_free_one_rx_ring_bufs;
1086	}
1087	return 0;
1088
1089err_free_one_rx_ring_bufs:
1090	bnge_free_one_rx_ring_bufs(bn, rxr);
1091	return rc;
1092}
1093
1094static void bnge_init_rxbd_pages(struct bnge_ring_struct *ring, u32 type)
1095{
1096	struct rx_bd **rx_desc_ring;
1097	u32 prod;
1098	int i;
1099
1100	rx_desc_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
1101	for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
1102		struct rx_bd *rxbd = rx_desc_ring[i];
1103		int j;
1104
1105		for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
1106			rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
1107			rxbd->rx_bd_opaque = prod;
1108		}
1109	}
1110}
1111
1112static void bnge_init_one_rx_ring_rxbd(struct bnge_net *bn,
1113				       struct bnge_rx_ring_info *rxr)
1114{
1115	struct bnge_ring_struct *ring;
1116	u32 type;
1117
1118	type = (bn->rx_buf_use_size << RX_BD_LEN_SHIFT) |
1119		RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
1120
1121	if (NET_IP_ALIGN == 2)
1122		type |= RX_BD_FLAGS_SOP;
1123
1124	ring = &rxr->rx_ring_struct;
1125	bnge_init_rxbd_pages(ring, type);
1126	ring->fw_ring_id = INVALID_HW_RING_ID;
1127}
1128
1129static void bnge_init_one_agg_ring_rxbd(struct bnge_net *bn,
1130					struct bnge_rx_ring_info *rxr)
1131{
1132	struct bnge_ring_struct *ring;
1133	u32 type;
1134
1135	ring = &rxr->rx_agg_ring_struct;
1136	ring->fw_ring_id = INVALID_HW_RING_ID;
1137	if (bnge_is_agg_reqd(bn->bd)) {
1138		type = ((u32)BNGE_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
1139			RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
1140
1141		bnge_init_rxbd_pages(ring, type);
1142	}
1143}
1144
1145static void bnge_init_one_rx_ring_pair(struct bnge_net *bn, int ring_nr)
1146{
1147	struct bnge_rx_ring_info *rxr;
1148
1149	rxr = &bn->rx_ring[ring_nr];
1150	bnge_init_one_rx_ring_rxbd(bn, rxr);
1151
1152	netif_queue_set_napi(bn->netdev, ring_nr, NETDEV_QUEUE_TYPE_RX,
1153			     &rxr->bnapi->napi);
1154
1155	bnge_init_one_agg_ring_rxbd(bn, rxr);
1156}
1157
1158static int bnge_alloc_rx_ring_pair_bufs(struct bnge_net *bn)
1159{
1160	int i, rc;
1161
1162	for (i = 0; i < bn->bd->rx_nr_rings; i++) {
1163		rc = bnge_alloc_one_rx_ring_pair_bufs(bn, i);
1164		if (rc)
1165			goto err_free_rx_ring_pair_bufs;
1166	}
1167	return 0;
1168
1169err_free_rx_ring_pair_bufs:
1170	bnge_free_rx_ring_pair_bufs(bn);
1171	return rc;
1172}
1173
1174static void bnge_init_rx_rings(struct bnge_net *bn)
1175{
1176	int i;
1177
1178#define BNGE_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
1179#define BNGE_RX_DMA_OFFSET NET_SKB_PAD
1180	bn->rx_offset = BNGE_RX_OFFSET;
1181	bn->rx_dma_offset = BNGE_RX_DMA_OFFSET;
1182
1183	for (i = 0; i < bn->bd->rx_nr_rings; i++)
1184		bnge_init_one_rx_ring_pair(bn, i);
1185}
1186
1187static void bnge_init_tx_rings(struct bnge_net *bn)
1188{
1189	int i;
1190
1191	bn->tx_wake_thresh = max(bn->tx_ring_size / 2, BNGE_MIN_TX_DESC_CNT);
1192
1193	for (i = 0; i < bn->bd->tx_nr_rings; i++) {
1194		struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
1195		struct bnge_ring_struct *ring = &txr->tx_ring_struct;
1196
1197		ring->fw_ring_id = INVALID_HW_RING_ID;
1198
1199		netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX,
1200				     &txr->bnapi->napi);
1201	}
1202}
1203
1204static void bnge_init_vnics(struct bnge_net *bn)
1205{
1206	struct bnge_vnic_info *vnic0 = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1207	int i;
1208
1209	for (i = 0; i < bn->nr_vnics; i++) {
1210		struct bnge_vnic_info *vnic = &bn->vnic_info[i];
1211		int j;
1212
1213		vnic->fw_vnic_id = INVALID_HW_RING_ID;
1214		vnic->vnic_id = i;
1215		for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++)
1216			vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
1217
1218		if (bn->vnic_info[i].rss_hash_key) {
1219			if (i == BNGE_VNIC_DEFAULT) {
1220				u8 *key = (void *)vnic->rss_hash_key;
1221				int k;
1222
1223				if (!bn->rss_hash_key_valid &&
1224				    !bn->rss_hash_key_updated) {
1225					get_random_bytes(bn->rss_hash_key,
1226							 HW_HASH_KEY_SIZE);
1227					bn->rss_hash_key_updated = true;
1228				}
1229
1230				memcpy(vnic->rss_hash_key, bn->rss_hash_key,
1231				       HW_HASH_KEY_SIZE);
1232
1233				if (!bn->rss_hash_key_updated)
1234					continue;
1235
1236				bn->rss_hash_key_updated = false;
1237				bn->rss_hash_key_valid = true;
1238
1239				bn->toeplitz_prefix = 0;
1240				for (k = 0; k < 8; k++) {
1241					bn->toeplitz_prefix <<= 8;
1242					bn->toeplitz_prefix |= key[k];
1243				}
1244			} else {
1245				memcpy(vnic->rss_hash_key, vnic0->rss_hash_key,
1246				       HW_HASH_KEY_SIZE);
1247			}
1248		}
1249	}
1250}
1251
1252static void bnge_set_db_mask(struct bnge_net *bn, struct bnge_db_info *db,
1253			     u32 ring_type)
1254{
1255	switch (ring_type) {
1256	case HWRM_RING_ALLOC_TX:
1257		db->db_ring_mask = bn->tx_ring_mask;
1258		break;
1259	case HWRM_RING_ALLOC_RX:
1260		db->db_ring_mask = bn->rx_ring_mask;
1261		break;
1262	case HWRM_RING_ALLOC_AGG:
1263		db->db_ring_mask = bn->rx_agg_ring_mask;
1264		break;
1265	case HWRM_RING_ALLOC_CMPL:
1266	case HWRM_RING_ALLOC_NQ:
1267		db->db_ring_mask = bn->cp_ring_mask;
1268		break;
1269	}
1270	db->db_epoch_mask = db->db_ring_mask + 1;
1271	db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
1272}
1273
1274static void bnge_set_db(struct bnge_net *bn, struct bnge_db_info *db,
1275			u32 ring_type, u32 map_idx, u32 xid)
1276{
1277	struct bnge_dev *bd = bn->bd;
1278
1279	switch (ring_type) {
1280	case HWRM_RING_ALLOC_TX:
1281		db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
1282		break;
1283	case HWRM_RING_ALLOC_RX:
1284	case HWRM_RING_ALLOC_AGG:
1285		db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
1286		break;
1287	case HWRM_RING_ALLOC_CMPL:
1288		db->db_key64 = DBR_PATH_L2;
1289		break;
1290	case HWRM_RING_ALLOC_NQ:
1291		db->db_key64 = DBR_PATH_L2;
1292		break;
1293	}
1294	db->db_key64 |= ((u64)xid << DBR_XID_SFT) | DBR_VALID;
1295
1296	db->doorbell = bd->bar1 + bd->db_offset;
1297	bnge_set_db_mask(bn, db, ring_type);
1298}
1299
1300static int bnge_hwrm_cp_ring_alloc(struct bnge_net *bn,
1301				   struct bnge_cp_ring_info *cpr)
1302{
1303	const u32 type = HWRM_RING_ALLOC_CMPL;
1304	struct bnge_napi *bnapi = cpr->bnapi;
1305	struct bnge_ring_struct *ring;
1306	u32 map_idx = bnapi->index;
1307	int rc;
1308
1309	ring = &cpr->ring_struct;
1310	ring->handle = BNGE_SET_NQ_HDL(cpr);
1311	rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1312	if (rc)
1313		return rc;
1314
1315	bnge_set_db(bn, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
1316	bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons);
1317
1318	return 0;
1319}
1320
1321static int bnge_hwrm_tx_ring_alloc(struct bnge_net *bn,
1322				   struct bnge_tx_ring_info *txr, u32 tx_idx)
1323{
1324	struct bnge_ring_struct *ring = &txr->tx_ring_struct;
1325	const u32 type = HWRM_RING_ALLOC_TX;
1326	int rc;
1327
1328	rc = hwrm_ring_alloc_send_msg(bn, ring, type, tx_idx);
1329	if (rc)
1330		return rc;
1331
1332	bnge_set_db(bn, &txr->tx_db, type, tx_idx, ring->fw_ring_id);
1333
1334	return 0;
1335}
1336
1337static int bnge_hwrm_rx_agg_ring_alloc(struct bnge_net *bn,
1338				       struct bnge_rx_ring_info *rxr)
1339{
1340	struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct;
1341	u32 type = HWRM_RING_ALLOC_AGG;
1342	struct bnge_dev *bd = bn->bd;
1343	u32 grp_idx = ring->grp_idx;
1344	u32 map_idx;
1345	int rc;
1346
1347	map_idx = grp_idx + bd->rx_nr_rings;
1348	rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1349	if (rc)
1350		return rc;
1351
1352	bnge_set_db(bn, &rxr->rx_agg_db, type, map_idx,
1353		    ring->fw_ring_id);
1354	bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod);
1355	bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod);
1356	bn->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
1357
1358	return 0;
1359}
1360
1361static int bnge_hwrm_rx_ring_alloc(struct bnge_net *bn,
1362				   struct bnge_rx_ring_info *rxr)
1363{
1364	struct bnge_ring_struct *ring = &rxr->rx_ring_struct;
1365	struct bnge_napi *bnapi = rxr->bnapi;
1366	u32 type = HWRM_RING_ALLOC_RX;
1367	u32 map_idx = bnapi->index;
1368	int rc;
1369
1370	rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1371	if (rc)
1372		return rc;
1373
1374	bnge_set_db(bn, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
1375	bn->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
1376
1377	return 0;
1378}
1379
1380static int bnge_hwrm_ring_alloc(struct bnge_net *bn)
1381{
1382	struct bnge_dev *bd = bn->bd;
1383	bool agg_rings;
1384	int i, rc = 0;
1385
1386	agg_rings = !!(bnge_is_agg_reqd(bd));
1387	for (i = 0; i < bd->nq_nr_rings; i++) {
1388		struct bnge_napi *bnapi = bn->bnapi[i];
1389		struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
1390		struct bnge_ring_struct *ring = &nqr->ring_struct;
1391		u32 type = HWRM_RING_ALLOC_NQ;
1392		u32 map_idx = ring->map_idx;
1393		unsigned int vector;
1394
1395		vector = bd->irq_tbl[map_idx].vector;
1396		disable_irq_nosync(vector);
1397		rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1398		if (rc) {
1399			enable_irq(vector);
1400			goto err_out;
1401		}
1402		bnge_set_db(bn, &nqr->nq_db, type, map_idx, ring->fw_ring_id);
1403		bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons);
1404		enable_irq(vector);
1405		bn->grp_info[i].nq_fw_ring_id = ring->fw_ring_id;
1406
1407		if (!i) {
1408			rc = bnge_hwrm_set_async_event_cr(bd, ring->fw_ring_id);
1409			if (rc)
1410				netdev_warn(bn->netdev, "Failed to set async event completion ring.\n");
1411		}
1412	}
1413
1414	for (i = 0; i < bd->tx_nr_rings; i++) {
1415		struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
1416
1417		rc = bnge_hwrm_cp_ring_alloc(bn, txr->tx_cpr);
1418		if (rc)
1419			goto err_out;
1420		rc = bnge_hwrm_tx_ring_alloc(bn, txr, i);
1421		if (rc)
1422			goto err_out;
1423	}
1424
1425	for (i = 0; i < bd->rx_nr_rings; i++) {
1426		struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
1427		struct bnge_cp_ring_info *cpr;
1428		struct bnge_ring_struct *ring;
1429		struct bnge_napi *bnapi;
1430		u32 map_idx, type;
1431
1432		rc = bnge_hwrm_rx_ring_alloc(bn, rxr);
1433		if (rc)
1434			goto err_out;
1435		/* If we have agg rings, post agg buffers first. */
1436		if (!agg_rings)
1437			bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod);
1438
1439		cpr = rxr->rx_cpr;
1440		bnapi = rxr->bnapi;
1441		type = HWRM_RING_ALLOC_CMPL;
1442		map_idx = bnapi->index;
1443
1444		ring = &cpr->ring_struct;
1445		ring->handle = BNGE_SET_NQ_HDL(cpr);
1446		rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1447		if (rc)
1448			goto err_out;
1449		bnge_set_db(bn, &cpr->cp_db, type, map_idx,
1450			    ring->fw_ring_id);
1451		bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons);
1452	}
1453
1454	if (agg_rings) {
1455		for (i = 0; i < bd->rx_nr_rings; i++) {
1456			rc = bnge_hwrm_rx_agg_ring_alloc(bn, &bn->rx_ring[i]);
1457			if (rc)
1458				goto err_out;
1459		}
1460	}
1461err_out:
1462	return rc;
1463}
1464
1465void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic)
1466{
1467	__le16 *ring_tbl = vnic->rss_table;
1468	struct bnge_rx_ring_info *rxr;
1469	struct bnge_dev *bd = bn->bd;
1470	u16 tbl_size, i;
1471
1472	tbl_size = bnge_get_rxfh_indir_size(bd);
1473
1474	for (i = 0; i < tbl_size; i++) {
1475		u16 ring_id, j;
1476
1477		j = bd->rss_indir_tbl[i];
1478		rxr = &bn->rx_ring[j];
1479
1480		ring_id = rxr->rx_ring_struct.fw_ring_id;
1481		*ring_tbl++ = cpu_to_le16(ring_id);
1482		ring_id = bnge_cp_ring_for_rx(rxr);
1483		*ring_tbl++ = cpu_to_le16(ring_id);
1484	}
1485}
1486
1487static int bnge_hwrm_vnic_rss_cfg(struct bnge_net *bn,
1488				  struct bnge_vnic_info *vnic)
1489{
1490	int rc;
1491
1492	rc = bnge_hwrm_vnic_set_rss(bn, vnic, true);
1493	if (rc) {
1494		netdev_err(bn->netdev, "hwrm vnic %d set rss failure rc: %d\n",
1495			   vnic->vnic_id, rc);
1496		return rc;
1497	}
1498	rc = bnge_hwrm_vnic_cfg(bn, vnic);
1499	if (rc)
1500		netdev_err(bn->netdev, "hwrm vnic %d cfg failure rc: %d\n",
1501			   vnic->vnic_id, rc);
1502	return rc;
1503}
1504
1505static int bnge_setup_vnic(struct bnge_net *bn, struct bnge_vnic_info *vnic)
1506{
1507	struct bnge_dev *bd = bn->bd;
1508	int rc, i, nr_ctxs;
1509
1510	nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings);
1511	for (i = 0; i < nr_ctxs; i++) {
1512		rc = bnge_hwrm_vnic_ctx_alloc(bd, vnic, i);
1513		if (rc) {
1514			netdev_err(bn->netdev, "hwrm vnic %d ctx %d alloc failure rc: %d\n",
1515				   vnic->vnic_id, i, rc);
1516			return -ENOMEM;
1517		}
1518		bn->rsscos_nr_ctxs++;
1519	}
1520
1521	rc = bnge_hwrm_vnic_rss_cfg(bn, vnic);
1522	if (rc)
1523		return rc;
1524
1525	if (bnge_is_agg_reqd(bd)) {
1526		rc = bnge_hwrm_vnic_set_hds(bn, vnic);
1527		if (rc)
1528			netdev_err(bn->netdev, "hwrm vnic %d set hds failure rc: %d\n",
1529				   vnic->vnic_id, rc);
1530	}
1531	return rc;
1532}
1533
1534static void bnge_del_l2_filter(struct bnge_net *bn, struct bnge_l2_filter *fltr)
1535{
1536	if (!refcount_dec_and_test(&fltr->refcnt))
1537		return;
1538	hlist_del_rcu(&fltr->base.hash);
1539	kfree_rcu(fltr, base.rcu);
1540}
1541
1542static void bnge_init_l2_filter(struct bnge_net *bn,
1543				struct bnge_l2_filter *fltr,
1544				struct bnge_l2_key *key, u32 idx)
1545{
1546	struct hlist_head *head;
1547
1548	ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
1549	fltr->l2_key.vlan = key->vlan;
1550	fltr->base.type = BNGE_FLTR_TYPE_L2;
1551
1552	head = &bn->l2_fltr_hash_tbl[idx];
1553	hlist_add_head_rcu(&fltr->base.hash, head);
1554	refcount_set(&fltr->refcnt, 1);
1555}
1556
1557static struct bnge_l2_filter *__bnge_lookup_l2_filter(struct bnge_net *bn,
1558						      struct bnge_l2_key *key,
1559						      u32 idx)
1560{
1561	struct bnge_l2_filter *fltr;
1562	struct hlist_head *head;
1563
1564	head = &bn->l2_fltr_hash_tbl[idx];
1565	hlist_for_each_entry_rcu(fltr, head, base.hash) {
1566		struct bnge_l2_key *l2_key = &fltr->l2_key;
1567
1568		if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
1569		    l2_key->vlan == key->vlan)
1570			return fltr;
1571	}
1572	return NULL;
1573}
1574
1575static struct bnge_l2_filter *bnge_lookup_l2_filter(struct bnge_net *bn,
1576						    struct bnge_l2_key *key,
1577						    u32 idx)
1578{
1579	struct bnge_l2_filter *fltr;
1580
1581	rcu_read_lock();
1582	fltr = __bnge_lookup_l2_filter(bn, key, idx);
1583	if (fltr)
1584		refcount_inc(&fltr->refcnt);
1585	rcu_read_unlock();
1586	return fltr;
1587}
1588
1589static struct bnge_l2_filter *bnge_alloc_l2_filter(struct bnge_net *bn,
1590						   struct bnge_l2_key *key,
1591						   gfp_t gfp)
1592{
1593	struct bnge_l2_filter *fltr;
1594	u32 idx;
1595
1596	idx = jhash2(&key->filter_key, BNGE_L2_KEY_SIZE, bn->hash_seed) &
1597	      BNGE_L2_FLTR_HASH_MASK;
1598	fltr = bnge_lookup_l2_filter(bn, key, idx);
1599	if (fltr)
1600		return fltr;
1601
1602	fltr = kzalloc(sizeof(*fltr), gfp);
1603	if (!fltr)
1604		return ERR_PTR(-ENOMEM);
1605
1606	bnge_init_l2_filter(bn, fltr, key, idx);
1607	return fltr;
1608}
1609
1610static int bnge_hwrm_set_vnic_filter(struct bnge_net *bn, u16 vnic_id, u16 idx,
1611				     const u8 *mac_addr)
1612{
1613	struct bnge_l2_filter *fltr;
1614	struct bnge_l2_key key;
1615	int rc;
1616
1617	ether_addr_copy(key.dst_mac_addr, mac_addr);
1618	key.vlan = 0;
1619	fltr = bnge_alloc_l2_filter(bn, &key, GFP_KERNEL);
1620	if (IS_ERR(fltr))
1621		return PTR_ERR(fltr);
1622
1623	fltr->base.fw_vnic_id = bn->vnic_info[vnic_id].fw_vnic_id;
1624	rc = bnge_hwrm_l2_filter_alloc(bn->bd, fltr);
1625	if (rc)
1626		goto err_del_l2_filter;
1627	bn->vnic_info[vnic_id].l2_filters[idx] = fltr;
1628	return rc;
1629
1630err_del_l2_filter:
1631	bnge_del_l2_filter(bn, fltr);
1632	return rc;
1633}
1634
1635static bool bnge_mc_list_updated(struct bnge_net *bn, u32 *rx_mask)
1636{
1637	struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1638	struct net_device *dev = bn->netdev;
1639	struct netdev_hw_addr *ha;
1640	int mc_count = 0, off = 0;
1641	bool update = false;
1642	u8 *haddr;
1643
1644	netdev_for_each_mc_addr(ha, dev) {
1645		if (mc_count >= BNGE_MAX_MC_ADDRS) {
1646			*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
1647			vnic->mc_list_count = 0;
1648			return false;
1649		}
1650		haddr = ha->addr;
1651		if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
1652			memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
1653			update = true;
1654		}
1655		off += ETH_ALEN;
1656		mc_count++;
1657	}
1658	if (mc_count)
1659		*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
1660
1661	if (mc_count != vnic->mc_list_count) {
1662		vnic->mc_list_count = mc_count;
1663		update = true;
1664	}
1665	return update;
1666}
1667
1668static bool bnge_uc_list_updated(struct bnge_net *bn)
1669{
1670	struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1671	struct net_device *dev = bn->netdev;
1672	struct netdev_hw_addr *ha;
1673	int off = 0;
1674
1675	if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
1676		return true;
1677
1678	netdev_for_each_uc_addr(ha, dev) {
1679		if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
1680			return true;
1681
1682		off += ETH_ALEN;
1683	}
1684	return false;
1685}
1686
1687static bool bnge_promisc_ok(struct bnge_net *bn)
1688{
1689	return true;
1690}
1691
1692static int bnge_cfg_def_vnic(struct bnge_net *bn)
1693{
1694	struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1695	struct net_device *dev = bn->netdev;
1696	struct bnge_dev *bd = bn->bd;
1697	struct netdev_hw_addr *ha;
1698	int i, off = 0, rc;
1699	bool uc_update;
1700
1701	netif_addr_lock_bh(dev);
1702	uc_update = bnge_uc_list_updated(bn);
1703	netif_addr_unlock_bh(dev);
1704
1705	if (!uc_update)
1706		goto skip_uc;
1707
1708	for (i = 1; i < vnic->uc_filter_count; i++) {
1709		struct bnge_l2_filter *fltr = vnic->l2_filters[i];
1710
1711		bnge_hwrm_l2_filter_free(bd, fltr);
1712		bnge_del_l2_filter(bn, fltr);
1713	}
1714
1715	vnic->uc_filter_count = 1;
1716
1717	netif_addr_lock_bh(dev);
1718	if (netdev_uc_count(dev) > (BNGE_MAX_UC_ADDRS - 1)) {
1719		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
1720	} else {
1721		netdev_for_each_uc_addr(ha, dev) {
1722			memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
1723			off += ETH_ALEN;
1724			vnic->uc_filter_count++;
1725		}
1726	}
1727	netif_addr_unlock_bh(dev);
1728
1729	for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
1730		rc = bnge_hwrm_set_vnic_filter(bn, 0, i, vnic->uc_list + off);
1731		if (rc) {
1732			netdev_err(dev, "HWRM vnic filter failure rc: %d\n", rc);
1733			vnic->uc_filter_count = i;
1734			return rc;
1735		}
1736	}
1737
1738skip_uc:
1739	if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
1740	    !bnge_promisc_ok(bn))
1741		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
1742	rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic);
1743	if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
1744		netdev_info(dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
1745			    rc);
1746		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
1747		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
1748		vnic->mc_list_count = 0;
1749		rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic);
1750	}
1751	if (rc)
1752		netdev_err(dev, "HWRM cfa l2 rx mask failure rc: %d\n",
1753			   rc);
1754
1755	return rc;
1756}
1757
1758static void bnge_hwrm_vnic_free(struct bnge_net *bn)
1759{
1760	int i;
1761
1762	for (i = 0; i < bn->nr_vnics; i++)
1763		bnge_hwrm_vnic_free_one(bn->bd, &bn->vnic_info[i]);
1764}
1765
1766static void bnge_hwrm_vnic_ctx_free(struct bnge_net *bn)
1767{
1768	int i, j;
1769
1770	for (i = 0; i < bn->nr_vnics; i++) {
1771		struct bnge_vnic_info *vnic = &bn->vnic_info[i];
1772
1773		for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) {
1774			if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
1775				bnge_hwrm_vnic_ctx_free_one(bn->bd, vnic, j);
1776		}
1777	}
1778	bn->rsscos_nr_ctxs = 0;
1779}
1780
1781static void bnge_hwrm_clear_vnic_filter(struct bnge_net *bn)
1782{
1783	struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1784	int i;
1785
1786	for (i = 0; i < vnic->uc_filter_count; i++) {
1787		struct bnge_l2_filter *fltr = vnic->l2_filters[i];
1788
1789		bnge_hwrm_l2_filter_free(bn->bd, fltr);
1790		bnge_del_l2_filter(bn, fltr);
1791	}
1792
1793	vnic->uc_filter_count = 0;
1794}
1795
1796static void bnge_clear_vnic(struct bnge_net *bn)
1797{
1798	bnge_hwrm_clear_vnic_filter(bn);
1799	bnge_hwrm_vnic_free(bn);
1800	bnge_hwrm_vnic_ctx_free(bn);
1801}
1802
1803static void bnge_hwrm_rx_ring_free(struct bnge_net *bn,
1804				   struct bnge_rx_ring_info *rxr,
1805				   bool close_path)
1806{
1807	struct bnge_ring_struct *ring = &rxr->rx_ring_struct;
1808	u32 grp_idx = rxr->bnapi->index;
1809	u32 cmpl_ring_id;
1810
1811	if (ring->fw_ring_id == INVALID_HW_RING_ID)
1812		return;
1813
1814	cmpl_ring_id = bnge_cp_ring_for_rx(rxr);
1815	hwrm_ring_free_send_msg(bn, ring,
1816				RING_FREE_REQ_RING_TYPE_RX,
1817				close_path ? cmpl_ring_id :
1818				INVALID_HW_RING_ID);
1819	ring->fw_ring_id = INVALID_HW_RING_ID;
1820	bn->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID;
1821}
1822
1823static void bnge_hwrm_rx_agg_ring_free(struct bnge_net *bn,
1824				       struct bnge_rx_ring_info *rxr,
1825				       bool close_path)
1826{
1827	struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct;
1828	u32 grp_idx = rxr->bnapi->index;
1829	u32 cmpl_ring_id;
1830
1831	if (ring->fw_ring_id == INVALID_HW_RING_ID)
1832		return;
1833
1834	cmpl_ring_id = bnge_cp_ring_for_rx(rxr);
1835	hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_RX_AGG,
1836				close_path ? cmpl_ring_id :
1837				INVALID_HW_RING_ID);
1838	ring->fw_ring_id = INVALID_HW_RING_ID;
1839	bn->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID;
1840}
1841
1842static void bnge_hwrm_tx_ring_free(struct bnge_net *bn,
1843				   struct bnge_tx_ring_info *txr,
1844				   bool close_path)
1845{
1846	struct bnge_ring_struct *ring = &txr->tx_ring_struct;
1847	u32 cmpl_ring_id;
1848
1849	if (ring->fw_ring_id == INVALID_HW_RING_ID)
1850		return;
1851
1852	cmpl_ring_id = close_path ? bnge_cp_ring_for_tx(txr) :
1853		       INVALID_HW_RING_ID;
1854	hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_TX,
1855				cmpl_ring_id);
1856	ring->fw_ring_id = INVALID_HW_RING_ID;
1857}
1858
1859static void bnge_hwrm_cp_ring_free(struct bnge_net *bn,
1860				   struct bnge_cp_ring_info *cpr)
1861{
1862	struct bnge_ring_struct *ring;
1863
1864	ring = &cpr->ring_struct;
1865	if (ring->fw_ring_id == INVALID_HW_RING_ID)
1866		return;
1867
1868	hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL,
1869				INVALID_HW_RING_ID);
1870	ring->fw_ring_id = INVALID_HW_RING_ID;
1871}
1872
1873static void bnge_hwrm_ring_free(struct bnge_net *bn, bool close_path)
1874{
1875	struct bnge_dev *bd = bn->bd;
1876	int i;
1877
1878	if (!bn->bnapi)
1879		return;
1880
1881	for (i = 0; i < bd->tx_nr_rings; i++)
1882		bnge_hwrm_tx_ring_free(bn, &bn->tx_ring[i], close_path);
1883
1884	for (i = 0; i < bd->rx_nr_rings; i++) {
1885		bnge_hwrm_rx_ring_free(bn, &bn->rx_ring[i], close_path);
1886		bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path);
1887	}
1888
1889	for (i = 0; i < bd->nq_nr_rings; i++) {
1890		struct bnge_napi *bnapi = bn->bnapi[i];
1891		struct bnge_nq_ring_info *nqr;
1892		struct bnge_ring_struct *ring;
1893		int j;
1894
1895		nqr = &bnapi->nq_ring;
1896		for (j = 0; j < nqr->cp_ring_count && nqr->cp_ring_arr; j++)
1897			bnge_hwrm_cp_ring_free(bn, &nqr->cp_ring_arr[j]);
1898
1899		ring = &nqr->ring_struct;
1900		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1901			hwrm_ring_free_send_msg(bn, ring,
1902						RING_FREE_REQ_RING_TYPE_NQ,
1903						INVALID_HW_RING_ID);
1904			ring->fw_ring_id = INVALID_HW_RING_ID;
1905			bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID;
1906		}
1907	}
1908}
1909
1910static void bnge_setup_msix(struct bnge_net *bn)
1911{
1912	struct net_device *dev = bn->netdev;
1913	struct bnge_dev *bd = bn->bd;
1914	int len, i;
1915
1916	len = sizeof(bd->irq_tbl[0].name);
1917	for (i = 0; i < bd->nq_nr_rings; i++) {
1918		int map_idx = bnge_cp_num_to_irq_num(bn, i);
1919		char *attr;
1920
1921		if (bd->flags & BNGE_EN_SHARED_CHNL)
1922			attr = "TxRx";
1923		else if (i < bd->rx_nr_rings)
1924			attr = "rx";
1925		else
1926			attr = "tx";
1927
1928		snprintf(bd->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
1929			 attr, i);
1930		bd->irq_tbl[map_idx].handler = bnge_msix;
1931	}
1932}
1933
1934static int bnge_setup_interrupts(struct bnge_net *bn)
1935{
1936	struct net_device *dev = bn->netdev;
1937	struct bnge_dev *bd = bn->bd;
1938
1939	bnge_setup_msix(bn);
1940
1941	return netif_set_real_num_queues(dev, bd->tx_nr_rings, bd->rx_nr_rings);
1942}
1943
1944static void bnge_hwrm_resource_free(struct bnge_net *bn, bool close_path)
1945{
1946	bnge_clear_vnic(bn);
1947	bnge_hwrm_ring_free(bn, close_path);
1948	bnge_hwrm_stat_ctx_free(bn);
1949}
1950
1951static void bnge_free_irq(struct bnge_net *bn)
1952{
1953	struct bnge_dev *bd = bn->bd;
1954	struct bnge_irq *irq;
1955	int i;
1956
1957	for (i = 0; i < bd->nq_nr_rings; i++) {
1958		int map_idx = bnge_cp_num_to_irq_num(bn, i);
1959
1960		irq = &bd->irq_tbl[map_idx];
1961		if (irq->requested) {
1962			if (irq->have_cpumask) {
1963				irq_set_affinity_hint(irq->vector, NULL);
1964				free_cpumask_var(irq->cpu_mask);
1965				irq->have_cpumask = 0;
1966			}
1967			free_irq(irq->vector, bn->bnapi[i]);
1968		}
1969
1970		irq->requested = 0;
1971	}
1972}
1973
1974static int bnge_request_irq(struct bnge_net *bn)
1975{
1976	struct bnge_dev *bd = bn->bd;
1977	int i, rc;
1978
1979	rc = bnge_setup_interrupts(bn);
1980	if (rc) {
1981		netdev_err(bn->netdev, "bnge_setup_interrupts err: %d\n", rc);
1982		return rc;
1983	}
1984	for (i = 0; i < bd->nq_nr_rings; i++) {
1985		int map_idx = bnge_cp_num_to_irq_num(bn, i);
1986		struct bnge_irq *irq = &bd->irq_tbl[map_idx];
1987
1988		rc = request_irq(irq->vector, irq->handler, 0, irq->name,
1989				 bn->bnapi[i]);
1990		if (rc)
1991			goto err_free_irq;
1992
1993		netif_napi_set_irq_locked(&bn->bnapi[i]->napi, irq->vector);
1994		irq->requested = 1;
1995
1996		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
1997			int numa_node = dev_to_node(&bd->pdev->dev);
1998
1999			irq->have_cpumask = 1;
2000			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
2001					irq->cpu_mask);
2002			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
2003			if (rc) {
2004				netdev_warn(bn->netdev,
2005					    "Set affinity failed, IRQ = %d\n",
2006					    irq->vector);
2007				goto err_free_irq;
2008			}
2009		}
2010	}
2011	return 0;
2012
2013err_free_irq:
2014	bnge_free_irq(bn);
2015	return rc;
2016}
2017
2018static int bnge_init_chip(struct bnge_net *bn)
2019{
2020	struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
2021	struct bnge_dev *bd = bn->bd;
2022	int rc;
2023
2024#define BNGE_DEF_STATS_COAL_TICKS	 1000000
2025	bn->stats_coal_ticks = BNGE_DEF_STATS_COAL_TICKS;
2026
2027	rc = bnge_hwrm_stat_ctx_alloc(bn);
2028	if (rc) {
2029		netdev_err(bn->netdev, "hwrm stat ctx alloc failure rc: %d\n", rc);
2030		goto err_out;
2031	}
2032
2033	rc = bnge_hwrm_ring_alloc(bn);
2034	if (rc) {
2035		netdev_err(bn->netdev, "hwrm ring alloc failure rc: %d\n", rc);
2036		goto err_out;
2037	}
2038
2039	rc = bnge_hwrm_vnic_alloc(bd, vnic, bd->rx_nr_rings);
2040	if (rc) {
2041		netdev_err(bn->netdev, "hwrm vnic alloc failure rc: %d\n", rc);
2042		goto err_out;
2043	}
2044
2045	rc = bnge_setup_vnic(bn, vnic);
2046	if (rc)
2047		goto err_out;
2048
2049	if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA)
2050		bnge_hwrm_update_rss_hash_cfg(bn);
2051
2052	/* Filter for default vnic 0 */
2053	rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr);
2054	if (rc) {
2055		netdev_err(bn->netdev, "HWRM vnic filter failure rc: %d\n", rc);
2056		goto err_out;
2057	}
2058	vnic->uc_filter_count = 1;
2059
2060	vnic->rx_mask = 0;
2061
2062	if (bn->netdev->flags & IFF_BROADCAST)
2063		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
2064
2065	if (bn->netdev->flags & IFF_PROMISC)
2066		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
2067
2068	if (bn->netdev->flags & IFF_ALLMULTI) {
2069		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
2070		vnic->mc_list_count = 0;
2071	} else if (bn->netdev->flags & IFF_MULTICAST) {
2072		u32 mask = 0;
2073
2074		bnge_mc_list_updated(bn, &mask);
2075		vnic->rx_mask |= mask;
2076	}
2077
2078	rc = bnge_cfg_def_vnic(bn);
2079	if (rc)
2080		goto err_out;
2081	return 0;
2082
2083err_out:
2084	bnge_hwrm_resource_free(bn, 0);
2085	return rc;
2086}
2087
2088static int bnge_napi_poll(struct napi_struct *napi, int budget)
2089{
2090	int work_done = 0;
2091
2092	/* defer NAPI implementation to next patch series */
2093	napi_complete_done(napi, work_done);
2094
2095	return work_done;
2096}
2097
2098static void bnge_init_napi(struct bnge_net *bn)
2099{
2100	struct bnge_dev *bd = bn->bd;
2101	struct bnge_napi *bnapi;
2102	int i;
2103
2104	for (i = 0; i < bd->nq_nr_rings; i++) {
2105		bnapi = bn->bnapi[i];
2106		netif_napi_add_config_locked(bn->netdev, &bnapi->napi,
2107					     bnge_napi_poll, bnapi->index);
2108	}
2109}
2110
2111static void bnge_del_napi(struct bnge_net *bn)
2112{
2113	struct bnge_dev *bd = bn->bd;
2114	int i;
2115
2116	for (i = 0; i < bd->rx_nr_rings; i++)
2117		netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_RX, NULL);
2118	for (i = 0; i < bd->tx_nr_rings; i++)
2119		netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, NULL);
2120
2121	for (i = 0; i < bd->nq_nr_rings; i++) {
2122		struct bnge_napi *bnapi = bn->bnapi[i];
2123
2124		__netif_napi_del_locked(&bnapi->napi);
2125	}
2126
2127	/* Wait for RCU grace period after removing NAPI instances */
2128	synchronize_net();
2129}
2130
2131static int bnge_init_nic(struct bnge_net *bn)
2132{
2133	int rc;
2134
2135	bnge_init_nq_tree(bn);
2136
2137	bnge_init_rx_rings(bn);
2138	rc = bnge_alloc_rx_ring_pair_bufs(bn);
2139	if (rc)
2140		return rc;
2141
2142	bnge_init_tx_rings(bn);
2143
2144	rc = bnge_init_ring_grps(bn);
2145	if (rc)
2146		goto err_free_rx_ring_pair_bufs;
2147
2148	bnge_init_vnics(bn);
2149
2150	rc = bnge_init_chip(bn);
2151	if (rc)
2152		goto err_free_ring_grps;
2153	return rc;
2154
2155err_free_ring_grps:
2156	bnge_free_ring_grps(bn);
2157	return rc;
2158
2159err_free_rx_ring_pair_bufs:
2160	bnge_free_rx_ring_pair_bufs(bn);
2161	return rc;
2162}
2163
2164static int bnge_open_core(struct bnge_net *bn)
2165{
2166	struct bnge_dev *bd = bn->bd;
2167	int rc;
2168
2169	netif_carrier_off(bn->netdev);
2170
2171	rc = bnge_reserve_rings(bd);
2172	if (rc) {
2173		netdev_err(bn->netdev, "bnge_reserve_rings err: %d\n", rc);
2174		return rc;
2175	}
2176
2177	rc = bnge_alloc_core(bn);
2178	if (rc) {
2179		netdev_err(bn->netdev, "bnge_alloc_core err: %d\n", rc);
2180		return rc;
2181	}
2182
2183	bnge_init_napi(bn);
2184	rc = bnge_request_irq(bn);
2185	if (rc) {
2186		netdev_err(bn->netdev, "bnge_request_irq err: %d\n", rc);
2187		goto err_del_napi;
2188	}
2189
2190	rc = bnge_init_nic(bn);
2191	if (rc) {
2192		netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc);
2193		goto err_free_irq;
2194	}
2195	set_bit(BNGE_STATE_OPEN, &bd->state);
2196	return 0;
2197
2198err_free_irq:
2199	bnge_free_irq(bn);
2200err_del_napi:
2201	bnge_del_napi(bn);
2202	bnge_free_core(bn);
2203	return rc;
2204}
2205
2206static netdev_tx_t bnge_start_xmit(struct sk_buff *skb, struct net_device *dev)
2207{
2208	dev_kfree_skb_any(skb);
2209
2210	return NETDEV_TX_OK;
2211}
2212
2213static int bnge_open(struct net_device *dev)
2214{
2215	struct bnge_net *bn = netdev_priv(dev);
2216	int rc;
2217
2218	rc = bnge_open_core(bn);
2219	if (rc)
2220		netdev_err(dev, "bnge_open_core err: %d\n", rc);
2221
2222	return rc;
2223}
2224
2225static int bnge_shutdown_nic(struct bnge_net *bn)
2226{
2227	/* TODO: close_path = 0 until we make NAPI functional */
2228	bnge_hwrm_resource_free(bn, 0);
2229	return 0;
2230}
2231
2232static void bnge_close_core(struct bnge_net *bn)
2233{
2234	struct bnge_dev *bd = bn->bd;
2235
2236	clear_bit(BNGE_STATE_OPEN, &bd->state);
2237	bnge_shutdown_nic(bn);
2238	bnge_free_all_rings_bufs(bn);
2239	bnge_free_irq(bn);
2240	bnge_del_napi(bn);
2241
2242	bnge_free_core(bn);
2243}
2244
2245static int bnge_close(struct net_device *dev)
2246{
2247	struct bnge_net *bn = netdev_priv(dev);
2248
2249	bnge_close_core(bn);
2250
2251	return 0;
2252}
2253
2254static const struct net_device_ops bnge_netdev_ops = {
2255	.ndo_open		= bnge_open,
2256	.ndo_stop		= bnge_close,
2257	.ndo_start_xmit		= bnge_start_xmit,
2258};
2259
2260static void bnge_init_mac_addr(struct bnge_dev *bd)
2261{
2262	eth_hw_addr_set(bd->netdev, bd->pf.mac_addr);
2263}
2264
2265static void bnge_set_tpa_flags(struct bnge_dev *bd)
2266{
2267	struct bnge_net *bn = netdev_priv(bd->netdev);
2268
2269	bn->priv_flags &= ~BNGE_NET_EN_TPA;
2270
2271	if (bd->netdev->features & NETIF_F_LRO)
2272		bn->priv_flags |= BNGE_NET_EN_LRO;
2273	else if (bd->netdev->features & NETIF_F_GRO_HW)
2274		bn->priv_flags |= BNGE_NET_EN_GRO;
2275}
2276
2277static void bnge_init_l2_fltr_tbl(struct bnge_net *bn)
2278{
2279	int i;
2280
2281	for (i = 0; i < BNGE_L2_FLTR_HASH_SIZE; i++)
2282		INIT_HLIST_HEAD(&bn->l2_fltr_hash_tbl[i]);
2283	get_random_bytes(&bn->hash_seed, sizeof(bn->hash_seed));
2284}
2285
2286void bnge_set_ring_params(struct bnge_dev *bd)
2287{
2288	struct bnge_net *bn = netdev_priv(bd->netdev);
2289	u32 ring_size, rx_size, rx_space, max_rx_cmpl;
2290	u32 agg_factor = 0, agg_ring_size = 0;
2291
2292	/* 8 for CRC and VLAN */
2293	rx_size = SKB_DATA_ALIGN(bn->netdev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2294
2295	rx_space = rx_size + ALIGN(NET_SKB_PAD, 8) +
2296		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2297
2298	bn->rx_copy_thresh = BNGE_RX_COPY_THRESH;
2299	ring_size = bn->rx_ring_size;
2300	bn->rx_agg_ring_size = 0;
2301	bn->rx_agg_nr_pages = 0;
2302
2303	if (bn->priv_flags & BNGE_NET_EN_TPA)
2304		agg_factor = min_t(u32, 4, 65536 / BNGE_RX_PAGE_SIZE);
2305
2306	bn->priv_flags &= ~BNGE_NET_EN_JUMBO;
2307	if (rx_space > PAGE_SIZE) {
2308		u32 jumbo_factor;
2309
2310		bn->priv_flags |= BNGE_NET_EN_JUMBO;
2311		jumbo_factor = PAGE_ALIGN(bn->netdev->mtu - 40) >> PAGE_SHIFT;
2312		if (jumbo_factor > agg_factor)
2313			agg_factor = jumbo_factor;
2314	}
2315	if (agg_factor) {
2316		if (ring_size > BNGE_MAX_RX_DESC_CNT_JUM_ENA) {
2317			ring_size = BNGE_MAX_RX_DESC_CNT_JUM_ENA;
2318			netdev_warn(bn->netdev, "RX ring size reduced from %d to %d due to jumbo ring\n",
2319				    bn->rx_ring_size, ring_size);
2320			bn->rx_ring_size = ring_size;
2321		}
2322		agg_ring_size = ring_size * agg_factor;
2323
2324		bn->rx_agg_nr_pages = bnge_adjust_pow_two(agg_ring_size,
2325							  RX_DESC_CNT);
2326		if (bn->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2327			u32 tmp = agg_ring_size;
2328
2329			bn->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2330			agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2331			netdev_warn(bn->netdev, "RX agg ring size %d reduced to %d.\n",
2332				    tmp, agg_ring_size);
2333		}
2334		bn->rx_agg_ring_size = agg_ring_size;
2335		bn->rx_agg_ring_mask = (bn->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2336
2337		rx_size = SKB_DATA_ALIGN(BNGE_RX_COPY_THRESH + NET_IP_ALIGN);
2338		rx_space = rx_size + NET_SKB_PAD +
2339			SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2340	}
2341
2342	bn->rx_buf_use_size = rx_size;
2343	bn->rx_buf_size = rx_space;
2344
2345	bn->rx_nr_pages = bnge_adjust_pow_two(ring_size, RX_DESC_CNT);
2346	bn->rx_ring_mask = (bn->rx_nr_pages * RX_DESC_CNT) - 1;
2347
2348	ring_size = bn->tx_ring_size;
2349	bn->tx_nr_pages = bnge_adjust_pow_two(ring_size, TX_DESC_CNT);
2350	bn->tx_ring_mask = (bn->tx_nr_pages * TX_DESC_CNT) - 1;
2351
2352	max_rx_cmpl = bn->rx_ring_size;
2353
2354	if (bn->priv_flags & BNGE_NET_EN_TPA)
2355		max_rx_cmpl += bd->max_tpa_v2;
2356	ring_size = max_rx_cmpl * 2 + agg_ring_size + bn->tx_ring_size;
2357	bn->cp_ring_size = ring_size;
2358
2359	bn->cp_nr_pages = bnge_adjust_pow_two(ring_size, CP_DESC_CNT);
2360	if (bn->cp_nr_pages > MAX_CP_PAGES) {
2361		bn->cp_nr_pages = MAX_CP_PAGES;
2362		bn->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2363		netdev_warn(bn->netdev, "completion ring size %d reduced to %d.\n",
2364			    ring_size, bn->cp_ring_size);
2365	}
2366	bn->cp_bit = bn->cp_nr_pages * CP_DESC_CNT;
2367	bn->cp_ring_mask = bn->cp_bit - 1;
2368}
2369
2370int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs)
2371{
2372	struct net_device *netdev;
2373	struct bnge_net *bn;
2374	int rc;
2375
2376	netdev = alloc_etherdev_mqs(sizeof(*bn), max_irqs * BNGE_MAX_QUEUE,
2377				    max_irqs);
2378	if (!netdev)
2379		return -ENOMEM;
2380
2381	SET_NETDEV_DEV(netdev, bd->dev);
2382	bd->netdev = netdev;
2383
2384	netdev->netdev_ops = &bnge_netdev_ops;
2385
2386	bnge_set_ethtool_ops(netdev);
2387
2388	bn = netdev_priv(netdev);
2389	bn->netdev = netdev;
2390	bn->bd = bd;
2391
2392	netdev->min_mtu = ETH_ZLEN;
2393	netdev->max_mtu = bd->max_mtu;
2394
2395	netdev->hw_features = NETIF_F_IP_CSUM |
2396			      NETIF_F_IPV6_CSUM |
2397			      NETIF_F_SG |
2398			      NETIF_F_TSO |
2399			      NETIF_F_TSO6 |
2400			      NETIF_F_GSO_UDP_TUNNEL |
2401			      NETIF_F_GSO_GRE |
2402			      NETIF_F_GSO_IPXIP4 |
2403			      NETIF_F_GSO_UDP_TUNNEL_CSUM |
2404			      NETIF_F_GSO_GRE_CSUM |
2405			      NETIF_F_GSO_PARTIAL |
2406			      NETIF_F_RXHASH |
2407			      NETIF_F_RXCSUM |
2408			      NETIF_F_GRO;
2409
2410	if (bd->flags & BNGE_EN_UDP_GSO_SUPP)
2411		netdev->hw_features |= NETIF_F_GSO_UDP_L4;
2412
2413	if (BNGE_SUPPORTS_TPA(bd))
2414		netdev->hw_features |= NETIF_F_LRO;
2415
2416	netdev->hw_enc_features = NETIF_F_IP_CSUM |
2417				  NETIF_F_IPV6_CSUM |
2418				  NETIF_F_SG |
2419				  NETIF_F_TSO |
2420				  NETIF_F_TSO6 |
2421				  NETIF_F_GSO_UDP_TUNNEL |
2422				  NETIF_F_GSO_GRE |
2423				  NETIF_F_GSO_UDP_TUNNEL_CSUM |
2424				  NETIF_F_GSO_GRE_CSUM |
2425				  NETIF_F_GSO_IPXIP4 |
2426				  NETIF_F_GSO_PARTIAL;
2427
2428	if (bd->flags & BNGE_EN_UDP_GSO_SUPP)
2429		netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
2430
2431	netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
2432				       NETIF_F_GSO_GRE_CSUM;
2433
2434	netdev->vlan_features = netdev->hw_features | NETIF_F_HIGHDMA;
2435	if (bd->fw_cap & BNGE_FW_CAP_VLAN_RX_STRIP)
2436		netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_RX;
2437	if (bd->fw_cap & BNGE_FW_CAP_VLAN_TX_INSERT)
2438		netdev->hw_features |= BNGE_HW_FEATURE_VLAN_ALL_TX;
2439
2440	if (BNGE_SUPPORTS_TPA(bd))
2441		netdev->hw_features |= NETIF_F_GRO_HW;
2442
2443	netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA;
2444
2445	if (netdev->features & NETIF_F_GRO_HW)
2446		netdev->features &= ~NETIF_F_LRO;
2447
2448	netdev->priv_flags |= IFF_UNICAST_FLT;
2449
2450	netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
2451	if (bd->tso_max_segs)
2452		netif_set_tso_max_segs(netdev, bd->tso_max_segs);
2453
2454	bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE;
2455	bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE;
2456	bn->rx_dir = DMA_FROM_DEVICE;
2457
2458	bnge_set_tpa_flags(bd);
2459	bnge_set_ring_params(bd);
2460
2461	bnge_init_l2_fltr_tbl(bn);
2462	bnge_init_mac_addr(bd);
2463
2464	netdev->request_ops_lock = true;
2465	rc = register_netdev(netdev);
2466	if (rc) {
2467		dev_err(bd->dev, "Register netdev failed rc: %d\n", rc);
2468		goto err_netdev;
2469	}
2470
2471	return 0;
2472
2473err_netdev:
2474	free_netdev(netdev);
2475	return rc;
2476}
2477
2478void bnge_netdev_free(struct bnge_dev *bd)
2479{
2480	struct net_device *netdev = bd->netdev;
2481
2482	unregister_netdev(netdev);
2483	free_netdev(netdev);
2484	bd->netdev = NULL;
2485}