fbnic_ethtool.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) Meta Platforms, Inc. and affiliates. */
   3
   4#include <linux/ethtool.h>
   5#include <linux/ethtool_netlink.h>
   6#include <linux/netdevice.h>
   7#include <linux/pci.h>
   8#include <net/ipv6.h>
   9
  10#include "fbnic.h"
  11#include "fbnic_netdev.h"
  12#include "fbnic_tlv.h"
  13
  14struct fbnic_stat {
  15	u8 string[ETH_GSTRING_LEN];
  16	unsigned int size;
  17	unsigned int offset;
  18};
  19
  20#define FBNIC_STAT_FIELDS(type, name, stat) { \
  21	.string = name, \
  22	.size = sizeof_field(struct type, stat), \
  23	.offset = offsetof(struct type, stat), \
  24}
  25
  26/* Hardware statistics not captured in rtnl_link_stats */
  27#define FBNIC_HW_STAT(name, stat) \
  28	FBNIC_STAT_FIELDS(fbnic_hw_stats, name, stat)
  29
  30static const struct fbnic_stat fbnic_gstrings_hw_stats[] = {
  31	/* TTI */
  32	FBNIC_HW_STAT("tti_cm_drop_frames", tti.cm_drop.frames),
  33	FBNIC_HW_STAT("tti_cm_drop_bytes", tti.cm_drop.bytes),
  34	FBNIC_HW_STAT("tti_frame_drop_frames", tti.frame_drop.frames),
  35	FBNIC_HW_STAT("tti_frame_drop_bytes", tti.frame_drop.bytes),
  36	FBNIC_HW_STAT("tti_tbi_drop_frames", tti.tbi_drop.frames),
  37	FBNIC_HW_STAT("tti_tbi_drop_bytes", tti.tbi_drop.bytes),
  38
  39	/* TMI */
  40	FBNIC_HW_STAT("ptp_illegal_req", tmi.ptp_illegal_req),
  41	FBNIC_HW_STAT("ptp_good_ts", tmi.ptp_good_ts),
  42	FBNIC_HW_STAT("ptp_bad_ts", tmi.ptp_bad_ts),
  43
  44	/* RPC */
  45	FBNIC_HW_STAT("rpc_unkn_etype", rpc.unkn_etype),
  46	FBNIC_HW_STAT("rpc_unkn_ext_hdr", rpc.unkn_ext_hdr),
  47	FBNIC_HW_STAT("rpc_ipv4_frag", rpc.ipv4_frag),
  48	FBNIC_HW_STAT("rpc_ipv6_frag", rpc.ipv6_frag),
  49	FBNIC_HW_STAT("rpc_ipv4_esp", rpc.ipv4_esp),
  50	FBNIC_HW_STAT("rpc_ipv6_esp", rpc.ipv6_esp),
  51	FBNIC_HW_STAT("rpc_tcp_opt_err", rpc.tcp_opt_err),
  52	FBNIC_HW_STAT("rpc_out_of_hdr_err", rpc.out_of_hdr_err),
  53};
  54
  55#define FBNIC_HW_FIXED_STATS_LEN ARRAY_SIZE(fbnic_gstrings_hw_stats)
  56
  57#define FBNIC_RXB_ENQUEUE_STAT(name, stat) \
  58	FBNIC_STAT_FIELDS(fbnic_rxb_enqueue_stats, name, stat)
  59
  60static const struct fbnic_stat fbnic_gstrings_rxb_enqueue_stats[] = {
  61	FBNIC_RXB_ENQUEUE_STAT("rxb_integrity_err%u", integrity_err),
  62	FBNIC_RXB_ENQUEUE_STAT("rxb_mac_err%u", mac_err),
  63	FBNIC_RXB_ENQUEUE_STAT("rxb_parser_err%u", parser_err),
  64	FBNIC_RXB_ENQUEUE_STAT("rxb_frm_err%u", frm_err),
  65
  66	FBNIC_RXB_ENQUEUE_STAT("rxb_drbo%u_frames", drbo.frames),
  67	FBNIC_RXB_ENQUEUE_STAT("rxb_drbo%u_bytes", drbo.bytes),
  68};
  69
  70#define FBNIC_HW_RXB_ENQUEUE_STATS_LEN \
  71	ARRAY_SIZE(fbnic_gstrings_rxb_enqueue_stats)
  72
  73#define FBNIC_RXB_FIFO_STAT(name, stat) \
  74	FBNIC_STAT_FIELDS(fbnic_rxb_fifo_stats, name, stat)
  75
  76static const struct fbnic_stat fbnic_gstrings_rxb_fifo_stats[] = {
  77	FBNIC_RXB_FIFO_STAT("rxb_fifo%u_drop", trans_drop),
  78	FBNIC_RXB_FIFO_STAT("rxb_fifo%u_dropped_frames", drop.frames),
  79	FBNIC_RXB_FIFO_STAT("rxb_fifo%u_ecn", trans_ecn),
  80	FBNIC_RXB_FIFO_STAT("rxb_fifo%u_level", level),
  81};
  82
  83#define FBNIC_HW_RXB_FIFO_STATS_LEN ARRAY_SIZE(fbnic_gstrings_rxb_fifo_stats)
  84
  85#define FBNIC_RXB_DEQUEUE_STAT(name, stat) \
  86	FBNIC_STAT_FIELDS(fbnic_rxb_dequeue_stats, name, stat)
  87
  88static const struct fbnic_stat fbnic_gstrings_rxb_dequeue_stats[] = {
  89	FBNIC_RXB_DEQUEUE_STAT("rxb_intf%u_frames", intf.frames),
  90	FBNIC_RXB_DEQUEUE_STAT("rxb_intf%u_bytes", intf.bytes),
  91	FBNIC_RXB_DEQUEUE_STAT("rxb_pbuf%u_frames", pbuf.frames),
  92	FBNIC_RXB_DEQUEUE_STAT("rxb_pbuf%u_bytes", pbuf.bytes),
  93};
  94
  95#define FBNIC_HW_RXB_DEQUEUE_STATS_LEN \
  96	ARRAY_SIZE(fbnic_gstrings_rxb_dequeue_stats)
  97
  98#define FBNIC_HW_Q_STAT(name, stat) \
  99	FBNIC_STAT_FIELDS(fbnic_hw_q_stats, name, stat.value)
 100
 101static const struct fbnic_stat fbnic_gstrings_hw_q_stats[] = {
 102	FBNIC_HW_Q_STAT("rde_%u_pkt_err", rde_pkt_err),
 103	FBNIC_HW_Q_STAT("rde_%u_pkt_cq_drop", rde_pkt_cq_drop),
 104	FBNIC_HW_Q_STAT("rde_%u_pkt_bdq_drop", rde_pkt_bdq_drop),
 105};
 106
 107#define FBNIC_HW_Q_STATS_LEN ARRAY_SIZE(fbnic_gstrings_hw_q_stats)
 108#define FBNIC_HW_STATS_LEN \
 109	(FBNIC_HW_FIXED_STATS_LEN + \
 110	 FBNIC_HW_RXB_ENQUEUE_STATS_LEN * FBNIC_RXB_ENQUEUE_INDICES + \
 111	 FBNIC_HW_RXB_FIFO_STATS_LEN * FBNIC_RXB_FIFO_INDICES + \
 112	 FBNIC_HW_RXB_DEQUEUE_STATS_LEN * FBNIC_RXB_DEQUEUE_INDICES + \
 113	 FBNIC_HW_Q_STATS_LEN * FBNIC_MAX_QUEUES)
 114
 115#define FBNIC_QUEUE_STAT(name, stat) \
 116	FBNIC_STAT_FIELDS(fbnic_ring, name, stat)
 117
 118static const struct fbnic_stat fbnic_gstrings_xdp_stats[] = {
 119	FBNIC_QUEUE_STAT("xdp_tx_queue_%u_packets", stats.packets),
 120	FBNIC_QUEUE_STAT("xdp_tx_queue_%u_bytes", stats.bytes),
 121	FBNIC_QUEUE_STAT("xdp_tx_queue_%u_dropped", stats.dropped),
 122};
 123
 124#define FBNIC_XDP_STATS_LEN ARRAY_SIZE(fbnic_gstrings_xdp_stats)
 125
 126#define FBNIC_STATS_LEN \
 127	(FBNIC_HW_STATS_LEN + FBNIC_XDP_STATS_LEN * FBNIC_MAX_XDPQS)
 128
 129static void
 130fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 131{
 132	struct fbnic_net *fbn = netdev_priv(netdev);
 133	struct fbnic_dev *fbd = fbn->fbd;
 134
 135	fbnic_get_fw_ver_commit_str(fbd, drvinfo->fw_version,
 136				    sizeof(drvinfo->fw_version));
 137}
 138
 139static int fbnic_get_regs_len(struct net_device *netdev)
 140{
 141	struct fbnic_net *fbn = netdev_priv(netdev);
 142
 143	return fbnic_csr_regs_len(fbn->fbd) * sizeof(u32);
 144}
 145
 146static void fbnic_get_regs(struct net_device *netdev,
 147			   struct ethtool_regs *regs, void *data)
 148{
 149	struct fbnic_net *fbn = netdev_priv(netdev);
 150
 151	fbnic_csr_get_regs(fbn->fbd, data, &regs->version);
 152}
 153
 154static struct fbnic_net *fbnic_clone_create(struct fbnic_net *orig)
 155{
 156	struct fbnic_net *clone;
 157
 158	clone = kmemdup(orig, sizeof(*orig), GFP_KERNEL);
 159	if (!clone)
 160		return NULL;
 161
 162	memset(clone->tx, 0, sizeof(clone->tx));
 163	memset(clone->rx, 0, sizeof(clone->rx));
 164	memset(clone->napi, 0, sizeof(clone->napi));
 165	return clone;
 166}
 167
 168static void fbnic_clone_swap_cfg(struct fbnic_net *orig,
 169				 struct fbnic_net *clone)
 170{
 171	swap(clone->rcq_size, orig->rcq_size);
 172	swap(clone->hpq_size, orig->hpq_size);
 173	swap(clone->ppq_size, orig->ppq_size);
 174	swap(clone->txq_size, orig->txq_size);
 175	swap(clone->num_rx_queues, orig->num_rx_queues);
 176	swap(clone->num_tx_queues, orig->num_tx_queues);
 177	swap(clone->num_napi, orig->num_napi);
 178	swap(clone->hds_thresh, orig->hds_thresh);
 179}
 180
 181static void fbnic_aggregate_vector_counters(struct fbnic_net *fbn,
 182					    struct fbnic_napi_vector *nv)
 183{
 184	int i, j;
 185
 186	for (i = 0; i < nv->txt_count; i++) {
 187		fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].sub0);
 188		fbnic_aggregate_ring_xdp_counters(fbn, &nv->qt[i].sub1);
 189		fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].cmpl);
 190	}
 191
 192	for (j = 0; j < nv->rxt_count; j++, i++) {
 193		fbnic_aggregate_ring_bdq_counters(fbn, &nv->qt[i].sub0);
 194		fbnic_aggregate_ring_bdq_counters(fbn, &nv->qt[i].sub1);
 195		fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].cmpl);
 196	}
 197}
 198
 199static void fbnic_clone_swap(struct fbnic_net *orig,
 200			     struct fbnic_net *clone)
 201{
 202	struct fbnic_dev *fbd = orig->fbd;
 203	unsigned int i;
 204
 205	for (i = 0; i < max(clone->num_napi, orig->num_napi); i++)
 206		fbnic_synchronize_irq(fbd, FBNIC_NON_NAPI_VECTORS + i);
 207	for (i = 0; i < orig->num_napi; i++)
 208		fbnic_aggregate_vector_counters(orig, orig->napi[i]);
 209
 210	fbnic_clone_swap_cfg(orig, clone);
 211
 212	for (i = 0; i < ARRAY_SIZE(orig->napi); i++)
 213		swap(clone->napi[i], orig->napi[i]);
 214	for (i = 0; i < ARRAY_SIZE(orig->tx); i++)
 215		swap(clone->tx[i], orig->tx[i]);
 216	for (i = 0; i < ARRAY_SIZE(orig->rx); i++)
 217		swap(clone->rx[i], orig->rx[i]);
 218}
 219
 220static void fbnic_clone_free(struct fbnic_net *clone)
 221{
 222	kfree(clone);
 223}
 224
 225static int fbnic_get_coalesce(struct net_device *netdev,
 226			      struct ethtool_coalesce *ec,
 227			      struct kernel_ethtool_coalesce *kernel_coal,
 228			      struct netlink_ext_ack *extack)
 229{
 230	struct fbnic_net *fbn = netdev_priv(netdev);
 231
 232	ec->tx_coalesce_usecs = fbn->tx_usecs;
 233	ec->rx_coalesce_usecs = fbn->rx_usecs;
 234	ec->rx_max_coalesced_frames = fbn->rx_max_frames;
 235
 236	return 0;
 237}
 238
 239static int fbnic_set_coalesce(struct net_device *netdev,
 240			      struct ethtool_coalesce *ec,
 241			      struct kernel_ethtool_coalesce *kernel_coal,
 242			      struct netlink_ext_ack *extack)
 243{
 244	struct fbnic_net *fbn = netdev_priv(netdev);
 245
 246	/* Verify against hardware limits */
 247	if (ec->rx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT)) {
 248		NL_SET_ERR_MSG_MOD(extack, "rx_usecs is above device max");
 249		return -EINVAL;
 250	}
 251	if (ec->tx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT)) {
 252		NL_SET_ERR_MSG_MOD(extack, "tx_usecs is above device max");
 253		return -EINVAL;
 254	}
 255	if (ec->rx_max_coalesced_frames >
 256	    FIELD_MAX(FBNIC_QUEUE_RIM_THRESHOLD_RCD_MASK) /
 257	    FBNIC_MIN_RXD_PER_FRAME) {
 258		NL_SET_ERR_MSG_MOD(extack, "rx_frames is above device max");
 259		return -EINVAL;
 260	}
 261
 262	fbn->tx_usecs = ec->tx_coalesce_usecs;
 263	fbn->rx_usecs = ec->rx_coalesce_usecs;
 264	fbn->rx_max_frames = ec->rx_max_coalesced_frames;
 265
 266	if (netif_running(netdev)) {
 267		int i;
 268
 269		for (i = 0; i < fbn->num_napi; i++) {
 270			struct fbnic_napi_vector *nv = fbn->napi[i];
 271
 272			fbnic_config_txrx_usecs(nv, 0);
 273			fbnic_config_rx_frames(nv);
 274		}
 275	}
 276
 277	return 0;
 278}
 279
 280static void
 281fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
 282		    struct kernel_ethtool_ringparam *kernel_ring,
 283		    struct netlink_ext_ack *extack)
 284{
 285	struct fbnic_net *fbn = netdev_priv(netdev);
 286
 287	ring->rx_max_pending = FBNIC_QUEUE_SIZE_MAX;
 288	ring->rx_mini_max_pending = FBNIC_QUEUE_SIZE_MAX;
 289	ring->rx_jumbo_max_pending = FBNIC_QUEUE_SIZE_MAX;
 290	ring->tx_max_pending = FBNIC_QUEUE_SIZE_MAX;
 291
 292	ring->rx_pending = fbn->rcq_size;
 293	ring->rx_mini_pending = fbn->hpq_size;
 294	ring->rx_jumbo_pending = fbn->ppq_size;
 295	ring->tx_pending = fbn->txq_size;
 296
 297	kernel_ring->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_ENABLED;
 298	kernel_ring->hds_thresh_max = FBNIC_HDS_THRESH_MAX;
 299	kernel_ring->hds_thresh = fbn->hds_thresh;
 300}
 301
 302static void fbnic_set_rings(struct fbnic_net *fbn,
 303			    struct ethtool_ringparam *ring,
 304			    struct kernel_ethtool_ringparam *kernel_ring)
 305{
 306	fbn->rcq_size = ring->rx_pending;
 307	fbn->hpq_size = ring->rx_mini_pending;
 308	fbn->ppq_size = ring->rx_jumbo_pending;
 309	fbn->txq_size = ring->tx_pending;
 310	fbn->hds_thresh = kernel_ring->hds_thresh;
 311}
 312
 313static int
 314fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
 315		    struct kernel_ethtool_ringparam *kernel_ring,
 316		    struct netlink_ext_ack *extack)
 317
 318{
 319	struct fbnic_net *fbn = netdev_priv(netdev);
 320	struct fbnic_net *clone;
 321	int err;
 322
 323	ring->rx_pending	= roundup_pow_of_two(ring->rx_pending);
 324	ring->rx_mini_pending	= roundup_pow_of_two(ring->rx_mini_pending);
 325	ring->rx_jumbo_pending	= roundup_pow_of_two(ring->rx_jumbo_pending);
 326	ring->tx_pending	= roundup_pow_of_two(ring->tx_pending);
 327
 328	/* These are absolute minimums allowing the device and driver to operate
 329	 * but not necessarily guarantee reasonable performance. Settings below
 330	 * Rx queue size of 128 and BDQs smaller than 64 are likely suboptimal
 331	 * at best.
 332	 */
 333	if (ring->rx_pending < max(FBNIC_QUEUE_SIZE_MIN, FBNIC_RX_DESC_MIN) ||
 334	    ring->rx_mini_pending < FBNIC_QUEUE_SIZE_MIN ||
 335	    ring->rx_jumbo_pending < FBNIC_QUEUE_SIZE_MIN ||
 336	    ring->tx_pending < max(FBNIC_QUEUE_SIZE_MIN, FBNIC_TX_DESC_MIN)) {
 337		NL_SET_ERR_MSG_MOD(extack, "requested ring size too small");
 338		return -EINVAL;
 339	}
 340
 341	if (kernel_ring->tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_DISABLED) {
 342		NL_SET_ERR_MSG_MOD(extack, "Cannot disable TCP data split");
 343		return -EINVAL;
 344	}
 345
 346	/* If an XDP program is attached, we should check for potential frame
 347	 * splitting. If the new HDS threshold can cause splitting, we should
 348	 * only allow if the attached XDP program can handle frags.
 349	 */
 350	if (fbnic_check_split_frames(fbn->xdp_prog, netdev->mtu,
 351				     kernel_ring->hds_thresh)) {
 352		NL_SET_ERR_MSG_MOD(extack,
 353				   "Use higher HDS threshold or multi-buf capable program");
 354		return -EINVAL;
 355	}
 356
 357	if (!netif_running(netdev)) {
 358		fbnic_set_rings(fbn, ring, kernel_ring);
 359		return 0;
 360	}
 361
 362	clone = fbnic_clone_create(fbn);
 363	if (!clone)
 364		return -ENOMEM;
 365
 366	fbnic_set_rings(clone, ring, kernel_ring);
 367
 368	err = fbnic_alloc_napi_vectors(clone);
 369	if (err)
 370		goto err_free_clone;
 371
 372	err = fbnic_alloc_resources(clone);
 373	if (err)
 374		goto err_free_napis;
 375
 376	fbnic_down_noidle(fbn);
 377	err = fbnic_wait_all_queues_idle(fbn->fbd, true);
 378	if (err)
 379		goto err_start_stack;
 380
 381	err = fbnic_set_netif_queues(clone);
 382	if (err)
 383		goto err_start_stack;
 384
 385	/* Nothing can fail past this point */
 386	fbnic_flush(fbn);
 387
 388	fbnic_clone_swap(fbn, clone);
 389
 390	fbnic_up(fbn);
 391
 392	fbnic_free_resources(clone);
 393	fbnic_free_napi_vectors(clone);
 394	fbnic_clone_free(clone);
 395
 396	return 0;
 397
 398err_start_stack:
 399	fbnic_flush(fbn);
 400	fbnic_up(fbn);
 401	fbnic_free_resources(clone);
 402err_free_napis:
 403	fbnic_free_napi_vectors(clone);
 404err_free_clone:
 405	fbnic_clone_free(clone);
 406	return err;
 407}
 408
 409static void fbnic_get_rxb_enqueue_strings(u8 **data, unsigned int idx)
 410{
 411	const struct fbnic_stat *stat;
 412	int i;
 413
 414	stat = fbnic_gstrings_rxb_enqueue_stats;
 415	for (i = 0; i < FBNIC_HW_RXB_ENQUEUE_STATS_LEN; i++, stat++)
 416		ethtool_sprintf(data, stat->string, idx);
 417}
 418
 419static void fbnic_get_rxb_fifo_strings(u8 **data, unsigned int idx)
 420{
 421	const struct fbnic_stat *stat;
 422	int i;
 423
 424	stat = fbnic_gstrings_rxb_fifo_stats;
 425	for (i = 0; i < FBNIC_HW_RXB_FIFO_STATS_LEN; i++, stat++)
 426		ethtool_sprintf(data, stat->string, idx);
 427}
 428
 429static void fbnic_get_rxb_dequeue_strings(u8 **data, unsigned int idx)
 430{
 431	const struct fbnic_stat *stat;
 432	int i;
 433
 434	stat = fbnic_gstrings_rxb_dequeue_stats;
 435	for (i = 0; i < FBNIC_HW_RXB_DEQUEUE_STATS_LEN; i++, stat++)
 436		ethtool_sprintf(data, stat->string, idx);
 437}
 438
 439static void fbnic_get_xdp_queue_strings(u8 **data, unsigned int idx)
 440{
 441	const struct fbnic_stat *stat;
 442	int i;
 443
 444	stat = fbnic_gstrings_xdp_stats;
 445	for (i = 0; i < FBNIC_XDP_STATS_LEN; i++, stat++)
 446		ethtool_sprintf(data, stat->string, idx);
 447}
 448
 449static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data)
 450{
 451	const struct fbnic_stat *stat;
 452	int i, idx;
 453
 454	switch (sset) {
 455	case ETH_SS_STATS:
 456		for (i = 0; i < FBNIC_HW_FIXED_STATS_LEN; i++)
 457			ethtool_puts(&data, fbnic_gstrings_hw_stats[i].string);
 458
 459		for (i = 0; i < FBNIC_RXB_ENQUEUE_INDICES; i++)
 460			fbnic_get_rxb_enqueue_strings(&data, i);
 461
 462		for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++)
 463			fbnic_get_rxb_fifo_strings(&data, i);
 464
 465		for (i = 0; i < FBNIC_RXB_DEQUEUE_INDICES; i++)
 466			fbnic_get_rxb_dequeue_strings(&data, i);
 467
 468		for (idx = 0; idx < FBNIC_MAX_QUEUES; idx++) {
 469			stat = fbnic_gstrings_hw_q_stats;
 470
 471			for (i = 0; i < FBNIC_HW_Q_STATS_LEN; i++, stat++)
 472				ethtool_sprintf(&data, stat->string, idx);
 473		}
 474
 475		for (i = 0; i < FBNIC_MAX_XDPQS; i++)
 476			fbnic_get_xdp_queue_strings(&data, i);
 477		break;
 478	}
 479}
 480
 481static void fbnic_report_hw_stats(const struct fbnic_stat *stat,
 482				  const void *base, int len, u64 **data)
 483{
 484	while (len--) {
 485		u8 *curr = (u8 *)base + stat->offset;
 486
 487		**data = *(u64 *)curr;
 488
 489		stat++;
 490		(*data)++;
 491	}
 492}
 493
 494static void fbnic_get_xdp_queue_stats(struct fbnic_ring *ring, u64 **data)
 495{
 496	const struct fbnic_stat *stat;
 497	int i;
 498
 499	if (!ring) {
 500		*data += FBNIC_XDP_STATS_LEN;
 501		return;
 502	}
 503
 504	stat = fbnic_gstrings_xdp_stats;
 505	for (i = 0; i < FBNIC_XDP_STATS_LEN; i++, stat++, (*data)++) {
 506		u8 *p = (u8 *)ring + stat->offset;
 507
 508		**data = *(u64 *)p;
 509	}
 510}
 511
 512static void fbnic_get_ethtool_stats(struct net_device *dev,
 513				    struct ethtool_stats *stats, u64 *data)
 514{
 515	struct fbnic_net *fbn = netdev_priv(dev);
 516	struct fbnic_dev *fbd = fbn->fbd;
 517	int i;
 518
 519	fbnic_get_hw_stats(fbn->fbd);
 520
 521	spin_lock(&fbd->hw_stats.lock);
 522	fbnic_report_hw_stats(fbnic_gstrings_hw_stats, &fbd->hw_stats,
 523			      FBNIC_HW_FIXED_STATS_LEN, &data);
 524
 525	for (i = 0; i < FBNIC_RXB_ENQUEUE_INDICES; i++) {
 526		const struct fbnic_rxb_enqueue_stats *enq;
 527
 528		enq = &fbd->hw_stats.rxb.enq[i];
 529		fbnic_report_hw_stats(fbnic_gstrings_rxb_enqueue_stats,
 530				      enq, FBNIC_HW_RXB_ENQUEUE_STATS_LEN,
 531				      &data);
 532	}
 533
 534	for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++) {
 535		const struct fbnic_rxb_fifo_stats *fifo;
 536
 537		fifo = &fbd->hw_stats.rxb.fifo[i];
 538		fbnic_report_hw_stats(fbnic_gstrings_rxb_fifo_stats,
 539				      fifo, FBNIC_HW_RXB_FIFO_STATS_LEN,
 540				      &data);
 541	}
 542
 543	for (i = 0; i < FBNIC_RXB_DEQUEUE_INDICES; i++) {
 544		const struct fbnic_rxb_dequeue_stats *deq;
 545
 546		deq = &fbd->hw_stats.rxb.deq[i];
 547		fbnic_report_hw_stats(fbnic_gstrings_rxb_dequeue_stats,
 548				      deq, FBNIC_HW_RXB_DEQUEUE_STATS_LEN,
 549				      &data);
 550	}
 551
 552	for (i  = 0; i < FBNIC_MAX_QUEUES; i++) {
 553		const struct fbnic_hw_q_stats *hw_q = &fbd->hw_stats.hw_q[i];
 554
 555		fbnic_report_hw_stats(fbnic_gstrings_hw_q_stats, hw_q,
 556				      FBNIC_HW_Q_STATS_LEN, &data);
 557	}
 558	spin_unlock(&fbd->hw_stats.lock);
 559
 560	for (i = 0; i < FBNIC_MAX_XDPQS; i++)
 561		fbnic_get_xdp_queue_stats(fbn->tx[i + FBNIC_MAX_TXQS], &data);
 562}
 563
 564static int fbnic_get_sset_count(struct net_device *dev, int sset)
 565{
 566	switch (sset) {
 567	case ETH_SS_STATS:
 568		return FBNIC_STATS_LEN;
 569	default:
 570		return -EOPNOTSUPP;
 571	}
 572}
 573
 574static int fbnic_get_rss_hash_idx(u32 flow_type)
 575{
 576	switch (flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS)) {
 577	case TCP_V4_FLOW:
 578		return FBNIC_TCP4_HASH_OPT;
 579	case TCP_V6_FLOW:
 580		return FBNIC_TCP6_HASH_OPT;
 581	case UDP_V4_FLOW:
 582		return FBNIC_UDP4_HASH_OPT;
 583	case UDP_V6_FLOW:
 584		return FBNIC_UDP6_HASH_OPT;
 585	case AH_V4_FLOW:
 586	case ESP_V4_FLOW:
 587	case AH_ESP_V4_FLOW:
 588	case SCTP_V4_FLOW:
 589	case IPV4_FLOW:
 590	case IPV4_USER_FLOW:
 591		return FBNIC_IPV4_HASH_OPT;
 592	case AH_V6_FLOW:
 593	case ESP_V6_FLOW:
 594	case AH_ESP_V6_FLOW:
 595	case SCTP_V6_FLOW:
 596	case IPV6_FLOW:
 597	case IPV6_USER_FLOW:
 598		return FBNIC_IPV6_HASH_OPT;
 599	case ETHER_FLOW:
 600		return FBNIC_ETHER_HASH_OPT;
 601	}
 602
 603	return -1;
 604}
 605
 606static int fbnic_get_cls_rule_all(struct fbnic_net *fbn,
 607				  struct ethtool_rxnfc *cmd,
 608				  u32 *rule_locs)
 609{
 610	struct fbnic_dev *fbd = fbn->fbd;
 611	int i, cnt = 0;
 612
 613	/* Report maximum rule count */
 614	cmd->data = FBNIC_RPC_ACT_TBL_NFC_ENTRIES;
 615
 616	for (i = 0; i < FBNIC_RPC_ACT_TBL_NFC_ENTRIES; i++) {
 617		int idx = i + FBNIC_RPC_ACT_TBL_NFC_OFFSET;
 618		struct fbnic_act_tcam *act_tcam;
 619
 620		act_tcam = &fbd->act_tcam[idx];
 621		if (act_tcam->state != FBNIC_TCAM_S_VALID)
 622			continue;
 623
 624		if (rule_locs) {
 625			if (cnt == cmd->rule_cnt)
 626				return -EMSGSIZE;
 627
 628			rule_locs[cnt] = i;
 629		}
 630
 631		cnt++;
 632	}
 633
 634	return cnt;
 635}
 636
 637static int fbnic_get_cls_rule(struct fbnic_net *fbn, struct ethtool_rxnfc *cmd)
 638{
 639	struct ethtool_rx_flow_spec *fsp;
 640	struct fbnic_dev *fbd = fbn->fbd;
 641	struct fbnic_act_tcam *act_tcam;
 642	int idx;
 643
 644	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
 645
 646	if (fsp->location >= FBNIC_RPC_ACT_TBL_NFC_ENTRIES)
 647		return -EINVAL;
 648
 649	idx = fsp->location + FBNIC_RPC_ACT_TBL_NFC_OFFSET;
 650	act_tcam = &fbd->act_tcam[idx];
 651
 652	if (act_tcam->state != FBNIC_TCAM_S_VALID)
 653		return -EINVAL;
 654
 655	/* Report maximum rule count */
 656	cmd->data = FBNIC_RPC_ACT_TBL_NFC_ENTRIES;
 657
 658	/* Set flow type field */
 659	if (!(act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_IP_VALID)) {
 660		fsp->flow_type = ETHER_FLOW;
 661		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX,
 662			       act_tcam->mask.tcam[1])) {
 663			struct fbnic_mac_addr *mac_addr;
 664
 665			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX,
 666					act_tcam->value.tcam[1]);
 667			mac_addr = &fbd->mac_addr[idx];
 668
 669			ether_addr_copy(fsp->h_u.ether_spec.h_dest,
 670					mac_addr->value.addr8);
 671			eth_broadcast_addr(fsp->m_u.ether_spec.h_dest);
 672		}
 673	} else if (act_tcam->value.tcam[1] &
 674		   FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID) {
 675		fsp->flow_type = IPV6_USER_FLOW;
 676		fsp->h_u.usr_ip6_spec.l4_proto = IPPROTO_IPV6;
 677		fsp->m_u.usr_ip6_spec.l4_proto = 0xff;
 678
 679		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX,
 680			       act_tcam->mask.tcam[0])) {
 681			struct fbnic_ip_addr *ip_addr;
 682			int i;
 683
 684			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX,
 685					act_tcam->value.tcam[0]);
 686			ip_addr = &fbd->ipo_src[idx];
 687
 688			for (i = 0; i < 4; i++) {
 689				fsp->h_u.usr_ip6_spec.ip6src[i] =
 690					ip_addr->value.s6_addr32[i];
 691				fsp->m_u.usr_ip6_spec.ip6src[i] =
 692					~ip_addr->mask.s6_addr32[i];
 693			}
 694		}
 695
 696		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX,
 697			       act_tcam->mask.tcam[0])) {
 698			struct fbnic_ip_addr *ip_addr;
 699			int i;
 700
 701			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX,
 702					act_tcam->value.tcam[0]);
 703			ip_addr = &fbd->ipo_dst[idx];
 704
 705			for (i = 0; i < 4; i++) {
 706				fsp->h_u.usr_ip6_spec.ip6dst[i] =
 707					ip_addr->value.s6_addr32[i];
 708				fsp->m_u.usr_ip6_spec.ip6dst[i] =
 709					~ip_addr->mask.s6_addr32[i];
 710			}
 711		}
 712	} else if ((act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_IP_IS_V6)) {
 713		if (act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L4_VALID) {
 714			if (act_tcam->value.tcam[1] &
 715			    FBNIC_RPC_TCAM_ACT1_L4_IS_UDP)
 716				fsp->flow_type = UDP_V6_FLOW;
 717			else
 718				fsp->flow_type = TCP_V6_FLOW;
 719			fsp->h_u.tcp_ip6_spec.psrc =
 720				cpu_to_be16(act_tcam->value.tcam[3]);
 721			fsp->m_u.tcp_ip6_spec.psrc =
 722				cpu_to_be16(~act_tcam->mask.tcam[3]);
 723			fsp->h_u.tcp_ip6_spec.pdst =
 724				cpu_to_be16(act_tcam->value.tcam[4]);
 725			fsp->m_u.tcp_ip6_spec.pdst =
 726				cpu_to_be16(~act_tcam->mask.tcam[4]);
 727		} else {
 728			fsp->flow_type = IPV6_USER_FLOW;
 729		}
 730
 731		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX,
 732			       act_tcam->mask.tcam[0])) {
 733			struct fbnic_ip_addr *ip_addr;
 734			int i;
 735
 736			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX,
 737					act_tcam->value.tcam[0]);
 738			ip_addr = &fbd->ip_src[idx];
 739
 740			for (i = 0; i < 4; i++) {
 741				fsp->h_u.usr_ip6_spec.ip6src[i] =
 742					ip_addr->value.s6_addr32[i];
 743				fsp->m_u.usr_ip6_spec.ip6src[i] =
 744					~ip_addr->mask.s6_addr32[i];
 745			}
 746		}
 747
 748		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX,
 749			       act_tcam->mask.tcam[0])) {
 750			struct fbnic_ip_addr *ip_addr;
 751			int i;
 752
 753			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX,
 754					act_tcam->value.tcam[0]);
 755			ip_addr = &fbd->ip_dst[idx];
 756
 757			for (i = 0; i < 4; i++) {
 758				fsp->h_u.usr_ip6_spec.ip6dst[i] =
 759					ip_addr->value.s6_addr32[i];
 760				fsp->m_u.usr_ip6_spec.ip6dst[i] =
 761					~ip_addr->mask.s6_addr32[i];
 762			}
 763		}
 764	} else {
 765		if (act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L4_VALID) {
 766			if (act_tcam->value.tcam[1] &
 767			    FBNIC_RPC_TCAM_ACT1_L4_IS_UDP)
 768				fsp->flow_type = UDP_V4_FLOW;
 769			else
 770				fsp->flow_type = TCP_V4_FLOW;
 771			fsp->h_u.tcp_ip4_spec.psrc =
 772				cpu_to_be16(act_tcam->value.tcam[3]);
 773			fsp->m_u.tcp_ip4_spec.psrc =
 774				cpu_to_be16(~act_tcam->mask.tcam[3]);
 775			fsp->h_u.tcp_ip4_spec.pdst =
 776				cpu_to_be16(act_tcam->value.tcam[4]);
 777			fsp->m_u.tcp_ip4_spec.pdst =
 778				cpu_to_be16(~act_tcam->mask.tcam[4]);
 779		} else {
 780			fsp->flow_type = IPV4_USER_FLOW;
 781			fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4;
 782		}
 783
 784		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX,
 785			       act_tcam->mask.tcam[0])) {
 786			struct fbnic_ip_addr *ip_addr;
 787
 788			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX,
 789					act_tcam->value.tcam[0]);
 790			ip_addr = &fbd->ip_src[idx];
 791
 792			fsp->h_u.usr_ip4_spec.ip4src =
 793				ip_addr->value.s6_addr32[3];
 794			fsp->m_u.usr_ip4_spec.ip4src =
 795				~ip_addr->mask.s6_addr32[3];
 796		}
 797
 798		if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX,
 799			       act_tcam->mask.tcam[0])) {
 800			struct fbnic_ip_addr *ip_addr;
 801
 802			idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX,
 803					act_tcam->value.tcam[0]);
 804			ip_addr = &fbd->ip_dst[idx];
 805
 806			fsp->h_u.usr_ip4_spec.ip4dst =
 807				ip_addr->value.s6_addr32[3];
 808			fsp->m_u.usr_ip4_spec.ip4dst =
 809				~ip_addr->mask.s6_addr32[3];
 810		}
 811	}
 812
 813	/* Record action */
 814	if (act_tcam->dest & FBNIC_RPC_ACT_TBL0_DROP)
 815		fsp->ring_cookie = RX_CLS_FLOW_DISC;
 816	else if (act_tcam->dest & FBNIC_RPC_ACT_TBL0_Q_SEL)
 817		fsp->ring_cookie = FIELD_GET(FBNIC_RPC_ACT_TBL0_Q_ID,
 818					     act_tcam->dest);
 819	else
 820		fsp->flow_type |= FLOW_RSS;
 821
 822	cmd->rss_context = FIELD_GET(FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID,
 823				     act_tcam->dest);
 824
 825	return 0;
 826}
 827
 828static int fbnic_get_rxnfc(struct net_device *netdev,
 829			   struct ethtool_rxnfc *cmd, u32 *rule_locs)
 830{
 831	struct fbnic_net *fbn = netdev_priv(netdev);
 832	int ret = -EOPNOTSUPP;
 833	u32 special = 0;
 834
 835	switch (cmd->cmd) {
 836	case ETHTOOL_GRXRINGS:
 837		cmd->data = fbn->num_rx_queues;
 838		ret = 0;
 839		break;
 840	case ETHTOOL_GRXCLSRULE:
 841		ret = fbnic_get_cls_rule(fbn, cmd);
 842		break;
 843	case ETHTOOL_GRXCLSRLCNT:
 844		rule_locs = NULL;
 845		special = RX_CLS_LOC_SPECIAL;
 846		fallthrough;
 847	case ETHTOOL_GRXCLSRLALL:
 848		ret = fbnic_get_cls_rule_all(fbn, cmd, rule_locs);
 849		if (ret < 0)
 850			break;
 851
 852		cmd->data |= special;
 853		cmd->rule_cnt = ret;
 854		ret = 0;
 855		break;
 856	}
 857
 858	return ret;
 859}
 860
 861static int fbnic_cls_rule_any_loc(struct fbnic_dev *fbd)
 862{
 863	int i;
 864
 865	for (i = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; i--;) {
 866		int idx = i + FBNIC_RPC_ACT_TBL_NFC_OFFSET;
 867
 868		if (fbd->act_tcam[idx].state != FBNIC_TCAM_S_VALID)
 869			return i;
 870	}
 871
 872	return -ENOSPC;
 873}
 874
 875static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn,
 876				  const struct ethtool_rxnfc *cmd)
 877{
 878	u16 flow_value = 0, flow_mask = 0xffff, ip_value = 0, ip_mask = 0xffff;
 879	u16 sport = 0, sport_mask = ~0, dport = 0, dport_mask = ~0;
 880	u16 misc = 0, misc_mask = ~0;
 881	u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK,
 882			      FBNIC_RPC_ACT_TBL0_DEST_HOST);
 883	struct fbnic_ip_addr *ip_src = NULL, *ip_dst = NULL;
 884	struct fbnic_mac_addr *mac_addr = NULL;
 885	struct ethtool_rx_flow_spec *fsp;
 886	struct fbnic_dev *fbd = fbn->fbd;
 887	struct fbnic_act_tcam *act_tcam;
 888	struct in6_addr *addr6, *mask6;
 889	struct in_addr *addr4, *mask4;
 890	int hash_idx, location;
 891	u32 flow_type;
 892	int idx, j;
 893
 894	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
 895
 896	if (fsp->location != RX_CLS_LOC_ANY)
 897		return -EINVAL;
 898	location = fbnic_cls_rule_any_loc(fbd);
 899	if (location < 0)
 900		return location;
 901
 902	if (fsp->ring_cookie == RX_CLS_FLOW_DISC) {
 903		dest = FBNIC_RPC_ACT_TBL0_DROP;
 904	} else if (fsp->flow_type & FLOW_RSS) {
 905		if (cmd->rss_context == 1)
 906			dest |= FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID;
 907	} else {
 908		u32 ring_idx = ethtool_get_flow_spec_ring(fsp->ring_cookie);
 909
 910		if (ring_idx >= fbn->num_rx_queues)
 911			return -EINVAL;
 912
 913		dest |= FBNIC_RPC_ACT_TBL0_Q_SEL |
 914			FIELD_PREP(FBNIC_RPC_ACT_TBL0_Q_ID, ring_idx);
 915	}
 916
 917	idx = location + FBNIC_RPC_ACT_TBL_NFC_OFFSET;
 918	act_tcam = &fbd->act_tcam[idx];
 919
 920	/* Do not allow overwriting for now.
 921	 * To support overwriting rules we will need to add logic to free
 922	 * any IP or MACDA TCAMs that may be associated with the old rule.
 923	 */
 924	if (act_tcam->state != FBNIC_TCAM_S_DISABLED)
 925		return -EBUSY;
 926
 927	flow_type = fsp->flow_type & ~(FLOW_EXT | FLOW_RSS);
 928	hash_idx = fbnic_get_rss_hash_idx(flow_type);
 929
 930	switch (flow_type) {
 931	case UDP_V4_FLOW:
 932udp4_flow:
 933		flow_value |= FBNIC_RPC_TCAM_ACT1_L4_IS_UDP;
 934		fallthrough;
 935	case TCP_V4_FLOW:
 936tcp4_flow:
 937		flow_value |= FBNIC_RPC_TCAM_ACT1_L4_VALID;
 938		flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_L4_IS_UDP |
 939			       FBNIC_RPC_TCAM_ACT1_L4_VALID);
 940
 941		sport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.psrc);
 942		sport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip4_spec.psrc);
 943		dport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.pdst);
 944		dport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip4_spec.pdst);
 945		goto ip4_flow;
 946	case IP_USER_FLOW:
 947		if (!fsp->m_u.usr_ip4_spec.proto)
 948			goto ip4_flow;
 949		if (fsp->m_u.usr_ip4_spec.proto != 0xff)
 950			return -EINVAL;
 951		if (fsp->h_u.usr_ip4_spec.proto == IPPROTO_UDP)
 952			goto udp4_flow;
 953		if (fsp->h_u.usr_ip4_spec.proto == IPPROTO_TCP)
 954			goto tcp4_flow;
 955		return -EINVAL;
 956ip4_flow:
 957		addr4 = (struct in_addr *)&fsp->h_u.usr_ip4_spec.ip4src;
 958		mask4 = (struct in_addr *)&fsp->m_u.usr_ip4_spec.ip4src;
 959		if (mask4->s_addr) {
 960			ip_src = __fbnic_ip4_sync(fbd, fbd->ip_src,
 961						  addr4, mask4);
 962			if (!ip_src)
 963				return -ENOSPC;
 964
 965			set_bit(idx, ip_src->act_tcam);
 966			ip_value |= FBNIC_RPC_TCAM_ACT0_IPSRC_VALID |
 967				    FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX,
 968					       ip_src - fbd->ip_src);
 969			ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPSRC_VALID |
 970				     FBNIC_RPC_TCAM_ACT0_IPSRC_IDX);
 971		}
 972
 973		addr4 = (struct in_addr *)&fsp->h_u.usr_ip4_spec.ip4dst;
 974		mask4 = (struct in_addr *)&fsp->m_u.usr_ip4_spec.ip4dst;
 975		if (mask4->s_addr) {
 976			ip_dst = __fbnic_ip4_sync(fbd, fbd->ip_dst,
 977						  addr4, mask4);
 978			if (!ip_dst) {
 979				if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD)
 980					memset(ip_src, 0, sizeof(*ip_src));
 981				return -ENOSPC;
 982			}
 983
 984			set_bit(idx, ip_dst->act_tcam);
 985			ip_value |= FBNIC_RPC_TCAM_ACT0_IPDST_VALID |
 986				    FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPDST_IDX,
 987					       ip_dst - fbd->ip_dst);
 988			ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPDST_VALID |
 989				     FBNIC_RPC_TCAM_ACT0_IPDST_IDX);
 990		}
 991		flow_value |= FBNIC_RPC_TCAM_ACT1_IP_VALID |
 992			      FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID;
 993		flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_IP_IS_V6 |
 994			       FBNIC_RPC_TCAM_ACT1_IP_VALID |
 995			       FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID);
 996		break;
 997	case UDP_V6_FLOW:
 998udp6_flow:
 999		flow_value |= FBNIC_RPC_TCAM_ACT1_L4_IS_UDP;
1000		fallthrough;
1001	case TCP_V6_FLOW:
1002tcp6_flow:
1003		flow_value |= FBNIC_RPC_TCAM_ACT1_L4_VALID;
1004		flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_L4_IS_UDP |
1005			  FBNIC_RPC_TCAM_ACT1_L4_VALID);
1006
1007		sport = be16_to_cpu(fsp->h_u.tcp_ip6_spec.psrc);
1008		sport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip6_spec.psrc);
1009		dport = be16_to_cpu(fsp->h_u.tcp_ip6_spec.pdst);
1010		dport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip6_spec.pdst);
1011		goto ipv6_flow;
1012	case IPV6_USER_FLOW:
1013		if (!fsp->m_u.usr_ip6_spec.l4_proto)
1014			goto ipv6_flow;
1015
1016		if (fsp->m_u.usr_ip6_spec.l4_proto != 0xff)
1017			return -EINVAL;
1018		if (fsp->h_u.usr_ip6_spec.l4_proto == IPPROTO_UDP)
1019			goto udp6_flow;
1020		if (fsp->h_u.usr_ip6_spec.l4_proto == IPPROTO_TCP)
1021			goto tcp6_flow;
1022		if (fsp->h_u.usr_ip6_spec.l4_proto != IPPROTO_IPV6)
1023			return -EINVAL;
1024
1025		addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6src;
1026		mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6src;
1027		if (!ipv6_addr_any(mask6)) {
1028			ip_src = __fbnic_ip6_sync(fbd, fbd->ipo_src,
1029						  addr6, mask6);
1030			if (!ip_src)
1031				return -ENOSPC;
1032
1033			set_bit(idx, ip_src->act_tcam);
1034			ip_value |=
1035				FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID |
1036				FIELD_PREP(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX,
1037					   ip_src - fbd->ipo_src);
1038			ip_mask &=
1039				~(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID |
1040				  FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX);
1041		}
1042
1043		addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6dst;
1044		mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6dst;
1045		if (!ipv6_addr_any(mask6)) {
1046			ip_dst = __fbnic_ip6_sync(fbd, fbd->ipo_dst,
1047						  addr6, mask6);
1048			if (!ip_dst) {
1049				if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD)
1050					memset(ip_src, 0, sizeof(*ip_src));
1051				return -ENOSPC;
1052			}
1053
1054			set_bit(idx, ip_dst->act_tcam);
1055			ip_value |=
1056				FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID |
1057				FIELD_PREP(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX,
1058					   ip_dst - fbd->ipo_dst);
1059			ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID |
1060				     FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX);
1061		}
1062
1063		flow_value |= FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID;
1064		flow_mask &= FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID;
1065ipv6_flow:
1066		addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6src;
1067		mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6src;
1068		if (!ip_src && !ipv6_addr_any(mask6)) {
1069			ip_src = __fbnic_ip6_sync(fbd, fbd->ip_src,
1070						  addr6, mask6);
1071			if (!ip_src)
1072				return -ENOSPC;
1073
1074			set_bit(idx, ip_src->act_tcam);
1075			ip_value |= FBNIC_RPC_TCAM_ACT0_IPSRC_VALID |
1076				    FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX,
1077					       ip_src - fbd->ip_src);
1078			ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPSRC_VALID |
1079				       FBNIC_RPC_TCAM_ACT0_IPSRC_IDX);
1080		}
1081
1082		addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6dst;
1083		mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6dst;
1084		if (!ip_dst && !ipv6_addr_any(mask6)) {
1085			ip_dst = __fbnic_ip6_sync(fbd, fbd->ip_dst,
1086						  addr6, mask6);
1087			if (!ip_dst) {
1088				if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD)
1089					memset(ip_src, 0, sizeof(*ip_src));
1090				return -ENOSPC;
1091			}
1092
1093			set_bit(idx, ip_dst->act_tcam);
1094			ip_value |= FBNIC_RPC_TCAM_ACT0_IPDST_VALID |
1095				    FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPDST_IDX,
1096					       ip_dst - fbd->ip_dst);
1097			ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPDST_VALID |
1098				       FBNIC_RPC_TCAM_ACT0_IPDST_IDX);
1099		}
1100
1101		flow_value |= FBNIC_RPC_TCAM_ACT1_IP_IS_V6 |
1102			      FBNIC_RPC_TCAM_ACT1_IP_VALID |
1103			      FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID;
1104		flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_IP_IS_V6 |
1105			       FBNIC_RPC_TCAM_ACT1_IP_VALID |
1106			       FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID);
1107		break;
1108	case ETHER_FLOW:
1109		if (!is_zero_ether_addr(fsp->m_u.ether_spec.h_dest)) {
1110			u8 *addr = fsp->h_u.ether_spec.h_dest;
1111			u8 *mask = fsp->m_u.ether_spec.h_dest;
1112
1113			/* Do not allow MAC addr of 0 */
1114			if (is_zero_ether_addr(addr))
1115				return -EINVAL;
1116
1117			/* Only support full MAC address to avoid
1118			 * conflicts with other MAC addresses.
1119			 */
1120			if (!is_broadcast_ether_addr(mask))
1121				return -EINVAL;
1122
1123			if (is_multicast_ether_addr(addr))
1124				mac_addr = __fbnic_mc_sync(fbd, addr);
1125			else
1126				mac_addr = __fbnic_uc_sync(fbd, addr);
1127
1128			if (!mac_addr)
1129				return -ENOSPC;
1130
1131			set_bit(idx, mac_addr->act_tcam);
1132			flow_value |=
1133				FIELD_PREP(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX,
1134					   mac_addr - fbd->mac_addr);
1135			flow_mask &= ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX;
1136		}
1137
1138		flow_value |= FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID;
1139		flow_mask &= ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID;
1140		break;
1141	default:
1142		return -EINVAL;
1143	}
1144
1145	dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT,
1146			   FBNIC_RCD_HDR_AL_DMA_HINT_L4);
1147
1148	/* Write action table values */
1149	act_tcam->dest = dest;
1150	act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx);
1151
1152	/* Write IP Match value/mask to action_tcam[0] */
1153	act_tcam->value.tcam[0] = ip_value;
1154	act_tcam->mask.tcam[0] = ip_mask;
1155
1156	/* Write flow type value/mask to action_tcam[1] */
1157	act_tcam->value.tcam[1] = flow_value;
1158	act_tcam->mask.tcam[1] = flow_mask;
1159
1160	/* Write error, DSCP, extra L4 matches to action_tcam[2] */
1161	act_tcam->value.tcam[2] = misc;
1162	act_tcam->mask.tcam[2] = misc_mask;
1163
1164	/* Write source/destination port values */
1165	act_tcam->value.tcam[3] = sport;
1166	act_tcam->mask.tcam[3] = sport_mask;
1167	act_tcam->value.tcam[4] = dport;
1168	act_tcam->mask.tcam[4] = dport_mask;
1169
1170	for (j = 5; j < FBNIC_RPC_TCAM_ACT_WORD_LEN; j++)
1171		act_tcam->mask.tcam[j] = 0xffff;
1172
1173	act_tcam->state = FBNIC_TCAM_S_UPDATE;
1174	fsp->location = location;
1175
1176	if (netif_running(fbn->netdev)) {
1177		fbnic_write_rules(fbd);
1178		if (ip_src || ip_dst)
1179			fbnic_write_ip_addr(fbd);
1180		if (mac_addr)
1181			fbnic_write_macda(fbd);
1182	}
1183
1184	return 0;
1185}
1186
1187static void fbnic_clear_nfc_macda(struct fbnic_net *fbn,
1188				  unsigned int tcam_idx)
1189{
1190	struct fbnic_dev *fbd = fbn->fbd;
1191	int idx;
1192
1193	for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;)
1194		__fbnic_xc_unsync(&fbd->mac_addr[idx], tcam_idx);
1195
1196	/* Write updates to hardware */
1197	if (netif_running(fbn->netdev))
1198		fbnic_write_macda(fbd);
1199}
1200
1201static void fbnic_clear_nfc_ip_addr(struct fbnic_net *fbn,
1202				    unsigned int tcam_idx)
1203{
1204	struct fbnic_dev *fbd = fbn->fbd;
1205	int idx;
1206
1207	for (idx = ARRAY_SIZE(fbd->ip_src); idx--;)
1208		__fbnic_ip_unsync(&fbd->ip_src[idx], tcam_idx);
1209	for (idx = ARRAY_SIZE(fbd->ip_dst); idx--;)
1210		__fbnic_ip_unsync(&fbd->ip_dst[idx], tcam_idx);
1211	for (idx = ARRAY_SIZE(fbd->ipo_src); idx--;)
1212		__fbnic_ip_unsync(&fbd->ipo_src[idx], tcam_idx);
1213	for (idx = ARRAY_SIZE(fbd->ipo_dst); idx--;)
1214		__fbnic_ip_unsync(&fbd->ipo_dst[idx], tcam_idx);
1215
1216	/* Write updates to hardware */
1217	if (netif_running(fbn->netdev))
1218		fbnic_write_ip_addr(fbd);
1219}
1220
1221static int fbnic_set_cls_rule_del(struct fbnic_net *fbn,
1222				  const struct ethtool_rxnfc *cmd)
1223{
1224	struct ethtool_rx_flow_spec *fsp;
1225	struct fbnic_dev *fbd = fbn->fbd;
1226	struct fbnic_act_tcam *act_tcam;
1227	int idx;
1228
1229	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
1230
1231	if (fsp->location >= FBNIC_RPC_ACT_TBL_NFC_ENTRIES)
1232		return -EINVAL;
1233
1234	idx = fsp->location + FBNIC_RPC_ACT_TBL_NFC_OFFSET;
1235	act_tcam = &fbd->act_tcam[idx];
1236
1237	if (act_tcam->state != FBNIC_TCAM_S_VALID)
1238		return -EINVAL;
1239
1240	act_tcam->state = FBNIC_TCAM_S_DELETE;
1241
1242	if ((act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID) &&
1243	    (~act_tcam->mask.tcam[1] & FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX))
1244		fbnic_clear_nfc_macda(fbn, idx);
1245
1246	if ((act_tcam->value.tcam[0] &
1247	     (FBNIC_RPC_TCAM_ACT0_IPSRC_VALID |
1248	      FBNIC_RPC_TCAM_ACT0_IPDST_VALID |
1249	      FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID |
1250	      FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID)) &&
1251	    (~act_tcam->mask.tcam[0] &
1252	     (FBNIC_RPC_TCAM_ACT0_IPSRC_IDX |
1253	      FBNIC_RPC_TCAM_ACT0_IPDST_IDX |
1254	      FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX |
1255	      FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX)))
1256		fbnic_clear_nfc_ip_addr(fbn, idx);
1257
1258	if (netif_running(fbn->netdev))
1259		fbnic_write_rules(fbd);
1260
1261	return 0;
1262}
1263
1264static int fbnic_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
1265{
1266	struct fbnic_net *fbn = netdev_priv(netdev);
1267	int ret = -EOPNOTSUPP;
1268
1269	switch (cmd->cmd) {
1270	case ETHTOOL_SRXCLSRLINS:
1271		ret = fbnic_set_cls_rule_ins(fbn, cmd);
1272		break;
1273	case ETHTOOL_SRXCLSRLDEL:
1274		ret = fbnic_set_cls_rule_del(fbn, cmd);
1275		break;
1276	}
1277
1278	return ret;
1279}
1280
1281static u32 fbnic_get_rxfh_key_size(struct net_device *netdev)
1282{
1283	return FBNIC_RPC_RSS_KEY_BYTE_LEN;
1284}
1285
1286static u32 fbnic_get_rxfh_indir_size(struct net_device *netdev)
1287{
1288	return FBNIC_RPC_RSS_TBL_SIZE;
1289}
1290
1291static int
1292fbnic_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh)
1293{
1294	struct fbnic_net *fbn = netdev_priv(netdev);
1295	unsigned int i;
1296
1297	rxfh->hfunc = ETH_RSS_HASH_TOP;
1298
1299	if (rxfh->key) {
1300		for (i = 0; i < FBNIC_RPC_RSS_KEY_BYTE_LEN; i++) {
1301			u32 rss_key = fbn->rss_key[i / 4] << ((i % 4) * 8);
1302
1303			rxfh->key[i] = rss_key >> 24;
1304		}
1305	}
1306
1307	if (rxfh->indir) {
1308		for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++)
1309			rxfh->indir[i] = fbn->indir_tbl[0][i];
1310	}
1311
1312	return 0;
1313}
1314
1315static unsigned int
1316fbnic_set_indir(struct fbnic_net *fbn, unsigned int idx, const u32 *indir)
1317{
1318	unsigned int i, changes = 0;
1319
1320	for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) {
1321		if (fbn->indir_tbl[idx][i] == indir[i])
1322			continue;
1323
1324		fbn->indir_tbl[idx][i] = indir[i];
1325		changes++;
1326	}
1327
1328	return changes;
1329}
1330
1331static int
1332fbnic_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh,
1333	       struct netlink_ext_ack *extack)
1334{
1335	struct fbnic_net *fbn = netdev_priv(netdev);
1336	unsigned int i, changes = 0;
1337
1338	if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
1339	    rxfh->hfunc != ETH_RSS_HASH_TOP)
1340		return -EINVAL;
1341
1342	if (rxfh->key) {
1343		u32 rss_key = 0;
1344
1345		for (i = FBNIC_RPC_RSS_KEY_BYTE_LEN; i--;) {
1346			rss_key >>= 8;
1347			rss_key |= (u32)(rxfh->key[i]) << 24;
1348
1349			if (i % 4)
1350				continue;
1351
1352			if (fbn->rss_key[i / 4] == rss_key)
1353				continue;
1354
1355			fbn->rss_key[i / 4] = rss_key;
1356			changes++;
1357		}
1358	}
1359
1360	if (rxfh->indir)
1361		changes += fbnic_set_indir(fbn, 0, rxfh->indir);
1362
1363	if (changes && netif_running(netdev))
1364		fbnic_rss_reinit_hw(fbn->fbd, fbn);
1365
1366	return 0;
1367}
1368
1369static int
1370fbnic_get_rss_hash_opts(struct net_device *netdev,
1371			struct ethtool_rxfh_fields *cmd)
1372{
1373	int hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type);
1374	struct fbnic_net *fbn = netdev_priv(netdev);
1375
1376	if (hash_opt_idx < 0)
1377		return -EINVAL;
1378
1379	/* Report options from rss_en table in fbn */
1380	cmd->data = fbn->rss_flow_hash[hash_opt_idx];
1381
1382	return 0;
1383}
1384
1385#define FBNIC_L2_HASH_OPTIONS \
1386	(RXH_L2DA | RXH_DISCARD)
1387#define FBNIC_L3_HASH_OPTIONS \
1388	(FBNIC_L2_HASH_OPTIONS | RXH_IP_SRC | RXH_IP_DST | RXH_IP6_FL)
1389#define FBNIC_L4_HASH_OPTIONS \
1390	(FBNIC_L3_HASH_OPTIONS | RXH_L4_B_0_1 | RXH_L4_B_2_3)
1391
1392static int
1393fbnic_set_rss_hash_opts(struct net_device *netdev,
1394			const struct ethtool_rxfh_fields *cmd,
1395			struct netlink_ext_ack *extack)
1396{
1397	struct fbnic_net *fbn = netdev_priv(netdev);
1398	int hash_opt_idx;
1399
1400	/* Verify the type requested is correct */
1401	hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type);
1402	if (hash_opt_idx < 0)
1403		return -EINVAL;
1404
1405	/* Verify the fields asked for can actually be assigned based on type */
1406	if (cmd->data & ~FBNIC_L4_HASH_OPTIONS ||
1407	    (hash_opt_idx > FBNIC_L4_HASH_OPT &&
1408	     cmd->data & ~FBNIC_L3_HASH_OPTIONS) ||
1409	    (hash_opt_idx > FBNIC_IP_HASH_OPT &&
1410	     cmd->data & ~FBNIC_L2_HASH_OPTIONS))
1411		return -EINVAL;
1412
1413	fbn->rss_flow_hash[hash_opt_idx] = cmd->data;
1414
1415	if (netif_running(fbn->netdev)) {
1416		fbnic_rss_reinit(fbn->fbd, fbn);
1417		fbnic_write_rules(fbn->fbd);
1418	}
1419
1420	return 0;
1421}
1422
1423static int
1424fbnic_modify_rxfh_context(struct net_device *netdev,
1425			  struct ethtool_rxfh_context *ctx,
1426			  const struct ethtool_rxfh_param *rxfh,
1427			  struct netlink_ext_ack *extack)
1428{
1429	struct fbnic_net *fbn = netdev_priv(netdev);
1430	const u32 *indir = rxfh->indir;
1431	unsigned int changes;
1432
1433	if (!indir)
1434		indir = ethtool_rxfh_context_indir(ctx);
1435
1436	changes = fbnic_set_indir(fbn, rxfh->rss_context, indir);
1437	if (changes && netif_running(netdev))
1438		fbnic_rss_reinit_hw(fbn->fbd, fbn);
1439
1440	return 0;
1441}
1442
1443static int
1444fbnic_create_rxfh_context(struct net_device *netdev,
1445			  struct ethtool_rxfh_context *ctx,
1446			  const struct ethtool_rxfh_param *rxfh,
1447			  struct netlink_ext_ack *extack)
1448{
1449	struct fbnic_net *fbn = netdev_priv(netdev);
1450
1451	if (rxfh->hfunc && rxfh->hfunc != ETH_RSS_HASH_TOP) {
1452		NL_SET_ERR_MSG_MOD(extack, "RSS hash function not supported");
1453		return -EOPNOTSUPP;
1454	}
1455	ctx->hfunc = ETH_RSS_HASH_TOP;
1456
1457	if (!rxfh->indir) {
1458		u32 *indir = ethtool_rxfh_context_indir(ctx);
1459		unsigned int num_rx = fbn->num_rx_queues;
1460		unsigned int i;
1461
1462		for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++)
1463			indir[i] = ethtool_rxfh_indir_default(i, num_rx);
1464	}
1465
1466	return fbnic_modify_rxfh_context(netdev, ctx, rxfh, extack);
1467}
1468
1469static int
1470fbnic_remove_rxfh_context(struct net_device *netdev,
1471			  struct ethtool_rxfh_context *ctx, u32 rss_context,
1472			  struct netlink_ext_ack *extack)
1473{
1474	/* Nothing to do, contexts are allocated statically */
1475	return 0;
1476}
1477
1478static void fbnic_get_channels(struct net_device *netdev,
1479			       struct ethtool_channels *ch)
1480{
1481	struct fbnic_net *fbn = netdev_priv(netdev);
1482	struct fbnic_dev *fbd = fbn->fbd;
1483
1484	ch->max_rx = fbd->max_num_queues;
1485	ch->max_tx = fbd->max_num_queues;
1486	ch->max_combined = min(ch->max_rx, ch->max_tx);
1487	ch->max_other =	FBNIC_NON_NAPI_VECTORS;
1488
1489	if (fbn->num_rx_queues > fbn->num_napi ||
1490	    fbn->num_tx_queues > fbn->num_napi)
1491		ch->combined_count = min(fbn->num_rx_queues,
1492					 fbn->num_tx_queues);
1493	else
1494		ch->combined_count =
1495			fbn->num_rx_queues + fbn->num_tx_queues - fbn->num_napi;
1496	ch->rx_count = fbn->num_rx_queues - ch->combined_count;
1497	ch->tx_count = fbn->num_tx_queues - ch->combined_count;
1498	ch->other_count = FBNIC_NON_NAPI_VECTORS;
1499}
1500
1501static void fbnic_set_queues(struct fbnic_net *fbn, struct ethtool_channels *ch,
1502			     unsigned int max_napis)
1503{
1504	fbn->num_rx_queues = ch->rx_count + ch->combined_count;
1505	fbn->num_tx_queues = ch->tx_count + ch->combined_count;
1506	fbn->num_napi = min(ch->rx_count + ch->tx_count + ch->combined_count,
1507			    max_napis);
1508}
1509
1510static int fbnic_set_channels(struct net_device *netdev,
1511			      struct ethtool_channels *ch)
1512{
1513	struct fbnic_net *fbn = netdev_priv(netdev);
1514	unsigned int max_napis, standalone;
1515	struct fbnic_dev *fbd = fbn->fbd;
1516	struct fbnic_net *clone;
1517	int err;
1518
1519	max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS;
1520	standalone = ch->rx_count + ch->tx_count;
1521
1522	/* Limits for standalone queues:
1523	 *  - each queue has its own NAPI (num_napi >= rx + tx + combined)
1524	 *  - combining queues (combined not 0, rx or tx must be 0)
1525	 */
1526	if ((ch->rx_count && ch->tx_count && ch->combined_count) ||
1527	    (standalone && standalone + ch->combined_count > max_napis) ||
1528	    ch->rx_count + ch->combined_count > fbd->max_num_queues ||
1529	    ch->tx_count + ch->combined_count > fbd->max_num_queues ||
1530	    ch->other_count != FBNIC_NON_NAPI_VECTORS)
1531		return -EINVAL;
1532
1533	if (!netif_running(netdev)) {
1534		fbnic_set_queues(fbn, ch, max_napis);
1535		fbnic_reset_indir_tbl(fbn);
1536		return 0;
1537	}
1538
1539	clone = fbnic_clone_create(fbn);
1540	if (!clone)
1541		return -ENOMEM;
1542
1543	fbnic_set_queues(clone, ch, max_napis);
1544
1545	err = fbnic_alloc_napi_vectors(clone);
1546	if (err)
1547		goto err_free_clone;
1548
1549	err = fbnic_alloc_resources(clone);
1550	if (err)
1551		goto err_free_napis;
1552
1553	fbnic_down_noidle(fbn);
1554	err = fbnic_wait_all_queues_idle(fbn->fbd, true);
1555	if (err)
1556		goto err_start_stack;
1557
1558	err = fbnic_set_netif_queues(clone);
1559	if (err)
1560		goto err_start_stack;
1561
1562	/* Nothing can fail past this point */
1563	fbnic_flush(fbn);
1564
1565	fbnic_clone_swap(fbn, clone);
1566
1567	/* Reset RSS indirection table */
1568	fbnic_reset_indir_tbl(fbn);
1569
1570	fbnic_up(fbn);
1571
1572	fbnic_free_resources(clone);
1573	fbnic_free_napi_vectors(clone);
1574	fbnic_clone_free(clone);
1575
1576	return 0;
1577
1578err_start_stack:
1579	fbnic_flush(fbn);
1580	fbnic_up(fbn);
1581	fbnic_free_resources(clone);
1582err_free_napis:
1583	fbnic_free_napi_vectors(clone);
1584err_free_clone:
1585	fbnic_clone_free(clone);
1586	return err;
1587}
1588
1589static int
1590fbnic_get_ts_info(struct net_device *netdev,
1591		  struct kernel_ethtool_ts_info *tsinfo)
1592{
1593	struct fbnic_net *fbn = netdev_priv(netdev);
1594
1595	tsinfo->phc_index = ptp_clock_index(fbn->fbd->ptp);
1596
1597	tsinfo->so_timestamping =
1598		SOF_TIMESTAMPING_TX_SOFTWARE |
1599		SOF_TIMESTAMPING_TX_HARDWARE |
1600		SOF_TIMESTAMPING_RX_HARDWARE |
1601		SOF_TIMESTAMPING_RAW_HARDWARE;
1602
1603	tsinfo->tx_types =
1604		BIT(HWTSTAMP_TX_OFF) |
1605		BIT(HWTSTAMP_TX_ON);
1606
1607	tsinfo->rx_filters =
1608		BIT(HWTSTAMP_FILTER_NONE) |
1609		BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
1610		BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) |
1611		BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
1612		BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
1613		BIT(HWTSTAMP_FILTER_ALL);
1614
1615	return 0;
1616}
1617
1618static void fbnic_get_ts_stats(struct net_device *netdev,
1619			       struct ethtool_ts_stats *ts_stats)
1620{
1621	struct fbnic_net *fbn = netdev_priv(netdev);
1622	u64 ts_packets, ts_lost;
1623	struct fbnic_ring *ring;
1624	unsigned int start;
1625	int i;
1626
1627	ts_stats->pkts = fbn->tx_stats.twq.ts_packets;
1628	ts_stats->lost = fbn->tx_stats.twq.ts_lost;
1629	for (i = 0; i < fbn->num_tx_queues; i++) {
1630		ring = fbn->tx[i];
1631		do {
1632			start = u64_stats_fetch_begin(&ring->stats.syncp);
1633			ts_packets = ring->stats.twq.ts_packets;
1634			ts_lost = ring->stats.twq.ts_lost;
1635		} while (u64_stats_fetch_retry(&ring->stats.syncp, start));
1636		ts_stats->pkts += ts_packets;
1637		ts_stats->lost += ts_lost;
1638	}
1639}
1640
1641static int
1642fbnic_get_module_eeprom_by_page(struct net_device *netdev,
1643				const struct ethtool_module_eeprom *page_data,
1644				struct netlink_ext_ack *extack)
1645{
1646	struct fbnic_net *fbn = netdev_priv(netdev);
1647	struct fbnic_fw_completion *fw_cmpl;
1648	struct fbnic_dev *fbd = fbn->fbd;
1649	int err;
1650
1651	if (page_data->i2c_address != 0x50) {
1652		NL_SET_ERR_MSG_MOD(extack,
1653				   "Invalid i2c address. Only 0x50 is supported");
1654		return -EINVAL;
1655	}
1656
1657	fw_cmpl = __fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_QSFP_READ_RESP,
1658					page_data->length);
1659	if (!fw_cmpl)
1660		return -ENOMEM;
1661
1662	/* Initialize completion and queue it for FW to process */
1663	fw_cmpl->u.qsfp.length = page_data->length;
1664	fw_cmpl->u.qsfp.offset = page_data->offset;
1665	fw_cmpl->u.qsfp.page = page_data->page;
1666	fw_cmpl->u.qsfp.bank = page_data->bank;
1667
1668	err = fbnic_fw_xmit_qsfp_read_msg(fbd, fw_cmpl, page_data->page,
1669					  page_data->bank, page_data->offset,
1670					  page_data->length);
1671	if (err) {
1672		NL_SET_ERR_MSG_MOD(extack,
1673				   "Failed to transmit EEPROM read request");
1674		goto exit_free;
1675	}
1676
1677	if (!wait_for_completion_timeout(&fw_cmpl->done, 2 * HZ)) {
1678		err = -ETIMEDOUT;
1679		NL_SET_ERR_MSG_MOD(extack,
1680				   "Timed out waiting for firmware response");
1681		goto exit_cleanup;
1682	}
1683
1684	if (fw_cmpl->result) {
1685		err = fw_cmpl->result;
1686		NL_SET_ERR_MSG_MOD(extack, "Failed to read EEPROM");
1687		goto exit_cleanup;
1688	}
1689
1690	memcpy(page_data->data, fw_cmpl->u.qsfp.data, page_data->length);
1691
1692exit_cleanup:
1693	fbnic_mbx_clear_cmpl(fbd, fw_cmpl);
1694exit_free:
1695	fbnic_fw_put_cmpl(fw_cmpl);
1696
1697	return err ? : page_data->length;
1698}
1699
1700static void fbnic_set_counter(u64 *stat, struct fbnic_stat_counter *counter)
1701{
1702	if (counter->reported)
1703		*stat = counter->value;
1704}
1705
1706static void
1707fbnic_get_pause_stats(struct net_device *netdev,
1708		      struct ethtool_pause_stats *pause_stats)
1709{
1710	struct fbnic_net *fbn = netdev_priv(netdev);
1711	struct fbnic_mac_stats *mac_stats;
1712	struct fbnic_dev *fbd = fbn->fbd;
1713
1714	mac_stats = &fbd->hw_stats.mac;
1715
1716	fbd->mac->get_pause_stats(fbd, false, &mac_stats->pause);
1717
1718	pause_stats->tx_pause_frames = mac_stats->pause.tx_pause_frames.value;
1719	pause_stats->rx_pause_frames = mac_stats->pause.rx_pause_frames.value;
1720}
1721
1722static void
1723fbnic_get_fec_stats(struct net_device *netdev,
1724		    struct ethtool_fec_stats *fec_stats,
1725		    struct ethtool_fec_hist *hist)
1726{
1727	struct fbnic_net *fbn = netdev_priv(netdev);
1728	struct fbnic_phy_stats *phy_stats;
1729	struct fbnic_dev *fbd = fbn->fbd;
1730
1731	fbnic_get_hw_stats32(fbd);
1732	phy_stats = &fbd->hw_stats.phy;
1733
1734	spin_lock(&fbd->hw_stats.lock);
1735	fec_stats->corrected_blocks.total =
1736		phy_stats->fec.corrected_blocks.value;
1737	fec_stats->uncorrectable_blocks.total =
1738		phy_stats->fec.uncorrectable_blocks.value;
1739	spin_unlock(&fbd->hw_stats.lock);
1740}
1741
1742static void
1743fbnic_get_eth_phy_stats(struct net_device *netdev,
1744			struct ethtool_eth_phy_stats *eth_phy_stats)
1745{
1746	struct fbnic_net *fbn = netdev_priv(netdev);
1747	struct fbnic_phy_stats *phy_stats;
1748	struct fbnic_dev *fbd = fbn->fbd;
1749	u64 total = 0;
1750	int i;
1751
1752	fbnic_get_hw_stats32(fbd);
1753	phy_stats = &fbd->hw_stats.phy;
1754
1755	spin_lock(&fbd->hw_stats.lock);
1756	for (i = 0; i < FBNIC_PCS_MAX_LANES; i++)
1757		total += phy_stats->pcs.SymbolErrorDuringCarrier.lanes[i].value;
1758
1759	eth_phy_stats->SymbolErrorDuringCarrier = total;
1760	spin_unlock(&fbd->hw_stats.lock);
1761}
1762
1763static void
1764fbnic_get_eth_mac_stats(struct net_device *netdev,
1765			struct ethtool_eth_mac_stats *eth_mac_stats)
1766{
1767	struct fbnic_net *fbn = netdev_priv(netdev);
1768	struct fbnic_mac_stats *mac_stats;
1769	struct fbnic_dev *fbd = fbn->fbd;
1770	const struct fbnic_mac *mac;
1771
1772	mac_stats = &fbd->hw_stats.mac;
1773	mac = fbd->mac;
1774
1775	mac->get_eth_mac_stats(fbd, false, &mac_stats->eth_mac);
1776
1777	fbnic_set_counter(&eth_mac_stats->FramesTransmittedOK,
1778			  &mac_stats->eth_mac.FramesTransmittedOK);
1779	fbnic_set_counter(&eth_mac_stats->FramesReceivedOK,
1780			  &mac_stats->eth_mac.FramesReceivedOK);
1781	fbnic_set_counter(&eth_mac_stats->FrameCheckSequenceErrors,
1782			  &mac_stats->eth_mac.FrameCheckSequenceErrors);
1783	fbnic_set_counter(&eth_mac_stats->AlignmentErrors,
1784			  &mac_stats->eth_mac.AlignmentErrors);
1785	fbnic_set_counter(&eth_mac_stats->OctetsTransmittedOK,
1786			  &mac_stats->eth_mac.OctetsTransmittedOK);
1787	fbnic_set_counter(&eth_mac_stats->FramesLostDueToIntMACXmitError,
1788			  &mac_stats->eth_mac.FramesLostDueToIntMACXmitError);
1789	fbnic_set_counter(&eth_mac_stats->OctetsReceivedOK,
1790			  &mac_stats->eth_mac.OctetsReceivedOK);
1791	fbnic_set_counter(&eth_mac_stats->FramesLostDueToIntMACRcvError,
1792			  &mac_stats->eth_mac.FramesLostDueToIntMACRcvError);
1793	fbnic_set_counter(&eth_mac_stats->MulticastFramesXmittedOK,
1794			  &mac_stats->eth_mac.MulticastFramesXmittedOK);
1795	fbnic_set_counter(&eth_mac_stats->BroadcastFramesXmittedOK,
1796			  &mac_stats->eth_mac.BroadcastFramesXmittedOK);
1797	fbnic_set_counter(&eth_mac_stats->MulticastFramesReceivedOK,
1798			  &mac_stats->eth_mac.MulticastFramesReceivedOK);
1799	fbnic_set_counter(&eth_mac_stats->BroadcastFramesReceivedOK,
1800			  &mac_stats->eth_mac.BroadcastFramesReceivedOK);
1801	fbnic_set_counter(&eth_mac_stats->FrameTooLongErrors,
1802			  &mac_stats->eth_mac.FrameTooLongErrors);
1803}
1804
1805static void
1806fbnic_get_eth_ctrl_stats(struct net_device *netdev,
1807			 struct ethtool_eth_ctrl_stats *eth_ctrl_stats)
1808{
1809	struct fbnic_net *fbn = netdev_priv(netdev);
1810	struct fbnic_mac_stats *mac_stats;
1811	struct fbnic_dev *fbd = fbn->fbd;
1812
1813	mac_stats = &fbd->hw_stats.mac;
1814
1815	fbd->mac->get_eth_ctrl_stats(fbd, false, &mac_stats->eth_ctrl);
1816
1817	eth_ctrl_stats->MACControlFramesReceived =
1818		mac_stats->eth_ctrl.MACControlFramesReceived.value;
1819	eth_ctrl_stats->MACControlFramesTransmitted =
1820		mac_stats->eth_ctrl.MACControlFramesTransmitted.value;
1821}
1822
1823static const struct ethtool_rmon_hist_range fbnic_rmon_ranges[] = {
1824	{    0,   64 },
1825	{   65,  127 },
1826	{  128,  255 },
1827	{  256,  511 },
1828	{  512, 1023 },
1829	{ 1024, 1518 },
1830	{ 1519, 2047 },
1831	{ 2048, 4095 },
1832	{ 4096, 8191 },
1833	{ 8192, 9216 },
1834	{ 9217, FBNIC_MAX_JUMBO_FRAME_SIZE },
1835	{}
1836};
1837
1838static void
1839fbnic_get_rmon_stats(struct net_device *netdev,
1840		     struct ethtool_rmon_stats *rmon_stats,
1841		     const struct ethtool_rmon_hist_range **ranges)
1842{
1843	struct fbnic_net *fbn = netdev_priv(netdev);
1844	struct fbnic_mac_stats *mac_stats;
1845	struct fbnic_dev *fbd = fbn->fbd;
1846	int i;
1847
1848	mac_stats = &fbd->hw_stats.mac;
1849
1850	fbd->mac->get_rmon_stats(fbd, false, &mac_stats->rmon);
1851
1852	rmon_stats->undersize_pkts =
1853		mac_stats->rmon.undersize_pkts.value;
1854	rmon_stats->oversize_pkts =
1855		mac_stats->rmon.oversize_pkts.value;
1856	rmon_stats->fragments =
1857		mac_stats->rmon.fragments.value;
1858	rmon_stats->jabbers =
1859		mac_stats->rmon.jabbers.value;
1860
1861	for (i = 0; fbnic_rmon_ranges[i].high; i++) {
1862		rmon_stats->hist[i] = mac_stats->rmon.hist[i].value;
1863		rmon_stats->hist_tx[i] = mac_stats->rmon.hist_tx[i].value;
1864	}
1865
1866	*ranges = fbnic_rmon_ranges;
1867}
1868
1869static void fbnic_get_link_ext_stats(struct net_device *netdev,
1870				     struct ethtool_link_ext_stats *stats)
1871{
1872	struct fbnic_net *fbn = netdev_priv(netdev);
1873
1874	stats->link_down_events = fbn->link_down_events;
1875}
1876
1877static const struct ethtool_ops fbnic_ethtool_ops = {
1878	.cap_link_lanes_supported	= true,
1879	.supported_coalesce_params	= ETHTOOL_COALESCE_USECS |
1880					  ETHTOOL_COALESCE_RX_MAX_FRAMES,
1881	.supported_ring_params		= ETHTOOL_RING_USE_TCP_DATA_SPLIT |
1882					  ETHTOOL_RING_USE_HDS_THRS,
1883	.rxfh_max_num_contexts		= FBNIC_RPC_RSS_TBL_COUNT,
1884	.get_drvinfo			= fbnic_get_drvinfo,
1885	.get_regs_len			= fbnic_get_regs_len,
1886	.get_regs			= fbnic_get_regs,
1887	.get_link			= ethtool_op_get_link,
1888	.get_link_ext_stats		= fbnic_get_link_ext_stats,
1889	.get_coalesce			= fbnic_get_coalesce,
1890	.set_coalesce			= fbnic_set_coalesce,
1891	.get_ringparam			= fbnic_get_ringparam,
1892	.set_ringparam			= fbnic_set_ringparam,
1893	.get_pause_stats		= fbnic_get_pause_stats,
1894	.get_pauseparam			= fbnic_phylink_get_pauseparam,
1895	.set_pauseparam			= fbnic_phylink_set_pauseparam,
1896	.get_strings			= fbnic_get_strings,
1897	.get_ethtool_stats		= fbnic_get_ethtool_stats,
1898	.get_sset_count			= fbnic_get_sset_count,
1899	.get_rxnfc			= fbnic_get_rxnfc,
1900	.set_rxnfc			= fbnic_set_rxnfc,
1901	.get_rxfh_key_size		= fbnic_get_rxfh_key_size,
1902	.get_rxfh_indir_size		= fbnic_get_rxfh_indir_size,
1903	.get_rxfh			= fbnic_get_rxfh,
1904	.set_rxfh			= fbnic_set_rxfh,
1905	.get_rxfh_fields		= fbnic_get_rss_hash_opts,
1906	.set_rxfh_fields		= fbnic_set_rss_hash_opts,
1907	.create_rxfh_context		= fbnic_create_rxfh_context,
1908	.modify_rxfh_context		= fbnic_modify_rxfh_context,
1909	.remove_rxfh_context		= fbnic_remove_rxfh_context,
1910	.get_channels			= fbnic_get_channels,
1911	.set_channels			= fbnic_set_channels,
1912	.get_ts_info			= fbnic_get_ts_info,
1913	.get_ts_stats			= fbnic_get_ts_stats,
1914	.get_link_ksettings		= fbnic_phylink_ethtool_ksettings_get,
1915	.get_fec_stats			= fbnic_get_fec_stats,
1916	.get_fecparam			= fbnic_phylink_get_fecparam,
1917	.get_module_eeprom_by_page	= fbnic_get_module_eeprom_by_page,
1918	.get_eth_phy_stats		= fbnic_get_eth_phy_stats,
1919	.get_eth_mac_stats		= fbnic_get_eth_mac_stats,
1920	.get_eth_ctrl_stats		= fbnic_get_eth_ctrl_stats,
1921	.get_rmon_stats			= fbnic_get_rmon_stats,
1922};
1923
1924void fbnic_set_ethtool_ops(struct net_device *dev)
1925{
1926	dev->ethtool_ops = &fbnic_ethtool_ops;
1927}