fbnic_netdev.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) Meta Platforms, Inc. and affiliates. */
  3
  4#include <linux/etherdevice.h>
  5#include <linux/ipv6.h>
  6#include <linux/types.h>
  7#include <net/netdev_queues.h>
  8
  9#include "fbnic.h"
 10#include "fbnic_netdev.h"
 11#include "fbnic_txrx.h"
 12
 13int __fbnic_open(struct fbnic_net *fbn)
 14{
 15	struct fbnic_dev *fbd = fbn->fbd;
 16	int err;
 17
 18	err = fbnic_alloc_napi_vectors(fbn);
 19	if (err)
 20		return err;
 21
 22	err = fbnic_alloc_resources(fbn);
 23	if (err)
 24		goto free_napi_vectors;
 25
 26	err = fbnic_set_netif_queues(fbn);
 27	if (err)
 28		goto free_resources;
 29
 30	/* Send ownership message and flush to verify FW has seen it */
 31	err = fbnic_fw_xmit_ownership_msg(fbd, true);
 32	if (err) {
 33		dev_warn(fbd->dev,
 34			 "Error %d sending host ownership message to the firmware\n",
 35			 err);
 36		goto err_reset_queues;
 37	}
 38
 39	err = fbnic_time_start(fbn);
 40	if (err)
 41		goto release_ownership;
 42
 43	err = fbnic_fw_init_heartbeat(fbd, false);
 44	if (err)
 45		goto time_stop;
 46
 47	err = fbnic_mac_request_irq(fbd);
 48	if (err)
 49		goto time_stop;
 50
 51	/* Pull the BMC config and initialize the RPC */
 52	fbnic_bmc_rpc_init(fbd);
 53	fbnic_rss_reinit(fbd, fbn);
 54
 55	phylink_resume(fbn->phylink);
 56
 57	return 0;
 58time_stop:
 59	fbnic_time_stop(fbn);
 60release_ownership:
 61	fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
 62err_reset_queues:
 63	fbnic_reset_netif_queues(fbn);
 64free_resources:
 65	fbnic_free_resources(fbn);
 66free_napi_vectors:
 67	fbnic_free_napi_vectors(fbn);
 68	return err;
 69}
 70
 71static int fbnic_open(struct net_device *netdev)
 72{
 73	struct fbnic_net *fbn = netdev_priv(netdev);
 74	int err;
 75
 76	fbnic_napi_name_irqs(fbn->fbd);
 77
 78	err = __fbnic_open(fbn);
 79	if (!err)
 80		fbnic_up(fbn);
 81
 82	return err;
 83}
 84
 85static int fbnic_stop(struct net_device *netdev)
 86{
 87	struct fbnic_net *fbn = netdev_priv(netdev);
 88
 89	fbnic_mac_free_irq(fbn->fbd);
 90	phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd));
 91
 92	fbnic_down(fbn);
 93
 94	fbnic_time_stop(fbn);
 95	fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
 96
 97	fbnic_reset_netif_queues(fbn);
 98	fbnic_free_resources(fbn);
 99	fbnic_free_napi_vectors(fbn);
100
101	return 0;
102}
103
104static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr)
105{
106	struct fbnic_net *fbn = netdev_priv(netdev);
107	struct fbnic_mac_addr *avail_addr;
108
109	if (WARN_ON(!is_valid_ether_addr(addr)))
110		return -EADDRNOTAVAIL;
111
112	avail_addr = __fbnic_uc_sync(fbn->fbd, addr);
113	if (!avail_addr)
114		return -ENOSPC;
115
116	/* Add type flag indicating this address is in use by the host */
117	set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam);
118
119	return 0;
120}
121
122static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr)
123{
124	struct fbnic_net *fbn = netdev_priv(netdev);
125	struct fbnic_dev *fbd = fbn->fbd;
126	int i, ret;
127
128	/* Scan from middle of list to bottom, filling bottom up.
129	 * Skip the first entry which is reserved for dev_addr and
130	 * leave the last entry to use for promiscuous filtering.
131	 */
132	for (i = fbd->mac_addr_boundary, ret = -ENOENT;
133	     i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) {
134		struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];
135
136		if (!ether_addr_equal(mac_addr->value.addr8, addr))
137			continue;
138
139		ret = __fbnic_uc_unsync(mac_addr);
140	}
141
142	return ret;
143}
144
145static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr)
146{
147	struct fbnic_net *fbn = netdev_priv(netdev);
148	struct fbnic_mac_addr *avail_addr;
149
150	if (WARN_ON(!is_multicast_ether_addr(addr)))
151		return -EADDRNOTAVAIL;
152
153	avail_addr = __fbnic_mc_sync(fbn->fbd, addr);
154	if (!avail_addr)
155		return -ENOSPC;
156
157	/* Add type flag indicating this address is in use by the host */
158	set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam);
159
160	return 0;
161}
162
163static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr)
164{
165	struct fbnic_net *fbn = netdev_priv(netdev);
166	struct fbnic_dev *fbd = fbn->fbd;
167	int i, ret;
168
169	/* Scan from middle of list to top, filling top down.
170	 * Skip over the address reserved for the BMC MAC and
171	 * exclude index 0 as that belongs to the broadcast address
172	 */
173	for (i = fbd->mac_addr_boundary, ret = -ENOENT;
174	     --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) {
175		struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];
176
177		if (!ether_addr_equal(mac_addr->value.addr8, addr))
178			continue;
179
180		ret = __fbnic_mc_unsync(mac_addr);
181	}
182
183	return ret;
184}
185
186void __fbnic_set_rx_mode(struct fbnic_dev *fbd)
187{
188	bool uc_promisc = false, mc_promisc = false;
189	struct net_device *netdev = fbd->netdev;
190	struct fbnic_mac_addr *mac_addr;
191	int err;
192
193	/* Populate host address from dev_addr */
194	mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX];
195	if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) ||
196	    mac_addr->state != FBNIC_TCAM_S_VALID) {
197		ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr);
198		mac_addr->state = FBNIC_TCAM_S_UPDATE;
199		set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam);
200	}
201
202	/* Populate broadcast address if broadcast is enabled */
203	mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX];
204	if (netdev->flags & IFF_BROADCAST) {
205		if (!is_broadcast_ether_addr(mac_addr->value.addr8) ||
206		    mac_addr->state != FBNIC_TCAM_S_VALID) {
207			eth_broadcast_addr(mac_addr->value.addr8);
208			mac_addr->state = FBNIC_TCAM_S_ADD;
209		}
210		set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam);
211	} else if (mac_addr->state == FBNIC_TCAM_S_VALID) {
212		__fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST);
213	}
214
215	/* Synchronize unicast and multicast address lists */
216	err = __dev_uc_sync(netdev, fbnic_uc_sync, fbnic_uc_unsync);
217	if (err == -ENOSPC)
218		uc_promisc = true;
219	err = __dev_mc_sync(netdev, fbnic_mc_sync, fbnic_mc_unsync);
220	if (err == -ENOSPC)
221		mc_promisc = true;
222
223	uc_promisc |= !!(netdev->flags & IFF_PROMISC);
224	mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc;
225
226	/* Update the promiscuous rules */
227	fbnic_promisc_sync(fbd, uc_promisc, mc_promisc);
228
229	/* Add rules for BMC all multicast if it is enabled */
230	fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc);
231
232	/* Sift out any unshared BMC rules and place them in BMC only section */
233	fbnic_sift_macda(fbd);
234
235	/* Write updates to hardware */
236	fbnic_write_rules(fbd);
237	fbnic_write_macda(fbd);
238	fbnic_write_tce_tcam(fbd);
239}
240
241static void fbnic_set_rx_mode(struct net_device *netdev)
242{
243	struct fbnic_net *fbn = netdev_priv(netdev);
244	struct fbnic_dev *fbd = fbn->fbd;
245
246	/* No need to update the hardware if we are not running */
247	if (netif_running(netdev))
248		__fbnic_set_rx_mode(fbd);
249}
250
251static int fbnic_set_mac(struct net_device *netdev, void *p)
252{
253	struct sockaddr *addr = p;
254
255	if (!is_valid_ether_addr(addr->sa_data))
256		return -EADDRNOTAVAIL;
257
258	eth_hw_addr_set(netdev, addr->sa_data);
259
260	fbnic_set_rx_mode(netdev);
261
262	return 0;
263}
264
265static int fbnic_change_mtu(struct net_device *dev, int new_mtu)
266{
267	struct fbnic_net *fbn = netdev_priv(dev);
268
269	if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) {
270		dev_err(&dev->dev,
271			"MTU %d is larger than HDS threshold %d in XDP mode\n",
272			new_mtu, fbn->hds_thresh);
273
274		return -EINVAL;
275	}
276
277	WRITE_ONCE(dev->mtu, new_mtu);
278
279	return 0;
280}
281
282void fbnic_clear_rx_mode(struct fbnic_dev *fbd)
283{
284	struct net_device *netdev = fbd->netdev;
285	int idx;
286
287	for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) {
288		struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx];
289
290		if (mac_addr->state != FBNIC_TCAM_S_VALID)
291			continue;
292
293		bitmap_clear(mac_addr->act_tcam,
294			     FBNIC_MAC_ADDR_T_HOST_START,
295			     FBNIC_MAC_ADDR_T_HOST_LEN);
296
297		if (bitmap_empty(mac_addr->act_tcam,
298				 FBNIC_RPC_TCAM_ACT_NUM_ENTRIES))
299			mac_addr->state = FBNIC_TCAM_S_DELETE;
300	}
301
302	/* Write updates to hardware */
303	fbnic_write_macda(fbd);
304
305	__dev_uc_unsync(netdev, NULL);
306	__dev_mc_unsync(netdev, NULL);
307}
308
309static int fbnic_hwtstamp_get(struct net_device *netdev,
310			      struct kernel_hwtstamp_config *config)
311{
312	struct fbnic_net *fbn = netdev_priv(netdev);
313
314	*config = fbn->hwtstamp_config;
315
316	return 0;
317}
318
319static int fbnic_hwtstamp_set(struct net_device *netdev,
320			      struct kernel_hwtstamp_config *config,
321			      struct netlink_ext_ack *extack)
322{
323	struct fbnic_net *fbn = netdev_priv(netdev);
324	int old_rx_filter;
325
326	if (config->source != HWTSTAMP_SOURCE_NETDEV)
327		return -EOPNOTSUPP;
328
329	if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config))
330		return 0;
331
332	/* Upscale the filters */
333	switch (config->rx_filter) {
334	case HWTSTAMP_FILTER_NONE:
335	case HWTSTAMP_FILTER_ALL:
336	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
337	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
338	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
339	case HWTSTAMP_FILTER_PTP_V2_EVENT:
340		break;
341	case HWTSTAMP_FILTER_NTP_ALL:
342		config->rx_filter = HWTSTAMP_FILTER_ALL;
343		break;
344	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
345	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
346		config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
347		break;
348	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
349	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
350		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
351		break;
352	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
353	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
354		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
355		break;
356	case HWTSTAMP_FILTER_PTP_V2_SYNC:
357	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
358		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
359		break;
360	default:
361		return -ERANGE;
362	}
363
364	/* Configure */
365	old_rx_filter = fbn->hwtstamp_config.rx_filter;
366	memcpy(&fbn->hwtstamp_config, config, sizeof(*config));
367
368	if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) {
369		fbnic_rss_reinit(fbn->fbd, fbn);
370		fbnic_write_rules(fbn->fbd);
371	}
372
373	/* Save / report back filter configuration
374	 * Note that our filter configuration is inexact. Instead of
375	 * filtering for a specific UDP port or L2 Ethertype we are
376	 * filtering in all UDP or all non-IP packets for timestamping. So
377	 * if anything other than FILTER_ALL is requested we report
378	 * FILTER_SOME indicating that we will be timestamping a few
379	 * additional packets.
380	 */
381	if (config->rx_filter > HWTSTAMP_FILTER_ALL)
382		config->rx_filter = HWTSTAMP_FILTER_SOME;
383
384	return 0;
385}
386
387static void fbnic_get_stats64(struct net_device *dev,
388			      struct rtnl_link_stats64 *stats64)
389{
390	u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0;
391	u64 rx_over = 0, rx_missed = 0, rx_length = 0;
392	u64 tx_bytes, tx_packets, tx_dropped = 0;
393	struct fbnic_net *fbn = netdev_priv(dev);
394	struct fbnic_dev *fbd = fbn->fbd;
395	struct fbnic_queue_stats *stats;
396
397	unsigned int start, i;
398
399	fbnic_get_hw_stats(fbd);
400
401	stats = &fbn->tx_stats;
402
403	tx_bytes = stats->bytes;
404	tx_packets = stats->packets;
405	tx_dropped = stats->dropped;
406
407	/* Record drops from Tx HW Datapath */
408	spin_lock(&fbd->hw_stats.lock);
409	tx_dropped += fbd->hw_stats.tmi.drop.frames.value +
410		      fbd->hw_stats.tti.cm_drop.frames.value +
411		      fbd->hw_stats.tti.frame_drop.frames.value +
412		      fbd->hw_stats.tti.tbi_drop.frames.value;
413	spin_unlock(&fbd->hw_stats.lock);
414
415	stats64->tx_bytes = tx_bytes;
416	stats64->tx_packets = tx_packets;
417	stats64->tx_dropped = tx_dropped;
418
419	for (i = 0; i < fbn->num_tx_queues; i++) {
420		struct fbnic_ring *txr = fbn->tx[i];
421
422		if (!txr)
423			continue;
424
425		stats = &txr->stats;
426		do {
427			start = u64_stats_fetch_begin(&stats->syncp);
428			tx_bytes = stats->bytes;
429			tx_packets = stats->packets;
430			tx_dropped = stats->dropped;
431		} while (u64_stats_fetch_retry(&stats->syncp, start));
432
433		stats64->tx_bytes += tx_bytes;
434		stats64->tx_packets += tx_packets;
435		stats64->tx_dropped += tx_dropped;
436	}
437
438	stats = &fbn->rx_stats;
439
440	rx_bytes = stats->bytes;
441	rx_packets = stats->packets;
442	rx_dropped = stats->dropped;
443
444	spin_lock(&fbd->hw_stats.lock);
445	/* Record drops for the host FIFOs.
446	 * 4: network to Host,	6: BMC to Host
447	 * Exclude the BMC and MC FIFOs as those stats may contain drops
448	 * due to unrelated items such as TCAM misses. They are still
449	 * accessible through the ethtool stats.
450	 */
451	i = FBNIC_RXB_FIFO_HOST;
452	rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value;
453	i = FBNIC_RXB_FIFO_BMC_TO_HOST;
454	rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value;
455
456	for (i = 0; i < fbd->max_num_queues; i++) {
457		/* Report packets dropped due to CQ/BDQ being full/empty */
458		rx_over += fbd->hw_stats.hw_q[i].rde_pkt_cq_drop.value;
459		rx_over += fbd->hw_stats.hw_q[i].rde_pkt_bdq_drop.value;
460
461		/* Report packets with errors */
462		rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value;
463	}
464	spin_unlock(&fbd->hw_stats.lock);
465
466	stats64->rx_bytes = rx_bytes;
467	stats64->rx_packets = rx_packets;
468	stats64->rx_dropped = rx_dropped;
469	stats64->rx_over_errors = rx_over;
470	stats64->rx_errors = rx_errors;
471	stats64->rx_missed_errors = rx_missed;
472
473	for (i = 0; i < fbn->num_rx_queues; i++) {
474		struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i];
475		struct fbnic_ring *rxr = fbn->rx[i];
476
477		if (!rxr)
478			continue;
479
480		stats = &rxr->stats;
481		do {
482			start = u64_stats_fetch_begin(&stats->syncp);
483			rx_bytes = stats->bytes;
484			rx_packets = stats->packets;
485			rx_dropped = stats->dropped;
486			rx_length = stats->rx.length_errors;
487		} while (u64_stats_fetch_retry(&stats->syncp, start));
488
489		stats64->rx_bytes += rx_bytes;
490		stats64->rx_packets += rx_packets;
491		stats64->rx_dropped += rx_dropped;
492		stats64->rx_errors += rx_length;
493		stats64->rx_length_errors += rx_length;
494
495		if (!xdpr)
496			continue;
497
498		stats = &xdpr->stats;
499		do {
500			start = u64_stats_fetch_begin(&stats->syncp);
501			tx_bytes = stats->bytes;
502			tx_packets = stats->packets;
503			tx_dropped = stats->dropped;
504		} while (u64_stats_fetch_retry(&stats->syncp, start));
505
506		stats64->tx_bytes += tx_bytes;
507		stats64->tx_packets += tx_packets;
508		stats64->tx_dropped += tx_dropped;
509	}
510}
511
512bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu,
513			      u32 hds_thresh)
514{
515	if (!prog)
516		return false;
517
518	if (prog->aux->xdp_has_frags)
519		return false;
520
521	return mtu + ETH_HLEN > hds_thresh;
522}
523
524static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf)
525{
526	struct bpf_prog *prog = bpf->prog, *prev_prog;
527	struct fbnic_net *fbn = netdev_priv(netdev);
528
529	if (bpf->command != XDP_SETUP_PROG)
530		return -EINVAL;
531
532	if (fbnic_check_split_frames(prog, netdev->mtu,
533				     fbn->hds_thresh)) {
534		NL_SET_ERR_MSG_MOD(bpf->extack,
535				   "MTU too high, or HDS threshold is too low for single buffer XDP");
536		return -EOPNOTSUPP;
537	}
538
539	prev_prog = xchg(&fbn->xdp_prog, prog);
540	if (prev_prog)
541		bpf_prog_put(prev_prog);
542
543	return 0;
544}
545
546static const struct net_device_ops fbnic_netdev_ops = {
547	.ndo_open		= fbnic_open,
548	.ndo_stop		= fbnic_stop,
549	.ndo_validate_addr	= eth_validate_addr,
550	.ndo_start_xmit		= fbnic_xmit_frame,
551	.ndo_features_check	= fbnic_features_check,
552	.ndo_set_mac_address	= fbnic_set_mac,
553	.ndo_change_mtu		= fbnic_change_mtu,
554	.ndo_set_rx_mode	= fbnic_set_rx_mode,
555	.ndo_get_stats64	= fbnic_get_stats64,
556	.ndo_bpf		= fbnic_bpf,
557	.ndo_hwtstamp_get	= fbnic_hwtstamp_get,
558	.ndo_hwtstamp_set	= fbnic_hwtstamp_set,
559};
560
561static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,
562				     struct netdev_queue_stats_rx *rx)
563{
564	u64 bytes, packets, alloc_fail, alloc_fail_bdq;
565	struct fbnic_net *fbn = netdev_priv(dev);
566	struct fbnic_ring *rxr = fbn->rx[idx];
567	struct fbnic_dev *fbd = fbn->fbd;
568	struct fbnic_queue_stats *stats;
569	u64 csum_complete, csum_none;
570	struct fbnic_q_triad *qt;
571	unsigned int start;
572
573	if (!rxr)
574		return;
575
576	/* fbn->rx points to completion queues */
577	qt = container_of(rxr, struct fbnic_q_triad, cmpl);
578
579	stats = &rxr->stats;
580	do {
581		start = u64_stats_fetch_begin(&stats->syncp);
582		bytes = stats->bytes;
583		packets = stats->packets;
584		alloc_fail = stats->rx.alloc_failed;
585		csum_complete = stats->rx.csum_complete;
586		csum_none = stats->rx.csum_none;
587	} while (u64_stats_fetch_retry(&stats->syncp, start));
588
589	stats = &qt->sub0.stats;
590	do {
591		start = u64_stats_fetch_begin(&stats->syncp);
592		alloc_fail_bdq = stats->bdq.alloc_failed;
593	} while (u64_stats_fetch_retry(&stats->syncp, start));
594	alloc_fail += alloc_fail_bdq;
595
596	stats = &qt->sub1.stats;
597	do {
598		start = u64_stats_fetch_begin(&stats->syncp);
599		alloc_fail_bdq = stats->bdq.alloc_failed;
600	} while (u64_stats_fetch_retry(&stats->syncp, start));
601	alloc_fail += alloc_fail_bdq;
602
603	rx->bytes = bytes;
604	rx->packets = packets;
605	rx->alloc_fail = alloc_fail;
606	rx->csum_complete = csum_complete;
607	rx->csum_none = csum_none;
608
609	fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q);
610
611	spin_lock(&fbd->hw_stats.lock);
612	rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value +
613			       fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value;
614	rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value +
615		       rx->hw_drop_overruns;
616	spin_unlock(&fbd->hw_stats.lock);
617}
618
619static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx,
620				     struct netdev_queue_stats_tx *tx)
621{
622	struct fbnic_net *fbn = netdev_priv(dev);
623	struct fbnic_ring *txr = fbn->tx[idx];
624	struct fbnic_queue_stats *stats;
625	u64 stop, wake, csum, lso;
626	struct fbnic_ring *xdpr;
627	unsigned int start;
628	u64 bytes, packets;
629
630	if (!txr)
631		return;
632
633	stats = &txr->stats;
634	do {
635		start = u64_stats_fetch_begin(&stats->syncp);
636		bytes = stats->bytes;
637		packets = stats->packets;
638		csum = stats->twq.csum_partial;
639		lso = stats->twq.lso;
640		stop = stats->twq.stop;
641		wake = stats->twq.wake;
642	} while (u64_stats_fetch_retry(&stats->syncp, start));
643
644	tx->bytes = bytes;
645	tx->packets = packets;
646	tx->needs_csum = csum + lso;
647	tx->hw_gso_wire_packets = lso;
648	tx->stop = stop;
649	tx->wake = wake;
650
651	xdpr = fbn->tx[FBNIC_MAX_TXQS + idx];
652	if (xdpr) {
653		stats = &xdpr->stats;
654		do {
655			start = u64_stats_fetch_begin(&stats->syncp);
656			bytes = stats->bytes;
657			packets = stats->packets;
658		} while (u64_stats_fetch_retry(&stats->syncp, start));
659
660		tx->bytes += bytes;
661		tx->packets += packets;
662	}
663}
664
665static void fbnic_get_base_stats(struct net_device *dev,
666				 struct netdev_queue_stats_rx *rx,
667				 struct netdev_queue_stats_tx *tx)
668{
669	struct fbnic_net *fbn = netdev_priv(dev);
670
671	tx->bytes = fbn->tx_stats.bytes;
672	tx->packets = fbn->tx_stats.packets;
673	tx->needs_csum = fbn->tx_stats.twq.csum_partial + fbn->tx_stats.twq.lso;
674	tx->hw_gso_wire_packets = fbn->tx_stats.twq.lso;
675	tx->stop = fbn->tx_stats.twq.stop;
676	tx->wake = fbn->tx_stats.twq.wake;
677
678	rx->bytes = fbn->rx_stats.bytes;
679	rx->packets = fbn->rx_stats.packets;
680	rx->alloc_fail = fbn->rx_stats.rx.alloc_failed +
681		fbn->bdq_stats.bdq.alloc_failed;
682	rx->csum_complete = fbn->rx_stats.rx.csum_complete;
683	rx->csum_none = fbn->rx_stats.rx.csum_none;
684}
685
686static const struct netdev_stat_ops fbnic_stat_ops = {
687	.get_queue_stats_rx	= fbnic_get_queue_stats_rx,
688	.get_queue_stats_tx	= fbnic_get_queue_stats_tx,
689	.get_base_stats		= fbnic_get_base_stats,
690};
691
692void fbnic_reset_queues(struct fbnic_net *fbn,
693			unsigned int tx, unsigned int rx)
694{
695	struct fbnic_dev *fbd = fbn->fbd;
696	unsigned int max_napis;
697
698	max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS;
699
700	tx = min(tx, max_napis);
701	fbn->num_tx_queues = tx;
702
703	rx = min(rx, max_napis);
704	fbn->num_rx_queues = rx;
705
706	fbn->num_napi = max(tx, rx);
707}
708
709/**
710 * fbnic_netdev_free - Free the netdev associate with fbnic
711 * @fbd: Driver specific structure to free netdev from
712 *
713 * Allocate and initialize the netdev and netdev private structure. Bind
714 * together the hardware, netdev, and pci data structures.
715 **/
716void fbnic_netdev_free(struct fbnic_dev *fbd)
717{
718	fbnic_phylink_destroy(fbd->netdev);
719
720	free_netdev(fbd->netdev);
721	fbd->netdev = NULL;
722}
723
724/**
725 * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic
726 * @fbd: Driver specific structure to associate netdev with
727 *
728 * Allocate and initialize the netdev and netdev private structure. Bind
729 * together the hardware, netdev, and pci data structures.
730 *
731 *  Return: Pointer to net_device on success, NULL on failure
732 **/
733struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)
734{
735	struct net_device *netdev;
736	struct fbnic_net *fbn;
737	int default_queues;
738
739	netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS);
740	if (!netdev)
741		return NULL;
742
743	SET_NETDEV_DEV(netdev, fbd->dev);
744	fbd->netdev = netdev;
745
746	netdev->netdev_ops = &fbnic_netdev_ops;
747	netdev->stat_ops = &fbnic_stat_ops;
748	netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops;
749	netdev->netmem_tx = true;
750
751	fbnic_set_ethtool_ops(netdev);
752
753	fbn = netdev_priv(netdev);
754
755	fbn->netdev = netdev;
756	fbn->fbd = fbd;
757
758	fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT;
759	fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT;
760	fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT;
761	fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT;
762
763	fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT;
764	fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT;
765	fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT;
766
767	/* Initialize the hds_thresh */
768	netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT;
769	fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT;
770
771	default_queues = netif_get_num_default_rss_queues();
772	if (default_queues > fbd->max_num_queues)
773		default_queues = fbd->max_num_queues;
774
775	fbnic_reset_queues(fbn, default_queues, default_queues);
776
777	fbnic_reset_indir_tbl(fbn);
778	fbnic_rss_key_fill(fbn->rss_key);
779	fbnic_rss_init_en_mask(fbn);
780
781	netdev->priv_flags |= IFF_UNICAST_FLT;
782
783	netdev->gso_partial_features =
784		NETIF_F_GSO_GRE |
785		NETIF_F_GSO_GRE_CSUM |
786		NETIF_F_GSO_IPXIP4 |
787		NETIF_F_GSO_UDP_TUNNEL |
788		NETIF_F_GSO_UDP_TUNNEL_CSUM;
789
790	netdev->features |=
791		netdev->gso_partial_features |
792		FBNIC_TUN_GSO_FEATURES |
793		NETIF_F_RXHASH |
794		NETIF_F_SG |
795		NETIF_F_HW_CSUM |
796		NETIF_F_RXCSUM |
797		NETIF_F_TSO |
798		NETIF_F_TSO_ECN |
799		NETIF_F_TSO6 |
800		NETIF_F_GSO_PARTIAL |
801		NETIF_F_GSO_UDP_L4;
802
803	netdev->hw_features |= netdev->features;
804	netdev->vlan_features |= netdev->features;
805	netdev->hw_enc_features |= netdev->features;
806	netdev->features |= NETIF_F_NTUPLE;
807
808	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG;
809
810	netdev->min_mtu = IPV6_MIN_MTU;
811	netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;
812
813	/* TBD: This is workaround for BMC as phylink doesn't have support
814	 * for leavling the link enabled if a BMC is present.
815	 */
816	netdev->ethtool->wol_enabled = true;
817
818	netif_carrier_off(netdev);
819
820	netif_tx_stop_all_queues(netdev);
821
822	if (fbnic_phylink_create(netdev)) {
823		fbnic_netdev_free(fbd);
824		return NULL;
825	}
826
827	return netdev;
828}
829
830static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr)
831{
832	addr[0] = (dsn >> 56) & 0xFF;
833	addr[1] = (dsn >> 48) & 0xFF;
834	addr[2] = (dsn >> 40) & 0xFF;
835	addr[3] = (dsn >> 16) & 0xFF;
836	addr[4] = (dsn >> 8) & 0xFF;
837	addr[5] = dsn & 0xFF;
838
839	return is_valid_ether_addr(addr) ? 0 : -EINVAL;
840}
841
842/**
843 * fbnic_netdev_register - Initialize general software structures
844 * @netdev: Netdev containing structure to initialize and register
845 *
846 * Initialize the MAC address for the netdev and register it.
847 *
848 *  Return: 0 on success, negative on failure
849 **/
850int fbnic_netdev_register(struct net_device *netdev)
851{
852	struct fbnic_net *fbn = netdev_priv(netdev);
853	struct fbnic_dev *fbd = fbn->fbd;
854	u64 dsn = fbd->dsn;
855	u8 addr[ETH_ALEN];
856	int err;
857
858	err = fbnic_dsn_to_mac_addr(dsn, addr);
859	if (!err) {
860		ether_addr_copy(netdev->perm_addr, addr);
861		eth_hw_addr_set(netdev, addr);
862	} else {
863		/* A randomly assigned MAC address will cause provisioning
864		 * issues so instead just fail to spawn the netdev and
865		 * avoid any confusion.
866		 */
867		dev_err(fbd->dev, "MAC addr %pM invalid\n", addr);
868		return err;
869	}
870
871	return register_netdev(netdev);
872}
873
874void fbnic_netdev_unregister(struct net_device *netdev)
875{
876	unregister_netdev(netdev);
877}