aq_main.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Atlantic Network Driver
  3 *
  4 * Copyright (C) 2014-2019 aQuantia Corporation
  5 * Copyright (C) 2019-2020 Marvell International Ltd.
  6 */
  7
  8/* File aq_main.c: Main file for aQuantia Linux driver. */
  9
 10#include "aq_main.h"
 11#include "aq_nic.h"
 12#include "aq_pci_func.h"
 13#include "aq_ethtool.h"
 14#include "aq_ptp.h"
 15#include "aq_filters.h"
 16#include "aq_hw_utils.h"
 17#include "aq_vec.h"
 18
 19#include <linux/netdevice.h>
 20#include <linux/module.h>
 21#include <linux/ip.h>
 22#include <linux/udp.h>
 23#include <net/pkt_cls.h>
 24#include <net/pkt_sched.h>
 25#include <linux/filter.h>
 26
 27MODULE_LICENSE("GPL v2");
 28MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);
 29MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);
 30
 31DEFINE_STATIC_KEY_FALSE(aq_xdp_locking_key);
 32EXPORT_SYMBOL(aq_xdp_locking_key);
 33
 34static const char aq_ndev_driver_name[] = AQ_CFG_DRV_NAME;
 35
 36static const struct net_device_ops aq_ndev_ops;
 37
 38static struct workqueue_struct *aq_ndev_wq;
 39
 40void aq_ndev_schedule_work(struct work_struct *work)
 41{
 42	queue_work(aq_ndev_wq, work);
 43}
 44
 45struct net_device *aq_ndev_alloc(void)
 46{
 47	struct net_device *ndev = NULL;
 48	struct aq_nic_s *aq_nic = NULL;
 49
 50	ndev = alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_HW_QUEUES_MAX);
 51	if (!ndev)
 52		return NULL;
 53
 54	aq_nic = netdev_priv(ndev);
 55	aq_nic->ndev = ndev;
 56	ndev->netdev_ops = &aq_ndev_ops;
 57	ndev->ethtool_ops = &aq_ethtool_ops;
 58
 59	return ndev;
 60}
 61
 62int aq_ndev_open(struct net_device *ndev)
 63{
 64	struct aq_nic_s *aq_nic = netdev_priv(ndev);
 65	int err = 0;
 66
 67	err = aq_nic_init(aq_nic);
 68	if (err < 0)
 69		goto err_exit;
 70
 71	err = aq_reapply_rxnfc_all_rules(aq_nic);
 72	if (err < 0)
 73		goto err_exit;
 74
 75	err = aq_filters_vlans_update(aq_nic);
 76	if (err < 0)
 77		goto err_exit;
 78
 79	err = aq_nic_start(aq_nic);
 80	if (err < 0) {
 81		aq_nic_stop(aq_nic);
 82		goto err_exit;
 83	}
 84
 85err_exit:
 86	if (err < 0)
 87		aq_nic_deinit(aq_nic, true);
 88
 89	return err;
 90}
 91
 92int aq_ndev_close(struct net_device *ndev)
 93{
 94	struct aq_nic_s *aq_nic = netdev_priv(ndev);
 95	int err = 0;
 96
 97	err = aq_nic_stop(aq_nic);
 98	aq_nic_deinit(aq_nic, true);
 99
100	return err;
101}
102
103static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *ndev)
104{
105	struct aq_nic_s *aq_nic = netdev_priv(ndev);
106
107#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
108	if (unlikely(aq_utils_obj_test(&aq_nic->flags, AQ_NIC_PTP_DPATH_UP))) {
109		/* Hardware adds the Timestamp for PTPv2 802.AS1
110		 * and PTPv2 IPv4 UDP.
111		 * We have to push even general 320 port messages to the ptp
112		 * queue explicitly. This is a limitation of current firmware
113		 * and hardware PTP design of the chip. Otherwise ptp stream
114		 * will fail to sync
115		 */
116		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ||
117		    unlikely((ip_hdr(skb)->version == 4) &&
118			     (ip_hdr(skb)->protocol == IPPROTO_UDP) &&
119			     ((udp_hdr(skb)->dest == htons(319)) ||
120			      (udp_hdr(skb)->dest == htons(320)))) ||
121		    unlikely(eth_hdr(skb)->h_proto == htons(ETH_P_1588)))
122			return aq_ptp_xmit(aq_nic, skb);
123	}
124#endif
125
126	return aq_nic_xmit(aq_nic, skb);
127}
128
129static int aq_ndev_change_mtu(struct net_device *ndev, int new_mtu)
130{
131	int new_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
132	struct aq_nic_s *aq_nic = netdev_priv(ndev);
133	struct bpf_prog *prog;
134	int err;
135
136	prog = READ_ONCE(aq_nic->xdp_prog);
137	if (prog && !prog->aux->xdp_has_frags &&
138	    new_frame_size > AQ_CFG_RX_FRAME_MAX) {
139		netdev_err(ndev, "Illegal MTU %d for XDP prog without frags\n",
140			   ndev->mtu);
141		return -EOPNOTSUPP;
142	}
143
144	err = aq_nic_set_mtu(aq_nic, new_mtu + ETH_HLEN);
145
146	if (err < 0)
147		goto err_exit;
148	WRITE_ONCE(ndev->mtu, new_mtu);
149
150err_exit:
151	return err;
152}
153
154static int aq_ndev_set_features(struct net_device *ndev,
155				netdev_features_t features)
156{
157	bool is_vlan_tx_insert = !!(features & NETIF_F_HW_VLAN_CTAG_TX);
158	bool is_vlan_rx_strip = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
159	struct aq_nic_s *aq_nic = netdev_priv(ndev);
160	bool need_ndev_restart = false;
161	struct aq_nic_cfg_s *aq_cfg;
162	bool is_lro = false;
163	int err = 0;
164
165	aq_cfg = aq_nic_get_cfg(aq_nic);
166
167	if (!(features & NETIF_F_NTUPLE)) {
168		if (aq_nic->ndev->features & NETIF_F_NTUPLE) {
169			err = aq_clear_rxnfc_all_rules(aq_nic);
170			if (unlikely(err))
171				goto err_exit;
172		}
173	}
174	if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER)) {
175		if (aq_nic->ndev->features & NETIF_F_HW_VLAN_CTAG_FILTER) {
176			err = aq_filters_vlan_offload_off(aq_nic);
177			if (unlikely(err))
178				goto err_exit;
179		}
180	}
181
182	aq_cfg->features = features;
183
184	if (aq_cfg->aq_hw_caps->hw_features & NETIF_F_LRO) {
185		is_lro = features & NETIF_F_LRO;
186
187		if (aq_cfg->is_lro != is_lro) {
188			aq_cfg->is_lro = is_lro;
189			need_ndev_restart = true;
190		}
191	}
192
193	if ((aq_nic->ndev->features ^ features) & NETIF_F_RXCSUM) {
194		err = aq_nic->aq_hw_ops->hw_set_offload(aq_nic->aq_hw,
195							aq_cfg);
196
197		if (unlikely(err))
198			goto err_exit;
199	}
200
201	if (aq_cfg->is_vlan_rx_strip != is_vlan_rx_strip) {
202		aq_cfg->is_vlan_rx_strip = is_vlan_rx_strip;
203		need_ndev_restart = true;
204	}
205	if (aq_cfg->is_vlan_tx_insert != is_vlan_tx_insert) {
206		aq_cfg->is_vlan_tx_insert = is_vlan_tx_insert;
207		need_ndev_restart = true;
208	}
209
210	if (need_ndev_restart && netif_running(ndev)) {
211		aq_ndev_close(ndev);
212		aq_ndev_open(ndev);
213	}
214
215err_exit:
216	return err;
217}
218
219static netdev_features_t aq_ndev_fix_features(struct net_device *ndev,
220					      netdev_features_t features)
221{
222	struct aq_nic_s *aq_nic = netdev_priv(ndev);
223	struct bpf_prog *prog;
224
225	if (!(features & NETIF_F_RXCSUM))
226		features &= ~NETIF_F_LRO;
227
228	prog = READ_ONCE(aq_nic->xdp_prog);
229	if (prog && !prog->aux->xdp_has_frags &&
230	    aq_nic->xdp_prog && features & NETIF_F_LRO) {
231		netdev_err(ndev, "LRO is not supported with single buffer XDP, disabling\n");
232		features &= ~NETIF_F_LRO;
233	}
234
235	return features;
236}
237
238static int aq_ndev_set_mac_address(struct net_device *ndev, void *addr)
239{
240	struct aq_nic_s *aq_nic = netdev_priv(ndev);
241	int err = 0;
242
243	err = eth_mac_addr(ndev, addr);
244	if (err < 0)
245		goto err_exit;
246	err = aq_nic_set_mac(aq_nic, ndev);
247	if (err < 0)
248		goto err_exit;
249
250err_exit:
251	return err;
252}
253
254static void aq_ndev_set_multicast_settings(struct net_device *ndev)
255{
256	struct aq_nic_s *aq_nic = netdev_priv(ndev);
257
258	(void)aq_nic_set_multicast_list(aq_nic, ndev);
259}
260
261static int aq_ndev_hwtstamp_set(struct net_device *netdev,
262				struct kernel_hwtstamp_config *config,
263				struct netlink_ext_ack *extack)
264{
265	struct aq_nic_s *aq_nic = netdev_priv(netdev);
266
267	if (!IS_REACHABLE(CONFIG_PTP_1588_CLOCK) || !aq_nic->aq_ptp)
268		return -EOPNOTSUPP;
269
270	switch (config->tx_type) {
271	case HWTSTAMP_TX_OFF:
272	case HWTSTAMP_TX_ON:
273		break;
274	default:
275		return -ERANGE;
276	}
277
278	switch (config->rx_filter) {
279	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
280	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
281	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
282	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
283	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
284	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
285	case HWTSTAMP_FILTER_PTP_V2_SYNC:
286	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
287		config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
288		break;
289	case HWTSTAMP_FILTER_PTP_V2_EVENT:
290	case HWTSTAMP_FILTER_NONE:
291		break;
292	default:
293		return -ERANGE;
294	}
295
296	return aq_ptp_hwtstamp_config_set(aq_nic->aq_ptp, config);
297}
298
299static int aq_ndev_hwtstamp_get(struct net_device *netdev,
300				struct kernel_hwtstamp_config *config)
301{
302	struct aq_nic_s *aq_nic = netdev_priv(netdev);
303
304	if (!aq_nic->aq_ptp)
305		return -EOPNOTSUPP;
306
307	aq_ptp_hwtstamp_config_get(aq_nic->aq_ptp, config);
308	return 0;
309}
310
311static int aq_ndo_vlan_rx_add_vid(struct net_device *ndev, __be16 proto,
312				  u16 vid)
313{
314	struct aq_nic_s *aq_nic = netdev_priv(ndev);
315
316	if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
317		return -EOPNOTSUPP;
318
319	set_bit(vid, aq_nic->active_vlans);
320
321	return aq_filters_vlans_update(aq_nic);
322}
323
324static int aq_ndo_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto,
325				   u16 vid)
326{
327	struct aq_nic_s *aq_nic = netdev_priv(ndev);
328
329	if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
330		return -EOPNOTSUPP;
331
332	clear_bit(vid, aq_nic->active_vlans);
333
334	if (-ENOENT == aq_del_fvlan_by_vlan(aq_nic, vid))
335		return aq_filters_vlans_update(aq_nic);
336
337	return 0;
338}
339
340static int aq_validate_mqprio_opt(struct aq_nic_s *self,
341				  struct tc_mqprio_qopt_offload *mqprio,
342				  const unsigned int num_tc)
343{
344	const bool has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
345	struct aq_nic_cfg_s *aq_nic_cfg = aq_nic_get_cfg(self);
346	const unsigned int tcs_max = min_t(u8, aq_nic_cfg->aq_hw_caps->tcs_max,
347					   AQ_CFG_TCS_MAX);
348
349	if (num_tc > tcs_max) {
350		netdev_err(self->ndev, "Too many TCs requested\n");
351		return -EOPNOTSUPP;
352	}
353
354	if (num_tc != 0 && !is_power_of_2(num_tc)) {
355		netdev_err(self->ndev, "TC count should be power of 2\n");
356		return -EOPNOTSUPP;
357	}
358
359	if (has_min_rate && !ATL_HW_IS_CHIP_FEATURE(self->aq_hw, ANTIGUA)) {
360		netdev_err(self->ndev, "Min tx rate is not supported\n");
361		return -EOPNOTSUPP;
362	}
363
364	return 0;
365}
366
367static int aq_ndo_setup_tc(struct net_device *dev, enum tc_setup_type type,
368			   void *type_data)
369{
370	struct tc_mqprio_qopt_offload *mqprio = type_data;
371	struct aq_nic_s *aq_nic = netdev_priv(dev);
372	bool has_min_rate;
373	bool has_max_rate;
374	int err;
375	int i;
376
377	if (type != TC_SETUP_QDISC_MQPRIO)
378		return -EOPNOTSUPP;
379
380	has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
381	has_max_rate = !!(mqprio->flags & TC_MQPRIO_F_MAX_RATE);
382
383	err = aq_validate_mqprio_opt(aq_nic, mqprio, mqprio->qopt.num_tc);
384	if (err)
385		return err;
386
387	for (i = 0; i < mqprio->qopt.num_tc; i++) {
388		if (has_max_rate) {
389			u64 max_rate = mqprio->max_rate[i];
390
391			do_div(max_rate, AQ_MBPS_DIVISOR);
392			aq_nic_setup_tc_max_rate(aq_nic, i, (u32)max_rate);
393		}
394
395		if (has_min_rate) {
396			u64 min_rate = mqprio->min_rate[i];
397
398			do_div(min_rate, AQ_MBPS_DIVISOR);
399			aq_nic_setup_tc_min_rate(aq_nic, i, (u32)min_rate);
400		}
401	}
402
403	return aq_nic_setup_tc_mqprio(aq_nic, mqprio->qopt.num_tc,
404				      mqprio->qopt.prio_tc_map);
405}
406
407static int aq_xdp_setup(struct net_device *ndev, struct bpf_prog *prog,
408			struct netlink_ext_ack *extack)
409{
410	bool need_update, running = netif_running(ndev);
411	struct aq_nic_s *aq_nic = netdev_priv(ndev);
412	struct bpf_prog *old_prog;
413
414	if (prog && !prog->aux->xdp_has_frags) {
415		if (ndev->mtu > AQ_CFG_RX_FRAME_MAX) {
416			NL_SET_ERR_MSG_MOD(extack,
417					   "prog does not support XDP frags");
418			return -EOPNOTSUPP;
419		}
420
421		if (prog && ndev->features & NETIF_F_LRO) {
422			netdev_err(ndev,
423				   "LRO is not supported with single buffer XDP, disabling\n");
424			ndev->features &= ~NETIF_F_LRO;
425		}
426	}
427
428	need_update = !!aq_nic->xdp_prog != !!prog;
429	if (running && need_update)
430		aq_ndev_close(ndev);
431
432	old_prog = xchg(&aq_nic->xdp_prog, prog);
433	if (old_prog)
434		bpf_prog_put(old_prog);
435
436	if (!old_prog && prog)
437		static_branch_inc(&aq_xdp_locking_key);
438	else if (old_prog && !prog)
439		static_branch_dec(&aq_xdp_locking_key);
440
441	if (running && need_update)
442		return aq_ndev_open(ndev);
443
444	return 0;
445}
446
447static int aq_xdp(struct net_device *dev, struct netdev_bpf *xdp)
448{
449	switch (xdp->command) {
450	case XDP_SETUP_PROG:
451		return aq_xdp_setup(dev, xdp->prog, xdp->extack);
452	default:
453		return -EINVAL;
454	}
455}
456
457static const struct net_device_ops aq_ndev_ops = {
458	.ndo_open = aq_ndev_open,
459	.ndo_stop = aq_ndev_close,
460	.ndo_start_xmit = aq_ndev_start_xmit,
461	.ndo_set_rx_mode = aq_ndev_set_multicast_settings,
462	.ndo_change_mtu = aq_ndev_change_mtu,
463	.ndo_set_mac_address = aq_ndev_set_mac_address,
464	.ndo_set_features = aq_ndev_set_features,
465	.ndo_fix_features = aq_ndev_fix_features,
466	.ndo_vlan_rx_add_vid = aq_ndo_vlan_rx_add_vid,
467	.ndo_vlan_rx_kill_vid = aq_ndo_vlan_rx_kill_vid,
468	.ndo_setup_tc = aq_ndo_setup_tc,
469	.ndo_bpf = aq_xdp,
470	.ndo_xdp_xmit = aq_xdp_xmit,
471	.ndo_hwtstamp_get = aq_ndev_hwtstamp_get,
472	.ndo_hwtstamp_set = aq_ndev_hwtstamp_set,
473};
474
475static int __init aq_ndev_init_module(void)
476{
477	int ret;
478
479	aq_ndev_wq = create_singlethread_workqueue(aq_ndev_driver_name);
480	if (!aq_ndev_wq) {
481		pr_err("Failed to create workqueue\n");
482		return -ENOMEM;
483	}
484
485	ret = aq_pci_func_register_driver();
486	if (ret) {
487		destroy_workqueue(aq_ndev_wq);
488		return ret;
489	}
490
491	return 0;
492}
493
494static void __exit aq_ndev_exit_module(void)
495{
496	aq_pci_func_unregister_driver();
497
498	if (aq_ndev_wq) {
499		destroy_workqueue(aq_ndev_wq);
500		aq_ndev_wq = NULL;
501	}
502}
503
504module_init(aq_ndev_init_module);
505module_exit(aq_ndev_exit_module);