mlx4_en.h

  1/*
  2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 *
 32 */
 33
 34#ifndef _MLX4_EN_H_
 35#define _MLX4_EN_H_
 36
 37#include <linux/bitops.h>
 38#include <linux/compiler.h>
 39#include <linux/ethtool.h>
 40#include <linux/list.h>
 41#include <linux/mutex.h>
 42#include <linux/netdevice.h>
 43#include <linux/if_vlan.h>
 44#include <linux/net_tstamp.h>
 45#ifdef CONFIG_MLX4_EN_DCB
 46#include <linux/dcbnl.h>
 47#endif
 48#include <linux/cpu_rmap.h>
 49#include <linux/ptp_clock_kernel.h>
 50#include <linux/irq.h>
 51#include <net/xdp.h>
 52#include <linux/notifier.h>
 53
 54#include <linux/mlx4/device.h>
 55#include <linux/mlx4/qp.h>
 56#include <linux/mlx4/cq.h>
 57#include <linux/mlx4/srq.h>
 58#include <linux/mlx4/doorbell.h>
 59#include <linux/mlx4/cmd.h>
 60
 61#include "en_port.h"
 62#include "mlx4_stats.h"
 63
 64#define DRV_NAME	"mlx4_en"
 65#define DRV_VERSION	"4.0-0"
 66
 67#define MLX4_EN_MSG_LEVEL	(NETIF_MSG_LINK | NETIF_MSG_IFDOWN)
 68
 69/*
 70 * Device constants
 71 */
 72
 73
 74#define MLX4_EN_PAGE_SHIFT	12
 75#define MLX4_EN_PAGE_SIZE	(1 << MLX4_EN_PAGE_SHIFT)
 76#define DEF_RX_RINGS		16
 77#define MAX_RX_RINGS		128
 78#define MIN_RX_RINGS		1
 79#define LOG_TXBB_SIZE		6
 80#define TXBB_SIZE		BIT(LOG_TXBB_SIZE)
 81#define HEADROOM		(2048 / TXBB_SIZE + 1)
 82#define STAMP_STRIDE		64
 83#define STAMP_DWORDS		(STAMP_STRIDE / 4)
 84#define STAMP_SHIFT		31
 85#define STAMP_VAL		0x7fffffff
 86#define STATS_DELAY		(HZ / 4)
 87#define SERVICE_TASK_DELAY	(HZ / 4)
 88#define MAX_NUM_OF_FS_RULES	256
 89
 90#define MLX4_EN_FILTER_HASH_SHIFT 4
 91#define MLX4_EN_FILTER_EXPIRY_QUOTA 60
 92
 93#define CTRL_SIZE	sizeof(struct mlx4_wqe_ctrl_seg)
 94#define DS_SIZE		sizeof(struct mlx4_wqe_data_seg)
 95
 96/* Maximal size of the bounce buffer:
 97 * 256 bytes for LSO headers.
 98 * CTRL_SIZE for control desc.
 99 * DS_SIZE if skb->head contains some payload.
100 * MAX_SKB_FRAGS frags.
101 */
102#define MLX4_TX_BOUNCE_BUFFER_SIZE \
103	ALIGN(256 + CTRL_SIZE + DS_SIZE + MAX_SKB_FRAGS * DS_SIZE, TXBB_SIZE)
104
105#define MLX4_MAX_DESC_TXBBS	   (MLX4_TX_BOUNCE_BUFFER_SIZE / TXBB_SIZE)
106
107/*
108 * OS related constants and tunables
109 */
110
111#define MLX4_EN_PRIV_FLAGS_BLUEFLAME 1
112#define MLX4_EN_PRIV_FLAGS_PHV	     2
113
114#define MLX4_EN_WATCHDOG_TIMEOUT	(15 * HZ)
115
116/* Use the maximum between 16384 and a single page */
117#define MLX4_EN_ALLOC_SIZE	PAGE_ALIGN(16384)
118
119#define MLX4_EN_MAX_RX_FRAGS	4
120
121/* Maximum ring sizes */
122#define MLX4_EN_MAX_TX_SIZE	8192
123#define MLX4_EN_MAX_RX_SIZE	8192
124
125/* Minimum ring size for our page-allocation scheme to work */
126#define MLX4_EN_MIN_RX_SIZE	(MLX4_EN_ALLOC_SIZE / SMP_CACHE_BYTES)
127#define MLX4_EN_MIN_TX_SIZE	(4096 / TXBB_SIZE)
128
129#define MLX4_EN_SMALL_PKT_SIZE		64
130#define MLX4_EN_MIN_TX_RING_P_UP	1
131#define MLX4_EN_MAX_TX_RING_P_UP	32
132#define MLX4_EN_NUM_UP_LOW		1
133#define MLX4_EN_NUM_UP_HIGH		8
134#define MLX4_EN_DEF_RX_RING_SIZE  	1024
135#define MLX4_EN_DEF_TX_RING_SIZE	MLX4_EN_DEF_RX_RING_SIZE
136#define MAX_TX_RINGS			(MLX4_EN_MAX_TX_RING_P_UP * \
137					 MLX4_EN_NUM_UP_HIGH)
138
139#define MLX4_EN_DEFAULT_TX_WORK		256
140
141/* Target number of packets to coalesce with interrupt moderation */
142#define MLX4_EN_RX_COAL_TARGET	44
143#define MLX4_EN_RX_COAL_TIME	0x10
144
145#define MLX4_EN_TX_COAL_PKTS	16
146#define MLX4_EN_TX_COAL_TIME	0x10
147
148#define MLX4_EN_MAX_COAL_PKTS	U16_MAX
149#define MLX4_EN_MAX_COAL_TIME	U16_MAX
150
151#define MLX4_EN_RX_RATE_LOW		400000
152#define MLX4_EN_RX_COAL_TIME_LOW	0
153#define MLX4_EN_RX_RATE_HIGH		450000
154#define MLX4_EN_RX_COAL_TIME_HIGH	128
155#define MLX4_EN_RX_SIZE_THRESH		1024
156#define MLX4_EN_RX_RATE_THRESH		(1000000 / MLX4_EN_RX_COAL_TIME_HIGH)
157#define MLX4_EN_SAMPLE_INTERVAL		0
158#define MLX4_EN_AVG_PKT_SMALL		256
159
160#define MLX4_EN_AUTO_CONF	0xffff
161
162#define MLX4_EN_DEF_RX_PAUSE	1
163#define MLX4_EN_DEF_TX_PAUSE	1
164
165/* Interval between successive polls in the Tx routine when polling is used
166   instead of interrupts (in per-core Tx rings) - should be power of 2 */
167#define MLX4_EN_TX_POLL_MODER	16
168#define MLX4_EN_TX_POLL_TIMEOUT	(HZ / 4)
169
170#define SMALL_PACKET_SIZE      (256 - NET_IP_ALIGN)
171#define HEADER_COPY_SIZE       (128 - NET_IP_ALIGN)
172#define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETH_HLEN)
173#define PREAMBLE_LEN           8
174#define MLX4_SELFTEST_LB_MIN_MTU (MLX4_LOOPBACK_TEST_PAYLOAD + NET_IP_ALIGN + \
175				  ETH_HLEN + PREAMBLE_LEN)
176
177/* VLAN_HLEN is added twice,to support skb vlan tagged with multiple
178 * headers. (For example: ETH_P_8021Q and ETH_P_8021AD).
179 */
180#define MLX4_EN_EFF_MTU(mtu)	((mtu) + ETH_HLEN + (2 * VLAN_HLEN))
181#define ETH_BCAST		0xffffffffffffULL
182
183#define MLX4_EN_LOOPBACK_RETRIES	5
184#define MLX4_EN_LOOPBACK_TIMEOUT	100
185
186/* Constants for TX flow */
187enum {
188	MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */
189	MAX_BF = 256,
190	MIN_PKT_LEN = 17,
191};
192
193/*
194 * Configurables
195 */
196
197enum cq_type {
198	/* keep tx types first */
199	TX,
200	TX_XDP,
201#define MLX4_EN_NUM_TX_TYPES (TX_XDP + 1)
202	RX,
203};
204
205
206/*
207 * Useful macros
208 */
209#define ROUNDUP_LOG2(x)		ilog2(roundup_pow_of_two(x))
210#define XNOR(x, y)		(!(x) == !(y))
211
212
213struct mlx4_en_tx_info {
214	union {
215		struct sk_buff *skb;
216		struct page *page;
217	};
218	dma_addr_t	map0_dma;
219	u32		map0_byte_count;
220	u32		nr_txbb;
221	u32		nr_bytes;
222	u8		linear;
223	u8		data_offset;
224	u8		inl;
225	u8		ts_requested;
226	u8		nr_maps;
227} ____cacheline_aligned_in_smp;
228
229
230#define MLX4_EN_BIT_DESC_OWN	0x80000000
231#define MLX4_EN_MEMTYPE_PAD	0x100
232
233
234struct mlx4_en_tx_desc {
235	struct mlx4_wqe_ctrl_seg ctrl;
236	union {
237		struct mlx4_wqe_data_seg data; /* at least one data segment */
238		struct mlx4_wqe_lso_seg lso;
239		struct mlx4_wqe_inline_seg inl;
240	};
241};
242
243#define MLX4_EN_USE_SRQ		0x01000000
244
245#define MLX4_EN_CX3_LOW_ID	0x1000
246#define MLX4_EN_CX3_HIGH_ID	0x1005
247
248struct mlx4_en_rx_alloc {
249	struct page	*page;
250	u32		page_offset;
251};
252
253#define MLX4_EN_CACHE_SIZE (2 * NAPI_POLL_WEIGHT)
254
255enum {
256	MLX4_EN_TX_RING_STATE_RECOVERING,
257};
258
259struct mlx4_en_priv;
260
261struct mlx4_en_tx_ring {
262	/* cache line used and dirtied in tx completion
263	 * (mlx4_en_free_tx_buf())
264	 */
265	u32			last_nr_txbb;
266	u32			cons;
267	unsigned long		wake_queue;
268	struct netdev_queue	*tx_queue;
269	u32			(*free_tx_desc)(struct mlx4_en_priv *priv,
270						struct mlx4_en_tx_ring *ring,
271						int index,
272						u64 timestamp, int napi_mode);
273	struct mlx4_en_rx_ring	*recycle_ring;
274
275	/* cache line used and dirtied in mlx4_en_xmit() */
276	u32			prod ____cacheline_aligned_in_smp;
277	unsigned int		tx_dropped;
278	unsigned long		bytes;
279	unsigned long		packets;
280	unsigned long		tx_csum;
281	unsigned long		tso_packets;
282	unsigned long		xmit_more;
283	struct mlx4_bf		bf;
284
285	/* Following part should be mostly read */
286	void __iomem		*doorbell_address;
287	__be32			doorbell_qpn;
288	__be32			mr_key;
289	u32			size; /* number of TXBBs */
290	u32			size_mask;
291	u32			full_size;
292	u32			buf_size;
293	void			*buf;
294	struct mlx4_en_tx_info	*tx_info;
295	int			qpn;
296	u8			queue_index;
297	bool			bf_enabled;
298	bool			bf_alloced;
299	u8			hwtstamp_tx_type;
300	u8			*bounce_buf;
301
302	/* Not used in fast path
303	 * Only queue_stopped might be used if BQL is not properly working.
304	 */
305	unsigned long		queue_stopped;
306	unsigned long		state;
307	struct mlx4_hwq_resources sp_wqres;
308	struct mlx4_qp		sp_qp;
309	struct mlx4_qp_context	sp_context;
310	cpumask_t		sp_affinity_mask;
311	enum mlx4_qp_state	sp_qp_state;
312	u16			sp_stride;
313	u16			sp_cqn;	/* index of port CQ associated with this ring */
314} ____cacheline_aligned_in_smp;
315
316struct mlx4_en_rx_desc {
317	/* actual number of entries depends on rx ring stride */
318	DECLARE_FLEX_ARRAY(struct mlx4_wqe_data_seg, data);
319};
320
321struct mlx4_en_rx_ring {
322	struct mlx4_hwq_resources wqres;
323	u32 size ;	/* number of Rx descs*/
324	u32 actual_size;
325	u32 size_mask;
326	u16 stride;
327	u16 log_stride;
328	u16 cqn;	/* index of port CQ associated with this ring */
329	u8  fcs_del;
330	u32 prod;
331	u32 cons;
332	u32 buf_size;
333	struct page_pool *pp;
334	void *buf;
335	void *rx_info;
336	struct bpf_prog __rcu *xdp_prog;
337	unsigned long bytes;
338	unsigned long packets;
339	unsigned long csum_ok;
340	unsigned long csum_none;
341	unsigned long csum_complete;
342	unsigned long rx_alloc_pages;
343	unsigned long xdp_drop;
344	unsigned long xdp_redirect;
345	unsigned long xdp_redirect_fail;
346	unsigned long xdp_tx;
347	unsigned long xdp_tx_full;
348	unsigned long dropped;
349	unsigned long alloc_fail;
350	int hwtstamp_rx_filter;
351	cpumask_var_t affinity_mask;
352	struct xdp_rxq_info xdp_rxq;
353};
354
355struct mlx4_en_cq {
356	struct mlx4_cq          mcq;
357	struct mlx4_hwq_resources wqres;
358	int                     ring;
359	struct net_device      *dev;
360	union {
361		struct napi_struct napi;
362		bool               xdp_busy;
363	};
364	int size;
365	int buf_size;
366	int vector;
367	enum cq_type type;
368	u16 moder_time;
369	u16 moder_cnt;
370	struct mlx4_cqe *buf;
371#define MLX4_EN_OPCODE_ERROR	0x1e
372
373	const struct cpumask *aff_mask;
374	int cq_idx;
375};
376
377struct mlx4_en_port_profile {
378	u32 flags;
379	u32 tx_ring_num[MLX4_EN_NUM_TX_TYPES];
380	u32 rx_ring_num;
381	u32 tx_ring_size;
382	u32 rx_ring_size;
383	u8 num_tx_rings_p_up;
384	u8 rx_pause;
385	u8 rx_ppp;
386	u8 tx_pause;
387	u8 tx_ppp;
388	u8 num_up;
389	int rss_rings;
390	int inline_thold;
391	struct kernel_hwtstamp_config hwtstamp_config;
392};
393
394struct mlx4_en_profile {
395	int udp_rss;
396	u8 rss_mask;
397	u32 active_ports;
398	u32 small_pkt_int;
399	u8 no_reset;
400	u8 max_num_tx_rings_p_up;
401	struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1];
402};
403
404struct mlx4_en_dev {
405	struct mlx4_dev         *dev;
406	struct pci_dev		*pdev;
407	struct mutex		state_lock;
408	struct net_device       *pndev[MLX4_MAX_PORTS + 1];
409	struct net_device       *upper[MLX4_MAX_PORTS + 1];
410	u32                     port_cnt;
411	bool			device_up;
412	struct mlx4_en_profile  profile;
413	u32			LSO_support;
414	struct workqueue_struct *workqueue;
415	struct device           *dma_device;
416	void __iomem            *uar_map;
417	struct mlx4_uar         priv_uar;
418	struct mlx4_mr		mr;
419	u32                     priv_pdn;
420	spinlock_t              uar_lock;
421	u8			mac_removed[MLX4_MAX_PORTS + 1];
422	u32			nominal_c_mult;
423	struct cyclecounter	cycles;
424	seqlock_t		clock_lock;
425	struct timecounter	clock;
426	unsigned long		last_overflow_check;
427	struct ptp_clock	*ptp_clock;
428	struct ptp_clock_info	ptp_clock_info;
429	struct notifier_block	netdev_nb;
430	struct notifier_block	mlx_nb;
431};
432
433
434struct mlx4_en_rss_map {
435	int base_qpn;
436	struct mlx4_qp qps[MAX_RX_RINGS];
437	enum mlx4_qp_state state[MAX_RX_RINGS];
438	struct mlx4_qp *indir_qp;
439	enum mlx4_qp_state indir_state;
440};
441
442enum mlx4_en_port_flag {
443	MLX4_EN_PORT_ANC = 1<<0, /* Auto-negotiation complete */
444	MLX4_EN_PORT_ANE = 1<<1, /* Auto-negotiation enabled */
445};
446
447struct mlx4_en_port_state {
448	int link_state;
449	int link_speed;
450	int transceiver;
451	u32 flags;
452};
453
454enum mlx4_en_mclist_act {
455	MCLIST_NONE,
456	MCLIST_REM,
457	MCLIST_ADD,
458};
459
460struct mlx4_en_mc_list {
461	struct list_head	list;
462	enum mlx4_en_mclist_act	action;
463	u8			addr[ETH_ALEN];
464	u64			reg_id;
465	u64			tunnel_reg_id;
466};
467
468struct mlx4_en_frag_info {
469	u16 frag_size;
470	u32 frag_stride;
471};
472
473#ifdef CONFIG_MLX4_EN_DCB
474/* Minimal TC BW - setting to 0 will block traffic */
475#define MLX4_EN_BW_MIN 1
476#define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */
477
478#define MLX4_EN_TC_VENDOR 0
479#define MLX4_EN_TC_ETS 7
480
481enum dcb_pfc_type {
482	pfc_disabled = 0,
483	pfc_enabled_full,
484	pfc_enabled_tx,
485	pfc_enabled_rx
486};
487
488struct mlx4_en_cee_config {
489	bool	pfc_state;
490	enum	dcb_pfc_type dcb_pfc[MLX4_EN_NUM_UP_HIGH];
491};
492#endif
493
494struct ethtool_flow_id {
495	struct list_head list;
496	struct ethtool_rx_flow_spec flow_spec;
497	u64 id;
498};
499
500enum {
501	MLX4_EN_FLAG_PROMISC		= (1 << 0),
502	MLX4_EN_FLAG_MC_PROMISC		= (1 << 1),
503	/* whether we need to enable hardware loopback by putting dmac
504	 * in Tx WQE
505	 */
506	MLX4_EN_FLAG_ENABLE_HW_LOOPBACK	= (1 << 2),
507	/* whether we need to drop packets that hardware loopback-ed */
508	MLX4_EN_FLAG_RX_FILTER_NEEDED	= (1 << 3),
509	MLX4_EN_FLAG_FORCE_PROMISC	= (1 << 4),
510	MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP	= (1 << 5),
511#ifdef CONFIG_MLX4_EN_DCB
512	MLX4_EN_FLAG_DCB_ENABLED        = (1 << 6),
513#endif
514};
515
516#define PORT_BEACON_MAX_LIMIT (65535)
517#define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE)
518#define MLX4_EN_MAC_HASH_IDX 5
519
520struct mlx4_en_stats_bitmap {
521	DECLARE_BITMAP(bitmap, NUM_ALL_STATS);
522	struct mutex mutex; /* for mutual access to stats bitmap */
523};
524
525enum {
526	MLX4_EN_STATE_FLAG_RESTARTING,
527};
528
529struct mlx4_en_priv {
530	struct mlx4_en_dev *mdev;
531	struct mlx4_en_port_profile *prof;
532	struct net_device *dev;
533	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
534	struct mlx4_en_port_state port_state;
535	spinlock_t stats_lock;
536	struct ethtool_flow_id ethtool_rules[MAX_NUM_OF_FS_RULES];
537	/* To allow rules removal while port is going down */
538	struct list_head ethtool_list;
539
540	unsigned long last_moder_packets[MAX_RX_RINGS];
541	unsigned long last_moder_tx_packets;
542	unsigned long last_moder_bytes[MAX_RX_RINGS];
543	unsigned long last_moder_jiffies;
544	int last_moder_time[MAX_RX_RINGS];
545	u16 rx_usecs;
546	u16 rx_frames;
547	u16 tx_usecs;
548	u16 tx_frames;
549	u32 pkt_rate_low;
550	u16 rx_usecs_low;
551	u32 pkt_rate_high;
552	u16 rx_usecs_high;
553	u32 sample_interval;
554	u32 adaptive_rx_coal;
555	u32 msg_enable;
556	u32 loopback_ok;
557	u32 validate_loopback;
558
559	struct mlx4_hwq_resources res;
560	int link_state;
561	bool port_up;
562	int port;
563	int registered;
564	int allocated;
565	int stride;
566	unsigned char current_mac[ETH_ALEN + 2];
567	int mac_index;
568	unsigned max_mtu;
569	int base_qpn;
570	int cqe_factor;
571	int cqe_size;
572
573	struct mlx4_en_rss_map rss_map;
574	__be32 ctrl_flags;
575	u32 flags;
576	u8 num_tx_rings_p_up;
577	u32 tx_work_limit;
578	u32 tx_ring_num[MLX4_EN_NUM_TX_TYPES];
579	u32 rx_ring_num;
580	u32 rx_skb_size;
581	struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS];
582	u8 num_frags;
583	u8 log_rx_info;
584	u8 dma_dir;
585	u16 rx_headroom;
586
587	struct mlx4_en_tx_ring **tx_ring[MLX4_EN_NUM_TX_TYPES];
588	struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS];
589	struct mlx4_en_cq **tx_cq[MLX4_EN_NUM_TX_TYPES];
590	struct mlx4_en_cq *rx_cq[MAX_RX_RINGS];
591	struct mlx4_qp drop_qp;
592	struct work_struct rx_mode_task;
593	struct work_struct restart_task;
594	struct work_struct linkstate_task;
595	struct delayed_work stats_task;
596	struct delayed_work service_task;
597	struct mlx4_en_pkt_stats pkstats;
598	struct mlx4_en_counter_stats pf_stats;
599	struct mlx4_en_flow_stats_rx rx_priority_flowstats[MLX4_NUM_PRIORITIES];
600	struct mlx4_en_flow_stats_tx tx_priority_flowstats[MLX4_NUM_PRIORITIES];
601	struct mlx4_en_flow_stats_rx rx_flowstats;
602	struct mlx4_en_flow_stats_tx tx_flowstats;
603	struct mlx4_en_port_stats port_stats;
604	struct mlx4_en_xdp_stats xdp_stats;
605	struct mlx4_en_phy_stats phy_stats;
606	struct mlx4_en_stats_bitmap stats_bitmap;
607	struct list_head mc_list;
608	struct list_head curr_list;
609	u64 broadcast_id;
610	struct mlx4_en_stat_out_mbox hw_stats;
611	int vids[128];
612	bool wol;
613	struct device *ddev;
614	struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE];
615	struct kernel_hwtstamp_config hwtstamp_config;
616	u32 counter_index;
617
618#ifdef CONFIG_MLX4_EN_DCB
619#define MLX4_EN_DCB_ENABLED	0x3
620	struct ieee_ets ets;
621	u16 maxrate[IEEE_8021QAZ_MAX_TCS];
622	enum dcbnl_cndd_states cndd_state[IEEE_8021QAZ_MAX_TCS];
623	struct mlx4_en_cee_config cee_config;
624	u8 dcbx_cap;
625#endif
626#ifdef CONFIG_RFS_ACCEL
627	spinlock_t filters_lock;
628	int last_filter_id;
629	struct list_head filters;
630	struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT];
631#endif
632	u64 tunnel_reg_id;
633	__be16 vxlan_port;
634
635	u32 pflags;
636	u8 rss_key[MLX4_EN_RSS_KEY_SIZE];
637	u8 rss_hash_fn;
638	unsigned long state;
639};
640
641enum mlx4_en_wol {
642	MLX4_EN_WOL_MAGIC = (1ULL << 61),
643	MLX4_EN_WOL_ENABLED = (1ULL << 62),
644};
645
646struct mlx4_mac_entry {
647	struct hlist_node hlist;
648	unsigned char mac[ETH_ALEN + 2];
649	u64 reg_id;
650	struct rcu_head rcu;
651};
652
653static inline struct mlx4_cqe *mlx4_en_get_cqe(void *buf, int idx, int cqe_sz)
654{
655	return buf + idx * cqe_sz;
656}
657
658#define MLX4_EN_WOL_DO_MODIFY (1ULL << 63)
659
660void mlx4_en_init_ptys2ethtool_map(void);
661void mlx4_en_update_loopback_state(struct net_device *dev,
662				   netdev_features_t features);
663
664void mlx4_en_destroy_netdev(struct net_device *dev);
665int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
666			struct mlx4_en_port_profile *prof);
667
668int mlx4_en_start_port(struct net_device *dev);
669void mlx4_en_stop_port(struct net_device *dev, int detach);
670
671void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev,
672			      struct mlx4_en_stats_bitmap *stats_bitmap,
673			      u8 rx_ppp, u8 rx_pause,
674			      u8 tx_ppp, u8 tx_pause);
675
676int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv,
677				struct mlx4_en_priv *tmp,
678				struct mlx4_en_port_profile *prof,
679				bool carry_xdp_prog);
680void mlx4_en_safe_replace_resources(struct mlx4_en_priv *priv,
681				    struct mlx4_en_priv *tmp);
682
683int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq,
684		      int entries, int ring, enum cq_type mode, int node);
685void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq);
686int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
687			int cq_idx);
688void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
689int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
690void mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
691
692void mlx4_en_tx_irq(struct mlx4_cq *mcq);
693u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
694			 struct net_device *sb_dev);
695netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev);
696netdev_tx_t mlx4_en_xmit_frame(struct mlx4_en_rx_ring *rx_ring,
697			       struct mlx4_en_rx_alloc *frame,
698			       struct mlx4_en_priv *priv, unsigned int length,
699			       int tx_ind, bool *doorbell_pending);
700void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring);
701
702int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
703			   struct mlx4_en_tx_ring **pring,
704			   u32 size, u16 stride,
705			   int node, int queue_index);
706void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv,
707			     struct mlx4_en_tx_ring **pring);
708void mlx4_en_init_tx_xdp_ring_descs(struct mlx4_en_priv *priv,
709				    struct mlx4_en_tx_ring *ring);
710int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv,
711			     struct mlx4_en_tx_ring *ring,
712			     int cq, int user_prio);
713void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv,
714				struct mlx4_en_tx_ring *ring);
715void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev);
716void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv);
717int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
718			   struct mlx4_en_rx_ring **pring,
719			   u32 size, u16 stride, int node, int queue_index);
720void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
721			     struct mlx4_en_rx_ring **pring,
722			     u32 size, u16 stride);
723int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv);
724void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
725				struct mlx4_en_rx_ring *ring);
726int mlx4_en_process_rx_cq(struct net_device *dev,
727			  struct mlx4_en_cq *cq,
728			  int budget);
729int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget);
730int mlx4_en_poll_tx_cq(struct napi_struct *napi, int budget);
731int mlx4_en_process_tx_cq(struct net_device *dev,
732			  struct mlx4_en_cq *cq, int napi_budget);
733u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv,
734			 struct mlx4_en_tx_ring *ring,
735			 int index, u64 timestamp,
736			 int napi_mode);
737u32 mlx4_en_recycle_tx_desc(struct mlx4_en_priv *priv,
738			    struct mlx4_en_tx_ring *ring,
739			    int index, u64 timestamp,
740			    int napi_mode);
741void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
742		int is_tx, int rss, int qpn, int cqn, int user_prio,
743		struct mlx4_qp_context *context);
744void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event);
745int mlx4_en_change_mcast_lb(struct mlx4_en_priv *priv, struct mlx4_qp *qp,
746			    int loopback);
747void mlx4_en_calc_rx_buf(struct net_device *dev);
748int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv);
749void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv);
750int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv);
751void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv);
752int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring);
753void mlx4_en_rx_irq(struct mlx4_cq *mcq);
754
755int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode);
756int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv);
757
758void mlx4_en_fold_software_stats(struct net_device *dev);
759int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset);
760int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port);
761
762#ifdef CONFIG_MLX4_EN_DCB
763extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops;
764extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops;
765#endif
766
767int mlx4_en_setup_tc(struct net_device *dev, u8 up);
768int mlx4_en_alloc_tx_queue_per_tc(struct net_device *dev, u8 tc);
769
770#ifdef CONFIG_RFS_ACCEL
771void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv);
772#endif
773
774#define MLX4_EN_NUM_SELF_TEST	5
775void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf);
776void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev);
777
778#define DEV_FEATURE_CHANGED(dev, new_features, feature) \
779	((dev->features & feature) ^ (new_features & feature))
780
781int mlx4_en_moderation_update(struct mlx4_en_priv *priv);
782int mlx4_en_reset_config(struct net_device *dev,
783			 struct kernel_hwtstamp_config *ts_config,
784			 netdev_features_t new_features);
785void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev,
786				     struct mlx4_en_stats_bitmap *stats_bitmap,
787				     u8 rx_ppp, u8 rx_pause,
788				     u8 tx_ppp, u8 tx_pause);
789int mlx4_en_netdev_event(struct notifier_block *this,
790			 unsigned long event, void *ptr);
791
792struct xdp_md;
793int mlx4_en_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp);
794int mlx4_en_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash,
795			enum xdp_rss_hash_type *rss_type);
796
797/*
798 * Functions for time stamping
799 */
800u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe);
801u64 mlx4_en_get_hwtstamp(struct mlx4_en_dev *mdev, u64 timestamp);
802void mlx4_en_fill_hwtstamps(struct mlx4_en_dev *mdev,
803			    struct skb_shared_hwtstamps *hwts,
804			    u64 timestamp);
805void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev);
806void mlx4_en_remove_timestamp(struct mlx4_en_dev *mdev);
807
808/* Globals
809 */
810extern const struct ethtool_ops mlx4_en_ethtool_ops;
811
812
813
814/*
815 * printk / logging functions
816 */
817
818__printf(3, 4)
819void en_print(const char *level, const struct mlx4_en_priv *priv,
820	      const char *format, ...);
821
822#define en_dbg(mlevel, priv, format, ...)				\
823do {									\
824	if (NETIF_MSG_##mlevel & (priv)->msg_enable)			\
825		en_print(KERN_DEBUG, priv, format, ##__VA_ARGS__);	\
826} while (0)
827#define en_warn(priv, format, ...)					\
828	en_print(KERN_WARNING, priv, format, ##__VA_ARGS__)
829#define en_err(priv, format, ...)					\
830	en_print(KERN_ERR, priv, format, ##__VA_ARGS__)
831#define en_info(priv, format, ...)					\
832	en_print(KERN_INFO, priv, format, ##__VA_ARGS__)
833
834#define mlx4_err(mdev, format, ...)					\
835	pr_err(DRV_NAME " %s: " format,					\
836	       dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
837#define mlx4_info(mdev, format, ...)					\
838	pr_info(DRV_NAME " %s: " format,				\
839		dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
840#define mlx4_warn(mdev, format, ...)					\
841	pr_warn(DRV_NAME " %s: " format,				\
842		dev_name(&(mdev)->pdev->dev), ##__VA_ARGS__)
843
844#endif