ice_txrx.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* Copyright (c) 2018, Intel Corporation. */
  3
  4#ifndef _ICE_TXRX_H_
  5#define _ICE_TXRX_H_
  6
  7#include <net/libeth/types.h>
  8
  9#include "ice_type.h"
 10
 11#define ICE_DFLT_IRQ_WORK	256
 12#define ICE_RXBUF_3072		3072
 13#define ICE_RXBUF_2048		2048
 14#define ICE_RXBUF_1664		1664
 15#define ICE_RXBUF_1536		1536
 16#define ICE_MAX_CHAINED_RX_BUFS	5
 17#define ICE_MAX_BUF_TXD		8
 18#define ICE_MIN_TX_LEN		17
 19#define ICE_MAX_FRAME_LEGACY_RX 8320
 20
 21/* The size limit for a transmit buffer in a descriptor is (16K - 1).
 22 * In order to align with the read requests we will align the value to
 23 * the nearest 4K which represents our maximum read request size.
 24 */
 25#define ICE_MAX_READ_REQ_SIZE	4096
 26#define ICE_MAX_DATA_PER_TXD	(16 * 1024 - 1)
 27#define ICE_MAX_DATA_PER_TXD_ALIGNED \
 28	(~(ICE_MAX_READ_REQ_SIZE - 1) & ICE_MAX_DATA_PER_TXD)
 29
 30#define ICE_MAX_TXQ_PER_TXQG	128
 31
 32/* We are assuming that the cache line is always 64 Bytes here for ice.
 33 * In order to make sure that is a correct assumption there is a check in probe
 34 * to print a warning if the read from GLPCI_CNF2 tells us that the cache line
 35 * size is 128 bytes. We do it this way because we do not want to read the
 36 * GLPCI_CNF2 register or a variable containing the value on every pass through
 37 * the Tx path.
 38 */
 39#define ICE_CACHE_LINE_BYTES		64
 40#define ICE_DESCS_PER_CACHE_LINE	(ICE_CACHE_LINE_BYTES / \
 41					 sizeof(struct ice_tx_desc))
 42#define ICE_DESCS_FOR_CTX_DESC		1
 43#define ICE_DESCS_FOR_SKB_DATA_PTR	1
 44/* Tx descriptors needed, worst case */
 45#define DESC_NEEDED (MAX_SKB_FRAGS + ICE_DESCS_FOR_CTX_DESC + \
 46		     ICE_DESCS_PER_CACHE_LINE + ICE_DESCS_FOR_SKB_DATA_PTR)
 47#define ICE_DESC_UNUSED(R)	\
 48	(u16)((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->count) + \
 49	      (R)->next_to_clean - (R)->next_to_use - 1)
 50
 51#define ICE_RING_QUARTER(R) ((R)->count >> 2)
 52
 53#define ICE_TX_FLAGS_TSO	BIT(0)
 54#define ICE_TX_FLAGS_HW_VLAN	BIT(1)
 55#define ICE_TX_FLAGS_SW_VLAN	BIT(2)
 56/* Free, was ICE_TX_FLAGS_DUMMY_PKT */
 57#define ICE_TX_FLAGS_TSYN	BIT(4)
 58#define ICE_TX_FLAGS_IPV4	BIT(5)
 59#define ICE_TX_FLAGS_IPV6	BIT(6)
 60#define ICE_TX_FLAGS_TUNNEL	BIT(7)
 61#define ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN	BIT(8)
 62
 63#define ICE_XDP_PASS		0
 64#define ICE_XDP_CONSUMED	BIT(0)
 65#define ICE_XDP_TX		BIT(1)
 66#define ICE_XDP_REDIR		BIT(2)
 67#define ICE_XDP_EXIT		BIT(3)
 68#define ICE_SKB_CONSUMED	ICE_XDP_CONSUMED
 69
 70#define ICE_RX_DMA_ATTR \
 71	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
 72
 73#define ICE_ETH_PKT_HDR_PAD	(ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))
 74
 75#define ICE_TXD_LAST_DESC_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS)
 76
 77/**
 78 * enum ice_tx_buf_type - type of &ice_tx_buf to act on Tx completion
 79 * @ICE_TX_BUF_EMPTY: unused OR XSk frame, no action required
 80 * @ICE_TX_BUF_DUMMY: dummy Flow Director packet, unmap and kfree()
 81 * @ICE_TX_BUF_FRAG: mapped skb OR &xdp_buff frag, only unmap DMA
 82 * @ICE_TX_BUF_SKB: &sk_buff, unmap and consume_skb(), update stats
 83 * @ICE_TX_BUF_XDP_TX: &xdp_buff, unmap and page_frag_free(), stats
 84 * @ICE_TX_BUF_XDP_XMIT: &xdp_frame, unmap and xdp_return_frame(), stats
 85 * @ICE_TX_BUF_XSK_TX: &xdp_buff on XSk queue, xsk_buff_free(), stats
 86 */
 87enum ice_tx_buf_type {
 88	ICE_TX_BUF_EMPTY	= 0U,
 89	ICE_TX_BUF_DUMMY,
 90	ICE_TX_BUF_FRAG,
 91	ICE_TX_BUF_SKB,
 92	ICE_TX_BUF_XDP_TX,
 93	ICE_TX_BUF_XDP_XMIT,
 94	ICE_TX_BUF_XSK_TX,
 95};
 96
 97struct ice_tx_buf {
 98	union {
 99		struct ice_tx_desc *next_to_watch;
100		u32 rs_idx;
101	};
102	union {
103		void *raw_buf;		/* used for XDP_TX and FDir rules */
104		struct sk_buff *skb;	/* used for .ndo_start_xmit() */
105		struct xdp_frame *xdpf;	/* used for .ndo_xdp_xmit() */
106		struct xdp_buff *xdp;	/* used for XDP_TX ZC */
107	};
108	unsigned int bytecount;
109	union {
110		unsigned int gso_segs;
111		unsigned int nr_frags;	/* used for mbuf XDP */
112	};
113	u32 tx_flags:12;
114	u32 type:4;			/* &ice_tx_buf_type */
115	u32 vid:16;
116	DEFINE_DMA_UNMAP_LEN(len);
117	DEFINE_DMA_UNMAP_ADDR(dma);
118};
119
120struct ice_tx_offload_params {
121	u64 cd_qw1;
122	struct ice_tx_ring *tx_ring;
123	u32 td_cmd;
124	u32 td_offset;
125	u32 td_l2tag1;
126	u32 cd_tunnel_params;
127	u16 cd_l2tag2;
128	u16 cd_gcs_params;
129	u8 header_len;
130};
131
132struct ice_q_stats {
133	u64 pkts;
134	u64 bytes;
135};
136
137struct ice_txq_stats {
138	u64 restart_q;
139	u64 tx_busy;
140	u64 tx_linearize;
141	int prev_pkt; /* negative if no pending Tx descriptors */
142};
143
144struct ice_rxq_stats {
145	u64 non_eop_descs;
146	u64 alloc_page_failed;
147	u64 alloc_buf_failed;
148};
149
150struct ice_ring_stats {
151	struct rcu_head rcu;	/* to avoid race on free */
152	struct ice_q_stats stats;
153	struct u64_stats_sync syncp;
154	union {
155		struct ice_txq_stats tx_stats;
156		struct ice_rxq_stats rx_stats;
157	};
158};
159
160enum ice_ring_state_t {
161	ICE_TX_XPS_INIT_DONE,
162	ICE_TX_NBITS,
163};
164
165/* this enum matches hardware bits and is meant to be used by DYN_CTLN
166 * registers and QINT registers or more generally anywhere in the manual
167 * mentioning ITR_INDX, ITR_NONE cannot be used as an index 'n' into any
168 * register but instead is a special value meaning "don't update" ITR0/1/2.
169 */
170enum ice_dyn_idx_t {
171	ICE_IDX_ITR0 = 0,
172	ICE_IDX_ITR1 = 1,
173	ICE_IDX_ITR2 = 2,
174	ICE_ITR_NONE = 3	/* ITR_NONE must not be used as an index */
175};
176
177/* Header split modes defined by DTYPE field of Rx RLAN context */
178enum ice_rx_dtype {
179	ICE_RX_DTYPE_NO_SPLIT		= 0,
180	ICE_RX_DTYPE_HEADER_SPLIT	= 1,
181	ICE_RX_DTYPE_SPLIT_ALWAYS	= 2,
182};
183
184struct ice_pkt_ctx {
185	u64 cached_phctime;
186	__be16 vlan_proto;
187};
188
189/* indices into GLINT_ITR registers */
190#define ICE_RX_ITR	ICE_IDX_ITR0
191#define ICE_TX_ITR	ICE_IDX_ITR1
192#define ICE_ITR_8K	124
193#define ICE_ITR_20K	50
194#define ICE_ITR_MAX	8160 /* 0x1FE0 */
195#define ICE_DFLT_TX_ITR	ICE_ITR_20K
196#define ICE_DFLT_RX_ITR	ICE_ITR_20K
197enum ice_dynamic_itr {
198	ITR_STATIC = 0,
199	ITR_DYNAMIC = 1
200};
201
202#define ITR_IS_DYNAMIC(rc) ((rc)->itr_mode == ITR_DYNAMIC)
203#define ICE_ITR_GRAN_S		1	/* ITR granularity is always 2us */
204#define ICE_ITR_GRAN_US		BIT(ICE_ITR_GRAN_S)
205#define ICE_ITR_MASK		0x1FFE	/* ITR register value alignment mask */
206#define ITR_REG_ALIGN(setting)	((setting) & ICE_ITR_MASK)
207
208#define ICE_DFLT_INTRL	0
209#define ICE_MAX_INTRL	236
210
211#define ICE_IN_WB_ON_ITR_MODE	255
212/* Sets WB_ON_ITR and assumes INTENA bit is already cleared, which allows
213 * setting the MSK_M bit to tell hardware to ignore the INTENA_M bit. Also,
214 * set the write-back latency to the usecs passed in.
215 */
216#define ICE_GLINT_DYN_CTL_WB_ON_ITR(usecs, itr_idx)	\
217	((((usecs) << (GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S)) & \
218	  GLINT_DYN_CTL_INTERVAL_M) | \
219	 (((itr_idx) << GLINT_DYN_CTL_ITR_INDX_S) & \
220	  GLINT_DYN_CTL_ITR_INDX_M) | GLINT_DYN_CTL_INTENA_MSK_M | \
221	 GLINT_DYN_CTL_WB_ON_ITR_M)
222
223/* Legacy or Advanced Mode Queue */
224#define ICE_TX_ADVANCED	0
225#define ICE_TX_LEGACY	1
226
227/* descriptor ring, associated with a VSI */
228struct ice_tstamp_ring {
229	struct ice_tx_ring *tx_ring;	/* Backreference to associated Tx ring */
230	dma_addr_t dma;			/* physical address of ring */
231	struct rcu_head rcu;            /* to avoid race on free */
232	u8 __iomem *tail;
233	void *desc;
234	u16 next_to_use;
235	u16 count;
236} ____cacheline_internodealigned_in_smp;
237
238struct ice_rx_ring {
239	/* CL1 - 1st cacheline starts here */
240	void *desc;			/* Descriptor ring memory */
241	struct page_pool *pp;
242	struct net_device *netdev;	/* netdev ring maps to */
243	struct ice_vsi *vsi;		/* Backreference to associated VSI */
244	struct ice_q_vector *q_vector;	/* Backreference to associated vector */
245	u8 __iomem *tail;
246	u16 q_index;			/* Queue number of ring */
247
248	u16 count;			/* Number of descriptors */
249	u16 reg_idx;			/* HW register index of the ring */
250	u16 next_to_alloc;
251
252	union {
253		struct libeth_fqe *rx_fqes;
254		struct xdp_buff **xdp_buf;
255	};
256
257	/* CL2 - 2nd cacheline starts here */
258	struct libeth_fqe *hdr_fqes;
259	struct page_pool *hdr_pp;
260
261	union {
262		struct libeth_xdp_buff_stash xdp;
263		struct libeth_xdp_buff *xsk;
264	};
265
266	/* CL3 - 3rd cacheline starts here */
267	union {
268		struct ice_pkt_ctx pkt_ctx;
269		struct {
270			u64 cached_phctime;
271			__be16 vlan_proto;
272		};
273	};
274	struct bpf_prog *xdp_prog;
275
276	/* used in interrupt processing */
277	u16 next_to_use;
278	u16 next_to_clean;
279
280	u32 hdr_truesize;
281	u32 truesize;
282
283	/* stats structs */
284	struct ice_ring_stats *ring_stats;
285
286	struct rcu_head rcu;		/* to avoid race on free */
287	/* CL4 - 4th cacheline starts here */
288	struct ice_channel *ch;
289	struct ice_tx_ring *xdp_ring;
290	struct ice_rx_ring *next;	/* pointer to next ring in q_vector */
291	struct xsk_buff_pool *xsk_pool;
292	u16 rx_hdr_len;
293	u16 rx_buf_len;
294	dma_addr_t dma;			/* physical address of ring */
295	u8 dcb_tc;			/* Traffic class of ring */
296	u8 ptp_rx;
297#define ICE_RX_FLAGS_CRC_STRIP_DIS	BIT(2)
298#define ICE_RX_FLAGS_MULTIDEV		BIT(3)
299#define ICE_RX_FLAGS_RING_GCS		BIT(4)
300	u8 flags;
301	/* CL5 - 5th cacheline starts here */
302	struct xdp_rxq_info xdp_rxq;
303} ____cacheline_internodealigned_in_smp;
304
305struct ice_tx_ring {
306	/* CL1 - 1st cacheline starts here */
307	struct ice_tx_ring *next;	/* pointer to next ring in q_vector */
308	void *desc;			/* Descriptor ring memory */
309	struct device *dev;		/* Used for DMA mapping */
310	u8 __iomem *tail;
311	struct ice_tx_buf *tx_buf;
312	struct ice_q_vector *q_vector;	/* Backreference to associated vector */
313	struct net_device *netdev;	/* netdev ring maps to */
314	struct ice_vsi *vsi;		/* Backreference to associated VSI */
315	/* CL2 - 2nd cacheline starts here */
316	dma_addr_t dma;			/* physical address of ring */
317	struct xsk_buff_pool *xsk_pool;
318	u16 next_to_use;
319	u16 next_to_clean;
320	u16 q_handle;			/* Queue handle per TC */
321	u16 reg_idx;			/* HW register index of the ring */
322	u16 count;			/* Number of descriptors */
323	u16 q_index;			/* Queue number of ring */
324	u16 xdp_tx_active;
325	/* stats structs */
326	struct ice_ring_stats *ring_stats;
327	/* CL3 - 3rd cacheline starts here */
328	struct rcu_head rcu;		/* to avoid race on free */
329	DECLARE_BITMAP(xps_state, ICE_TX_NBITS);	/* XPS Config State */
330	struct ice_channel *ch;
331	struct ice_ptp_tx *tx_tstamps;
332	spinlock_t tx_lock;
333	u32 txq_teid;			/* Added Tx queue TEID */
334	/* CL4 - 4th cacheline starts here */
335	struct ice_tstamp_ring *tstamp_ring;
336#define ICE_TX_FLAGS_RING_XDP		BIT(0)
337#define ICE_TX_FLAGS_RING_VLAN_L2TAG1	BIT(1)
338#define ICE_TX_FLAGS_RING_VLAN_L2TAG2	BIT(2)
339#define ICE_TX_FLAGS_TXTIME		BIT(3)
340	u8 flags;
341	u8 dcb_tc;			/* Traffic class of ring */
342	u16 quanta_prof_id;
343} ____cacheline_internodealigned_in_smp;
344
345static inline bool ice_ring_ch_enabled(struct ice_tx_ring *ring)
346{
347	return !!ring->ch;
348}
349
350static inline bool ice_ring_is_xdp(struct ice_tx_ring *ring)
351{
352	return !!(ring->flags & ICE_TX_FLAGS_RING_XDP);
353}
354
355enum ice_container_type {
356	ICE_RX_CONTAINER,
357	ICE_TX_CONTAINER,
358};
359
360struct ice_ring_container {
361	/* head of linked-list of rings */
362	union {
363		struct ice_rx_ring *rx_ring;
364		struct ice_tx_ring *tx_ring;
365	};
366	struct dim dim;		/* data for net_dim algorithm */
367	u16 itr_idx;		/* index in the interrupt vector */
368	/* this matches the maximum number of ITR bits, but in usec
369	 * values, so it is shifted left one bit (bit zero is ignored)
370	 */
371	union {
372		struct {
373			u16 itr_setting:13;
374			u16 itr_reserved:2;
375			u16 itr_mode:1;
376		};
377		u16 itr_settings;
378	};
379	enum ice_container_type type;
380};
381
382struct ice_coalesce_stored {
383	u16 itr_tx;
384	u16 itr_rx;
385	u8 intrl;
386	u8 tx_valid;
387	u8 rx_valid;
388};
389
390/* iterator for handling rings in ring container */
391#define ice_for_each_rx_ring(pos, head) \
392	for (pos = (head).rx_ring; pos; pos = pos->next)
393
394#define ice_for_each_tx_ring(pos, head) \
395	for (pos = (head).tx_ring; pos; pos = pos->next)
396
397static inline unsigned int ice_rx_pg_order(struct ice_rx_ring *ring)
398{
399	return 0;
400}
401
402union ice_32b_rx_flex_desc;
403
404void ice_init_ctrl_rx_descs(struct ice_rx_ring *rx_ring, u32 num_descs);
405void ice_rxq_pp_destroy(struct ice_rx_ring *rq);
406bool ice_alloc_rx_bufs(struct ice_rx_ring *rxr, unsigned int cleaned_count);
407netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev);
408u16
409ice_select_queue(struct net_device *dev, struct sk_buff *skb,
410		 struct net_device *sb_dev);
411void ice_clean_tx_ring(struct ice_tx_ring *tx_ring);
412void ice_clean_rx_ring(struct ice_rx_ring *rx_ring);
413int ice_setup_tx_ring(struct ice_tx_ring *tx_ring);
414int ice_setup_rx_ring(struct ice_rx_ring *rx_ring);
415int ice_alloc_setup_tstamp_ring(struct ice_tx_ring *tx_ring);
416void ice_free_tx_ring(struct ice_tx_ring *tx_ring);
417void ice_free_rx_ring(struct ice_rx_ring *rx_ring);
418int ice_napi_poll(struct napi_struct *napi, int budget);
419int
420ice_prgm_fdir_fltr(struct ice_vsi *vsi, struct ice_fltr_desc *fdir_desc,
421		   u8 *raw_packet);
422void ice_clean_ctrl_tx_irq(struct ice_tx_ring *tx_ring);
423void ice_clean_ctrl_rx_irq(struct ice_rx_ring *rx_ring);
424void ice_free_tx_tstamp_ring(struct ice_tx_ring *tx_ring);
425void ice_free_tstamp_ring(struct ice_tx_ring *tx_ring);
426#endif /* _ICE_TXRX_H_ */