enic_rq.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2// Copyright 2024 Cisco Systems, Inc.  All rights reserved.
  3
  4#include <linux/skbuff.h>
  5#include <linux/if_vlan.h>
  6#include <net/busy_poll.h>
  7#include "enic.h"
  8#include "enic_res.h"
  9#include "enic_rq.h"
 10#include "vnic_rq.h"
 11#include "cq_enet_desc.h"
 12
 13#define ENIC_LARGE_PKT_THRESHOLD                1000
 14
 15static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size,
 16				      u32 pkt_len)
 17{
 18	if (pkt_len > ENIC_LARGE_PKT_THRESHOLD)
 19		pkt_size->large_pkt_bytes_cnt += pkt_len;
 20	else
 21		pkt_size->small_pkt_bytes_cnt += pkt_len;
 22}
 23
 24static void enic_rq_cq_desc_dec(void *cq_desc, u8 cq_desc_size, u8 *type,
 25				u8 *color, u16 *q_number, u16 *completed_index)
 26{
 27	/* type_color is the last field for all cq structs */
 28	u8 type_color;
 29
 30	switch (cq_desc_size) {
 31	case VNIC_RQ_CQ_ENTRY_SIZE_16: {
 32		struct cq_enet_rq_desc *desc =
 33			(struct cq_enet_rq_desc *)cq_desc;
 34		type_color = desc->type_color;
 35
 36		/* Make sure color bit is read from desc *before* other fields
 37		 * are read from desc.  Hardware guarantees color bit is last
 38		 * bit (byte) written.  Adding the rmb() prevents the compiler
 39		 * and/or CPU from reordering the reads which would potentially
 40		 * result in reading stale values.
 41		 */
 42		rmb();
 43
 44		*q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
 45			    CQ_DESC_Q_NUM_MASK;
 46		*completed_index = le16_to_cpu(desc->completed_index_flags) &
 47				   CQ_DESC_COMP_NDX_MASK;
 48		break;
 49	}
 50	case VNIC_RQ_CQ_ENTRY_SIZE_32: {
 51		struct cq_enet_rq_desc_32 *desc =
 52			(struct cq_enet_rq_desc_32 *)cq_desc;
 53		type_color = desc->type_color;
 54
 55		/* Make sure color bit is read from desc *before* other fields
 56		 * are read from desc.  Hardware guarantees color bit is last
 57		 * bit (byte) written.  Adding the rmb() prevents the compiler
 58		 * and/or CPU from reordering the reads which would potentially
 59		 * result in reading stale values.
 60		 */
 61		rmb();
 62
 63		*q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
 64			    CQ_DESC_Q_NUM_MASK;
 65		*completed_index = le16_to_cpu(desc->completed_index_flags) &
 66				   CQ_DESC_COMP_NDX_MASK;
 67		*completed_index |= (desc->fetch_index_flags & CQ_DESC_32_FI_MASK) <<
 68				CQ_DESC_COMP_NDX_BITS;
 69		break;
 70	}
 71	case VNIC_RQ_CQ_ENTRY_SIZE_64: {
 72		struct cq_enet_rq_desc_64 *desc =
 73			(struct cq_enet_rq_desc_64 *)cq_desc;
 74		type_color = desc->type_color;
 75
 76		/* Make sure color bit is read from desc *before* other fields
 77		 * are read from desc.  Hardware guarantees color bit is last
 78		 * bit (byte) written.  Adding the rmb() prevents the compiler
 79		 * and/or CPU from reordering the reads which would potentially
 80		 * result in reading stale values.
 81		 */
 82		rmb();
 83
 84		*q_number = le16_to_cpu(desc->q_number_rss_type_flags) &
 85			    CQ_DESC_Q_NUM_MASK;
 86		*completed_index = le16_to_cpu(desc->completed_index_flags) &
 87				   CQ_DESC_COMP_NDX_MASK;
 88		*completed_index |= (desc->fetch_index_flags & CQ_DESC_64_FI_MASK) <<
 89				CQ_DESC_COMP_NDX_BITS;
 90		break;
 91	}
 92	}
 93
 94	*color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
 95	*type = type_color & CQ_DESC_TYPE_MASK;
 96}
 97
 98static void enic_rq_set_skb_flags(struct vnic_rq *vrq, u8 type, u32 rss_hash,
 99				  u8 rss_type, u8 fcoe, u8 fcoe_fc_crc_ok,
100				  u8 vlan_stripped, u8 csum_not_calc,
101				  u8 tcp_udp_csum_ok, u8 ipv6, u8 ipv4_csum_ok,
102				  u16 vlan_tci, struct sk_buff *skb)
103{
104	struct enic *enic = vnic_dev_priv(vrq->vdev);
105	struct net_device *netdev = enic->netdev;
106	struct enic_rq_stats *rqstats =  &enic->rq[vrq->index].stats;
107	bool outer_csum_ok = true, encap = false;
108
109	if ((netdev->features & NETIF_F_RXHASH) && rss_hash && type == 3) {
110		switch (rss_type) {
111		case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4:
112		case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6:
113		case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX:
114			skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4);
115			rqstats->l4_rss_hash++;
116			break;
117		case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4:
118		case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6:
119		case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX:
120			skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3);
121			rqstats->l3_rss_hash++;
122			break;
123		}
124	}
125	if (enic->vxlan.vxlan_udp_port_number) {
126		switch (enic->vxlan.patch_level) {
127		case 0:
128			if (fcoe) {
129				encap = true;
130				outer_csum_ok = fcoe_fc_crc_ok;
131			}
132			break;
133		case 2:
134			if (type == 7 && (rss_hash & BIT(0))) {
135				encap = true;
136				outer_csum_ok = (rss_hash & BIT(1)) &&
137						(rss_hash & BIT(2));
138			}
139			break;
140		}
141	}
142
143	/* Hardware does not provide whole packet checksum. It only
144	 * provides pseudo checksum. Since hw validates the packet
145	 * checksum but not provide us the checksum value. use
146	 * CHECSUM_UNNECESSARY.
147	 *
148	 * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is
149	 * inner csum_ok. outer_csum_ok is set by hw when outer udp
150	 * csum is correct or is zero.
151	 */
152	if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc &&
153	    tcp_udp_csum_ok && outer_csum_ok && (ipv4_csum_ok || ipv6)) {
154		skb->ip_summed = CHECKSUM_UNNECESSARY;
155		skb->csum_level = encap;
156		if (encap)
157			rqstats->csum_unnecessary_encap++;
158		else
159			rqstats->csum_unnecessary++;
160	}
161
162	if (vlan_stripped) {
163		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
164		rqstats->vlan_stripped++;
165	}
166}
167
168/*
169 * cq_enet_rq_desc accesses section uses only the 1st 15 bytes of the cq which
170 * is identical for all type (16,32 and 64 byte) of cqs.
171 */
172static void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc, u8 *ingress_port,
173				u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type,
174				u8 *csum_not_calc, u32 *rss_hash,
175				u16 *bytes_written, u8 *packet_error,
176				u8 *vlan_stripped, u16 *vlan_tci,
177				u16 *checksum, u8 *fcoe_sof,
178				u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error,
179				u8 *fcoe_eof, u8 *tcp_udp_csum_ok, u8 *udp,
180				u8 *tcp, u8 *ipv4_csum_ok, u8 *ipv6, u8 *ipv4,
181				u8 *ipv4_fragment, u8 *fcs_ok)
182{
183	u16 completed_index_flags;
184	u16 q_number_rss_type_flags;
185	u16 bytes_written_flags;
186
187	completed_index_flags = le16_to_cpu(desc->completed_index_flags);
188	q_number_rss_type_flags =
189		le16_to_cpu(desc->q_number_rss_type_flags);
190	bytes_written_flags = le16_to_cpu(desc->bytes_written_flags);
191
192	*ingress_port = (completed_index_flags &
193		CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0;
194	*fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ?
195		1 : 0;
196	*eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ?
197		1 : 0;
198	*sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ?
199		1 : 0;
200
201	*rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) &
202		CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
203	*csum_not_calc = (q_number_rss_type_flags &
204		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0;
205
206	*rss_hash = le32_to_cpu(desc->rss_hash);
207
208	*bytes_written = bytes_written_flags &
209		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
210	*packet_error = (bytes_written_flags &
211		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0;
212	*vlan_stripped = (bytes_written_flags &
213		CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0;
214
215	/*
216	 * Tag Control Information(16) = user_priority(3) + cfi(1) + vlan(12)
217	 */
218	*vlan_tci = le16_to_cpu(desc->vlan);
219
220	if (*fcoe) {
221		*fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) &
222			CQ_ENET_RQ_DESC_FCOE_SOF_MASK);
223		*fcoe_fc_crc_ok = (desc->flags &
224			CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0;
225		*fcoe_enc_error = (desc->flags &
226			CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0;
227		*fcoe_eof = (u8)((le16_to_cpu(desc->checksum_fcoe) >>
228			CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) &
229			CQ_ENET_RQ_DESC_FCOE_EOF_MASK);
230		*checksum = 0;
231	} else {
232		*fcoe_sof = 0;
233		*fcoe_fc_crc_ok = 0;
234		*fcoe_enc_error = 0;
235		*fcoe_eof = 0;
236		*checksum = le16_to_cpu(desc->checksum_fcoe);
237	}
238
239	*tcp_udp_csum_ok =
240		(desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0;
241	*udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0;
242	*tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0;
243	*ipv4_csum_ok =
244		(desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0;
245	*ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0;
246	*ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0;
247	*ipv4_fragment =
248		(desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0;
249	*fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0;
250}
251
252static bool enic_rq_pkt_error(struct vnic_rq *vrq, u8 packet_error, u8 fcs_ok,
253			      u16 bytes_written)
254{
255	struct enic *enic = vnic_dev_priv(vrq->vdev);
256	struct enic_rq_stats *rqstats = &enic->rq[vrq->index].stats;
257
258	if (packet_error) {
259		if (!fcs_ok) {
260			if (bytes_written > 0)
261				rqstats->bad_fcs++;
262			else if (bytes_written == 0)
263				rqstats->pkt_truncated++;
264		}
265		return true;
266	}
267	return false;
268}
269
270int enic_rq_alloc_buf(struct vnic_rq *rq)
271{
272	struct enic *enic = vnic_dev_priv(rq->vdev);
273	struct net_device *netdev = enic->netdev;
274	struct enic_rq *erq = &enic->rq[rq->index];
275	struct enic_rq_stats *rqstats = &erq->stats;
276	unsigned int offset = 0;
277	unsigned int len = netdev->mtu + VLAN_ETH_HLEN;
278	unsigned int os_buf_index = 0;
279	dma_addr_t dma_addr;
280	struct vnic_rq_buf *buf = rq->to_use;
281	struct page *page;
282	unsigned int truesize = len;
283
284	if (buf->os_buf) {
285		enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr,
286				   buf->len);
287
288		return 0;
289	}
290
291	page = page_pool_dev_alloc(erq->pool, &offset, &truesize);
292	if (unlikely(!page)) {
293		rqstats->pp_alloc_fail++;
294		return -ENOMEM;
295	}
296	buf->offset = offset;
297	buf->truesize = truesize;
298	dma_addr = page_pool_get_dma_addr(page) + offset;
299	enic_queue_rq_desc(rq, (void *)page, os_buf_index, dma_addr, len);
300
301	return 0;
302}
303
304void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf)
305{
306	struct enic *enic = vnic_dev_priv(rq->vdev);
307	struct enic_rq *erq = &enic->rq[rq->index];
308
309	if (!buf->os_buf)
310		return;
311
312	page_pool_put_full_page(erq->pool, (struct page *)buf->os_buf, true);
313	buf->os_buf = NULL;
314}
315
316static void enic_rq_indicate_buf(struct enic *enic, struct vnic_rq *rq,
317				 struct vnic_rq_buf *buf, void *cq_desc,
318				 u8 type, u16 q_number, u16 completed_index)
319{
320	struct sk_buff *skb;
321	struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)];
322	struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats;
323	struct napi_struct *napi;
324
325	u8 eop, sop, ingress_port, vlan_stripped;
326	u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
327	u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
328	u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
329	u8 packet_error;
330	u16 bytes_written, vlan_tci, checksum;
331	u32 rss_hash;
332
333	rqstats->packets++;
334
335	cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, &ingress_port,
336			    &fcoe, &eop, &sop, &rss_type, &csum_not_calc,
337			    &rss_hash, &bytes_written, &packet_error,
338			    &vlan_stripped, &vlan_tci, &checksum, &fcoe_sof,
339			    &fcoe_fc_crc_ok, &fcoe_enc_error, &fcoe_eof,
340			    &tcp_udp_csum_ok, &udp, &tcp, &ipv4_csum_ok, &ipv6,
341			    &ipv4, &ipv4_fragment, &fcs_ok);
342
343	if (enic_rq_pkt_error(rq, packet_error, fcs_ok, bytes_written))
344		return;
345
346	if (eop && bytes_written > 0) {
347		/* Good receive
348		 */
349		rqstats->bytes += bytes_written;
350		napi = &enic->napi[rq->index];
351		skb = napi_get_frags(napi);
352		if (unlikely(!skb)) {
353			net_warn_ratelimited("%s: skb alloc error rq[%d], desc[%d]\n",
354					     enic->netdev->name, rq->index,
355					     completed_index);
356			rqstats->no_skb++;
357			return;
358		}
359
360		prefetch(skb->data - NET_IP_ALIGN);
361
362		dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr,
363					bytes_written, DMA_FROM_DEVICE);
364		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
365				(struct page *)buf->os_buf, buf->offset,
366				bytes_written, buf->truesize);
367		skb_record_rx_queue(skb, q_number);
368		enic_rq_set_skb_flags(rq, type, rss_hash, rss_type, fcoe,
369				      fcoe_fc_crc_ok, vlan_stripped,
370				      csum_not_calc, tcp_udp_csum_ok, ipv6,
371				      ipv4_csum_ok, vlan_tci, skb);
372		skb_mark_for_recycle(skb);
373		napi_gro_frags(napi);
374		if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
375			enic_intr_update_pkt_size(&cq->pkt_size_counter,
376						  bytes_written);
377		buf->os_buf = NULL;
378		buf->dma_addr = 0;
379		buf = buf->next;
380	} else {
381		/* Buffer overflow
382		 */
383		rqstats->pkt_truncated++;
384	}
385}
386
387static void enic_rq_service(struct enic *enic, void *cq_desc, u8 type,
388			    u16 q_number, u16 completed_index)
389{
390	struct enic_rq_stats *rqstats = &enic->rq[q_number].stats;
391	struct vnic_rq *vrq = &enic->rq[q_number].vrq;
392	struct vnic_rq_buf *vrq_buf = vrq->to_clean;
393	int skipped;
394
395	while (1) {
396		skipped = (vrq_buf->index != completed_index);
397		if (!skipped)
398			enic_rq_indicate_buf(enic, vrq, vrq_buf, cq_desc, type,
399					     q_number, completed_index);
400		else
401			rqstats->desc_skip++;
402
403		vrq->ring.desc_avail++;
404		vrq->to_clean = vrq_buf->next;
405		vrq_buf = vrq_buf->next;
406		if (!skipped)
407			break;
408	}
409}
410
411unsigned int enic_rq_cq_service(struct enic *enic, unsigned int cq_index,
412				unsigned int work_to_do)
413{
414	struct vnic_cq *cq = &enic->cq[cq_index];
415	void *cq_desc = vnic_cq_to_clean(cq);
416	u16 q_number, completed_index;
417	unsigned int work_done = 0;
418	u8 type, color;
419
420	enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color, &q_number,
421			    &completed_index);
422
423	while (color != cq->last_color) {
424		enic_rq_service(enic, cq_desc, type, q_number, completed_index);
425		vnic_cq_inc_to_clean(cq);
426
427		if (++work_done >= work_to_do)
428			break;
429
430		cq_desc = vnic_cq_to_clean(cq);
431		enic_rq_cq_desc_dec(cq_desc, enic->ext_cq, &type, &color,
432				    &q_number, &completed_index);
433	}
434
435	return work_done;
436}