hinic3_rx.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
  3
  4#include <linux/etherdevice.h>
  5#include <linux/if_vlan.h>
  6#include <linux/netdevice.h>
  7#include <net/gro.h>
  8#include <net/page_pool/helpers.h>
  9
 10#include "hinic3_hwdev.h"
 11#include "hinic3_nic_dev.h"
 12#include "hinic3_nic_io.h"
 13#include "hinic3_rx.h"
 14
 15#define HINIC3_RX_HDR_SIZE              256
 16#define HINIC3_RX_BUFFER_WRITE          16
 17
 18#define HINIC3_RX_TCP_PKT               0x3
 19#define HINIC3_RX_UDP_PKT               0x4
 20#define HINIC3_RX_SCTP_PKT              0x7
 21
 22#define HINIC3_RX_IPV4_PKT              0
 23#define HINIC3_RX_IPV6_PKT              1
 24#define HINIC3_RX_INVALID_IP_TYPE       2
 25
 26#define HINIC3_RX_PKT_FORMAT_NON_TUNNEL 0
 27#define HINIC3_RX_PKT_FORMAT_VXLAN      1
 28
 29#define HINIC3_LRO_PKT_HDR_LEN_IPV4     66
 30#define HINIC3_LRO_PKT_HDR_LEN_IPV6     86
 31#define HINIC3_LRO_PKT_HDR_LEN(cqe) \
 32	(RQ_CQE_OFFOLAD_TYPE_GET((cqe)->offload_type, IP_TYPE) == \
 33	 HINIC3_RX_IPV6_PKT ? HINIC3_LRO_PKT_HDR_LEN_IPV6 : \
 34	 HINIC3_LRO_PKT_HDR_LEN_IPV4)
 35
 36int hinic3_alloc_rxqs(struct net_device *netdev)
 37{
 38	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
 39	struct pci_dev *pdev = nic_dev->pdev;
 40	u16 num_rxqs = nic_dev->max_qps;
 41	struct hinic3_rxq *rxq;
 42	u16 q_id;
 43
 44	nic_dev->rxqs = kcalloc(num_rxqs, sizeof(*nic_dev->rxqs), GFP_KERNEL);
 45	if (!nic_dev->rxqs)
 46		return -ENOMEM;
 47
 48	for (q_id = 0; q_id < num_rxqs; q_id++) {
 49		rxq = &nic_dev->rxqs[q_id];
 50		rxq->netdev = netdev;
 51		rxq->dev = &pdev->dev;
 52		rxq->q_id = q_id;
 53		rxq->buf_len = nic_dev->rx_buf_len;
 54		rxq->buf_len_shift = ilog2(nic_dev->rx_buf_len);
 55		rxq->q_depth = nic_dev->q_params.rq_depth;
 56		rxq->q_mask = nic_dev->q_params.rq_depth - 1;
 57	}
 58
 59	return 0;
 60}
 61
 62void hinic3_free_rxqs(struct net_device *netdev)
 63{
 64	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
 65
 66	kfree(nic_dev->rxqs);
 67}
 68
 69static int rx_alloc_mapped_page(struct page_pool *page_pool,
 70				struct hinic3_rx_info *rx_info, u16 buf_len)
 71{
 72	struct page *page;
 73	u32 page_offset;
 74
 75	if (likely(rx_info->page))
 76		return 0;
 77
 78	page = page_pool_dev_alloc_frag(page_pool, &page_offset, buf_len);
 79	if (unlikely(!page))
 80		return -ENOMEM;
 81
 82	rx_info->page = page;
 83	rx_info->page_offset = page_offset;
 84
 85	return 0;
 86}
 87
 88/* Associate fixed completion element to every wqe in the rq. Every rq wqe will
 89 * always post completion to the same place.
 90 */
 91static void rq_associate_cqes(struct hinic3_rxq *rxq)
 92{
 93	struct hinic3_queue_pages *qpages;
 94	struct hinic3_rq_wqe *rq_wqe;
 95	dma_addr_t cqe_dma;
 96	u32 i;
 97
 98	qpages = &rxq->rq->wq.qpages;
 99
100	for (i = 0; i < rxq->q_depth; i++) {
101		rq_wqe = get_q_element(qpages, i, NULL);
102		cqe_dma = rxq->cqe_start_paddr +
103			  i * sizeof(struct hinic3_rq_cqe);
104		rq_wqe->cqe_hi_addr = cpu_to_le32(upper_32_bits(cqe_dma));
105		rq_wqe->cqe_lo_addr = cpu_to_le32(lower_32_bits(cqe_dma));
106	}
107}
108
109static void rq_wqe_buf_set(struct hinic3_io_queue *rq, uint32_t wqe_idx,
110			   dma_addr_t dma_addr, u16 len)
111{
112	struct hinic3_rq_wqe *rq_wqe;
113
114	rq_wqe = get_q_element(&rq->wq.qpages, wqe_idx, NULL);
115	rq_wqe->buf_hi_addr = cpu_to_le32(upper_32_bits(dma_addr));
116	rq_wqe->buf_lo_addr = cpu_to_le32(lower_32_bits(dma_addr));
117}
118
119static u32 hinic3_rx_fill_buffers(struct hinic3_rxq *rxq)
120{
121	u32 i, free_wqebbs = rxq->delta - 1;
122	struct hinic3_rx_info *rx_info;
123	dma_addr_t dma_addr;
124	int err;
125
126	for (i = 0; i < free_wqebbs; i++) {
127		rx_info = &rxq->rx_info[rxq->next_to_update];
128
129		err = rx_alloc_mapped_page(rxq->page_pool, rx_info,
130					   rxq->buf_len);
131		if (unlikely(err))
132			break;
133
134		dma_addr = page_pool_get_dma_addr(rx_info->page) +
135			rx_info->page_offset;
136		rq_wqe_buf_set(rxq->rq, rxq->next_to_update, dma_addr,
137			       rxq->buf_len);
138		rxq->next_to_update = (rxq->next_to_update + 1) & rxq->q_mask;
139	}
140
141	if (likely(i)) {
142		hinic3_write_db(rxq->rq, rxq->q_id & 3, DB_CFLAG_DP_RQ,
143				rxq->next_to_update << HINIC3_NORMAL_RQ_WQE);
144		rxq->delta -= i;
145		rxq->next_to_alloc = rxq->next_to_update;
146	}
147
148	return i;
149}
150
151static u32 hinic3_alloc_rx_buffers(struct hinic3_dyna_rxq_res *rqres,
152				   u32 rq_depth, u16 buf_len)
153{
154	u32 free_wqebbs = rq_depth - 1;
155	u32 idx;
156	int err;
157
158	for (idx = 0; idx < free_wqebbs; idx++) {
159		err = rx_alloc_mapped_page(rqres->page_pool,
160					   &rqres->rx_info[idx], buf_len);
161		if (err)
162			break;
163	}
164
165	return idx;
166}
167
168static void hinic3_free_rx_buffers(struct hinic3_dyna_rxq_res *rqres,
169				   u32 q_depth)
170{
171	struct hinic3_rx_info *rx_info;
172	u32 i;
173
174	/* Free all the Rx ring sk_buffs */
175	for (i = 0; i < q_depth; i++) {
176		rx_info = &rqres->rx_info[i];
177
178		if (rx_info->page) {
179			page_pool_put_full_page(rqres->page_pool,
180						rx_info->page, false);
181			rx_info->page = NULL;
182		}
183	}
184}
185
186static void hinic3_add_rx_frag(struct hinic3_rxq *rxq,
187			       struct hinic3_rx_info *rx_info,
188			       struct sk_buff *skb, u32 size)
189{
190	struct page *page;
191	u8 *va;
192
193	page = rx_info->page;
194	va = (u8 *)page_address(page) + rx_info->page_offset;
195	net_prefetch(va);
196
197	page_pool_dma_sync_for_cpu(rxq->page_pool, page, rx_info->page_offset,
198				   rxq->buf_len);
199
200	if (size <= HINIC3_RX_HDR_SIZE && !skb_is_nonlinear(skb)) {
201		memcpy(__skb_put(skb, size), va,
202		       ALIGN(size, sizeof(long)));
203		page_pool_put_full_page(rxq->page_pool, page, false);
204
205		return;
206	}
207
208	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
209			rx_info->page_offset, size, rxq->buf_len);
210	skb_mark_for_recycle(skb);
211}
212
213static void packaging_skb(struct hinic3_rxq *rxq, struct sk_buff *skb,
214			  u32 sge_num, u32 pkt_len)
215{
216	struct hinic3_rx_info *rx_info;
217	u32 temp_pkt_len = pkt_len;
218	u32 temp_sge_num = sge_num;
219	u32 sw_ci;
220	u32 size;
221
222	sw_ci = rxq->cons_idx & rxq->q_mask;
223	while (temp_sge_num) {
224		rx_info = &rxq->rx_info[sw_ci];
225		sw_ci = (sw_ci + 1) & rxq->q_mask;
226		if (unlikely(temp_pkt_len > rxq->buf_len)) {
227			size = rxq->buf_len;
228			temp_pkt_len -= rxq->buf_len;
229		} else {
230			size = temp_pkt_len;
231		}
232
233		hinic3_add_rx_frag(rxq, rx_info, skb, size);
234
235		/* clear contents of buffer_info */
236		rx_info->page = NULL;
237		temp_sge_num--;
238	}
239}
240
241static u32 hinic3_get_sge_num(struct hinic3_rxq *rxq, u32 pkt_len)
242{
243	u32 sge_num;
244
245	sge_num = pkt_len >> rxq->buf_len_shift;
246	sge_num += (pkt_len & (rxq->buf_len - 1)) ? 1 : 0;
247
248	return sge_num;
249}
250
251static struct sk_buff *hinic3_fetch_rx_buffer(struct hinic3_rxq *rxq,
252					      u32 pkt_len)
253{
254	struct sk_buff *skb;
255	u32 sge_num;
256
257	skb = napi_alloc_skb(&rxq->irq_cfg->napi, HINIC3_RX_HDR_SIZE);
258	if (unlikely(!skb))
259		return NULL;
260
261	sge_num = hinic3_get_sge_num(rxq, pkt_len);
262
263	net_prefetchw(skb->data);
264	packaging_skb(rxq, skb, sge_num, pkt_len);
265
266	rxq->cons_idx += sge_num;
267	rxq->delta += sge_num;
268
269	return skb;
270}
271
272static void hinic3_pull_tail(struct sk_buff *skb)
273{
274	skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
275	unsigned int pull_len;
276	unsigned char *va;
277
278	va = skb_frag_address(frag);
279
280	/* we need the header to contain the greater of either ETH_HLEN or
281	 * 60 bytes if the skb->len is less than 60 for skb_pad.
282	 */
283	pull_len = eth_get_headlen(skb->dev, va, HINIC3_RX_HDR_SIZE);
284
285	/* align pull length to size of long to optimize memcpy performance */
286	skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
287
288	/* update all of the pointers */
289	skb_frag_size_sub(frag, pull_len);
290	skb_frag_off_add(frag, pull_len);
291
292	skb->data_len -= pull_len;
293	skb->tail += pull_len;
294}
295
296static void hinic3_rx_csum(struct hinic3_rxq *rxq, u32 offload_type,
297			   u32 status, struct sk_buff *skb)
298{
299	u32 pkt_fmt = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, TUNNEL_PKT_FORMAT);
300	u32 pkt_type = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE);
301	u32 ip_type = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, IP_TYPE);
302	u32 csum_err = RQ_CQE_STATUS_GET(status, CSUM_ERR);
303	struct net_device *netdev = rxq->netdev;
304
305	if (!(netdev->features & NETIF_F_RXCSUM))
306		return;
307
308	if (unlikely(csum_err)) {
309		/* pkt type is recognized by HW, and csum is wrong */
310		skb->ip_summed = CHECKSUM_NONE;
311		return;
312	}
313
314	if (ip_type == HINIC3_RX_INVALID_IP_TYPE ||
315	    !(pkt_fmt == HINIC3_RX_PKT_FORMAT_NON_TUNNEL ||
316	      pkt_fmt == HINIC3_RX_PKT_FORMAT_VXLAN)) {
317		skb->ip_summed = CHECKSUM_NONE;
318		return;
319	}
320
321	switch (pkt_type) {
322	case HINIC3_RX_TCP_PKT:
323	case HINIC3_RX_UDP_PKT:
324	case HINIC3_RX_SCTP_PKT:
325		skb->ip_summed = CHECKSUM_UNNECESSARY;
326		break;
327	default:
328		skb->ip_summed = CHECKSUM_NONE;
329		break;
330	}
331}
332
333static void hinic3_lro_set_gso_params(struct sk_buff *skb, u16 num_lro)
334{
335	struct ethhdr *eth = (struct ethhdr *)(skb->data);
336	__be16 proto;
337
338	proto = __vlan_get_protocol(skb, eth->h_proto, NULL);
339
340	skb_shinfo(skb)->gso_size = DIV_ROUND_UP(skb->len - skb_headlen(skb),
341						 num_lro);
342	skb_shinfo(skb)->gso_type = proto == htons(ETH_P_IP) ?
343				    SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
344	skb_shinfo(skb)->gso_segs = num_lro;
345}
346
347static int recv_one_pkt(struct hinic3_rxq *rxq, struct hinic3_rq_cqe *rx_cqe,
348			u32 pkt_len, u32 vlan_len, u32 status)
349{
350	struct net_device *netdev = rxq->netdev;
351	struct sk_buff *skb;
352	u32 offload_type;
353	u16 num_lro;
354
355	skb = hinic3_fetch_rx_buffer(rxq, pkt_len);
356	if (unlikely(!skb))
357		return -ENOMEM;
358
359	/* place header in linear portion of buffer */
360	if (skb_is_nonlinear(skb))
361		hinic3_pull_tail(skb);
362
363	offload_type = le32_to_cpu(rx_cqe->offload_type);
364	hinic3_rx_csum(rxq, offload_type, status, skb);
365
366	num_lro = RQ_CQE_STATUS_GET(status, NUM_LRO);
367	if (num_lro)
368		hinic3_lro_set_gso_params(skb, num_lro);
369
370	skb_record_rx_queue(skb, rxq->q_id);
371	skb->protocol = eth_type_trans(skb, netdev);
372
373	if (skb_has_frag_list(skb)) {
374		napi_gro_flush(&rxq->irq_cfg->napi, false);
375		netif_receive_skb(skb);
376	} else {
377		napi_gro_receive(&rxq->irq_cfg->napi, skb);
378	}
379
380	return 0;
381}
382
383int hinic3_alloc_rxqs_res(struct net_device *netdev, u16 num_rq,
384			  u32 rq_depth, struct hinic3_dyna_rxq_res *rxqs_res)
385{
386	u64 cqe_mem_size = sizeof(struct hinic3_rq_cqe) * rq_depth;
387	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
388	struct page_pool_params pp_params = {};
389	struct hinic3_dyna_rxq_res *rqres;
390	u32 pkt_idx;
391	int idx;
392
393	for (idx = 0; idx < num_rq; idx++) {
394		rqres = &rxqs_res[idx];
395		rqres->rx_info = kcalloc(rq_depth, sizeof(*rqres->rx_info),
396					 GFP_KERNEL);
397		if (!rqres->rx_info)
398			goto err_free_rqres;
399
400		rqres->cqe_start_vaddr =
401			dma_alloc_coherent(&nic_dev->pdev->dev, cqe_mem_size,
402					   &rqres->cqe_start_paddr, GFP_KERNEL);
403		if (!rqres->cqe_start_vaddr) {
404			netdev_err(netdev, "Failed to alloc rxq%d rx cqe\n",
405				   idx);
406			goto err_free_rx_info;
407		}
408
409		pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
410		pp_params.pool_size = rq_depth * nic_dev->rx_buf_len /
411				      PAGE_SIZE;
412		pp_params.nid = dev_to_node(&nic_dev->pdev->dev);
413		pp_params.dev = &nic_dev->pdev->dev;
414		pp_params.dma_dir = DMA_FROM_DEVICE;
415		pp_params.max_len = PAGE_SIZE;
416		rqres->page_pool = page_pool_create(&pp_params);
417		if (IS_ERR(rqres->page_pool)) {
418			netdev_err(netdev, "Failed to create rxq%d page pool\n",
419				   idx);
420			goto err_free_cqe;
421		}
422
423		pkt_idx = hinic3_alloc_rx_buffers(rqres, rq_depth,
424						  nic_dev->rx_buf_len);
425		if (!pkt_idx) {
426			netdev_err(netdev, "Failed to alloc rxq%d rx buffers\n",
427				   idx);
428			goto err_destroy_page_pool;
429		}
430		rqres->next_to_alloc = pkt_idx;
431	}
432
433	return 0;
434
435err_destroy_page_pool:
436	page_pool_destroy(rqres->page_pool);
437err_free_cqe:
438	dma_free_coherent(&nic_dev->pdev->dev, cqe_mem_size,
439			  rqres->cqe_start_vaddr,
440			  rqres->cqe_start_paddr);
441err_free_rx_info:
442	kfree(rqres->rx_info);
443err_free_rqres:
444	hinic3_free_rxqs_res(netdev, idx, rq_depth, rxqs_res);
445
446	return -ENOMEM;
447}
448
449void hinic3_free_rxqs_res(struct net_device *netdev, u16 num_rq,
450			  u32 rq_depth, struct hinic3_dyna_rxq_res *rxqs_res)
451{
452	u64 cqe_mem_size = sizeof(struct hinic3_rq_cqe) * rq_depth;
453	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
454	struct hinic3_dyna_rxq_res *rqres;
455	int idx;
456
457	for (idx = 0; idx < num_rq; idx++) {
458		rqres = &rxqs_res[idx];
459
460		hinic3_free_rx_buffers(rqres, rq_depth);
461		page_pool_destroy(rqres->page_pool);
462		dma_free_coherent(&nic_dev->pdev->dev, cqe_mem_size,
463				  rqres->cqe_start_vaddr,
464				  rqres->cqe_start_paddr);
465		kfree(rqres->rx_info);
466	}
467}
468
469int hinic3_configure_rxqs(struct net_device *netdev, u16 num_rq,
470			  u32 rq_depth, struct hinic3_dyna_rxq_res *rxqs_res)
471{
472	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
473	struct hinic3_dyna_rxq_res *rqres;
474	struct msix_entry *msix_entry;
475	struct hinic3_rxq *rxq;
476	u16 q_id;
477	u32 pkts;
478
479	for (q_id = 0; q_id < num_rq; q_id++) {
480		rxq = &nic_dev->rxqs[q_id];
481		rqres = &rxqs_res[q_id];
482		msix_entry = &nic_dev->qps_msix_entries[q_id];
483
484		rxq->irq_id = msix_entry->vector;
485		rxq->msix_entry_idx = msix_entry->entry;
486		rxq->next_to_update = 0;
487		rxq->next_to_alloc = rqres->next_to_alloc;
488		rxq->q_depth = rq_depth;
489		rxq->delta = rxq->q_depth;
490		rxq->q_mask = rxq->q_depth - 1;
491		rxq->cons_idx = 0;
492
493		rxq->cqe_arr = rqres->cqe_start_vaddr;
494		rxq->cqe_start_paddr = rqres->cqe_start_paddr;
495		rxq->rx_info = rqres->rx_info;
496		rxq->page_pool = rqres->page_pool;
497
498		rxq->rq = &nic_dev->nic_io->rq[rxq->q_id];
499
500		rq_associate_cqes(rxq);
501
502		pkts = hinic3_rx_fill_buffers(rxq);
503		if (!pkts) {
504			netdev_err(netdev, "Failed to fill Rx buffer\n");
505			return -ENOMEM;
506		}
507	}
508
509	return 0;
510}
511
512int hinic3_rx_poll(struct hinic3_rxq *rxq, int budget)
513{
514	struct hinic3_nic_dev *nic_dev = netdev_priv(rxq->netdev);
515	u32 sw_ci, status, pkt_len, vlan_len;
516	struct hinic3_rq_cqe *rx_cqe;
517	u32 num_wqe = 0;
518	int nr_pkts = 0;
519	u16 num_lro;
520
521	while (likely(nr_pkts < budget)) {
522		sw_ci = rxq->cons_idx & rxq->q_mask;
523		rx_cqe = rxq->cqe_arr + sw_ci;
524		status = le32_to_cpu(rx_cqe->status);
525		if (!RQ_CQE_STATUS_GET(status, RXDONE))
526			break;
527
528		/* make sure we read rx_done before packet length */
529		rmb();
530
531		vlan_len = le32_to_cpu(rx_cqe->vlan_len);
532		pkt_len = RQ_CQE_SGE_GET(vlan_len, LEN);
533		if (recv_one_pkt(rxq, rx_cqe, pkt_len, vlan_len, status))
534			break;
535
536		nr_pkts++;
537		num_lro = RQ_CQE_STATUS_GET(status, NUM_LRO);
538		if (num_lro)
539			num_wqe += hinic3_get_sge_num(rxq, pkt_len);
540
541		rx_cqe->status = 0;
542
543		if (num_wqe >= nic_dev->lro_replenish_thld)
544			break;
545	}
546
547	if (rxq->delta >= HINIC3_RX_BUFFER_WRITE)
548		hinic3_rx_fill_buffers(rxq);
549
550	return nr_pkts;
551}