hbg_txrx.c

  1// SPDX-License-Identifier: GPL-2.0+
  2// Copyright (c) 2024 Hisilicon Limited.
  3
  4#include <net/netdev_queues.h>
  5#include "hbg_common.h"
  6#include "hbg_irq.h"
  7#include "hbg_reg.h"
  8#include "hbg_txrx.h"
  9
 10#define CREATE_TRACE_POINTS
 11#include "hbg_trace.h"
 12
 13#define netdev_get_tx_ring(netdev) \
 14			(&(((struct hbg_priv *)netdev_priv(netdev))->tx_ring))
 15
 16#define buffer_to_dma_dir(buffer) (((buffer)->dir == HBG_DIR_RX) ? \
 17				   DMA_FROM_DEVICE : DMA_TO_DEVICE)
 18
 19#define hbg_queue_used_num(head, tail, ring) ({ \
 20	typeof(ring) _ring = (ring); \
 21	((tail) + _ring->len - (head)) % _ring->len; })
 22#define hbg_queue_left_num(head, tail, ring) ({ \
 23	typeof(ring) _r = (ring); \
 24	_r->len - hbg_queue_used_num((head), (tail), _r) - 1; })
 25#define hbg_queue_is_empty(head, tail, ring) \
 26	(hbg_queue_used_num((head), (tail), (ring)) == 0)
 27#define hbg_queue_is_full(head, tail, ring) \
 28	(hbg_queue_left_num((head), (tail), (ring)) == 0)
 29#define hbg_queue_next_prt(p, ring) (((p) + 1) % (ring)->len)
 30#define hbg_queue_move_next(p, ring) ({ \
 31	typeof(ring) _ring = (ring); \
 32	_ring->p = hbg_queue_next_prt(_ring->p, _ring); })
 33
 34#define hbg_get_page_order(ring) ({ \
 35	typeof(ring) _ring = (ring); \
 36	get_order(hbg_spec_max_frame_len(_ring->priv, _ring->dir)); })
 37#define hbg_get_page_size(ring) (PAGE_SIZE << hbg_get_page_order((ring)))
 38
 39#define HBG_TX_STOP_THRS	2
 40#define HBG_TX_START_THRS	(2 * HBG_TX_STOP_THRS)
 41
 42static int hbg_dma_map(struct hbg_buffer *buffer)
 43{
 44	struct hbg_priv *priv = buffer->priv;
 45
 46	buffer->skb_dma = dma_map_single(&priv->pdev->dev,
 47					 buffer->skb->data, buffer->skb_len,
 48					 buffer_to_dma_dir(buffer));
 49	if (unlikely(dma_mapping_error(&priv->pdev->dev, buffer->skb_dma))) {
 50		if (buffer->dir == HBG_DIR_RX)
 51			priv->stats.rx_dma_err_cnt++;
 52		else
 53			priv->stats.tx_dma_err_cnt++;
 54
 55		return -ENOMEM;
 56	}
 57
 58	return 0;
 59}
 60
 61static void hbg_dma_unmap(struct hbg_buffer *buffer)
 62{
 63	struct hbg_priv *priv = buffer->priv;
 64
 65	if (unlikely(!buffer->skb_dma))
 66		return;
 67
 68	dma_unmap_single(&priv->pdev->dev, buffer->skb_dma, buffer->skb_len,
 69			 buffer_to_dma_dir(buffer));
 70	buffer->skb_dma = 0;
 71}
 72
 73static void hbg_buffer_free_page(struct hbg_buffer *buffer)
 74{
 75	struct hbg_ring *ring = buffer->ring;
 76
 77	if (unlikely(!buffer->page))
 78		return;
 79
 80	page_pool_put_full_page(ring->page_pool, buffer->page, false);
 81
 82	buffer->page = NULL;
 83	buffer->page_dma = 0;
 84	buffer->page_addr = NULL;
 85	buffer->page_size = 0;
 86	buffer->page_offset = 0;
 87}
 88
 89static int hbg_buffer_alloc_page(struct hbg_buffer *buffer)
 90{
 91	struct hbg_ring *ring = buffer->ring;
 92	u32 len = hbg_get_page_size(ring);
 93	u32 offset;
 94
 95	if (unlikely(!ring->page_pool))
 96		return 0;
 97
 98	buffer->page = page_pool_dev_alloc_frag(ring->page_pool, &offset, len);
 99	if (unlikely(!buffer->page))
100		return -ENOMEM;
101
102	buffer->page_dma = page_pool_get_dma_addr(buffer->page) + offset;
103	buffer->page_addr = page_address(buffer->page) + offset;
104	buffer->page_size = len;
105	buffer->page_offset = offset;
106
107	return 0;
108}
109
110static void hbg_init_tx_desc(struct hbg_buffer *buffer,
111			     struct hbg_tx_desc *tx_desc)
112{
113	u32 ip_offset = buffer->skb->network_header - buffer->skb->mac_header;
114	u32 word0 = 0;
115
116	word0 |= FIELD_PREP(HBG_TX_DESC_W0_WB_B, HBG_STATUS_ENABLE);
117	word0 |= FIELD_PREP(HBG_TX_DESC_W0_IP_OFF_M, ip_offset);
118	if (likely(buffer->skb->ip_summed == CHECKSUM_PARTIAL)) {
119		word0 |= FIELD_PREP(HBG_TX_DESC_W0_l3_CS_B, HBG_STATUS_ENABLE);
120		word0 |= FIELD_PREP(HBG_TX_DESC_W0_l4_CS_B, HBG_STATUS_ENABLE);
121	}
122
123	tx_desc->word0 = word0;
124	tx_desc->word1 = FIELD_PREP(HBG_TX_DESC_W1_SEND_LEN_M,
125				    buffer->skb->len);
126	tx_desc->word2 = buffer->skb_dma;
127	tx_desc->word3 = buffer->state_dma;
128}
129
130netdev_tx_t hbg_net_start_xmit(struct sk_buff *skb, struct net_device *netdev)
131{
132	struct hbg_ring *ring = netdev_get_tx_ring(netdev);
133	struct hbg_priv *priv = netdev_priv(netdev);
134	/* This smp_load_acquire() pairs with smp_store_release() in
135	 * hbg_napi_tx_recycle() called in tx interrupt handle process.
136	 */
137	u32 ntc = smp_load_acquire(&ring->ntc);
138	struct hbg_buffer *buffer;
139	struct hbg_tx_desc tx_desc;
140	u32 ntu = ring->ntu;
141
142	if (unlikely(!skb->len ||
143		     skb->len > hbg_spec_max_frame_len(priv, HBG_DIR_TX))) {
144		dev_kfree_skb_any(skb);
145		netdev->stats.tx_errors++;
146		return NETDEV_TX_OK;
147	}
148
149	if (!netif_subqueue_maybe_stop(netdev, 0,
150				       hbg_queue_left_num(ntc, ntu, ring),
151				       HBG_TX_STOP_THRS, HBG_TX_START_THRS))
152		return NETDEV_TX_BUSY;
153
154	buffer = &ring->queue[ntu];
155	buffer->skb = skb;
156	buffer->skb_len = skb->len;
157	if (unlikely(hbg_dma_map(buffer))) {
158		dev_kfree_skb_any(skb);
159		return NETDEV_TX_OK;
160	}
161
162	buffer->state = HBG_TX_STATE_START;
163	hbg_init_tx_desc(buffer, &tx_desc);
164	hbg_hw_set_tx_desc(priv, &tx_desc);
165
166	/* This smp_store_release() pairs with smp_load_acquire() in
167	 * hbg_napi_tx_recycle() called in tx interrupt handle process.
168	 */
169	smp_store_release(&ring->ntu, hbg_queue_next_prt(ntu, ring));
170	dev_sw_netstats_tx_add(netdev, 1, skb->len);
171	return NETDEV_TX_OK;
172}
173
174static void hbg_buffer_free_skb(struct hbg_buffer *buffer)
175{
176	if (unlikely(!buffer->skb))
177		return;
178
179	dev_kfree_skb_any(buffer->skb);
180	buffer->skb = NULL;
181}
182
183static void hbg_buffer_free(struct hbg_buffer *buffer)
184{
185	if (buffer->skb) {
186		hbg_dma_unmap(buffer);
187		return hbg_buffer_free_skb(buffer);
188	}
189
190	hbg_buffer_free_page(buffer);
191}
192
193static int hbg_napi_tx_recycle(struct napi_struct *napi, int budget)
194{
195	struct hbg_ring *ring = container_of(napi, struct hbg_ring, napi);
196	/* This smp_load_acquire() pairs with smp_store_release() in
197	 * hbg_net_start_xmit() called in xmit process.
198	 */
199	u32 ntu = smp_load_acquire(&ring->ntu);
200	struct hbg_priv *priv = ring->priv;
201	struct hbg_buffer *buffer;
202	u32 ntc = ring->ntc;
203	int packet_done = 0;
204
205	/* We need do cleanup even if budget is 0.
206	 * Per NAPI documentation budget is for Rx.
207	 * So We hardcode the amount of work Tx NAPI does to 128.
208	 */
209	budget = 128;
210	while (packet_done < budget) {
211		if (unlikely(hbg_queue_is_empty(ntc, ntu, ring)))
212			break;
213
214		/* make sure HW write desc complete */
215		dma_rmb();
216
217		buffer = &ring->queue[ntc];
218		if (buffer->state != HBG_TX_STATE_COMPLETE)
219			break;
220
221		hbg_buffer_free(buffer);
222		ntc = hbg_queue_next_prt(ntc, ring);
223		packet_done++;
224	}
225
226	/* This smp_store_release() pairs with smp_load_acquire() in
227	 * hbg_net_start_xmit() called in xmit process.
228	 */
229	smp_store_release(&ring->ntc, ntc);
230	netif_wake_queue(priv->netdev);
231
232	if (likely(packet_done < budget &&
233		   napi_complete_done(napi, packet_done)))
234		hbg_hw_irq_enable(priv, HBG_INT_MSK_TX_B, true);
235
236	return packet_done;
237}
238
239static bool hbg_rx_check_l3l4_error(struct hbg_priv *priv,
240				    struct hbg_rx_desc *desc,
241				    struct sk_buff *skb)
242{
243	bool rx_checksum_offload = !!(priv->netdev->features & NETIF_F_RXCSUM);
244
245	skb->ip_summed = rx_checksum_offload ?
246			 CHECKSUM_UNNECESSARY : CHECKSUM_NONE;
247
248	if (likely(!FIELD_GET(HBG_RX_DESC_W4_L3_ERR_CODE_M, desc->word4) &&
249		   !FIELD_GET(HBG_RX_DESC_W4_L4_ERR_CODE_M, desc->word4)))
250		return true;
251
252	switch (FIELD_GET(HBG_RX_DESC_W4_L3_ERR_CODE_M, desc->word4)) {
253	case HBG_L3_OK:
254		break;
255	case HBG_L3_WRONG_HEAD:
256		priv->stats.rx_desc_l3_wrong_head_cnt++;
257		return false;
258	case HBG_L3_CSUM_ERR:
259		skb->ip_summed = CHECKSUM_NONE;
260		priv->stats.rx_desc_l3_csum_err_cnt++;
261
262		/* Don't drop packets on csum validation failure,
263		 * suggest by Jakub
264		 */
265		break;
266	case HBG_L3_LEN_ERR:
267		priv->stats.rx_desc_l3_len_err_cnt++;
268		return false;
269	case HBG_L3_ZERO_TTL:
270		priv->stats.rx_desc_l3_zero_ttl_cnt++;
271		return false;
272	default:
273		priv->stats.rx_desc_l3_other_cnt++;
274		return false;
275	}
276
277	switch (FIELD_GET(HBG_RX_DESC_W4_L4_ERR_CODE_M, desc->word4)) {
278	case HBG_L4_OK:
279		break;
280	case HBG_L4_WRONG_HEAD:
281		priv->stats.rx_desc_l4_wrong_head_cnt++;
282		return false;
283	case HBG_L4_LEN_ERR:
284		priv->stats.rx_desc_l4_len_err_cnt++;
285		return false;
286	case HBG_L4_CSUM_ERR:
287		skb->ip_summed = CHECKSUM_NONE;
288		priv->stats.rx_desc_l4_csum_err_cnt++;
289
290		/* Don't drop packets on csum validation failure,
291		 * suggest by Jakub
292		 */
293		break;
294	case HBG_L4_ZERO_PORT_NUM:
295		priv->stats.rx_desc_l4_zero_port_num_cnt++;
296		return false;
297	default:
298		priv->stats.rx_desc_l4_other_cnt++;
299		return false;
300	}
301
302	return true;
303}
304
305static void hbg_update_rx_ip_protocol_stats(struct hbg_priv *priv,
306					    struct hbg_rx_desc *desc)
307{
308	if (unlikely(!FIELD_GET(HBG_RX_DESC_W4_IP_TCP_UDP_M, desc->word4))) {
309		priv->stats.rx_desc_no_ip_pkt_cnt++;
310		return;
311	}
312
313	if (unlikely(FIELD_GET(HBG_RX_DESC_W4_IP_VERSION_ERR_B, desc->word4))) {
314		priv->stats.rx_desc_ip_ver_err_cnt++;
315		return;
316	}
317
318	/* 0:ipv4, 1:ipv6 */
319	if (FIELD_GET(HBG_RX_DESC_W4_IP_VERSION_B, desc->word4))
320		priv->stats.rx_desc_ipv6_pkt_cnt++;
321	else
322		priv->stats.rx_desc_ipv4_pkt_cnt++;
323
324	switch (FIELD_GET(HBG_RX_DESC_W4_IP_TCP_UDP_M, desc->word4)) {
325	case HBG_IP_PKT:
326		priv->stats.rx_desc_ip_pkt_cnt++;
327		if (FIELD_GET(HBG_RX_DESC_W4_OPT_B, desc->word4))
328			priv->stats.rx_desc_ip_opt_pkt_cnt++;
329		if (FIELD_GET(HBG_RX_DESC_W4_FRAG_B, desc->word4))
330			priv->stats.rx_desc_frag_cnt++;
331
332		if (FIELD_GET(HBG_RX_DESC_W4_ICMP_B, desc->word4))
333			priv->stats.rx_desc_icmp_pkt_cnt++;
334		else if (FIELD_GET(HBG_RX_DESC_W4_IPSEC_B, desc->word4))
335			priv->stats.rx_desc_ipsec_pkt_cnt++;
336		break;
337	case HBG_TCP_PKT:
338		priv->stats.rx_desc_tcp_pkt_cnt++;
339		break;
340	case HBG_UDP_PKT:
341		priv->stats.rx_desc_udp_pkt_cnt++;
342		break;
343	default:
344		priv->stats.rx_desc_no_ip_pkt_cnt++;
345		break;
346	}
347}
348
349static void hbg_update_rx_protocol_stats(struct hbg_priv *priv,
350					 struct hbg_rx_desc *desc)
351{
352	if (unlikely(!FIELD_GET(HBG_RX_DESC_W4_IDX_MATCH_B, desc->word4))) {
353		priv->stats.rx_desc_key_not_match_cnt++;
354		return;
355	}
356
357	if (FIELD_GET(HBG_RX_DESC_W4_BRD_CST_B, desc->word4))
358		priv->stats.rx_desc_broadcast_pkt_cnt++;
359	else if (FIELD_GET(HBG_RX_DESC_W4_MUL_CST_B, desc->word4))
360		priv->stats.rx_desc_multicast_pkt_cnt++;
361
362	if (FIELD_GET(HBG_RX_DESC_W4_VLAN_FLAG_B, desc->word4))
363		priv->stats.rx_desc_vlan_pkt_cnt++;
364
365	if (FIELD_GET(HBG_RX_DESC_W4_ARP_B, desc->word4)) {
366		priv->stats.rx_desc_arp_pkt_cnt++;
367		return;
368	} else if (FIELD_GET(HBG_RX_DESC_W4_RARP_B, desc->word4)) {
369		priv->stats.rx_desc_rarp_pkt_cnt++;
370		return;
371	}
372
373	hbg_update_rx_ip_protocol_stats(priv, desc);
374}
375
376static bool hbg_rx_pkt_check(struct hbg_priv *priv, struct hbg_rx_desc *desc,
377			     struct sk_buff *skb)
378{
379	if (unlikely(FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, desc->word2) >
380		     priv->dev_specs.max_frame_len)) {
381		priv->stats.rx_desc_pkt_len_err_cnt++;
382		return false;
383	}
384
385	if (unlikely(FIELD_GET(HBG_RX_DESC_W2_PORT_NUM_M, desc->word2) !=
386		     priv->dev_specs.mac_id ||
387		     FIELD_GET(HBG_RX_DESC_W4_DROP_B, desc->word4))) {
388		priv->stats.rx_desc_drop++;
389		return false;
390	}
391
392	if (unlikely(FIELD_GET(HBG_RX_DESC_W4_L2_ERR_B, desc->word4))) {
393		priv->stats.rx_desc_l2_err_cnt++;
394		return false;
395	}
396
397	if (unlikely(!hbg_rx_check_l3l4_error(priv, desc, skb))) {
398		priv->stats.rx_desc_l3l4_err_cnt++;
399		return false;
400	}
401
402	hbg_update_rx_protocol_stats(priv, desc);
403	return true;
404}
405
406static int hbg_rx_fill_one_buffer(struct hbg_priv *priv)
407{
408	struct hbg_ring *ring = &priv->rx_ring;
409	struct hbg_buffer *buffer;
410	int ret;
411
412	if (hbg_queue_is_full(ring->ntc, ring->ntu, ring) ||
413	    hbg_fifo_is_full(priv, ring->dir))
414		return 0;
415
416	buffer = &ring->queue[ring->ntu];
417	ret = hbg_buffer_alloc_page(buffer);
418	if (unlikely(ret))
419		return ret;
420
421	memset(buffer->page_addr, 0, HBG_PACKET_HEAD_SIZE);
422	dma_sync_single_for_device(&priv->pdev->dev, buffer->page_dma,
423				   HBG_PACKET_HEAD_SIZE, DMA_TO_DEVICE);
424
425	hbg_hw_fill_buffer(priv, buffer->page_dma);
426	hbg_queue_move_next(ntu, ring);
427	return 0;
428}
429
430static int hbg_rx_fill_buffers(struct hbg_priv *priv)
431{
432	u32 remained = hbg_hw_get_fifo_used_num(priv, HBG_DIR_RX);
433	u32 max_count = priv->dev_specs.rx_fifo_num;
434	u32 refill_count;
435	int ret;
436
437	if (unlikely(remained >= max_count))
438		return 0;
439
440	refill_count = max_count - remained;
441	while (refill_count--) {
442		ret = hbg_rx_fill_one_buffer(priv);
443		if (unlikely(ret))
444			break;
445	}
446
447	return ret;
448}
449
450static bool hbg_sync_data_from_hw(struct hbg_priv *priv,
451				  struct hbg_buffer *buffer)
452{
453	struct hbg_rx_desc *rx_desc;
454
455	/* make sure HW write desc complete */
456	dma_rmb();
457
458	dma_sync_single_for_cpu(&priv->pdev->dev, buffer->page_dma,
459				buffer->page_size, DMA_FROM_DEVICE);
460
461	rx_desc = (struct hbg_rx_desc *)buffer->page_addr;
462	return FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2) != 0;
463}
464
465static int hbg_build_skb(struct hbg_priv *priv,
466			 struct hbg_buffer *buffer, u32 pkt_len)
467{
468	net_prefetch(buffer->page_addr);
469
470	buffer->skb = napi_build_skb(buffer->page_addr, buffer->page_size);
471	if (unlikely(!buffer->skb))
472		return -ENOMEM;
473	skb_mark_for_recycle(buffer->skb);
474
475	/* page will be freed together with the skb */
476	buffer->page = NULL;
477
478	return 0;
479}
480
481static int hbg_napi_rx_poll(struct napi_struct *napi, int budget)
482{
483	struct hbg_ring *ring = container_of(napi, struct hbg_ring, napi);
484	struct hbg_priv *priv = ring->priv;
485	struct hbg_rx_desc *rx_desc;
486	struct hbg_buffer *buffer;
487	u32 packet_done = 0;
488	u32 pkt_len;
489
490	hbg_rx_fill_buffers(priv);
491	while (packet_done < budget) {
492		if (unlikely(hbg_queue_is_empty(ring->ntc, ring->ntu, ring)))
493			break;
494
495		buffer = &ring->queue[ring->ntc];
496		if (unlikely(!buffer->page))
497			goto next_buffer;
498
499		if (unlikely(!hbg_sync_data_from_hw(priv, buffer)))
500			break;
501		rx_desc = (struct hbg_rx_desc *)buffer->page_addr;
502		pkt_len = FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2);
503		trace_hbg_rx_desc(priv, ring->ntc, rx_desc);
504
505		if (unlikely(hbg_build_skb(priv, buffer, pkt_len))) {
506			hbg_buffer_free_page(buffer);
507			goto next_buffer;
508		}
509
510		if (unlikely(!hbg_rx_pkt_check(priv, rx_desc, buffer->skb))) {
511			hbg_buffer_free_skb(buffer);
512			goto next_buffer;
513		}
514
515		skb_reserve(buffer->skb, HBG_PACKET_HEAD_SIZE + NET_IP_ALIGN);
516		skb_put(buffer->skb, pkt_len);
517		buffer->skb->protocol = eth_type_trans(buffer->skb,
518						       priv->netdev);
519		dev_sw_netstats_rx_add(priv->netdev, pkt_len);
520		napi_gro_receive(napi, buffer->skb);
521		buffer->skb = NULL;
522		buffer->page = NULL;
523
524next_buffer:
525		hbg_rx_fill_one_buffer(priv);
526		hbg_queue_move_next(ntc, ring);
527		packet_done++;
528	}
529
530	if (likely(packet_done < budget &&
531		   napi_complete_done(napi, packet_done)))
532		hbg_hw_irq_enable(priv, HBG_INT_MSK_RX_B, true);
533
534	return packet_done;
535}
536
537static void hbg_ring_page_pool_destory(struct hbg_ring *ring)
538{
539	if (!ring->page_pool)
540		return;
541
542	page_pool_destroy(ring->page_pool);
543	ring->page_pool = NULL;
544}
545
546static int hbg_ring_page_pool_init(struct hbg_priv *priv, struct hbg_ring *ring)
547{
548	u32 buf_size = hbg_spec_max_frame_len(priv, ring->dir);
549	struct page_pool_params pp_params = {
550		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
551		.order = hbg_get_page_order(ring),
552		.pool_size = ring->len * buf_size / hbg_get_page_size(ring),
553		.nid = dev_to_node(&priv->pdev->dev),
554		.dev = &priv->pdev->dev,
555		.napi = &ring->napi,
556		.dma_dir = DMA_FROM_DEVICE,
557		.offset = 0,
558		.max_len = hbg_get_page_size(ring),
559	};
560	int ret = 0;
561
562	ring->page_pool = page_pool_create(&pp_params);
563	if (IS_ERR(ring->page_pool)) {
564		ret = PTR_ERR(ring->page_pool);
565		dev_err(&priv->pdev->dev,
566			"failed to create page pool, ret = %d\n", ret);
567		ring->page_pool = NULL;
568	}
569
570	return ret;
571}
572
573static void hbg_ring_uninit(struct hbg_ring *ring)
574{
575	struct hbg_buffer *buffer;
576	u32 i;
577
578	if (!ring->queue)
579		return;
580
581	napi_disable(&ring->napi);
582	netif_napi_del(&ring->napi);
583
584	for (i = 0; i < ring->len; i++) {
585		buffer = &ring->queue[i];
586		hbg_buffer_free(buffer);
587		buffer->ring = NULL;
588		buffer->priv = NULL;
589	}
590
591	hbg_ring_page_pool_destory(ring);
592	dma_free_coherent(&ring->priv->pdev->dev,
593			  ring->len * sizeof(*ring->queue),
594			  ring->queue, ring->queue_dma);
595	ring->queue = NULL;
596	ring->queue_dma = 0;
597	ring->len = 0;
598	ring->priv = NULL;
599}
600
601static int hbg_ring_init(struct hbg_priv *priv, struct hbg_ring *ring,
602			 int (*napi_poll)(struct napi_struct *, int),
603			 enum hbg_dir dir)
604{
605	struct hbg_buffer *buffer;
606	u32 i, len;
607	int ret;
608
609	len = hbg_get_spec_fifo_max_num(priv, dir) + 1;
610	/* To improve receiving performance under high-stress scenarios,
611	 * in the `hbg_napi_rx_poll()`, we first use the other half of
612	 * the buffer to receive packets from the hardware via the
613	 * `hbg_rx_fill_buffers()`, and then process the packets in the
614	 * original half of the buffer to avoid packet loss caused by
615	 * hardware overflow as much as possible.
616	 */
617	if (dir == HBG_DIR_RX)
618		len += hbg_get_spec_fifo_max_num(priv, dir);
619
620	ring->queue = dma_alloc_coherent(&priv->pdev->dev,
621					 len * sizeof(*ring->queue),
622					 &ring->queue_dma, GFP_KERNEL);
623	if (!ring->queue)
624		return -ENOMEM;
625
626	for (i = 0; i < len; i++) {
627		buffer = &ring->queue[i];
628		buffer->skb_len = 0;
629		buffer->dir = dir;
630		buffer->ring = ring;
631		buffer->priv = priv;
632		buffer->state_dma = ring->queue_dma + (i * sizeof(*buffer));
633	}
634
635	ring->dir = dir;
636	ring->priv = priv;
637	ring->ntc = 0;
638	ring->ntu = 0;
639	ring->len = len;
640
641	if (dir == HBG_DIR_TX) {
642		netif_napi_add_tx(priv->netdev, &ring->napi, napi_poll);
643	} else {
644		netif_napi_add(priv->netdev, &ring->napi, napi_poll);
645
646		ret = hbg_ring_page_pool_init(priv, ring);
647		if (ret) {
648			netif_napi_del(&ring->napi);
649			dma_free_coherent(&ring->priv->pdev->dev,
650					  ring->len * sizeof(*ring->queue),
651					  ring->queue, ring->queue_dma);
652			ring->queue = NULL;
653			ring->len = 0;
654			return ret;
655		}
656	}
657
658	napi_enable(&ring->napi);
659	return 0;
660}
661
662static int hbg_tx_ring_init(struct hbg_priv *priv)
663{
664	struct hbg_ring *tx_ring = &priv->tx_ring;
665
666	if (!tx_ring->tout_log_buf)
667		tx_ring->tout_log_buf = devm_kmalloc(&priv->pdev->dev,
668						     HBG_TX_TIMEOUT_BUF_LEN,
669						     GFP_KERNEL);
670
671	if (!tx_ring->tout_log_buf)
672		return -ENOMEM;
673
674	return hbg_ring_init(priv, tx_ring, hbg_napi_tx_recycle, HBG_DIR_TX);
675}
676
677static int hbg_rx_ring_init(struct hbg_priv *priv)
678{
679	int ret;
680
681	ret = hbg_ring_init(priv, &priv->rx_ring, hbg_napi_rx_poll, HBG_DIR_RX);
682	if (ret)
683		return ret;
684
685	ret = hbg_rx_fill_buffers(priv);
686	if (ret)
687		hbg_ring_uninit(&priv->rx_ring);
688
689	return ret;
690}
691
692int hbg_txrx_init(struct hbg_priv *priv)
693{
694	int ret;
695
696	ret = hbg_tx_ring_init(priv);
697	if (ret) {
698		dev_err(&priv->pdev->dev,
699			"failed to init tx ring, ret = %d\n", ret);
700		return ret;
701	}
702
703	ret = hbg_rx_ring_init(priv);
704	if (ret) {
705		dev_err(&priv->pdev->dev,
706			"failed to init rx ring, ret = %d\n", ret);
707		hbg_ring_uninit(&priv->tx_ring);
708	}
709
710	return ret;
711}
712
713void hbg_txrx_uninit(struct hbg_priv *priv)
714{
715	hbg_ring_uninit(&priv->tx_ring);
716	hbg_ring_uninit(&priv->rx_ring);
717}