hinic3_tx.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
  3
  4#include <linux/if_vlan.h>
  5#include <linux/iopoll.h>
  6#include <net/ip6_checksum.h>
  7#include <net/ipv6.h>
  8#include <net/netdev_queues.h>
  9
 10#include "hinic3_hwdev.h"
 11#include "hinic3_nic_cfg.h"
 12#include "hinic3_nic_dev.h"
 13#include "hinic3_nic_io.h"
 14#include "hinic3_tx.h"
 15#include "hinic3_wq.h"
 16
 17#define MIN_SKB_LEN                32
 18
 19int hinic3_alloc_txqs(struct net_device *netdev)
 20{
 21	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
 22	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
 23	u16 q_id, num_txqs = nic_dev->max_qps;
 24	struct pci_dev *pdev = nic_dev->pdev;
 25	struct hinic3_txq *txq;
 26
 27	if (!num_txqs) {
 28		dev_err(hwdev->dev, "Cannot allocate zero size txqs\n");
 29		return -EINVAL;
 30	}
 31
 32	nic_dev->txqs = kcalloc(num_txqs, sizeof(*nic_dev->txqs),  GFP_KERNEL);
 33	if (!nic_dev->txqs)
 34		return -ENOMEM;
 35
 36	for (q_id = 0; q_id < num_txqs; q_id++) {
 37		txq = &nic_dev->txqs[q_id];
 38		txq->netdev = netdev;
 39		txq->q_id = q_id;
 40		txq->q_depth = nic_dev->q_params.sq_depth;
 41		txq->q_mask = nic_dev->q_params.sq_depth - 1;
 42		txq->dev = &pdev->dev;
 43	}
 44
 45	return 0;
 46}
 47
 48void hinic3_free_txqs(struct net_device *netdev)
 49{
 50	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
 51
 52	kfree(nic_dev->txqs);
 53}
 54
 55static void hinic3_set_buf_desc(struct hinic3_sq_bufdesc *buf_descs,
 56				dma_addr_t addr, u32 len)
 57{
 58	buf_descs->hi_addr = cpu_to_le32(upper_32_bits(addr));
 59	buf_descs->lo_addr = cpu_to_le32(lower_32_bits(addr));
 60	buf_descs->len = cpu_to_le32(len);
 61}
 62
 63static int hinic3_tx_map_skb(struct net_device *netdev, struct sk_buff *skb,
 64			     struct hinic3_txq *txq,
 65			     struct hinic3_tx_info *tx_info,
 66			     struct hinic3_sq_wqe_combo *wqe_combo)
 67{
 68	struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->ctrl_bd0;
 69	struct hinic3_sq_bufdesc *buf_desc = wqe_combo->bds_head;
 70	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
 71	struct hinic3_dma_info *dma_info = tx_info->dma_info;
 72	struct pci_dev *pdev = nic_dev->pdev;
 73	skb_frag_t *frag;
 74	u32 i, idx;
 75	int err;
 76
 77	dma_info[0].dma = dma_map_single(&pdev->dev, skb->data,
 78					 skb_headlen(skb), DMA_TO_DEVICE);
 79	if (dma_mapping_error(&pdev->dev, dma_info[0].dma))
 80		return -EFAULT;
 81
 82	dma_info[0].len = skb_headlen(skb);
 83
 84	wqe_desc->hi_addr = cpu_to_le32(upper_32_bits(dma_info[0].dma));
 85	wqe_desc->lo_addr = cpu_to_le32(lower_32_bits(dma_info[0].dma));
 86
 87	wqe_desc->ctrl_len = cpu_to_le32(dma_info[0].len);
 88
 89	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 90		frag = &(skb_shinfo(skb)->frags[i]);
 91		if (unlikely(i == wqe_combo->first_bds_num))
 92			buf_desc = wqe_combo->bds_sec2;
 93
 94		idx = i + 1;
 95		dma_info[idx].dma = skb_frag_dma_map(&pdev->dev, frag, 0,
 96						     skb_frag_size(frag),
 97						     DMA_TO_DEVICE);
 98		if (dma_mapping_error(&pdev->dev, dma_info[idx].dma)) {
 99			err = -EFAULT;
100			goto err_unmap_page;
101		}
102		dma_info[idx].len = skb_frag_size(frag);
103
104		hinic3_set_buf_desc(buf_desc, dma_info[idx].dma,
105				    dma_info[idx].len);
106		buf_desc++;
107	}
108
109	return 0;
110
111err_unmap_page:
112	while (idx > 1) {
113		idx--;
114		dma_unmap_page(&pdev->dev, dma_info[idx].dma,
115			       dma_info[idx].len, DMA_TO_DEVICE);
116	}
117	dma_unmap_single(&pdev->dev, dma_info[0].dma, dma_info[0].len,
118			 DMA_TO_DEVICE);
119
120	return err;
121}
122
123static void hinic3_tx_unmap_skb(struct net_device *netdev,
124				struct sk_buff *skb,
125				struct hinic3_dma_info *dma_info)
126{
127	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
128	struct pci_dev *pdev = nic_dev->pdev;
129	int i;
130
131	for (i = 0; i < skb_shinfo(skb)->nr_frags;) {
132		i++;
133		dma_unmap_page(&pdev->dev,
134			       dma_info[i].dma,
135			       dma_info[i].len, DMA_TO_DEVICE);
136	}
137
138	dma_unmap_single(&pdev->dev, dma_info[0].dma,
139			 dma_info[0].len, DMA_TO_DEVICE);
140}
141
142static void free_all_tx_skbs(struct net_device *netdev, u32 sq_depth,
143			     struct hinic3_tx_info *tx_info_arr)
144{
145	struct hinic3_tx_info *tx_info;
146	u32 idx;
147
148	for (idx = 0; idx < sq_depth; idx++) {
149		tx_info = &tx_info_arr[idx];
150		if (tx_info->skb) {
151			hinic3_tx_unmap_skb(netdev, tx_info->skb,
152					    tx_info->dma_info);
153			dev_kfree_skb_any(tx_info->skb);
154			tx_info->skb = NULL;
155		}
156	}
157}
158
159union hinic3_ip {
160	struct iphdr   *v4;
161	struct ipv6hdr *v6;
162	unsigned char  *hdr;
163};
164
165union hinic3_l4 {
166	struct tcphdr *tcp;
167	struct udphdr *udp;
168	unsigned char *hdr;
169};
170
171enum hinic3_l3_type {
172	HINIC3_L3_UNKNOWN         = 0,
173	HINIC3_L3_IP6_PKT         = 1,
174	HINIC3_L3_IP4_PKT_NO_CSUM = 2,
175	HINIC3_L3_IP4_PKT_CSUM    = 3,
176};
177
178enum hinic3_l4_offload_type {
179	HINIC3_L4_OFFLOAD_DISABLE = 0,
180	HINIC3_L4_OFFLOAD_TCP     = 1,
181	HINIC3_L4_OFFLOAD_STCP    = 2,
182	HINIC3_L4_OFFLOAD_UDP     = 3,
183};
184
185/* initialize l4 offset and offload */
186static void get_inner_l4_info(struct sk_buff *skb, union hinic3_l4 *l4,
187			      u8 l4_proto, u32 *offset,
188			      enum hinic3_l4_offload_type *l4_offload)
189{
190	switch (l4_proto) {
191	case IPPROTO_TCP:
192		*l4_offload = HINIC3_L4_OFFLOAD_TCP;
193		/* To be same with TSO, payload offset begins from payload */
194		*offset = (l4->tcp->doff << TCP_HDR_DATA_OFF_UNIT_SHIFT) +
195			   TRANSPORT_OFFSET(l4->hdr, skb);
196		break;
197
198	case IPPROTO_UDP:
199		*l4_offload = HINIC3_L4_OFFLOAD_UDP;
200		*offset = TRANSPORT_OFFSET(l4->hdr, skb);
201		break;
202	default:
203		*l4_offload = HINIC3_L4_OFFLOAD_DISABLE;
204		*offset = 0;
205	}
206}
207
208static int hinic3_tx_csum(struct hinic3_txq *txq, struct hinic3_sq_task *task,
209			  struct sk_buff *skb)
210{
211	if (skb->ip_summed != CHECKSUM_PARTIAL)
212		return 0;
213
214	if (skb->encapsulation) {
215		union hinic3_ip ip;
216		u8 l4_proto;
217
218		task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
219								 TUNNEL_FLAG));
220
221		ip.hdr = skb_network_header(skb);
222		if (ip.v4->version == 4) {
223			l4_proto = ip.v4->protocol;
224		} else if (ip.v4->version == 6) {
225			union hinic3_l4 l4;
226			unsigned char *exthdr;
227			__be16 frag_off;
228
229			exthdr = ip.hdr + sizeof(*ip.v6);
230			l4_proto = ip.v6->nexthdr;
231			l4.hdr = skb_transport_header(skb);
232			if (l4.hdr != exthdr)
233				ipv6_skip_exthdr(skb, exthdr - skb->data,
234						 &l4_proto, &frag_off);
235		} else {
236			l4_proto = IPPROTO_RAW;
237		}
238
239		if (l4_proto != IPPROTO_UDP ||
240		    ((struct udphdr *)skb_transport_header(skb))->dest !=
241		    VXLAN_OFFLOAD_PORT_LE) {
242			/* Unsupported tunnel packet, disable csum offload */
243			skb_checksum_help(skb);
244			return 0;
245		}
246	}
247
248	task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, INNER_L4_EN));
249
250	return 1;
251}
252
253static void get_inner_l3_l4_type(struct sk_buff *skb, union hinic3_ip *ip,
254				 union hinic3_l4 *l4,
255				 enum hinic3_l3_type *l3_type, u8 *l4_proto)
256{
257	unsigned char *exthdr;
258	__be16 frag_off;
259
260	if (ip->v4->version == 4) {
261		*l3_type = HINIC3_L3_IP4_PKT_CSUM;
262		*l4_proto = ip->v4->protocol;
263	} else if (ip->v4->version == 6) {
264		*l3_type = HINIC3_L3_IP6_PKT;
265		exthdr = ip->hdr + sizeof(*ip->v6);
266		*l4_proto = ip->v6->nexthdr;
267		if (exthdr != l4->hdr) {
268			ipv6_skip_exthdr(skb, exthdr - skb->data,
269					 l4_proto, &frag_off);
270		}
271	} else {
272		*l3_type = HINIC3_L3_UNKNOWN;
273		*l4_proto = 0;
274	}
275}
276
277static void hinic3_set_tso_info(struct hinic3_sq_task *task, __le32 *queue_info,
278				enum hinic3_l4_offload_type l4_offload,
279				u32 offset, u32 mss)
280{
281	if (l4_offload == HINIC3_L4_OFFLOAD_TCP) {
282		*queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, TSO));
283		task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
284								 INNER_L4_EN));
285	} else if (l4_offload == HINIC3_L4_OFFLOAD_UDP) {
286		*queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, UFO));
287		task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
288								 INNER_L4_EN));
289	}
290
291	/* enable L3 calculation */
292	task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, INNER_L3_EN));
293
294	*queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(offset >> 1, PLDOFF));
295
296	/* set MSS value */
297	*queue_info &= cpu_to_le32(~SQ_CTRL_QUEUE_INFO_MSS_MASK);
298	*queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(mss, MSS));
299}
300
301static __sum16 csum_magic(union hinic3_ip *ip, unsigned short proto)
302{
303	return (ip->v4->version == 4) ?
304		csum_tcpudp_magic(ip->v4->saddr, ip->v4->daddr, 0, proto, 0) :
305		csum_ipv6_magic(&ip->v6->saddr, &ip->v6->daddr, 0, proto, 0);
306}
307
308static int hinic3_tso(struct hinic3_sq_task *task, __le32 *queue_info,
309		      struct sk_buff *skb)
310{
311	enum hinic3_l4_offload_type l4_offload;
312	enum hinic3_l3_type l3_type;
313	union hinic3_ip ip;
314	union hinic3_l4 l4;
315	u8 l4_proto;
316	u32 offset;
317	int err;
318
319	if (!skb_is_gso(skb))
320		return 0;
321
322	err = skb_cow_head(skb, 0);
323	if (err < 0)
324		return err;
325
326	if (skb->encapsulation) {
327		u32 gso_type = skb_shinfo(skb)->gso_type;
328		/* L3 checksum is always enabled */
329		task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1, OUT_L3_EN));
330		task->pkt_info0 |= cpu_to_le32(SQ_TASK_INFO0_SET(1,
331								 TUNNEL_FLAG));
332
333		l4.hdr = skb_transport_header(skb);
334		ip.hdr = skb_network_header(skb);
335
336		if (gso_type & SKB_GSO_UDP_TUNNEL_CSUM) {
337			l4.udp->check = ~csum_magic(&ip, IPPROTO_UDP);
338			task->pkt_info0 |=
339				cpu_to_le32(SQ_TASK_INFO0_SET(1, OUT_L4_EN));
340		}
341
342		ip.hdr = skb_inner_network_header(skb);
343		l4.hdr = skb_inner_transport_header(skb);
344	} else {
345		ip.hdr = skb_network_header(skb);
346		l4.hdr = skb_transport_header(skb);
347	}
348
349	get_inner_l3_l4_type(skb, &ip, &l4, &l3_type, &l4_proto);
350
351	if (l4_proto == IPPROTO_TCP)
352		l4.tcp->check = ~csum_magic(&ip, IPPROTO_TCP);
353
354	get_inner_l4_info(skb, &l4, l4_proto, &offset, &l4_offload);
355
356	hinic3_set_tso_info(task, queue_info, l4_offload, offset,
357			    skb_shinfo(skb)->gso_size);
358
359	return 1;
360}
361
362static void hinic3_set_vlan_tx_offload(struct hinic3_sq_task *task,
363				       u16 vlan_tag, u8 vlan_tpid)
364{
365	/* vlan_tpid: 0=select TPID0 in IPSU, 1=select TPID1 in IPSU
366	 * 2=select TPID2 in IPSU, 3=select TPID3 in IPSU,
367	 * 4=select TPID4 in IPSU
368	 */
369	task->vlan_offload =
370		cpu_to_le32(SQ_TASK_INFO3_SET(vlan_tag, VLAN_TAG) |
371			    SQ_TASK_INFO3_SET(vlan_tpid, VLAN_TPID) |
372			    SQ_TASK_INFO3_SET(1, VLAN_TAG_VALID));
373}
374
375static u32 hinic3_tx_offload(struct sk_buff *skb, struct hinic3_sq_task *task,
376			     __le32 *queue_info, struct hinic3_txq *txq)
377{
378	u32 offload = 0;
379	int tso_cs_en;
380
381	task->pkt_info0 = 0;
382	task->ip_identify = 0;
383	task->rsvd = 0;
384	task->vlan_offload = 0;
385
386	tso_cs_en = hinic3_tso(task, queue_info, skb);
387	if (tso_cs_en < 0) {
388		offload = HINIC3_TX_OFFLOAD_INVALID;
389		return offload;
390	} else if (tso_cs_en) {
391		offload |= HINIC3_TX_OFFLOAD_TSO;
392	} else {
393		tso_cs_en = hinic3_tx_csum(txq, task, skb);
394		if (tso_cs_en)
395			offload |= HINIC3_TX_OFFLOAD_CSUM;
396	}
397
398#define VLAN_INSERT_MODE_MAX 5
399	if (unlikely(skb_vlan_tag_present(skb))) {
400		/* select vlan insert mode by qid, default 802.1Q Tag type */
401		hinic3_set_vlan_tx_offload(task, skb_vlan_tag_get(skb),
402					   txq->q_id % VLAN_INSERT_MODE_MAX);
403		offload |= HINIC3_TX_OFFLOAD_VLAN;
404	}
405
406	if (unlikely(SQ_CTRL_QUEUE_INFO_GET(*queue_info, PLDOFF) >
407		     SQ_CTRL_MAX_PLDOFF)) {
408		offload = HINIC3_TX_OFFLOAD_INVALID;
409		return offload;
410	}
411
412	return offload;
413}
414
415static u16 hinic3_get_and_update_sq_owner(struct hinic3_io_queue *sq,
416					  u16 curr_pi, u16 wqebb_cnt)
417{
418	u16 owner = sq->owner;
419
420	if (unlikely(curr_pi + wqebb_cnt >= sq->wq.q_depth))
421		sq->owner = !sq->owner;
422
423	return owner;
424}
425
426static u16 hinic3_set_wqe_combo(struct hinic3_txq *txq,
427				struct hinic3_sq_wqe_combo *wqe_combo,
428				u32 offload, u16 num_sge, u16 *curr_pi)
429{
430	struct hinic3_sq_bufdesc *first_part_wqebbs, *second_part_wqebbs;
431	u16 first_part_wqebbs_num, tmp_pi;
432
433	wqe_combo->ctrl_bd0 = hinic3_wq_get_one_wqebb(&txq->sq->wq, curr_pi);
434	if (!offload && num_sge == 1) {
435		wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE;
436		return hinic3_get_and_update_sq_owner(txq->sq, *curr_pi, 1);
437	}
438
439	wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE;
440
441	if (offload) {
442		wqe_combo->task = hinic3_wq_get_one_wqebb(&txq->sq->wq,
443							  &tmp_pi);
444		wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES;
445	} else {
446		wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS;
447	}
448
449	if (num_sge > 1) {
450		/* first wqebb contain bd0, and bd size is equal to sq wqebb
451		 * size, so we use (num_sge - 1) as wanted weqbb_cnt
452		 */
453		hinic3_wq_get_multi_wqebbs(&txq->sq->wq, num_sge - 1, &tmp_pi,
454					   &first_part_wqebbs,
455					   &second_part_wqebbs,
456					   &first_part_wqebbs_num);
457		wqe_combo->bds_head = first_part_wqebbs;
458		wqe_combo->bds_sec2 = second_part_wqebbs;
459		wqe_combo->first_bds_num = first_part_wqebbs_num;
460	}
461
462	return hinic3_get_and_update_sq_owner(txq->sq, *curr_pi,
463					      num_sge + !!offload);
464}
465
466static void hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo,
467				   __le32 queue_info, int nr_descs, u16 owner)
468{
469	struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->ctrl_bd0;
470
471	if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE) {
472		wqe_desc->ctrl_len |=
473			cpu_to_le32(SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |
474				    SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) |
475				    SQ_CTRL_SET(owner, OWNER));
476
477		/* compact wqe queue_info will transfer to chip */
478		wqe_desc->queue_info = 0;
479		return;
480	}
481
482	wqe_desc->ctrl_len |=
483		cpu_to_le32(SQ_CTRL_SET(nr_descs, BUFDESC_NUM) |
484			    SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) |
485			    SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |
486			    SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) |
487			    SQ_CTRL_SET(owner, OWNER));
488
489	wqe_desc->queue_info = queue_info;
490	wqe_desc->queue_info |= cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(1, UC));
491
492	if (!SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS)) {
493		wqe_desc->queue_info |=
494		    cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(HINIC3_TX_MSS_DEFAULT, MSS));
495	} else if (SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS) <
496		   HINIC3_TX_MSS_MIN) {
497		/* mss should not be less than 80 */
498		wqe_desc->queue_info &=
499		    cpu_to_le32(~SQ_CTRL_QUEUE_INFO_MSS_MASK);
500		wqe_desc->queue_info |=
501		    cpu_to_le32(SQ_CTRL_QUEUE_INFO_SET(HINIC3_TX_MSS_MIN, MSS));
502	}
503}
504
505static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb,
506				       struct net_device *netdev,
507				       struct hinic3_txq *txq)
508{
509	struct hinic3_sq_wqe_combo wqe_combo = {};
510	struct hinic3_tx_info *tx_info;
511	struct hinic3_sq_task task;
512	u16 wqebb_cnt, num_sge;
513	__le32 queue_info = 0;
514	u16 saved_wq_prod_idx;
515	u16 owner, pi = 0;
516	u8 saved_sq_owner;
517	u32 offload;
518	int err;
519
520	if (unlikely(skb->len < MIN_SKB_LEN)) {
521		if (skb_pad(skb, MIN_SKB_LEN - skb->len))
522			goto err_out;
523
524		skb->len = MIN_SKB_LEN;
525	}
526
527	num_sge = skb_shinfo(skb)->nr_frags + 1;
528	/* assume normal wqe format + 1 wqebb for task info */
529	wqebb_cnt = num_sge + 1;
530
531	if (unlikely(hinic3_wq_free_wqebbs(&txq->sq->wq) < wqebb_cnt)) {
532		if (likely(wqebb_cnt > txq->tx_stop_thrs))
533			txq->tx_stop_thrs = min(wqebb_cnt, txq->tx_start_thrs);
534
535		netif_subqueue_try_stop(netdev, txq->sq->q_id,
536					hinic3_wq_free_wqebbs(&txq->sq->wq),
537					txq->tx_start_thrs);
538
539		return NETDEV_TX_BUSY;
540	}
541
542	offload = hinic3_tx_offload(skb, &task, &queue_info, txq);
543	if (unlikely(offload == HINIC3_TX_OFFLOAD_INVALID)) {
544		goto err_drop_pkt;
545	} else if (!offload) {
546		wqebb_cnt -= 1;
547		if (unlikely(num_sge == 1 &&
548			     skb->len > HINIC3_COMPACT_WQEE_SKB_MAX_LEN))
549			goto err_drop_pkt;
550	}
551
552	saved_wq_prod_idx = txq->sq->wq.prod_idx;
553	saved_sq_owner = txq->sq->owner;
554
555	owner = hinic3_set_wqe_combo(txq, &wqe_combo, offload, num_sge, &pi);
556	if (offload)
557		*wqe_combo.task = task;
558
559	tx_info = &txq->tx_info[pi];
560	tx_info->skb = skb;
561	tx_info->wqebb_cnt = wqebb_cnt;
562
563	err = hinic3_tx_map_skb(netdev, skb, txq, tx_info, &wqe_combo);
564	if (err) {
565		/* Rollback work queue to reclaim the wqebb we did not use */
566		txq->sq->wq.prod_idx = saved_wq_prod_idx;
567		txq->sq->owner = saved_sq_owner;
568		goto err_drop_pkt;
569	}
570
571	netif_subqueue_sent(netdev, txq->sq->q_id, skb->len);
572	netif_subqueue_maybe_stop(netdev, txq->sq->q_id,
573				  hinic3_wq_free_wqebbs(&txq->sq->wq),
574				  txq->tx_stop_thrs,
575				  txq->tx_start_thrs);
576
577	hinic3_prepare_sq_ctrl(&wqe_combo, queue_info, num_sge, owner);
578	hinic3_write_db(txq->sq, 0, DB_CFLAG_DP_SQ,
579			hinic3_get_sq_local_pi(txq->sq));
580
581	return NETDEV_TX_OK;
582
583err_drop_pkt:
584	dev_kfree_skb_any(skb);
585
586err_out:
587	return NETDEV_TX_OK;
588}
589
590netdev_tx_t hinic3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
591{
592	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
593	u16 q_id = skb_get_queue_mapping(skb);
594
595	if (unlikely(!netif_carrier_ok(netdev)))
596		goto err_drop_pkt;
597
598	if (unlikely(q_id >= nic_dev->q_params.num_qps))
599		goto err_drop_pkt;
600
601	return hinic3_send_one_skb(skb, netdev, &nic_dev->txqs[q_id]);
602
603err_drop_pkt:
604	dev_kfree_skb_any(skb);
605
606	return NETDEV_TX_OK;
607}
608
609static bool is_hw_complete_sq_process(struct hinic3_io_queue *sq)
610{
611	u16 sw_pi, hw_ci;
612
613	sw_pi = hinic3_get_sq_local_pi(sq);
614	hw_ci = hinic3_get_sq_hw_ci(sq);
615
616	return sw_pi == hw_ci;
617}
618
619#define HINIC3_FLUSH_QUEUE_POLL_SLEEP_US   10000
620#define HINIC3_FLUSH_QUEUE_POLL_TIMEOUT_US 10000000
621static int hinic3_stop_sq(struct hinic3_txq *txq)
622{
623	struct hinic3_nic_dev *nic_dev = netdev_priv(txq->netdev);
624	int err, rc;
625
626	err = read_poll_timeout(hinic3_force_drop_tx_pkt, rc,
627				is_hw_complete_sq_process(txq->sq) || rc,
628				HINIC3_FLUSH_QUEUE_POLL_SLEEP_US,
629				HINIC3_FLUSH_QUEUE_POLL_TIMEOUT_US,
630				true, nic_dev->hwdev);
631	if (rc)
632		return rc;
633	else
634		return err;
635}
636
637/* packet transmission should be stopped before calling this function */
638void hinic3_flush_txqs(struct net_device *netdev)
639{
640	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
641	u16 qid;
642	int err;
643
644	for (qid = 0; qid < nic_dev->q_params.num_qps; qid++) {
645		err = hinic3_stop_sq(&nic_dev->txqs[qid]);
646		netdev_tx_reset_subqueue(netdev, qid);
647		if (err)
648			netdev_err(netdev, "Failed to stop sq%u\n", qid);
649	}
650}
651
652#define HINIC3_BDS_PER_SQ_WQEBB \
653	(HINIC3_SQ_WQEBB_SIZE / sizeof(struct hinic3_sq_bufdesc))
654
655int hinic3_alloc_txqs_res(struct net_device *netdev, u16 num_sq,
656			  u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res)
657{
658	struct hinic3_dyna_txq_res *tqres;
659	int idx;
660
661	for (idx = 0; idx < num_sq; idx++) {
662		tqres = &txqs_res[idx];
663
664		tqres->tx_info = kcalloc(sq_depth, sizeof(*tqres->tx_info),
665					 GFP_KERNEL);
666		if (!tqres->tx_info)
667			goto err_free_tqres;
668
669		tqres->bds = kcalloc(sq_depth * HINIC3_BDS_PER_SQ_WQEBB +
670				     HINIC3_MAX_SQ_SGE, sizeof(*tqres->bds),
671				     GFP_KERNEL);
672		if (!tqres->bds) {
673			kfree(tqres->tx_info);
674			goto err_free_tqres;
675		}
676	}
677
678	return 0;
679
680err_free_tqres:
681	while (idx > 0) {
682		idx--;
683		tqres = &txqs_res[idx];
684
685		kfree(tqres->bds);
686		kfree(tqres->tx_info);
687	}
688
689	return -ENOMEM;
690}
691
692void hinic3_free_txqs_res(struct net_device *netdev, u16 num_sq,
693			  u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res)
694{
695	struct hinic3_dyna_txq_res *tqres;
696	int idx;
697
698	for (idx = 0; idx < num_sq; idx++) {
699		tqres = &txqs_res[idx];
700
701		free_all_tx_skbs(netdev, sq_depth, tqres->tx_info);
702		kfree(tqres->bds);
703		kfree(tqres->tx_info);
704	}
705}
706
707int hinic3_configure_txqs(struct net_device *netdev, u16 num_sq,
708			  u32 sq_depth, struct hinic3_dyna_txq_res *txqs_res)
709{
710	struct hinic3_nic_dev *nic_dev = netdev_priv(netdev);
711	struct hinic3_dyna_txq_res *tqres;
712	struct hinic3_txq *txq;
713	u16 q_id;
714	u32 idx;
715
716	for (q_id = 0; q_id < num_sq; q_id++) {
717		txq = &nic_dev->txqs[q_id];
718		tqres = &txqs_res[q_id];
719
720		txq->q_depth = sq_depth;
721		txq->q_mask = sq_depth - 1;
722
723		txq->tx_stop_thrs = min(HINIC3_DEFAULT_STOP_THRS,
724					sq_depth / 20);
725		txq->tx_start_thrs = min(HINIC3_DEFAULT_START_THRS,
726					 sq_depth / 10);
727
728		txq->tx_info = tqres->tx_info;
729		for (idx = 0; idx < sq_depth; idx++)
730			txq->tx_info[idx].dma_info =
731				&tqres->bds[idx * HINIC3_BDS_PER_SQ_WQEBB];
732
733		txq->sq = &nic_dev->nic_io->sq[q_id];
734	}
735
736	return 0;
737}
738
739bool hinic3_tx_poll(struct hinic3_txq *txq, int budget)
740{
741	struct net_device *netdev = txq->netdev;
742	u16 hw_ci, sw_ci, q_id = txq->sq->q_id;
743	struct hinic3_tx_info *tx_info;
744	unsigned int bytes_compl = 0;
745	unsigned int pkts = 0;
746	u16 wqebb_cnt = 0;
747
748	hw_ci = hinic3_get_sq_hw_ci(txq->sq);
749	dma_rmb();
750	sw_ci = hinic3_get_sq_local_ci(txq->sq);
751
752	do {
753		tx_info = &txq->tx_info[sw_ci];
754
755		/* Did all wqebb of this wqe complete? */
756		if (hw_ci == sw_ci ||
757		    ((hw_ci - sw_ci) & txq->q_mask) < tx_info->wqebb_cnt)
758			break;
759
760		sw_ci = (sw_ci + tx_info->wqebb_cnt) & txq->q_mask;
761		net_prefetch(&txq->tx_info[sw_ci]);
762
763		wqebb_cnt += tx_info->wqebb_cnt;
764		bytes_compl += tx_info->skb->len;
765		pkts++;
766
767		hinic3_tx_unmap_skb(netdev, tx_info->skb, tx_info->dma_info);
768		napi_consume_skb(tx_info->skb, budget);
769		tx_info->skb = NULL;
770	} while (likely(pkts < HINIC3_TX_POLL_WEIGHT));
771
772	hinic3_wq_put_wqebbs(&txq->sq->wq, wqebb_cnt);
773
774	netif_subqueue_completed_wake(netdev, q_id, pkts, bytes_compl,
775				      hinic3_wq_free_wqebbs(&txq->sq->wq),
776				      txq->tx_start_thrs);
777
778	return pkts == HINIC3_TX_POLL_WEIGHT;
779}