hinic3_nic_io.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
  3
  4#include "hinic3_cmdq.h"
  5#include "hinic3_hw_comm.h"
  6#include "hinic3_hw_intf.h"
  7#include "hinic3_hwdev.h"
  8#include "hinic3_hwif.h"
  9#include "hinic3_nic_cfg.h"
 10#include "hinic3_nic_dev.h"
 11#include "hinic3_nic_io.h"
 12
 13#define HINIC3_DEFAULT_TX_CI_PENDING_LIMIT    1
 14#define HINIC3_DEFAULT_TX_CI_COALESCING_TIME  1
 15#define HINIC3_DEFAULT_DROP_THD_ON            (0xFFFF)
 16#define HINIC3_DEFAULT_DROP_THD_OFF           0
 17
 18#define HINIC3_CI_Q_ADDR_SIZE                (64)
 19
 20#define HINIC3_CI_TABLE_SIZE(num_qps)  \
 21	(ALIGN((num_qps) * HINIC3_CI_Q_ADDR_SIZE, HINIC3_MIN_PAGE_SIZE))
 22
 23#define HINIC3_CI_VADDR(base_addr, q_id)  \
 24	((u8 *)(base_addr) + (q_id) * HINIC3_CI_Q_ADDR_SIZE)
 25
 26#define HINIC3_CI_PADDR(base_paddr, q_id)  \
 27	((base_paddr) + (q_id) * HINIC3_CI_Q_ADDR_SIZE)
 28
 29#define SQ_WQ_PREFETCH_MAX        1
 30#define SQ_WQ_PREFETCH_MIN        1
 31#define SQ_WQ_PREFETCH_THRESHOLD  16
 32
 33#define RQ_WQ_PREFETCH_MAX        4
 34#define RQ_WQ_PREFETCH_MIN        1
 35#define RQ_WQ_PREFETCH_THRESHOLD  256
 36
 37/* (2048 - 8) / 64 */
 38#define HINIC3_Q_CTXT_MAX         31
 39
 40enum hinic3_qp_ctxt_type {
 41	HINIC3_QP_CTXT_TYPE_SQ = 0,
 42	HINIC3_QP_CTXT_TYPE_RQ = 1,
 43};
 44
 45struct hinic3_qp_ctxt_hdr {
 46	__le16 num_queues;
 47	__le16 queue_type;
 48	__le16 start_qid;
 49	__le16 rsvd;
 50};
 51
 52struct hinic3_sq_ctxt {
 53	__le32 ci_pi;
 54	__le32 drop_mode_sp;
 55	__le32 wq_pfn_hi_owner;
 56	__le32 wq_pfn_lo;
 57
 58	__le32 rsvd0;
 59	__le32 pkt_drop_thd;
 60	__le32 global_sq_id;
 61	__le32 vlan_ceq_attr;
 62
 63	__le32 pref_cache;
 64	__le32 pref_ci_owner;
 65	__le32 pref_wq_pfn_hi_ci;
 66	__le32 pref_wq_pfn_lo;
 67
 68	__le32 rsvd8;
 69	__le32 rsvd9;
 70	__le32 wq_block_pfn_hi;
 71	__le32 wq_block_pfn_lo;
 72};
 73
 74struct hinic3_rq_ctxt {
 75	__le32 ci_pi;
 76	__le32 ceq_attr;
 77	__le32 wq_pfn_hi_type_owner;
 78	__le32 wq_pfn_lo;
 79
 80	__le32 rsvd[3];
 81	__le32 cqe_sge_len;
 82
 83	__le32 pref_cache;
 84	__le32 pref_ci_owner;
 85	__le32 pref_wq_pfn_hi_ci;
 86	__le32 pref_wq_pfn_lo;
 87
 88	__le32 pi_paddr_hi;
 89	__le32 pi_paddr_lo;
 90	__le32 wq_block_pfn_hi;
 91	__le32 wq_block_pfn_lo;
 92};
 93
 94struct hinic3_sq_ctxt_block {
 95	struct hinic3_qp_ctxt_hdr cmdq_hdr;
 96	struct hinic3_sq_ctxt     sq_ctxt[HINIC3_Q_CTXT_MAX];
 97};
 98
 99struct hinic3_rq_ctxt_block {
100	struct hinic3_qp_ctxt_hdr cmdq_hdr;
101	struct hinic3_rq_ctxt     rq_ctxt[HINIC3_Q_CTXT_MAX];
102};
103
104struct hinic3_clean_queue_ctxt {
105	struct hinic3_qp_ctxt_hdr cmdq_hdr;
106	__le32                    rsvd;
107};
108
109#define SQ_CTXT_SIZE(num_sqs)  \
110	(sizeof(struct hinic3_qp_ctxt_hdr) +  \
111	(num_sqs) * sizeof(struct hinic3_sq_ctxt))
112
113#define RQ_CTXT_SIZE(num_rqs)  \
114	(sizeof(struct hinic3_qp_ctxt_hdr) +  \
115	(num_rqs) * sizeof(struct hinic3_rq_ctxt))
116
117#define SQ_CTXT_PREF_CI_HI_SHIFT           12
118#define SQ_CTXT_PREF_CI_HI(val)            ((val) >> SQ_CTXT_PREF_CI_HI_SHIFT)
119
120#define SQ_CTXT_PI_IDX_MASK                GENMASK(15, 0)
121#define SQ_CTXT_CI_IDX_MASK                GENMASK(31, 16)
122#define SQ_CTXT_CI_PI_SET(val, member)  \
123	FIELD_PREP(SQ_CTXT_##member##_MASK, val)
124
125#define SQ_CTXT_MODE_SP_FLAG_MASK          BIT(0)
126#define SQ_CTXT_MODE_PKT_DROP_MASK         BIT(1)
127#define SQ_CTXT_MODE_SET(val, member)  \
128	FIELD_PREP(SQ_CTXT_MODE_##member##_MASK, val)
129
130#define SQ_CTXT_WQ_PAGE_HI_PFN_MASK        GENMASK(19, 0)
131#define SQ_CTXT_WQ_PAGE_OWNER_MASK         BIT(23)
132#define SQ_CTXT_WQ_PAGE_SET(val, member)  \
133	FIELD_PREP(SQ_CTXT_WQ_PAGE_##member##_MASK, val)
134
135#define SQ_CTXT_PKT_DROP_THD_ON_MASK       GENMASK(15, 0)
136#define SQ_CTXT_PKT_DROP_THD_OFF_MASK      GENMASK(31, 16)
137#define SQ_CTXT_PKT_DROP_THD_SET(val, member)  \
138	FIELD_PREP(SQ_CTXT_PKT_DROP_##member##_MASK, val)
139
140#define SQ_CTXT_GLOBAL_SQ_ID_MASK          GENMASK(12, 0)
141#define SQ_CTXT_GLOBAL_QUEUE_ID_SET(val, member)  \
142	FIELD_PREP(SQ_CTXT_##member##_MASK, val)
143
144#define SQ_CTXT_VLAN_INSERT_MODE_MASK      GENMASK(20, 19)
145#define SQ_CTXT_VLAN_CEQ_EN_MASK           BIT(23)
146#define SQ_CTXT_VLAN_CEQ_SET(val, member)  \
147	FIELD_PREP(SQ_CTXT_VLAN_##member##_MASK, val)
148
149#define SQ_CTXT_PREF_CACHE_THRESHOLD_MASK  GENMASK(13, 0)
150#define SQ_CTXT_PREF_CACHE_MAX_MASK        GENMASK(24, 14)
151#define SQ_CTXT_PREF_CACHE_MIN_MASK        GENMASK(31, 25)
152
153#define SQ_CTXT_PREF_CI_HI_MASK            GENMASK(3, 0)
154#define SQ_CTXT_PREF_OWNER_MASK            BIT(4)
155
156#define SQ_CTXT_PREF_WQ_PFN_HI_MASK        GENMASK(19, 0)
157#define SQ_CTXT_PREF_CI_LOW_MASK           GENMASK(31, 20)
158#define SQ_CTXT_PREF_SET(val, member)  \
159	FIELD_PREP(SQ_CTXT_PREF_##member##_MASK, val)
160
161#define SQ_CTXT_WQ_BLOCK_PFN_HI_MASK       GENMASK(22, 0)
162#define SQ_CTXT_WQ_BLOCK_SET(val, member)  \
163	FIELD_PREP(SQ_CTXT_WQ_BLOCK_##member##_MASK, val)
164
165#define RQ_CTXT_PI_IDX_MASK                GENMASK(15, 0)
166#define RQ_CTXT_CI_IDX_MASK                GENMASK(31, 16)
167#define RQ_CTXT_CI_PI_SET(val, member)  \
168	FIELD_PREP(RQ_CTXT_##member##_MASK, val)
169
170#define RQ_CTXT_CEQ_ATTR_INTR_MASK         GENMASK(30, 21)
171#define RQ_CTXT_CEQ_ATTR_EN_MASK           BIT(31)
172#define RQ_CTXT_CEQ_ATTR_SET(val, member)  \
173	FIELD_PREP(RQ_CTXT_CEQ_ATTR_##member##_MASK, val)
174
175#define RQ_CTXT_WQ_PAGE_HI_PFN_MASK        GENMASK(19, 0)
176#define RQ_CTXT_WQ_PAGE_WQE_TYPE_MASK      GENMASK(29, 28)
177#define RQ_CTXT_WQ_PAGE_OWNER_MASK         BIT(31)
178#define RQ_CTXT_WQ_PAGE_SET(val, member)  \
179	FIELD_PREP(RQ_CTXT_WQ_PAGE_##member##_MASK, val)
180
181#define RQ_CTXT_CQE_LEN_MASK               GENMASK(29, 28)
182#define RQ_CTXT_CQE_LEN_SET(val, member)  \
183	FIELD_PREP(RQ_CTXT_##member##_MASK, val)
184
185#define RQ_CTXT_PREF_CACHE_THRESHOLD_MASK  GENMASK(13, 0)
186#define RQ_CTXT_PREF_CACHE_MAX_MASK        GENMASK(24, 14)
187#define RQ_CTXT_PREF_CACHE_MIN_MASK        GENMASK(31, 25)
188
189#define RQ_CTXT_PREF_CI_HI_MASK            GENMASK(3, 0)
190#define RQ_CTXT_PREF_OWNER_MASK            BIT(4)
191
192#define RQ_CTXT_PREF_WQ_PFN_HI_MASK        GENMASK(19, 0)
193#define RQ_CTXT_PREF_CI_LOW_MASK           GENMASK(31, 20)
194#define RQ_CTXT_PREF_SET(val, member)  \
195	FIELD_PREP(RQ_CTXT_PREF_##member##_MASK, val)
196
197#define RQ_CTXT_WQ_BLOCK_PFN_HI_MASK       GENMASK(22, 0)
198#define RQ_CTXT_WQ_BLOCK_SET(val, member)  \
199	FIELD_PREP(RQ_CTXT_WQ_BLOCK_##member##_MASK, val)
200
201#define WQ_PAGE_PFN_SHIFT       12
202#define WQ_BLOCK_PFN_SHIFT      9
203#define WQ_PAGE_PFN(page_addr)  ((page_addr) >> WQ_PAGE_PFN_SHIFT)
204#define WQ_BLOCK_PFN(page_addr) ((page_addr) >> WQ_BLOCK_PFN_SHIFT)
205
206int hinic3_init_nic_io(struct hinic3_nic_dev *nic_dev)
207{
208	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
209	struct hinic3_nic_io *nic_io;
210	int err;
211
212	nic_io = kzalloc(sizeof(*nic_io), GFP_KERNEL);
213	if (!nic_io)
214		return -ENOMEM;
215
216	nic_dev->nic_io = nic_io;
217
218	err = hinic3_set_func_svc_used_state(hwdev, COMM_FUNC_SVC_T_NIC, 1);
219	if (err) {
220		dev_err(hwdev->dev, "Failed to set function svc used state\n");
221		goto err_free_nicio;
222	}
223
224	err = hinic3_init_function_table(nic_dev);
225	if (err) {
226		dev_err(hwdev->dev, "Failed to init function table\n");
227		goto err_clear_func_svc_used_state;
228	}
229
230	nic_io->rx_buf_len = nic_dev->rx_buf_len;
231
232	err = hinic3_get_nic_feature_from_hw(nic_dev);
233	if (err) {
234		dev_err(hwdev->dev, "Failed to get nic features\n");
235		goto err_clear_func_svc_used_state;
236	}
237
238	nic_io->feature_cap &= HINIC3_NIC_F_ALL_MASK;
239	nic_io->feature_cap &= HINIC3_NIC_DRV_DEFAULT_FEATURE;
240	dev_dbg(hwdev->dev, "nic features: 0x%llx\n\n", nic_io->feature_cap);
241
242	return 0;
243
244err_clear_func_svc_used_state:
245	hinic3_set_func_svc_used_state(hwdev, COMM_FUNC_SVC_T_NIC, 0);
246err_free_nicio:
247	nic_dev->nic_io = NULL;
248	kfree(nic_io);
249
250	return err;
251}
252
253void hinic3_free_nic_io(struct hinic3_nic_dev *nic_dev)
254{
255	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
256
257	hinic3_set_func_svc_used_state(nic_dev->hwdev, COMM_FUNC_SVC_T_NIC, 0);
258	nic_dev->nic_io = NULL;
259	kfree(nic_io);
260}
261
262int hinic3_init_nicio_res(struct hinic3_nic_dev *nic_dev)
263{
264	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
265	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
266	void __iomem *db_base;
267	int err;
268
269	nic_io->max_qps = hinic3_func_max_qnum(hwdev);
270
271	err = hinic3_alloc_db_addr(hwdev, &db_base, NULL);
272	if (err) {
273		dev_err(hwdev->dev, "Failed to allocate doorbell for sqs\n");
274		return err;
275	}
276	nic_io->sqs_db_addr = db_base;
277
278	err = hinic3_alloc_db_addr(hwdev, &db_base, NULL);
279	if (err) {
280		hinic3_free_db_addr(hwdev, nic_io->sqs_db_addr);
281		dev_err(hwdev->dev, "Failed to allocate doorbell for rqs\n");
282		return err;
283	}
284	nic_io->rqs_db_addr = db_base;
285
286	nic_io->ci_vaddr_base =
287		dma_alloc_coherent(hwdev->dev,
288				   HINIC3_CI_TABLE_SIZE(nic_io->max_qps),
289				   &nic_io->ci_dma_base,
290				   GFP_KERNEL);
291	if (!nic_io->ci_vaddr_base) {
292		hinic3_free_db_addr(hwdev, nic_io->sqs_db_addr);
293		hinic3_free_db_addr(hwdev, nic_io->rqs_db_addr);
294		return -ENOMEM;
295	}
296
297	return 0;
298}
299
300void hinic3_free_nicio_res(struct hinic3_nic_dev *nic_dev)
301{
302	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
303	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
304
305	dma_free_coherent(hwdev->dev,
306			  HINIC3_CI_TABLE_SIZE(nic_io->max_qps),
307			  nic_io->ci_vaddr_base, nic_io->ci_dma_base);
308
309	hinic3_free_db_addr(hwdev, nic_io->sqs_db_addr);
310	hinic3_free_db_addr(hwdev, nic_io->rqs_db_addr);
311}
312
313static int hinic3_create_sq(struct hinic3_hwdev *hwdev,
314			    struct hinic3_io_queue *sq,
315			    u16 q_id, u32 sq_depth, u16 sq_msix_idx)
316{
317	int err;
318
319	/* sq used & hardware request init 1 */
320	sq->owner = 1;
321
322	sq->q_id = q_id;
323	sq->msix_entry_idx = sq_msix_idx;
324
325	err = hinic3_wq_create(hwdev, &sq->wq, sq_depth,
326			       BIT(HINIC3_SQ_WQEBB_SHIFT));
327	if (err) {
328		dev_err(hwdev->dev, "Failed to create tx queue %u wq\n",
329			q_id);
330		return err;
331	}
332
333	return 0;
334}
335
336static int hinic3_create_rq(struct hinic3_hwdev *hwdev,
337			    struct hinic3_io_queue *rq,
338			    u16 q_id, u32 rq_depth, u16 rq_msix_idx)
339{
340	int err;
341
342	rq->q_id = q_id;
343	rq->msix_entry_idx = rq_msix_idx;
344
345	err = hinic3_wq_create(hwdev, &rq->wq, rq_depth,
346			       BIT(HINIC3_RQ_WQEBB_SHIFT +
347				   HINIC3_NORMAL_RQ_WQE));
348	if (err) {
349		dev_err(hwdev->dev, "Failed to create rx queue %u wq\n",
350			q_id);
351		return err;
352	}
353
354	return 0;
355}
356
357static int hinic3_create_qp(struct hinic3_hwdev *hwdev,
358			    struct hinic3_io_queue *sq,
359			    struct hinic3_io_queue *rq, u16 q_id, u32 sq_depth,
360			    u32 rq_depth, u16 qp_msix_idx)
361{
362	int err;
363
364	err = hinic3_create_sq(hwdev, sq, q_id, sq_depth, qp_msix_idx);
365	if (err) {
366		dev_err(hwdev->dev, "Failed to create sq, qid: %u\n",
367			q_id);
368		return err;
369	}
370
371	err = hinic3_create_rq(hwdev, rq, q_id, rq_depth, qp_msix_idx);
372	if (err) {
373		dev_err(hwdev->dev, "Failed to create rq, qid: %u\n",
374			q_id);
375		goto err_destroy_sq_wq;
376	}
377
378	return 0;
379
380err_destroy_sq_wq:
381	hinic3_wq_destroy(hwdev, &sq->wq);
382
383	return err;
384}
385
386static void hinic3_destroy_qp(struct hinic3_hwdev *hwdev,
387			      struct hinic3_io_queue *sq,
388			      struct hinic3_io_queue *rq)
389{
390	hinic3_wq_destroy(hwdev, &sq->wq);
391	hinic3_wq_destroy(hwdev, &rq->wq);
392}
393
394int hinic3_alloc_qps(struct hinic3_nic_dev *nic_dev,
395		     struct hinic3_dyna_qp_params *qp_params)
396{
397	struct msix_entry *qps_msix_entries = nic_dev->qps_msix_entries;
398	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
399	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
400	struct hinic3_io_queue *sqs;
401	struct hinic3_io_queue *rqs;
402	u16 q_id;
403	int err;
404
405	if (qp_params->num_qps > nic_io->max_qps || !qp_params->num_qps)
406		return -EINVAL;
407
408	sqs = kcalloc(qp_params->num_qps, sizeof(*sqs), GFP_KERNEL);
409	if (!sqs) {
410		err = -ENOMEM;
411		goto err_out;
412	}
413
414	rqs = kcalloc(qp_params->num_qps, sizeof(*rqs), GFP_KERNEL);
415	if (!rqs) {
416		err = -ENOMEM;
417		goto err_free_sqs;
418	}
419
420	for (q_id = 0; q_id < qp_params->num_qps; q_id++) {
421		err = hinic3_create_qp(hwdev, &sqs[q_id], &rqs[q_id], q_id,
422				       qp_params->sq_depth, qp_params->rq_depth,
423				       qps_msix_entries[q_id].entry);
424		if (err) {
425			dev_err(hwdev->dev, "Failed to allocate qp %u, err: %d\n",
426				q_id, err);
427			goto err_destroy_qp;
428		}
429	}
430
431	qp_params->sqs = sqs;
432	qp_params->rqs = rqs;
433
434	return 0;
435
436err_destroy_qp:
437	while (q_id > 0) {
438		q_id--;
439		hinic3_destroy_qp(hwdev, &sqs[q_id], &rqs[q_id]);
440	}
441	kfree(rqs);
442err_free_sqs:
443	kfree(sqs);
444err_out:
445	return err;
446}
447
448void hinic3_free_qps(struct hinic3_nic_dev *nic_dev,
449		     struct hinic3_dyna_qp_params *qp_params)
450{
451	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
452	u16 q_id;
453
454	for (q_id = 0; q_id < qp_params->num_qps; q_id++)
455		hinic3_destroy_qp(hwdev, &qp_params->sqs[q_id],
456				  &qp_params->rqs[q_id]);
457
458	kfree(qp_params->sqs);
459	kfree(qp_params->rqs);
460}
461
462void hinic3_init_qps(struct hinic3_nic_dev *nic_dev,
463		     struct hinic3_dyna_qp_params *qp_params)
464{
465	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
466	struct hinic3_io_queue *sqs = qp_params->sqs;
467	struct hinic3_io_queue *rqs = qp_params->rqs;
468	u16 q_id;
469
470	nic_io->num_qps = qp_params->num_qps;
471	nic_io->sq = qp_params->sqs;
472	nic_io->rq = qp_params->rqs;
473	for (q_id = 0; q_id < nic_io->num_qps; q_id++) {
474		sqs[q_id].cons_idx_addr =
475			(u16 *)HINIC3_CI_VADDR(nic_io->ci_vaddr_base, q_id);
476		/* clear ci value */
477		WRITE_ONCE(*sqs[q_id].cons_idx_addr, 0);
478
479		sqs[q_id].db_addr = nic_io->sqs_db_addr;
480		rqs[q_id].db_addr = nic_io->rqs_db_addr;
481	}
482}
483
484void hinic3_uninit_qps(struct hinic3_nic_dev *nic_dev,
485		       struct hinic3_dyna_qp_params *qp_params)
486{
487	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
488
489	qp_params->sqs = nic_io->sq;
490	qp_params->rqs = nic_io->rq;
491	qp_params->num_qps = nic_io->num_qps;
492}
493
494static void hinic3_qp_prepare_cmdq_header(struct hinic3_qp_ctxt_hdr *qp_ctxt_hdr,
495					  enum hinic3_qp_ctxt_type ctxt_type,
496					  u16 num_queues, u16 q_id)
497{
498	qp_ctxt_hdr->queue_type = cpu_to_le16(ctxt_type);
499	qp_ctxt_hdr->num_queues = cpu_to_le16(num_queues);
500	qp_ctxt_hdr->start_qid = cpu_to_le16(q_id);
501	qp_ctxt_hdr->rsvd = 0;
502}
503
504static void hinic3_sq_prepare_ctxt(struct hinic3_io_queue *sq, u16 sq_id,
505				   struct hinic3_sq_ctxt *sq_ctxt)
506{
507	u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
508	u32 wq_block_pfn_hi, wq_block_pfn_lo;
509	u32 wq_page_pfn_hi, wq_page_pfn_lo;
510	u16 pi_start, ci_start;
511
512	ci_start = hinic3_get_sq_local_ci(sq);
513	pi_start = hinic3_get_sq_local_pi(sq);
514
515	wq_page_addr = hinic3_wq_get_first_wqe_page_addr(&sq->wq);
516
517	wq_page_pfn = WQ_PAGE_PFN(wq_page_addr);
518	wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
519	wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
520
521	wq_block_pfn = WQ_BLOCK_PFN(sq->wq.wq_block_paddr);
522	wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
523	wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
524
525	sq_ctxt->ci_pi =
526		cpu_to_le32(SQ_CTXT_CI_PI_SET(ci_start, CI_IDX) |
527			    SQ_CTXT_CI_PI_SET(pi_start, PI_IDX));
528
529	sq_ctxt->drop_mode_sp =
530		cpu_to_le32(SQ_CTXT_MODE_SET(0, SP_FLAG) |
531			    SQ_CTXT_MODE_SET(0, PKT_DROP));
532
533	sq_ctxt->wq_pfn_hi_owner =
534		cpu_to_le32(SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
535			    SQ_CTXT_WQ_PAGE_SET(1, OWNER));
536
537	sq_ctxt->wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo);
538
539	sq_ctxt->pkt_drop_thd =
540		cpu_to_le32(SQ_CTXT_PKT_DROP_THD_SET(HINIC3_DEFAULT_DROP_THD_ON, THD_ON) |
541			    SQ_CTXT_PKT_DROP_THD_SET(HINIC3_DEFAULT_DROP_THD_OFF, THD_OFF));
542
543	sq_ctxt->global_sq_id =
544		cpu_to_le32(SQ_CTXT_GLOBAL_QUEUE_ID_SET((u32)sq_id,
545							GLOBAL_SQ_ID));
546
547	/* enable insert c-vlan by default */
548	sq_ctxt->vlan_ceq_attr =
549		cpu_to_le32(SQ_CTXT_VLAN_CEQ_SET(0, CEQ_EN) |
550			    SQ_CTXT_VLAN_CEQ_SET(1, INSERT_MODE));
551
552	sq_ctxt->rsvd0 = 0;
553
554	sq_ctxt->pref_cache =
555		cpu_to_le32(SQ_CTXT_PREF_SET(SQ_WQ_PREFETCH_MIN, CACHE_MIN) |
556			    SQ_CTXT_PREF_SET(SQ_WQ_PREFETCH_MAX, CACHE_MAX) |
557			    SQ_CTXT_PREF_SET(SQ_WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD));
558
559	sq_ctxt->pref_ci_owner =
560		cpu_to_le32(SQ_CTXT_PREF_SET(SQ_CTXT_PREF_CI_HI(ci_start), CI_HI) |
561			    SQ_CTXT_PREF_SET(1, OWNER));
562
563	sq_ctxt->pref_wq_pfn_hi_ci =
564		cpu_to_le32(SQ_CTXT_PREF_SET(ci_start, CI_LOW) |
565			    SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_PFN_HI));
566
567	sq_ctxt->pref_wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo);
568
569	sq_ctxt->wq_block_pfn_hi =
570		cpu_to_le32(SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, PFN_HI));
571
572	sq_ctxt->wq_block_pfn_lo = cpu_to_le32(wq_block_pfn_lo);
573}
574
575static void hinic3_rq_prepare_ctxt_get_wq_info(struct hinic3_io_queue *rq,
576					       u32 *wq_page_pfn_hi,
577					       u32 *wq_page_pfn_lo,
578					       u32 *wq_block_pfn_hi,
579					       u32 *wq_block_pfn_lo)
580{
581	u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
582
583	wq_page_addr = hinic3_wq_get_first_wqe_page_addr(&rq->wq);
584
585	wq_page_pfn = WQ_PAGE_PFN(wq_page_addr);
586	*wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
587	*wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
588
589	wq_block_pfn = WQ_BLOCK_PFN(rq->wq.wq_block_paddr);
590	*wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
591	*wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
592}
593
594static void hinic3_rq_prepare_ctxt(struct hinic3_io_queue *rq,
595				   struct hinic3_rq_ctxt *rq_ctxt)
596{
597	u32 wq_block_pfn_hi, wq_block_pfn_lo;
598	u32 wq_page_pfn_hi, wq_page_pfn_lo;
599	u16 pi_start, ci_start;
600
601	ci_start = (rq->wq.cons_idx & rq->wq.idx_mask) << HINIC3_NORMAL_RQ_WQE;
602	pi_start = (rq->wq.prod_idx & rq->wq.idx_mask) << HINIC3_NORMAL_RQ_WQE;
603
604	hinic3_rq_prepare_ctxt_get_wq_info(rq, &wq_page_pfn_hi, &wq_page_pfn_lo,
605					   &wq_block_pfn_hi, &wq_block_pfn_lo);
606
607	rq_ctxt->ci_pi =
608		cpu_to_le32(RQ_CTXT_CI_PI_SET(ci_start, CI_IDX) |
609			    RQ_CTXT_CI_PI_SET(pi_start, PI_IDX));
610
611	rq_ctxt->ceq_attr =
612		cpu_to_le32(RQ_CTXT_CEQ_ATTR_SET(0, EN) |
613			    RQ_CTXT_CEQ_ATTR_SET(rq->msix_entry_idx, INTR));
614
615	rq_ctxt->wq_pfn_hi_type_owner =
616		cpu_to_le32(RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
617			    RQ_CTXT_WQ_PAGE_SET(1, OWNER));
618
619	/* use 16Byte WQE */
620	rq_ctxt->wq_pfn_hi_type_owner |=
621		cpu_to_le32(RQ_CTXT_WQ_PAGE_SET(2, WQE_TYPE));
622	rq_ctxt->cqe_sge_len = cpu_to_le32(RQ_CTXT_CQE_LEN_SET(1, CQE_LEN));
623
624	rq_ctxt->wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo);
625
626	rq_ctxt->pref_cache =
627		cpu_to_le32(RQ_CTXT_PREF_SET(RQ_WQ_PREFETCH_MIN, CACHE_MIN) |
628			    RQ_CTXT_PREF_SET(RQ_WQ_PREFETCH_MAX, CACHE_MAX) |
629			    RQ_CTXT_PREF_SET(RQ_WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD));
630
631	rq_ctxt->pref_ci_owner =
632		cpu_to_le32(RQ_CTXT_PREF_SET(SQ_CTXT_PREF_CI_HI(ci_start), CI_HI) |
633			    RQ_CTXT_PREF_SET(1, OWNER));
634
635	rq_ctxt->pref_wq_pfn_hi_ci =
636		cpu_to_le32(RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_PFN_HI) |
637			    RQ_CTXT_PREF_SET(ci_start, CI_LOW));
638
639	rq_ctxt->pref_wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo);
640
641	rq_ctxt->wq_block_pfn_hi =
642		cpu_to_le32(RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, PFN_HI));
643
644	rq_ctxt->wq_block_pfn_lo = cpu_to_le32(wq_block_pfn_lo);
645}
646
647static int init_sq_ctxts(struct hinic3_nic_dev *nic_dev)
648{
649	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
650	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
651	struct hinic3_sq_ctxt_block *sq_ctxt_block;
652	u16 q_id, curr_id, max_ctxts, i;
653	struct hinic3_sq_ctxt *sq_ctxt;
654	struct hinic3_cmd_buf *cmd_buf;
655	struct hinic3_io_queue *sq;
656	__le64 out_param;
657	int err = 0;
658
659	cmd_buf = hinic3_alloc_cmd_buf(hwdev);
660	if (!cmd_buf) {
661		dev_err(hwdev->dev, "Failed to allocate cmd buf\n");
662		return -ENOMEM;
663	}
664
665	q_id = 0;
666	while (q_id < nic_io->num_qps) {
667		sq_ctxt_block = cmd_buf->buf;
668		sq_ctxt = sq_ctxt_block->sq_ctxt;
669
670		max_ctxts = (nic_io->num_qps - q_id) > HINIC3_Q_CTXT_MAX ?
671			     HINIC3_Q_CTXT_MAX : (nic_io->num_qps - q_id);
672
673		hinic3_qp_prepare_cmdq_header(&sq_ctxt_block->cmdq_hdr,
674					      HINIC3_QP_CTXT_TYPE_SQ, max_ctxts,
675					      q_id);
676
677		for (i = 0; i < max_ctxts; i++) {
678			curr_id = q_id + i;
679			sq = &nic_io->sq[curr_id];
680			hinic3_sq_prepare_ctxt(sq, curr_id, &sq_ctxt[i]);
681		}
682
683		hinic3_cmdq_buf_swab32(sq_ctxt_block, sizeof(*sq_ctxt_block));
684
685		cmd_buf->size = cpu_to_le16(SQ_CTXT_SIZE(max_ctxts));
686		err = hinic3_cmdq_direct_resp(hwdev, MGMT_MOD_L2NIC,
687					      L2NIC_UCODE_CMD_MODIFY_QUEUE_CTX,
688					      cmd_buf, &out_param);
689		if (err || out_param) {
690			dev_err(hwdev->dev, "Failed to set SQ ctxts, err: %d, out_param: 0x%llx\n",
691				err, out_param);
692			err = -EFAULT;
693			break;
694		}
695
696		q_id += max_ctxts;
697	}
698
699	hinic3_free_cmd_buf(hwdev, cmd_buf);
700
701	return err;
702}
703
704static int init_rq_ctxts(struct hinic3_nic_dev *nic_dev)
705{
706	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
707	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
708	struct hinic3_rq_ctxt_block *rq_ctxt_block;
709	u16 q_id, curr_id, max_ctxts, i;
710	struct hinic3_rq_ctxt *rq_ctxt;
711	struct hinic3_cmd_buf *cmd_buf;
712	struct hinic3_io_queue *rq;
713	__le64 out_param;
714	int err = 0;
715
716	cmd_buf = hinic3_alloc_cmd_buf(hwdev);
717	if (!cmd_buf) {
718		dev_err(hwdev->dev, "Failed to allocate cmd buf\n");
719		return -ENOMEM;
720	}
721
722	q_id = 0;
723	while (q_id < nic_io->num_qps) {
724		rq_ctxt_block = cmd_buf->buf;
725		rq_ctxt = rq_ctxt_block->rq_ctxt;
726
727		max_ctxts = (nic_io->num_qps - q_id) > HINIC3_Q_CTXT_MAX ?
728				HINIC3_Q_CTXT_MAX : (nic_io->num_qps - q_id);
729
730		hinic3_qp_prepare_cmdq_header(&rq_ctxt_block->cmdq_hdr,
731					      HINIC3_QP_CTXT_TYPE_RQ, max_ctxts,
732					      q_id);
733
734		for (i = 0; i < max_ctxts; i++) {
735			curr_id = q_id + i;
736			rq = &nic_io->rq[curr_id];
737			hinic3_rq_prepare_ctxt(rq, &rq_ctxt[i]);
738		}
739
740		hinic3_cmdq_buf_swab32(rq_ctxt_block, sizeof(*rq_ctxt_block));
741
742		cmd_buf->size = cpu_to_le16(RQ_CTXT_SIZE(max_ctxts));
743
744		err = hinic3_cmdq_direct_resp(hwdev, MGMT_MOD_L2NIC,
745					      L2NIC_UCODE_CMD_MODIFY_QUEUE_CTX,
746					      cmd_buf, &out_param);
747		if (err || out_param) {
748			dev_err(hwdev->dev, "Failed to set RQ ctxts, err: %d, out_param: 0x%llx\n",
749				err, out_param);
750			err = -EFAULT;
751			break;
752		}
753
754		q_id += max_ctxts;
755	}
756
757	hinic3_free_cmd_buf(hwdev, cmd_buf);
758
759	return err;
760}
761
762static int init_qp_ctxts(struct hinic3_nic_dev *nic_dev)
763{
764	int err;
765
766	err = init_sq_ctxts(nic_dev);
767	if (err)
768		return err;
769
770	err = init_rq_ctxts(nic_dev);
771	if (err)
772		return err;
773
774	return 0;
775}
776
777static int clean_queue_offload_ctxt(struct hinic3_nic_dev *nic_dev,
778				    enum hinic3_qp_ctxt_type ctxt_type)
779{
780	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
781	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
782	struct hinic3_clean_queue_ctxt *ctxt_block;
783	struct hinic3_cmd_buf *cmd_buf;
784	__le64 out_param;
785	int err;
786
787	cmd_buf = hinic3_alloc_cmd_buf(hwdev);
788	if (!cmd_buf) {
789		dev_err(hwdev->dev, "Failed to allocate cmd buf\n");
790		return -ENOMEM;
791	}
792
793	ctxt_block = cmd_buf->buf;
794	ctxt_block->cmdq_hdr.num_queues = cpu_to_le16(nic_io->max_qps);
795	ctxt_block->cmdq_hdr.queue_type = cpu_to_le16(ctxt_type);
796	ctxt_block->cmdq_hdr.start_qid = 0;
797	ctxt_block->cmdq_hdr.rsvd = 0;
798	ctxt_block->rsvd = 0;
799
800	hinic3_cmdq_buf_swab32(ctxt_block, sizeof(*ctxt_block));
801
802	cmd_buf->size = cpu_to_le16(sizeof(*ctxt_block));
803
804	err = hinic3_cmdq_direct_resp(hwdev, MGMT_MOD_L2NIC,
805				      L2NIC_UCODE_CMD_CLEAN_QUEUE_CTX,
806				      cmd_buf, &out_param);
807	if (err || out_param) {
808		dev_err(hwdev->dev, "Failed to clean queue offload ctxts, err: %d,out_param: 0x%llx\n",
809			err, out_param);
810
811		err = -EFAULT;
812	}
813
814	hinic3_free_cmd_buf(hwdev, cmd_buf);
815
816	return err;
817}
818
819static int clean_qp_offload_ctxt(struct hinic3_nic_dev *nic_dev)
820{
821	/* clean LRO/TSO context space */
822	return clean_queue_offload_ctxt(nic_dev, HINIC3_QP_CTXT_TYPE_SQ) ||
823	       clean_queue_offload_ctxt(nic_dev, HINIC3_QP_CTXT_TYPE_RQ);
824}
825
826/* init qps ctxt and set sq ci attr and arm all sq */
827int hinic3_init_qp_ctxts(struct hinic3_nic_dev *nic_dev)
828{
829	struct hinic3_nic_io *nic_io = nic_dev->nic_io;
830	struct hinic3_hwdev *hwdev = nic_dev->hwdev;
831	struct hinic3_sq_attr sq_attr;
832	u32 rq_depth;
833	u16 q_id;
834	int err;
835
836	err = init_qp_ctxts(nic_dev);
837	if (err) {
838		dev_err(hwdev->dev, "Failed to init QP ctxts\n");
839		return err;
840	}
841
842	/* clean LRO/TSO context space */
843	err = clean_qp_offload_ctxt(nic_dev);
844	if (err) {
845		dev_err(hwdev->dev, "Failed to clean qp offload ctxts\n");
846		return err;
847	}
848
849	rq_depth = nic_io->rq[0].wq.q_depth << HINIC3_NORMAL_RQ_WQE;
850
851	err = hinic3_set_root_ctxt(hwdev, rq_depth, nic_io->sq[0].wq.q_depth,
852				   nic_io->rx_buf_len);
853	if (err) {
854		dev_err(hwdev->dev, "Failed to set root context\n");
855		return err;
856	}
857
858	for (q_id = 0; q_id < nic_io->num_qps; q_id++) {
859		sq_attr.ci_dma_base =
860			HINIC3_CI_PADDR(nic_io->ci_dma_base, q_id) >> 0x2;
861		sq_attr.pending_limit = HINIC3_DEFAULT_TX_CI_PENDING_LIMIT;
862		sq_attr.coalescing_time = HINIC3_DEFAULT_TX_CI_COALESCING_TIME;
863		sq_attr.intr_en = 1;
864		sq_attr.intr_idx = nic_io->sq[q_id].msix_entry_idx;
865		sq_attr.l2nic_sqn = q_id;
866		sq_attr.dma_attr_off = 0;
867		err = hinic3_set_ci_table(hwdev, &sq_attr);
868		if (err) {
869			dev_err(hwdev->dev, "Failed to set ci table\n");
870			goto err_clean_root_ctxt;
871		}
872	}
873
874	return 0;
875
876err_clean_root_ctxt:
877	hinic3_clean_root_ctxt(hwdev);
878
879	return err;
880}
881
882void hinic3_free_qp_ctxts(struct hinic3_nic_dev *nic_dev)
883{
884	hinic3_clean_root_ctxt(nic_dev->hwdev);
885}