hinic3_cmdq.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved.
  3
  4#include <linux/bitfield.h>
  5#include <linux/delay.h>
  6#include <linux/dma-mapping.h>
  7
  8#include "hinic3_cmdq.h"
  9#include "hinic3_hwdev.h"
 10#include "hinic3_hwif.h"
 11#include "hinic3_mbox.h"
 12
 13#define CMDQ_BUF_SIZE             2048
 14#define CMDQ_WQEBB_SIZE           64
 15
 16#define CMDQ_CMD_TIMEOUT          5000
 17#define CMDQ_ENABLE_WAIT_TIMEOUT  300
 18
 19#define CMDQ_CTXT_CURR_WQE_PAGE_PFN_MASK  GENMASK_ULL(51, 0)
 20#define CMDQ_CTXT_EQ_ID_MASK              GENMASK_ULL(60, 53)
 21#define CMDQ_CTXT_CEQ_ARM_MASK            BIT_ULL(61)
 22#define CMDQ_CTXT_CEQ_EN_MASK             BIT_ULL(62)
 23#define CMDQ_CTXT_HW_BUSY_BIT_MASK        BIT_ULL(63)
 24
 25#define CMDQ_CTXT_WQ_BLOCK_PFN_MASK       GENMASK_ULL(51, 0)
 26#define CMDQ_CTXT_CI_MASK                 GENMASK_ULL(63, 52)
 27#define CMDQ_CTXT_SET(val, member)  \
 28	FIELD_PREP(CMDQ_CTXT_##member##_MASK, val)
 29
 30#define CMDQ_WQE_HDR_BUFDESC_LEN_MASK        GENMASK(7, 0)
 31#define CMDQ_WQE_HDR_COMPLETE_FMT_MASK       BIT(15)
 32#define CMDQ_WQE_HDR_DATA_FMT_MASK           BIT(22)
 33#define CMDQ_WQE_HDR_COMPLETE_REQ_MASK       BIT(23)
 34#define CMDQ_WQE_HDR_COMPLETE_SECT_LEN_MASK  GENMASK(28, 27)
 35#define CMDQ_WQE_HDR_CTRL_LEN_MASK           GENMASK(30, 29)
 36#define CMDQ_WQE_HDR_HW_BUSY_BIT_MASK        BIT(31)
 37#define CMDQ_WQE_HDR_SET(val, member)  \
 38	FIELD_PREP(CMDQ_WQE_HDR_##member##_MASK, val)
 39#define CMDQ_WQE_HDR_GET(val, member)  \
 40	FIELD_GET(CMDQ_WQE_HDR_##member##_MASK, le32_to_cpu(val))
 41
 42#define CMDQ_CTRL_PI_MASK              GENMASK(15, 0)
 43#define CMDQ_CTRL_CMD_MASK             GENMASK(23, 16)
 44#define CMDQ_CTRL_MOD_MASK             GENMASK(28, 24)
 45#define CMDQ_CTRL_HW_BUSY_BIT_MASK     BIT(31)
 46#define CMDQ_CTRL_SET(val, member)  \
 47	FIELD_PREP(CMDQ_CTRL_##member##_MASK, val)
 48#define CMDQ_CTRL_GET(val, member)  \
 49	FIELD_GET(CMDQ_CTRL_##member##_MASK, val)
 50
 51#define CMDQ_WQE_ERRCODE_VAL_MASK      GENMASK(30, 0)
 52#define CMDQ_WQE_ERRCODE_GET(val, member)  \
 53	FIELD_GET(CMDQ_WQE_ERRCODE_##member##_MASK, le32_to_cpu(val))
 54
 55#define CMDQ_DB_INFO_HI_PROD_IDX_MASK  GENMASK(7, 0)
 56#define CMDQ_DB_INFO_SET(val, member)  \
 57	FIELD_PREP(CMDQ_DB_INFO_##member##_MASK, val)
 58
 59#define CMDQ_DB_HEAD_QUEUE_TYPE_MASK   BIT(23)
 60#define CMDQ_DB_HEAD_CMDQ_TYPE_MASK    GENMASK(26, 24)
 61#define CMDQ_DB_HEAD_SET(val, member)  \
 62	FIELD_PREP(CMDQ_DB_HEAD_##member##_MASK, val)
 63
 64#define CMDQ_CEQE_TYPE_MASK            GENMASK(2, 0)
 65#define CMDQ_CEQE_GET(val, member)  \
 66	FIELD_GET(CMDQ_CEQE_##member##_MASK, le32_to_cpu(val))
 67
 68#define CMDQ_WQE_HEADER(wqe)           ((struct cmdq_header *)(wqe))
 69#define CMDQ_WQE_COMPLETED(ctrl_info)  \
 70	CMDQ_CTRL_GET(le32_to_cpu(ctrl_info), HW_BUSY_BIT)
 71
 72#define CMDQ_PFN(addr)  ((addr) >> 12)
 73
 74/* cmdq work queue's chip logical address table is up to 512B */
 75#define CMDQ_WQ_CLA_SIZE  512
 76
 77/* Completion codes: send, direct sync, force stop */
 78#define CMDQ_SEND_CMPT_CODE         10
 79#define CMDQ_DIRECT_SYNC_CMPT_CODE  11
 80#define CMDQ_FORCE_STOP_CMPT_CODE   12
 81
 82enum cmdq_data_format {
 83	CMDQ_DATA_SGE    = 0,
 84	CMDQ_DATA_DIRECT = 1,
 85};
 86
 87enum cmdq_ctrl_sect_len {
 88	CMDQ_CTRL_SECT_LEN        = 1,
 89	CMDQ_CTRL_DIRECT_SECT_LEN = 2,
 90};
 91
 92enum cmdq_bufdesc_len {
 93	CMDQ_BUFDESC_LCMD_LEN = 2,
 94	CMDQ_BUFDESC_SCMD_LEN = 3,
 95};
 96
 97enum cmdq_completion_format {
 98	CMDQ_COMPLETE_DIRECT = 0,
 99	CMDQ_COMPLETE_SGE    = 1,
100};
101
102enum cmdq_cmd_type {
103	CMDQ_CMD_DIRECT_RESP,
104	CMDQ_CMD_SGE_RESP,
105};
106
107#define CMDQ_WQE_NUM_WQEBBS  1
108
109static struct cmdq_wqe *cmdq_read_wqe(struct hinic3_wq *wq, u16 *ci)
110{
111	if (hinic3_wq_get_used(wq) == 0)
112		return NULL;
113
114	*ci = wq->cons_idx & wq->idx_mask;
115
116	return get_q_element(&wq->qpages, wq->cons_idx, NULL);
117}
118
119struct hinic3_cmd_buf *hinic3_alloc_cmd_buf(struct hinic3_hwdev *hwdev)
120{
121	struct hinic3_cmd_buf *cmd_buf;
122	struct hinic3_cmdqs *cmdqs;
123
124	cmdqs = hwdev->cmdqs;
125
126	cmd_buf = kmalloc(sizeof(*cmd_buf), GFP_ATOMIC);
127	if (!cmd_buf)
128		return NULL;
129
130	cmd_buf->buf = dma_pool_alloc(cmdqs->cmd_buf_pool, GFP_ATOMIC,
131				      &cmd_buf->dma_addr);
132	if (!cmd_buf->buf) {
133		dev_err(hwdev->dev, "Failed to allocate cmdq cmd buf from the pool\n");
134		goto err_free_cmd_buf;
135	}
136
137	cmd_buf->size = cpu_to_le16(CMDQ_BUF_SIZE);
138	refcount_set(&cmd_buf->ref_cnt, 1);
139
140	return cmd_buf;
141
142err_free_cmd_buf:
143	kfree(cmd_buf);
144
145	return NULL;
146}
147
148void hinic3_free_cmd_buf(struct hinic3_hwdev *hwdev,
149			 struct hinic3_cmd_buf *cmd_buf)
150{
151	struct hinic3_cmdqs *cmdqs;
152
153	if (!refcount_dec_and_test(&cmd_buf->ref_cnt))
154		return;
155
156	cmdqs = hwdev->cmdqs;
157
158	dma_pool_free(cmdqs->cmd_buf_pool, cmd_buf->buf, cmd_buf->dma_addr);
159	kfree(cmd_buf);
160}
161
162static void cmdq_clear_cmd_buf(struct hinic3_cmdq_cmd_info *cmd_info,
163			       struct hinic3_hwdev *hwdev)
164{
165	if (cmd_info->buf_in) {
166		hinic3_free_cmd_buf(hwdev, cmd_info->buf_in);
167		cmd_info->buf_in = NULL;
168	}
169}
170
171static void clear_wqe_complete_bit(struct hinic3_cmdq *cmdq,
172				   struct cmdq_wqe *wqe, u16 ci)
173{
174	struct cmdq_header *hdr = CMDQ_WQE_HEADER(wqe);
175	__le32 header_info = hdr->header_info;
176	enum cmdq_data_format df;
177	struct cmdq_ctrl *ctrl;
178
179	df = CMDQ_WQE_HDR_GET(header_info, DATA_FMT);
180	if (df == CMDQ_DATA_SGE)
181		ctrl = &wqe->wqe_lcmd.ctrl;
182	else
183		ctrl = &wqe->wqe_scmd.ctrl;
184
185	/* clear HW busy bit */
186	ctrl->ctrl_info = 0;
187	cmdq->cmd_infos[ci].cmd_type = HINIC3_CMD_TYPE_NONE;
188	wmb(); /* verify wqe is clear before updating ci */
189	hinic3_wq_put_wqebbs(&cmdq->wq, CMDQ_WQE_NUM_WQEBBS);
190}
191
192static void cmdq_update_cmd_status(struct hinic3_cmdq *cmdq, u16 prod_idx,
193				   struct cmdq_wqe *wqe)
194{
195	struct hinic3_cmdq_cmd_info *cmd_info;
196	struct cmdq_wqe_lcmd *wqe_lcmd;
197	__le32 status_info;
198
199	wqe_lcmd = &wqe->wqe_lcmd;
200	cmd_info = &cmdq->cmd_infos[prod_idx];
201	if (cmd_info->errcode) {
202		status_info = wqe_lcmd->status.status_info;
203		*cmd_info->errcode = CMDQ_WQE_ERRCODE_GET(status_info, VAL);
204	}
205
206	if (cmd_info->direct_resp)
207		*cmd_info->direct_resp = wqe_lcmd->completion.resp.direct.val;
208}
209
210static void cmdq_sync_cmd_handler(struct hinic3_cmdq *cmdq,
211				  struct cmdq_wqe *wqe, u16 ci)
212{
213	spin_lock(&cmdq->cmdq_lock);
214	cmdq_update_cmd_status(cmdq, ci, wqe);
215	if (cmdq->cmd_infos[ci].cmpt_code) {
216		*cmdq->cmd_infos[ci].cmpt_code = CMDQ_DIRECT_SYNC_CMPT_CODE;
217		cmdq->cmd_infos[ci].cmpt_code = NULL;
218	}
219
220	/* Ensure that completion code has been updated before updating done */
221	smp_wmb();
222	if (cmdq->cmd_infos[ci].done) {
223		complete(cmdq->cmd_infos[ci].done);
224		cmdq->cmd_infos[ci].done = NULL;
225	}
226	spin_unlock(&cmdq->cmdq_lock);
227
228	cmdq_clear_cmd_buf(&cmdq->cmd_infos[ci], cmdq->hwdev);
229	clear_wqe_complete_bit(cmdq, wqe, ci);
230}
231
232void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data)
233{
234	enum hinic3_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE);
235	struct hinic3_cmdqs *cmdqs = hwdev->cmdqs;
236	struct hinic3_cmdq_cmd_info *cmd_info;
237	struct cmdq_wqe_lcmd *wqe_lcmd;
238	struct hinic3_cmdq *cmdq;
239	struct cmdq_wqe *wqe;
240	__le32 ctrl_info;
241	u16 ci;
242
243	if (unlikely(cmdq_type >= ARRAY_SIZE(cmdqs->cmdq)))
244		return;
245
246	cmdq = &cmdqs->cmdq[cmdq_type];
247	while ((wqe = cmdq_read_wqe(&cmdq->wq, &ci)) != NULL) {
248		cmd_info = &cmdq->cmd_infos[ci];
249		switch (cmd_info->cmd_type) {
250		case HINIC3_CMD_TYPE_NONE:
251			return;
252		case HINIC3_CMD_TYPE_TIMEOUT:
253			dev_warn(hwdev->dev, "Cmdq timeout, q_id: %u, ci: %u\n",
254				 cmdq_type, ci);
255			fallthrough;
256		case HINIC3_CMD_TYPE_FAKE_TIMEOUT:
257			cmdq_clear_cmd_buf(cmd_info, hwdev);
258			clear_wqe_complete_bit(cmdq, wqe, ci);
259			break;
260		default:
261			/* only arm bit is using scmd wqe,
262			 * the other wqe is lcmd
263			 */
264			wqe_lcmd = &wqe->wqe_lcmd;
265			ctrl_info = wqe_lcmd->ctrl.ctrl_info;
266			if (!CMDQ_WQE_COMPLETED(ctrl_info))
267				return;
268
269			dma_rmb();
270			/* For FORCE_STOP cmd_type, we also need to wait for
271			 * the firmware processing to complete to prevent the
272			 * firmware from accessing the released cmd_buf
273			 */
274			if (cmd_info->cmd_type == HINIC3_CMD_TYPE_FORCE_STOP) {
275				cmdq_clear_cmd_buf(cmd_info, hwdev);
276				clear_wqe_complete_bit(cmdq, wqe, ci);
277			} else {
278				cmdq_sync_cmd_handler(cmdq, wqe, ci);
279			}
280
281			break;
282		}
283	}
284}
285
286static int wait_cmdqs_enable(struct hinic3_cmdqs *cmdqs)
287{
288	unsigned long end;
289
290	end = jiffies + msecs_to_jiffies(CMDQ_ENABLE_WAIT_TIMEOUT);
291	do {
292		if (cmdqs->status & HINIC3_CMDQ_ENABLE)
293			return 0;
294		usleep_range(1000, 2000);
295	} while (time_before(jiffies, end) && !cmdqs->disable_flag);
296
297	cmdqs->disable_flag = 1;
298
299	return -EBUSY;
300}
301
302static void cmdq_set_completion(struct cmdq_completion *complete,
303				struct hinic3_cmd_buf *buf_out)
304{
305	struct hinic3_sge *sge = &complete->resp.sge;
306
307	hinic3_set_sge(sge, buf_out->dma_addr, cpu_to_le32(CMDQ_BUF_SIZE));
308}
309
310static struct cmdq_wqe *cmdq_get_wqe(struct hinic3_wq *wq, u16 *pi)
311{
312	if (!hinic3_wq_free_wqebbs(wq))
313		return NULL;
314
315	return hinic3_wq_get_one_wqebb(wq, pi);
316}
317
318static void cmdq_set_lcmd_bufdesc(struct cmdq_wqe_lcmd *wqe,
319				  struct hinic3_cmd_buf *buf_in)
320{
321	hinic3_set_sge(&wqe->buf_desc.sge, buf_in->dma_addr,
322		       (__force __le32)buf_in->size);
323}
324
325static void cmdq_set_db(struct hinic3_cmdq *cmdq,
326			enum hinic3_cmdq_type cmdq_type, u16 prod_idx)
327{
328	u8 __iomem *db_base = cmdq->hwdev->cmdqs->cmdqs_db_base;
329	u16 db_ofs = (prod_idx & 0xFF) << 3;
330	struct cmdq_db db;
331
332	db.db_info = cpu_to_le32(CMDQ_DB_INFO_SET(prod_idx >> 8, HI_PROD_IDX));
333	db.db_head = cpu_to_le32(CMDQ_DB_HEAD_SET(1, QUEUE_TYPE) |
334				 CMDQ_DB_HEAD_SET(cmdq_type, CMDQ_TYPE));
335	writeq(*(u64 *)&db, db_base + db_ofs);
336}
337
338static void cmdq_wqe_fill(struct cmdq_wqe *hw_wqe,
339			  const struct cmdq_wqe *shadow_wqe)
340{
341	const struct cmdq_header *src = (struct cmdq_header *)shadow_wqe;
342	struct cmdq_header *dst = (struct cmdq_header *)hw_wqe;
343	size_t len;
344
345	len = sizeof(struct cmdq_wqe) - sizeof(struct cmdq_header);
346	memcpy(dst + 1, src + 1, len);
347	/* Ensure buffer len before updating header */
348	wmb();
349	WRITE_ONCE(*dst, *src);
350}
351
352static void cmdq_prepare_wqe_ctrl(struct cmdq_wqe *wqe, u8 wrapped,
353				  u8 mod, u8 cmd, u16 prod_idx,
354				  enum cmdq_completion_format complete_format,
355				  enum cmdq_data_format data_format,
356				  enum cmdq_bufdesc_len buf_len)
357{
358	struct cmdq_header *hdr = CMDQ_WQE_HEADER(wqe);
359	enum cmdq_ctrl_sect_len ctrl_len;
360	struct cmdq_wqe_lcmd *wqe_lcmd;
361	struct cmdq_wqe_scmd *wqe_scmd;
362	struct cmdq_ctrl *ctrl;
363
364	if (data_format == CMDQ_DATA_SGE) {
365		wqe_lcmd = &wqe->wqe_lcmd;
366		wqe_lcmd->status.status_info = 0;
367		ctrl = &wqe_lcmd->ctrl;
368		ctrl_len = CMDQ_CTRL_SECT_LEN;
369	} else {
370		wqe_scmd = &wqe->wqe_scmd;
371		wqe_scmd->status.status_info = 0;
372		ctrl = &wqe_scmd->ctrl;
373		ctrl_len = CMDQ_CTRL_DIRECT_SECT_LEN;
374	}
375
376	ctrl->ctrl_info =
377		cpu_to_le32(CMDQ_CTRL_SET(prod_idx, PI) |
378			    CMDQ_CTRL_SET(cmd, CMD) |
379			    CMDQ_CTRL_SET(mod, MOD));
380
381	hdr->header_info =
382		cpu_to_le32(CMDQ_WQE_HDR_SET(buf_len, BUFDESC_LEN) |
383			    CMDQ_WQE_HDR_SET(complete_format, COMPLETE_FMT) |
384			    CMDQ_WQE_HDR_SET(data_format, DATA_FMT) |
385			    CMDQ_WQE_HDR_SET(1, COMPLETE_REQ) |
386			    CMDQ_WQE_HDR_SET(3, COMPLETE_SECT_LEN) |
387			    CMDQ_WQE_HDR_SET(ctrl_len, CTRL_LEN) |
388			    CMDQ_WQE_HDR_SET(wrapped, HW_BUSY_BIT));
389}
390
391static void cmdq_set_lcmd_wqe(struct cmdq_wqe *wqe,
392			      enum cmdq_cmd_type cmd_type,
393			      struct hinic3_cmd_buf *buf_in,
394			      struct hinic3_cmd_buf *buf_out,
395			      u8 wrapped, u8 mod, u8 cmd, u16 prod_idx)
396{
397	enum cmdq_completion_format complete_format = CMDQ_COMPLETE_DIRECT;
398	struct cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
399
400	switch (cmd_type) {
401	case CMDQ_CMD_DIRECT_RESP:
402		wqe_lcmd->completion.resp.direct.val = 0;
403		break;
404	case CMDQ_CMD_SGE_RESP:
405		if (buf_out) {
406			complete_format = CMDQ_COMPLETE_SGE;
407			cmdq_set_completion(&wqe_lcmd->completion, buf_out);
408		}
409		break;
410	}
411
412	cmdq_prepare_wqe_ctrl(wqe, wrapped, mod, cmd, prod_idx, complete_format,
413			      CMDQ_DATA_SGE, CMDQ_BUFDESC_LCMD_LEN);
414	cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
415}
416
417static int hinic3_cmdq_sync_timeout_check(struct hinic3_cmdq *cmdq,
418					  struct cmdq_wqe *wqe, u16 pi)
419{
420	struct cmdq_wqe_lcmd *wqe_lcmd;
421	struct cmdq_ctrl *ctrl;
422	__le32 ctrl_info;
423
424	wqe_lcmd = &wqe->wqe_lcmd;
425	ctrl = &wqe_lcmd->ctrl;
426	ctrl_info = ctrl->ctrl_info;
427	if (!CMDQ_WQE_COMPLETED(ctrl_info)) {
428		dev_dbg(cmdq->hwdev->dev, "Cmdq sync command check busy bit not set\n");
429		return -EFAULT;
430	}
431	cmdq_update_cmd_status(cmdq, pi, wqe);
432
433	return 0;
434}
435
436static void clear_cmd_info(struct hinic3_cmdq_cmd_info *cmd_info,
437			   const struct hinic3_cmdq_cmd_info *saved_cmd_info)
438{
439	if (cmd_info->errcode == saved_cmd_info->errcode)
440		cmd_info->errcode = NULL;
441
442	if (cmd_info->done == saved_cmd_info->done)
443		cmd_info->done = NULL;
444
445	if (cmd_info->direct_resp == saved_cmd_info->direct_resp)
446		cmd_info->direct_resp = NULL;
447}
448
449static int wait_cmdq_sync_cmd_completion(struct hinic3_cmdq *cmdq,
450					 struct hinic3_cmdq_cmd_info *cmd_info,
451					 struct hinic3_cmdq_cmd_info *saved_cmd_info,
452					 u64 curr_msg_id, u16 curr_prod_idx,
453					 struct cmdq_wqe *curr_wqe,
454					 u32 timeout)
455{
456	ulong timeo = msecs_to_jiffies(timeout);
457	int err;
458
459	if (wait_for_completion_timeout(saved_cmd_info->done, timeo))
460		return 0;
461
462	spin_lock_bh(&cmdq->cmdq_lock);
463	if (cmd_info->cmpt_code == saved_cmd_info->cmpt_code)
464		cmd_info->cmpt_code = NULL;
465
466	if (*saved_cmd_info->cmpt_code == CMDQ_DIRECT_SYNC_CMPT_CODE) {
467		dev_dbg(cmdq->hwdev->dev, "Cmdq direct sync command has been completed\n");
468		spin_unlock_bh(&cmdq->cmdq_lock);
469		return 0;
470	}
471
472	if (curr_msg_id == cmd_info->cmdq_msg_id) {
473		err = hinic3_cmdq_sync_timeout_check(cmdq, curr_wqe,
474						     curr_prod_idx);
475		if (err)
476			cmd_info->cmd_type = HINIC3_CMD_TYPE_TIMEOUT;
477		else
478			cmd_info->cmd_type = HINIC3_CMD_TYPE_FAKE_TIMEOUT;
479	} else {
480		err = -ETIMEDOUT;
481		dev_err(cmdq->hwdev->dev,
482			"Cmdq sync command current msg id mismatch cmd_info msg id\n");
483	}
484
485	clear_cmd_info(cmd_info, saved_cmd_info);
486	spin_unlock_bh(&cmdq->cmdq_lock);
487
488	return err;
489}
490
491static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd,
492				     struct hinic3_cmd_buf *buf_in,
493				     __le64 *out_param)
494{
495	struct hinic3_cmdq_cmd_info *cmd_info, saved_cmd_info;
496	int cmpt_code = CMDQ_SEND_CMPT_CODE;
497	struct cmdq_wqe *curr_wqe, wqe = {};
498	struct hinic3_wq *wq = &cmdq->wq;
499	u16 curr_prod_idx, next_prod_idx;
500	struct completion done;
501	u64 curr_msg_id;
502	int errcode;
503	u8 wrapped;
504	int err;
505
506	spin_lock_bh(&cmdq->cmdq_lock);
507	curr_wqe = cmdq_get_wqe(wq, &curr_prod_idx);
508	if (!curr_wqe) {
509		spin_unlock_bh(&cmdq->cmdq_lock);
510		return -EBUSY;
511	}
512
513	wrapped = cmdq->wrapped;
514	next_prod_idx = curr_prod_idx + CMDQ_WQE_NUM_WQEBBS;
515	if (next_prod_idx >= wq->q_depth) {
516		cmdq->wrapped ^= 1;
517		next_prod_idx -= wq->q_depth;
518	}
519
520	cmd_info = &cmdq->cmd_infos[curr_prod_idx];
521	init_completion(&done);
522	refcount_inc(&buf_in->ref_cnt);
523	cmd_info->cmd_type = HINIC3_CMD_TYPE_DIRECT_RESP;
524	cmd_info->done = &done;
525	cmd_info->errcode = &errcode;
526	cmd_info->direct_resp = out_param;
527	cmd_info->cmpt_code = &cmpt_code;
528	cmd_info->buf_in = buf_in;
529	saved_cmd_info = *cmd_info;
530	cmdq_set_lcmd_wqe(&wqe, CMDQ_CMD_DIRECT_RESP, buf_in, NULL,
531			  wrapped, mod, cmd, curr_prod_idx);
532
533	cmdq_wqe_fill(curr_wqe, &wqe);
534	(cmd_info->cmdq_msg_id)++;
535	curr_msg_id = cmd_info->cmdq_msg_id;
536	cmdq_set_db(cmdq, HINIC3_CMDQ_SYNC, next_prod_idx);
537	spin_unlock_bh(&cmdq->cmdq_lock);
538
539	err = wait_cmdq_sync_cmd_completion(cmdq, cmd_info, &saved_cmd_info,
540					    curr_msg_id, curr_prod_idx,
541					    curr_wqe, CMDQ_CMD_TIMEOUT);
542	if (err) {
543		dev_err(cmdq->hwdev->dev,
544			"Cmdq sync command timeout, mod: %u, cmd: %u, prod idx: 0x%x\n",
545			mod, cmd, curr_prod_idx);
546		err = -ETIMEDOUT;
547	}
548
549	if (cmpt_code == CMDQ_FORCE_STOP_CMPT_CODE) {
550		dev_dbg(cmdq->hwdev->dev,
551			"Force stop cmdq cmd, mod: %u, cmd: %u\n", mod, cmd);
552		err = -EAGAIN;
553	}
554
555	smp_rmb(); /* read error code after completion */
556
557	return err ? err : errcode;
558}
559
560int hinic3_cmdq_direct_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd,
561			    struct hinic3_cmd_buf *buf_in, __le64 *out_param)
562{
563	struct hinic3_cmdqs *cmdqs;
564	int err;
565
566	cmdqs = hwdev->cmdqs;
567	err = wait_cmdqs_enable(cmdqs);
568	if (err) {
569		dev_err(hwdev->dev, "Cmdq is disabled\n");
570		return err;
571	}
572
573	err = cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC3_CMDQ_SYNC],
574					mod, cmd, buf_in, out_param);
575
576	return err;
577}
578
579static void cmdq_init_queue_ctxt(struct hinic3_hwdev *hwdev, u8 cmdq_id,
580				 struct comm_cmdq_ctxt_info *ctxt_info)
581{
582	const struct hinic3_cmdqs *cmdqs;
583	u64 cmdq_first_block_paddr, pfn;
584	const struct hinic3_wq *wq;
585
586	cmdqs = hwdev->cmdqs;
587	wq = &cmdqs->cmdq[cmdq_id].wq;
588	pfn = CMDQ_PFN(hinic3_wq_get_first_wqe_page_addr(wq));
589
590	ctxt_info->curr_wqe_page_pfn =
591		cpu_to_le64(CMDQ_CTXT_SET(1, HW_BUSY_BIT) |
592			    CMDQ_CTXT_SET(1, CEQ_EN)	|
593			    CMDQ_CTXT_SET(1, CEQ_ARM)	|
594			    CMDQ_CTXT_SET(0, EQ_ID) |
595			    CMDQ_CTXT_SET(pfn, CURR_WQE_PAGE_PFN));
596
597	if (!hinic3_wq_is_0_level_cla(wq)) {
598		cmdq_first_block_paddr = cmdqs->wq_block_paddr;
599		pfn = CMDQ_PFN(cmdq_first_block_paddr);
600	}
601
602	ctxt_info->wq_block_pfn = cpu_to_le64(CMDQ_CTXT_SET(wq->cons_idx, CI) |
603					      CMDQ_CTXT_SET(pfn, WQ_BLOCK_PFN));
604}
605
606static int init_cmdq(struct hinic3_cmdq *cmdq, struct hinic3_hwdev *hwdev,
607		     enum hinic3_cmdq_type q_type)
608{
609	int err;
610
611	cmdq->cmdq_type = q_type;
612	cmdq->wrapped = 1;
613	cmdq->hwdev = hwdev;
614
615	spin_lock_init(&cmdq->cmdq_lock);
616
617	cmdq->cmd_infos = kcalloc(cmdq->wq.q_depth, sizeof(*cmdq->cmd_infos),
618				  GFP_KERNEL);
619	if (!cmdq->cmd_infos) {
620		err = -ENOMEM;
621		return err;
622	}
623
624	return 0;
625}
626
627static int hinic3_set_cmdq_ctxt(struct hinic3_hwdev *hwdev, u8 cmdq_id)
628{
629	struct comm_cmd_set_cmdq_ctxt cmdq_ctxt = {};
630	struct mgmt_msg_params msg_params = {};
631	int err;
632
633	cmdq_init_queue_ctxt(hwdev, cmdq_id, &cmdq_ctxt.ctxt);
634	cmdq_ctxt.func_id = hinic3_global_func_id(hwdev);
635	cmdq_ctxt.cmdq_id = cmdq_id;
636
637	mgmt_msg_params_init_default(&msg_params, &cmdq_ctxt,
638				     sizeof(cmdq_ctxt));
639
640	err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_COMM,
641				       COMM_CMD_SET_CMDQ_CTXT, &msg_params);
642	if (err || cmdq_ctxt.head.status) {
643		dev_err(hwdev->dev, "Failed to set cmdq ctxt, err: %d, status: 0x%x\n",
644			err, cmdq_ctxt.head.status);
645		return -EFAULT;
646	}
647
648	return 0;
649}
650
651static int hinic3_set_cmdq_ctxts(struct hinic3_hwdev *hwdev)
652{
653	struct hinic3_cmdqs *cmdqs = hwdev->cmdqs;
654	u8 cmdq_type;
655	int err;
656
657	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) {
658		err = hinic3_set_cmdq_ctxt(hwdev, cmdq_type);
659		if (err)
660			return err;
661	}
662
663	cmdqs->status |= HINIC3_CMDQ_ENABLE;
664	cmdqs->disable_flag = 0;
665
666	return 0;
667}
668
669static int create_cmdq_wq(struct hinic3_hwdev *hwdev,
670			  struct hinic3_cmdqs *cmdqs)
671{
672	u8 cmdq_type;
673	int err;
674
675	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) {
676		err = hinic3_wq_create(hwdev, &cmdqs->cmdq[cmdq_type].wq,
677				       CMDQ_DEPTH, CMDQ_WQEBB_SIZE);
678		if (err) {
679			dev_err(hwdev->dev, "Failed to create cmdq wq\n");
680			goto err_destroy_wq;
681		}
682	}
683
684	/* 1-level Chip Logical Address (CLA) must put all
685	 * cmdq's wq page addr in one wq block
686	 */
687	if (!hinic3_wq_is_0_level_cla(&cmdqs->cmdq[HINIC3_CMDQ_SYNC].wq)) {
688		if (cmdqs->cmdq[HINIC3_CMDQ_SYNC].wq.qpages.num_pages >
689		    CMDQ_WQ_CLA_SIZE / sizeof(u64)) {
690			err = -EINVAL;
691			dev_err(hwdev->dev,
692				"Cmdq number of wq pages exceeds limit: %lu\n",
693				CMDQ_WQ_CLA_SIZE / sizeof(u64));
694			goto err_destroy_wq;
695		}
696
697		cmdqs->wq_block_vaddr =
698			dma_alloc_coherent(hwdev->dev, HINIC3_MIN_PAGE_SIZE,
699					   &cmdqs->wq_block_paddr, GFP_KERNEL);
700		if (!cmdqs->wq_block_vaddr) {
701			err = -ENOMEM;
702			goto err_destroy_wq;
703		}
704
705		for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++)
706			memcpy((u8 *)cmdqs->wq_block_vaddr +
707			       CMDQ_WQ_CLA_SIZE * cmdq_type,
708			       cmdqs->cmdq[cmdq_type].wq.wq_block_vaddr,
709			       cmdqs->cmdq[cmdq_type].wq.qpages.num_pages *
710			       sizeof(__be64));
711	}
712
713	return 0;
714
715err_destroy_wq:
716	while (cmdq_type > 0) {
717		cmdq_type--;
718		hinic3_wq_destroy(hwdev, &cmdqs->cmdq[cmdq_type].wq);
719	}
720
721	return err;
722}
723
724static void destroy_cmdq_wq(struct hinic3_hwdev *hwdev,
725			    struct hinic3_cmdqs *cmdqs)
726{
727	u8 cmdq_type;
728
729	if (cmdqs->wq_block_vaddr)
730		dma_free_coherent(hwdev->dev, HINIC3_MIN_PAGE_SIZE,
731				  cmdqs->wq_block_vaddr, cmdqs->wq_block_paddr);
732
733	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++)
734		hinic3_wq_destroy(hwdev, &cmdqs->cmdq[cmdq_type].wq);
735}
736
737static int init_cmdqs(struct hinic3_hwdev *hwdev)
738{
739	struct hinic3_cmdqs *cmdqs;
740
741	cmdqs = kzalloc(sizeof(*cmdqs), GFP_KERNEL);
742	if (!cmdqs)
743		return -ENOMEM;
744
745	hwdev->cmdqs = cmdqs;
746	cmdqs->hwdev = hwdev;
747	cmdqs->cmdq_num = hwdev->max_cmdq;
748
749	cmdqs->cmd_buf_pool = dma_pool_create("hinic3_cmdq", hwdev->dev,
750					      CMDQ_BUF_SIZE, CMDQ_BUF_SIZE, 0);
751	if (!cmdqs->cmd_buf_pool) {
752		dev_err(hwdev->dev, "Failed to create cmdq buffer pool\n");
753		kfree(cmdqs);
754		return -ENOMEM;
755	}
756
757	return 0;
758}
759
760static void cmdq_flush_sync_cmd(struct hinic3_cmdq_cmd_info *cmd_info)
761{
762	if (cmd_info->cmd_type != HINIC3_CMD_TYPE_DIRECT_RESP)
763		return;
764
765	cmd_info->cmd_type = HINIC3_CMD_TYPE_FORCE_STOP;
766
767	if (cmd_info->cmpt_code &&
768	    *cmd_info->cmpt_code == CMDQ_SEND_CMPT_CODE)
769		*cmd_info->cmpt_code = CMDQ_FORCE_STOP_CMPT_CODE;
770
771	if (cmd_info->done) {
772		complete(cmd_info->done);
773		cmd_info->done = NULL;
774		cmd_info->cmpt_code = NULL;
775		cmd_info->direct_resp = NULL;
776		cmd_info->errcode = NULL;
777	}
778}
779
780static void hinic3_cmdq_flush_cmd(struct hinic3_cmdq *cmdq)
781{
782	struct hinic3_cmdq_cmd_info *cmd_info;
783	u16 ci;
784
785	spin_lock_bh(&cmdq->cmdq_lock);
786	while (cmdq_read_wqe(&cmdq->wq, &ci)) {
787		hinic3_wq_put_wqebbs(&cmdq->wq, CMDQ_WQE_NUM_WQEBBS);
788		cmd_info = &cmdq->cmd_infos[ci];
789		if (cmd_info->cmd_type == HINIC3_CMD_TYPE_DIRECT_RESP)
790			cmdq_flush_sync_cmd(cmd_info);
791	}
792	spin_unlock_bh(&cmdq->cmdq_lock);
793}
794
795void hinic3_cmdq_flush_sync_cmd(struct hinic3_hwdev *hwdev)
796{
797	struct hinic3_cmdq *cmdq;
798	u16 wqe_cnt, wqe_idx, i;
799	struct hinic3_wq *wq;
800
801	cmdq = &hwdev->cmdqs->cmdq[HINIC3_CMDQ_SYNC];
802	spin_lock_bh(&cmdq->cmdq_lock);
803	wq = &cmdq->wq;
804	wqe_cnt = hinic3_wq_get_used(wq);
805	for (i = 0; i < wqe_cnt; i++) {
806		wqe_idx = (wq->cons_idx + i) & wq->idx_mask;
807		cmdq_flush_sync_cmd(cmdq->cmd_infos + wqe_idx);
808	}
809	spin_unlock_bh(&cmdq->cmdq_lock);
810}
811
812static void hinic3_cmdq_reset_all_cmd_buf(struct hinic3_cmdq *cmdq)
813{
814	u16 i;
815
816	for (i = 0; i < cmdq->wq.q_depth; i++)
817		cmdq_clear_cmd_buf(&cmdq->cmd_infos[i], cmdq->hwdev);
818}
819
820int hinic3_reinit_cmdq_ctxts(struct hinic3_hwdev *hwdev)
821{
822	struct hinic3_cmdqs *cmdqs = hwdev->cmdqs;
823	u8 cmdq_type;
824
825	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) {
826		hinic3_cmdq_flush_cmd(&cmdqs->cmdq[cmdq_type]);
827		hinic3_cmdq_reset_all_cmd_buf(&cmdqs->cmdq[cmdq_type]);
828		cmdqs->cmdq[cmdq_type].wrapped = 1;
829		hinic3_wq_reset(&cmdqs->cmdq[cmdq_type].wq);
830	}
831
832	return hinic3_set_cmdq_ctxts(hwdev);
833}
834
835int hinic3_cmdqs_init(struct hinic3_hwdev *hwdev)
836{
837	struct hinic3_cmdqs *cmdqs;
838	void __iomem *db_base;
839	u8 cmdq_type;
840	int err;
841
842	err = init_cmdqs(hwdev);
843	if (err)
844		goto err_out;
845
846	cmdqs = hwdev->cmdqs;
847	err = create_cmdq_wq(hwdev, cmdqs);
848	if (err)
849		goto err_free_cmdqs;
850
851	err = hinic3_alloc_db_addr(hwdev, &db_base, NULL);
852	if (err) {
853		dev_err(hwdev->dev, "Failed to allocate doorbell address\n");
854		goto err_destroy_cmdq_wq;
855	}
856	cmdqs->cmdqs_db_base = db_base;
857
858	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) {
859		err = init_cmdq(&cmdqs->cmdq[cmdq_type], hwdev, cmdq_type);
860		if (err) {
861			dev_err(hwdev->dev,
862				"Failed to initialize cmdq type : %d\n",
863				cmdq_type);
864			goto err_free_cmd_infos;
865		}
866	}
867
868	err = hinic3_set_cmdq_ctxts(hwdev);
869	if (err)
870		goto err_free_cmd_infos;
871
872	return 0;
873
874err_free_cmd_infos:
875	while (cmdq_type > 0) {
876		cmdq_type--;
877		kfree(cmdqs->cmdq[cmdq_type].cmd_infos);
878	}
879
880	hinic3_free_db_addr(hwdev, cmdqs->cmdqs_db_base);
881
882err_destroy_cmdq_wq:
883	destroy_cmdq_wq(hwdev, cmdqs);
884
885err_free_cmdqs:
886	dma_pool_destroy(cmdqs->cmd_buf_pool);
887	kfree(cmdqs);
888
889err_out:
890	return err;
891}
892
893void hinic3_cmdqs_free(struct hinic3_hwdev *hwdev)
894{
895	struct hinic3_cmdqs *cmdqs = hwdev->cmdqs;
896	u8 cmdq_type;
897
898	cmdqs->status &= ~HINIC3_CMDQ_ENABLE;
899
900	for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) {
901		hinic3_cmdq_flush_cmd(&cmdqs->cmdq[cmdq_type]);
902		hinic3_cmdq_reset_all_cmd_buf(&cmdqs->cmdq[cmdq_type]);
903		kfree(cmdqs->cmdq[cmdq_type].cmd_infos);
904	}
905
906	hinic3_free_db_addr(hwdev, cmdqs->cmdqs_db_base);
907	destroy_cmdq_wq(hwdev, cmdqs);
908	dma_pool_destroy(cmdqs->cmd_buf_pool);
909	kfree(cmdqs);
910}
911
912bool hinic3_cmdq_idle(struct hinic3_cmdq *cmdq)
913{
914	return hinic3_wq_get_used(&cmdq->wq) == 0;
915}