erdma_cmdq.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2
  3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
  4/*          Kai Shen <kaishen@linux.alibaba.com> */
  5/* Copyright (c) 2020-2022, Alibaba Group. */
  6
  7#include "erdma.h"
  8
  9static void arm_cmdq_cq(struct erdma_cmdq *cmdq)
 10{
 11	struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq);
 12	u64 db_data = FIELD_PREP(ERDMA_CQDB_CI_MASK, cmdq->cq.ci) |
 13		      FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) |
 14		      FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cmdq->cq.cmdsn) |
 15		      FIELD_PREP(ERDMA_CQDB_IDX_MASK, cmdq->cq.cmdsn);
 16
 17	*cmdq->cq.dbrec = db_data;
 18	writeq(db_data, dev->func_bar + ERDMA_CMDQ_CQDB_REG);
 19
 20	atomic64_inc(&cmdq->cq.armed_num);
 21}
 22
 23static void kick_cmdq_db(struct erdma_cmdq *cmdq)
 24{
 25	struct erdma_dev *dev = container_of(cmdq, struct erdma_dev, cmdq);
 26	u64 db_data = FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi);
 27
 28	*cmdq->sq.dbrec = db_data;
 29	writeq(db_data, dev->func_bar + ERDMA_CMDQ_SQDB_REG);
 30}
 31
 32static struct erdma_comp_wait *get_comp_wait(struct erdma_cmdq *cmdq)
 33{
 34	int comp_idx;
 35
 36	spin_lock(&cmdq->lock);
 37	comp_idx = find_first_zero_bit(cmdq->comp_wait_bitmap,
 38				       cmdq->max_outstandings);
 39	if (comp_idx == cmdq->max_outstandings) {
 40		spin_unlock(&cmdq->lock);
 41		return ERR_PTR(-ENOMEM);
 42	}
 43
 44	__set_bit(comp_idx, cmdq->comp_wait_bitmap);
 45	spin_unlock(&cmdq->lock);
 46
 47	return &cmdq->wait_pool[comp_idx];
 48}
 49
 50static void put_comp_wait(struct erdma_cmdq *cmdq,
 51			  struct erdma_comp_wait *comp_wait)
 52{
 53	int used;
 54
 55	cmdq->wait_pool[comp_wait->ctx_id].cmd_status = ERDMA_CMD_STATUS_INIT;
 56	spin_lock(&cmdq->lock);
 57	used = __test_and_clear_bit(comp_wait->ctx_id, cmdq->comp_wait_bitmap);
 58	spin_unlock(&cmdq->lock);
 59
 60	WARN_ON(!used);
 61}
 62
 63static int erdma_cmdq_wait_res_init(struct erdma_dev *dev,
 64				    struct erdma_cmdq *cmdq)
 65{
 66	int i;
 67
 68	cmdq->wait_pool =
 69		devm_kcalloc(&dev->pdev->dev, cmdq->max_outstandings,
 70			     sizeof(struct erdma_comp_wait), GFP_KERNEL);
 71	if (!cmdq->wait_pool)
 72		return -ENOMEM;
 73
 74	spin_lock_init(&cmdq->lock);
 75	cmdq->comp_wait_bitmap = devm_bitmap_zalloc(
 76		&dev->pdev->dev, cmdq->max_outstandings, GFP_KERNEL);
 77	if (!cmdq->comp_wait_bitmap)
 78		return -ENOMEM;
 79
 80	for (i = 0; i < cmdq->max_outstandings; i++) {
 81		init_completion(&cmdq->wait_pool[i].wait_event);
 82		cmdq->wait_pool[i].ctx_id = i;
 83	}
 84
 85	return 0;
 86}
 87
 88static int erdma_cmdq_sq_init(struct erdma_dev *dev)
 89{
 90	struct erdma_cmdq *cmdq = &dev->cmdq;
 91	struct erdma_cmdq_sq *sq = &cmdq->sq;
 92
 93	sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE);
 94	sq->depth = cmdq->max_outstandings * sq->wqebb_cnt;
 95
 96	sq->qbuf = dma_alloc_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT,
 97				      &sq->qbuf_dma_addr, GFP_KERNEL);
 98	if (!sq->qbuf)
 99		return -ENOMEM;
100
101	sq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &sq->dbrec_dma);
102	if (!sq->dbrec)
103		goto err_out;
104
105	spin_lock_init(&sq->lock);
106
107	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_H_REG,
108			  upper_32_bits(sq->qbuf_dma_addr));
109	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_SQ_ADDR_L_REG,
110			  lower_32_bits(sq->qbuf_dma_addr));
111	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth);
112	erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, sq->dbrec_dma);
113
114	return 0;
115
116err_out:
117	dma_free_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT,
118			  sq->qbuf, sq->qbuf_dma_addr);
119
120	return -ENOMEM;
121}
122
123static int erdma_cmdq_cq_init(struct erdma_dev *dev)
124{
125	struct erdma_cmdq *cmdq = &dev->cmdq;
126	struct erdma_cmdq_cq *cq = &cmdq->cq;
127
128	cq->depth = cmdq->sq.depth;
129	cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
130				      &cq->qbuf_dma_addr, GFP_KERNEL);
131	if (!cq->qbuf)
132		return -ENOMEM;
133
134	spin_lock_init(&cq->lock);
135
136	cq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &cq->dbrec_dma);
137	if (!cq->dbrec)
138		goto err_out;
139
140	atomic64_set(&cq->armed_num, 0);
141
142	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_H_REG,
143			  upper_32_bits(cq->qbuf_dma_addr));
144	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG,
145			  lower_32_bits(cq->qbuf_dma_addr));
146	erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, cq->dbrec_dma);
147
148	return 0;
149
150err_out:
151	dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->qbuf,
152			  cq->qbuf_dma_addr);
153
154	return -ENOMEM;
155}
156
157static int erdma_cmdq_eq_init(struct erdma_dev *dev)
158{
159	struct erdma_cmdq *cmdq = &dev->cmdq;
160	struct erdma_eq *eq = &cmdq->eq;
161	int ret;
162
163	ret = erdma_eq_common_init(dev, eq, cmdq->max_outstandings);
164	if (ret)
165		return ret;
166
167	eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG;
168
169	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,
170			  upper_32_bits(eq->qbuf_dma_addr));
171	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_L_REG,
172			  lower_32_bits(eq->qbuf_dma_addr));
173	erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth);
174	erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, eq->dbrec_dma);
175
176	return 0;
177}
178
179int erdma_cmdq_init(struct erdma_dev *dev)
180{
181	struct erdma_cmdq *cmdq = &dev->cmdq;
182	int err;
183
184	cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;
185
186	sema_init(&cmdq->credits, cmdq->max_outstandings);
187
188	err = erdma_cmdq_wait_res_init(dev, cmdq);
189	if (err)
190		return err;
191
192	err = erdma_cmdq_sq_init(dev);
193	if (err)
194		return err;
195
196	err = erdma_cmdq_cq_init(dev);
197	if (err)
198		goto err_destroy_sq;
199
200	err = erdma_cmdq_eq_init(dev);
201	if (err)
202		goto err_destroy_cq;
203
204	set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
205
206	return 0;
207
208err_destroy_cq:
209	dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT,
210			  cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);
211
212	dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma);
213
214err_destroy_sq:
215	dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT,
216			  cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);
217
218	dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma);
219
220	return err;
221}
222
223void erdma_finish_cmdq_init(struct erdma_dev *dev)
224{
225	arm_cmdq_cq(&dev->cmdq);
226}
227
228void erdma_cmdq_destroy(struct erdma_dev *dev)
229{
230	struct erdma_cmdq *cmdq = &dev->cmdq;
231
232	clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
233
234	erdma_eq_destroy(dev, &cmdq->eq);
235
236	dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT,
237			  cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr);
238
239	dma_pool_free(dev->db_pool, cmdq->sq.dbrec, cmdq->sq.dbrec_dma);
240
241	dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT,
242			  cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr);
243
244	dma_pool_free(dev->db_pool, cmdq->cq.dbrec, cmdq->cq.dbrec_dma);
245}
246
247static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)
248{
249	__be32 *cqe = get_queue_entry(cmdq->cq.qbuf, cmdq->cq.ci,
250				      cmdq->cq.depth, CQE_SHIFT);
251	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
252			      be32_to_cpu(READ_ONCE(*cqe)));
253
254	return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe : NULL;
255}
256
257static void push_cmdq_sqe(struct erdma_cmdq *cmdq, u64 *req, size_t req_len,
258			  struct erdma_comp_wait *comp_wait)
259{
260	__le64 *wqe;
261	u64 hdr = *req;
262
263	comp_wait->cmd_status = ERDMA_CMD_STATUS_ISSUED;
264	reinit_completion(&comp_wait->wait_event);
265	comp_wait->sq_pi = cmdq->sq.pi;
266
267	wqe = get_queue_entry(cmdq->sq.qbuf, cmdq->sq.pi, cmdq->sq.depth,
268			      SQEBB_SHIFT);
269	memcpy(wqe, req, req_len);
270
271	cmdq->sq.pi += cmdq->sq.wqebb_cnt;
272	hdr |= FIELD_PREP(ERDMA_CMD_HDR_WQEBB_INDEX_MASK, cmdq->sq.pi) |
273	       FIELD_PREP(ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK,
274			  comp_wait->ctx_id) |
275	       FIELD_PREP(ERDMA_CMD_HDR_WQEBB_CNT_MASK, cmdq->sq.wqebb_cnt - 1);
276	*wqe = cpu_to_le64(hdr);
277
278	kick_cmdq_db(cmdq);
279}
280
281static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
282{
283	struct erdma_comp_wait *comp_wait;
284	u32 hdr0, sqe_idx;
285	__be32 *cqe;
286	u16 ctx_id;
287	u64 *sqe;
288
289	cqe = get_next_valid_cmdq_cqe(cmdq);
290	if (!cqe)
291		return -EAGAIN;
292
293	cmdq->cq.ci++;
294
295	dma_rmb();
296	hdr0 = be32_to_cpu(*cqe);
297	sqe_idx = be32_to_cpu(*(cqe + 1));
298
299	sqe = get_queue_entry(cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,
300			      SQEBB_SHIFT);
301	ctx_id = FIELD_GET(ERDMA_CMD_HDR_CONTEXT_COOKIE_MASK, *sqe);
302	comp_wait = &cmdq->wait_pool[ctx_id];
303	if (comp_wait->cmd_status != ERDMA_CMD_STATUS_ISSUED)
304		return -EIO;
305
306	comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;
307	comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);
308	cmdq->sq.ci += cmdq->sq.wqebb_cnt;
309	/* Copy 16B comp data after cqe hdr to outer */
310	be32_to_cpu_array(comp_wait->comp_data, cqe + 2, 4);
311
312	complete(&comp_wait->wait_event);
313
314	return 0;
315}
316
317static void erdma_polling_cmd_completions(struct erdma_cmdq *cmdq)
318{
319	unsigned long flags;
320	u16 comp_num;
321
322	spin_lock_irqsave(&cmdq->cq.lock, flags);
323
324	/* We must have less than # of max_outstandings
325	 * completions at one time.
326	 */
327	for (comp_num = 0; comp_num < cmdq->max_outstandings; comp_num++)
328		if (erdma_poll_single_cmd_completion(cmdq))
329			break;
330
331	spin_unlock_irqrestore(&cmdq->cq.lock, flags);
332}
333
334void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq)
335{
336	int got_event = 0;
337
338	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state))
339		return;
340
341	while (get_next_valid_eqe(&cmdq->eq)) {
342		cmdq->eq.ci++;
343		got_event++;
344	}
345
346	if (got_event) {
347		cmdq->cq.cmdsn++;
348		erdma_polling_cmd_completions(cmdq);
349		arm_cmdq_cq(cmdq);
350	}
351
352	notify_eq(&cmdq->eq);
353}
354
355static int erdma_poll_cmd_completion(struct erdma_comp_wait *comp_ctx,
356				     struct erdma_cmdq *cmdq, u32 timeout)
357{
358	unsigned long comp_timeout = jiffies + msecs_to_jiffies(timeout);
359
360	while (1) {
361		erdma_polling_cmd_completions(cmdq);
362		if (comp_ctx->cmd_status != ERDMA_CMD_STATUS_ISSUED)
363			break;
364
365		if (time_is_before_jiffies(comp_timeout))
366			return -ETIME;
367
368		udelay(20);
369	}
370
371	return 0;
372}
373
374static int erdma_wait_cmd_completion(struct erdma_comp_wait *comp_ctx,
375				     struct erdma_cmdq *cmdq, u32 timeout)
376{
377	unsigned long flags = 0;
378
379	wait_for_completion_timeout(&comp_ctx->wait_event,
380				    msecs_to_jiffies(timeout));
381
382	if (unlikely(comp_ctx->cmd_status != ERDMA_CMD_STATUS_FINISHED)) {
383		spin_lock_irqsave(&cmdq->cq.lock, flags);
384		comp_ctx->cmd_status = ERDMA_CMD_STATUS_TIMEOUT;
385		spin_unlock_irqrestore(&cmdq->cq.lock, flags);
386		return -ETIME;
387	}
388
389	return 0;
390}
391
392void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op)
393{
394	*hdr = FIELD_PREP(ERDMA_CMD_HDR_SUB_MOD_MASK, mod) |
395	       FIELD_PREP(ERDMA_CMD_HDR_OPCODE_MASK, op);
396}
397
398int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size,
399			u64 *resp0, u64 *resp1, bool sleepable)
400{
401	struct erdma_comp_wait *comp_wait;
402	int ret;
403
404	if (!test_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state))
405		return -ENODEV;
406
407	if (!sleepable) {
408		while (down_trylock(&cmdq->credits))
409			;
410	} else {
411		down(&cmdq->credits);
412	}
413
414	comp_wait = get_comp_wait(cmdq);
415	if (IS_ERR(comp_wait)) {
416		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
417		set_bit(ERDMA_CMDQ_STATE_CTX_ERR_BIT, &cmdq->state);
418		up(&cmdq->credits);
419		return PTR_ERR(comp_wait);
420	}
421
422	spin_lock(&cmdq->sq.lock);
423	push_cmdq_sqe(cmdq, req, req_size, comp_wait);
424	spin_unlock(&cmdq->sq.lock);
425
426	if (sleepable)
427		ret = erdma_wait_cmd_completion(comp_wait, cmdq,
428						ERDMA_CMDQ_TIMEOUT_MS);
429	else
430		ret = erdma_poll_cmd_completion(comp_wait, cmdq,
431						ERDMA_CMDQ_TIMEOUT_MS);
432
433	if (ret) {
434		set_bit(ERDMA_CMDQ_STATE_TIMEOUT_BIT, &cmdq->state);
435		clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
436		goto out;
437	}
438
439	if (comp_wait->comp_status)
440		ret = -EIO;
441
442	if (resp0 && resp1) {
443		*resp0 = *((u64 *)&comp_wait->comp_data[0]);
444		*resp1 = *((u64 *)&comp_wait->comp_data[2]);
445	}
446	put_comp_wait(cmdq, comp_wait);
447
448out:
449	up(&cmdq->credits);
450
451	return ret;
452}