hw_channel.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2/* Copyright (c) 2021, Microsoft Corporation. */
  3
  4#include <net/mana/gdma.h>
  5#include <net/mana/mana.h>
  6#include <net/mana/hw_channel.h>
  7#include <linux/vmalloc.h>
  8
  9static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id)
 10{
 11	struct gdma_resource *r = &hwc->inflight_msg_res;
 12	unsigned long flags;
 13	u32 index;
 14
 15	down(&hwc->sema);
 16
 17	spin_lock_irqsave(&r->lock, flags);
 18
 19	index = find_first_zero_bit(hwc->inflight_msg_res.map,
 20				    hwc->inflight_msg_res.size);
 21
 22	bitmap_set(hwc->inflight_msg_res.map, index, 1);
 23
 24	spin_unlock_irqrestore(&r->lock, flags);
 25
 26	*msg_id = index;
 27
 28	return 0;
 29}
 30
 31static void mana_hwc_put_msg_index(struct hw_channel_context *hwc, u16 msg_id)
 32{
 33	struct gdma_resource *r = &hwc->inflight_msg_res;
 34	unsigned long flags;
 35
 36	spin_lock_irqsave(&r->lock, flags);
 37	bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1);
 38	spin_unlock_irqrestore(&r->lock, flags);
 39
 40	up(&hwc->sema);
 41}
 42
 43static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,
 44				    const struct gdma_resp_hdr *resp_msg,
 45				    u32 resp_len)
 46{
 47	if (resp_len < sizeof(*resp_msg))
 48		return -EPROTO;
 49
 50	if (resp_len > caller_ctx->output_buflen)
 51		return -EPROTO;
 52
 53	return 0;
 54}
 55
 56static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
 57				struct hwc_work_request *req)
 58{
 59	struct device *dev = hwc_rxq->hwc->dev;
 60	struct gdma_sge *sge;
 61	int err;
 62
 63	sge = &req->sge;
 64	sge->address = (u64)req->buf_sge_addr;
 65	sge->mem_key = hwc_rxq->msg_buf->gpa_mkey;
 66	sge->size = req->buf_len;
 67
 68	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
 69	req->wqe_req.sgl = sge;
 70	req->wqe_req.num_sge = 1;
 71	req->wqe_req.client_data_unit = 0;
 72
 73	err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL);
 74	if (err)
 75		dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err);
 76	return err;
 77}
 78
 79static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
 80				 struct hwc_work_request *rx_req)
 81{
 82	const struct gdma_resp_hdr *resp_msg = rx_req->buf_va;
 83	struct hwc_caller_ctx *ctx;
 84	int err;
 85
 86	if (!test_bit(resp_msg->response.hwc_msg_id,
 87		      hwc->inflight_msg_res.map)) {
 88		dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
 89			resp_msg->response.hwc_msg_id);
 90		mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
 91		return;
 92	}
 93
 94	ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id;
 95	err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
 96	if (err)
 97		goto out;
 98
 99	ctx->status_code = resp_msg->status;
100
101	memcpy(ctx->output_buf, resp_msg, resp_len);
102out:
103	ctx->error = err;
104
105	/* Must post rx wqe before complete(), otherwise the next rx may
106	 * hit no_wqe error.
107	 */
108	mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
109
110	complete(&ctx->comp_event);
111}
112
113static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
114					struct gdma_event *event)
115{
116	union hwc_init_soc_service_type service_data;
117	struct hw_channel_context *hwc = ctx;
118	struct gdma_dev *gd = hwc->gdma_dev;
119	union hwc_init_type_data type_data;
120	union hwc_init_eq_id_db eq_db;
121	struct mana_context *ac;
122	u32 type, val;
123	int ret;
124
125	switch (event->type) {
126	case GDMA_EQE_HWC_INIT_EQ_ID_DB:
127		eq_db.as_uint32 = event->details[0];
128		hwc->cq->gdma_eq->id = eq_db.eq_id;
129		gd->doorbell = eq_db.doorbell;
130		break;
131
132	case GDMA_EQE_HWC_INIT_DATA:
133		type_data.as_uint32 = event->details[0];
134		type = type_data.type;
135		val = type_data.value;
136
137		switch (type) {
138		case HWC_INIT_DATA_CQID:
139			hwc->cq->gdma_cq->id = val;
140			break;
141
142		case HWC_INIT_DATA_RQID:
143			hwc->rxq->gdma_wq->id = val;
144			break;
145
146		case HWC_INIT_DATA_SQID:
147			hwc->txq->gdma_wq->id = val;
148			break;
149
150		case HWC_INIT_DATA_QUEUE_DEPTH:
151			hwc->hwc_init_q_depth_max = (u16)val;
152			break;
153
154		case HWC_INIT_DATA_MAX_REQUEST:
155			hwc->hwc_init_max_req_msg_size = val;
156			break;
157
158		case HWC_INIT_DATA_MAX_RESPONSE:
159			hwc->hwc_init_max_resp_msg_size = val;
160			break;
161
162		case HWC_INIT_DATA_MAX_NUM_CQS:
163			gd->gdma_context->max_num_cqs = val;
164			break;
165
166		case HWC_INIT_DATA_PDID:
167			hwc->gdma_dev->pdid = val;
168			break;
169
170		case HWC_INIT_DATA_GPA_MKEY:
171			hwc->rxq->msg_buf->gpa_mkey = val;
172			hwc->txq->msg_buf->gpa_mkey = val;
173			break;
174
175		case HWC_INIT_DATA_PF_DEST_RQ_ID:
176			hwc->pf_dest_vrq_id = val;
177			break;
178
179		case HWC_INIT_DATA_PF_DEST_CQ_ID:
180			hwc->pf_dest_vrcq_id = val;
181			break;
182		}
183
184		break;
185
186	case GDMA_EQE_HWC_INIT_DONE:
187		complete(&hwc->hwc_init_eqe_comp);
188		break;
189
190	case GDMA_EQE_HWC_SOC_RECONFIG_DATA:
191		type_data.as_uint32 = event->details[0];
192		type = type_data.type;
193		val = type_data.value;
194
195		switch (type) {
196		case HWC_DATA_CFG_HWC_TIMEOUT:
197			hwc->hwc_timeout = val;
198			break;
199
200		case HWC_DATA_HW_LINK_CONNECT:
201		case HWC_DATA_HW_LINK_DISCONNECT:
202			ac = gd->gdma_context->mana.driver_data;
203			if (!ac)
204				break;
205
206			WRITE_ONCE(ac->link_event, type);
207			schedule_work(&ac->link_change_work);
208
209			break;
210
211		default:
212			dev_warn(hwc->dev, "Received unknown reconfig type %u\n", type);
213			break;
214		}
215
216		break;
217	case GDMA_EQE_HWC_SOC_SERVICE:
218		service_data.as_uint32 = event->details[0];
219		type = service_data.type;
220
221		switch (type) {
222		case GDMA_SERVICE_TYPE_RDMA_SUSPEND:
223		case GDMA_SERVICE_TYPE_RDMA_RESUME:
224			ret = mana_rdma_service_event(gd->gdma_context, type);
225			if (ret)
226				dev_err(hwc->dev, "Failed to schedule adev service event: %d\n",
227					ret);
228			break;
229		default:
230			dev_warn(hwc->dev, "Received unknown SOC service type %u\n", type);
231			break;
232		}
233
234		break;
235	default:
236		dev_warn(hwc->dev, "Received unknown gdma event %u\n", event->type);
237		/* Ignore unknown events, which should never happen. */
238		break;
239	}
240}
241
242static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
243				      const struct hwc_rx_oob *rx_oob)
244{
245	struct hw_channel_context *hwc = ctx;
246	struct hwc_wq *hwc_rxq = hwc->rxq;
247	struct hwc_work_request *rx_req;
248	struct gdma_resp_hdr *resp;
249	struct gdma_wqe *dma_oob;
250	struct gdma_queue *rq;
251	struct gdma_sge *sge;
252	u64 rq_base_addr;
253	u64 rx_req_idx;
254	u8 *wqe;
255
256	if (WARN_ON_ONCE(hwc_rxq->gdma_wq->id != gdma_rxq_id))
257		return;
258
259	rq = hwc_rxq->gdma_wq;
260	wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE);
261	dma_oob = (struct gdma_wqe *)wqe;
262
263	sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4);
264
265	/* Select the RX work request for virtual address and for reposting. */
266	rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle;
267	rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size;
268
269	rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
270	resp = (struct gdma_resp_hdr *)rx_req->buf_va;
271
272	if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) {
273		dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n",
274			resp->response.hwc_msg_id);
275		return;
276	}
277
278	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req);
279
280	/* Can no longer use 'resp', because the buffer is posted to the HW
281	 * in mana_hwc_handle_resp() above.
282	 */
283	resp = NULL;
284}
285
286static void mana_hwc_tx_event_handler(void *ctx, u32 gdma_txq_id,
287				      const struct hwc_rx_oob *rx_oob)
288{
289	struct hw_channel_context *hwc = ctx;
290	struct hwc_wq *hwc_txq = hwc->txq;
291
292	WARN_ON_ONCE(!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id);
293}
294
295static int mana_hwc_create_gdma_wq(struct hw_channel_context *hwc,
296				   enum gdma_queue_type type, u64 queue_size,
297				   struct gdma_queue **queue)
298{
299	struct gdma_queue_spec spec = {};
300
301	if (type != GDMA_SQ && type != GDMA_RQ)
302		return -EINVAL;
303
304	spec.type = type;
305	spec.monitor_avl_buf = false;
306	spec.queue_size = queue_size;
307
308	return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
309}
310
311static int mana_hwc_create_gdma_cq(struct hw_channel_context *hwc,
312				   u64 queue_size,
313				   void *ctx, gdma_cq_callback *cb,
314				   struct gdma_queue *parent_eq,
315				   struct gdma_queue **queue)
316{
317	struct gdma_queue_spec spec = {};
318
319	spec.type = GDMA_CQ;
320	spec.monitor_avl_buf = false;
321	spec.queue_size = queue_size;
322	spec.cq.context = ctx;
323	spec.cq.callback = cb;
324	spec.cq.parent_eq = parent_eq;
325
326	return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
327}
328
329static int mana_hwc_create_gdma_eq(struct hw_channel_context *hwc,
330				   u64 queue_size,
331				   void *ctx, gdma_eq_callback *cb,
332				   struct gdma_queue **queue)
333{
334	struct gdma_queue_spec spec = {};
335
336	spec.type = GDMA_EQ;
337	spec.monitor_avl_buf = false;
338	spec.queue_size = queue_size;
339	spec.eq.context = ctx;
340	spec.eq.callback = cb;
341	spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ;
342	spec.eq.msix_index = 0;
343
344	return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue);
345}
346
347static void mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self)
348{
349	struct hwc_rx_oob comp_data = {};
350	struct gdma_comp *completions;
351	struct hwc_cq *hwc_cq = ctx;
352	int comp_read, i;
353
354	WARN_ON_ONCE(hwc_cq->gdma_cq != q_self);
355
356	completions = hwc_cq->comp_buf;
357	comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth);
358	WARN_ON_ONCE(comp_read <= 0 || comp_read > hwc_cq->queue_depth);
359
360	for (i = 0; i < comp_read; ++i) {
361		comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data;
362
363		if (completions[i].is_sq)
364			hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx,
365						completions[i].wq_num,
366						&comp_data);
367		else
368			hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx,
369						completions[i].wq_num,
370						&comp_data);
371	}
372
373	mana_gd_ring_cq(q_self, SET_ARM_BIT);
374}
375
376static void mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq)
377{
378	kfree(hwc_cq->comp_buf);
379
380	if (hwc_cq->gdma_cq)
381		mana_gd_destroy_queue(gc, hwc_cq->gdma_cq);
382
383	if (hwc_cq->gdma_eq)
384		mana_gd_destroy_queue(gc, hwc_cq->gdma_eq);
385
386	kfree(hwc_cq);
387}
388
389static int mana_hwc_create_cq(struct hw_channel_context *hwc, u16 q_depth,
390			      gdma_eq_callback *callback, void *ctx,
391			      hwc_rx_event_handler_t *rx_ev_hdlr,
392			      void *rx_ev_ctx,
393			      hwc_tx_event_handler_t *tx_ev_hdlr,
394			      void *tx_ev_ctx, struct hwc_cq **hwc_cq_ptr)
395{
396	struct gdma_queue *eq, *cq;
397	struct gdma_comp *comp_buf;
398	struct hwc_cq *hwc_cq;
399	u32 eq_size, cq_size;
400	int err;
401
402	eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth);
403	if (eq_size < MANA_MIN_QSIZE)
404		eq_size = MANA_MIN_QSIZE;
405
406	cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth);
407	if (cq_size < MANA_MIN_QSIZE)
408		cq_size = MANA_MIN_QSIZE;
409
410	hwc_cq = kzalloc(sizeof(*hwc_cq), GFP_KERNEL);
411	if (!hwc_cq)
412		return -ENOMEM;
413
414	err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq);
415	if (err) {
416		dev_err(hwc->dev, "Failed to create HWC EQ for RQ: %d\n", err);
417		goto out;
418	}
419	hwc_cq->gdma_eq = eq;
420
421	err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq, mana_hwc_comp_event,
422				      eq, &cq);
423	if (err) {
424		dev_err(hwc->dev, "Failed to create HWC CQ for RQ: %d\n", err);
425		goto out;
426	}
427	hwc_cq->gdma_cq = cq;
428
429	comp_buf = kcalloc(q_depth, sizeof(*comp_buf), GFP_KERNEL);
430	if (!comp_buf) {
431		err = -ENOMEM;
432		goto out;
433	}
434
435	hwc_cq->hwc = hwc;
436	hwc_cq->comp_buf = comp_buf;
437	hwc_cq->queue_depth = q_depth;
438	hwc_cq->rx_event_handler = rx_ev_hdlr;
439	hwc_cq->rx_event_ctx = rx_ev_ctx;
440	hwc_cq->tx_event_handler = tx_ev_hdlr;
441	hwc_cq->tx_event_ctx = tx_ev_ctx;
442
443	*hwc_cq_ptr = hwc_cq;
444	return 0;
445out:
446	mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq);
447	return err;
448}
449
450static int mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, u16 q_depth,
451				  u32 max_msg_size,
452				  struct hwc_dma_buf **dma_buf_ptr)
453{
454	struct gdma_context *gc = hwc->gdma_dev->gdma_context;
455	struct hwc_work_request *hwc_wr;
456	struct hwc_dma_buf *dma_buf;
457	struct gdma_mem_info *gmi;
458	void *virt_addr;
459	u32 buf_size;
460	u8 *base_pa;
461	int err;
462	u16 i;
463
464	dma_buf = kzalloc(struct_size(dma_buf, reqs, q_depth), GFP_KERNEL);
465	if (!dma_buf)
466		return -ENOMEM;
467
468	dma_buf->num_reqs = q_depth;
469
470	buf_size = MANA_PAGE_ALIGN(q_depth * max_msg_size);
471
472	gmi = &dma_buf->mem_info;
473	err = mana_gd_alloc_memory(gc, buf_size, gmi);
474	if (err) {
475		dev_err(hwc->dev, "Failed to allocate DMA buffer size: %u, err %d\n",
476			buf_size, err);
477		goto out;
478	}
479
480	virt_addr = dma_buf->mem_info.virt_addr;
481	base_pa = (u8 *)dma_buf->mem_info.dma_handle;
482
483	for (i = 0; i < q_depth; i++) {
484		hwc_wr = &dma_buf->reqs[i];
485
486		hwc_wr->buf_va = virt_addr + i * max_msg_size;
487		hwc_wr->buf_sge_addr = base_pa + i * max_msg_size;
488
489		hwc_wr->buf_len = max_msg_size;
490	}
491
492	*dma_buf_ptr = dma_buf;
493	return 0;
494out:
495	kfree(dma_buf);
496	return err;
497}
498
499static void mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc,
500				     struct hwc_dma_buf *dma_buf)
501{
502	if (!dma_buf)
503		return;
504
505	mana_gd_free_memory(&dma_buf->mem_info);
506
507	kfree(dma_buf);
508}
509
510static void mana_hwc_destroy_wq(struct hw_channel_context *hwc,
511				struct hwc_wq *hwc_wq)
512{
513	mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf);
514
515	if (hwc_wq->gdma_wq)
516		mana_gd_destroy_queue(hwc->gdma_dev->gdma_context,
517				      hwc_wq->gdma_wq);
518
519	kfree(hwc_wq);
520}
521
522static int mana_hwc_create_wq(struct hw_channel_context *hwc,
523			      enum gdma_queue_type q_type, u16 q_depth,
524			      u32 max_msg_size, struct hwc_cq *hwc_cq,
525			      struct hwc_wq **hwc_wq_ptr)
526{
527	struct gdma_queue *queue;
528	struct hwc_wq *hwc_wq;
529	u32 queue_size;
530	int err;
531
532	WARN_ON(q_type != GDMA_SQ && q_type != GDMA_RQ);
533
534	if (q_type == GDMA_RQ)
535		queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth);
536	else
537		queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth);
538
539	if (queue_size < MANA_MIN_QSIZE)
540		queue_size = MANA_MIN_QSIZE;
541
542	hwc_wq = kzalloc(sizeof(*hwc_wq), GFP_KERNEL);
543	if (!hwc_wq)
544		return -ENOMEM;
545
546	err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue);
547	if (err)
548		goto out;
549
550	hwc_wq->hwc = hwc;
551	hwc_wq->gdma_wq = queue;
552	hwc_wq->queue_depth = q_depth;
553	hwc_wq->hwc_cq = hwc_cq;
554
555	err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,
556				     &hwc_wq->msg_buf);
557	if (err)
558		goto out;
559
560	*hwc_wq_ptr = hwc_wq;
561	return 0;
562out:
563	if (err)
564		mana_hwc_destroy_wq(hwc, hwc_wq);
565
566	dev_err(hwc->dev, "Failed to create HWC queue size= %u type= %d err= %d\n",
567		queue_size, q_type, err);
568	return err;
569}
570
571static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,
572				struct hwc_work_request *req,
573				u32 dest_virt_rq_id, u32 dest_virt_rcq_id,
574				bool dest_pf)
575{
576	struct device *dev = hwc_txq->hwc->dev;
577	struct hwc_tx_oob *tx_oob;
578	struct gdma_sge *sge;
579	int err;
580
581	if (req->msg_size == 0 || req->msg_size > req->buf_len) {
582		dev_err(dev, "wrong msg_size: %u, buf_len: %u\n",
583			req->msg_size, req->buf_len);
584		return -EINVAL;
585	}
586
587	tx_oob = &req->tx_oob;
588
589	tx_oob->vrq_id = dest_virt_rq_id;
590	tx_oob->dest_vfid = 0;
591	tx_oob->vrcq_id = dest_virt_rcq_id;
592	tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id;
593	tx_oob->loopback = false;
594	tx_oob->lso_override = false;
595	tx_oob->dest_pf = dest_pf;
596	tx_oob->vsq_id = hwc_txq->gdma_wq->id;
597
598	sge = &req->sge;
599	sge->address = (u64)req->buf_sge_addr;
600	sge->mem_key = hwc_txq->msg_buf->gpa_mkey;
601	sge->size = req->msg_size;
602
603	memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request));
604	req->wqe_req.sgl = sge;
605	req->wqe_req.num_sge = 1;
606	req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob);
607	req->wqe_req.inline_oob_data = tx_oob;
608	req->wqe_req.client_data_unit = 0;
609
610	err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL);
611	if (err)
612		dev_err(dev, "Failed to post WQE on HWC SQ: %d\n", err);
613	return err;
614}
615
616static int mana_hwc_init_inflight_msg(struct hw_channel_context *hwc,
617				      u16 num_msg)
618{
619	int err;
620
621	sema_init(&hwc->sema, num_msg);
622
623	err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res);
624	if (err)
625		dev_err(hwc->dev, "Failed to init inflight_msg_res: %d\n", err);
626	return err;
627}
628
629static int mana_hwc_test_channel(struct hw_channel_context *hwc, u16 q_depth,
630				 u32 max_req_msg_size, u32 max_resp_msg_size)
631{
632	struct gdma_context *gc = hwc->gdma_dev->gdma_context;
633	struct hwc_wq *hwc_rxq = hwc->rxq;
634	struct hwc_work_request *req;
635	struct hwc_caller_ctx *ctx;
636	int err;
637	int i;
638
639	/* Post all WQEs on the RQ */
640	for (i = 0; i < q_depth; i++) {
641		req = &hwc_rxq->msg_buf->reqs[i];
642		err = mana_hwc_post_rx_wqe(hwc_rxq, req);
643		if (err)
644			return err;
645	}
646
647	ctx = kcalloc(q_depth, sizeof(*ctx), GFP_KERNEL);
648	if (!ctx)
649		return -ENOMEM;
650
651	for (i = 0; i < q_depth; ++i)
652		init_completion(&ctx[i].comp_event);
653
654	hwc->caller_ctx = ctx;
655
656	return mana_gd_test_eq(gc, hwc->cq->gdma_eq);
657}
658
659static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth,
660				      u32 *max_req_msg_size,
661				      u32 *max_resp_msg_size)
662{
663	struct hw_channel_context *hwc = gc->hwc.driver_data;
664	struct gdma_queue *rq = hwc->rxq->gdma_wq;
665	struct gdma_queue *sq = hwc->txq->gdma_wq;
666	struct gdma_queue *eq = hwc->cq->gdma_eq;
667	struct gdma_queue *cq = hwc->cq->gdma_cq;
668	int err;
669
670	init_completion(&hwc->hwc_init_eqe_comp);
671
672	err = mana_smc_setup_hwc(&gc->shm_channel, false,
673				 eq->mem_info.dma_handle,
674				 cq->mem_info.dma_handle,
675				 rq->mem_info.dma_handle,
676				 sq->mem_info.dma_handle,
677				 eq->eq.msix_index);
678	if (err)
679		return err;
680
681	if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * HZ))
682		return -ETIMEDOUT;
683
684	*q_depth = hwc->hwc_init_q_depth_max;
685	*max_req_msg_size = hwc->hwc_init_max_req_msg_size;
686	*max_resp_msg_size = hwc->hwc_init_max_resp_msg_size;
687
688	/* Both were set in mana_hwc_init_event_handler(). */
689	if (WARN_ON(cq->id >= gc->max_num_cqs))
690		return -EPROTO;
691
692	gc->cq_table = vcalloc(gc->max_num_cqs, sizeof(struct gdma_queue *));
693	if (!gc->cq_table)
694		return -ENOMEM;
695
696	gc->cq_table[cq->id] = cq;
697
698	return 0;
699}
700
701static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,
702				u32 max_req_msg_size, u32 max_resp_msg_size)
703{
704	int err;
705
706	err = mana_hwc_init_inflight_msg(hwc, q_depth);
707	if (err)
708		return err;
709
710	/* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ
711	 * queue depth and RQ queue depth.
712	 */
713	err = mana_hwc_create_cq(hwc, q_depth * 2,
714				 mana_hwc_init_event_handler, hwc,
715				 mana_hwc_rx_event_handler, hwc,
716				 mana_hwc_tx_event_handler, hwc, &hwc->cq);
717	if (err) {
718		dev_err(hwc->dev, "Failed to create HWC CQ: %d\n", err);
719		goto out;
720	}
721
722	err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,
723				 hwc->cq, &hwc->rxq);
724	if (err) {
725		dev_err(hwc->dev, "Failed to create HWC RQ: %d\n", err);
726		goto out;
727	}
728
729	err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,
730				 hwc->cq, &hwc->txq);
731	if (err) {
732		dev_err(hwc->dev, "Failed to create HWC SQ: %d\n", err);
733		goto out;
734	}
735
736	hwc->num_inflight_msg = q_depth;
737	hwc->max_req_msg_size = max_req_msg_size;
738
739	return 0;
740out:
741	/* mana_hwc_create_channel() will do the cleanup.*/
742	return err;
743}
744
745int mana_hwc_create_channel(struct gdma_context *gc)
746{
747	u32 max_req_msg_size, max_resp_msg_size;
748	struct gdma_dev *gd = &gc->hwc;
749	struct hw_channel_context *hwc;
750	u16 q_depth_max;
751	int err;
752
753	hwc = kzalloc(sizeof(*hwc), GFP_KERNEL);
754	if (!hwc)
755		return -ENOMEM;
756
757	gd->gdma_context = gc;
758	gd->driver_data = hwc;
759	hwc->gdma_dev = gd;
760	hwc->dev = gc->dev;
761	hwc->hwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS;
762
763	/* HWC's instance number is always 0. */
764	gd->dev_id.as_uint32 = 0;
765	gd->dev_id.type = GDMA_DEVICE_HWC;
766
767	gd->pdid = INVALID_PDID;
768	gd->doorbell = INVALID_DOORBELL;
769
770	/* mana_hwc_init_queues() only creates the required data structures,
771	 * and doesn't touch the HWC device.
772	 */
773	err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
774				   HW_CHANNEL_MAX_REQUEST_SIZE,
775				   HW_CHANNEL_MAX_RESPONSE_SIZE);
776	if (err) {
777		dev_err(hwc->dev, "Failed to initialize HWC: %d\n", err);
778		goto out;
779	}
780
781	err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size,
782					 &max_resp_msg_size);
783	if (err) {
784		dev_err(hwc->dev, "Failed to establish HWC: %d\n", err);
785		goto out;
786	}
787
788	err = mana_hwc_test_channel(gc->hwc.driver_data,
789				    HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
790				    max_req_msg_size, max_resp_msg_size);
791	if (err) {
792		dev_err(hwc->dev, "Failed to test HWC: %d\n", err);
793		goto out;
794	}
795
796	return 0;
797out:
798	mana_hwc_destroy_channel(gc);
799	return err;
800}
801
802void mana_hwc_destroy_channel(struct gdma_context *gc)
803{
804	struct hw_channel_context *hwc = gc->hwc.driver_data;
805
806	if (!hwc)
807		return;
808
809	/* gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's
810	 * non-zero, the HWC worked and we should tear down the HWC here.
811	 */
812	if (gc->max_num_cqs > 0) {
813		mana_smc_teardown_hwc(&gc->shm_channel, false);
814		gc->max_num_cqs = 0;
815	}
816
817	if (hwc->txq)
818		mana_hwc_destroy_wq(hwc, hwc->txq);
819
820	if (hwc->rxq)
821		mana_hwc_destroy_wq(hwc, hwc->rxq);
822
823	if (hwc->cq)
824		mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);
825
826	kfree(hwc->caller_ctx);
827	hwc->caller_ctx = NULL;
828
829	mana_gd_free_res_map(&hwc->inflight_msg_res);
830
831	hwc->num_inflight_msg = 0;
832
833	hwc->gdma_dev->doorbell = INVALID_DOORBELL;
834	hwc->gdma_dev->pdid = INVALID_PDID;
835
836	hwc->hwc_timeout = 0;
837
838	kfree(hwc);
839	gc->hwc.driver_data = NULL;
840	gc->hwc.gdma_context = NULL;
841
842	vfree(gc->cq_table);
843	gc->cq_table = NULL;
844}
845
846int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
847			  const void *req, u32 resp_len, void *resp)
848{
849	struct gdma_context *gc = hwc->gdma_dev->gdma_context;
850	struct hwc_work_request *tx_wr;
851	struct hwc_wq *txq = hwc->txq;
852	struct gdma_req_hdr *req_msg;
853	struct hwc_caller_ctx *ctx;
854	u32 dest_vrcq = 0;
855	u32 dest_vrq = 0;
856	u16 msg_id;
857	int err;
858
859	mana_hwc_get_msg_index(hwc, &msg_id);
860
861	tx_wr = &txq->msg_buf->reqs[msg_id];
862
863	if (req_len > tx_wr->buf_len) {
864		dev_err(hwc->dev, "HWC: req msg size: %d > %d\n", req_len,
865			tx_wr->buf_len);
866		err = -EINVAL;
867		goto out;
868	}
869
870	ctx = hwc->caller_ctx + msg_id;
871	ctx->output_buf = resp;
872	ctx->output_buflen = resp_len;
873
874	req_msg = (struct gdma_req_hdr *)tx_wr->buf_va;
875	if (req)
876		memcpy(req_msg, req, req_len);
877
878	req_msg->req.hwc_msg_id = msg_id;
879
880	tx_wr->msg_size = req_len;
881
882	if (gc->is_pf) {
883		dest_vrq = hwc->pf_dest_vrq_id;
884		dest_vrcq = hwc->pf_dest_vrcq_id;
885	}
886
887	err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false);
888	if (err) {
889		dev_err(hwc->dev, "HWC: Failed to post send WQE: %d\n", err);
890		goto out;
891	}
892
893	if (!wait_for_completion_timeout(&ctx->comp_event,
894					 (msecs_to_jiffies(hwc->hwc_timeout)))) {
895		if (hwc->hwc_timeout != 0)
896			dev_err(hwc->dev, "HWC: Request timed out: %u ms\n",
897				hwc->hwc_timeout);
898
899		/* Reduce further waiting if HWC no response */
900		if (hwc->hwc_timeout > 1)
901			hwc->hwc_timeout = 1;
902
903		err = -ETIMEDOUT;
904		goto out;
905	}
906
907	if (ctx->error) {
908		err = ctx->error;
909		goto out;
910	}
911
912	if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) {
913		if (ctx->status_code == GDMA_STATUS_CMD_UNSUPPORTED) {
914			err = -EOPNOTSUPP;
915			goto out;
916		}
917		if (req_msg->req.msg_type != MANA_QUERY_PHY_STAT)
918			dev_err(hwc->dev, "HWC: Failed hw_channel req: 0x%x\n",
919				ctx->status_code);
920		err = -EPROTO;
921		goto out;
922	}
923out:
924	mana_hwc_put_msg_index(hwc, msg_id);
925	return err;
926}