cq.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
  4 */
  5
  6#include "mana_ib.h"
  7
  8int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
  9		      struct uverbs_attr_bundle *attrs)
 10{
 11	struct ib_udata *udata = &attrs->driver_udata;
 12	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 13	struct mana_ib_create_cq_resp resp = {};
 14	struct mana_ib_ucontext *mana_ucontext;
 15	struct ib_device *ibdev = ibcq->device;
 16	struct mana_ib_create_cq ucmd = {};
 17	struct mana_ib_dev *mdev;
 18	bool is_rnic_cq;
 19	u32 doorbell;
 20	u32 buf_size;
 21	int err;
 22
 23	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
 24
 25	cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors;
 26	cq->cq_handle = INVALID_MANA_HANDLE;
 27
 28	if (udata) {
 29		if (udata->inlen < offsetof(struct mana_ib_create_cq, flags))
 30			return -EINVAL;
 31
 32		err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
 33		if (err) {
 34			ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err);
 35			return err;
 36		}
 37
 38		is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ);
 39
 40		if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) ||
 41		    attr->cqe > U32_MAX / COMP_ENTRY_SIZE) {
 42			ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
 43			return -EINVAL;
 44		}
 45
 46		cq->cqe = attr->cqe;
 47		err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
 48					   &cq->queue);
 49		if (err) {
 50			ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
 51			return err;
 52		}
 53
 54		mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
 55							  ibucontext);
 56		doorbell = mana_ucontext->doorbell;
 57	} else {
 58		is_rnic_cq = true;
 59		if (attr->cqe > U32_MAX / COMP_ENTRY_SIZE / 2 + 1) {
 60			ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
 61			return -EINVAL;
 62		}
 63		buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE));
 64		cq->cqe = buf_size / COMP_ENTRY_SIZE;
 65		err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue);
 66		if (err) {
 67			ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err);
 68			return err;
 69		}
 70		doorbell = mdev->gdma_dev->doorbell;
 71	}
 72
 73	if (is_rnic_cq) {
 74		err = mana_ib_gd_create_cq(mdev, cq, doorbell);
 75		if (err) {
 76			ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err);
 77			goto err_destroy_queue;
 78		}
 79
 80		err = mana_ib_install_cq_cb(mdev, cq);
 81		if (err) {
 82			ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err);
 83			goto err_destroy_rnic_cq;
 84		}
 85	}
 86
 87	if (udata) {
 88		resp.cqid = cq->queue.id;
 89		err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
 90		if (err) {
 91			ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
 92			goto err_remove_cq_cb;
 93		}
 94	}
 95
 96	spin_lock_init(&cq->cq_lock);
 97	INIT_LIST_HEAD(&cq->list_send_qp);
 98	INIT_LIST_HEAD(&cq->list_recv_qp);
 99
100	return 0;
101
102err_remove_cq_cb:
103	mana_ib_remove_cq_cb(mdev, cq);
104err_destroy_rnic_cq:
105	mana_ib_gd_destroy_cq(mdev, cq);
106err_destroy_queue:
107	mana_ib_destroy_queue(mdev, &cq->queue);
108
109	return err;
110}
111
112int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
113{
114	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
115	struct ib_device *ibdev = ibcq->device;
116	struct mana_ib_dev *mdev;
117
118	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
119
120	mana_ib_remove_cq_cb(mdev, cq);
121
122	/* Ignore return code as there is not much we can do about it.
123	 * The error message is printed inside.
124	 */
125	mana_ib_gd_destroy_cq(mdev, cq);
126
127	mana_ib_destroy_queue(mdev, &cq->queue);
128
129	return 0;
130}
131
132static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
133{
134	struct mana_ib_cq *cq = ctx;
135
136	if (cq->ibcq.comp_handler)
137		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
138}
139
140int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
141{
142	struct gdma_context *gc = mdev_to_gc(mdev);
143	struct gdma_queue *gdma_cq;
144
145	if (cq->queue.id >= gc->max_num_cqs)
146		return -EINVAL;
147	/* Create CQ table entry */
148	WARN_ON(gc->cq_table[cq->queue.id]);
149	if (cq->queue.kmem)
150		gdma_cq = cq->queue.kmem;
151	else
152		gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
153	if (!gdma_cq)
154		return -ENOMEM;
155
156	gdma_cq->cq.context = cq;
157	gdma_cq->type = GDMA_CQ;
158	gdma_cq->cq.callback = mana_ib_cq_handler;
159	gdma_cq->id = cq->queue.id;
160	gc->cq_table[cq->queue.id] = gdma_cq;
161	return 0;
162}
163
164void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
165{
166	struct gdma_context *gc = mdev_to_gc(mdev);
167
168	if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID)
169		return;
170
171	if (cq->queue.kmem)
172	/* Then it will be cleaned and removed by the mana */
173		return;
174
175	kfree(gc->cq_table[cq->queue.id]);
176	gc->cq_table[cq->queue.id] = NULL;
177}
178
179int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
180{
181	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
182	struct gdma_queue *gdma_cq = cq->queue.kmem;
183
184	if (!gdma_cq)
185		return -EINVAL;
186
187	mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
188	return 0;
189}
190
191static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
192{
193	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
194	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
195	struct ud_sq_shadow_wqe *shadow_wqe;
196
197	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
198	if (!shadow_wqe)
199		return;
200
201	shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
202
203	wq->tail += shadow_wqe->header.posted_wqe_size;
204	shadow_queue_advance_next_to_complete(&qp->shadow_sq);
205}
206
207static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
208{
209	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
210	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
211	struct ud_rq_shadow_wqe *shadow_wqe;
212
213	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
214	if (!shadow_wqe)
215		return;
216
217	shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
218	shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
219	shadow_wqe->header.error_code = IB_WC_SUCCESS;
220
221	wq->tail += shadow_wqe->header.posted_wqe_size;
222	shadow_queue_advance_next_to_complete(&qp->shadow_rq);
223}
224
225static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
226{
227	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
228
229	if (!qp)
230		return;
231
232	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
233		if (cqe->is_sq)
234			handle_ud_sq_cqe(qp, cqe);
235		else
236			handle_ud_rq_cqe(qp, cqe);
237	}
238
239	mana_put_qp_ref(qp);
240}
241
242static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
243				       const struct shadow_wqe_header *shadow_wqe)
244{
245	const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
246
247	wc->wr_id = shadow_wqe->wr_id;
248	wc->status = shadow_wqe->error_code;
249	wc->opcode = shadow_wqe->opcode;
250	wc->vendor_err = shadow_wqe->error_code;
251	wc->wc_flags = 0;
252	wc->qp = &qp->ibqp;
253	wc->pkey_index = 0;
254
255	if (shadow_wqe->opcode == IB_WC_RECV) {
256		wc->byte_len = ud_wqe->byte_len;
257		wc->src_qp = ud_wqe->src_qpn;
258		wc->wc_flags |= IB_WC_GRH;
259	}
260}
261
262static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
263{
264	struct shadow_wqe_header *shadow_wqe;
265	struct mana_ib_qp *qp;
266	int wc_index = 0;
267
268	/* process send shadow queue completions  */
269	list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
270		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
271				!= NULL) {
272			if (wc_index >= nwc)
273				goto out;
274
275			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
276			shadow_queue_advance_consumer(&qp->shadow_sq);
277			wc_index++;
278		}
279	}
280
281	/* process recv shadow queue completions */
282	list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
283		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
284				!= NULL) {
285			if (wc_index >= nwc)
286				goto out;
287
288			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
289			shadow_queue_advance_consumer(&qp->shadow_rq);
290			wc_index++;
291		}
292	}
293
294out:
295	return wc_index;
296}
297
298void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
299{
300	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false);
301	struct ud_sq_shadow_wqe *shadow_wqe;
302	struct mana_ib_cq *cq;
303	unsigned long flags;
304
305	if (!qp)
306		return;
307
308	cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
309
310	spin_lock_irqsave(&cq->cq_lock, flags);
311	while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq))
312			!= NULL) {
313		shadow_wqe->header.error_code = IB_WC_GENERAL_ERR;
314		shadow_queue_advance_next_to_complete(&qp->shadow_sq);
315	}
316	spin_unlock_irqrestore(&cq->cq_lock, flags);
317
318	if (cq->ibcq.comp_handler)
319		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
320
321	mana_put_qp_ref(qp);
322}
323
324int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
325{
326	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
327	struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
328	struct gdma_queue *queue = cq->queue.kmem;
329	struct gdma_comp gdma_cqe;
330	unsigned long flags;
331	int num_polled = 0;
332	int comp_read, i;
333
334	spin_lock_irqsave(&cq->cq_lock, flags);
335	for (i = 0; i < num_entries; i++) {
336		comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
337		if (comp_read < 1)
338			break;
339		mana_handle_cqe(mdev, &gdma_cqe);
340	}
341
342	num_polled = mana_process_completions(cq, num_entries, wc);
343	spin_unlock_irqrestore(&cq->cq_lock, flags);
344
345	return num_polled;
346}