qp.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
  4 */
  5
  6#include "mana_ib.h"
  7
  8static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev,
  9				      struct net_device *ndev,
 10				      mana_handle_t default_rxobj,
 11				      mana_handle_t ind_table[],
 12				      u32 log_ind_tbl_size, u32 rx_hash_key_len,
 13				      u8 *rx_hash_key)
 14{
 15	struct mana_port_context *mpc = netdev_priv(ndev);
 16	struct mana_cfg_rx_steer_req_v2 *req;
 17	struct mana_cfg_rx_steer_resp resp = {};
 18	struct gdma_context *gc;
 19	u32 req_buf_size;
 20	int i, err;
 21
 22	gc = mdev_to_gc(dev);
 23
 24	req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE);
 25	req = kzalloc(req_buf_size, GFP_KERNEL);
 26	if (!req)
 27		return -ENOMEM;
 28
 29	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
 30			     sizeof(resp));
 31
 32	req->hdr.req.msg_version = GDMA_MESSAGE_V2;
 33
 34	req->vport = mpc->port_handle;
 35	req->rx_enable = 1;
 36	req->update_default_rxobj = 1;
 37	req->default_rxobj = default_rxobj;
 38	req->hdr.dev_id = gc->mana.dev_id;
 39
 40	/* If there are more than 1 entries in indirection table, enable RSS */
 41	if (log_ind_tbl_size)
 42		req->rss_enable = true;
 43
 44	req->num_indir_entries = MANA_INDIRECT_TABLE_DEF_SIZE;
 45	req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2,
 46					 indir_tab);
 47	req->update_indir_tab = true;
 48	req->cqe_coalescing_enable = 1;
 49
 50	/* The ind table passed to the hardware must have
 51	 * MANA_INDIRECT_TABLE_DEF_SIZE entries. Adjust the verb
 52	 * ind_table to MANA_INDIRECT_TABLE_SIZE if required
 53	 */
 54	ibdev_dbg(&dev->ib_dev, "ind table size %u\n", 1 << log_ind_tbl_size);
 55	for (i = 0; i < MANA_INDIRECT_TABLE_DEF_SIZE; i++) {
 56		req->indir_tab[i] = ind_table[i % (1 << log_ind_tbl_size)];
 57		ibdev_dbg(&dev->ib_dev, "index %u handle 0x%llx\n", i,
 58			  req->indir_tab[i]);
 59	}
 60
 61	req->update_hashkey = true;
 62	if (rx_hash_key_len)
 63		memcpy(req->hashkey, rx_hash_key, rx_hash_key_len);
 64	else
 65		netdev_rss_key_fill(req->hashkey, MANA_HASH_KEY_SIZE);
 66
 67	ibdev_dbg(&dev->ib_dev, "vport handle %llu default_rxobj 0x%llx\n",
 68		  req->vport, default_rxobj);
 69
 70	err = mana_gd_send_request(gc, req_buf_size, req, sizeof(resp), &resp);
 71	if (err) {
 72		netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
 73		goto out;
 74	}
 75
 76	if (resp.hdr.status) {
 77		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
 78			   resp.hdr.status);
 79		err = -EPROTO;
 80		goto out;
 81	}
 82
 83	netdev_info(ndev, "Configured steering vPort %llu log_entries %u\n",
 84		    mpc->port_handle, log_ind_tbl_size);
 85
 86out:
 87	kfree(req);
 88	return err;
 89}
 90
 91static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 92				 struct ib_qp_init_attr *attr,
 93				 struct ib_udata *udata)
 94{
 95	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
 96	struct mana_ib_dev *mdev =
 97		container_of(pd->device, struct mana_ib_dev, ib_dev);
 98	struct ib_rwq_ind_table *ind_tbl = attr->rwq_ind_tbl;
 99	struct mana_ib_create_qp_rss_resp resp = {};
100	struct mana_ib_create_qp_rss ucmd = {};
101	mana_handle_t *mana_ind_table;
102	struct mana_port_context *mpc;
103	unsigned int ind_tbl_size;
104	struct net_device *ndev;
105	struct mana_ib_cq *cq;
106	struct mana_ib_wq *wq;
107	struct mana_eq *eq;
108	struct ib_cq *ibcq;
109	struct ib_wq *ibwq;
110	int i = 0;
111	u32 port;
112	int ret;
113
114	if (!udata || udata->inlen < sizeof(ucmd))
115		return -EINVAL;
116
117	ret = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
118	if (ret) {
119		ibdev_dbg(&mdev->ib_dev,
120			  "Failed copy from udata for create rss-qp, err %d\n",
121			  ret);
122		return ret;
123	}
124
125	if (attr->cap.max_recv_wr > mdev->adapter_caps.max_qp_wr) {
126		ibdev_dbg(&mdev->ib_dev,
127			  "Requested max_recv_wr %d exceeding limit\n",
128			  attr->cap.max_recv_wr);
129		return -EINVAL;
130	}
131
132	if (attr->cap.max_recv_sge > MAX_RX_WQE_SGL_ENTRIES) {
133		ibdev_dbg(&mdev->ib_dev,
134			  "Requested max_recv_sge %d exceeding limit\n",
135			  attr->cap.max_recv_sge);
136		return -EINVAL;
137	}
138
139	ind_tbl_size = 1 << ind_tbl->log_ind_tbl_size;
140	if (ind_tbl_size > MANA_INDIRECT_TABLE_DEF_SIZE) {
141		ibdev_dbg(&mdev->ib_dev,
142			  "Indirect table size %d exceeding limit\n",
143			  ind_tbl_size);
144		return -EINVAL;
145	}
146
147	if (ucmd.rx_hash_function != MANA_IB_RX_HASH_FUNC_TOEPLITZ) {
148		ibdev_dbg(&mdev->ib_dev,
149			  "RX Hash function is not supported, %d\n",
150			  ucmd.rx_hash_function);
151		return -EINVAL;
152	}
153
154	/* IB ports start with 1, MANA start with 0 */
155	port = ucmd.port;
156	ndev = mana_ib_get_netdev(pd->device, port);
157	if (!ndev) {
158		ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n",
159			  port);
160		return -EINVAL;
161	}
162	mpc = netdev_priv(ndev);
163
164	ibdev_dbg(&mdev->ib_dev, "rx_hash_function %d port %d\n",
165		  ucmd.rx_hash_function, port);
166
167	mana_ind_table = kcalloc(ind_tbl_size, sizeof(mana_handle_t),
168				 GFP_KERNEL);
169	if (!mana_ind_table) {
170		ret = -ENOMEM;
171		goto fail;
172	}
173
174	qp->port = port;
175
176	for (i = 0; i < ind_tbl_size; i++) {
177		struct mana_obj_spec wq_spec = {};
178		struct mana_obj_spec cq_spec = {};
179
180		ibwq = ind_tbl->ind_tbl[i];
181		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
182
183		ibcq = ibwq->cq;
184		cq = container_of(ibcq, struct mana_ib_cq, ibcq);
185
186		wq_spec.gdma_region = wq->queue.gdma_region;
187		wq_spec.queue_size = wq->wq_buf_size;
188
189		cq_spec.gdma_region = cq->queue.gdma_region;
190		cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
191		cq_spec.modr_ctx_id = 0;
192		eq = &mpc->ac->eqs[cq->comp_vector];
193		cq_spec.attached_eq = eq->eq->id;
194
195		ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
196					 &wq_spec, &cq_spec, &wq->rx_object);
197		if (ret) {
198			/* Do cleanup starting with index i-1 */
199			i--;
200			goto fail;
201		}
202
203		/* The GDMA regions are now owned by the WQ object */
204		wq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
205		cq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
206
207		wq->queue.id = wq_spec.queue_index;
208		cq->queue.id = cq_spec.queue_index;
209
210		ibdev_dbg(&mdev->ib_dev,
211			  "rx_object 0x%llx wq id %llu cq id %llu\n",
212			  wq->rx_object, wq->queue.id, cq->queue.id);
213
214		resp.entries[i].cqid = cq->queue.id;
215		resp.entries[i].wqid = wq->queue.id;
216
217		mana_ind_table[i] = wq->rx_object;
218
219		/* Create CQ table entry */
220		ret = mana_ib_install_cq_cb(mdev, cq);
221		if (ret)
222			goto fail;
223	}
224	resp.num_entries = i;
225
226	ret = mana_ib_cfg_vport_steering(mdev, ndev, wq->rx_object,
227					 mana_ind_table,
228					 ind_tbl->log_ind_tbl_size,
229					 ucmd.rx_hash_key_len,
230					 ucmd.rx_hash_key);
231	if (ret)
232		goto fail;
233
234	ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
235	if (ret) {
236		ibdev_dbg(&mdev->ib_dev,
237			  "Failed to copy to udata create rss-qp, %d\n",
238			  ret);
239		goto fail;
240	}
241
242	kfree(mana_ind_table);
243
244	return 0;
245
246fail:
247	while (i-- > 0) {
248		ibwq = ind_tbl->ind_tbl[i];
249		ibcq = ibwq->cq;
250		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
251		cq = container_of(ibcq, struct mana_ib_cq, ibcq);
252
253		mana_ib_remove_cq_cb(mdev, cq);
254		mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
255	}
256
257	kfree(mana_ind_table);
258
259	return ret;
260}
261
262static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
263				 struct ib_qp_init_attr *attr,
264				 struct ib_udata *udata)
265{
266	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
267	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
268	struct mana_ib_dev *mdev =
269		container_of(ibpd->device, struct mana_ib_dev, ib_dev);
270	struct mana_ib_cq *send_cq =
271		container_of(attr->send_cq, struct mana_ib_cq, ibcq);
272	struct mana_ib_ucontext *mana_ucontext =
273		rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
274					  ibucontext);
275	struct mana_ib_create_qp_resp resp = {};
276	struct mana_ib_create_qp ucmd = {};
277	struct mana_obj_spec wq_spec = {};
278	struct mana_obj_spec cq_spec = {};
279	struct mana_port_context *mpc;
280	struct net_device *ndev;
281	struct mana_eq *eq;
282	int eq_vec;
283	u32 port;
284	int err;
285
286	if (!mana_ucontext || udata->inlen < sizeof(ucmd))
287		return -EINVAL;
288
289	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
290	if (err) {
291		ibdev_dbg(&mdev->ib_dev,
292			  "Failed to copy from udata create qp-raw, %d\n", err);
293		return err;
294	}
295
296	if (attr->cap.max_send_wr > mdev->adapter_caps.max_qp_wr) {
297		ibdev_dbg(&mdev->ib_dev,
298			  "Requested max_send_wr %d exceeding limit\n",
299			  attr->cap.max_send_wr);
300		return -EINVAL;
301	}
302
303	if (attr->cap.max_send_sge > MAX_TX_WQE_SGL_ENTRIES) {
304		ibdev_dbg(&mdev->ib_dev,
305			  "Requested max_send_sge %d exceeding limit\n",
306			  attr->cap.max_send_sge);
307		return -EINVAL;
308	}
309
310	port = ucmd.port;
311	ndev = mana_ib_get_netdev(ibpd->device, port);
312	if (!ndev) {
313		ibdev_dbg(&mdev->ib_dev, "Invalid port %u in creating qp\n",
314			  port);
315		return -EINVAL;
316	}
317	mpc = netdev_priv(ndev);
318	ibdev_dbg(&mdev->ib_dev, "port %u ndev %p mpc %p\n", port, ndev, mpc);
319
320	err = mana_ib_cfg_vport(mdev, port, pd, mana_ucontext->doorbell);
321	if (err)
322		return -ENODEV;
323
324	qp->port = port;
325
326	ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
327		  ucmd.sq_buf_addr, ucmd.port);
328
329	err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq);
330	if (err) {
331		ibdev_dbg(&mdev->ib_dev,
332			  "Failed to create queue for create qp-raw, err %d\n", err);
333		goto err_free_vport;
334	}
335
336	/* Create a WQ on the same port handle used by the Ethernet */
337	wq_spec.gdma_region = qp->raw_sq.gdma_region;
338	wq_spec.queue_size = ucmd.sq_buf_size;
339
340	cq_spec.gdma_region = send_cq->queue.gdma_region;
341	cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
342	cq_spec.modr_ctx_id = 0;
343	eq_vec = send_cq->comp_vector;
344	eq = &mpc->ac->eqs[eq_vec];
345	cq_spec.attached_eq = eq->eq->id;
346
347	err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec,
348				 &cq_spec, &qp->qp_handle);
349	if (err) {
350		ibdev_dbg(&mdev->ib_dev,
351			  "Failed to create wq for create raw-qp, err %d\n",
352			  err);
353		goto err_destroy_queue;
354	}
355
356	/* The GDMA regions are now owned by the WQ object */
357	qp->raw_sq.gdma_region = GDMA_INVALID_DMA_REGION;
358	send_cq->queue.gdma_region = GDMA_INVALID_DMA_REGION;
359
360	qp->raw_sq.id = wq_spec.queue_index;
361	send_cq->queue.id = cq_spec.queue_index;
362
363	/* Create CQ table entry */
364	err = mana_ib_install_cq_cb(mdev, send_cq);
365	if (err)
366		goto err_destroy_wq_obj;
367
368	ibdev_dbg(&mdev->ib_dev,
369		  "qp->qp_handle 0x%llx sq id %llu cq id %llu\n",
370		  qp->qp_handle, qp->raw_sq.id, send_cq->queue.id);
371
372	resp.sqid = qp->raw_sq.id;
373	resp.cqid = send_cq->queue.id;
374	resp.tx_vp_offset = pd->tx_vp_offset;
375
376	err = ib_copy_to_udata(udata, &resp, sizeof(resp));
377	if (err) {
378		ibdev_dbg(&mdev->ib_dev,
379			  "Failed copy udata for create qp-raw, %d\n",
380			  err);
381		goto err_remove_cq_cb;
382	}
383
384	return 0;
385
386err_remove_cq_cb:
387	mana_ib_remove_cq_cb(mdev, send_cq);
388
389err_destroy_wq_obj:
390	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle);
391
392err_destroy_queue:
393	mana_ib_destroy_queue(mdev, &qp->raw_sq);
394
395err_free_vport:
396	mana_ib_uncfg_vport(mdev, pd, port);
397
398	return err;
399}
400
401static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
402{
403	u32 wqe_size = sge * sizeof(struct gdma_sge) + sizeof(struct gdma_wqe) + oob_size;
404
405	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);
406}
407
408static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
409{
410	u32 queue_size;
411
412	switch (attr->qp_type) {
413	case IB_QPT_UD:
414	case IB_QPT_GSI:
415		if (queue_type == MANA_UD_SEND_QUEUE)
416			queue_size = attr->cap.max_send_wr *
417				mana_ib_wqe_size(attr->cap.max_send_sge, INLINE_OOB_LARGE_SIZE);
418		else
419			queue_size = attr->cap.max_recv_wr *
420				mana_ib_wqe_size(attr->cap.max_recv_sge, INLINE_OOB_SMALL_SIZE);
421		break;
422	default:
423		return 0;
424	}
425
426	return MANA_PAGE_ALIGN(roundup_pow_of_two(queue_size));
427}
428
429static enum gdma_queue_type mana_ib_queue_type(struct ib_qp_init_attr *attr, u32 queue_type)
430{
431	enum gdma_queue_type type;
432
433	switch (attr->qp_type) {
434	case IB_QPT_UD:
435	case IB_QPT_GSI:
436		if (queue_type == MANA_UD_SEND_QUEUE)
437			type = GDMA_SQ;
438		else
439			type = GDMA_RQ;
440		break;
441	default:
442		type = GDMA_INVALID_QUEUE;
443	}
444	return type;
445}
446
447static int mana_table_store_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
448{
449	return xa_insert_irq(&mdev->qp_table_wq, qp->ibqp.qp_num, qp,
450			     GFP_KERNEL);
451}
452
453static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
454{
455	xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num);
456}
457
458static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
459{
460	u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;
461	u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
462	int err;
463
464	err = xa_insert_irq(&mdev->qp_table_wq, qids, qp, GFP_KERNEL);
465	if (err)
466		return err;
467
468	err = xa_insert_irq(&mdev->qp_table_wq, qidr, qp, GFP_KERNEL);
469	if (err)
470		goto remove_sq;
471
472	return 0;
473
474remove_sq:
475	xa_erase_irq(&mdev->qp_table_wq, qids);
476	return err;
477}
478
479static void mana_table_remove_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
480{
481	u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK;
482	u32 qidr = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
483
484	xa_erase_irq(&mdev->qp_table_wq, qids);
485	xa_erase_irq(&mdev->qp_table_wq, qidr);
486}
487
488static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
489{
490	refcount_set(&qp->refcount, 1);
491	init_completion(&qp->free);
492
493	switch (qp->ibqp.qp_type) {
494	case IB_QPT_RC:
495		return mana_table_store_rc_qp(mdev, qp);
496	case IB_QPT_UD:
497	case IB_QPT_GSI:
498		return mana_table_store_ud_qp(mdev, qp);
499	default:
500		ibdev_dbg(&mdev->ib_dev, "Unknown QP type for storing in mana table, %d\n",
501			  qp->ibqp.qp_type);
502	}
503
504	return -EINVAL;
505}
506
507static void mana_table_remove_qp(struct mana_ib_dev *mdev,
508				 struct mana_ib_qp *qp)
509{
510	switch (qp->ibqp.qp_type) {
511	case IB_QPT_RC:
512		mana_table_remove_rc_qp(mdev, qp);
513		break;
514	case IB_QPT_UD:
515	case IB_QPT_GSI:
516		mana_table_remove_ud_qp(mdev, qp);
517		break;
518	default:
519		ibdev_dbg(&mdev->ib_dev, "Unknown QP type for removing from mana table, %d\n",
520			  qp->ibqp.qp_type);
521		return;
522	}
523	mana_put_qp_ref(qp);
524	wait_for_completion(&qp->free);
525}
526
527static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
528				struct ib_qp_init_attr *attr, struct ib_udata *udata)
529{
530	struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
531	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
532	struct mana_ib_create_rc_qp_resp resp = {};
533	struct mana_ib_ucontext *mana_ucontext;
534	struct mana_ib_create_rc_qp ucmd = {};
535	int i, err, j;
536	u64 flags = 0;
537	u32 doorbell;
538
539	if (!udata || udata->inlen < sizeof(ucmd))
540		return -EINVAL;
541
542	mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
543	doorbell = mana_ucontext->doorbell;
544	flags = MANA_RC_FLAG_NO_FMR;
545	err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
546	if (err) {
547		ibdev_dbg(&mdev->ib_dev, "Failed to copy from udata, %d\n", err);
548		return err;
549	}
550
551	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
552		/* skip FMR for user-level RC QPs */
553		if (i == MANA_RC_SEND_QUEUE_FMR) {
554			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
555			qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
556			continue;
557		}
558		err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
559					   &qp->rc_qp.queues[i]);
560		if (err) {
561			ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
562			goto destroy_queues;
563		}
564		j++;
565	}
566
567	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
568	if (err) {
569		ibdev_err(&mdev->ib_dev, "Failed to create rc qp  %d\n", err);
570		goto destroy_queues;
571	}
572	qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id;
573	qp->port = attr->port_num;
574
575	if (udata) {
576		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
577			if (i == MANA_RC_SEND_QUEUE_FMR)
578				continue;
579			resp.queue_id[j] = qp->rc_qp.queues[i].id;
580			j++;
581		}
582		err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
583		if (err) {
584			ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
585			goto destroy_qp;
586		}
587	}
588
589	err = mana_table_store_qp(mdev, qp);
590	if (err)
591		goto destroy_qp;
592
593	return 0;
594
595destroy_qp:
596	mana_ib_gd_destroy_rc_qp(mdev, qp);
597destroy_queues:
598	while (i-- > 0)
599		mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
600	return err;
601}
602
603static void mana_add_qp_to_cqs(struct mana_ib_qp *qp)
604{
605	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
606	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
607	unsigned long flags;
608
609	spin_lock_irqsave(&send_cq->cq_lock, flags);
610	list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
611	spin_unlock_irqrestore(&send_cq->cq_lock, flags);
612
613	spin_lock_irqsave(&recv_cq->cq_lock, flags);
614	list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
615	spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
616}
617
618static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp)
619{
620	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
621	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
622	unsigned long flags;
623
624	spin_lock_irqsave(&send_cq->cq_lock, flags);
625	list_del(&qp->cq_send_list);
626	spin_unlock_irqrestore(&send_cq->cq_lock, flags);
627
628	spin_lock_irqsave(&recv_cq->cq_lock, flags);
629	list_del(&qp->cq_recv_list);
630	spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
631}
632
633static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
634				struct ib_qp_init_attr *attr, struct ib_udata *udata)
635{
636	struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
637	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
638	u32 doorbell, queue_size;
639	int i, err;
640
641	if (udata) {
642		ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not supported\n");
643		return -EOPNOTSUPP;
644	}
645
646	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) {
647		queue_size = mana_ib_queue_size(attr, i);
648		err = mana_ib_create_kernel_queue(mdev, queue_size, mana_ib_queue_type(attr, i),
649						  &qp->ud_qp.queues[i]);
650		if (err) {
651			ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n",
652				  i, err);
653			goto destroy_queues;
654		}
655	}
656	doorbell = mdev->gdma_dev->doorbell;
657
658	err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr,
659				  sizeof(struct ud_rq_shadow_wqe));
660	if (err) {
661		ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err);
662		goto destroy_queues;
663	}
664	err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr,
665				  sizeof(struct ud_sq_shadow_wqe));
666	if (err) {
667		ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err);
668		goto destroy_shadow_queues;
669	}
670
671	err = mana_ib_gd_create_ud_qp(mdev, qp, attr, doorbell, attr->qp_type);
672	if (err) {
673		ibdev_err(&mdev->ib_dev, "Failed to create ud qp  %d\n", err);
674		goto destroy_shadow_queues;
675	}
676	qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
677	qp->port = attr->port_num;
678
679	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
680		qp->ud_qp.queues[i].kmem->id = qp->ud_qp.queues[i].id;
681
682	err = mana_table_store_qp(mdev, qp);
683	if (err)
684		goto destroy_qp;
685
686	mana_add_qp_to_cqs(qp);
687
688	return 0;
689
690destroy_qp:
691	mana_ib_gd_destroy_ud_qp(mdev, qp);
692destroy_shadow_queues:
693	destroy_shadow_queue(&qp->shadow_rq);
694	destroy_shadow_queue(&qp->shadow_sq);
695destroy_queues:
696	while (i-- > 0)
697		mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]);
698	return err;
699}
700
701int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
702		      struct ib_udata *udata)
703{
704	switch (attr->qp_type) {
705	case IB_QPT_RAW_PACKET:
706		/* When rwq_ind_tbl is used, it's for creating WQs for RSS */
707		if (attr->rwq_ind_tbl)
708			return mana_ib_create_qp_rss(ibqp, ibqp->pd, attr,
709						     udata);
710
711		return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
712	case IB_QPT_RC:
713		return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
714	case IB_QPT_UD:
715	case IB_QPT_GSI:
716		return mana_ib_create_ud_qp(ibqp, ibqp->pd, attr, udata);
717	default:
718		ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
719			  attr->qp_type);
720	}
721
722	return -EINVAL;
723}
724
725static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
726				int attr_mask, struct ib_udata *udata)
727{
728	struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev);
729	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
730	struct mana_rnic_set_qp_state_resp resp = {};
731	struct mana_rnic_set_qp_state_req req = {};
732	struct gdma_context *gc = mdev_to_gc(mdev);
733	struct mana_port_context *mpc;
734	struct net_device *ndev;
735	int err;
736
737	mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp));
738
739	req.hdr.req.msg_version = GDMA_MESSAGE_V3;
740	req.hdr.dev_id = mdev->gdma_dev->dev_id;
741	req.adapter = mdev->adapter_handle;
742	req.qp_handle = qp->qp_handle;
743	req.qp_state = attr->qp_state;
744	req.attr_mask = attr_mask;
745	req.path_mtu = attr->path_mtu;
746	req.rq_psn = attr->rq_psn;
747	req.sq_psn = attr->sq_psn;
748	req.dest_qpn = attr->dest_qp_num;
749	req.max_dest_rd_atomic = attr->max_dest_rd_atomic;
750	req.retry_cnt = attr->retry_cnt;
751	req.rnr_retry = attr->rnr_retry;
752	req.min_rnr_timer = attr->min_rnr_timer;
753	req.rate_limit = attr->rate_limit;
754	req.qkey = attr->qkey;
755	req.local_ack_timeout = attr->timeout;
756	req.qp_access_flags = attr->qp_access_flags;
757	req.max_rd_atomic = attr->max_rd_atomic;
758
759	if (attr_mask & IB_QP_AV) {
760		ndev = mana_ib_get_netdev(&mdev->ib_dev, ibqp->port);
761		if (!ndev) {
762			ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n",
763				  ibqp->port, ibqp->qp_num);
764			return -EINVAL;
765		}
766		mpc = netdev_priv(ndev);
767		copy_in_reverse(req.ah_attr.src_mac, mpc->mac_addr, ETH_ALEN);
768		copy_in_reverse(req.ah_attr.dest_mac, attr->ah_attr.roce.dmac, ETH_ALEN);
769		copy_in_reverse(req.ah_attr.src_addr, attr->ah_attr.grh.sgid_attr->gid.raw,
770				sizeof(union ib_gid));
771		copy_in_reverse(req.ah_attr.dest_addr, attr->ah_attr.grh.dgid.raw,
772				sizeof(union ib_gid));
773		if (rdma_gid_attr_network_type(attr->ah_attr.grh.sgid_attr) == RDMA_NETWORK_IPV4) {
774			req.ah_attr.src_addr_type = SGID_TYPE_IPV4;
775			req.ah_attr.dest_addr_type = SGID_TYPE_IPV4;
776		} else {
777			req.ah_attr.src_addr_type = SGID_TYPE_IPV6;
778			req.ah_attr.dest_addr_type = SGID_TYPE_IPV6;
779		}
780		req.ah_attr.dest_port = ROCE_V2_UDP_DPORT;
781		req.ah_attr.src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
782							  ibqp->qp_num, attr->dest_qp_num);
783		req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class >> 2;
784		req.ah_attr.hop_limit = attr->ah_attr.grh.hop_limit;
785		req.ah_attr.flow_label = attr->ah_attr.grh.flow_label;
786	}
787
788	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
789	if (err) {
790		ibdev_err(&mdev->ib_dev, "Failed modify qp err %d", err);
791		return err;
792	}
793
794	return 0;
795}
796
797int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
798		      int attr_mask, struct ib_udata *udata)
799{
800	switch (ibqp->qp_type) {
801	case IB_QPT_RC:
802	case IB_QPT_UD:
803	case IB_QPT_GSI:
804		return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
805	default:
806		ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type);
807		return -EOPNOTSUPP;
808	}
809}
810
811static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
812				  struct ib_rwq_ind_table *ind_tbl,
813				  struct ib_udata *udata)
814{
815	struct mana_ib_dev *mdev =
816		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
817	struct mana_port_context *mpc;
818	struct net_device *ndev;
819	struct mana_ib_wq *wq;
820	struct ib_wq *ibwq;
821	int i;
822
823	ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port);
824	mpc = netdev_priv(ndev);
825
826	for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
827		ibwq = ind_tbl->ind_tbl[i];
828		wq = container_of(ibwq, struct mana_ib_wq, ibwq);
829		ibdev_dbg(&mdev->ib_dev, "destroying wq->rx_object %llu\n",
830			  wq->rx_object);
831		mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object);
832	}
833
834	return 0;
835}
836
837static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata)
838{
839	struct mana_ib_dev *mdev =
840		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
841	struct ib_pd *ibpd = qp->ibqp.pd;
842	struct mana_port_context *mpc;
843	struct net_device *ndev;
844	struct mana_ib_pd *pd;
845
846	ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port);
847	mpc = netdev_priv(ndev);
848	pd = container_of(ibpd, struct mana_ib_pd, ibpd);
849
850	mana_destroy_wq_obj(mpc, GDMA_SQ, qp->qp_handle);
851
852	mana_ib_destroy_queue(mdev, &qp->raw_sq);
853
854	mana_ib_uncfg_vport(mdev, pd, qp->port);
855
856	return 0;
857}
858
859static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
860{
861	struct mana_ib_dev *mdev =
862		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
863	int i;
864
865	mana_table_remove_qp(mdev, qp);
866
867	/* Ignore return code as there is not much we can do about it.
868	 * The error message is printed inside.
869	 */
870	mana_ib_gd_destroy_rc_qp(mdev, qp);
871	for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i)
872		mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
873
874	return 0;
875}
876
877static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
878{
879	struct mana_ib_dev *mdev =
880		container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
881	int i;
882
883	mana_remove_qp_from_cqs(qp);
884	mana_table_remove_qp(mdev, qp);
885
886	destroy_shadow_queue(&qp->shadow_rq);
887	destroy_shadow_queue(&qp->shadow_sq);
888
889	/* Ignore return code as there is not much we can do about it.
890	 * The error message is printed inside.
891	 */
892	mana_ib_gd_destroy_ud_qp(mdev, qp);
893	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
894		mana_ib_destroy_queue(mdev, &qp->ud_qp.queues[i]);
895
896	return 0;
897}
898
899int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
900{
901	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
902
903	switch (ibqp->qp_type) {
904	case IB_QPT_RAW_PACKET:
905		if (ibqp->rwq_ind_tbl)
906			return mana_ib_destroy_qp_rss(qp, ibqp->rwq_ind_tbl,
907						      udata);
908
909		return mana_ib_destroy_qp_raw(qp, udata);
910	case IB_QPT_RC:
911		return mana_ib_destroy_rc_qp(qp, udata);
912	case IB_QPT_UD:
913	case IB_QPT_GSI:
914		return mana_ib_destroy_ud_qp(qp, udata);
915	default:
916		ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
917			  ibqp->qp_type);
918	}
919
920	return -ENOENT;
921}