conn.c

  1/*
  2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 *
 32 */
 33
 34#include <net/addrconf.h>
 35#include <linux/etherdevice.h>
 36#include <linux/mlx5/vport.h>
 37
 38#include "mlx5_core.h"
 39#include "lib/mlx5.h"
 40#include "fpga/conn.h"
 41
 42#define MLX5_FPGA_PKEY 0xFFFF
 43#define MLX5_FPGA_PKEY_INDEX 0 /* RoCE PKEY 0xFFFF is always at index 0 */
 44#define MLX5_FPGA_RECV_SIZE 2048
 45#define MLX5_FPGA_PORT_NUM 1
 46#define MLX5_FPGA_CQ_BUDGET 64
 47
 48static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn,
 49				  struct mlx5_fpga_dma_buf *buf)
 50{
 51	struct device *dma_device;
 52	int err = 0;
 53
 54	if (unlikely(!buf->sg[0].data))
 55		goto out;
 56
 57	dma_device = mlx5_core_dma_dev(conn->fdev->mdev);
 58	buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data,
 59					     buf->sg[0].size, buf->dma_dir);
 60	err = dma_mapping_error(dma_device, buf->sg[0].dma_addr);
 61	if (unlikely(err)) {
 62		mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err);
 63		err = -ENOMEM;
 64		goto out;
 65	}
 66
 67	if (!buf->sg[1].data)
 68		goto out;
 69
 70	buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data,
 71					     buf->sg[1].size, buf->dma_dir);
 72	err = dma_mapping_error(dma_device, buf->sg[1].dma_addr);
 73	if (unlikely(err)) {
 74		mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err);
 75		dma_unmap_single(dma_device, buf->sg[0].dma_addr,
 76				 buf->sg[0].size, buf->dma_dir);
 77		err = -ENOMEM;
 78	}
 79
 80out:
 81	return err;
 82}
 83
 84static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn,
 85				     struct mlx5_fpga_dma_buf *buf)
 86{
 87	struct device *dma_device;
 88
 89	dma_device = mlx5_core_dma_dev(conn->fdev->mdev);
 90	if (buf->sg[1].data)
 91		dma_unmap_single(dma_device, buf->sg[1].dma_addr,
 92				 buf->sg[1].size, buf->dma_dir);
 93
 94	if (likely(buf->sg[0].data))
 95		dma_unmap_single(dma_device, buf->sg[0].dma_addr,
 96				 buf->sg[0].size, buf->dma_dir);
 97}
 98
 99static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
100				    struct mlx5_fpga_dma_buf *buf)
101{
102	struct mlx5_wqe_data_seg *data;
103	unsigned int ix;
104	int err = 0;
105
106	err = mlx5_fpga_conn_map_buf(conn, buf);
107	if (unlikely(err))
108		goto out;
109
110	if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) {
111		mlx5_fpga_conn_unmap_buf(conn, buf);
112		return -EBUSY;
113	}
114
115	ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
116	data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
117	data->byte_count = cpu_to_be32(buf->sg[0].size);
118	data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey);
119	data->addr = cpu_to_be64(buf->sg[0].dma_addr);
120
121	conn->qp.rq.pc++;
122	conn->qp.rq.bufs[ix] = buf;
123
124	/* Make sure that descriptors are written before doorbell record. */
125	dma_wmb();
126	*conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff);
127out:
128	return err;
129}
130
131static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe)
132{
133	/* ensure wqe is visible to device before updating doorbell record */
134	dma_wmb();
135	*conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc);
136	/* Make sure that doorbell record is visible before ringing */
137	wmb();
138	mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET);
139}
140
141static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn,
142				     struct mlx5_fpga_dma_buf *buf)
143{
144	struct mlx5_wqe_ctrl_seg *ctrl;
145	struct mlx5_wqe_data_seg *data;
146	unsigned int ix, sgi;
147	int size = 1;
148
149	ix = conn->qp.sq.pc & (conn->qp.sq.size - 1);
150
151	ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix);
152	data = (void *)(ctrl + 1);
153
154	for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) {
155		if (!buf->sg[sgi].data)
156			break;
157		data->byte_count = cpu_to_be32(buf->sg[sgi].size);
158		data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey);
159		data->addr = cpu_to_be64(buf->sg[sgi].dma_addr);
160		data++;
161		size++;
162	}
163
164	ctrl->imm = 0;
165	ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
166	ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) |
167					     MLX5_OPCODE_SEND);
168	ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.qpn << 8));
169
170	conn->qp.sq.pc++;
171	conn->qp.sq.bufs[ix] = buf;
172	mlx5_fpga_conn_notify_hw(conn, ctrl);
173}
174
175int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn,
176			struct mlx5_fpga_dma_buf *buf)
177{
178	unsigned long flags;
179	int err;
180
181	if (!conn->qp.active)
182		return -ENOTCONN;
183
184	buf->dma_dir = DMA_TO_DEVICE;
185	err = mlx5_fpga_conn_map_buf(conn, buf);
186	if (err)
187		return err;
188
189	spin_lock_irqsave(&conn->qp.sq.lock, flags);
190
191	if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) {
192		list_add_tail(&buf->list, &conn->qp.sq.backlog);
193		goto out_unlock;
194	}
195
196	mlx5_fpga_conn_post_send(conn, buf);
197
198out_unlock:
199	spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
200	return err;
201}
202
203static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn)
204{
205	struct mlx5_fpga_dma_buf *buf;
206	int err;
207
208	buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0);
209	if (!buf)
210		return -ENOMEM;
211
212	buf->sg[0].data = (void *)(buf + 1);
213	buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
214	buf->dma_dir = DMA_FROM_DEVICE;
215
216	err = mlx5_fpga_conn_post_recv(conn, buf);
217	if (err)
218		kfree(buf);
219
220	return err;
221}
222
223static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
224				      u32 *mkey)
225{
226	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
227	void *mkc;
228	u32 *in;
229	int err;
230
231	in = kvzalloc(inlen, GFP_KERNEL);
232	if (!in)
233		return -ENOMEM;
234
235	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
236	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
237	MLX5_SET(mkc, mkc, lw, 1);
238	MLX5_SET(mkc, mkc, lr, 1);
239
240	MLX5_SET(mkc, mkc, pd, pdn);
241	MLX5_SET(mkc, mkc, length64, 1);
242	MLX5_SET(mkc, mkc, qpn, 0xffffff);
243
244	err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
245
246	kvfree(in);
247	return err;
248}
249
250static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn,
251				  struct mlx5_cqe64 *cqe, u8 status)
252{
253	struct mlx5_fpga_dma_buf *buf;
254	int ix, err;
255
256	ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1);
257	buf = conn->qp.rq.bufs[ix];
258	conn->qp.rq.bufs[ix] = NULL;
259	conn->qp.rq.cc++;
260
261	if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
262		mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
263			       buf, conn->fpga_qpn, status);
264	else
265		mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
266			      buf, conn->fpga_qpn, status);
267
268	mlx5_fpga_conn_unmap_buf(conn, buf);
269
270	if (unlikely(status || !conn->qp.active)) {
271		conn->qp.active = false;
272		kfree(buf);
273		return;
274	}
275
276	buf->sg[0].size = be32_to_cpu(cqe->byte_cnt);
277	mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n",
278		      buf->sg[0].size);
279	conn->recv_cb(conn->cb_arg, buf);
280
281	buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
282	err = mlx5_fpga_conn_post_recv(conn, buf);
283	if (unlikely(err)) {
284		mlx5_fpga_warn(conn->fdev,
285			       "Failed to re-post recv buf: %d\n", err);
286		kfree(buf);
287	}
288}
289
290static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn,
291				  struct mlx5_cqe64 *cqe, u8 status)
292{
293	struct mlx5_fpga_dma_buf *buf, *nextbuf;
294	unsigned long flags;
295	int ix;
296
297	spin_lock_irqsave(&conn->qp.sq.lock, flags);
298
299	ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1);
300	buf = conn->qp.sq.bufs[ix];
301	conn->qp.sq.bufs[ix] = NULL;
302	conn->qp.sq.cc++;
303
304	/* Handle backlog still under the spinlock to ensure message post order */
305	if (unlikely(!list_empty(&conn->qp.sq.backlog))) {
306		if (likely(conn->qp.active)) {
307			nextbuf = list_first_entry(&conn->qp.sq.backlog,
308						   struct mlx5_fpga_dma_buf, list);
309			list_del(&nextbuf->list);
310			mlx5_fpga_conn_post_send(conn, nextbuf);
311		}
312	}
313
314	spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
315
316	if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
317		mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
318			       buf, conn->fpga_qpn, status);
319	else
320		mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
321			      buf, conn->fpga_qpn, status);
322
323	mlx5_fpga_conn_unmap_buf(conn, buf);
324
325	if (likely(buf->complete))
326		buf->complete(conn, conn->fdev, buf, status);
327
328	if (unlikely(status))
329		conn->qp.active = false;
330}
331
332static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn,
333				      struct mlx5_cqe64 *cqe)
334{
335	u8 opcode, status = 0;
336
337	opcode = get_cqe_opcode(cqe);
338
339	switch (opcode) {
340	case MLX5_CQE_REQ_ERR:
341		status = ((struct mlx5_err_cqe *)cqe)->syndrome;
342		fallthrough;
343	case MLX5_CQE_REQ:
344		mlx5_fpga_conn_sq_cqe(conn, cqe, status);
345		break;
346
347	case MLX5_CQE_RESP_ERR:
348		status = ((struct mlx5_err_cqe *)cqe)->syndrome;
349		fallthrough;
350	case MLX5_CQE_RESP_SEND:
351		mlx5_fpga_conn_rq_cqe(conn, cqe, status);
352		break;
353	default:
354		mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n",
355			       opcode);
356	}
357}
358
359static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn)
360{
361	mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT,
362		    conn->fdev->conn_res.uar->map, conn->cq.wq.cc);
363}
364
365static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn,
366				       unsigned int budget)
367{
368	struct mlx5_cqe64 *cqe;
369
370	while (budget) {
371		cqe = mlx5_cqwq_get_cqe(&conn->cq.wq);
372		if (!cqe)
373			break;
374
375		budget--;
376		mlx5_cqwq_pop(&conn->cq.wq);
377		mlx5_fpga_conn_handle_cqe(conn, cqe);
378		mlx5_cqwq_update_db_record(&conn->cq.wq);
379	}
380	if (!budget) {
381		tasklet_schedule(&conn->cq.tasklet);
382		return;
383	}
384
385	mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc);
386	/* ensure cq space is freed before enabling more cqes */
387	wmb();
388	mlx5_fpga_conn_arm_cq(conn);
389}
390
391static void mlx5_fpga_conn_cq_tasklet(struct tasklet_struct *t)
392{
393	struct mlx5_fpga_conn *conn = from_tasklet(conn, t, cq.tasklet);
394
395	if (unlikely(!conn->qp.active))
396		return;
397	mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
398}
399
400static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq,
401				       struct mlx5_eqe *eqe)
402{
403	struct mlx5_fpga_conn *conn;
404
405	conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
406	if (unlikely(!conn->qp.active))
407		return;
408	mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
409}
410
411static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
412{
413	struct mlx5_fpga_device *fdev = conn->fdev;
414	struct mlx5_core_dev *mdev = fdev->mdev;
415	u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0};
416	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
417	struct mlx5_wq_param wqp;
418	struct mlx5_cqe64 *cqe;
419	int inlen, err, eqn;
420	void *cqc, *in;
421	__be64 *pas;
422	u32 i;
423
424	conn->cq.mcq.cqe_sz     = 64;
425	conn->cq.mcq.set_ci_db  = conn->cq.wq_ctrl.db.db;
426	conn->cq.mcq.arm_db     = conn->cq.wq_ctrl.db.db + 1;
427	*conn->cq.mcq.set_ci_db = 0;
428	conn->cq.mcq.vector     = 0;
429	conn->cq.mcq.comp       = mlx5_fpga_conn_cq_complete;
430
431	cq_size = roundup_pow_of_two(cq_size);
432	MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size));
433
434	wqp.buf_numa_node = mdev->priv.numa_node;
435	wqp.db_numa_node  = mdev->priv.numa_node;
436
437	err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq,
438			       &conn->cq.wq_ctrl);
439	if (err)
440		return err;
441
442	for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) {
443		cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i);
444		cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK;
445	}
446
447	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
448		sizeof(u64) * conn->cq.wq_ctrl.buf.npages;
449	in = kvzalloc(inlen, GFP_KERNEL);
450	if (!in) {
451		err = -ENOMEM;
452		goto err_cqwq;
453	}
454
455	err = mlx5_comp_eqn_get(mdev, smp_processor_id(), &eqn);
456	if (err) {
457		kvfree(in);
458		goto err_cqwq;
459	}
460
461	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
462	MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size));
463	MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
464	MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index);
465	MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.buf.page_shift -
466			   MLX5_ADAPTER_PAGE_SHIFT);
467	MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma);
468
469	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
470	mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.buf, pas);
471
472	err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen, out, sizeof(out));
473	kvfree(in);
474
475	if (err)
476		goto err_cqwq;
477
478	tasklet_setup(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet);
479	mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn);
480
481	goto out;
482
483err_cqwq:
484	mlx5_wq_destroy(&conn->cq.wq_ctrl);
485out:
486	return err;
487}
488
489static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn)
490{
491	tasklet_disable(&conn->cq.tasklet);
492	tasklet_kill(&conn->cq.tasklet);
493	mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq);
494	mlx5_wq_destroy(&conn->cq.wq_ctrl);
495}
496
497static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc)
498{
499	struct mlx5_fpga_device *fdev = conn->fdev;
500	struct mlx5_core_dev *mdev = fdev->mdev;
501	struct mlx5_wq_param wqp;
502
503	wqp.buf_numa_node = mdev->priv.numa_node;
504	wqp.db_numa_node  = mdev->priv.numa_node;
505
506	return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq,
507				 &conn->qp.wq_ctrl);
508}
509
510static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn,
511				    unsigned int tx_size, unsigned int rx_size)
512{
513	struct mlx5_fpga_device *fdev = conn->fdev;
514	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
515	struct mlx5_core_dev *mdev = fdev->mdev;
516	u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {};
517	void *in = NULL, *qpc;
518	int err, inlen;
519
520	conn->qp.rq.pc = 0;
521	conn->qp.rq.cc = 0;
522	conn->qp.rq.size = roundup_pow_of_two(rx_size);
523	conn->qp.sq.pc = 0;
524	conn->qp.sq.cc = 0;
525	conn->qp.sq.size = roundup_pow_of_two(tx_size);
526
527	MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
528	MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size));
529	MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size));
530	err = mlx5_fpga_conn_create_wq(conn, temp_qpc);
531	if (err)
532		goto out;
533
534	conn->qp.rq.bufs = kvcalloc(conn->qp.rq.size,
535				    sizeof(conn->qp.rq.bufs[0]),
536				    GFP_KERNEL);
537	if (!conn->qp.rq.bufs) {
538		err = -ENOMEM;
539		goto err_wq;
540	}
541
542	conn->qp.sq.bufs = kvcalloc(conn->qp.sq.size,
543				    sizeof(conn->qp.sq.bufs[0]),
544				    GFP_KERNEL);
545	if (!conn->qp.sq.bufs) {
546		err = -ENOMEM;
547		goto err_rq_bufs;
548	}
549
550	inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
551		MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) *
552		conn->qp.wq_ctrl.buf.npages;
553	in = kvzalloc(inlen, GFP_KERNEL);
554	if (!in) {
555		err = -ENOMEM;
556		goto err_sq_bufs;
557	}
558
559	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
560	MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index);
561	MLX5_SET(qpc, qpc, log_page_size,
562		 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
563	MLX5_SET(qpc, qpc, fre, 1);
564	MLX5_SET(qpc, qpc, rlky, 1);
565	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
566	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
567	MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn);
568	MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
569	MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size));
570	MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
571	MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size));
572	MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
573	MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
574	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(mdev));
575	MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
576	if (MLX5_CAP_GEN(mdev, cqe_version) == 1)
577		MLX5_SET(qpc, qpc, user_index, 0xFFFFFF);
578
579	mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf,
580				  (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas));
581
582	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
583	err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
584	if (err)
585		goto err_sq_bufs;
586
587	conn->qp.qpn = MLX5_GET(create_qp_out, out, qpn);
588	mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.qpn);
589
590	goto out;
591
592err_sq_bufs:
593	kvfree(conn->qp.sq.bufs);
594err_rq_bufs:
595	kvfree(conn->qp.rq.bufs);
596err_wq:
597	mlx5_wq_destroy(&conn->qp.wq_ctrl);
598out:
599	kvfree(in);
600	return err;
601}
602
603static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn)
604{
605	int ix;
606
607	for (ix = 0; ix < conn->qp.rq.size; ix++) {
608		if (!conn->qp.rq.bufs[ix])
609			continue;
610		mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]);
611		kfree(conn->qp.rq.bufs[ix]);
612		conn->qp.rq.bufs[ix] = NULL;
613	}
614}
615
616static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn)
617{
618	struct mlx5_fpga_dma_buf *buf, *temp;
619	int ix;
620
621	for (ix = 0; ix < conn->qp.sq.size; ix++) {
622		buf = conn->qp.sq.bufs[ix];
623		if (!buf)
624			continue;
625		conn->qp.sq.bufs[ix] = NULL;
626		mlx5_fpga_conn_unmap_buf(conn, buf);
627		if (!buf->complete)
628			continue;
629		buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
630	}
631	list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) {
632		mlx5_fpga_conn_unmap_buf(conn, buf);
633		if (!buf->complete)
634			continue;
635		buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
636	}
637}
638
639static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn)
640{
641	struct mlx5_core_dev *dev = conn->fdev->mdev;
642	u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
643
644	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
645	MLX5_SET(destroy_qp_in, in, qpn, conn->qp.qpn);
646	mlx5_cmd_exec_in(dev, destroy_qp, in);
647
648	mlx5_fpga_conn_free_recv_bufs(conn);
649	mlx5_fpga_conn_flush_send_bufs(conn);
650	kvfree(conn->qp.sq.bufs);
651	kvfree(conn->qp.rq.bufs);
652	mlx5_wq_destroy(&conn->qp.wq_ctrl);
653}
654
655static int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn)
656{
657	struct mlx5_core_dev *mdev = conn->fdev->mdev;
658	u32 in[MLX5_ST_SZ_DW(qp_2rst_in)] = {};
659
660	mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.qpn);
661
662	MLX5_SET(qp_2rst_in, in, opcode, MLX5_CMD_OP_2RST_QP);
663	MLX5_SET(qp_2rst_in, in, qpn, conn->qp.qpn);
664
665	return mlx5_cmd_exec_in(mdev, qp_2rst, in);
666}
667
668static int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn)
669{
670	u32 in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {};
671	struct mlx5_fpga_device *fdev = conn->fdev;
672	struct mlx5_core_dev *mdev = fdev->mdev;
673	u32 *qpc;
674
675	mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.qpn);
676
677	qpc = MLX5_ADDR_OF(rst2init_qp_in, in, qpc);
678
679	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
680	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
681	MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
682	MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
683	MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn);
684	MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
685	MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
686	MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
687
688	MLX5_SET(rst2init_qp_in, in, opcode, MLX5_CMD_OP_RST2INIT_QP);
689	MLX5_SET(rst2init_qp_in, in, qpn, conn->qp.qpn);
690
691	return mlx5_cmd_exec_in(mdev, rst2init_qp, in);
692}
693
694static int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn)
695{
696	u32 in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {};
697	struct mlx5_fpga_device *fdev = conn->fdev;
698	struct mlx5_core_dev *mdev = fdev->mdev;
699	u32 *qpc;
700
701	mlx5_fpga_dbg(conn->fdev, "QP RTR\n");
702
703	qpc = MLX5_ADDR_OF(init2rtr_qp_in, in, qpc);
704
705	MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES);
706	MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg));
707	MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn);
708	MLX5_SET(qpc, qpc, next_rcv_psn,
709		 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn));
710	MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
711	MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
712	ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
713			MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32));
714	MLX5_SET(qpc, qpc, primary_address_path.udp_sport,
715		 MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port));
716	MLX5_SET(qpc, qpc, primary_address_path.src_addr_index,
717		 conn->qp.sgid_index);
718	MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0);
719	memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip),
720	       MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip),
721	       MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip));
722
723	MLX5_SET(init2rtr_qp_in, in, opcode, MLX5_CMD_OP_INIT2RTR_QP);
724	MLX5_SET(init2rtr_qp_in, in, qpn, conn->qp.qpn);
725
726	return mlx5_cmd_exec_in(mdev, init2rtr_qp, in);
727}
728
729static int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn)
730{
731	struct mlx5_fpga_device *fdev = conn->fdev;
732	u32 in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {};
733	struct mlx5_core_dev *mdev = fdev->mdev;
734	u32 *qpc;
735
736	mlx5_fpga_dbg(conn->fdev, "QP RTS\n");
737
738	qpc = MLX5_ADDR_OF(rtr2rts_qp_in, in, qpc);
739
740	MLX5_SET(qpc, qpc, log_ack_req_freq, 8);
741	MLX5_SET(qpc, qpc, min_rnr_nak, 0x12);
742	MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12); /* ~1.07s */
743	MLX5_SET(qpc, qpc, next_send_psn,
744		 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn));
745	MLX5_SET(qpc, qpc, retry_count, 7);
746	MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */
747
748	MLX5_SET(rtr2rts_qp_in, in, opcode, MLX5_CMD_OP_RTR2RTS_QP);
749	MLX5_SET(rtr2rts_qp_in, in, qpn, conn->qp.qpn);
750	MLX5_SET(rtr2rts_qp_in, in, opt_param_mask, MLX5_QP_OPTPAR_RNR_TIMEOUT);
751
752	return mlx5_cmd_exec_in(mdev, rtr2rts_qp, in);
753}
754
755static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn)
756{
757	struct mlx5_fpga_device *fdev = conn->fdev;
758	int err;
759
760	MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE);
761	err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
762				  MLX5_FPGA_QPC_STATE, &conn->fpga_qpc);
763	if (err) {
764		mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err);
765		goto out;
766	}
767
768	err = mlx5_fpga_conn_reset_qp(conn);
769	if (err) {
770		mlx5_fpga_err(fdev, "Failed to change QP state to reset\n");
771		goto err_fpga_qp;
772	}
773
774	err = mlx5_fpga_conn_init_qp(conn);
775	if (err) {
776		mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n");
777		goto err_fpga_qp;
778	}
779	conn->qp.active = true;
780
781	while (!mlx5_fpga_conn_post_recv_buf(conn))
782		;
783
784	err = mlx5_fpga_conn_rtr_qp(conn);
785	if (err) {
786		mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n");
787		goto err_recv_bufs;
788	}
789
790	err = mlx5_fpga_conn_rts_qp(conn);
791	if (err) {
792		mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n");
793		goto err_recv_bufs;
794	}
795	goto out;
796
797err_recv_bufs:
798	mlx5_fpga_conn_free_recv_bufs(conn);
799err_fpga_qp:
800	MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
801	if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
802				MLX5_FPGA_QPC_STATE, &conn->fpga_qpc))
803		mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n");
804out:
805	return err;
806}
807
808struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev,
809					     struct mlx5_fpga_conn_attr *attr,
810					     enum mlx5_ifc_fpga_qp_type qp_type)
811{
812	struct mlx5_fpga_conn *ret, *conn;
813	u8 *remote_mac, *remote_ip;
814	int err;
815
816	if (!attr->recv_cb)
817		return ERR_PTR(-EINVAL);
818
819	conn = kzalloc(sizeof(*conn), GFP_KERNEL);
820	if (!conn)
821		return ERR_PTR(-ENOMEM);
822
823	conn->fdev = fdev;
824	INIT_LIST_HEAD(&conn->qp.sq.backlog);
825
826	spin_lock_init(&conn->qp.sq.lock);
827
828	conn->recv_cb = attr->recv_cb;
829	conn->cb_arg = attr->cb_arg;
830
831	remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32);
832	err = mlx5_query_mac_address(fdev->mdev, remote_mac);
833	if (err) {
834		mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err);
835		ret = ERR_PTR(err);
836		goto err;
837	}
838
839	/* Build Modified EUI-64 IPv6 address from the MAC address */
840	remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip);
841	remote_ip[0] = 0xfe;
842	remote_ip[1] = 0x80;
843	addrconf_addr_eui48(&remote_ip[8], remote_mac);
844
845	err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index);
846	if (err) {
847		mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err);
848		ret = ERR_PTR(err);
849		goto err;
850	}
851
852	err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index,
853				     MLX5_ROCE_VERSION_2,
854				     MLX5_ROCE_L3_TYPE_IPV6,
855				     remote_ip, remote_mac, true, 0,
856				     MLX5_FPGA_PORT_NUM);
857	if (err) {
858		mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err);
859		ret = ERR_PTR(err);
860		goto err_rsvd_gid;
861	}
862	mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index);
863
864	/* Allow for one cqe per rx/tx wqe, plus one cqe for the next wqe,
865	 * created during processing of the cqe
866	 */
867	err = mlx5_fpga_conn_create_cq(conn,
868				       (attr->tx_size + attr->rx_size) * 2);
869	if (err) {
870		mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err);
871		ret = ERR_PTR(err);
872		goto err_gid;
873	}
874
875	mlx5_fpga_conn_arm_cq(conn);
876
877	err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size);
878	if (err) {
879		mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err);
880		ret = ERR_PTR(err);
881		goto err_cq;
882	}
883
884	MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
885	MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type);
886	MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC);
887	MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q);
888	MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0);
889	MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1);
890	MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0);
891	MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY);
892	MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.qpn);
893	MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7);
894	MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7);
895
896	err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc,
897				  &conn->fpga_qpn);
898	if (err) {
899		mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err);
900		ret = ERR_PTR(err);
901		goto err_qp;
902	}
903
904	err = mlx5_fpga_conn_connect(conn);
905	if (err) {
906		ret = ERR_PTR(err);
907		goto err_conn;
908	}
909
910	mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn);
911	ret = conn;
912	goto out;
913
914err_conn:
915	mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
916err_qp:
917	mlx5_fpga_conn_destroy_qp(conn);
918err_cq:
919	mlx5_fpga_conn_destroy_cq(conn);
920err_gid:
921	mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL,
922			       NULL, false, 0, MLX5_FPGA_PORT_NUM);
923err_rsvd_gid:
924	mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index);
925err:
926	kfree(conn);
927out:
928	return ret;
929}
930
931void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn)
932{
933	conn->qp.active = false;
934	tasklet_disable(&conn->cq.tasklet);
935	synchronize_irq(conn->cq.mcq.irqn);
936
937	mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
938	mlx5_fpga_conn_destroy_qp(conn);
939	mlx5_fpga_conn_destroy_cq(conn);
940
941	mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0,
942			       NULL, NULL, false, 0, MLX5_FPGA_PORT_NUM);
943	mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index);
944	kfree(conn);
945}
946
947int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev)
948{
949	int err;
950
951	err = mlx5_nic_vport_enable_roce(fdev->mdev);
952	if (err) {
953		mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err);
954		goto out;
955	}
956
957	fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev);
958	if (IS_ERR(fdev->conn_res.uar)) {
959		err = PTR_ERR(fdev->conn_res.uar);
960		mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err);
961		goto err_roce;
962	}
963	mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n",
964		      fdev->conn_res.uar->index);
965
966	err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn);
967	if (err) {
968		mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err);
969		goto err_uar;
970	}
971	mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn);
972
973	err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn,
974					 &fdev->conn_res.mkey);
975	if (err) {
976		mlx5_fpga_err(fdev, "create mkey failed, %d\n", err);
977		goto err_dealloc_pd;
978	}
979	mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey);
980
981	return 0;
982
983err_dealloc_pd:
984	mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
985err_uar:
986	mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
987err_roce:
988	mlx5_nic_vport_disable_roce(fdev->mdev);
989out:
990	return err;
991}
992
993void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev)
994{
995	mlx5_core_destroy_mkey(fdev->mdev, fdev->conn_res.mkey);
996	mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
997	mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
998	mlx5_nic_vport_disable_roce(fdev->mdev);
999}