gsi.c

  1/*
  2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#include "mlx5_ib.h"
 34
 35struct mlx5_ib_gsi_wr {
 36	struct ib_cqe cqe;
 37	struct ib_wc wc;
 38	bool completed:1;
 39};
 40
 41static bool mlx5_ib_deth_sqpn_cap(struct mlx5_ib_dev *dev)
 42{
 43	return MLX5_CAP_GEN(dev->mdev, set_deth_sqpn);
 44}
 45
 46/* Call with gsi->lock locked */
 47static void generate_completions(struct mlx5_ib_qp *mqp)
 48{
 49	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
 50	struct ib_cq *gsi_cq = mqp->ibqp.send_cq;
 51	struct mlx5_ib_gsi_wr *wr;
 52	u32 index;
 53
 54	for (index = gsi->outstanding_ci; index != gsi->outstanding_pi;
 55	     index++) {
 56		wr = &gsi->outstanding_wrs[index % gsi->cap.max_send_wr];
 57
 58		if (!wr->completed)
 59			break;
 60
 61		WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &wr->wc));
 62		wr->completed = false;
 63	}
 64
 65	gsi->outstanding_ci = index;
 66}
 67
 68static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc)
 69{
 70	struct mlx5_ib_gsi_qp *gsi = cq->cq_context;
 71	struct mlx5_ib_gsi_wr *wr =
 72		container_of(wc->wr_cqe, struct mlx5_ib_gsi_wr, cqe);
 73	struct mlx5_ib_qp *mqp = container_of(gsi, struct mlx5_ib_qp, gsi);
 74	u64 wr_id;
 75	unsigned long flags;
 76
 77	spin_lock_irqsave(&gsi->lock, flags);
 78	wr->completed = true;
 79	wr_id = wr->wc.wr_id;
 80	wr->wc = *wc;
 81	wr->wc.wr_id = wr_id;
 82	wr->wc.qp = &mqp->ibqp;
 83
 84	generate_completions(mqp);
 85	spin_unlock_irqrestore(&gsi->lock, flags);
 86}
 87
 88int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp,
 89		       struct ib_qp_init_attr *attr)
 90{
 91	struct mlx5_ib_dev *dev = to_mdev(pd->device);
 92	struct mlx5_ib_gsi_qp *gsi;
 93	struct ib_qp_init_attr hw_init_attr = *attr;
 94	const u8 port_num = attr->port_num;
 95	int num_qps = 0;
 96	int ret;
 97
 98	if (mlx5_ib_deth_sqpn_cap(dev)) {
 99		if (MLX5_CAP_GEN(dev->mdev,
100				 port_type) == MLX5_CAP_PORT_TYPE_IB)
101			num_qps = pd->device->attrs.max_pkeys;
102		else if (dev->lag_active)
103			num_qps = dev->lag_ports;
104	}
105
106	gsi = &mqp->gsi;
107	gsi->tx_qps = kcalloc(num_qps, sizeof(*gsi->tx_qps), GFP_KERNEL);
108	if (!gsi->tx_qps)
109		return -ENOMEM;
110
111	gsi->outstanding_wrs =
112		kcalloc(attr->cap.max_send_wr, sizeof(*gsi->outstanding_wrs),
113			GFP_KERNEL);
114	if (!gsi->outstanding_wrs) {
115		ret = -ENOMEM;
116		goto err_free_tx;
117	}
118
119	if (dev->devr.ports[port_num - 1].gsi) {
120		mlx5_ib_warn(dev, "GSI QP already exists on port %d\n",
121			     port_num);
122		ret = -EBUSY;
123		goto err_free_wrs;
124	}
125	gsi->num_qps = num_qps;
126	spin_lock_init(&gsi->lock);
127
128	gsi->cap = attr->cap;
129	gsi->port_num = port_num;
130
131	gsi->cq = ib_alloc_cq(pd->device, gsi, attr->cap.max_send_wr, 0,
132			      IB_POLL_SOFTIRQ);
133	if (IS_ERR(gsi->cq)) {
134		mlx5_ib_warn(dev,
135			     "unable to create send CQ for GSI QP. error %pe\n",
136			     gsi->cq);
137		ret = PTR_ERR(gsi->cq);
138		goto err_free_wrs;
139	}
140
141	hw_init_attr.qp_type = MLX5_IB_QPT_HW_GSI;
142	hw_init_attr.send_cq = gsi->cq;
143	if (num_qps) {
144		hw_init_attr.cap.max_send_wr = 0;
145		hw_init_attr.cap.max_send_sge = 0;
146		hw_init_attr.cap.max_inline_data = 0;
147	}
148
149	gsi->rx_qp = ib_create_qp(pd, &hw_init_attr);
150	if (IS_ERR(gsi->rx_qp)) {
151		mlx5_ib_warn(dev,
152			     "unable to create hardware GSI QP. error %pe\n",
153			     gsi->rx_qp);
154		ret = PTR_ERR(gsi->rx_qp);
155		goto err_destroy_cq;
156	}
157
158	dev->devr.ports[attr->port_num - 1].gsi = gsi;
159	return 0;
160
161err_destroy_cq:
162	ib_free_cq(gsi->cq);
163err_free_wrs:
164	kfree(gsi->outstanding_wrs);
165err_free_tx:
166	kfree(gsi->tx_qps);
167	return ret;
168}
169
170int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp)
171{
172	struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
173	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
174	const int port_num = gsi->port_num;
175	int qp_index;
176	int ret;
177
178	ret = ib_destroy_qp(gsi->rx_qp);
179	if (ret) {
180		mlx5_ib_warn(dev, "unable to destroy hardware GSI QP. error %d\n",
181			     ret);
182		return ret;
183	}
184	dev->devr.ports[port_num - 1].gsi = NULL;
185	gsi->rx_qp = NULL;
186
187	for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index) {
188		if (!gsi->tx_qps[qp_index])
189			continue;
190		WARN_ON_ONCE(ib_destroy_qp(gsi->tx_qps[qp_index]));
191		gsi->tx_qps[qp_index] = NULL;
192	}
193
194	ib_free_cq(gsi->cq);
195
196	kfree(gsi->outstanding_wrs);
197	kfree(gsi->tx_qps);
198	return 0;
199}
200
201static struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi)
202{
203	struct ib_pd *pd = gsi->rx_qp->pd;
204	struct ib_qp_init_attr init_attr = {
205		.event_handler = gsi->rx_qp->event_handler,
206		.qp_context = gsi->rx_qp->qp_context,
207		.send_cq = gsi->cq,
208		.recv_cq = gsi->rx_qp->recv_cq,
209		.cap = {
210			.max_send_wr = gsi->cap.max_send_wr,
211			.max_send_sge = gsi->cap.max_send_sge,
212			.max_inline_data = gsi->cap.max_inline_data,
213		},
214		.qp_type = IB_QPT_UD,
215		.create_flags = MLX5_IB_QP_CREATE_SQPN_QP1,
216	};
217
218	return ib_create_qp(pd, &init_attr);
219}
220
221static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,
222			 u16 pkey_index)
223{
224	struct mlx5_ib_dev *dev = to_mdev(qp->device);
225	struct ib_qp_attr attr;
226	int mask;
227	int ret;
228
229	mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY | IB_QP_PORT;
230	attr.qp_state = IB_QPS_INIT;
231	attr.pkey_index = pkey_index;
232	attr.qkey = IB_QP1_QKEY;
233	attr.port_num = gsi->port_num;
234	ret = ib_modify_qp(qp, &attr, mask);
235	if (ret) {
236		mlx5_ib_err(dev, "could not change QP%d state to INIT: %d\n",
237			    qp->qp_num, ret);
238		return ret;
239	}
240
241	attr.qp_state = IB_QPS_RTR;
242	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
243	if (ret) {
244		mlx5_ib_err(dev, "could not change QP%d state to RTR: %d\n",
245			    qp->qp_num, ret);
246		return ret;
247	}
248
249	attr.qp_state = IB_QPS_RTS;
250	attr.sq_psn = 0;
251	ret = ib_modify_qp(qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN);
252	if (ret) {
253		mlx5_ib_err(dev, "could not change QP%d state to RTS: %d\n",
254			    qp->qp_num, ret);
255		return ret;
256	}
257
258	return 0;
259}
260
261static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)
262{
263	struct ib_device *device = gsi->rx_qp->device;
264	struct mlx5_ib_dev *dev = to_mdev(device);
265	int pkey_index = qp_index;
266	struct mlx5_ib_qp *mqp;
267	struct ib_qp *qp;
268	unsigned long flags;
269	u16 pkey;
270	int ret;
271
272	if (MLX5_CAP_GEN(dev->mdev,  port_type) != MLX5_CAP_PORT_TYPE_IB)
273		pkey_index = 0;
274
275	ret = ib_query_pkey(device, gsi->port_num, pkey_index, &pkey);
276	if (ret) {
277		mlx5_ib_warn(dev, "unable to read P_Key at port %d, index %d\n",
278			     gsi->port_num, qp_index);
279		return;
280	}
281
282	if (!pkey) {
283		mlx5_ib_dbg(dev, "invalid P_Key at port %d, index %d.  Skipping.\n",
284			    gsi->port_num, qp_index);
285		return;
286	}
287
288	spin_lock_irqsave(&gsi->lock, flags);
289	qp = gsi->tx_qps[qp_index];
290	spin_unlock_irqrestore(&gsi->lock, flags);
291	if (qp) {
292		mlx5_ib_dbg(dev, "already existing GSI TX QP at port %d, index %d. Skipping\n",
293			    gsi->port_num, qp_index);
294		return;
295	}
296
297	qp = create_gsi_ud_qp(gsi);
298	if (IS_ERR(qp)) {
299		mlx5_ib_warn(dev,
300			     "unable to create hardware UD QP for GSI: %pe\n",
301			     qp);
302		return;
303	}
304
305	mqp = to_mqp(qp);
306	if (dev->lag_active)
307		mqp->gsi_lag_port = qp_index + 1;
308	ret = modify_to_rts(gsi, qp, pkey_index);
309	if (ret)
310		goto err_destroy_qp;
311
312	spin_lock_irqsave(&gsi->lock, flags);
313	WARN_ON_ONCE(gsi->tx_qps[qp_index]);
314	gsi->tx_qps[qp_index] = qp;
315	spin_unlock_irqrestore(&gsi->lock, flags);
316
317	return;
318
319err_destroy_qp:
320	WARN_ON_ONCE(qp);
321}
322
323int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
324			  int attr_mask)
325{
326	struct mlx5_ib_dev *dev = to_mdev(qp->device);
327	struct mlx5_ib_qp *mqp = to_mqp(qp);
328	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
329	u16 qp_index;
330	int ret;
331
332	mlx5_ib_dbg(dev, "modifying GSI QP to state %d\n", attr->qp_state);
333
334	ret = ib_modify_qp(gsi->rx_qp, attr, attr_mask);
335	if (ret) {
336		mlx5_ib_warn(dev, "unable to modify GSI rx QP: %d\n", ret);
337		return ret;
338	}
339
340	if (to_mqp(gsi->rx_qp)->state != IB_QPS_RTS)
341		return 0;
342
343	for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index)
344		setup_qp(gsi, qp_index);
345	return 0;
346}
347
348int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
349			 int qp_attr_mask,
350			 struct ib_qp_init_attr *qp_init_attr)
351{
352	struct mlx5_ib_qp *mqp = to_mqp(qp);
353	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
354	int ret;
355
356	ret = ib_query_qp(gsi->rx_qp, qp_attr, qp_attr_mask, qp_init_attr);
357	qp_init_attr->cap = gsi->cap;
358	return ret;
359}
360
361/* Call with gsi->lock locked */
362static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_qp *mqp,
363				      struct ib_ud_wr *wr, struct ib_wc *wc)
364{
365	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
366	struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device);
367	struct mlx5_ib_gsi_wr *gsi_wr;
368
369	if (gsi->outstanding_pi == gsi->outstanding_ci + gsi->cap.max_send_wr) {
370		mlx5_ib_warn(dev, "no available GSI work request.\n");
371		return -ENOMEM;
372	}
373
374	gsi_wr = &gsi->outstanding_wrs[gsi->outstanding_pi %
375				       gsi->cap.max_send_wr];
376	gsi->outstanding_pi++;
377
378	if (!wc) {
379		memset(&gsi_wr->wc, 0, sizeof(gsi_wr->wc));
380		gsi_wr->wc.pkey_index = wr->pkey_index;
381		gsi_wr->wc.wr_id = wr->wr.wr_id;
382	} else {
383		gsi_wr->wc = *wc;
384		gsi_wr->completed = true;
385	}
386
387	gsi_wr->cqe.done = &handle_single_completion;
388	wr->wr.wr_cqe = &gsi_wr->cqe;
389
390	return 0;
391}
392
393/* Call with gsi->lock locked */
394static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_qp *mqp, struct ib_ud_wr *wr)
395{
396	struct ib_wc wc = {
397		{ .wr_id = wr->wr.wr_id },
398		.status = IB_WC_SUCCESS,
399		.opcode = IB_WC_SEND,
400		.qp = &mqp->ibqp,
401	};
402	int ret;
403
404	ret = mlx5_ib_add_outstanding_wr(mqp, wr, &wc);
405	if (ret)
406		return ret;
407
408	generate_completions(mqp);
409
410	return 0;
411}
412
413/* Call with gsi->lock locked */
414static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr)
415{
416	struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device);
417	struct mlx5_ib_ah *ah = to_mah(wr->ah);
418	int qp_index = wr->pkey_index;
419
420	if (!gsi->num_qps)
421		return gsi->rx_qp;
422
423	if (dev->lag_active && ah->xmit_port)
424		qp_index = ah->xmit_port - 1;
425
426	if (qp_index >= gsi->num_qps)
427		return NULL;
428
429	return gsi->tx_qps[qp_index];
430}
431
432int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,
433			  const struct ib_send_wr **bad_wr)
434{
435	struct mlx5_ib_qp *mqp = to_mqp(qp);
436	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
437	struct ib_qp *tx_qp;
438	unsigned long flags;
439	int ret;
440
441	for (; wr; wr = wr->next) {
442		struct ib_ud_wr cur_wr = *ud_wr(wr);
443
444		cur_wr.wr.next = NULL;
445
446		spin_lock_irqsave(&gsi->lock, flags);
447		tx_qp = get_tx_qp(gsi, &cur_wr);
448		if (!tx_qp) {
449			ret = mlx5_ib_gsi_silent_drop(mqp, &cur_wr);
450			if (ret)
451				goto err;
452			spin_unlock_irqrestore(&gsi->lock, flags);
453			continue;
454		}
455
456		ret = mlx5_ib_add_outstanding_wr(mqp, &cur_wr, NULL);
457		if (ret)
458			goto err;
459
460		ret = ib_post_send(tx_qp, &cur_wr.wr, bad_wr);
461		if (ret) {
462			/* Undo the effect of adding the outstanding wr */
463			gsi->outstanding_pi--;
464			goto err;
465		}
466		spin_unlock_irqrestore(&gsi->lock, flags);
467	}
468
469	return 0;
470
471err:
472	spin_unlock_irqrestore(&gsi->lock, flags);
473	*bad_wr = wr;
474	return ret;
475}
476
477int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,
478			  const struct ib_recv_wr **bad_wr)
479{
480	struct mlx5_ib_qp *mqp = to_mqp(qp);
481	struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
482
483	return ib_post_recv(gsi->rx_qp, wr, bad_wr);
484}
485
486void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi)
487{
488	u16 qp_index;
489
490	for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index)
491		setup_qp(gsi, qp_index);
492}