aso.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  3
  4#include <linux/mlx5/device.h>
  5#include <linux/mlx5/transobj.h>
  6#include "clock.h"
  7#include "aso.h"
  8#include "wq.h"
  9
 10struct mlx5_aso_cq {
 11	/* data path - accessed per cqe */
 12	struct mlx5_cqwq           wq;
 13
 14	/* data path - accessed per napi poll */
 15	struct mlx5_core_cq        mcq;
 16
 17	/* control */
 18	struct mlx5_core_dev      *mdev;
 19	struct mlx5_wq_ctrl        wq_ctrl;
 20} ____cacheline_aligned_in_smp;
 21
 22struct mlx5_aso {
 23	/* data path */
 24	u16                        cc;
 25	u16                        pc;
 26
 27	struct mlx5_wqe_ctrl_seg  *doorbell_cseg;
 28	struct mlx5_aso_cq         cq;
 29
 30	/* read only */
 31	struct mlx5_wq_cyc         wq;
 32	void __iomem              *uar_map;
 33	u32                        sqn;
 34
 35	/* control path */
 36	struct mlx5_wq_ctrl        wq_ctrl;
 37
 38} ____cacheline_aligned_in_smp;
 39
 40static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
 41{
 42	mlx5_wq_destroy(&cq->wq_ctrl);
 43}
 44
 45static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
 46			     void *cqc_data, struct mlx5_aso_cq *cq)
 47{
 48	struct mlx5_core_cq *mcq = &cq->mcq;
 49	struct mlx5_wq_param param;
 50	int err;
 51	u32 i;
 52
 53	param.buf_numa_node = numa_node;
 54	param.db_numa_node = numa_node;
 55
 56	err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
 57	if (err)
 58		return err;
 59
 60	mcq->cqe_sz     = 64;
 61	mcq->set_ci_db  = cq->wq_ctrl.db.db;
 62	mcq->arm_db     = cq->wq_ctrl.db.db + 1;
 63
 64	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
 65		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
 66
 67		cqe->op_own = 0xf1;
 68	}
 69
 70	cq->mdev = mdev;
 71
 72	return 0;
 73}
 74
 75static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data)
 76{
 77	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
 78	struct mlx5_core_dev *mdev = cq->mdev;
 79	struct mlx5_core_cq *mcq = &cq->mcq;
 80	void *in, *cqc;
 81	int inlen, eqn;
 82	int err;
 83
 84	err = mlx5_comp_eqn_get(mdev, 0, &eqn);
 85	if (err)
 86		return err;
 87
 88	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
 89		sizeof(u64) * cq->wq_ctrl.buf.npages;
 90	in = kvzalloc(inlen, GFP_KERNEL);
 91	if (!in)
 92		return -ENOMEM;
 93
 94	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 95
 96	memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));
 97
 98	mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
 99				  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
100
101	MLX5_SET(cqc,   cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
102	MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
103	MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.bfreg.up->index);
104	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
105					    MLX5_ADAPTER_PAGE_SHIFT);
106	MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
107
108	err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
109
110	kvfree(in);
111
112	return err;
113}
114
115static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
116{
117	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
118	mlx5_wq_destroy(&cq->wq_ctrl);
119}
120
121static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
122			      struct mlx5_aso_cq *cq)
123{
124	void *cqc_data;
125	int err;
126
127	cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
128	if (!cqc_data)
129		return -ENOMEM;
130
131	MLX5_SET(cqc, cqc_data, log_cq_size, 1);
132	MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.bfreg.up->index);
133	if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
134		MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);
135
136	err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
137	if (err) {
138		mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
139		goto err_out;
140	}
141
142	err = create_aso_cq(cq, cqc_data);
143	if (err) {
144		mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
145		goto err_free_cq;
146	}
147
148	kvfree(cqc_data);
149	return 0;
150
151err_free_cq:
152	mlx5_aso_free_cq(cq);
153err_out:
154	kvfree(cqc_data);
155	return err;
156}
157
158static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
159			     void *sqc_data, struct mlx5_aso *sq)
160{
161	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
162	struct mlx5_wq_cyc *wq = &sq->wq;
163	struct mlx5_wq_param param;
164	int err;
165
166	sq->uar_map = mdev->priv.bfreg.map;
167
168	param.db_numa_node = numa_node;
169	param.buf_numa_node = numa_node;
170	err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
171	if (err)
172		return err;
173	wq->db = &wq->db[MLX5_SND_DBR];
174
175	return 0;
176}
177
178static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
179			 void *sqc_data, struct mlx5_aso *sq)
180{
181	void *in, *sqc, *wq;
182	int inlen, err;
183	u8 ts_format;
184
185	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
186		sizeof(u64) * sq->wq_ctrl.buf.npages;
187	in = kvzalloc(inlen, GFP_KERNEL);
188	if (!in)
189		return -ENOMEM;
190
191	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
192	wq = MLX5_ADDR_OF(sqc, sqc, wq);
193
194	memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
195	MLX5_SET(sqc,  sqc, cqn, sq->cq.mcq.cqn);
196
197	MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
198	MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
199
200	ts_format = mlx5_is_real_time_sq(mdev) ?
201			MLX5_TIMESTAMP_FORMAT_REAL_TIME :
202			MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
203	MLX5_SET(sqc, sqc, ts_format, ts_format);
204
205	MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
206	MLX5_SET(wq,   wq, uar_page,      mdev->priv.bfreg.index);
207	MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
208					  MLX5_ADAPTER_PAGE_SHIFT);
209	MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);
210
211	mlx5_fill_page_frag_array(&sq->wq_ctrl.buf,
212				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
213
214	err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
215
216	kvfree(in);
217
218	return err;
219}
220
221static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
222{
223	void *in, *sqc;
224	int inlen, err;
225
226	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
227	in = kvzalloc(inlen, GFP_KERNEL);
228	if (!in)
229		return -ENOMEM;
230
231	MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
232	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
233	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
234
235	err = mlx5_core_modify_sq(mdev, sqn, in);
236
237	kvfree(in);
238
239	return err;
240}
241
242static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
243				  void *sqc_data, struct mlx5_aso *sq)
244{
245	int err;
246
247	err = create_aso_sq(mdev, pdn, sqc_data, sq);
248	if (err)
249		return err;
250
251	err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
252	if (err)
253		mlx5_core_destroy_sq(mdev, sq->sqn);
254
255	return err;
256}
257
258static void mlx5_aso_free_sq(struct mlx5_aso *sq)
259{
260	mlx5_wq_destroy(&sq->wq_ctrl);
261}
262
263static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
264{
265	mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
266	mlx5_aso_free_sq(sq);
267}
268
269static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
270			      u32 pdn, struct mlx5_aso *sq)
271{
272	void *sqc_data, *wq;
273	int err;
274
275	sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
276	if (!sqc_data)
277		return -ENOMEM;
278
279	wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
280	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
281	MLX5_SET(wq, wq, pd, pdn);
282	MLX5_SET(wq, wq, log_wq_sz, 1);
283
284	err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
285	if (err) {
286		mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
287		goto err_out;
288	}
289
290	err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
291	if (err) {
292		mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
293		goto err_free_asosq;
294	}
295
296	mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);
297
298	kvfree(sqc_data);
299	return 0;
300
301err_free_asosq:
302	mlx5_aso_free_sq(sq);
303err_out:
304	kvfree(sqc_data);
305	return err;
306}
307
308struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn)
309{
310	int numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
311	struct mlx5_aso *aso;
312	int err;
313
314	aso = kzalloc(sizeof(*aso), GFP_KERNEL);
315	if (!aso)
316		return ERR_PTR(-ENOMEM);
317
318	err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
319	if (err)
320		goto err_cq;
321
322	err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
323	if (err)
324		goto err_sq;
325
326	return aso;
327
328err_sq:
329	mlx5_aso_destroy_cq(&aso->cq);
330err_cq:
331	kfree(aso);
332	return ERR_PTR(err);
333}
334
335void mlx5_aso_destroy(struct mlx5_aso *aso)
336{
337	mlx5_aso_destroy_sq(aso);
338	mlx5_aso_destroy_cq(&aso->cq);
339	kfree(aso);
340}
341
342void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
343			struct mlx5_aso_wqe *aso_wqe,
344			u32 obj_id, u32 opc_mode)
345{
346	struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;
347
348	cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) |
349					     (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
350					     MLX5_OPCODE_ACCESS_ASO);
351	cseg->qpn_ds     = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
352	cseg->fm_ce_se   = MLX5_WQE_CTRL_CQ_UPDATE;
353	cseg->general_id = cpu_to_be32(obj_id);
354}
355
356struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso)
357{
358	struct mlx5_aso_wqe *wqe;
359	u16 pi;
360
361	pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
362	wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
363	memset(wqe, 0, sizeof(*wqe));
364	return wqe;
365}
366
367void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
368		       struct mlx5_wqe_ctrl_seg *doorbell_cseg)
369{
370	doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
371	/* ensure wqe is visible to device before updating doorbell record */
372	dma_wmb();
373
374	if (with_data)
375		aso->pc += MLX5_ASO_WQEBBS_DATA;
376	else
377		aso->pc += MLX5_ASO_WQEBBS;
378	*aso->wq.db = cpu_to_be32(aso->pc);
379
380	/* ensure doorbell record is visible to device before ringing the
381	 * doorbell
382	 */
383	wmb();
384
385	mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map);
386
387	/* Ensure doorbell is written on uar_page before poll_cq */
388	WRITE_ONCE(doorbell_cseg, NULL);
389}
390
391int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
392{
393	struct mlx5_aso_cq *cq = &aso->cq;
394	struct mlx5_cqe64 *cqe;
395
396	cqe = mlx5_cqwq_get_cqe(&cq->wq);
397	if (!cqe)
398		return -ETIMEDOUT;
399
400	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
401	 * otherwise a cq overrun may occur
402	 */
403	mlx5_cqwq_pop(&cq->wq);
404
405	if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
406		struct mlx5_err_cqe *err_cqe;
407
408		mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
409			      get_cqe_opcode(cqe));
410
411		err_cqe = (struct mlx5_err_cqe *)cqe;
412		mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
413			      err_cqe->vendor_err_synd);
414		mlx5_core_err(cq->mdev, "syndrome=%x\n",
415			      err_cqe->syndrome);
416		print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
417			       16, 1, err_cqe,
418			       sizeof(*err_cqe), false);
419	}
420
421	mlx5_cqwq_update_db_record(&cq->wq);
422
423	/* ensure cq space is freed before enabling more cqes */
424	wmb();
425
426	if (with_data)
427		aso->cc += MLX5_ASO_WQEBBS_DATA;
428	else
429		aso->cc += MLX5_ASO_WQEBBS;
430
431	return 0;
432}