bng_dev.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) 2025 Broadcom.
  3
  4#include <linux/module.h>
  5#include <linux/pci.h>
  6#include <linux/auxiliary_bus.h>
  7
  8#include <rdma/ib_verbs.h>
  9
 10#include "bng_res.h"
 11#include "bng_sp.h"
 12#include "bng_fw.h"
 13#include "bnge.h"
 14#include "bnge_auxr.h"
 15#include "bng_re.h"
 16#include "bnge_hwrm.h"
 17#include "bng_debugfs.h"
 18
 19MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@broadcom.com>");
 20MODULE_DESCRIPTION(BNG_RE_DESC);
 21MODULE_LICENSE("Dual BSD/GPL");
 22
 23static struct bng_re_dev *bng_re_dev_add(struct auxiliary_device *adev,
 24					 struct bnge_auxr_dev *aux_dev)
 25{
 26	struct bng_re_dev *rdev;
 27
 28	/* Allocate bng_re_dev instance */
 29	rdev = ib_alloc_device(bng_re_dev, ibdev);
 30	if (!rdev) {
 31		pr_err("%s: bng_re_dev allocation failure!", KBUILD_MODNAME);
 32		return NULL;
 33	}
 34
 35	/* Assign auxiliary device specific data */
 36	rdev->netdev = aux_dev->net;
 37	rdev->aux_dev = aux_dev;
 38	rdev->adev = adev;
 39	rdev->fn_id = rdev->aux_dev->pdev->devfn;
 40
 41	return rdev;
 42}
 43
 44
 45static int bng_re_register_netdev(struct bng_re_dev *rdev)
 46{
 47	struct bnge_auxr_dev *aux_dev;
 48
 49	aux_dev = rdev->aux_dev;
 50	return bnge_register_dev(aux_dev, rdev->adev);
 51}
 52
 53static void bng_re_destroy_chip_ctx(struct bng_re_dev *rdev)
 54{
 55	struct bng_re_chip_ctx *chip_ctx;
 56
 57	kfree(rdev->dev_attr);
 58	rdev->dev_attr = NULL;
 59
 60	chip_ctx = rdev->chip_ctx;
 61	rdev->chip_ctx = NULL;
 62	rdev->rcfw.res = NULL;
 63	rdev->bng_res.cctx = NULL;
 64	rdev->bng_res.pdev = NULL;
 65	kfree(chip_ctx);
 66}
 67
 68static int bng_re_setup_chip_ctx(struct bng_re_dev *rdev)
 69{
 70	struct bng_re_chip_ctx *chip_ctx;
 71	struct bnge_auxr_dev *aux_dev;
 72	int rc = -ENOMEM;
 73
 74	aux_dev = rdev->aux_dev;
 75	rdev->bng_res.pdev = aux_dev->pdev;
 76	rdev->rcfw.res = &rdev->bng_res;
 77	chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
 78	if (!chip_ctx)
 79		return -ENOMEM;
 80	chip_ctx->chip_num = aux_dev->chip_num;
 81	chip_ctx->hw_stats_size = aux_dev->hw_ring_stats_size;
 82
 83	rdev->chip_ctx = chip_ctx;
 84	rdev->bng_res.cctx = rdev->chip_ctx;
 85	rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL);
 86	if (!rdev->dev_attr)
 87		goto free_chip_ctx;
 88	rdev->bng_res.dattr = rdev->dev_attr;
 89
 90	return 0;
 91free_chip_ctx:
 92	kfree(rdev->chip_ctx);
 93	rdev->chip_ctx = NULL;
 94	return rc;
 95}
 96
 97static void bng_re_init_hwrm_hdr(struct input *hdr, u16 opcd)
 98{
 99	hdr->req_type = cpu_to_le16(opcd);
100	hdr->cmpl_ring = cpu_to_le16(-1);
101	hdr->target_id = cpu_to_le16(-1);
102}
103
104static void bng_re_fill_fw_msg(struct bnge_fw_msg *fw_msg, void *msg,
105			       int msg_len, void *resp, int resp_max_len,
106			       int timeout)
107{
108	fw_msg->msg = msg;
109	fw_msg->msg_len = msg_len;
110	fw_msg->resp = resp;
111	fw_msg->resp_max_len = resp_max_len;
112	fw_msg->timeout = timeout;
113}
114
115static int bng_re_net_ring_free(struct bng_re_dev *rdev,
116				u16 fw_ring_id, int type)
117{
118	struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
119	struct hwrm_ring_free_input req = {};
120	struct hwrm_ring_free_output resp;
121	struct bnge_fw_msg fw_msg = {};
122	int rc = -EINVAL;
123
124	bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_FREE);
125	req.ring_type = type;
126	req.ring_id = cpu_to_le16(fw_ring_id);
127	bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
128			    sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
129	rc = bnge_send_msg(aux_dev, &fw_msg);
130	if (rc)
131		ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
132			  req.ring_id, rc);
133	return rc;
134}
135
136static int bng_re_net_ring_alloc(struct bng_re_dev *rdev,
137				 struct bng_re_ring_attr *ring_attr,
138				 u16 *fw_ring_id)
139{
140	struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
141	struct hwrm_ring_alloc_input req = {};
142	struct hwrm_ring_alloc_output resp;
143	struct bnge_fw_msg fw_msg = {};
144	int rc;
145
146	bng_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC);
147	req.enables = 0;
148	req.page_tbl_addr =  cpu_to_le64(ring_attr->dma_arr[0]);
149	if (ring_attr->pages > 1) {
150		/* Page size is in log2 units */
151		req.page_size = BNGE_PAGE_SHIFT;
152		req.page_tbl_depth = 1;
153	}
154	req.fbo = 0;
155	/* Association of ring index with doorbell index and MSIX number */
156	req.logical_id = cpu_to_le16(ring_attr->lrid);
157	req.length = cpu_to_le32(ring_attr->depth + 1);
158	req.ring_type = ring_attr->type;
159	req.int_mode = ring_attr->mode;
160	bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
161			   sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
162	rc = bnge_send_msg(aux_dev, &fw_msg);
163	if (!rc)
164		*fw_ring_id = le16_to_cpu(resp.ring_id);
165
166	return rc;
167}
168
169static int bng_re_stats_ctx_free(struct bng_re_dev *rdev)
170{
171	struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
172	struct hwrm_stat_ctx_free_input req = {};
173	struct hwrm_stat_ctx_free_output resp = {};
174	struct bnge_fw_msg fw_msg = {};
175	int rc;
176
177	bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_FREE);
178	req.stat_ctx_id = cpu_to_le32(rdev->stats_ctx.fw_id);
179	bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
180			   sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
181	rc = bnge_send_msg(aux_dev, &fw_msg);
182	if (rc)
183		ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x",
184			  rc);
185
186	return rc;
187}
188
189static int bng_re_stats_ctx_alloc(struct bng_re_dev *rdev)
190{
191	struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
192	struct bng_re_stats *stats = &rdev->stats_ctx;
193	struct hwrm_stat_ctx_alloc_output resp = {};
194	struct hwrm_stat_ctx_alloc_input req = {};
195	struct bnge_fw_msg fw_msg = {};
196	int rc;
197
198	stats->fw_id = BNGE_INVALID_STATS_CTX_ID;
199
200	bng_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_ALLOC);
201	req.update_period_ms = cpu_to_le32(1000);
202	req.stats_dma_addr = cpu_to_le64(stats->dma_map);
203	req.stats_dma_length = cpu_to_le16(rdev->chip_ctx->hw_stats_size);
204	req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;
205	bng_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
206			   sizeof(resp), BNGE_DFLT_HWRM_CMD_TIMEOUT);
207	rc = bnge_send_msg(aux_dev, &fw_msg);
208	if (!rc)
209		stats->fw_id = le32_to_cpu(resp.stat_ctx_id);
210	return rc;
211}
212
213static int bng_re_query_hwrm_version(struct bng_re_dev *rdev)
214{
215	struct bnge_auxr_dev *aux_dev = rdev->aux_dev;
216	struct hwrm_ver_get_output ver_get_resp = {};
217	struct hwrm_ver_get_input ver_get_req = {};
218	struct bng_re_chip_ctx *cctx;
219	struct bnge_fw_msg fw_msg = {};
220	int rc;
221
222	bng_re_init_hwrm_hdr((void *)&ver_get_req, HWRM_VER_GET);
223	ver_get_req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
224	ver_get_req.hwrm_intf_min = HWRM_VERSION_MINOR;
225	ver_get_req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
226	bng_re_fill_fw_msg(&fw_msg, (void *)&ver_get_req, sizeof(ver_get_req),
227			    (void *)&ver_get_resp, sizeof(ver_get_resp),
228			    BNGE_DFLT_HWRM_CMD_TIMEOUT);
229	rc = bnge_send_msg(aux_dev, &fw_msg);
230	if (rc) {
231		ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
232			  rc);
233		return rc;
234	}
235
236	cctx = rdev->chip_ctx;
237	cctx->hwrm_intf_ver =
238		(u64)le16_to_cpu(ver_get_resp.hwrm_intf_major) << 48 |
239		(u64)le16_to_cpu(ver_get_resp.hwrm_intf_minor) << 32 |
240		(u64)le16_to_cpu(ver_get_resp.hwrm_intf_build) << 16 |
241		le16_to_cpu(ver_get_resp.hwrm_intf_patch);
242
243	cctx->hwrm_cmd_max_timeout = le16_to_cpu(ver_get_resp.max_req_timeout);
244
245	if (!cctx->hwrm_cmd_max_timeout)
246		cctx->hwrm_cmd_max_timeout = BNG_ROCE_FW_MAX_TIMEOUT;
247
248	return 0;
249}
250
251static void bng_re_dev_uninit(struct bng_re_dev *rdev)
252{
253	int rc;
254	bng_re_debugfs_rem_pdev(rdev);
255
256	if (test_and_clear_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
257		rc = bng_re_deinit_rcfw(&rdev->rcfw);
258		if (rc)
259			ibdev_warn(&rdev->ibdev,
260				   "Failed to deinitialize RCFW: %#x", rc);
261		bng_re_stats_ctx_free(rdev);
262		bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx);
263		bng_re_disable_rcfw_channel(&rdev->rcfw);
264		bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id,
265			     RING_ALLOC_REQ_RING_TYPE_NQ);
266		bng_re_free_rcfw_channel(&rdev->rcfw);
267	}
268
269	kfree(rdev->nqr);
270	rdev->nqr = NULL;
271	bng_re_destroy_chip_ctx(rdev);
272	if (test_and_clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
273		bnge_unregister_dev(rdev->aux_dev);
274}
275
276static int bng_re_dev_init(struct bng_re_dev *rdev)
277{
278	struct bng_re_ring_attr rattr = {};
279	struct bng_re_creq_ctx *creq;
280	u32 db_offt;
281	int vid;
282	u8 type;
283	int rc;
284
285	/* Registered a new RoCE device instance to netdev */
286	rc = bng_re_register_netdev(rdev);
287	if (rc) {
288		ibdev_err(&rdev->ibdev,
289				"Failed to register with netedev: %#x\n", rc);
290		goto reg_netdev_fail;
291	}
292
293	set_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
294
295	if (rdev->aux_dev->auxr_info->msix_requested < BNG_RE_MIN_MSIX) {
296		ibdev_err(&rdev->ibdev,
297			  "RoCE requires minimum 2 MSI-X vectors, but only %d reserved\n",
298			  rdev->aux_dev->auxr_info->msix_requested);
299		rc = -EINVAL;
300		goto msix_ctx_fail;
301	}
302	ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
303		  rdev->aux_dev->auxr_info->msix_requested);
304
305	rc = bng_re_setup_chip_ctx(rdev);
306	if (rc) {
307		ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
308		goto msix_ctx_fail;
309	}
310
311	rc = bng_re_query_hwrm_version(rdev);
312	if (rc)
313		goto destroy_chip_ctx;
314
315	rc = bng_re_alloc_fw_channel(&rdev->bng_res, &rdev->rcfw);
316	if (rc) {
317		ibdev_err(&rdev->ibdev,
318			  "Failed to allocate RCFW Channel: %#x\n", rc);
319		goto destroy_chip_ctx;
320	}
321
322	/* Allocate nq record memory */
323	rdev->nqr = kzalloc(sizeof(*rdev->nqr), GFP_KERNEL);
324	if (!rdev->nqr) {
325		rc = -ENOMEM;
326		goto nq_alloc_fail;
327	}
328
329	rdev->nqr->num_msix = rdev->aux_dev->auxr_info->msix_requested;
330	memcpy(rdev->nqr->msix_entries, rdev->aux_dev->msix_info,
331	       sizeof(struct bnge_msix_info) * rdev->nqr->num_msix);
332
333	type = RING_ALLOC_REQ_RING_TYPE_NQ;
334	creq = &rdev->rcfw.creq;
335	rattr.dma_arr = creq->hwq.pbl[BNG_PBL_LVL_0].pg_map_arr;
336	rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
337	rattr.type = type;
338	rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
339	rattr.depth = BNG_FW_CREQE_MAX_CNT - 1;
340	rattr.lrid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].ring_idx;
341	rc = bng_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
342	if (rc) {
343		ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
344		goto free_rcfw;
345	}
346	db_offt = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].db_offset;
347	vid = rdev->nqr->msix_entries[BNG_RE_CREQ_NQ_IDX].vector;
348
349	rc = bng_re_enable_fw_channel(&rdev->rcfw,
350					vid, db_offt);
351	if (rc) {
352		ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n",
353			  rc);
354		goto free_ring;
355	}
356
357	rc = bng_re_get_dev_attr(&rdev->rcfw);
358	if (rc)
359		goto disable_rcfw;
360
361	bng_re_debugfs_add_pdev(rdev);
362	rc = bng_re_alloc_stats_ctx_mem(rdev->bng_res.pdev, rdev->chip_ctx,
363					&rdev->stats_ctx);
364	if (rc) {
365		ibdev_err(&rdev->ibdev,
366			  "Failed to allocate stats context: %#x\n", rc);
367		goto disable_rcfw;
368	}
369
370	rc = bng_re_stats_ctx_alloc(rdev);
371	if (rc) {
372		ibdev_err(&rdev->ibdev,
373			  "Failed to allocate QPLIB context: %#x\n", rc);
374		goto free_stats_ctx;
375	}
376
377	rc = bng_re_init_rcfw(&rdev->rcfw, &rdev->stats_ctx);
378	if (rc) {
379		ibdev_err(&rdev->ibdev,
380			  "Failed to initialize RCFW: %#x\n", rc);
381		goto free_sctx;
382	}
383	set_bit(BNG_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
384
385	return 0;
386free_sctx:
387	bng_re_stats_ctx_free(rdev);
388free_stats_ctx:
389	bng_re_free_stats_ctx_mem(rdev->bng_res.pdev, &rdev->stats_ctx);
390disable_rcfw:
391	bng_re_disable_rcfw_channel(&rdev->rcfw);
392free_ring:
393	bng_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
394free_rcfw:
395	kfree(rdev->nqr);
396nq_alloc_fail:
397	bng_re_free_rcfw_channel(&rdev->rcfw);
398destroy_chip_ctx:
399	bng_re_destroy_chip_ctx(rdev);
400msix_ctx_fail:
401	bnge_unregister_dev(rdev->aux_dev);
402	clear_bit(BNG_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
403reg_netdev_fail:
404	return rc;
405}
406
407static int bng_re_add_device(struct auxiliary_device *adev)
408{
409	struct bnge_auxr_priv *auxr_priv =
410		container_of(adev, struct bnge_auxr_priv, aux_dev);
411	struct bng_re_en_dev_info *dev_info;
412	struct bng_re_dev *rdev;
413	int rc;
414
415	dev_info = auxiliary_get_drvdata(adev);
416
417	rdev = bng_re_dev_add(adev, auxr_priv->auxr_dev);
418	if (!rdev) {
419		rc = -ENOMEM;
420		goto exit;
421	}
422
423	dev_info->rdev = rdev;
424
425	rc = bng_re_dev_init(rdev);
426	if (rc)
427		goto re_dev_dealloc;
428
429	return 0;
430
431re_dev_dealloc:
432	ib_dealloc_device(&rdev->ibdev);
433exit:
434	return rc;
435}
436
437
438static void bng_re_remove_device(struct bng_re_dev *rdev,
439				 struct auxiliary_device *aux_dev)
440{
441	bng_re_dev_uninit(rdev);
442	ib_dealloc_device(&rdev->ibdev);
443}
444
445
446static int bng_re_probe(struct auxiliary_device *adev,
447			const struct auxiliary_device_id *id)
448{
449	struct bnge_auxr_priv *aux_priv =
450		container_of(adev, struct bnge_auxr_priv, aux_dev);
451	struct bng_re_en_dev_info *en_info;
452	int rc;
453
454	en_info = kzalloc(sizeof(*en_info), GFP_KERNEL);
455	if (!en_info)
456		return -ENOMEM;
457
458	en_info->auxr_dev = aux_priv->auxr_dev;
459
460	auxiliary_set_drvdata(adev, en_info);
461
462	rc = bng_re_add_device(adev);
463	if (rc)
464		kfree(en_info);
465
466	return rc;
467}
468
469static void bng_re_remove(struct auxiliary_device *adev)
470{
471	struct bng_re_en_dev_info *dev_info = auxiliary_get_drvdata(adev);
472	struct bng_re_dev *rdev;
473
474	rdev = dev_info->rdev;
475
476	bng_re_remove_device(rdev, adev);
477	kfree(dev_info);
478}
479
480static const struct auxiliary_device_id bng_re_id_table[] = {
481	{ .name = BNG_RE_ADEV_NAME ".rdma", },
482	{},
483};
484
485MODULE_DEVICE_TABLE(auxiliary, bng_re_id_table);
486
487static struct auxiliary_driver bng_re_driver = {
488	.name = "rdma",
489	.probe = bng_re_probe,
490	.remove = bng_re_remove,
491	.id_table = bng_re_id_table,
492};
493
494static int __init bng_re_mod_init(void)
495{
496	int rc;
497
498
499	bng_re_register_debugfs();
500
501	rc = auxiliary_driver_register(&bng_re_driver);
502	if (rc) {
503		pr_err("%s: Failed to register auxiliary driver\n",
504		       KBUILD_MODNAME);
505		goto unreg_debugfs;
506	}
507	return 0;
508unreg_debugfs:
509	bng_re_unregister_debugfs();
510	return rc;
511}
512
513static void __exit bng_re_mod_exit(void)
514{
515	auxiliary_driver_unregister(&bng_re_driver);
516	bng_re_unregister_debugfs();
517}
518
519module_init(bng_re_mod_init);
520module_exit(bng_re_mod_exit);