bng_res.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) 2025 Broadcom.
  3
  4#include <linux/pci.h>
  5#include <linux/vmalloc.h>
  6#include <rdma/ib_umem.h>
  7
  8#include <linux/bnxt/hsi.h>
  9#include "bng_res.h"
 10#include "roce_hsi.h"
 11
 12/* Stats */
 13void bng_re_free_stats_ctx_mem(struct pci_dev *pdev,
 14			       struct bng_re_stats *stats)
 15{
 16	if (stats->dma) {
 17		dma_free_coherent(&pdev->dev, stats->size,
 18				  stats->dma, stats->dma_map);
 19	}
 20	memset(stats, 0, sizeof(*stats));
 21	stats->fw_id = -1;
 22}
 23
 24int bng_re_alloc_stats_ctx_mem(struct pci_dev *pdev,
 25			       struct bng_re_chip_ctx *cctx,
 26			       struct bng_re_stats *stats)
 27{
 28	memset(stats, 0, sizeof(*stats));
 29	stats->fw_id = -1;
 30	stats->size = cctx->hw_stats_size;
 31	stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
 32					&stats->dma_map, GFP_KERNEL);
 33	if (!stats->dma)
 34		return -ENOMEM;
 35
 36	return 0;
 37}
 38
 39static void bng_free_pbl(struct bng_re_res  *res, struct bng_re_pbl *pbl)
 40{
 41	struct pci_dev *pdev = res->pdev;
 42	int i;
 43
 44	for (i = 0; i < pbl->pg_count; i++) {
 45		if (pbl->pg_arr[i])
 46			dma_free_coherent(&pdev->dev, pbl->pg_size,
 47					  (void *)((unsigned long)
 48					     pbl->pg_arr[i] &
 49						PAGE_MASK),
 50					  pbl->pg_map_arr[i]);
 51		else
 52			dev_warn(&pdev->dev,
 53					"PBL free pg_arr[%d] empty?!\n", i);
 54		pbl->pg_arr[i] = NULL;
 55	}
 56
 57	vfree(pbl->pg_arr);
 58	pbl->pg_arr = NULL;
 59	vfree(pbl->pg_map_arr);
 60	pbl->pg_map_arr = NULL;
 61	pbl->pg_count = 0;
 62	pbl->pg_size = 0;
 63}
 64
 65static int bng_alloc_pbl(struct bng_re_res  *res,
 66			 struct bng_re_pbl *pbl,
 67			 struct bng_re_sg_info *sginfo)
 68{
 69	struct pci_dev *pdev = res->pdev;
 70	u32 pages;
 71	int i;
 72
 73	if (sginfo->nopte)
 74		return 0;
 75	pages = sginfo->npages;
 76
 77	/* page ptr arrays */
 78	pbl->pg_arr = vmalloc_array(pages, sizeof(void *));
 79	if (!pbl->pg_arr)
 80		return -ENOMEM;
 81
 82	pbl->pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t));
 83	if (!pbl->pg_map_arr) {
 84		vfree(pbl->pg_arr);
 85		pbl->pg_arr = NULL;
 86		return -ENOMEM;
 87	}
 88	pbl->pg_count = 0;
 89	pbl->pg_size = sginfo->pgsize;
 90
 91	for (i = 0; i < pages; i++) {
 92		pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
 93				pbl->pg_size,
 94				&pbl->pg_map_arr[i],
 95				GFP_KERNEL);
 96		if (!pbl->pg_arr[i])
 97			goto fail;
 98		pbl->pg_count++;
 99	}
100
101	return 0;
102fail:
103	bng_free_pbl(res, pbl);
104	return -ENOMEM;
105}
106
107void bng_re_free_hwq(struct bng_re_res *res,
108		     struct bng_re_hwq *hwq)
109{
110	int i;
111
112	if (!hwq->max_elements)
113		return;
114	if (hwq->level >= BNG_PBL_LVL_MAX)
115		return;
116
117	for (i = 0; i < hwq->level + 1; i++)
118		bng_free_pbl(res, &hwq->pbl[i]);
119
120	hwq->level = BNG_PBL_LVL_MAX;
121	hwq->max_elements = 0;
122	hwq->element_size = 0;
123	hwq->prod = 0;
124	hwq->cons = 0;
125}
126
127/* All HWQs are power of 2 in size */
128int bng_re_alloc_init_hwq(struct bng_re_hwq *hwq,
129			  struct bng_re_hwq_attr *hwq_attr)
130{
131	u32 npages, pg_size;
132	struct bng_re_sg_info sginfo = {};
133	u32 depth, stride, npbl, npde;
134	dma_addr_t *src_phys_ptr, **dst_virt_ptr;
135	struct bng_re_res *res;
136	struct pci_dev *pdev;
137	int i, rc, lvl;
138
139	res = hwq_attr->res;
140	pdev = res->pdev;
141	pg_size = hwq_attr->sginfo->pgsize;
142	hwq->level = BNG_PBL_LVL_MAX;
143
144	depth = roundup_pow_of_two(hwq_attr->depth);
145	stride = roundup_pow_of_two(hwq_attr->stride);
146
147	npages = (depth * stride) / pg_size;
148	if ((depth * stride) % pg_size)
149		npages++;
150	if (!npages)
151		return -EINVAL;
152	hwq_attr->sginfo->npages = npages;
153
154	if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
155		/* This request is Level 0, map PTE */
156		rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_0], hwq_attr->sginfo);
157		if (rc)
158			goto fail;
159		hwq->level = BNG_PBL_LVL_0;
160		goto done;
161	}
162
163	if (npages >= MAX_PBL_LVL_0_PGS) {
164		if (npages > MAX_PBL_LVL_1_PGS) {
165			u32 flag = PTU_PTE_VALID;
166			/* 2 levels of indirection */
167			npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
168			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
169				npbl++;
170			npde = npbl >> MAX_PDL_LVL_SHIFT;
171			if (npbl % BIT(MAX_PDL_LVL_SHIFT))
172				npde++;
173			/* Alloc PDE pages */
174			sginfo.pgsize = npde * pg_size;
175			sginfo.npages = 1;
176			rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_0], &sginfo);
177			if (rc)
178				goto fail;
179
180			/* Alloc PBL pages */
181			sginfo.npages = npbl;
182			sginfo.pgsize = PAGE_SIZE;
183			rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_1], &sginfo);
184			if (rc)
185				goto fail;
186			/* Fill PDL with PBL page pointers */
187			dst_virt_ptr =
188				(dma_addr_t **)hwq->pbl[BNG_PBL_LVL_0].pg_arr;
189			src_phys_ptr = hwq->pbl[BNG_PBL_LVL_1].pg_map_arr;
190			for (i = 0; i < hwq->pbl[BNG_PBL_LVL_1].pg_count; i++)
191				dst_virt_ptr[0][i] = src_phys_ptr[i] | flag;
192
193			/* Alloc or init PTEs */
194			rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_2],
195					 hwq_attr->sginfo);
196			if (rc)
197				goto fail;
198			hwq->level = BNG_PBL_LVL_2;
199			if (hwq_attr->sginfo->nopte)
200				goto done;
201			/* Fill PBLs with PTE pointers */
202			dst_virt_ptr =
203				(dma_addr_t **)hwq->pbl[BNG_PBL_LVL_1].pg_arr;
204			src_phys_ptr = hwq->pbl[BNG_PBL_LVL_2].pg_map_arr;
205			for (i = 0; i < hwq->pbl[BNG_PBL_LVL_2].pg_count; i++) {
206				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
207					src_phys_ptr[i] | PTU_PTE_VALID;
208			}
209			if (hwq_attr->type == BNG_HWQ_TYPE_QUEUE) {
210				/* Find the last pg of the size */
211				i = hwq->pbl[BNG_PBL_LVL_2].pg_count;
212				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
213								  PTU_PTE_LAST;
214				if (i > 1)
215					dst_virt_ptr[PTR_PG(i - 2)]
216						    [PTR_IDX(i - 2)] |=
217						    PTU_PTE_NEXT_TO_LAST;
218			}
219		} else { /* pages < 512 npbl = 1, npde = 0 */
220			u32 flag = PTU_PTE_VALID;
221
222			/* 1 level of indirection */
223			npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
224			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
225				npbl++;
226			sginfo.npages = npbl;
227			sginfo.pgsize = PAGE_SIZE;
228			/* Alloc PBL page */
229			rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_0], &sginfo);
230			if (rc)
231				goto fail;
232			/* Alloc or init  PTEs */
233			rc = bng_alloc_pbl(res, &hwq->pbl[BNG_PBL_LVL_1],
234					 hwq_attr->sginfo);
235			if (rc)
236				goto fail;
237			hwq->level = BNG_PBL_LVL_1;
238			if (hwq_attr->sginfo->nopte)
239				goto done;
240			/* Fill PBL with PTE pointers */
241			dst_virt_ptr =
242				(dma_addr_t **)hwq->pbl[BNG_PBL_LVL_0].pg_arr;
243			src_phys_ptr = hwq->pbl[BNG_PBL_LVL_1].pg_map_arr;
244			for (i = 0; i < hwq->pbl[BNG_PBL_LVL_1].pg_count; i++)
245				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
246					src_phys_ptr[i] | flag;
247			if (hwq_attr->type == BNG_HWQ_TYPE_QUEUE) {
248				/* Find the last pg of the size */
249				i = hwq->pbl[BNG_PBL_LVL_1].pg_count;
250				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
251								  PTU_PTE_LAST;
252				if (i > 1)
253					dst_virt_ptr[PTR_PG(i - 2)]
254						    [PTR_IDX(i - 2)] |=
255						    PTU_PTE_NEXT_TO_LAST;
256			}
257		}
258	}
259done:
260	hwq->prod = 0;
261	hwq->cons = 0;
262	hwq->pdev = pdev;
263	hwq->depth = hwq_attr->depth;
264	hwq->max_elements = hwq->depth;
265	hwq->element_size = stride;
266	hwq->qe_ppg = pg_size / stride;
267	/* For direct access to the elements */
268	lvl = hwq->level;
269	if (hwq_attr->sginfo->nopte && hwq->level)
270		lvl = hwq->level - 1;
271	hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
272	hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
273	spin_lock_init(&hwq->lock);
274
275	return 0;
276fail:
277	bng_re_free_hwq(res, hwq);
278	return -ENOMEM;
279}