bnge_rmem.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) 2025 Broadcom.
  3
  4#include <linux/etherdevice.h>
  5#include <linux/init.h>
  6#include <linux/module.h>
  7#include <linux/pci.h>
  8#include <linux/mm.h>
  9#include <linux/dma-mapping.h>
 10#include <linux/vmalloc.h>
 11#include <linux/crash_dump.h>
 12#include <linux/bnxt/hsi.h>
 13
 14#include "bnge.h"
 15#include "bnge_hwrm_lib.h"
 16#include "bnge_rmem.h"
 17
 18static void bnge_init_ctx_mem(struct bnge_ctx_mem_type *ctxm,
 19			      void *p, int len)
 20{
 21	u8 init_val = ctxm->init_value;
 22	u16 offset = ctxm->init_offset;
 23	u8 *p2 = p;
 24	int i;
 25
 26	if (!init_val)
 27		return;
 28	if (offset == BNGE_CTX_INIT_INVALID_OFFSET) {
 29		memset(p, init_val, len);
 30		return;
 31	}
 32	for (i = 0; i < len; i += ctxm->entry_size)
 33		*(p2 + i + offset) = init_val;
 34}
 35
 36void bnge_free_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem)
 37{
 38	struct pci_dev *pdev = bd->pdev;
 39	int i;
 40
 41	if (!rmem->pg_arr)
 42		goto skip_pages;
 43
 44	for (i = 0; i < rmem->nr_pages; i++) {
 45		if (!rmem->pg_arr[i])
 46			continue;
 47
 48		dma_free_coherent(&pdev->dev, rmem->page_size,
 49				  rmem->pg_arr[i], rmem->dma_arr[i]);
 50
 51		rmem->pg_arr[i] = NULL;
 52	}
 53skip_pages:
 54	if (rmem->pg_tbl) {
 55		size_t pg_tbl_size = rmem->nr_pages * 8;
 56
 57		if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG)
 58			pg_tbl_size = rmem->page_size;
 59		dma_free_coherent(&pdev->dev, pg_tbl_size,
 60				  rmem->pg_tbl, rmem->dma_pg_tbl);
 61		rmem->pg_tbl = NULL;
 62	}
 63	if (rmem->vmem_size && *rmem->vmem) {
 64		vfree(*rmem->vmem);
 65		*rmem->vmem = NULL;
 66	}
 67}
 68
 69int bnge_alloc_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem)
 70{
 71	struct pci_dev *pdev = bd->pdev;
 72	u64 valid_bit = 0;
 73	int i;
 74
 75	if (rmem->flags & (BNGE_RMEM_VALID_PTE_FLAG | BNGE_RMEM_RING_PTE_FLAG))
 76		valid_bit = PTU_PTE_VALID;
 77
 78	if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
 79		size_t pg_tbl_size = rmem->nr_pages * 8;
 80
 81		if (rmem->flags & BNGE_RMEM_USE_FULL_PAGE_FLAG)
 82			pg_tbl_size = rmem->page_size;
 83		rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
 84						  &rmem->dma_pg_tbl,
 85						  GFP_KERNEL);
 86		if (!rmem->pg_tbl)
 87			return -ENOMEM;
 88	}
 89
 90	for (i = 0; i < rmem->nr_pages; i++) {
 91		u64 extra_bits = valid_bit;
 92
 93		rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
 94						     rmem->page_size,
 95						     &rmem->dma_arr[i],
 96						     GFP_KERNEL);
 97		if (!rmem->pg_arr[i])
 98			goto err_free_ring;
 99
100		if (rmem->ctx_mem)
101			bnge_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
102					  rmem->page_size);
103
104		if (rmem->nr_pages > 1 || rmem->depth > 0) {
105			if (i == rmem->nr_pages - 2 &&
106			    (rmem->flags & BNGE_RMEM_RING_PTE_FLAG))
107				extra_bits |= PTU_PTE_NEXT_TO_LAST;
108			else if (i == rmem->nr_pages - 1 &&
109				 (rmem->flags & BNGE_RMEM_RING_PTE_FLAG))
110				extra_bits |= PTU_PTE_LAST;
111			rmem->pg_tbl[i] =
112				cpu_to_le64(rmem->dma_arr[i] | extra_bits);
113		}
114	}
115
116	if (rmem->vmem_size) {
117		*rmem->vmem = vzalloc(rmem->vmem_size);
118		if (!(*rmem->vmem))
119			goto err_free_ring;
120	}
121	return 0;
122
123err_free_ring:
124	bnge_free_ring(bd, rmem);
125	return -ENOMEM;
126}
127
128static int bnge_alloc_ctx_one_lvl(struct bnge_dev *bd,
129				  struct bnge_ctx_pg_info *ctx_pg)
130{
131	struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem;
132
133	rmem->page_size = BNGE_PAGE_SIZE;
134	rmem->pg_arr = ctx_pg->ctx_pg_arr;
135	rmem->dma_arr = ctx_pg->ctx_dma_arr;
136	rmem->flags = BNGE_RMEM_VALID_PTE_FLAG;
137	if (rmem->depth >= 1)
138		rmem->flags |= BNGE_RMEM_USE_FULL_PAGE_FLAG;
139	return bnge_alloc_ring(bd, rmem);
140}
141
142static int bnge_alloc_ctx_pg_tbls(struct bnge_dev *bd,
143				  struct bnge_ctx_pg_info *ctx_pg, u32 mem_size,
144				  u8 depth, struct bnge_ctx_mem_type *ctxm)
145{
146	struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem;
147	int rc;
148
149	if (!mem_size)
150		return -EINVAL;
151
152	ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE);
153	if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
154		ctx_pg->nr_pages = 0;
155		return -EINVAL;
156	}
157	if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
158		int nr_tbls, i;
159
160		rmem->depth = 2;
161		ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
162					     GFP_KERNEL);
163		if (!ctx_pg->ctx_pg_tbl)
164			return -ENOMEM;
165		nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
166		rmem->nr_pages = nr_tbls;
167		rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg);
168		if (rc)
169			return rc;
170		for (i = 0; i < nr_tbls; i++) {
171			struct bnge_ctx_pg_info *pg_tbl;
172
173			pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
174			if (!pg_tbl)
175				return -ENOMEM;
176			ctx_pg->ctx_pg_tbl[i] = pg_tbl;
177			rmem = &pg_tbl->ring_mem;
178			rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
179			rmem->dma_pg_tbl = ctx_pg->ctx_dma_arr[i];
180			rmem->depth = 1;
181			rmem->nr_pages = MAX_CTX_PAGES;
182			rmem->ctx_mem = ctxm;
183			if (i == (nr_tbls - 1)) {
184				int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
185
186				if (rem)
187					rmem->nr_pages = rem;
188			}
189			rc = bnge_alloc_ctx_one_lvl(bd, pg_tbl);
190			if (rc)
191				break;
192		}
193	} else {
194		rmem->nr_pages = DIV_ROUND_UP(mem_size, BNGE_PAGE_SIZE);
195		if (rmem->nr_pages > 1 || depth)
196			rmem->depth = 1;
197		rmem->ctx_mem = ctxm;
198		rc = bnge_alloc_ctx_one_lvl(bd, ctx_pg);
199	}
200
201	return rc;
202}
203
204static void bnge_free_ctx_pg_tbls(struct bnge_dev *bd,
205				  struct bnge_ctx_pg_info *ctx_pg)
206{
207	struct bnge_ring_mem_info *rmem = &ctx_pg->ring_mem;
208
209	if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
210	    ctx_pg->ctx_pg_tbl) {
211		int i, nr_tbls = rmem->nr_pages;
212
213		for (i = 0; i < nr_tbls; i++) {
214			struct bnge_ctx_pg_info *pg_tbl;
215			struct bnge_ring_mem_info *rmem2;
216
217			pg_tbl = ctx_pg->ctx_pg_tbl[i];
218			if (!pg_tbl)
219				continue;
220			rmem2 = &pg_tbl->ring_mem;
221			bnge_free_ring(bd, rmem2);
222			ctx_pg->ctx_pg_arr[i] = NULL;
223			kfree(pg_tbl);
224			ctx_pg->ctx_pg_tbl[i] = NULL;
225		}
226		kfree(ctx_pg->ctx_pg_tbl);
227		ctx_pg->ctx_pg_tbl = NULL;
228	}
229	bnge_free_ring(bd, rmem);
230	ctx_pg->nr_pages = 0;
231}
232
233static int bnge_setup_ctxm_pg_tbls(struct bnge_dev *bd,
234				   struct bnge_ctx_mem_type *ctxm, u32 entries,
235				   u8 pg_lvl)
236{
237	struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info;
238	int i, rc = 0, n = 1;
239	u32 mem_size;
240
241	if (!ctxm->entry_size || !ctx_pg)
242		return -EINVAL;
243	if (ctxm->instance_bmap)
244		n = hweight32(ctxm->instance_bmap);
245	if (ctxm->entry_multiple)
246		entries = roundup(entries, ctxm->entry_multiple);
247	entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
248	mem_size = entries * ctxm->entry_size;
249	for (i = 0; i < n && !rc; i++) {
250		ctx_pg[i].entries = entries;
251		rc = bnge_alloc_ctx_pg_tbls(bd, &ctx_pg[i], mem_size, pg_lvl,
252					    ctxm->init_value ? ctxm : NULL);
253	}
254
255	return rc;
256}
257
258static int bnge_backing_store_cfg(struct bnge_dev *bd, u32 ena)
259{
260	struct bnge_ctx_mem_info *ctx = bd->ctx;
261	struct bnge_ctx_mem_type *ctxm;
262	u16 last_type;
263	int rc = 0;
264	u16 type;
265
266	if (!ena)
267		return 0;
268	else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
269		last_type = BNGE_CTX_MAX - 1;
270	else
271		last_type = BNGE_CTX_L2_MAX - 1;
272	ctx->ctx_arr[last_type].last = 1;
273
274	for (type = 0 ; type < BNGE_CTX_V2_MAX; type++) {
275		ctxm = &ctx->ctx_arr[type];
276
277		rc = bnge_hwrm_func_backing_store(bd, ctxm, ctxm->last);
278		if (rc)
279			return rc;
280	}
281
282	return 0;
283}
284
285void bnge_free_ctx_mem(struct bnge_dev *bd)
286{
287	struct bnge_ctx_mem_info *ctx = bd->ctx;
288	u16 type;
289
290	if (!ctx)
291		return;
292
293	for (type = 0; type < BNGE_CTX_V2_MAX; type++) {
294		struct bnge_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
295		struct bnge_ctx_pg_info *ctx_pg = ctxm->pg_info;
296		int i, n = 1;
297
298		if (!ctx_pg)
299			continue;
300		if (ctxm->instance_bmap)
301			n = hweight32(ctxm->instance_bmap);
302		for (i = 0; i < n; i++)
303			bnge_free_ctx_pg_tbls(bd, &ctx_pg[i]);
304
305		kfree(ctx_pg);
306		ctxm->pg_info = NULL;
307	}
308
309	ctx->flags &= ~BNGE_CTX_FLAG_INITED;
310	kfree(ctx);
311	bd->ctx = NULL;
312}
313
314#define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES			\
315	(FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |		\
316	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |		\
317	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |		\
318	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |		\
319	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
320
321int bnge_alloc_ctx_mem(struct bnge_dev *bd)
322{
323	struct bnge_ctx_mem_type *ctxm;
324	struct bnge_ctx_mem_info *ctx;
325	u32 l2_qps, qp1_qps, max_qps;
326	u32 ena, entries_sp, entries;
327	u32 srqs, max_srqs, min;
328	u32 num_mr, num_ah;
329	u32 extra_srqs = 0;
330	u32 extra_qps = 0;
331	u32 fast_qpmd_qps;
332	u8 pg_lvl = 1;
333	int i, rc;
334
335	rc = bnge_hwrm_func_backing_store_qcaps(bd);
336	if (rc) {
337		dev_err(bd->dev, "Failed querying ctx mem caps, rc: %d\n", rc);
338		return rc;
339	}
340
341	ctx = bd->ctx;
342	if (!ctx || (ctx->flags & BNGE_CTX_FLAG_INITED))
343		return 0;
344
345	ctxm = &ctx->ctx_arr[BNGE_CTX_QP];
346	l2_qps = ctxm->qp_l2_entries;
347	qp1_qps = ctxm->qp_qp1_entries;
348	fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
349	max_qps = ctxm->max_entries;
350	ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ];
351	srqs = ctxm->srq_l2_entries;
352	max_srqs = ctxm->max_entries;
353	ena = 0;
354	if (bnge_is_roce_en(bd) && !is_kdump_kernel()) {
355		pg_lvl = 2;
356		extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
357		/* allocate extra qps if fast qp destroy feature enabled */
358		extra_qps += fast_qpmd_qps;
359		extra_srqs = min_t(u32, 8192, max_srqs - srqs);
360		if (fast_qpmd_qps)
361			ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
362	}
363
364	ctxm = &ctx->ctx_arr[BNGE_CTX_QP];
365	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps,
366				     pg_lvl);
367	if (rc)
368		return rc;
369
370	ctxm = &ctx->ctx_arr[BNGE_CTX_SRQ];
371	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, srqs + extra_srqs, pg_lvl);
372	if (rc)
373		return rc;
374
375	ctxm = &ctx->ctx_arr[BNGE_CTX_CQ];
376	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->cq_l2_entries +
377				     extra_qps * 2, pg_lvl);
378	if (rc)
379		return rc;
380
381	ctxm = &ctx->ctx_arr[BNGE_CTX_VNIC];
382	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1);
383	if (rc)
384		return rc;
385
386	ctxm = &ctx->ctx_arr[BNGE_CTX_STAT];
387	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, ctxm->max_entries, 1);
388	if (rc)
389		return rc;
390
391	if (!bnge_is_roce_en(bd))
392		goto skip_rdma;
393
394	ctxm = &ctx->ctx_arr[BNGE_CTX_MRAV];
395	/* 128K extra is needed to accommodate static AH context
396	 * allocation by f/w.
397	 */
398	num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
399	num_ah = min_t(u32, num_mr, 1024 * 128);
400	ctxm->split_entry_cnt = BNGE_CTX_MRAV_AV_SPLIT_ENTRY + 1;
401	if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
402		ctxm->mrav_av_entries = num_ah;
403
404	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, num_mr + num_ah, 2);
405	if (rc)
406		return rc;
407	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
408
409	ctxm = &ctx->ctx_arr[BNGE_CTX_TIM];
410	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 1);
411	if (rc)
412		return rc;
413	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
414
415skip_rdma:
416	ctxm = &ctx->ctx_arr[BNGE_CTX_STQM];
417	min = ctxm->min_entries;
418	entries_sp = ctx->ctx_arr[BNGE_CTX_VNIC].vnic_entries + l2_qps +
419		     2 * (extra_qps + qp1_qps) + min;
420	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries_sp, 2);
421	if (rc)
422		return rc;
423
424	ctxm = &ctx->ctx_arr[BNGE_CTX_FTQM];
425	entries = l2_qps + 2 * (extra_qps + qp1_qps);
426	rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, entries, 2);
427	if (rc)
428		return rc;
429	for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
430		ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
431	ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
432
433	rc = bnge_backing_store_cfg(bd, ena);
434	if (rc) {
435		dev_err(bd->dev, "Failed configuring ctx mem, rc: %d\n", rc);
436		return rc;
437	}
438	ctx->flags |= BNGE_CTX_FLAG_INITED;
439
440	return 0;
441}
442
443void bnge_init_ring_struct(struct bnge_net *bn)
444{
445	struct bnge_dev *bd = bn->bd;
446	int i, j;
447
448	for (i = 0; i < bd->nq_nr_rings; i++) {
449		struct bnge_napi *bnapi = bn->bnapi[i];
450		struct bnge_ring_mem_info *rmem;
451		struct bnge_nq_ring_info *nqr;
452		struct bnge_rx_ring_info *rxr;
453		struct bnge_tx_ring_info *txr;
454		struct bnge_ring_struct *ring;
455
456		nqr = &bnapi->nq_ring;
457		ring = &nqr->ring_struct;
458		rmem = &ring->ring_mem;
459		rmem->nr_pages = bn->cp_nr_pages;
460		rmem->page_size = HW_CMPD_RING_SIZE;
461		rmem->pg_arr = (void **)nqr->desc_ring;
462		rmem->dma_arr = nqr->desc_mapping;
463		rmem->vmem_size = 0;
464
465		rxr = bnapi->rx_ring;
466		if (!rxr)
467			goto skip_rx;
468
469		ring = &rxr->rx_ring_struct;
470		rmem = &ring->ring_mem;
471		rmem->nr_pages = bn->rx_nr_pages;
472		rmem->page_size = HW_RXBD_RING_SIZE;
473		rmem->pg_arr = (void **)rxr->rx_desc_ring;
474		rmem->dma_arr = rxr->rx_desc_mapping;
475		rmem->vmem_size = SW_RXBD_RING_SIZE * bn->rx_nr_pages;
476		rmem->vmem = (void **)&rxr->rx_buf_ring;
477
478		ring = &rxr->rx_agg_ring_struct;
479		rmem = &ring->ring_mem;
480		rmem->nr_pages = bn->rx_agg_nr_pages;
481		rmem->page_size = HW_RXBD_RING_SIZE;
482		rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
483		rmem->dma_arr = rxr->rx_agg_desc_mapping;
484		rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bn->rx_agg_nr_pages;
485		rmem->vmem = (void **)&rxr->rx_agg_buf_ring;
486
487skip_rx:
488		bnge_for_each_napi_tx(j, bnapi, txr) {
489			ring = &txr->tx_ring_struct;
490			rmem = &ring->ring_mem;
491			rmem->nr_pages = bn->tx_nr_pages;
492			rmem->page_size = HW_TXBD_RING_SIZE;
493			rmem->pg_arr = (void **)txr->tx_desc_ring;
494			rmem->dma_arr = txr->tx_desc_mapping;
495			rmem->vmem_size = SW_TXBD_RING_SIZE * bn->tx_nr_pages;
496			rmem->vmem = (void **)&txr->tx_buf_ring;
497		}
498	}
499}