qplib_res.c

  1/*
  2 * Broadcom NetXtreme-E RoCE driver.
  3 *
  4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
  5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
  6 *
  7 * This software is available to you under a choice of one of two
  8 * licenses.  You may choose to be licensed under the terms of the GNU
  9 * General Public License (GPL) Version 2, available from the file
 10 * COPYING in the main directory of this source tree, or the
 11 * BSD license below:
 12 *
 13 * Redistribution and use in source and binary forms, with or without
 14 * modification, are permitted provided that the following conditions
 15 * are met:
 16 *
 17 * 1. Redistributions of source code must retain the above copyright
 18 *    notice, this list of conditions and the following disclaimer.
 19 * 2. Redistributions in binary form must reproduce the above copyright
 20 *    notice, this list of conditions and the following disclaimer in
 21 *    the documentation and/or other materials provided with the
 22 *    distribution.
 23 *
 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
 25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 35 *
 36 * Description: QPLib resource manager
 37 */
 38
 39#define dev_fmt(fmt) "QPLIB: " fmt
 40
 41#include <linux/spinlock.h>
 42#include <linux/pci.h>
 43#include <linux/interrupt.h>
 44#include <linux/inetdevice.h>
 45#include <linux/dma-mapping.h>
 46#include <linux/if_vlan.h>
 47#include <linux/vmalloc.h>
 48#include <rdma/ib_verbs.h>
 49#include <rdma/ib_umem.h>
 50
 51#include "roce_hsi.h"
 52#include "qplib_res.h"
 53#include "qplib_sp.h"
 54#include "qplib_rcfw.h"
 55
 56/* PBL */
 57static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
 58		       bool is_umem)
 59{
 60	struct pci_dev *pdev = res->pdev;
 61	int i;
 62
 63	if (!is_umem) {
 64		for (i = 0; i < pbl->pg_count; i++) {
 65			if (pbl->pg_arr[i])
 66				dma_free_coherent(&pdev->dev, pbl->pg_size,
 67						  pbl->pg_arr[i],
 68						  pbl->pg_map_arr[i]);
 69			else
 70				dev_warn(&pdev->dev,
 71					 "PBL free pg_arr[%d] empty?!\n", i);
 72			pbl->pg_arr[i] = NULL;
 73		}
 74	}
 75	vfree(pbl->pg_arr);
 76	pbl->pg_arr = NULL;
 77	vfree(pbl->pg_map_arr);
 78	pbl->pg_map_arr = NULL;
 79	pbl->pg_count = 0;
 80	pbl->pg_size = 0;
 81}
 82
 83static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
 84					   struct bnxt_qplib_sg_info *sginfo)
 85{
 86	struct ib_block_iter biter;
 87	int i = 0;
 88
 89	rdma_umem_for_each_dma_block(sginfo->umem, &biter, sginfo->pgsize) {
 90		pbl->pg_map_arr[i] = rdma_block_iter_dma_address(&biter);
 91		pbl->pg_arr[i] = NULL;
 92		pbl->pg_count++;
 93		i++;
 94	}
 95}
 96
 97static int __alloc_pbl(struct bnxt_qplib_res *res,
 98		       struct bnxt_qplib_pbl *pbl,
 99		       struct bnxt_qplib_sg_info *sginfo)
100{
101	struct pci_dev *pdev = res->pdev;
102	bool is_umem = false;
103	u32 pages;
104	int i;
105
106	if (sginfo->nopte)
107		return 0;
108	if (sginfo->umem)
109		pages = ib_umem_num_dma_blocks(sginfo->umem, sginfo->pgsize);
110	else
111		pages = sginfo->npages;
112	/* page ptr arrays */
113	pbl->pg_arr = vmalloc_array(pages, sizeof(void *));
114	if (!pbl->pg_arr)
115		return -ENOMEM;
116	memset(pbl->pg_arr, 0, pages * sizeof(void *));
117
118	pbl->pg_map_arr = vmalloc_array(pages, sizeof(dma_addr_t));
119	if (!pbl->pg_map_arr) {
120		vfree(pbl->pg_arr);
121		pbl->pg_arr = NULL;
122		return -ENOMEM;
123	}
124	memset(pbl->pg_map_arr, 0, pages * sizeof(dma_addr_t));
125	pbl->pg_count = 0;
126	pbl->pg_size = sginfo->pgsize;
127
128	if (!sginfo->umem) {
129		for (i = 0; i < pages; i++) {
130			pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
131							    pbl->pg_size,
132							    &pbl->pg_map_arr[i],
133							    GFP_KERNEL);
134			if (!pbl->pg_arr[i])
135				goto fail;
136			pbl->pg_count++;
137		}
138	} else {
139		is_umem = true;
140		bnxt_qplib_fill_user_dma_pages(pbl, sginfo);
141	}
142
143	return 0;
144fail:
145	__free_pbl(res, pbl, is_umem);
146	return -ENOMEM;
147}
148
149/* HWQ */
150void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
151			 struct bnxt_qplib_hwq *hwq)
152{
153	int i;
154
155	if (!hwq->max_elements)
156		return;
157	if (hwq->level >= PBL_LVL_MAX)
158		return;
159
160	for (i = 0; i < hwq->level + 1; i++) {
161		if (i == hwq->level)
162			__free_pbl(res, &hwq->pbl[i], hwq->is_user);
163		else
164			__free_pbl(res, &hwq->pbl[i], false);
165	}
166
167	hwq->level = PBL_LVL_MAX;
168	hwq->max_elements = 0;
169	hwq->element_size = 0;
170	hwq->prod = 0;
171	hwq->cons = 0;
172	hwq->cp_bit = 0;
173}
174
175/* All HWQs are power of 2 in size */
176
177int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
178			      struct bnxt_qplib_hwq_attr *hwq_attr)
179{
180	u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;
181	struct bnxt_qplib_sg_info sginfo = {};
182	u32 depth, stride, npbl, npde;
183	dma_addr_t *src_phys_ptr, **dst_virt_ptr;
184	struct bnxt_qplib_res *res;
185	struct pci_dev *pdev;
186	int i, rc, lvl;
187
188	res = hwq_attr->res;
189	pdev = res->pdev;
190	pg_size = hwq_attr->sginfo->pgsize;
191	hwq->level = PBL_LVL_MAX;
192
193	depth = roundup_pow_of_two(hwq_attr->depth);
194	stride = roundup_pow_of_two(hwq_attr->stride);
195	if (hwq_attr->aux_depth) {
196		aux_slots = hwq_attr->aux_depth;
197		aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
198		aux_pages = (aux_slots * aux_size) / pg_size;
199		if ((aux_slots * aux_size) % pg_size)
200			aux_pages++;
201	}
202
203	if (!hwq_attr->sginfo->umem) {
204		hwq->is_user = false;
205		npages = (depth * stride) / pg_size + aux_pages;
206		if ((depth * stride) % pg_size)
207			npages++;
208		if (!npages)
209			return -EINVAL;
210		hwq_attr->sginfo->npages = npages;
211	} else {
212		npages = ib_umem_num_dma_blocks(hwq_attr->sginfo->umem,
213						hwq_attr->sginfo->pgsize);
214		hwq->is_user = true;
215	}
216
217	if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
218		/* This request is Level 0, map PTE */
219		rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
220		if (rc)
221			goto fail;
222		hwq->level = PBL_LVL_0;
223		goto done;
224	}
225
226	if (npages >= MAX_PBL_LVL_0_PGS) {
227		if (npages > MAX_PBL_LVL_1_PGS) {
228			u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
229				    0 : PTU_PTE_VALID;
230			/* 2 levels of indirection */
231			npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
232			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
233				npbl++;
234			npde = npbl >> MAX_PDL_LVL_SHIFT;
235			if (npbl % BIT(MAX_PDL_LVL_SHIFT))
236				npde++;
237			/* Alloc PDE pages */
238			sginfo.pgsize = npde * ROCE_PG_SIZE_4K;
239			sginfo.npages = 1;
240			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
241			if (rc)
242				goto fail;
243
244			/* Alloc PBL pages */
245			sginfo.npages = npbl;
246			sginfo.pgsize = ROCE_PG_SIZE_4K;
247			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
248			if (rc)
249				goto fail;
250			/* Fill PDL with PBL page pointers */
251			dst_virt_ptr =
252				(dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
253			src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
254			for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
255				dst_virt_ptr[0][i] = src_phys_ptr[i] | flag;
256
257			/* Alloc or init PTEs */
258			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
259					 hwq_attr->sginfo);
260			if (rc)
261				goto fail;
262			hwq->level = PBL_LVL_2;
263			if (hwq_attr->sginfo->nopte)
264				goto done;
265			/* Fill PBLs with PTE pointers */
266			dst_virt_ptr =
267				(dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
268			src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
269			for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
270				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
271					src_phys_ptr[i] | PTU_PTE_VALID;
272			}
273			if (hwq_attr->type == HWQ_TYPE_QUEUE) {
274				/* Find the last pg of the size */
275				i = hwq->pbl[PBL_LVL_2].pg_count;
276				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
277								  PTU_PTE_LAST;
278				if (i > 1)
279					dst_virt_ptr[PTR_PG(i - 2)]
280						    [PTR_IDX(i - 2)] |=
281						    PTU_PTE_NEXT_TO_LAST;
282			}
283		} else { /* pages < 512 npbl = 1, npde = 0 */
284			u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
285				    0 : PTU_PTE_VALID;
286
287			/* 1 level of indirection */
288			npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
289			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
290				npbl++;
291			sginfo.npages = npbl;
292			sginfo.pgsize = PAGE_SIZE;
293			/* Alloc PBL page */
294			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
295			if (rc)
296				goto fail;
297			/* Alloc or init  PTEs */
298			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
299					 hwq_attr->sginfo);
300			if (rc)
301				goto fail;
302			hwq->level = PBL_LVL_1;
303			if (hwq_attr->sginfo->nopte)
304				goto done;
305			/* Fill PBL with PTE pointers */
306			dst_virt_ptr =
307				(dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
308			src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
309			for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
310				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
311					src_phys_ptr[i] | flag;
312			if (hwq_attr->type == HWQ_TYPE_QUEUE) {
313				/* Find the last pg of the size */
314				i = hwq->pbl[PBL_LVL_1].pg_count;
315				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
316								  PTU_PTE_LAST;
317				if (i > 1)
318					dst_virt_ptr[PTR_PG(i - 2)]
319						    [PTR_IDX(i - 2)] |=
320						    PTU_PTE_NEXT_TO_LAST;
321			}
322		}
323	}
324done:
325	hwq->prod = 0;
326	hwq->cons = 0;
327	hwq->pdev = pdev;
328	hwq->depth = hwq_attr->depth;
329	hwq->max_elements = hwq->depth;
330	hwq->element_size = stride;
331	hwq->qe_ppg = pg_size / stride;
332	/* For direct access to the elements */
333	lvl = hwq->level;
334	if (hwq_attr->sginfo->nopte && hwq->level)
335		lvl = hwq->level - 1;
336	hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
337	hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
338	spin_lock_init(&hwq->lock);
339
340	return 0;
341fail:
342	bnxt_qplib_free_hwq(res, hwq);
343	return -ENOMEM;
344}
345
346/* Context Tables */
347void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res,
348			   struct bnxt_qplib_ctx *ctx)
349{
350	int i;
351
352	bnxt_qplib_free_hwq(res, &ctx->qpc_tbl);
353	bnxt_qplib_free_hwq(res, &ctx->mrw_tbl);
354	bnxt_qplib_free_hwq(res, &ctx->srqc_tbl);
355	bnxt_qplib_free_hwq(res, &ctx->cq_tbl);
356	bnxt_qplib_free_hwq(res, &ctx->tim_tbl);
357	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
358		bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.qtbl[i]);
359	/* restore original pde level before destroy */
360	ctx->tqm_ctx.pde.level = ctx->tqm_ctx.pde_level;
361	bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.pde);
362}
363
364static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
365				      struct bnxt_qplib_ctx *ctx)
366{
367	struct bnxt_qplib_hwq_attr hwq_attr = {};
368	struct bnxt_qplib_sg_info sginfo = {};
369	struct bnxt_qplib_tqm_ctx *tqmctx;
370	int rc;
371	int i;
372
373	tqmctx = &ctx->tqm_ctx;
374
375	sginfo.pgsize = PAGE_SIZE;
376	sginfo.pgshft = PAGE_SHIFT;
377	hwq_attr.sginfo = &sginfo;
378	hwq_attr.res = res;
379	hwq_attr.type = HWQ_TYPE_CTX;
380	hwq_attr.depth = 512;
381	hwq_attr.stride = sizeof(u64);
382	/* Alloc pdl buffer */
383	rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
384	if (rc)
385		goto out;
386	/* Save original pdl level */
387	tqmctx->pde_level = tqmctx->pde.level;
388
389	hwq_attr.stride = 1;
390	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
391		if (!tqmctx->qcount[i])
392			continue;
393		hwq_attr.depth = ctx->qpc_count * tqmctx->qcount[i];
394		rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
395		if (rc)
396			goto out;
397	}
398out:
399	return rc;
400}
401
402static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
403{
404	struct bnxt_qplib_hwq *tbl;
405	dma_addr_t *dma_ptr;
406	__le64 **pbl_ptr, *ptr;
407	int i, j, k;
408	int fnz_idx = -1;
409	int pg_count;
410
411	pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
412
413	for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
414	     i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
415		tbl = &ctx->qtbl[i];
416		if (!tbl->max_elements)
417			continue;
418		if (fnz_idx == -1)
419			fnz_idx = i; /* first non-zero index */
420		switch (tbl->level) {
421		case PBL_LVL_2:
422			pg_count = tbl->pbl[PBL_LVL_1].pg_count;
423			for (k = 0; k < pg_count; k++) {
424				ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
425				dma_ptr = &tbl->pbl[PBL_LVL_1].pg_map_arr[k];
426				*ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
427			}
428			break;
429		case PBL_LVL_1:
430		case PBL_LVL_0:
431		default:
432			ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
433			*ptr = cpu_to_le64(tbl->pbl[PBL_LVL_0].pg_map_arr[0] |
434					   PTU_PTE_VALID);
435			break;
436		}
437	}
438	if (fnz_idx == -1)
439		fnz_idx = 0;
440	/* update pde level as per page table programming */
441	ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
442			  ctx->qtbl[fnz_idx].level + 1;
443}
444
445static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
446				      struct bnxt_qplib_ctx *ctx)
447{
448	int rc;
449
450	rc = bnxt_qplib_alloc_tqm_rings(res, ctx);
451	if (rc)
452		goto fail;
453
454	bnxt_qplib_map_tqm_pgtbl(&ctx->tqm_ctx);
455fail:
456	return rc;
457}
458
459/*
460 * Routine: bnxt_qplib_alloc_hwctx
461 * Description:
462 *     Context tables are memories which are used by the chip fw.
463 *     The 6 tables defined are:
464 *             QPC ctx - holds QP states
465 *             MRW ctx - holds memory region and window
466 *             SRQ ctx - holds shared RQ states
467 *             CQ ctx - holds completion queue states
468 *             TQM ctx - holds Tx Queue Manager context
469 *             TIM ctx - holds timer context
470 *     Depending on the size of the tbl requested, either a 1 Page Buffer List
471 *     or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
472 *     instead.
473 *     Table might be employed as follows:
474 *             For 0      < ctx size <= 1 PAGE, 0 level of ind is used
475 *             For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
476 *             For 512    < ctx size <= MAX, 2 levels of ind is used
477 * Returns:
478 *     0 if success, else -ERRORS
479 */
480int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res,
481			   struct bnxt_qplib_ctx *ctx)
482{
483	struct bnxt_qplib_hwq_attr hwq_attr = {};
484	struct bnxt_qplib_sg_info sginfo = {};
485	int rc;
486
487	/* QPC Tables */
488	sginfo.pgsize = PAGE_SIZE;
489	sginfo.pgshft = PAGE_SHIFT;
490	hwq_attr.sginfo = &sginfo;
491
492	hwq_attr.res = res;
493	hwq_attr.depth = ctx->qpc_count;
494	hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
495	hwq_attr.type = HWQ_TYPE_CTX;
496	rc = bnxt_qplib_alloc_init_hwq(&ctx->qpc_tbl, &hwq_attr);
497	if (rc)
498		goto fail;
499
500	/* MRW Tables */
501	hwq_attr.depth = ctx->mrw_count;
502	hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
503	rc = bnxt_qplib_alloc_init_hwq(&ctx->mrw_tbl, &hwq_attr);
504	if (rc)
505		goto fail;
506
507	/* SRQ Tables */
508	hwq_attr.depth = ctx->srqc_count;
509	hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
510	rc = bnxt_qplib_alloc_init_hwq(&ctx->srqc_tbl, &hwq_attr);
511	if (rc)
512		goto fail;
513
514	/* CQ Tables */
515	hwq_attr.depth = ctx->cq_count;
516	hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
517	rc = bnxt_qplib_alloc_init_hwq(&ctx->cq_tbl, &hwq_attr);
518	if (rc)
519		goto fail;
520
521	/* TQM Buffer */
522	rc = bnxt_qplib_setup_tqm_rings(res, ctx);
523	if (rc)
524		goto fail;
525	/* TIM Buffer */
526	ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
527	hwq_attr.depth = ctx->qpc_count * 16;
528	hwq_attr.stride = 1;
529	rc = bnxt_qplib_alloc_init_hwq(&ctx->tim_tbl, &hwq_attr);
530	if (rc)
531		goto fail;
532
533	return 0;
534
535fail:
536	bnxt_qplib_free_hwctx(res, ctx);
537	return rc;
538}
539
540static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
541				     struct bnxt_qplib_sgid_tbl *sgid_tbl)
542{
543	kfree(sgid_tbl->tbl);
544	kfree(sgid_tbl->hw_id);
545	kfree(sgid_tbl->ctx);
546	kfree(sgid_tbl->vlan);
547	sgid_tbl->tbl = NULL;
548	sgid_tbl->hw_id = NULL;
549	sgid_tbl->ctx = NULL;
550	sgid_tbl->vlan = NULL;
551	sgid_tbl->max = 0;
552	sgid_tbl->active = 0;
553}
554
555static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
556				     struct bnxt_qplib_sgid_tbl *sgid_tbl,
557				     u16 max)
558{
559	sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
560	if (!sgid_tbl->tbl)
561		return -ENOMEM;
562
563	sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
564	if (!sgid_tbl->hw_id)
565		goto out_free1;
566
567	sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
568	if (!sgid_tbl->ctx)
569		goto out_free2;
570
571	sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
572	if (!sgid_tbl->vlan)
573		goto out_free3;
574
575	sgid_tbl->max = max;
576	return 0;
577out_free3:
578	kfree(sgid_tbl->ctx);
579	sgid_tbl->ctx = NULL;
580out_free2:
581	kfree(sgid_tbl->hw_id);
582	sgid_tbl->hw_id = NULL;
583out_free1:
584	kfree(sgid_tbl->tbl);
585	sgid_tbl->tbl = NULL;
586	return -ENOMEM;
587};
588
589static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
590					struct bnxt_qplib_sgid_tbl *sgid_tbl)
591{
592	int i;
593
594	for (i = 0; i < sgid_tbl->max; i++) {
595		if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
596			   sizeof(bnxt_qplib_gid_zero)))
597			bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
598					    sgid_tbl->tbl[i].vlan_id, true);
599	}
600	memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
601	memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
602	memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
603	sgid_tbl->active = 0;
604}
605
606static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
607				     struct net_device *netdev)
608{
609	u32 i;
610
611	for (i = 0; i < sgid_tbl->max; i++)
612		sgid_tbl->tbl[i].vlan_id = 0xffff;
613
614	memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
615}
616
617/* PDs */
618int bnxt_qplib_alloc_pd(struct bnxt_qplib_res  *res, struct bnxt_qplib_pd *pd)
619{
620	struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl;
621	u32 bit_num;
622	int rc = 0;
623
624	mutex_lock(&res->pd_tbl_lock);
625	bit_num = find_first_bit(pdt->tbl, pdt->max);
626	if (bit_num == pdt->max) {
627		rc = -ENOMEM;
628		goto exit;
629	}
630
631	/* Found unused PD */
632	clear_bit(bit_num, pdt->tbl);
633	pd->id = bit_num;
634exit:
635	mutex_unlock(&res->pd_tbl_lock);
636	return rc;
637}
638
639int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
640			  struct bnxt_qplib_pd_tbl *pdt,
641			  struct bnxt_qplib_pd *pd)
642{
643	int rc = 0;
644
645	mutex_lock(&res->pd_tbl_lock);
646	if (test_and_set_bit(pd->id, pdt->tbl)) {
647		dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
648			 pd->id);
649		rc = -EINVAL;
650		goto exit;
651	}
652	pd->id = 0;
653exit:
654	mutex_unlock(&res->pd_tbl_lock);
655	return rc;
656}
657
658static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
659{
660	kfree(pdt->tbl);
661	pdt->tbl = NULL;
662	pdt->max = 0;
663}
664
665static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
666				   struct bnxt_qplib_pd_tbl *pdt,
667				   u32 max)
668{
669	u32 bytes;
670
671	bytes = max >> 3;
672	if (!bytes)
673		bytes = 1;
674	pdt->tbl = kmalloc(bytes, GFP_KERNEL);
675	if (!pdt->tbl)
676		return -ENOMEM;
677
678	pdt->max = max;
679	memset((u8 *)pdt->tbl, 0xFF, bytes);
680	mutex_init(&res->pd_tbl_lock);
681
682	return 0;
683}
684
685/* DPIs */
686int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
687			 struct bnxt_qplib_dpi *dpi,
688			 void *app, u8 type)
689{
690	struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
691	struct bnxt_qplib_reg_desc *reg;
692	u32 bit_num;
693	u64 umaddr;
694
695	reg = &dpit->wcreg;
696	mutex_lock(&res->dpi_tbl_lock);
697
698	bit_num = find_first_bit(dpit->tbl, dpit->max);
699	if (bit_num == dpit->max) {
700		mutex_unlock(&res->dpi_tbl_lock);
701		return -ENOMEM;
702	}
703
704	/* Found unused DPI */
705	clear_bit(bit_num, dpit->tbl);
706	dpit->app_tbl[bit_num] = app;
707
708	dpi->bit = bit_num;
709	dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE;
710
711	umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE;
712	dpi->umdbr = umaddr;
713
714	switch (type) {
715	case BNXT_QPLIB_DPI_TYPE_KERNEL:
716		/* privileged dbr was already mapped just initialize it. */
717		dpi->umdbr = dpit->ucreg.bar_base +
718			     dpit->ucreg.offset + bit_num * PAGE_SIZE;
719		dpi->dbr = dpit->priv_db;
720		dpi->dpi = dpi->bit;
721		break;
722	case BNXT_QPLIB_DPI_TYPE_WC:
723		dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE);
724		break;
725	default:
726		dpi->dbr = ioremap(umaddr, PAGE_SIZE);
727		break;
728	}
729
730	dpi->type = type;
731	mutex_unlock(&res->dpi_tbl_lock);
732	return 0;
733
734}
735
736int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
737			   struct bnxt_qplib_dpi *dpi)
738{
739	struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
740
741	mutex_lock(&res->dpi_tbl_lock);
742	if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL)
743		pci_iounmap(res->pdev, dpi->dbr);
744
745	if (test_and_set_bit(dpi->bit, dpit->tbl)) {
746		dev_warn(&res->pdev->dev,
747			 "Freeing an unused DPI? dpi = %d, bit = %d\n",
748				dpi->dpi, dpi->bit);
749		mutex_unlock(&res->dpi_tbl_lock);
750		return -EINVAL;
751	}
752	if (dpit->app_tbl)
753		dpit->app_tbl[dpi->bit] = NULL;
754	memset(dpi, 0, sizeof(*dpi));
755	mutex_unlock(&res->dpi_tbl_lock);
756	return 0;
757}
758
759static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res     *res,
760				    struct bnxt_qplib_dpi_tbl *dpit)
761{
762	kfree(dpit->tbl);
763	kfree(dpit->app_tbl);
764	dpit->tbl = NULL;
765	dpit->app_tbl = NULL;
766	dpit->max = 0;
767}
768
769static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
770				    struct bnxt_qplib_dev_attr *dev_attr)
771{
772	struct bnxt_qplib_dpi_tbl *dpit;
773	struct bnxt_qplib_reg_desc *reg;
774	unsigned long bar_len;
775	u32 dbr_offset;
776	u32 bytes;
777
778	dpit = &res->dpi_tbl;
779	reg = &dpit->wcreg;
780
781	if (!bnxt_qplib_is_chip_gen_p5_p7(res->cctx)) {
782		/* Offest should come from L2 driver */
783		dbr_offset = dev_attr->l2_db_size;
784		dpit->ucreg.offset = dbr_offset;
785		dpit->wcreg.offset = dbr_offset;
786	}
787
788	bar_len = pci_resource_len(res->pdev, reg->bar_id);
789	dpit->max = (bar_len - reg->offset) / PAGE_SIZE;
790	if (dev_attr->max_dpi)
791		dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi);
792
793	dpit->app_tbl = kcalloc(dpit->max,  sizeof(void *), GFP_KERNEL);
794	if (!dpit->app_tbl)
795		return -ENOMEM;
796
797	bytes = dpit->max >> 3;
798	if (!bytes)
799		bytes = 1;
800
801	dpit->tbl = kmalloc(bytes, GFP_KERNEL);
802	if (!dpit->tbl) {
803		kfree(dpit->app_tbl);
804		dpit->app_tbl = NULL;
805		return -ENOMEM;
806	}
807
808	memset((u8 *)dpit->tbl, 0xFF, bytes);
809	mutex_init(&res->dpi_tbl_lock);
810	dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset;
811
812	return 0;
813
814}
815
816/* Stats */
817void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
818			       struct bnxt_qplib_stats *stats)
819{
820	if (stats->dma) {
821		dma_free_coherent(&pdev->dev, stats->size,
822				  stats->dma, stats->dma_map);
823	}
824	memset(stats, 0, sizeof(*stats));
825	stats->fw_id = -1;
826}
827
828int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
829			       struct bnxt_qplib_chip_ctx *cctx,
830			       struct bnxt_qplib_stats *stats)
831{
832	memset(stats, 0, sizeof(*stats));
833	stats->fw_id = -1;
834	stats->size = cctx->hw_stats_size;
835	stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
836					&stats->dma_map, GFP_KERNEL);
837	if (!stats->dma) {
838		dev_err(&pdev->dev, "Stats DMA allocation failed\n");
839		return -ENOMEM;
840	}
841	return 0;
842}
843
844void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
845{
846	bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
847}
848
849int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
850{
851	bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
852
853	return 0;
854}
855
856void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
857{
858	kfree(res->rcfw->qp_tbl);
859	bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
860	bnxt_qplib_free_pd_tbl(&res->pd_tbl);
861	bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
862}
863
864int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev)
865{
866	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
867	struct bnxt_qplib_dev_attr *dev_attr;
868	int rc;
869
870	res->netdev = netdev;
871	dev_attr = res->dattr;
872
873	/* Allocate one extra to hold the QP1 entries */
874	rcfw->qp_tbl_size = max_t(u32, BNXT_RE_MAX_QPC_COUNT + 1, dev_attr->max_qp);
875	rcfw->qp_tbl = kcalloc(rcfw->qp_tbl_size, sizeof(struct bnxt_qplib_qp_node),
876			       GFP_KERNEL);
877	if (!rcfw->qp_tbl)
878		return -ENOMEM;
879
880	rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
881	if (rc)
882		goto fail;
883
884	rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
885	if (rc)
886		goto fail;
887
888	rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr);
889	if (rc)
890		goto fail;
891
892	return 0;
893fail:
894	bnxt_qplib_free_res(res);
895	return rc;
896}
897
898void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res)
899{
900	struct bnxt_qplib_reg_desc *reg;
901
902	reg = &res->dpi_tbl.ucreg;
903	if (reg->bar_reg)
904		pci_iounmap(res->pdev, reg->bar_reg);
905	reg->bar_reg = NULL;
906	reg->bar_base = 0;
907	reg->len = 0;
908	reg->bar_id = 0;
909}
910
911int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res)
912{
913	struct bnxt_qplib_reg_desc *ucreg;
914	struct bnxt_qplib_reg_desc *wcreg;
915
916	wcreg = &res->dpi_tbl.wcreg;
917	wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
918	wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id);
919
920	ucreg = &res->dpi_tbl.ucreg;
921	ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
922	ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id);
923	ucreg->len = ucreg->offset + PAGE_SIZE;
924	if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) {
925		dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d",
926			(int)ucreg->len);
927		return -EINVAL;
928	}
929	ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len);
930	if (!ucreg->bar_reg) {
931		dev_err(&res->pdev->dev, "privileged dpi map failed!");
932		return -ENOMEM;
933	}
934
935	return 0;
936}
937
938int bnxt_qplib_determine_atomics(struct pci_dev *dev)
939{
940	int comp;
941	u16 ctl2;
942
943	comp = pci_enable_atomic_ops_to_root(dev,
944					     PCI_EXP_DEVCAP2_ATOMIC_COMP32);
945	if (comp)
946		return -EOPNOTSUPP;
947	comp = pci_enable_atomic_ops_to_root(dev,
948					     PCI_EXP_DEVCAP2_ATOMIC_COMP64);
949	if (comp)
950		return -EOPNOTSUPP;
951	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
952	return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
953}