rvu.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Marvell RVU Admin Function driver
   3 *
   4 * Copyright (C) 2018 Marvell.
   5 *
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/interrupt.h>
  10#include <linux/delay.h>
  11#include <linux/irq.h>
  12#include <linux/pci.h>
  13#include <linux/sysfs.h>
  14
  15#include "cgx.h"
  16#include "rvu.h"
  17#include "rvu_reg.h"
  18#include "ptp.h"
  19#include "mcs.h"
  20
  21#include "rvu_trace.h"
  22#include "rvu_npc_hash.h"
  23#include "cn20k/reg.h"
  24#include "cn20k/api.h"
  25
  26#define DRV_NAME	"rvu_af"
  27#define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"
  28
  29static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
  30				struct rvu_block *block, int lf);
  31static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
  32				  struct rvu_block *block, int lf);
  33static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);
  34
  35static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
  36			 int type, int num,
  37			 void (mbox_handler)(struct work_struct *),
  38			 void (mbox_up_handler)(struct work_struct *));
  39static irqreturn_t rvu_mbox_pf_intr_handler(int irq, void *rvu_irq);
  40static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq);
  41
  42/* Supported devices */
  43static const struct pci_device_id rvu_id_table[] = {
  44	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AF) },
  45	{ 0, }  /* end of table */
  46};
  47
  48MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
  49MODULE_DESCRIPTION(DRV_STRING);
  50MODULE_LICENSE("GPL v2");
  51MODULE_DEVICE_TABLE(pci, rvu_id_table);
  52
  53static char *mkex_profile; /* MKEX profile name */
  54module_param(mkex_profile, charp, 0000);
  55MODULE_PARM_DESC(mkex_profile, "MKEX profile name string");
  56
  57static char *kpu_profile; /* KPU profile name */
  58module_param(kpu_profile, charp, 0000);
  59MODULE_PARM_DESC(kpu_profile, "KPU profile name string");
  60
  61static void rvu_setup_hw_capabilities(struct rvu *rvu)
  62{
  63	struct rvu_hwinfo *hw = rvu->hw;
  64
  65	hw->cap.nix_tx_aggr_lvl = NIX_TXSCH_LVL_TL1;
  66	hw->cap.nix_fixed_txschq_mapping = false;
  67	hw->cap.nix_shaping = true;
  68	hw->cap.nix_tx_link_bp = true;
  69	hw->cap.nix_rx_multicast = true;
  70	hw->cap.nix_shaper_toggle_wait = false;
  71	hw->cap.npc_hash_extract = false;
  72	hw->cap.npc_exact_match_enabled = false;
  73	hw->rvu = rvu;
  74
  75	if (is_rvu_pre_96xx_C0(rvu)) {
  76		hw->cap.nix_fixed_txschq_mapping = true;
  77		hw->cap.nix_txsch_per_cgx_lmac = 4;
  78		hw->cap.nix_txsch_per_lbk_lmac = 132;
  79		hw->cap.nix_txsch_per_sdp_lmac = 76;
  80		hw->cap.nix_shaping = false;
  81		hw->cap.nix_tx_link_bp = false;
  82		if (is_rvu_96xx_A0(rvu) || is_rvu_95xx_A0(rvu))
  83			hw->cap.nix_rx_multicast = false;
  84	}
  85	if (!is_rvu_pre_96xx_C0(rvu))
  86		hw->cap.nix_shaper_toggle_wait = true;
  87
  88	if (!is_rvu_otx2(rvu))
  89		hw->cap.per_pf_mbox_regs = true;
  90
  91	if (is_rvu_npc_hash_extract_en(rvu))
  92		hw->cap.npc_hash_extract = true;
  93}
  94
  95/* Poll a RVU block's register 'offset', for a 'zero'
  96 * or 'nonzero' at bits specified by 'mask'
  97 */
  98int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero)
  99{
 100	unsigned long timeout = jiffies + usecs_to_jiffies(20000);
 101	bool twice = false;
 102	void __iomem *reg;
 103	u64 reg_val;
 104
 105	reg = rvu->afreg_base + ((block << 28) | offset);
 106again:
 107	reg_val = readq(reg);
 108	if (zero && !(reg_val & mask))
 109		return 0;
 110	if (!zero && (reg_val & mask))
 111		return 0;
 112	if (time_before(jiffies, timeout)) {
 113		usleep_range(1, 5);
 114		goto again;
 115	}
 116	/* In scenarios where CPU is scheduled out before checking
 117	 * 'time_before' (above) and gets scheduled in such that
 118	 * jiffies are beyond timeout value, then check again if HW is
 119	 * done with the operation in the meantime.
 120	 */
 121	if (!twice) {
 122		twice = true;
 123		goto again;
 124	}
 125	return -EBUSY;
 126}
 127
 128int rvu_alloc_rsrc(struct rsrc_bmap *rsrc)
 129{
 130	int id;
 131
 132	if (!rsrc->bmap)
 133		return -EINVAL;
 134
 135	id = find_first_zero_bit(rsrc->bmap, rsrc->max);
 136	if (id >= rsrc->max)
 137		return -ENOSPC;
 138
 139	__set_bit(id, rsrc->bmap);
 140
 141	return id;
 142}
 143
 144int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc)
 145{
 146	int start;
 147
 148	if (!rsrc->bmap)
 149		return -EINVAL;
 150
 151	start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
 152	if (start >= rsrc->max)
 153		return -ENOSPC;
 154
 155	bitmap_set(rsrc->bmap, start, nrsrc);
 156	return start;
 157}
 158
 159void rvu_free_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc, int start)
 160{
 161	if (!rsrc->bmap)
 162		return;
 163	if (start >= rsrc->max)
 164		return;
 165
 166	bitmap_clear(rsrc->bmap, start, nrsrc);
 167}
 168
 169bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc)
 170{
 171	int start;
 172
 173	if (!rsrc->bmap)
 174		return false;
 175
 176	start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
 177	if (start >= rsrc->max)
 178		return false;
 179
 180	return true;
 181}
 182
 183void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id)
 184{
 185	if (!rsrc->bmap)
 186		return;
 187
 188	__clear_bit(id, rsrc->bmap);
 189}
 190
 191int rvu_rsrc_free_count(struct rsrc_bmap *rsrc)
 192{
 193	int used;
 194
 195	if (!rsrc->bmap)
 196		return 0;
 197
 198	used = bitmap_weight(rsrc->bmap, rsrc->max);
 199	return (rsrc->max - used);
 200}
 201
 202bool is_rsrc_free(struct rsrc_bmap *rsrc, int id)
 203{
 204	if (!rsrc->bmap)
 205		return false;
 206
 207	return !test_bit(id, rsrc->bmap);
 208}
 209
 210int rvu_alloc_bitmap(struct rsrc_bmap *rsrc)
 211{
 212	rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max),
 213			     sizeof(long), GFP_KERNEL);
 214	if (!rsrc->bmap)
 215		return -ENOMEM;
 216	return 0;
 217}
 218
 219void rvu_free_bitmap(struct rsrc_bmap *rsrc)
 220{
 221	kfree(rsrc->bmap);
 222}
 223
 224/* Get block LF's HW index from a PF_FUNC's block slot number */
 225int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot)
 226{
 227	u16 match = 0;
 228	int lf;
 229
 230	mutex_lock(&rvu->rsrc_lock);
 231	for (lf = 0; lf < block->lf.max; lf++) {
 232		if (block->fn_map[lf] == pcifunc) {
 233			if (slot == match) {
 234				mutex_unlock(&rvu->rsrc_lock);
 235				return lf;
 236			}
 237			match++;
 238		}
 239	}
 240	mutex_unlock(&rvu->rsrc_lock);
 241	return -ENODEV;
 242}
 243
 244/* Convert BLOCK_TYPE_E to a BLOCK_ADDR_E.
 245 * Some silicon variants of OcteonTX2 supports
 246 * multiple blocks of same type.
 247 *
 248 * @pcifunc has to be zero when no LF is yet attached.
 249 *
 250 * For a pcifunc if LFs are attached from multiple blocks of same type, then
 251 * return blkaddr of first encountered block.
 252 */
 253int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc)
 254{
 255	int devnum, blkaddr = -ENODEV;
 256	u64 cfg, reg;
 257	bool is_pf;
 258
 259	switch (blktype) {
 260	case BLKTYPE_NPC:
 261		blkaddr = BLKADDR_NPC;
 262		goto exit;
 263	case BLKTYPE_NPA:
 264		blkaddr = BLKADDR_NPA;
 265		goto exit;
 266	case BLKTYPE_NIX:
 267		/* For now assume NIX0 */
 268		if (!pcifunc) {
 269			blkaddr = BLKADDR_NIX0;
 270			goto exit;
 271		}
 272		break;
 273	case BLKTYPE_SSO:
 274		blkaddr = BLKADDR_SSO;
 275		goto exit;
 276	case BLKTYPE_SSOW:
 277		blkaddr = BLKADDR_SSOW;
 278		goto exit;
 279	case BLKTYPE_TIM:
 280		blkaddr = BLKADDR_TIM;
 281		goto exit;
 282	case BLKTYPE_CPT:
 283		/* For now assume CPT0 */
 284		if (!pcifunc) {
 285			blkaddr = BLKADDR_CPT0;
 286			goto exit;
 287		}
 288		break;
 289	}
 290
 291	/* Check if this is a RVU PF or VF */
 292	if (pcifunc & RVU_PFVF_FUNC_MASK) {
 293		is_pf = false;
 294		devnum = rvu_get_hwvf(rvu, pcifunc);
 295	} else {
 296		is_pf = true;
 297		devnum = rvu_get_pf(rvu->pdev, pcifunc);
 298	}
 299
 300	/* Check if the 'pcifunc' has a NIX LF from 'BLKADDR_NIX0' or
 301	 * 'BLKADDR_NIX1'.
 302	 */
 303	if (blktype == BLKTYPE_NIX) {
 304		reg = is_pf ? RVU_PRIV_PFX_NIXX_CFG(0) :
 305			RVU_PRIV_HWVFX_NIXX_CFG(0);
 306		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 307		if (cfg) {
 308			blkaddr = BLKADDR_NIX0;
 309			goto exit;
 310		}
 311
 312		reg = is_pf ? RVU_PRIV_PFX_NIXX_CFG(1) :
 313			RVU_PRIV_HWVFX_NIXX_CFG(1);
 314		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 315		if (cfg)
 316			blkaddr = BLKADDR_NIX1;
 317	}
 318
 319	if (blktype == BLKTYPE_CPT) {
 320		reg = is_pf ? RVU_PRIV_PFX_CPTX_CFG(0) :
 321			RVU_PRIV_HWVFX_CPTX_CFG(0);
 322		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 323		if (cfg) {
 324			blkaddr = BLKADDR_CPT0;
 325			goto exit;
 326		}
 327
 328		reg = is_pf ? RVU_PRIV_PFX_CPTX_CFG(1) :
 329			RVU_PRIV_HWVFX_CPTX_CFG(1);
 330		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 331		if (cfg)
 332			blkaddr = BLKADDR_CPT1;
 333	}
 334
 335exit:
 336	if (is_block_implemented(rvu->hw, blkaddr))
 337		return blkaddr;
 338	return -ENODEV;
 339}
 340
 341static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf,
 342				struct rvu_block *block, u16 pcifunc,
 343				u16 lf, bool attach)
 344{
 345	int devnum, num_lfs = 0;
 346	bool is_pf;
 347	u64 reg;
 348
 349	if (lf >= block->lf.max) {
 350		dev_err(&rvu->pdev->dev,
 351			"%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",
 352			__func__, lf, block->name, block->lf.max);
 353		return;
 354	}
 355
 356	/* Check if this is for a RVU PF or VF */
 357	if (pcifunc & RVU_PFVF_FUNC_MASK) {
 358		is_pf = false;
 359		devnum = rvu_get_hwvf(rvu, pcifunc);
 360	} else {
 361		is_pf = true;
 362		devnum = rvu_get_pf(rvu->pdev, pcifunc);
 363	}
 364
 365	block->fn_map[lf] = attach ? pcifunc : 0;
 366
 367	switch (block->addr) {
 368	case BLKADDR_NPA:
 369		pfvf->npalf = attach ? true : false;
 370		num_lfs = pfvf->npalf;
 371		break;
 372	case BLKADDR_NIX0:
 373	case BLKADDR_NIX1:
 374		pfvf->nixlf = attach ? true : false;
 375		num_lfs = pfvf->nixlf;
 376		break;
 377	case BLKADDR_SSO:
 378		attach ? pfvf->sso++ : pfvf->sso--;
 379		num_lfs = pfvf->sso;
 380		break;
 381	case BLKADDR_SSOW:
 382		attach ? pfvf->ssow++ : pfvf->ssow--;
 383		num_lfs = pfvf->ssow;
 384		break;
 385	case BLKADDR_TIM:
 386		attach ? pfvf->timlfs++ : pfvf->timlfs--;
 387		num_lfs = pfvf->timlfs;
 388		break;
 389	case BLKADDR_CPT0:
 390		attach ? pfvf->cptlfs++ : pfvf->cptlfs--;
 391		num_lfs = pfvf->cptlfs;
 392		break;
 393	case BLKADDR_CPT1:
 394		attach ? pfvf->cpt1_lfs++ : pfvf->cpt1_lfs--;
 395		num_lfs = pfvf->cpt1_lfs;
 396		break;
 397	}
 398
 399	reg = is_pf ? block->pf_lfcnt_reg : block->vf_lfcnt_reg;
 400	rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs);
 401}
 402
 403void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)
 404{
 405	u64 cfg;
 406
 407	/* Get numVFs attached to this PF and first HWVF */
 408	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 409	if (numvfs)
 410		*numvfs = (cfg >> 12) & 0xFF;
 411	if (hwvf)
 412		*hwvf = cfg & 0xFFF;
 413}
 414
 415int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
 416{
 417	int pf, func;
 418	u64 cfg;
 419
 420	pf = rvu_get_pf(rvu->pdev, pcifunc);
 421	func = pcifunc & RVU_PFVF_FUNC_MASK;
 422
 423	/* Get first HWVF attached to this PF */
 424	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 425
 426	return ((cfg & 0xFFF) + func - 1);
 427}
 428
 429struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
 430{
 431	/* Check if it is a PF or VF */
 432	if (pcifunc & RVU_PFVF_FUNC_MASK)
 433		return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)];
 434	else
 435		return &rvu->pf[rvu_get_pf(rvu->pdev, pcifunc)];
 436}
 437
 438static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
 439{
 440	int pf, vf, nvfs;
 441	u64 cfg;
 442
 443	pf = rvu_get_pf(rvu->pdev, pcifunc);
 444	if (pf >= rvu->hw->total_pfs)
 445		return false;
 446
 447	if (!(pcifunc & RVU_PFVF_FUNC_MASK))
 448		return true;
 449
 450	/* Check if VF is within number of VFs attached to this PF */
 451	vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
 452	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 453	nvfs = (cfg >> 12) & 0xFF;
 454	if (vf >= nvfs)
 455		return false;
 456
 457	return true;
 458}
 459
 460bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr)
 461{
 462	struct rvu_block *block;
 463
 464	if (blkaddr < BLKADDR_RVUM || blkaddr >= BLK_COUNT)
 465		return false;
 466
 467	block = &hw->block[blkaddr];
 468	return block->implemented;
 469}
 470
 471static void rvu_check_block_implemented(struct rvu *rvu)
 472{
 473	struct rvu_hwinfo *hw = rvu->hw;
 474	struct rvu_block *block;
 475	int blkid;
 476	u64 cfg;
 477
 478	/* For each block check if 'implemented' bit is set */
 479	for (blkid = 0; blkid < BLK_COUNT; blkid++) {
 480		block = &hw->block[blkid];
 481		cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid));
 482		if (cfg & BIT_ULL(11))
 483			block->implemented = true;
 484	}
 485}
 486
 487static void rvu_setup_rvum_blk_revid(struct rvu *rvu)
 488{
 489	rvu_write64(rvu, BLKADDR_RVUM,
 490		    RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM),
 491		    RVU_BLK_RVUM_REVID);
 492}
 493
 494static void rvu_clear_rvum_blk_revid(struct rvu *rvu)
 495{
 496	rvu_write64(rvu, BLKADDR_RVUM,
 497		    RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM), 0x00);
 498}
 499
 500int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf)
 501{
 502	int err;
 503
 504	if (!block->implemented)
 505		return 0;
 506
 507	rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12));
 508	err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12),
 509			   true);
 510	return err;
 511}
 512
 513static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)
 514{
 515	struct rvu_block *block = &rvu->hw->block[blkaddr];
 516	int err;
 517
 518	if (!block->implemented)
 519		return;
 520
 521	rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));
 522	err = rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);
 523	if (err) {
 524		dev_err(rvu->dev, "HW block:%d reset timeout retrying again\n", blkaddr);
 525		while (rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true) == -EBUSY)
 526			;
 527	}
 528}
 529
 530static void rvu_reset_all_blocks(struct rvu *rvu)
 531{
 532	/* Do a HW reset of all RVU blocks */
 533	rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST);
 534	rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST);
 535	rvu_block_reset(rvu, BLKADDR_NIX1, NIX_AF_BLK_RST);
 536	rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST);
 537	rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST);
 538	rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST);
 539	rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST);
 540	rvu_block_reset(rvu, BLKADDR_CPT1, CPT_AF_BLK_RST);
 541	rvu_block_reset(rvu, BLKADDR_NDC_NIX0_RX, NDC_AF_BLK_RST);
 542	rvu_block_reset(rvu, BLKADDR_NDC_NIX0_TX, NDC_AF_BLK_RST);
 543	rvu_block_reset(rvu, BLKADDR_NDC_NIX1_RX, NDC_AF_BLK_RST);
 544	rvu_block_reset(rvu, BLKADDR_NDC_NIX1_TX, NDC_AF_BLK_RST);
 545	rvu_block_reset(rvu, BLKADDR_NDC_NPA0, NDC_AF_BLK_RST);
 546}
 547
 548static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block)
 549{
 550	struct rvu_pfvf *pfvf;
 551	u64 cfg;
 552	int lf;
 553
 554	for (lf = 0; lf < block->lf.max; lf++) {
 555		cfg = rvu_read64(rvu, block->addr,
 556				 block->lfcfg_reg | (lf << block->lfshift));
 557		if (!(cfg & BIT_ULL(63)))
 558			continue;
 559
 560		/* Set this resource as being used */
 561		__set_bit(lf, block->lf.bmap);
 562
 563		/* Get, to whom this LF is attached */
 564		pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF);
 565		rvu_update_rsrc_map(rvu, pfvf, block,
 566				    (cfg >> 8) & 0xFFFF, lf, true);
 567
 568		/* Set start MSIX vector for this LF within this PF/VF */
 569		rvu_set_msix_offset(rvu, pfvf, block, lf);
 570	}
 571}
 572
 573static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)
 574{
 575	int min_vecs;
 576
 577	if (!vf)
 578		goto check_pf;
 579
 580	if (!nvecs) {
 581		dev_warn(rvu->dev,
 582			 "PF%d:VF%d is configured with zero msix vectors, %d\n",
 583			 pf, vf - 1, nvecs);
 584	}
 585	return;
 586
 587check_pf:
 588	if (pf == 0)
 589		min_vecs = RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT;
 590	else
 591		min_vecs = RVU_PF_INT_VEC_CNT;
 592
 593	if (!(nvecs < min_vecs))
 594		return;
 595	dev_warn(rvu->dev,
 596		 "PF%d is configured with too few vectors, %d, min is %d\n",
 597		 pf, nvecs, min_vecs);
 598}
 599
 600static int rvu_setup_msix_resources(struct rvu *rvu)
 601{
 602	struct rvu_hwinfo *hw = rvu->hw;
 603	int pf, vf, numvfs, hwvf, err;
 604	int nvecs, offset, max_msix;
 605	struct rvu_pfvf *pfvf;
 606	u64 cfg, phy_addr;
 607	dma_addr_t iova;
 608
 609	for (pf = 0; pf < hw->total_pfs; pf++) {
 610		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 611		/* If PF is not enabled, nothing to do */
 612		if (!((cfg >> 20) & 0x01))
 613			continue;
 614
 615		rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
 616
 617		pfvf = &rvu->pf[pf];
 618		/* Get num of MSIX vectors attached to this PF */
 619		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf));
 620		pfvf->msix.max = ((cfg >> 32) & 0xFFF) + 1;
 621		rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0);
 622
 623		/* Alloc msix bitmap for this PF */
 624		err = rvu_alloc_bitmap(&pfvf->msix);
 625		if (err)
 626			return err;
 627
 628		/* Allocate memory for MSIX vector to RVU block LF mapping */
 629		pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max,
 630						sizeof(u16), GFP_KERNEL);
 631		if (!pfvf->msix_lfmap)
 632			return -ENOMEM;
 633
 634		/* For PF0 (AF) firmware will set msix vector offsets for
 635		 * AF, block AF and PF0_INT vectors, so jump to VFs.
 636		 */
 637		if (!pf)
 638			goto setup_vfmsix;
 639
 640		/* Set MSIX offset for PF's 'RVU_PF_INT_VEC' vectors.
 641		 * These are allocated on driver init and never freed,
 642		 * so no need to set 'msix_lfmap' for these.
 643		 */
 644		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf));
 645		nvecs = (cfg >> 12) & 0xFF;
 646		cfg &= ~0x7FFULL;
 647		offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
 648		rvu_write64(rvu, BLKADDR_RVUM,
 649			    RVU_PRIV_PFX_INT_CFG(pf), cfg | offset);
 650setup_vfmsix:
 651		/* Alloc msix bitmap for VFs */
 652		for (vf = 0; vf < numvfs; vf++) {
 653			pfvf =  &rvu->hwvf[hwvf + vf];
 654			/* Get num of MSIX vectors attached to this VF */
 655			cfg = rvu_read64(rvu, BLKADDR_RVUM,
 656					 RVU_PRIV_PFX_MSIX_CFG(pf));
 657			pfvf->msix.max = (cfg & 0xFFF) + 1;
 658			rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1);
 659
 660			/* Alloc msix bitmap for this VF */
 661			err = rvu_alloc_bitmap(&pfvf->msix);
 662			if (err)
 663				return err;
 664
 665			pfvf->msix_lfmap =
 666				devm_kcalloc(rvu->dev, pfvf->msix.max,
 667					     sizeof(u16), GFP_KERNEL);
 668			if (!pfvf->msix_lfmap)
 669				return -ENOMEM;
 670
 671			/* Set MSIX offset for HWVF's 'RVU_VF_INT_VEC' vectors.
 672			 * These are allocated on driver init and never freed,
 673			 * so no need to set 'msix_lfmap' for these.
 674			 */
 675			cfg = rvu_read64(rvu, BLKADDR_RVUM,
 676					 RVU_PRIV_HWVFX_INT_CFG(hwvf + vf));
 677			nvecs = (cfg >> 12) & 0xFF;
 678			cfg &= ~0x7FFULL;
 679			offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
 680			rvu_write64(rvu, BLKADDR_RVUM,
 681				    RVU_PRIV_HWVFX_INT_CFG(hwvf + vf),
 682				    cfg | offset);
 683		}
 684	}
 685
 686	/* HW interprets RVU_AF_MSIXTR_BASE address as an IOVA, hence
 687	 * create an IOMMU mapping for the physical address configured by
 688	 * firmware and reconfig RVU_AF_MSIXTR_BASE with IOVA.
 689	 */
 690	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 691	max_msix = cfg & 0xFFFFF;
 692	if (rvu->fwdata && rvu->fwdata->msixtr_base)
 693		phy_addr = rvu->fwdata->msixtr_base;
 694	else
 695		phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE);
 696
 697	iova = dma_map_resource(rvu->dev, phy_addr,
 698				max_msix * PCI_MSIX_ENTRY_SIZE,
 699				DMA_BIDIRECTIONAL, 0);
 700
 701	if (dma_mapping_error(rvu->dev, iova))
 702		return -ENOMEM;
 703
 704	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova);
 705	rvu->msix_base_iova = iova;
 706	rvu->msixtr_base_phy = phy_addr;
 707
 708	return 0;
 709}
 710
 711static void rvu_reset_msix(struct rvu *rvu)
 712{
 713	/* Restore msixtr base register */
 714	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE,
 715		    rvu->msixtr_base_phy);
 716}
 717
 718static void rvu_free_hw_resources(struct rvu *rvu)
 719{
 720	struct rvu_hwinfo *hw = rvu->hw;
 721	struct rvu_block *block;
 722	struct rvu_pfvf  *pfvf;
 723	int id, max_msix;
 724	u64 cfg;
 725
 726	rvu_npa_freemem(rvu);
 727	rvu_npc_freemem(rvu);
 728	rvu_nix_freemem(rvu);
 729
 730	/* Free block LF bitmaps */
 731	for (id = 0; id < BLK_COUNT; id++) {
 732		block = &hw->block[id];
 733		kfree(block->lf.bmap);
 734	}
 735
 736	/* Free MSIX bitmaps */
 737	for (id = 0; id < hw->total_pfs; id++) {
 738		pfvf = &rvu->pf[id];
 739		kfree(pfvf->msix.bmap);
 740	}
 741
 742	for (id = 0; id < hw->total_vfs; id++) {
 743		pfvf = &rvu->hwvf[id];
 744		kfree(pfvf->msix.bmap);
 745	}
 746
 747	/* Unmap MSIX vector base IOVA mapping */
 748	if (!rvu->msix_base_iova)
 749		return;
 750	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 751	max_msix = cfg & 0xFFFFF;
 752	dma_unmap_resource(rvu->dev, rvu->msix_base_iova,
 753			   max_msix * PCI_MSIX_ENTRY_SIZE,
 754			   DMA_BIDIRECTIONAL, 0);
 755
 756	rvu_reset_msix(rvu);
 757	mutex_destroy(&rvu->rsrc_lock);
 758
 759	/* Free the QINT/CINT memory */
 760	pfvf = &rvu->pf[RVU_AFPF];
 761	qmem_free(rvu->dev, pfvf->nix_qints_ctx);
 762	qmem_free(rvu->dev, pfvf->cq_ints_ctx);
 763}
 764
 765static void rvu_setup_pfvf_macaddress(struct rvu *rvu)
 766{
 767	struct rvu_hwinfo *hw = rvu->hw;
 768	int pf, vf, numvfs, hwvf;
 769	struct rvu_pfvf *pfvf;
 770	u64 *mac;
 771
 772	for (pf = 0; pf < hw->total_pfs; pf++) {
 773		/* For PF0(AF), Assign MAC address to only VFs (LBKVFs) */
 774		if (!pf)
 775			goto lbkvf;
 776
 777		if (!is_pf_cgxmapped(rvu, pf))
 778			continue;
 779		/* Assign MAC address to PF */
 780		pfvf = &rvu->pf[pf];
 781		if (rvu->fwdata && pf < PF_MACNUM_MAX) {
 782			mac = &rvu->fwdata->pf_macs[pf];
 783			if (*mac)
 784				u64_to_ether_addr(*mac, pfvf->mac_addr);
 785			else
 786				eth_random_addr(pfvf->mac_addr);
 787		} else {
 788			eth_random_addr(pfvf->mac_addr);
 789		}
 790		ether_addr_copy(pfvf->default_mac, pfvf->mac_addr);
 791
 792lbkvf:
 793		/* Assign MAC address to VFs*/
 794		rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
 795		for (vf = 0; vf < numvfs; vf++, hwvf++) {
 796			pfvf = &rvu->hwvf[hwvf];
 797			if (rvu->fwdata && hwvf < VF_MACNUM_MAX) {
 798				mac = &rvu->fwdata->vf_macs[hwvf];
 799				if (*mac)
 800					u64_to_ether_addr(*mac, pfvf->mac_addr);
 801				else
 802					eth_random_addr(pfvf->mac_addr);
 803			} else {
 804				eth_random_addr(pfvf->mac_addr);
 805			}
 806			ether_addr_copy(pfvf->default_mac, pfvf->mac_addr);
 807		}
 808	}
 809}
 810
 811static int rvu_fwdata_init(struct rvu *rvu)
 812{
 813	u64 fwdbase;
 814	int err;
 815
 816	/* Get firmware data base address */
 817	err = cgx_get_fwdata_base(&fwdbase);
 818	if (err)
 819		goto fail;
 820
 821	BUILD_BUG_ON(offsetof(struct rvu_fwdata, cgx_fw_data) > FWDATA_CGX_LMAC_OFFSET);
 822	rvu->fwdata = ioremap_wc(fwdbase, sizeof(struct rvu_fwdata));
 823	if (!rvu->fwdata)
 824		goto fail;
 825	if (!is_rvu_fwdata_valid(rvu)) {
 826		dev_err(rvu->dev,
 827			"Mismatch in 'fwdata' struct btw kernel and firmware\n");
 828		iounmap(rvu->fwdata);
 829		rvu->fwdata = NULL;
 830		return -EINVAL;
 831	}
 832	return 0;
 833fail:
 834	dev_info(rvu->dev, "Unable to fetch 'fwdata' from firmware\n");
 835	return -EIO;
 836}
 837
 838static void rvu_fwdata_exit(struct rvu *rvu)
 839{
 840	if (rvu->fwdata)
 841		iounmap(rvu->fwdata);
 842}
 843
 844static int rvu_setup_nix_hw_resource(struct rvu *rvu, int blkaddr)
 845{
 846	struct rvu_hwinfo *hw = rvu->hw;
 847	struct rvu_block *block;
 848	int blkid;
 849	u64 cfg;
 850
 851	/* Init NIX LF's bitmap */
 852	block = &hw->block[blkaddr];
 853	if (!block->implemented)
 854		return 0;
 855	blkid = (blkaddr == BLKADDR_NIX0) ? 0 : 1;
 856	cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
 857	block->lf.max = cfg & 0xFFF;
 858	block->addr = blkaddr;
 859	block->type = BLKTYPE_NIX;
 860	block->lfshift = 8;
 861	block->lookup_reg = NIX_AF_RVU_LF_CFG_DEBUG;
 862	block->pf_lfcnt_reg = RVU_PRIV_PFX_NIXX_CFG(blkid);
 863	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NIXX_CFG(blkid);
 864	block->lfcfg_reg = NIX_PRIV_LFX_CFG;
 865	block->msixcfg_reg = NIX_PRIV_LFX_INT_CFG;
 866	block->lfreset_reg = NIX_AF_LF_RST;
 867	block->rvu = rvu;
 868	sprintf(block->name, "NIX%d", blkid);
 869	rvu->nix_blkaddr[blkid] = blkaddr;
 870	return rvu_alloc_bitmap(&block->lf);
 871}
 872
 873static int rvu_setup_cpt_hw_resource(struct rvu *rvu, int blkaddr)
 874{
 875	struct rvu_hwinfo *hw = rvu->hw;
 876	struct rvu_block *block;
 877	int blkid;
 878	u64 cfg;
 879
 880	/* Init CPT LF's bitmap */
 881	block = &hw->block[blkaddr];
 882	if (!block->implemented)
 883		return 0;
 884	blkid = (blkaddr == BLKADDR_CPT0) ? 0 : 1;
 885	cfg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0);
 886	block->lf.max = cfg & 0xFF;
 887	block->addr = blkaddr;
 888	block->type = BLKTYPE_CPT;
 889	block->multislot = true;
 890	block->lfshift = 3;
 891	block->lookup_reg = CPT_AF_RVU_LF_CFG_DEBUG;
 892	block->pf_lfcnt_reg = RVU_PRIV_PFX_CPTX_CFG(blkid);
 893	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_CPTX_CFG(blkid);
 894	block->lfcfg_reg = CPT_PRIV_LFX_CFG;
 895	block->msixcfg_reg = CPT_PRIV_LFX_INT_CFG;
 896	block->lfreset_reg = CPT_AF_LF_RST;
 897	block->rvu = rvu;
 898	sprintf(block->name, "CPT%d", blkid);
 899	return rvu_alloc_bitmap(&block->lf);
 900}
 901
 902static void rvu_get_lbk_bufsize(struct rvu *rvu)
 903{
 904	struct pci_dev *pdev = NULL;
 905	void __iomem *base;
 906	u64 lbk_const;
 907
 908	pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
 909			      PCI_DEVID_OCTEONTX2_LBK, pdev);
 910	if (!pdev)
 911		return;
 912
 913	base = pci_ioremap_bar(pdev, 0);
 914	if (!base)
 915		goto err_put;
 916
 917	lbk_const = readq(base + LBK_CONST);
 918
 919	/* cache fifo size */
 920	rvu->hw->lbk_bufsize = FIELD_GET(LBK_CONST_BUF_SIZE, lbk_const);
 921
 922	iounmap(base);
 923err_put:
 924	pci_dev_put(pdev);
 925}
 926
 927static int rvu_setup_hw_resources(struct rvu *rvu)
 928{
 929	struct rvu_hwinfo *hw = rvu->hw;
 930	struct rvu_block *block;
 931	int blkid, err;
 932	u64 cfg;
 933
 934	/* Get HW supported max RVU PF & VF count */
 935	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 936	hw->total_pfs = (cfg >> 32) & 0xFF;
 937	hw->total_vfs = (cfg >> 20) & 0xFFF;
 938	hw->max_vfs_per_pf = (cfg >> 40) & 0xFF;
 939
 940	if (!is_rvu_otx2(rvu))
 941		rvu_apr_block_cn10k_init(rvu);
 942
 943	/* Init NPA LF's bitmap */
 944	block = &hw->block[BLKADDR_NPA];
 945	if (!block->implemented)
 946		goto nix;
 947	cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST);
 948	block->lf.max = (cfg >> 16) & 0xFFF;
 949	block->addr = BLKADDR_NPA;
 950	block->type = BLKTYPE_NPA;
 951	block->lfshift = 8;
 952	block->lookup_reg = NPA_AF_RVU_LF_CFG_DEBUG;
 953	block->pf_lfcnt_reg = RVU_PRIV_PFX_NPA_CFG;
 954	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NPA_CFG;
 955	block->lfcfg_reg = NPA_PRIV_LFX_CFG;
 956	block->msixcfg_reg = NPA_PRIV_LFX_INT_CFG;
 957	block->lfreset_reg = NPA_AF_LF_RST;
 958	block->rvu = rvu;
 959	sprintf(block->name, "NPA");
 960	err = rvu_alloc_bitmap(&block->lf);
 961	if (err) {
 962		dev_err(rvu->dev,
 963			"%s: Failed to allocate NPA LF bitmap\n", __func__);
 964		return err;
 965	}
 966
 967nix:
 968	err = rvu_setup_nix_hw_resource(rvu, BLKADDR_NIX0);
 969	if (err) {
 970		dev_err(rvu->dev,
 971			"%s: Failed to allocate NIX0 LFs bitmap\n", __func__);
 972		return err;
 973	}
 974
 975	err = rvu_setup_nix_hw_resource(rvu, BLKADDR_NIX1);
 976	if (err) {
 977		dev_err(rvu->dev,
 978			"%s: Failed to allocate NIX1 LFs bitmap\n", __func__);
 979		return err;
 980	}
 981
 982	/* Init SSO group's bitmap */
 983	block = &hw->block[BLKADDR_SSO];
 984	if (!block->implemented)
 985		goto ssow;
 986	cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST);
 987	block->lf.max = cfg & 0xFFFF;
 988	block->addr = BLKADDR_SSO;
 989	block->type = BLKTYPE_SSO;
 990	block->multislot = true;
 991	block->lfshift = 3;
 992	block->lookup_reg = SSO_AF_RVU_LF_CFG_DEBUG;
 993	block->pf_lfcnt_reg = RVU_PRIV_PFX_SSO_CFG;
 994	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSO_CFG;
 995	block->lfcfg_reg = SSO_PRIV_LFX_HWGRP_CFG;
 996	block->msixcfg_reg = SSO_PRIV_LFX_HWGRP_INT_CFG;
 997	block->lfreset_reg = SSO_AF_LF_HWGRP_RST;
 998	block->rvu = rvu;
 999	sprintf(block->name, "SSO GROUP");
1000	err = rvu_alloc_bitmap(&block->lf);
1001	if (err) {
1002		dev_err(rvu->dev,
1003			"%s: Failed to allocate SSO LF bitmap\n", __func__);
1004		return err;
1005	}
1006
1007ssow:
1008	/* Init SSO workslot's bitmap */
1009	block = &hw->block[BLKADDR_SSOW];
1010	if (!block->implemented)
1011		goto tim;
1012	block->lf.max = (cfg >> 56) & 0xFF;
1013	block->addr = BLKADDR_SSOW;
1014	block->type = BLKTYPE_SSOW;
1015	block->multislot = true;
1016	block->lfshift = 3;
1017	block->lookup_reg = SSOW_AF_RVU_LF_HWS_CFG_DEBUG;
1018	block->pf_lfcnt_reg = RVU_PRIV_PFX_SSOW_CFG;
1019	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSOW_CFG;
1020	block->lfcfg_reg = SSOW_PRIV_LFX_HWS_CFG;
1021	block->msixcfg_reg = SSOW_PRIV_LFX_HWS_INT_CFG;
1022	block->lfreset_reg = SSOW_AF_LF_HWS_RST;
1023	block->rvu = rvu;
1024	sprintf(block->name, "SSOWS");
1025	err = rvu_alloc_bitmap(&block->lf);
1026	if (err) {
1027		dev_err(rvu->dev,
1028			"%s: Failed to allocate SSOW LF bitmap\n", __func__);
1029		return err;
1030	}
1031
1032tim:
1033	/* Init TIM LF's bitmap */
1034	block = &hw->block[BLKADDR_TIM];
1035	if (!block->implemented)
1036		goto cpt;
1037	cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST);
1038	block->lf.max = cfg & 0xFFFF;
1039	block->addr = BLKADDR_TIM;
1040	block->type = BLKTYPE_TIM;
1041	block->multislot = true;
1042	block->lfshift = 3;
1043	block->lookup_reg = TIM_AF_RVU_LF_CFG_DEBUG;
1044	block->pf_lfcnt_reg = RVU_PRIV_PFX_TIM_CFG;
1045	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_TIM_CFG;
1046	block->lfcfg_reg = TIM_PRIV_LFX_CFG;
1047	block->msixcfg_reg = TIM_PRIV_LFX_INT_CFG;
1048	block->lfreset_reg = TIM_AF_LF_RST;
1049	block->rvu = rvu;
1050	sprintf(block->name, "TIM");
1051	err = rvu_alloc_bitmap(&block->lf);
1052	if (err) {
1053		dev_err(rvu->dev,
1054			"%s: Failed to allocate TIM LF bitmap\n", __func__);
1055		return err;
1056	}
1057
1058cpt:
1059	err = rvu_setup_cpt_hw_resource(rvu, BLKADDR_CPT0);
1060	if (err) {
1061		dev_err(rvu->dev,
1062			"%s: Failed to allocate CPT0 LF bitmap\n", __func__);
1063		return err;
1064	}
1065	err = rvu_setup_cpt_hw_resource(rvu, BLKADDR_CPT1);
1066	if (err) {
1067		dev_err(rvu->dev,
1068			"%s: Failed to allocate CPT1 LF bitmap\n", __func__);
1069		return err;
1070	}
1071
1072	/* Allocate memory for PFVF data */
1073	rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs,
1074			       sizeof(struct rvu_pfvf), GFP_KERNEL);
1075	if (!rvu->pf) {
1076		dev_err(rvu->dev,
1077			"%s: Failed to allocate memory for PF's rvu_pfvf struct\n", __func__);
1078		return -ENOMEM;
1079	}
1080
1081	rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs,
1082				 sizeof(struct rvu_pfvf), GFP_KERNEL);
1083	if (!rvu->hwvf) {
1084		dev_err(rvu->dev,
1085			"%s: Failed to allocate memory for VF's rvu_pfvf struct\n", __func__);
1086		return -ENOMEM;
1087	}
1088
1089	mutex_init(&rvu->rsrc_lock);
1090
1091	rvu_fwdata_init(rvu);
1092
1093	err = rvu_setup_msix_resources(rvu);
1094	if (err) {
1095		dev_err(rvu->dev,
1096			"%s: Failed to setup MSIX resources\n", __func__);
1097		return err;
1098	}
1099
1100	for (blkid = 0; blkid < BLK_COUNT; blkid++) {
1101		block = &hw->block[blkid];
1102		if (!block->lf.bmap)
1103			continue;
1104
1105		/* Allocate memory for block LF/slot to pcifunc mapping info */
1106		block->fn_map = devm_kcalloc(rvu->dev, block->lf.max,
1107					     sizeof(u16), GFP_KERNEL);
1108		if (!block->fn_map) {
1109			err = -ENOMEM;
1110			goto msix_err;
1111		}
1112
1113		/* Scan all blocks to check if low level firmware has
1114		 * already provisioned any of the resources to a PF/VF.
1115		 */
1116		rvu_scan_block(rvu, block);
1117	}
1118
1119	err = rvu_set_channels_base(rvu);
1120	if (err)
1121		goto msix_err;
1122
1123	err = rvu_npc_init(rvu);
1124	if (err) {
1125		dev_err(rvu->dev, "%s: Failed to initialize npc\n", __func__);
1126		goto npc_err;
1127	}
1128
1129	err = rvu_cgx_init(rvu);
1130	if (err) {
1131		dev_err(rvu->dev, "%s: Failed to initialize cgx\n", __func__);
1132		goto cgx_err;
1133	}
1134
1135	err = rvu_npc_exact_init(rvu);
1136	if (err) {
1137		dev_err(rvu->dev, "failed to initialize exact match table\n");
1138		return err;
1139	}
1140
1141	/* Assign MACs for CGX mapped functions */
1142	rvu_setup_pfvf_macaddress(rvu);
1143
1144	err = rvu_npa_init(rvu);
1145	if (err) {
1146		dev_err(rvu->dev, "%s: Failed to initialize npa\n", __func__);
1147		goto npa_err;
1148	}
1149
1150	rvu_get_lbk_bufsize(rvu);
1151
1152	err = rvu_nix_init(rvu);
1153	if (err) {
1154		dev_err(rvu->dev, "%s: Failed to initialize nix\n", __func__);
1155		goto nix_err;
1156	}
1157
1158	err = rvu_sdp_init(rvu);
1159	if (err) {
1160		dev_err(rvu->dev, "%s: Failed to initialize sdp\n", __func__);
1161		goto nix_err;
1162	}
1163
1164	rvu_program_channels(rvu);
1165	cgx_start_linkup(rvu);
1166
1167	rvu_block_bcast_xon(rvu, BLKADDR_NIX0);
1168	rvu_block_bcast_xon(rvu, BLKADDR_NIX1);
1169
1170	err = rvu_mcs_init(rvu);
1171	if (err) {
1172		dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__);
1173		goto nix_err;
1174	}
1175
1176	err = rvu_cpt_init(rvu);
1177	if (err) {
1178		dev_err(rvu->dev, "%s: Failed to initialize cpt\n", __func__);
1179		goto mcs_err;
1180	}
1181
1182	return 0;
1183
1184mcs_err:
1185	rvu_mcs_exit(rvu);
1186nix_err:
1187	rvu_nix_freemem(rvu);
1188npa_err:
1189	rvu_npa_freemem(rvu);
1190cgx_err:
1191	rvu_cgx_exit(rvu);
1192npc_err:
1193	rvu_npc_freemem(rvu);
1194	rvu_fwdata_exit(rvu);
1195msix_err:
1196	rvu_reset_msix(rvu);
1197	return err;
1198}
1199
1200/* NPA and NIX admin queue APIs */
1201void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq)
1202{
1203	if (!aq)
1204		return;
1205
1206	qmem_free(rvu->dev, aq->inst);
1207	qmem_free(rvu->dev, aq->res);
1208	devm_kfree(rvu->dev, aq);
1209}
1210
1211int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue,
1212		 int qsize, int inst_size, int res_size)
1213{
1214	struct admin_queue *aq;
1215	int err;
1216
1217	*ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL);
1218	if (!*ad_queue)
1219		return -ENOMEM;
1220	aq = *ad_queue;
1221
1222	/* Alloc memory for instructions i.e AQ */
1223	err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size);
1224	if (err) {
1225		devm_kfree(rvu->dev, aq);
1226		return err;
1227	}
1228
1229	/* Alloc memory for results */
1230	err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size);
1231	if (err) {
1232		rvu_aq_free(rvu, aq);
1233		return err;
1234	}
1235
1236	spin_lock_init(&aq->lock);
1237	return 0;
1238}
1239
1240int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req,
1241			   struct ready_msg_rsp *rsp)
1242{
1243	if (rvu->fwdata) {
1244		rsp->rclk_freq = rvu->fwdata->rclk;
1245		rsp->sclk_freq = rvu->fwdata->sclk;
1246	}
1247	return 0;
1248}
1249
1250/* Get current count of a RVU block's LF/slots
1251 * provisioned to a given RVU func.
1252 */
1253u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blkaddr)
1254{
1255	switch (blkaddr) {
1256	case BLKADDR_NPA:
1257		return pfvf->npalf ? 1 : 0;
1258	case BLKADDR_NIX0:
1259	case BLKADDR_NIX1:
1260		return pfvf->nixlf ? 1 : 0;
1261	case BLKADDR_SSO:
1262		return pfvf->sso;
1263	case BLKADDR_SSOW:
1264		return pfvf->ssow;
1265	case BLKADDR_TIM:
1266		return pfvf->timlfs;
1267	case BLKADDR_CPT0:
1268		return pfvf->cptlfs;
1269	case BLKADDR_CPT1:
1270		return pfvf->cpt1_lfs;
1271	}
1272	return 0;
1273}
1274
1275/* Return true if LFs of block type are attached to pcifunc */
1276static bool is_blktype_attached(struct rvu_pfvf *pfvf, int blktype)
1277{
1278	switch (blktype) {
1279	case BLKTYPE_NPA:
1280		return pfvf->npalf ? 1 : 0;
1281	case BLKTYPE_NIX:
1282		return pfvf->nixlf ? 1 : 0;
1283	case BLKTYPE_SSO:
1284		return !!pfvf->sso;
1285	case BLKTYPE_SSOW:
1286		return !!pfvf->ssow;
1287	case BLKTYPE_TIM:
1288		return !!pfvf->timlfs;
1289	case BLKTYPE_CPT:
1290		return pfvf->cptlfs || pfvf->cpt1_lfs;
1291	}
1292
1293	return false;
1294}
1295
1296bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype)
1297{
1298	struct rvu_pfvf *pfvf;
1299
1300	if (!is_pf_func_valid(rvu, pcifunc))
1301		return false;
1302
1303	pfvf = rvu_get_pfvf(rvu, pcifunc);
1304
1305	/* Check if this PFFUNC has a LF of type blktype attached */
1306	if (!is_blktype_attached(pfvf, blktype))
1307		return false;
1308
1309	return true;
1310}
1311
1312static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block,
1313			   int pcifunc, int slot)
1314{
1315	u64 val;
1316
1317	val = ((u64)pcifunc << 24) | (slot << 16) | (1ULL << 13);
1318	rvu_write64(rvu, block->addr, block->lookup_reg, val);
1319	/* Wait for the lookup to finish */
1320	/* TODO: put some timeout here */
1321	while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13))
1322		;
1323
1324	val = rvu_read64(rvu, block->addr, block->lookup_reg);
1325
1326	/* Check LF valid bit */
1327	if (!(val & (1ULL << 12)))
1328		return -1;
1329
1330	return (val & 0xFFF);
1331}
1332
1333int rvu_get_blkaddr_from_slot(struct rvu *rvu, int blktype, u16 pcifunc,
1334			      u16 global_slot, u16 *slot_in_block)
1335{
1336	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1337	int numlfs, total_lfs = 0, nr_blocks = 0;
1338	int i, num_blkaddr[BLK_COUNT] = { 0 };
1339	struct rvu_block *block;
1340	int blkaddr;
1341	u16 start_slot;
1342
1343	if (!is_blktype_attached(pfvf, blktype))
1344		return -ENODEV;
1345
1346	/* Get all the block addresses from which LFs are attached to
1347	 * the given pcifunc in num_blkaddr[].
1348	 */
1349	for (blkaddr = BLKADDR_RVUM; blkaddr < BLK_COUNT; blkaddr++) {
1350		block = &rvu->hw->block[blkaddr];
1351		if (block->type != blktype)
1352			continue;
1353		if (!is_block_implemented(rvu->hw, blkaddr))
1354			continue;
1355
1356		numlfs = rvu_get_rsrc_mapcount(pfvf, blkaddr);
1357		if (numlfs) {
1358			total_lfs += numlfs;
1359			num_blkaddr[nr_blocks] = blkaddr;
1360			nr_blocks++;
1361		}
1362	}
1363
1364	if (global_slot >= total_lfs)
1365		return -ENODEV;
1366
1367	/* Based on the given global slot number retrieve the
1368	 * correct block address out of all attached block
1369	 * addresses and slot number in that block.
1370	 */
1371	total_lfs = 0;
1372	blkaddr = -ENODEV;
1373	for (i = 0; i < nr_blocks; i++) {
1374		numlfs = rvu_get_rsrc_mapcount(pfvf, num_blkaddr[i]);
1375		total_lfs += numlfs;
1376		if (global_slot < total_lfs) {
1377			blkaddr = num_blkaddr[i];
1378			start_slot = total_lfs - numlfs;
1379			*slot_in_block = global_slot - start_slot;
1380			break;
1381		}
1382	}
1383
1384	return blkaddr;
1385}
1386
1387static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)
1388{
1389	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1390	struct rvu_hwinfo *hw = rvu->hw;
1391	struct rvu_block *block;
1392	int slot, lf, num_lfs;
1393	int blkaddr;
1394
1395	blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc);
1396	if (blkaddr < 0)
1397		return;
1398
1399
1400	block = &hw->block[blkaddr];
1401
1402	num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
1403	if (!num_lfs)
1404		return;
1405
1406	for (slot = 0; slot < num_lfs; slot++) {
1407		lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot);
1408		if (lf < 0) /* This should never happen */
1409			continue;
1410
1411		if (blktype == BLKTYPE_NIX) {
1412			rvu_nix_reset_mac(pfvf, pcifunc);
1413			rvu_npc_clear_ucast_entry(rvu, pcifunc, lf);
1414		}
1415		/* Disable the LF */
1416		rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1417			    (lf << block->lfshift), 0x00ULL);
1418
1419		/* Update SW maintained mapping info as well */
1420		rvu_update_rsrc_map(rvu, pfvf, block,
1421				    pcifunc, lf, false);
1422
1423		/* Free the resource */
1424		rvu_free_rsrc(&block->lf, lf);
1425
1426		/* Clear MSIX vector offset for this LF */
1427		rvu_clear_msix_offset(rvu, pfvf, block, lf);
1428	}
1429}
1430
1431static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,
1432			    u16 pcifunc)
1433{
1434	struct rvu_hwinfo *hw = rvu->hw;
1435	bool detach_all = true;
1436	struct rvu_block *block;
1437	int blkid;
1438
1439	mutex_lock(&rvu->rsrc_lock);
1440
1441	/* Check for partial resource detach */
1442	if (detach && detach->partial)
1443		detach_all = false;
1444
1445	/* Check for RVU block's LFs attached to this func,
1446	 * if so, detach them.
1447	 */
1448	for (blkid = 0; blkid < BLK_COUNT; blkid++) {
1449		block = &hw->block[blkid];
1450		if (!block->lf.bmap)
1451			continue;
1452		if (!detach_all && detach) {
1453			if (blkid == BLKADDR_NPA && !detach->npalf)
1454				continue;
1455			else if ((blkid == BLKADDR_NIX0) && !detach->nixlf)
1456				continue;
1457			else if ((blkid == BLKADDR_NIX1) && !detach->nixlf)
1458				continue;
1459			else if ((blkid == BLKADDR_SSO) && !detach->sso)
1460				continue;
1461			else if ((blkid == BLKADDR_SSOW) && !detach->ssow)
1462				continue;
1463			else if ((blkid == BLKADDR_TIM) && !detach->timlfs)
1464				continue;
1465			else if ((blkid == BLKADDR_CPT0) && !detach->cptlfs)
1466				continue;
1467			else if ((blkid == BLKADDR_CPT1) && !detach->cptlfs)
1468				continue;
1469		}
1470		rvu_detach_block(rvu, pcifunc, block->type);
1471	}
1472
1473	mutex_unlock(&rvu->rsrc_lock);
1474	return 0;
1475}
1476
1477int rvu_mbox_handler_detach_resources(struct rvu *rvu,
1478				      struct rsrc_detach *detach,
1479				      struct msg_rsp *rsp)
1480{
1481	return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc);
1482}
1483
1484int rvu_get_nix_blkaddr(struct rvu *rvu, u16 pcifunc)
1485{
1486	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1487	int blkaddr = BLKADDR_NIX0, vf;
1488	struct rvu_pfvf *pf;
1489
1490	pf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
1491
1492	/* All CGX mapped PFs are set with assigned NIX block during init */
1493	if (is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc))) {
1494		blkaddr = pf->nix_blkaddr;
1495	} else if (is_lbk_vf(rvu, pcifunc)) {
1496		vf = pcifunc - 1;
1497		/* Assign NIX based on VF number. All even numbered VFs get
1498		 * NIX0 and odd numbered gets NIX1
1499		 */
1500		blkaddr = (vf & 1) ? BLKADDR_NIX1 : BLKADDR_NIX0;
1501		/* NIX1 is not present on all silicons */
1502		if (!is_block_implemented(rvu->hw, BLKADDR_NIX1))
1503			blkaddr = BLKADDR_NIX0;
1504	}
1505
1506	/* if SDP1 then the blkaddr is NIX1 */
1507	if (is_sdp_pfvf(rvu, pcifunc) && pf->sdp_info->node_id == 1)
1508		blkaddr = BLKADDR_NIX1;
1509
1510	switch (blkaddr) {
1511	case BLKADDR_NIX1:
1512		pfvf->nix_blkaddr = BLKADDR_NIX1;
1513		pfvf->nix_rx_intf = NIX_INTFX_RX(1);
1514		pfvf->nix_tx_intf = NIX_INTFX_TX(1);
1515		break;
1516	case BLKADDR_NIX0:
1517	default:
1518		pfvf->nix_blkaddr = BLKADDR_NIX0;
1519		pfvf->nix_rx_intf = NIX_INTFX_RX(0);
1520		pfvf->nix_tx_intf = NIX_INTFX_TX(0);
1521		break;
1522	}
1523
1524	return pfvf->nix_blkaddr;
1525}
1526
1527static int rvu_get_attach_blkaddr(struct rvu *rvu, int blktype,
1528				  u16 pcifunc, struct rsrc_attach *attach)
1529{
1530	int blkaddr;
1531
1532	switch (blktype) {
1533	case BLKTYPE_NIX:
1534		blkaddr = rvu_get_nix_blkaddr(rvu, pcifunc);
1535		break;
1536	case BLKTYPE_CPT:
1537		if (attach->hdr.ver < RVU_MULTI_BLK_VER)
1538			return rvu_get_blkaddr(rvu, blktype, 0);
1539		blkaddr = attach->cpt_blkaddr ? attach->cpt_blkaddr :
1540			  BLKADDR_CPT0;
1541		if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1)
1542			return -ENODEV;
1543		break;
1544	default:
1545		return rvu_get_blkaddr(rvu, blktype, 0);
1546	}
1547
1548	if (is_block_implemented(rvu->hw, blkaddr))
1549		return blkaddr;
1550
1551	return -ENODEV;
1552}
1553
1554static int rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype,
1555			    int num_lfs, struct rsrc_attach *attach)
1556{
1557	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1558	struct rvu_hwinfo *hw = rvu->hw;
1559	struct rvu_block *block;
1560	int slot, lf;
1561	int blkaddr;
1562	u64 cfg;
1563
1564	if (!num_lfs)
1565		return -EINVAL;
1566
1567	blkaddr = rvu_get_attach_blkaddr(rvu, blktype, pcifunc, attach);
1568	if (blkaddr < 0)
1569		return -EFAULT;
1570
1571	block = &hw->block[blkaddr];
1572	if (!block->lf.bmap)
1573		return -ESRCH;
1574
1575	for (slot = 0; slot < num_lfs; slot++) {
1576		/* Allocate the resource */
1577		lf = rvu_alloc_rsrc(&block->lf);
1578		if (lf < 0)
1579			return -EFAULT;
1580
1581		cfg = (1ULL << 63) | (pcifunc << 8) | slot;
1582		rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1583			    (lf << block->lfshift), cfg);
1584		rvu_update_rsrc_map(rvu, pfvf, block,
1585				    pcifunc, lf, true);
1586
1587		/* Set start MSIX vector for this LF within this PF/VF */
1588		rvu_set_msix_offset(rvu, pfvf, block, lf);
1589	}
1590
1591	return 0;
1592}
1593
1594static int rvu_check_rsrc_availability(struct rvu *rvu,
1595				       struct rsrc_attach *req, u16 pcifunc)
1596{
1597	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1598	int free_lfs, mappedlfs, blkaddr;
1599	struct rvu_hwinfo *hw = rvu->hw;
1600	struct rvu_block *block;
1601
1602	/* Only one NPA LF can be attached */
1603	if (req->npalf && !is_blktype_attached(pfvf, BLKTYPE_NPA)) {
1604		block = &hw->block[BLKADDR_NPA];
1605		free_lfs = rvu_rsrc_free_count(&block->lf);
1606		if (!free_lfs)
1607			goto fail;
1608	} else if (req->npalf) {
1609		dev_err(&rvu->pdev->dev,
1610			"Func 0x%x: Invalid req, already has NPA\n",
1611			 pcifunc);
1612		return -EINVAL;
1613	}
1614
1615	/* Only one NIX LF can be attached */
1616	if (req->nixlf && !is_blktype_attached(pfvf, BLKTYPE_NIX)) {
1617		blkaddr = rvu_get_attach_blkaddr(rvu, BLKTYPE_NIX,
1618						 pcifunc, req);
1619		if (blkaddr < 0)
1620			return blkaddr;
1621		block = &hw->block[blkaddr];
1622		free_lfs = rvu_rsrc_free_count(&block->lf);
1623		if (!free_lfs)
1624			goto fail;
1625	} else if (req->nixlf) {
1626		dev_err(&rvu->pdev->dev,
1627			"Func 0x%x: Invalid req, already has NIX\n",
1628			pcifunc);
1629		return -EINVAL;
1630	}
1631
1632	if (req->sso) {
1633		block = &hw->block[BLKADDR_SSO];
1634		/* Is request within limits ? */
1635		if (req->sso > block->lf.max) {
1636			dev_err(&rvu->pdev->dev,
1637				"Func 0x%x: Invalid SSO req, %d > max %d\n",
1638				 pcifunc, req->sso, block->lf.max);
1639			return -EINVAL;
1640		}
1641		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
1642		free_lfs = rvu_rsrc_free_count(&block->lf);
1643		/* Check if additional resources are available */
1644		if (req->sso > mappedlfs &&
1645		    ((req->sso - mappedlfs) > free_lfs))
1646			goto fail;
1647	}
1648
1649	if (req->ssow) {
1650		block = &hw->block[BLKADDR_SSOW];
1651		if (req->ssow > block->lf.max) {
1652			dev_err(&rvu->pdev->dev,
1653				"Func 0x%x: Invalid SSOW req, %d > max %d\n",
1654				 pcifunc, req->ssow, block->lf.max);
1655			return -EINVAL;
1656		}
1657		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
1658		free_lfs = rvu_rsrc_free_count(&block->lf);
1659		if (req->ssow > mappedlfs &&
1660		    ((req->ssow - mappedlfs) > free_lfs))
1661			goto fail;
1662	}
1663
1664	if (req->timlfs) {
1665		block = &hw->block[BLKADDR_TIM];
1666		if (req->timlfs > block->lf.max) {
1667			dev_err(&rvu->pdev->dev,
1668				"Func 0x%x: Invalid TIMLF req, %d > max %d\n",
1669				 pcifunc, req->timlfs, block->lf.max);
1670			return -EINVAL;
1671		}
1672		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
1673		free_lfs = rvu_rsrc_free_count(&block->lf);
1674		if (req->timlfs > mappedlfs &&
1675		    ((req->timlfs - mappedlfs) > free_lfs))
1676			goto fail;
1677	}
1678
1679	if (req->cptlfs) {
1680		blkaddr = rvu_get_attach_blkaddr(rvu, BLKTYPE_CPT,
1681						 pcifunc, req);
1682		if (blkaddr < 0)
1683			return blkaddr;
1684		block = &hw->block[blkaddr];
1685		if (req->cptlfs > block->lf.max) {
1686			dev_err(&rvu->pdev->dev,
1687				"Func 0x%x: Invalid CPTLF req, %d > max %d\n",
1688				 pcifunc, req->cptlfs, block->lf.max);
1689			return -EINVAL;
1690		}
1691		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
1692		free_lfs = rvu_rsrc_free_count(&block->lf);
1693		if (req->cptlfs > mappedlfs &&
1694		    ((req->cptlfs - mappedlfs) > free_lfs))
1695			goto fail;
1696	}
1697
1698	return 0;
1699
1700fail:
1701	dev_info(rvu->dev, "Request for %s failed\n", block->name);
1702	return -ENOSPC;
1703}
1704
1705static bool rvu_attach_from_same_block(struct rvu *rvu, int blktype,
1706				       struct rsrc_attach *attach)
1707{
1708	int blkaddr, num_lfs;
1709
1710	blkaddr = rvu_get_attach_blkaddr(rvu, blktype,
1711					 attach->hdr.pcifunc, attach);
1712	if (blkaddr < 0)
1713		return false;
1714
1715	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, attach->hdr.pcifunc),
1716					blkaddr);
1717	/* Requester already has LFs from given block ? */
1718	return !!num_lfs;
1719}
1720
1721int rvu_mbox_handler_attach_resources(struct rvu *rvu,
1722				      struct rsrc_attach *attach,
1723				      struct msg_rsp *rsp)
1724{
1725	u16 pcifunc = attach->hdr.pcifunc;
1726	int err;
1727
1728	/* If first request, detach all existing attached resources */
1729	if (!attach->modify) {
1730		err = rvu_detach_rsrcs(rvu, NULL, pcifunc);
1731		if (err)
1732			return err;
1733	}
1734
1735	mutex_lock(&rvu->rsrc_lock);
1736
1737	/* Check if the request can be accommodated */
1738	err = rvu_check_rsrc_availability(rvu, attach, pcifunc);
1739	if (err)
1740		goto fail1;
1741
1742	/* Now attach the requested resources */
1743	if (attach->npalf) {
1744		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1, attach);
1745		if (err)
1746			goto fail1;
1747	}
1748
1749	if (attach->nixlf) {
1750		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach);
1751		if (err)
1752			goto fail2;
1753	}
1754
1755	if (attach->sso) {
1756		/* RVU func doesn't know which exact LF or slot is attached
1757		 * to it, it always sees as slot 0,1,2. So for a 'modify'
1758		 * request, simply detach all existing attached LFs/slots
1759		 * and attach a fresh.
1760		 */
1761		if (attach->modify)
1762			rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
1763		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO,
1764				       attach->sso, attach);
1765		if (err)
1766			goto fail3;
1767	}
1768
1769	if (attach->ssow) {
1770		if (attach->modify)
1771			rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
1772		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW,
1773				       attach->ssow, attach);
1774		if (err)
1775			goto fail4;
1776	}
1777
1778	if (attach->timlfs) {
1779		if (attach->modify)
1780			rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
1781		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM,
1782				       attach->timlfs, attach);
1783		if (err)
1784			goto fail5;
1785	}
1786
1787	if (attach->cptlfs) {
1788		if (attach->modify &&
1789		    rvu_attach_from_same_block(rvu, BLKTYPE_CPT, attach))
1790			rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT);
1791		err = rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT,
1792				       attach->cptlfs, attach);
1793		if (err)
1794			goto fail6;
1795	}
1796
1797	mutex_unlock(&rvu->rsrc_lock);
1798	return 0;
1799
1800fail6:
1801	if (attach->timlfs)
1802		rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
1803
1804fail5:
1805	if (attach->ssow)
1806		rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
1807
1808fail4:
1809	if (attach->sso)
1810		rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
1811
1812fail3:
1813	if (attach->nixlf)
1814		rvu_detach_block(rvu, pcifunc, BLKTYPE_NIX);
1815
1816fail2:
1817	if (attach->npalf)
1818		rvu_detach_block(rvu, pcifunc, BLKTYPE_NPA);
1819
1820fail1:
1821	mutex_unlock(&rvu->rsrc_lock);
1822	return err;
1823}
1824
1825static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1826			       int blkaddr, int lf)
1827{
1828	u16 vec;
1829
1830	if (lf < 0)
1831		return MSIX_VECTOR_INVALID;
1832
1833	for (vec = 0; vec < pfvf->msix.max; vec++) {
1834		if (pfvf->msix_lfmap[vec] == MSIX_BLKLF(blkaddr, lf))
1835			return vec;
1836	}
1837	return MSIX_VECTOR_INVALID;
1838}
1839
1840static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1841				struct rvu_block *block, int lf)
1842{
1843	u16 nvecs, vec, offset;
1844	u64 cfg;
1845
1846	cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1847			 (lf << block->lfshift));
1848	nvecs = (cfg >> 12) & 0xFF;
1849
1850	/* Check and alloc MSIX vectors, must be contiguous */
1851	if (!rvu_rsrc_check_contig(&pfvf->msix, nvecs))
1852		return;
1853
1854	offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
1855
1856	/* Config MSIX offset in LF */
1857	rvu_write64(rvu, block->addr, block->msixcfg_reg |
1858		    (lf << block->lfshift), (cfg & ~0x7FFULL) | offset);
1859
1860	/* Update the bitmap as well */
1861	for (vec = 0; vec < nvecs; vec++)
1862		pfvf->msix_lfmap[offset + vec] = MSIX_BLKLF(block->addr, lf);
1863}
1864
1865static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1866				  struct rvu_block *block, int lf)
1867{
1868	u16 nvecs, vec, offset;
1869	u64 cfg;
1870
1871	cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1872			 (lf << block->lfshift));
1873	nvecs = (cfg >> 12) & 0xFF;
1874
1875	/* Clear MSIX offset in LF */
1876	rvu_write64(rvu, block->addr, block->msixcfg_reg |
1877		    (lf << block->lfshift), cfg & ~0x7FFULL);
1878
1879	offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf);
1880
1881	/* Update the mapping */
1882	for (vec = 0; vec < nvecs; vec++)
1883		pfvf->msix_lfmap[offset + vec] = 0;
1884
1885	/* Free the same in MSIX bitmap */
1886	rvu_free_rsrc_contig(&pfvf->msix, nvecs, offset);
1887}
1888
1889int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req,
1890				 struct msix_offset_rsp *rsp)
1891{
1892	struct rvu_hwinfo *hw = rvu->hw;
1893	u16 pcifunc = req->hdr.pcifunc;
1894	struct rvu_pfvf *pfvf;
1895	int lf, slot, blkaddr;
1896
1897	pfvf = rvu_get_pfvf(rvu, pcifunc);
1898	if (!pfvf->msix.bmap)
1899		return 0;
1900
1901	/* Set MSIX offsets for each block's LFs attached to this PF/VF */
1902	lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0);
1903	rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf);
1904
1905	/* Get BLKADDR from which LFs are attached to pcifunc */
1906	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1907	if (blkaddr < 0) {
1908		rsp->nix_msixoff = MSIX_VECTOR_INVALID;
1909	} else {
1910		lf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
1911		rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, blkaddr, lf);
1912	}
1913
1914	rsp->sso = pfvf->sso;
1915	for (slot = 0; slot < rsp->sso; slot++) {
1916		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot);
1917		rsp->sso_msixoff[slot] =
1918			rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf);
1919	}
1920
1921	rsp->ssow = pfvf->ssow;
1922	for (slot = 0; slot < rsp->ssow; slot++) {
1923		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot);
1924		rsp->ssow_msixoff[slot] =
1925			rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf);
1926	}
1927
1928	rsp->timlfs = pfvf->timlfs;
1929	for (slot = 0; slot < rsp->timlfs; slot++) {
1930		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot);
1931		rsp->timlf_msixoff[slot] =
1932			rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf);
1933	}
1934
1935	rsp->cptlfs = pfvf->cptlfs;
1936	for (slot = 0; slot < rsp->cptlfs; slot++) {
1937		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot);
1938		rsp->cptlf_msixoff[slot] =
1939			rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf);
1940	}
1941
1942	rsp->cpt1_lfs = pfvf->cpt1_lfs;
1943	for (slot = 0; slot < rsp->cpt1_lfs; slot++) {
1944		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT1], pcifunc, slot);
1945		rsp->cpt1_lf_msixoff[slot] =
1946			rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT1, lf);
1947	}
1948
1949	return 0;
1950}
1951
1952int rvu_mbox_handler_free_rsrc_cnt(struct rvu *rvu, struct msg_req *req,
1953				   struct free_rsrcs_rsp *rsp)
1954{
1955	struct rvu_hwinfo *hw = rvu->hw;
1956	struct rvu_block *block;
1957	struct nix_txsch *txsch;
1958	struct nix_hw *nix_hw;
1959
1960	mutex_lock(&rvu->rsrc_lock);
1961
1962	block = &hw->block[BLKADDR_NPA];
1963	rsp->npa = rvu_rsrc_free_count(&block->lf);
1964
1965	block = &hw->block[BLKADDR_NIX0];
1966	rsp->nix = rvu_rsrc_free_count(&block->lf);
1967
1968	block = &hw->block[BLKADDR_NIX1];
1969	rsp->nix1 = rvu_rsrc_free_count(&block->lf);
1970
1971	block = &hw->block[BLKADDR_SSO];
1972	rsp->sso = rvu_rsrc_free_count(&block->lf);
1973
1974	block = &hw->block[BLKADDR_SSOW];
1975	rsp->ssow = rvu_rsrc_free_count(&block->lf);
1976
1977	block = &hw->block[BLKADDR_TIM];
1978	rsp->tim = rvu_rsrc_free_count(&block->lf);
1979
1980	block = &hw->block[BLKADDR_CPT0];
1981	rsp->cpt = rvu_rsrc_free_count(&block->lf);
1982
1983	block = &hw->block[BLKADDR_CPT1];
1984	rsp->cpt1 = rvu_rsrc_free_count(&block->lf);
1985
1986	if (rvu->hw->cap.nix_fixed_txschq_mapping) {
1987		rsp->schq[NIX_TXSCH_LVL_SMQ] = 1;
1988		rsp->schq[NIX_TXSCH_LVL_TL4] = 1;
1989		rsp->schq[NIX_TXSCH_LVL_TL3] = 1;
1990		rsp->schq[NIX_TXSCH_LVL_TL2] = 1;
1991		/* NIX1 */
1992		if (!is_block_implemented(rvu->hw, BLKADDR_NIX1))
1993			goto out;
1994		rsp->schq_nix1[NIX_TXSCH_LVL_SMQ] = 1;
1995		rsp->schq_nix1[NIX_TXSCH_LVL_TL4] = 1;
1996		rsp->schq_nix1[NIX_TXSCH_LVL_TL3] = 1;
1997		rsp->schq_nix1[NIX_TXSCH_LVL_TL2] = 1;
1998	} else {
1999		nix_hw = get_nix_hw(hw, BLKADDR_NIX0);
2000		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
2001		rsp->schq[NIX_TXSCH_LVL_SMQ] =
2002				rvu_rsrc_free_count(&txsch->schq);
2003
2004		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL4];
2005		rsp->schq[NIX_TXSCH_LVL_TL4] =
2006				rvu_rsrc_free_count(&txsch->schq);
2007
2008		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL3];
2009		rsp->schq[NIX_TXSCH_LVL_TL3] =
2010				rvu_rsrc_free_count(&txsch->schq);
2011
2012		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2];
2013		rsp->schq[NIX_TXSCH_LVL_TL2] =
2014				rvu_rsrc_free_count(&txsch->schq);
2015
2016		if (!is_block_implemented(rvu->hw, BLKADDR_NIX1))
2017			goto out;
2018
2019		nix_hw = get_nix_hw(hw, BLKADDR_NIX1);
2020		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
2021		rsp->schq_nix1[NIX_TXSCH_LVL_SMQ] =
2022				rvu_rsrc_free_count(&txsch->schq);
2023
2024		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL4];
2025		rsp->schq_nix1[NIX_TXSCH_LVL_TL4] =
2026				rvu_rsrc_free_count(&txsch->schq);
2027
2028		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL3];
2029		rsp->schq_nix1[NIX_TXSCH_LVL_TL3] =
2030				rvu_rsrc_free_count(&txsch->schq);
2031
2032		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2];
2033		rsp->schq_nix1[NIX_TXSCH_LVL_TL2] =
2034				rvu_rsrc_free_count(&txsch->schq);
2035	}
2036
2037	rsp->schq_nix1[NIX_TXSCH_LVL_TL1] = 1;
2038out:
2039	rsp->schq[NIX_TXSCH_LVL_TL1] = 1;
2040	mutex_unlock(&rvu->rsrc_lock);
2041
2042	return 0;
2043}
2044
2045int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req,
2046			    struct msg_rsp *rsp)
2047{
2048	u16 pcifunc = req->hdr.pcifunc;
2049	u16 vf, numvfs;
2050	u64 cfg;
2051
2052	vf = pcifunc & RVU_PFVF_FUNC_MASK;
2053	cfg = rvu_read64(rvu, BLKADDR_RVUM,
2054			 RVU_PRIV_PFX_CFG(rvu_get_pf(rvu->pdev, pcifunc)));
2055	numvfs = (cfg >> 12) & 0xFF;
2056
2057	if (vf && vf <= numvfs)
2058		__rvu_flr_handler(rvu, pcifunc);
2059	else
2060		return RVU_INVALID_VF_ID;
2061
2062	return 0;
2063}
2064
2065int rvu_ndc_sync(struct rvu *rvu, int lfblkaddr, int lfidx, u64 lfoffset)
2066{
2067	/* Sync cached info for this LF in NDC to LLC/DRAM */
2068	rvu_write64(rvu, lfblkaddr, lfoffset, BIT_ULL(12) | lfidx);
2069	return rvu_poll_reg(rvu, lfblkaddr, lfoffset, BIT_ULL(12), true);
2070}
2071
2072int rvu_mbox_handler_get_hw_cap(struct rvu *rvu, struct msg_req *req,
2073				struct get_hw_cap_rsp *rsp)
2074{
2075	struct rvu_hwinfo *hw = rvu->hw;
2076
2077	rsp->nix_fixed_txschq_mapping = hw->cap.nix_fixed_txschq_mapping;
2078	rsp->nix_shaping = hw->cap.nix_shaping;
2079	rsp->npc_hash_extract = hw->cap.npc_hash_extract;
2080
2081	if (rvu->mcs_blk_cnt)
2082		rsp->hw_caps = HW_CAP_MACSEC;
2083
2084	return 0;
2085}
2086
2087int rvu_mbox_handler_set_vf_perm(struct rvu *rvu, struct set_vf_perm *req,
2088				 struct msg_rsp *rsp)
2089{
2090	struct rvu_hwinfo *hw = rvu->hw;
2091	u16 pcifunc = req->hdr.pcifunc;
2092	struct rvu_pfvf *pfvf;
2093	int blkaddr, nixlf;
2094	u16 target;
2095
2096	/* Only PF can add VF permissions */
2097	if ((pcifunc & RVU_PFVF_FUNC_MASK) || is_lbk_vf(rvu, pcifunc))
2098		return -EOPNOTSUPP;
2099
2100	target = (pcifunc & ~RVU_PFVF_FUNC_MASK) | (req->vf + 1);
2101	pfvf = rvu_get_pfvf(rvu, target);
2102
2103	if (req->flags & RESET_VF_PERM) {
2104		pfvf->flags &= RVU_CLEAR_VF_PERM;
2105	} else if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) ^
2106		 (req->flags & VF_TRUSTED)) {
2107		change_bit(PF_SET_VF_TRUSTED, &pfvf->flags);
2108		/* disable multicast and promisc entries */
2109		if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags)) {
2110			blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, target);
2111			if (blkaddr < 0)
2112				return 0;
2113			nixlf = rvu_get_lf(rvu, &hw->block[blkaddr],
2114					   target, 0);
2115			if (nixlf < 0)
2116				return 0;
2117			npc_enadis_default_mce_entry(rvu, target, nixlf,
2118						     NIXLF_ALLMULTI_ENTRY,
2119						     false);
2120			npc_enadis_default_mce_entry(rvu, target, nixlf,
2121						     NIXLF_PROMISC_ENTRY,
2122						     false);
2123		}
2124	}
2125
2126	return 0;
2127}
2128
2129int rvu_mbox_handler_ndc_sync_op(struct rvu *rvu,
2130				 struct ndc_sync_op *req,
2131				 struct msg_rsp *rsp)
2132{
2133	struct rvu_hwinfo *hw = rvu->hw;
2134	u16 pcifunc = req->hdr.pcifunc;
2135	int err, lfidx, lfblkaddr;
2136
2137	if (req->npa_lf_sync) {
2138		/* Get NPA LF data */
2139		lfblkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
2140		if (lfblkaddr < 0)
2141			return NPA_AF_ERR_AF_LF_INVALID;
2142
2143		lfidx = rvu_get_lf(rvu, &hw->block[lfblkaddr], pcifunc, 0);
2144		if (lfidx < 0)
2145			return NPA_AF_ERR_AF_LF_INVALID;
2146
2147		/* Sync NPA NDC */
2148		err = rvu_ndc_sync(rvu, lfblkaddr,
2149				   lfidx, NPA_AF_NDC_SYNC);
2150		if (err)
2151			dev_err(rvu->dev,
2152				"NDC-NPA sync failed for LF %u\n", lfidx);
2153	}
2154
2155	if (!req->nix_lf_tx_sync && !req->nix_lf_rx_sync)
2156		return 0;
2157
2158	/* Get NIX LF data */
2159	lfblkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2160	if (lfblkaddr < 0)
2161		return NIX_AF_ERR_AF_LF_INVALID;
2162
2163	lfidx = rvu_get_lf(rvu, &hw->block[lfblkaddr], pcifunc, 0);
2164	if (lfidx < 0)
2165		return NIX_AF_ERR_AF_LF_INVALID;
2166
2167	if (req->nix_lf_tx_sync) {
2168		/* Sync NIX TX NDC */
2169		err = rvu_ndc_sync(rvu, lfblkaddr,
2170				   lfidx, NIX_AF_NDC_TX_SYNC);
2171		if (err)
2172			dev_err(rvu->dev,
2173				"NDC-NIX-TX sync fail for LF %u\n", lfidx);
2174	}
2175
2176	if (req->nix_lf_rx_sync) {
2177		/* Sync NIX RX NDC */
2178		err = rvu_ndc_sync(rvu, lfblkaddr,
2179				   lfidx, NIX_AF_NDC_RX_SYNC);
2180		if (err)
2181			dev_err(rvu->dev,
2182				"NDC-NIX-RX sync failed for LF %u\n", lfidx);
2183	}
2184
2185	return 0;
2186}
2187
2188static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
2189				struct mbox_msghdr *req)
2190{
2191	struct rvu *rvu = pci_get_drvdata(mbox->pdev);
2192
2193	/* Check if valid, if not reply with a invalid msg */
2194	if (req->sig != OTX2_MBOX_REQ_SIG)
2195		goto bad_message;
2196
2197	switch (req->id) {
2198#define M(_name, _id, _fn_name, _req_type, _rsp_type)			\
2199	case _id: {							\
2200		struct _rsp_type *rsp;					\
2201		int err;						\
2202									\
2203		rsp = (struct _rsp_type *)otx2_mbox_alloc_msg(		\
2204			mbox, devid,					\
2205			sizeof(struct _rsp_type));			\
2206		/* some handlers should complete even if reply */	\
2207		/* could not be allocated */				\
2208		if (!rsp &&						\
2209		    _id != MBOX_MSG_DETACH_RESOURCES &&			\
2210		    _id != MBOX_MSG_NIX_TXSCH_FREE &&			\
2211		    _id != MBOX_MSG_VF_FLR)				\
2212			return -ENOMEM;					\
2213		if (rsp) {						\
2214			rsp->hdr.id = _id;				\
2215			rsp->hdr.sig = OTX2_MBOX_RSP_SIG;		\
2216			rsp->hdr.pcifunc = req->pcifunc;		\
2217			rsp->hdr.rc = 0;				\
2218		}							\
2219									\
2220		err = rvu_mbox_handler_ ## _fn_name(rvu,		\
2221						    (struct _req_type *)req, \
2222						    rsp);		\
2223		if (rsp && err)						\
2224			rsp->hdr.rc = err;				\
2225									\
2226		trace_otx2_msg_process(mbox->pdev, _id, err, req->pcifunc); \
2227		return rsp ? err : -ENOMEM;				\
2228	}
2229MBOX_MESSAGES
2230#undef M
2231
2232bad_message:
2233	default:
2234		otx2_reply_invalid_msg(mbox, devid, req->pcifunc, req->id);
2235		return -ENODEV;
2236	}
2237}
2238
2239static void __rvu_mbox_handler(struct rvu_work *mwork, int type, bool poll)
2240{
2241	struct rvu *rvu = mwork->rvu;
2242	int offset, err, id, devid;
2243	struct otx2_mbox_dev *mdev;
2244	struct mbox_hdr *req_hdr;
2245	struct mbox_msghdr *msg;
2246	struct mbox_wq_info *mw;
2247	struct otx2_mbox *mbox;
2248
2249	switch (type) {
2250	case TYPE_AFPF:
2251		mw = &rvu->afpf_wq_info;
2252		break;
2253	case TYPE_AFVF:
2254		mw = &rvu->afvf_wq_info;
2255		break;
2256	default:
2257		return;
2258	}
2259
2260	devid = mwork - mw->mbox_wrk;
2261	mbox = &mw->mbox;
2262	mdev = &mbox->dev[devid];
2263
2264	/* Process received mbox messages */
2265	req_hdr = mdev->mbase + mbox->rx_start;
2266	if (mw->mbox_wrk[devid].num_msgs == 0)
2267		return;
2268
2269	offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
2270
2271	if (req_hdr->sig && !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) {
2272		req_hdr->opt_msg = mw->mbox_wrk[devid].num_msgs;
2273		rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL,
2274			    RVU_AF_BAR2_PFID);
2275		if (type == TYPE_AFPF)
2276			rvu_write64(rvu, BLKADDR_NIX0,
2277				    AF_BAR2_ALIASX(0, NIX_CINTX_INT_W1S(devid)),
2278				    0x1);
2279		else
2280			rvu_write64(rvu, BLKADDR_NIX0,
2281				    AF_BAR2_ALIASX(0, NIX_QINTX_CNT(devid)),
2282				    0x1);
2283		usleep_range(5000, 6000);
2284		goto done;
2285	}
2286
2287	for (id = 0; id < mw->mbox_wrk[devid].num_msgs; id++) {
2288		msg = mdev->mbase + offset;
2289
2290		/* Set which PF/VF sent this message based on mbox IRQ */
2291		switch (type) {
2292		case TYPE_AFPF:
2293			msg->pcifunc &= rvu_pcifunc_pf_mask(rvu->pdev);
2294			msg->pcifunc |= rvu_make_pcifunc(rvu->pdev, devid, 0);
2295			break;
2296		case TYPE_AFVF:
2297			msg->pcifunc &=
2298				~(RVU_PFVF_FUNC_MASK << RVU_PFVF_FUNC_SHIFT);
2299			msg->pcifunc |= (devid << RVU_PFVF_FUNC_SHIFT) + 1;
2300			break;
2301		}
2302
2303		err = rvu_process_mbox_msg(mbox, devid, msg);
2304		if (!err) {
2305			offset = mbox->rx_start + msg->next_msgoff;
2306			continue;
2307		}
2308
2309		if (msg->pcifunc & RVU_PFVF_FUNC_MASK)
2310			dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",
2311				 err, otx2_mbox_id2name(msg->id),
2312				 msg->id, rvu_get_pf(rvu->pdev, msg->pcifunc),
2313				 (msg->pcifunc & RVU_PFVF_FUNC_MASK) - 1);
2314		else
2315			dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n",
2316				 err, otx2_mbox_id2name(msg->id),
2317				 msg->id, devid);
2318	}
2319done:
2320	mw->mbox_wrk[devid].num_msgs = 0;
2321
2322	if (!is_cn20k(mbox->pdev) && poll)
2323		otx2_mbox_wait_for_zero(mbox, devid);
2324
2325	/* Send mbox responses to VF/PF */
2326	otx2_mbox_msg_send(mbox, devid);
2327}
2328
2329static inline void rvu_afpf_mbox_handler(struct work_struct *work)
2330{
2331	struct rvu_work *mwork = container_of(work, struct rvu_work, work);
2332	struct rvu *rvu = mwork->rvu;
2333
2334	mutex_lock(&rvu->mbox_lock);
2335	__rvu_mbox_handler(mwork, TYPE_AFPF, true);
2336	mutex_unlock(&rvu->mbox_lock);
2337}
2338
2339static inline void rvu_afvf_mbox_handler(struct work_struct *work)
2340{
2341	struct rvu_work *mwork = container_of(work, struct rvu_work, work);
2342
2343	__rvu_mbox_handler(mwork, TYPE_AFVF, false);
2344}
2345
2346static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)
2347{
2348	struct rvu *rvu = mwork->rvu;
2349	struct otx2_mbox_dev *mdev;
2350	struct mbox_hdr *rsp_hdr;
2351	struct mbox_msghdr *msg;
2352	struct mbox_wq_info *mw;
2353	struct otx2_mbox *mbox;
2354	int offset, id, devid;
2355
2356	switch (type) {
2357	case TYPE_AFPF:
2358		mw = &rvu->afpf_wq_info;
2359		break;
2360	case TYPE_AFVF:
2361		mw = &rvu->afvf_wq_info;
2362		break;
2363	default:
2364		return;
2365	}
2366
2367	devid = mwork - mw->mbox_wrk_up;
2368	mbox = &mw->mbox_up;
2369	mdev = &mbox->dev[devid];
2370
2371	rsp_hdr = mdev->mbase + mbox->rx_start;
2372	if (mw->mbox_wrk_up[devid].up_num_msgs == 0) {
2373		dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n");
2374		return;
2375	}
2376
2377	offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
2378
2379	for (id = 0; id < mw->mbox_wrk_up[devid].up_num_msgs; id++) {
2380		msg = mdev->mbase + offset;
2381
2382		if (msg->id >= MBOX_MSG_MAX) {
2383			dev_err(rvu->dev,
2384				"Mbox msg with unknown ID 0x%x\n", msg->id);
2385			goto end;
2386		}
2387
2388		if (msg->sig != OTX2_MBOX_RSP_SIG) {
2389			dev_err(rvu->dev,
2390				"Mbox msg with wrong signature %x, ID 0x%x\n",
2391				msg->sig, msg->id);
2392			goto end;
2393		}
2394
2395		switch (msg->id) {
2396		case MBOX_MSG_CGX_LINK_EVENT:
2397			break;
2398		default:
2399			if (msg->rc)
2400				dev_err(rvu->dev,
2401					"Mbox msg response has err %d, ID 0x%x\n",
2402					msg->rc, msg->id);
2403			break;
2404		}
2405end:
2406		offset = mbox->rx_start + msg->next_msgoff;
2407		mdev->msgs_acked++;
2408	}
2409	mw->mbox_wrk_up[devid].up_num_msgs = 0;
2410
2411	otx2_mbox_reset(mbox, devid);
2412}
2413
2414static inline void rvu_afpf_mbox_up_handler(struct work_struct *work)
2415{
2416	struct rvu_work *mwork = container_of(work, struct rvu_work, work);
2417
2418	__rvu_mbox_up_handler(mwork, TYPE_AFPF);
2419}
2420
2421static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)
2422{
2423	struct rvu_work *mwork = container_of(work, struct rvu_work, work);
2424
2425	__rvu_mbox_up_handler(mwork, TYPE_AFVF);
2426}
2427
2428static int rvu_get_mbox_regions(struct rvu *rvu, void __iomem **mbox_addr,
2429				int num, int type, unsigned long *pf_bmap)
2430{
2431	struct rvu_hwinfo *hw = rvu->hw;
2432	int region;
2433	u64 bar4;
2434
2435	/* For cn20k platform AF mailbox region is allocated by software
2436	 * and the corresponding IOVA is programmed in hardware unlike earlier
2437	 * silicons where software uses the hardware region after ioremap.
2438	 */
2439	if (is_cn20k(rvu->pdev))
2440		return cn20k_rvu_get_mbox_regions(rvu, (void *)mbox_addr,
2441						  num, type, pf_bmap);
2442
2443	/* For cn10k platform VF mailbox regions of a PF follows after the
2444	 * PF <-> AF mailbox region. Whereas for Octeontx2 it is read from
2445	 * RVU_PF_VF_BAR4_ADDR register.
2446	 */
2447	if (type == TYPE_AFVF) {
2448		for (region = 0; region < num; region++) {
2449			if (!test_bit(region, pf_bmap))
2450				continue;
2451
2452			if (hw->cap.per_pf_mbox_regs) {
2453				bar4 = rvu_read64(rvu, BLKADDR_RVUM,
2454						  RVU_AF_PFX_BAR4_ADDR(0)) +
2455						  MBOX_SIZE;
2456				bar4 += region * MBOX_SIZE;
2457			} else {
2458				bar4 = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);
2459				bar4 += region * MBOX_SIZE;
2460			}
2461			mbox_addr[region] = ioremap_wc(bar4, MBOX_SIZE);
2462			if (!mbox_addr[region])
2463				goto error;
2464		}
2465		return 0;
2466	}
2467
2468	/* For cn10k platform AF <-> PF mailbox region of a PF is read from per
2469	 * PF registers. Whereas for Octeontx2 it is read from
2470	 * RVU_AF_PF_BAR4_ADDR register.
2471	 */
2472	for (region = 0; region < num; region++) {
2473		if (!test_bit(region, pf_bmap))
2474			continue;
2475
2476		if (hw->cap.per_pf_mbox_regs) {
2477			bar4 = rvu_read64(rvu, BLKADDR_RVUM,
2478					  RVU_AF_PFX_BAR4_ADDR(region));
2479		} else {
2480			bar4 = rvu_read64(rvu, BLKADDR_RVUM,
2481					  RVU_AF_PF_BAR4_ADDR);
2482			bar4 += region * MBOX_SIZE;
2483		}
2484		mbox_addr[region] = ioremap_wc(bar4, MBOX_SIZE);
2485		if (!mbox_addr[region])
2486			goto error;
2487	}
2488	return 0;
2489
2490error:
2491	while (region--)
2492		iounmap(mbox_addr[region]);
2493	return -ENOMEM;
2494}
2495
2496static struct mbox_ops rvu_mbox_ops = {
2497	.pf_intr_handler = rvu_mbox_pf_intr_handler,
2498	.afvf_intr_handler = rvu_mbox_intr_handler,
2499};
2500
2501static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
2502			 int type, int num,
2503			 void (mbox_handler)(struct work_struct *),
2504			 void (mbox_up_handler)(struct work_struct *))
2505{
2506	void __iomem **mbox_regions;
2507	struct ng_rvu *ng_rvu_mbox;
2508	int err, i, dir, dir_up;
2509	void __iomem *reg_base;
2510	struct rvu_work *mwork;
2511	unsigned long *pf_bmap;
2512	const char *name;
2513	u64 cfg;
2514
2515	pf_bmap = bitmap_zalloc(num, GFP_KERNEL);
2516	if (!pf_bmap)
2517		return -ENOMEM;
2518
2519	ng_rvu_mbox = kzalloc(sizeof(*ng_rvu_mbox), GFP_KERNEL);
2520	if (!ng_rvu_mbox) {
2521		err = -ENOMEM;
2522		goto free_bitmap;
2523	}
2524
2525	/* RVU VFs */
2526	if (type == TYPE_AFVF)
2527		bitmap_set(pf_bmap, 0, num);
2528
2529	if (type == TYPE_AFPF) {
2530		/* Mark enabled PFs in bitmap */
2531		for (i = 0; i < num; i++) {
2532			cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(i));
2533			if (cfg & BIT_ULL(20))
2534				set_bit(i, pf_bmap);
2535		}
2536	}
2537
2538	rvu->ng_rvu = ng_rvu_mbox;
2539
2540	rvu->ng_rvu->rvu_mbox_ops = &rvu_mbox_ops;
2541
2542	err = cn20k_rvu_mbox_init(rvu, type, num);
2543	if (err)
2544		goto free_mem;
2545
2546	mutex_init(&rvu->mbox_lock);
2547
2548	mbox_regions = kcalloc(num, sizeof(void __iomem *), GFP_KERNEL);
2549	if (!mbox_regions) {
2550		err = -ENOMEM;
2551		goto free_qmem;
2552	}
2553
2554	switch (type) {
2555	case TYPE_AFPF:
2556		name = "rvu_afpf_mailbox";
2557		dir = MBOX_DIR_AFPF;
2558		dir_up = MBOX_DIR_AFPF_UP;
2559		reg_base = rvu->afreg_base;
2560		err = rvu_get_mbox_regions(rvu, mbox_regions, num, TYPE_AFPF, pf_bmap);
2561		if (err)
2562			goto free_regions;
2563		break;
2564	case TYPE_AFVF:
2565		name = "rvu_afvf_mailbox";
2566		dir = MBOX_DIR_PFVF;
2567		dir_up = MBOX_DIR_PFVF_UP;
2568		reg_base = rvu->pfreg_base;
2569		err = rvu_get_mbox_regions(rvu, mbox_regions, num, TYPE_AFVF, pf_bmap);
2570		if (err)
2571			goto free_regions;
2572		break;
2573	default:
2574		err = -EINVAL;
2575		goto free_regions;
2576	}
2577
2578	mw->mbox_wq = alloc_workqueue("%s",
2579				      WQ_HIGHPRI | WQ_MEM_RECLAIM,
2580				      num, name);
2581	if (!mw->mbox_wq) {
2582		err = -ENOMEM;
2583		goto unmap_regions;
2584	}
2585
2586	mw->mbox_wrk = devm_kcalloc(rvu->dev, num,
2587				    sizeof(struct rvu_work), GFP_KERNEL);
2588	if (!mw->mbox_wrk) {
2589		err = -ENOMEM;
2590		goto exit;
2591	}
2592
2593	mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num,
2594				       sizeof(struct rvu_work), GFP_KERNEL);
2595	if (!mw->mbox_wrk_up) {
2596		err = -ENOMEM;
2597		goto exit;
2598	}
2599
2600	err = otx2_mbox_regions_init(&mw->mbox, mbox_regions, rvu->pdev,
2601				     reg_base, dir, num, pf_bmap);
2602	if (err)
2603		goto exit;
2604
2605	err = otx2_mbox_regions_init(&mw->mbox_up, mbox_regions, rvu->pdev,
2606				     reg_base, dir_up, num, pf_bmap);
2607	if (err)
2608		goto exit;
2609
2610	for (i = 0; i < num; i++) {
2611		if (!test_bit(i, pf_bmap))
2612			continue;
2613
2614		mwork = &mw->mbox_wrk[i];
2615		mwork->rvu = rvu;
2616		INIT_WORK(&mwork->work, mbox_handler);
2617
2618		mwork = &mw->mbox_wrk_up[i];
2619		mwork->rvu = rvu;
2620		INIT_WORK(&mwork->work, mbox_up_handler);
2621	}
2622
2623	kfree(mbox_regions);
2624	bitmap_free(pf_bmap);
2625
2626	return 0;
2627
2628exit:
2629	destroy_workqueue(mw->mbox_wq);
2630unmap_regions:
2631	while (num--)
2632		iounmap((void __iomem *)mbox_regions[num]);
2633free_regions:
2634	kfree(mbox_regions);
2635free_qmem:
2636	cn20k_free_mbox_memory(rvu);
2637free_mem:
2638	kfree(rvu->ng_rvu);
2639free_bitmap:
2640	bitmap_free(pf_bmap);
2641	return err;
2642}
2643
2644static void rvu_mbox_destroy(struct mbox_wq_info *mw)
2645{
2646	struct otx2_mbox *mbox = &mw->mbox;
2647	struct otx2_mbox_dev *mdev;
2648	int devid;
2649
2650	if (mw->mbox_wq) {
2651		destroy_workqueue(mw->mbox_wq);
2652		mw->mbox_wq = NULL;
2653	}
2654
2655	for (devid = 0; devid < mbox->ndevs; devid++) {
2656		mdev = &mbox->dev[devid];
2657		if (mdev->hwbase)
2658			iounmap((void __iomem *)mdev->hwbase);
2659	}
2660
2661	otx2_mbox_destroy(&mw->mbox);
2662	otx2_mbox_destroy(&mw->mbox_up);
2663}
2664
2665void rvu_queue_work(struct mbox_wq_info *mw, int first,
2666		    int mdevs, u64 intr)
2667{
2668	struct otx2_mbox_dev *mdev;
2669	struct otx2_mbox *mbox;
2670	struct mbox_hdr *hdr;
2671	int i;
2672
2673	for (i = first; i < mdevs; i++) {
2674		/* start from 0 */
2675		if (!(intr & BIT_ULL(i - first)))
2676			continue;
2677
2678		mbox = &mw->mbox;
2679		mdev = &mbox->dev[i];
2680		hdr = mdev->mbase + mbox->rx_start;
2681
2682		/*The hdr->num_msgs is set to zero immediately in the interrupt
2683		 * handler to  ensure that it holds a correct value next time
2684		 * when the interrupt handler is called.
2685		 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler
2686		 * pf>mbox.up_num_msgs holds the data for use in
2687		 * pfaf_mbox_up_handler.
2688		 */
2689
2690		if (hdr->num_msgs) {
2691			mw->mbox_wrk[i].num_msgs = hdr->num_msgs;
2692			hdr->num_msgs = 0;
2693			queue_work(mw->mbox_wq, &mw->mbox_wrk[i].work);
2694		}
2695		mbox = &mw->mbox_up;
2696		mdev = &mbox->dev[i];
2697		hdr = mdev->mbase + mbox->rx_start;
2698		if (hdr->num_msgs) {
2699			mw->mbox_wrk_up[i].up_num_msgs = hdr->num_msgs;
2700			hdr->num_msgs = 0;
2701			queue_work(mw->mbox_wq, &mw->mbox_wrk_up[i].work);
2702		}
2703	}
2704}
2705
2706static irqreturn_t rvu_mbox_pf_intr_handler(int irq, void *rvu_irq)
2707{
2708	struct rvu *rvu = (struct rvu *)rvu_irq;
2709	u64 intr;
2710
2711	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
2712	/* Clear interrupts */
2713	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);
2714	if (intr)
2715		trace_otx2_msg_interrupt(rvu->pdev, "PF(s) to AF", intr);
2716
2717	/* Sync with mbox memory region */
2718	rmb();
2719
2720	rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);
2721
2722	return IRQ_HANDLED;
2723}
2724
2725static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
2726{
2727	struct rvu *rvu = (struct rvu *)rvu_irq;
2728	int vfs = rvu->vfs;
2729	u64 intr;
2730
2731	/* Sync with mbox memory region */
2732	rmb();
2733
2734	/* Handle VF interrupts */
2735	if (vfs > 64) {
2736		intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));
2737		rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr);
2738
2739		rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr);
2740		vfs = 64;
2741	}
2742
2743	intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));
2744	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);
2745	if (intr)
2746		trace_otx2_msg_interrupt(rvu->pdev, "VF(s) to AF", intr);
2747
2748	rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);
2749
2750	return IRQ_HANDLED;
2751}
2752
2753static void rvu_enable_mbox_intr(struct rvu *rvu)
2754{
2755	struct rvu_hwinfo *hw = rvu->hw;
2756
2757	if (is_cn20k(rvu->pdev)) {
2758		cn20k_rvu_enable_mbox_intr(rvu);
2759		return;
2760	}
2761
2762	/* Clear spurious irqs, if any */
2763	rvu_write64(rvu, BLKADDR_RVUM,
2764		    RVU_AF_PFAF_MBOX_INT, INTR_MASK(hw->total_pfs));
2765
2766	/* Enable mailbox interrupt for all PFs except PF0 i.e AF itself */
2767	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S,
2768		    INTR_MASK(hw->total_pfs) & ~1ULL);
2769}
2770
2771static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)
2772{
2773	struct rvu_block *block;
2774	int slot, lf, num_lfs;
2775	int err;
2776
2777	block = &rvu->hw->block[blkaddr];
2778	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
2779					block->addr);
2780	if (!num_lfs)
2781		return;
2782	for (slot = 0; slot < num_lfs; slot++) {
2783		lf = rvu_get_lf(rvu, block, pcifunc, slot);
2784		if (lf < 0)
2785			continue;
2786
2787		/* Cleanup LF and reset it */
2788		if (block->addr == BLKADDR_NIX0 || block->addr == BLKADDR_NIX1)
2789			rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf);
2790		else if (block->addr == BLKADDR_NPA)
2791			rvu_npa_lf_teardown(rvu, pcifunc, lf);
2792		else if ((block->addr == BLKADDR_CPT0) ||
2793			 (block->addr == BLKADDR_CPT1))
2794			rvu_cpt_lf_teardown(rvu, pcifunc, block->addr, lf,
2795					    slot);
2796
2797		err = rvu_lf_reset(rvu, block, lf);
2798		if (err) {
2799			dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
2800				block->addr, lf);
2801		}
2802	}
2803}
2804
2805static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc)
2806{
2807	if (rvu_npc_exact_has_match_table(rvu))
2808		rvu_npc_exact_reset(rvu, pcifunc);
2809
2810	mutex_lock(&rvu->flr_lock);
2811	/* Reset order should reflect inter-block dependencies:
2812	 * 1. Reset any packet/work sources (NIX, CPT, TIM)
2813	 * 2. Flush and reset SSO/SSOW
2814	 * 3. Cleanup pools (NPA)
2815	 */
2816
2817	/* Free allocated BPIDs */
2818	rvu_nix_flr_free_bpids(rvu, pcifunc);
2819
2820	/* Free multicast/mirror node associated with the 'pcifunc' */
2821	rvu_nix_mcast_flr_free_entries(rvu, pcifunc);
2822
2823	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0);
2824	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX1);
2825	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0);
2826	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT1);
2827	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM);
2828	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW);
2829	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO);
2830	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA);
2831	rvu_reset_lmt_map_tbl(rvu, pcifunc);
2832	rvu_detach_rsrcs(rvu, NULL, pcifunc);
2833	/* In scenarios where PF/VF drivers detach NIXLF without freeing MCAM
2834	 * entries, check and free the MCAM entries explicitly to avoid leak.
2835	 * Since LF is detached use LF number as -1.
2836	 */
2837	rvu_npc_free_mcam_entries(rvu, pcifunc, -1);
2838	rvu_mac_reset(rvu, pcifunc);
2839
2840	if (rvu->mcs_blk_cnt)
2841		rvu_mcs_flr_handler(rvu, pcifunc);
2842
2843	mutex_unlock(&rvu->flr_lock);
2844}
2845
2846static void rvu_afvf_flr_handler(struct rvu *rvu, int vf)
2847{
2848	int reg = 0;
2849
2850	/* pcifunc = 0(PF0) | (vf + 1) */
2851	__rvu_flr_handler(rvu, vf + 1);
2852
2853	if (vf >= 64) {
2854		reg = 1;
2855		vf = vf - 64;
2856	}
2857
2858	/* Signal FLR finish and enable IRQ */
2859	rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
2860	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
2861}
2862
2863static void rvu_flr_handler(struct work_struct *work)
2864{
2865	struct rvu_work *flrwork = container_of(work, struct rvu_work, work);
2866	struct rvu *rvu = flrwork->rvu;
2867	u16 pcifunc, numvfs, vf;
2868	u64 cfg;
2869	int pf;
2870
2871	pf = flrwork - rvu->flr_wrk;
2872	if (pf >= rvu->hw->total_pfs) {
2873		rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs);
2874		return;
2875	}
2876
2877	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2878	numvfs = (cfg >> 12) & 0xFF;
2879	pcifunc  = rvu_make_pcifunc(rvu->pdev, pf, 0);
2880
2881	for (vf = 0; vf < numvfs; vf++)
2882		__rvu_flr_handler(rvu, (pcifunc | (vf + 1)));
2883
2884	__rvu_flr_handler(rvu, pcifunc);
2885
2886	/* Signal FLR finish */
2887	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf));
2888
2889	/* Enable interrupt */
2890	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,  BIT_ULL(pf));
2891}
2892
2893static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs)
2894{
2895	int dev, vf, reg = 0;
2896	u64 intr;
2897
2898	if (start_vf >= 64)
2899		reg = 1;
2900
2901	intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg));
2902	if (!intr)
2903		return;
2904
2905	for (vf = 0; vf < numvfs; vf++) {
2906		if (!(intr & BIT_ULL(vf)))
2907			continue;
2908		/* Clear and disable the interrupt */
2909		rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
2910		rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf));
2911
2912		dev = vf + start_vf + rvu->hw->total_pfs;
2913		queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work);
2914	}
2915}
2916
2917static irqreturn_t rvu_flr_intr_handler(int irq, void *rvu_irq)
2918{
2919	struct rvu *rvu = (struct rvu *)rvu_irq;
2920	u64 intr;
2921	u8  pf;
2922
2923	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT);
2924	if (!intr)
2925		goto afvf_flr;
2926
2927	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2928		if (intr & (1ULL << pf)) {
2929			/* clear interrupt */
2930			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT,
2931				    BIT_ULL(pf));
2932			/* Disable the interrupt */
2933			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
2934				    BIT_ULL(pf));
2935			/* PF is already dead do only AF related operations */
2936			queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work);
2937		}
2938	}
2939
2940afvf_flr:
2941	rvu_afvf_queue_flr_work(rvu, 0, 64);
2942	if (rvu->vfs > 64)
2943		rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64);
2944
2945	return IRQ_HANDLED;
2946}
2947
2948static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr)
2949{
2950	int vf;
2951
2952	/* Nothing to be done here other than clearing the
2953	 * TRPEND bit.
2954	 */
2955	for (vf = 0; vf < 64; vf++) {
2956		if (intr & (1ULL << vf)) {
2957			/* clear the trpend due to ME(master enable) */
2958			rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf));
2959			/* clear interrupt */
2960			rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf));
2961		}
2962	}
2963}
2964
2965/* Handles ME interrupts from VFs of AF */
2966static irqreturn_t rvu_me_vf_intr_handler(int irq, void *rvu_irq)
2967{
2968	struct rvu *rvu = (struct rvu *)rvu_irq;
2969	int vfset;
2970	u64 intr;
2971
2972	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
2973
2974	for (vfset = 0; vfset <= 1; vfset++) {
2975		intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset));
2976		if (intr)
2977			rvu_me_handle_vfset(rvu, vfset, intr);
2978	}
2979
2980	return IRQ_HANDLED;
2981}
2982
2983/* Handles ME interrupts from PFs */
2984static irqreturn_t rvu_me_pf_intr_handler(int irq, void *rvu_irq)
2985{
2986	struct rvu *rvu = (struct rvu *)rvu_irq;
2987	u64 intr;
2988	u8  pf;
2989
2990	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
2991
2992	/* Nothing to be done here other than clearing the
2993	 * TRPEND bit.
2994	 */
2995	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2996		if (intr & (1ULL << pf)) {
2997			/* clear the trpend due to ME(master enable) */
2998			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND,
2999				    BIT_ULL(pf));
3000			/* clear interrupt */
3001			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT,
3002				    BIT_ULL(pf));
3003		}
3004	}
3005
3006	return IRQ_HANDLED;
3007}
3008
3009static void rvu_unregister_interrupts(struct rvu *rvu)
3010{
3011	int irq;
3012
3013	rvu_cpt_unregister_interrupts(rvu);
3014
3015	if (!is_cn20k(rvu->pdev))
3016		/* Disable the Mbox interrupt */
3017		rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C,
3018			    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
3019	else
3020		cn20k_rvu_unregister_interrupts(rvu);
3021
3022	/* Disable the PF FLR interrupt */
3023	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
3024		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
3025
3026	/* Disable the PF ME interrupt */
3027	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C,
3028		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
3029
3030	for (irq = 0; irq < rvu->num_vec; irq++) {
3031		if (rvu->irq_allocated[irq]) {
3032			free_irq(pci_irq_vector(rvu->pdev, irq), rvu);
3033			rvu->irq_allocated[irq] = false;
3034		}
3035	}
3036
3037	pci_free_irq_vectors(rvu->pdev);
3038	rvu->num_vec = 0;
3039}
3040
3041static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)
3042{
3043	struct rvu_pfvf *pfvf = &rvu->pf[0];
3044	int offset;
3045
3046	pfvf = &rvu->pf[0];
3047	offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
3048
3049	/* Make sure there are enough MSIX vectors configured so that
3050	 * VF interrupts can be handled. Offset equal to zero means
3051	 * that PF vectors are not configured and overlapping AF vectors.
3052	 */
3053	if (is_cn20k(rvu->pdev))
3054		return (pfvf->msix.max >= RVU_AF_CN20K_INT_VEC_CNT +
3055			RVU_MBOX_PF_INT_VEC_CNT) && offset;
3056
3057	return (pfvf->msix.max >= RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT) &&
3058	       offset;
3059}
3060
3061static int rvu_register_interrupts(struct rvu *rvu)
3062{
3063	int ret, offset, pf_vec_start;
3064
3065	rvu->num_vec = pci_msix_vec_count(rvu->pdev);
3066
3067	rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,
3068					   NAME_SIZE, GFP_KERNEL);
3069	if (!rvu->irq_name)
3070		return -ENOMEM;
3071
3072	rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec,
3073					  sizeof(bool), GFP_KERNEL);
3074	if (!rvu->irq_allocated)
3075		return -ENOMEM;
3076
3077	/* Enable MSI-X */
3078	ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec,
3079				    rvu->num_vec, PCI_IRQ_MSIX);
3080	if (ret < 0) {
3081		dev_err(rvu->dev,
3082			"RVUAF: Request for %d msix vectors failed, ret %d\n",
3083			rvu->num_vec, ret);
3084		return ret;
3085	}
3086
3087	if (!is_cn20k(rvu->pdev)) {
3088		/* Register mailbox interrupt handler */
3089		sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE],
3090			"RVUAF Mbox");
3091		ret = request_irq(pci_irq_vector
3092				  (rvu->pdev, RVU_AF_INT_VEC_MBOX),
3093				  rvu->ng_rvu->rvu_mbox_ops->pf_intr_handler, 0,
3094				  &rvu->irq_name[RVU_AF_INT_VEC_MBOX *
3095				  NAME_SIZE], rvu);
3096		if (ret) {
3097			dev_err(rvu->dev,
3098				"RVUAF: IRQ registration failed for mbox\n");
3099			goto fail;
3100		}
3101
3102		rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true;
3103	} else {
3104		ret = cn20k_register_afpf_mbox_intr(rvu);
3105		if (ret) {
3106			dev_err(rvu->dev,
3107				"RVUAF: IRQ registration failed for mbox\n");
3108			goto fail;
3109		}
3110	}
3111
3112	/* Enable mailbox interrupts from all PFs */
3113	rvu_enable_mbox_intr(rvu);
3114
3115	/* Register FLR interrupt handler */
3116	sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
3117		"RVUAF FLR");
3118	ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR),
3119			  rvu_flr_intr_handler, 0,
3120			  &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
3121			  rvu);
3122	if (ret) {
3123		dev_err(rvu->dev,
3124			"RVUAF: IRQ registration failed for FLR\n");
3125		goto fail;
3126	}
3127	rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true;
3128
3129	/* Enable FLR interrupt for all PFs*/
3130	rvu_write64(rvu, BLKADDR_RVUM,
3131		    RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs));
3132
3133	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,
3134		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
3135
3136	/* Register ME interrupt handler */
3137	sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
3138		"RVUAF ME");
3139	ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME),
3140			  rvu_me_pf_intr_handler, 0,
3141			  &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
3142			  rvu);
3143	if (ret) {
3144		dev_err(rvu->dev,
3145			"RVUAF: IRQ registration failed for ME\n");
3146	}
3147	rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true;
3148
3149	/* Clear TRPEND bit for all PF */
3150	rvu_write64(rvu, BLKADDR_RVUM,
3151		    RVU_AF_PFTRPEND, INTR_MASK(rvu->hw->total_pfs));
3152	/* Enable ME interrupt for all PFs*/
3153	rvu_write64(rvu, BLKADDR_RVUM,
3154		    RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs));
3155
3156	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S,
3157		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
3158
3159	if (!rvu_afvf_msix_vectors_num_ok(rvu))
3160		return 0;
3161
3162	/* Get PF MSIX vectors offset. */
3163	pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM,
3164				  RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
3165	if (!is_cn20k(rvu->pdev)) {
3166		/* Register MBOX0 interrupt. */
3167		offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX0;
3168		sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0");
3169		ret = request_irq(pci_irq_vector(rvu->pdev, offset),
3170				  rvu->ng_rvu->rvu_mbox_ops->afvf_intr_handler, 0,
3171				  &rvu->irq_name[offset * NAME_SIZE],
3172				  rvu);
3173		if (ret)
3174			dev_err(rvu->dev,
3175				"RVUAF: IRQ registration failed for Mbox0\n");
3176
3177		rvu->irq_allocated[offset] = true;
3178
3179		/* Register MBOX1 interrupt. MBOX1 IRQ number follows MBOX0 so
3180		 * simply increment current offset by 1.
3181		 */
3182		offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX1;
3183		sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1");
3184		ret = request_irq(pci_irq_vector(rvu->pdev, offset),
3185				  rvu->ng_rvu->rvu_mbox_ops->afvf_intr_handler, 0,
3186				  &rvu->irq_name[offset * NAME_SIZE],
3187				  rvu);
3188		if (ret)
3189			dev_err(rvu->dev,
3190				"RVUAF: IRQ registration failed for Mbox1\n");
3191
3192		rvu->irq_allocated[offset] = true;
3193	} else {
3194		ret = cn20k_register_afvf_mbox_intr(rvu, pf_vec_start);
3195		if (ret)
3196			dev_err(rvu->dev,
3197				"RVUAF: IRQ registration failed for Mbox\n");
3198	}
3199
3200	/* Register FLR interrupt handler for AF's VFs */
3201	offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR0;
3202	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0");
3203	ret = request_irq(pci_irq_vector(rvu->pdev, offset),
3204			  rvu_flr_intr_handler, 0,
3205			  &rvu->irq_name[offset * NAME_SIZE], rvu);
3206	if (ret) {
3207		dev_err(rvu->dev,
3208			"RVUAF: IRQ registration failed for RVUAFVF FLR0\n");
3209		goto fail;
3210	}
3211	rvu->irq_allocated[offset] = true;
3212
3213	offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR1;
3214	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1");
3215	ret = request_irq(pci_irq_vector(rvu->pdev, offset),
3216			  rvu_flr_intr_handler, 0,
3217			  &rvu->irq_name[offset * NAME_SIZE], rvu);
3218	if (ret) {
3219		dev_err(rvu->dev,
3220			"RVUAF: IRQ registration failed for RVUAFVF FLR1\n");
3221		goto fail;
3222	}
3223	rvu->irq_allocated[offset] = true;
3224
3225	/* Register ME interrupt handler for AF's VFs */
3226	offset = pf_vec_start + RVU_PF_INT_VEC_VFME0;
3227	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0");
3228	ret = request_irq(pci_irq_vector(rvu->pdev, offset),
3229			  rvu_me_vf_intr_handler, 0,
3230			  &rvu->irq_name[offset * NAME_SIZE], rvu);
3231	if (ret) {
3232		dev_err(rvu->dev,
3233			"RVUAF: IRQ registration failed for RVUAFVF ME0\n");
3234		goto fail;
3235	}
3236	rvu->irq_allocated[offset] = true;
3237
3238	offset = pf_vec_start + RVU_PF_INT_VEC_VFME1;
3239	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1");
3240	ret = request_irq(pci_irq_vector(rvu->pdev, offset),
3241			  rvu_me_vf_intr_handler, 0,
3242			  &rvu->irq_name[offset * NAME_SIZE], rvu);
3243	if (ret) {
3244		dev_err(rvu->dev,
3245			"RVUAF: IRQ registration failed for RVUAFVF ME1\n");
3246		goto fail;
3247	}
3248	rvu->irq_allocated[offset] = true;
3249
3250	ret = rvu_cpt_register_interrupts(rvu);
3251	if (ret)
3252		goto fail;
3253
3254	return 0;
3255
3256fail:
3257	rvu_unregister_interrupts(rvu);
3258	return ret;
3259}
3260
3261static void rvu_flr_wq_destroy(struct rvu *rvu)
3262{
3263	if (rvu->flr_wq) {
3264		destroy_workqueue(rvu->flr_wq);
3265		rvu->flr_wq = NULL;
3266	}
3267}
3268
3269static int rvu_flr_init(struct rvu *rvu)
3270{
3271	int dev, num_devs;
3272	u64 cfg;
3273	int pf;
3274
3275	/* Enable FLR for all PFs*/
3276	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
3277		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
3278		rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf),
3279			    cfg | BIT_ULL(22));
3280	}
3281
3282	rvu->flr_wq = alloc_ordered_workqueue("rvu_afpf_flr",
3283					      WQ_HIGHPRI | WQ_MEM_RECLAIM);
3284	if (!rvu->flr_wq)
3285		return -ENOMEM;
3286
3287	num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev);
3288	rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs,
3289				    sizeof(struct rvu_work), GFP_KERNEL);
3290	if (!rvu->flr_wrk) {
3291		destroy_workqueue(rvu->flr_wq);
3292		return -ENOMEM;
3293	}
3294
3295	for (dev = 0; dev < num_devs; dev++) {
3296		rvu->flr_wrk[dev].rvu = rvu;
3297		INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler);
3298	}
3299
3300	mutex_init(&rvu->flr_lock);
3301
3302	return 0;
3303}
3304
3305static void rvu_disable_afvf_intr(struct rvu *rvu)
3306{
3307	int vfs = rvu->vfs;
3308
3309	if (is_cn20k(rvu->pdev))
3310		return cn20k_rvu_disable_afvf_intr(rvu, vfs);
3311
3312	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs));
3313	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
3314	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
3315	if (vfs <= 64)
3316		return;
3317
3318	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),
3319		      INTR_MASK(vfs - 64));
3320	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
3321	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
3322}
3323
3324static void rvu_enable_afvf_intr(struct rvu *rvu)
3325{
3326	int vfs = rvu->vfs;
3327
3328	if (is_cn20k(rvu->pdev))
3329		return cn20k_rvu_enable_afvf_intr(rvu, vfs);
3330
3331	/* Clear any pending interrupts and enable AF VF interrupts for
3332	 * the first 64 VFs.
3333	 */
3334	/* Mbox */
3335	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs));
3336	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs));
3337
3338	/* FLR */
3339	rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs));
3340	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs));
3341	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs));
3342
3343	/* Same for remaining VFs, if any. */
3344	if (vfs <= 64)
3345		return;
3346
3347	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64));
3348	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
3349		      INTR_MASK(vfs - 64));
3350
3351	rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64));
3352	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
3353	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
3354}
3355
3356int rvu_get_num_lbk_chans(void)
3357{
3358	struct pci_dev *pdev;
3359	void __iomem *base;
3360	int ret = -EIO;
3361
3362	pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_LBK,
3363			      NULL);
3364	if (!pdev)
3365		goto err;
3366
3367	base = pci_ioremap_bar(pdev, 0);
3368	if (!base)
3369		goto err_put;
3370
3371	/* Read number of available LBK channels from LBK(0)_CONST register. */
3372	ret = (readq(base + 0x10) >> 32) & 0xffff;
3373	iounmap(base);
3374err_put:
3375	pci_dev_put(pdev);
3376err:
3377	return ret;
3378}
3379
3380static int rvu_enable_sriov(struct rvu *rvu)
3381{
3382	struct pci_dev *pdev = rvu->pdev;
3383	int err, chans, vfs;
3384	int pos = 0;
3385
3386	if (!rvu_afvf_msix_vectors_num_ok(rvu)) {
3387		dev_warn(&pdev->dev,
3388			 "Skipping SRIOV enablement since not enough IRQs are available\n");
3389		return 0;
3390	}
3391
3392	/* Get RVU VFs device id */
3393	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
3394	if (!pos)
3395		return 0;
3396	pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &rvu->vf_devid);
3397
3398	chans = rvu_get_num_lbk_chans();
3399	if (chans < 0)
3400		return chans;
3401
3402	vfs = pci_sriov_get_totalvfs(pdev);
3403
3404	/* Limit VFs in case we have more VFs than LBK channels available. */
3405	if (vfs > chans)
3406		vfs = chans;
3407
3408	if (!vfs)
3409		return 0;
3410
3411	/* LBK channel number 63 is used for switching packets between
3412	 * CGX mapped VFs. Hence limit LBK pairs till 62 only.
3413	 */
3414	if (vfs > 62)
3415		vfs = 62;
3416
3417	/* Save VFs number for reference in VF interrupts handlers.
3418	 * Since interrupts might start arriving during SRIOV enablement
3419	 * ordinary API cannot be used to get number of enabled VFs.
3420	 */
3421	rvu->vfs = vfs;
3422
3423	err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs,
3424			    rvu_afvf_mbox_handler, rvu_afvf_mbox_up_handler);
3425	if (err)
3426		return err;
3427
3428	rvu_enable_afvf_intr(rvu);
3429	/* Make sure IRQs are enabled before SRIOV. */
3430	mb();
3431
3432	err = pci_enable_sriov(pdev, vfs);
3433	if (err) {
3434		rvu_disable_afvf_intr(rvu);
3435		rvu_mbox_destroy(&rvu->afvf_wq_info);
3436		return err;
3437	}
3438
3439	return 0;
3440}
3441
3442static void rvu_disable_sriov(struct rvu *rvu)
3443{
3444	rvu_disable_afvf_intr(rvu);
3445	rvu_mbox_destroy(&rvu->afvf_wq_info);
3446	pci_disable_sriov(rvu->pdev);
3447}
3448
3449static void rvu_update_module_params(struct rvu *rvu)
3450{
3451	const char *default_pfl_name = "default";
3452
3453	strscpy(rvu->mkex_pfl_name,
3454		mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN);
3455	strscpy(rvu->kpu_pfl_name,
3456		kpu_profile ? kpu_profile : default_pfl_name, KPU_NAME_LEN);
3457}
3458
3459static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3460{
3461	struct device *dev = &pdev->dev;
3462	struct rvu *rvu;
3463	int    err;
3464
3465	rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL);
3466	if (!rvu)
3467		return -ENOMEM;
3468
3469	rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL);
3470	if (!rvu->hw) {
3471		devm_kfree(dev, rvu);
3472		return -ENOMEM;
3473	}
3474
3475	pci_set_drvdata(pdev, rvu);
3476	rvu->pdev = pdev;
3477	rvu->dev = &pdev->dev;
3478
3479	err = pci_enable_device(pdev);
3480	if (err) {
3481		dev_err(dev, "Failed to enable PCI device\n");
3482		goto err_freemem;
3483	}
3484
3485	err = pci_request_regions(pdev, DRV_NAME);
3486	if (err) {
3487		dev_err(dev, "PCI request regions failed 0x%x\n", err);
3488		goto err_disable_device;
3489	}
3490
3491	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
3492	if (err) {
3493		dev_err(dev, "DMA mask config failed, abort\n");
3494		goto err_release_regions;
3495	}
3496
3497	pci_set_master(pdev);
3498
3499	rvu->ptp = ptp_get();
3500	if (IS_ERR(rvu->ptp)) {
3501		err = PTR_ERR(rvu->ptp);
3502		if (err)
3503			goto err_release_regions;
3504		rvu->ptp = NULL;
3505	}
3506
3507	/* Map Admin function CSRs */
3508	rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0);
3509	rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
3510	if (!rvu->afreg_base || !rvu->pfreg_base) {
3511		dev_err(dev, "Unable to map admin function CSRs, aborting\n");
3512		err = -ENOMEM;
3513		goto err_put_ptp;
3514	}
3515
3516	/* Store module params in rvu structure */
3517	rvu_update_module_params(rvu);
3518
3519	/* Check which blocks the HW supports */
3520	rvu_check_block_implemented(rvu);
3521
3522	rvu_reset_all_blocks(rvu);
3523
3524	rvu_setup_hw_capabilities(rvu);
3525
3526	err = rvu_setup_hw_resources(rvu);
3527	if (err)
3528		goto err_put_ptp;
3529
3530	/* Init mailbox btw AF and PFs */
3531	err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF,
3532			    rvu->hw->total_pfs, rvu_afpf_mbox_handler,
3533			    rvu_afpf_mbox_up_handler);
3534	if (err) {
3535		dev_err(dev, "%s: Failed to initialize mbox\n", __func__);
3536		goto err_hwsetup;
3537	}
3538
3539	err = rvu_flr_init(rvu);
3540	if (err) {
3541		dev_err(dev, "%s: Failed to initialize flr\n", __func__);
3542		goto err_mbox;
3543	}
3544
3545	err = rvu_register_interrupts(rvu);
3546	if (err) {
3547		dev_err(dev, "%s: Failed to register interrupts\n", __func__);
3548		goto err_flr;
3549	}
3550
3551	err = rvu_register_dl(rvu);
3552	if (err) {
3553		dev_err(dev, "%s: Failed to register devlink\n", __func__);
3554		goto err_irq;
3555	}
3556
3557	rvu_setup_rvum_blk_revid(rvu);
3558
3559	/* Enable AF's VFs (if any) */
3560	err = rvu_enable_sriov(rvu);
3561	if (err) {
3562		dev_err(dev, "%s: Failed to enable sriov\n", __func__);
3563		goto err_dl;
3564	}
3565
3566	/* Initialize debugfs */
3567	rvu_dbg_init(rvu);
3568
3569	mutex_init(&rvu->rswitch.switch_lock);
3570
3571	if (rvu->fwdata)
3572		ptp_start(rvu, rvu->fwdata->sclk, rvu->fwdata->ptp_ext_clk_rate,
3573			  rvu->fwdata->ptp_ext_tstamp);
3574
3575	/* Alloc CINT and QINT memory */
3576	rvu_alloc_cint_qint_mem(rvu, &rvu->pf[RVU_AFPF], BLKADDR_NIX0,
3577				(rvu->hw->block[BLKADDR_NIX0].lf.max));
3578	return 0;
3579err_dl:
3580	rvu_unregister_dl(rvu);
3581err_irq:
3582	rvu_unregister_interrupts(rvu);
3583err_flr:
3584	rvu_flr_wq_destroy(rvu);
3585err_mbox:
3586	rvu_mbox_destroy(&rvu->afpf_wq_info);
3587err_hwsetup:
3588	rvu_cgx_exit(rvu);
3589	rvu_fwdata_exit(rvu);
3590	rvu_mcs_exit(rvu);
3591	rvu_reset_all_blocks(rvu);
3592	rvu_free_hw_resources(rvu);
3593	rvu_clear_rvum_blk_revid(rvu);
3594err_put_ptp:
3595	ptp_put(rvu->ptp);
3596err_release_regions:
3597	pci_release_regions(pdev);
3598err_disable_device:
3599	pci_disable_device(pdev);
3600err_freemem:
3601	pci_set_drvdata(pdev, NULL);
3602	devm_kfree(&pdev->dev, rvu->hw);
3603	devm_kfree(dev, rvu);
3604	return err;
3605}
3606
3607static void rvu_remove(struct pci_dev *pdev)
3608{
3609	struct rvu *rvu = pci_get_drvdata(pdev);
3610
3611	rvu_dbg_exit(rvu);
3612	rvu_unregister_dl(rvu);
3613	rvu_unregister_interrupts(rvu);
3614	rvu_flr_wq_destroy(rvu);
3615	rvu_cgx_exit(rvu);
3616	rvu_fwdata_exit(rvu);
3617	rvu_mcs_exit(rvu);
3618	rvu_mbox_destroy(&rvu->afpf_wq_info);
3619	rvu_disable_sriov(rvu);
3620	rvu_reset_all_blocks(rvu);
3621	rvu_free_hw_resources(rvu);
3622	rvu_clear_rvum_blk_revid(rvu);
3623	ptp_put(rvu->ptp);
3624	pci_release_regions(pdev);
3625	pci_disable_device(pdev);
3626	pci_set_drvdata(pdev, NULL);
3627
3628	devm_kfree(&pdev->dev, rvu->hw);
3629	if (is_cn20k(rvu->pdev))
3630		cn20k_free_mbox_memory(rvu);
3631	kfree(rvu->ng_rvu);
3632	devm_kfree(&pdev->dev, rvu);
3633}
3634
3635static void rvu_shutdown(struct pci_dev *pdev)
3636{
3637	struct rvu *rvu = pci_get_drvdata(pdev);
3638
3639	if (!rvu)
3640		return;
3641
3642	rvu_clear_rvum_blk_revid(rvu);
3643}
3644
3645static struct pci_driver rvu_driver = {
3646	.name = DRV_NAME,
3647	.id_table = rvu_id_table,
3648	.probe = rvu_probe,
3649	.remove = rvu_remove,
3650	.shutdown = rvu_shutdown,
3651};
3652
3653static int __init rvu_init_module(void)
3654{
3655	int err;
3656
3657	pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
3658
3659	err = pci_register_driver(&cgx_driver);
3660	if (err < 0)
3661		return err;
3662
3663	err = pci_register_driver(&ptp_driver);
3664	if (err < 0)
3665		goto ptp_err;
3666
3667	err = pci_register_driver(&mcs_driver);
3668	if (err < 0)
3669		goto mcs_err;
3670
3671	err =  pci_register_driver(&rvu_driver);
3672	if (err < 0)
3673		goto rvu_err;
3674
3675	return 0;
3676rvu_err:
3677	pci_unregister_driver(&mcs_driver);
3678mcs_err:
3679	pci_unregister_driver(&ptp_driver);
3680ptp_err:
3681	pci_unregister_driver(&cgx_driver);
3682
3683	return err;
3684}
3685
3686static void __exit rvu_cleanup_module(void)
3687{
3688	pci_unregister_driver(&rvu_driver);
3689	pci_unregister_driver(&mcs_driver);
3690	pci_unregister_driver(&ptp_driver);
3691	pci_unregister_driver(&cgx_driver);
3692}
3693
3694module_init(rvu_init_module);
3695module_exit(rvu_cleanup_module);