FILE
rvu_cpt.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/marvell/octeontx2/af/rvu_cpt.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8#include <linux/bitfield.h> 9#include <linux/pci.h> 10#include "rvu_struct.h" 11#include "rvu_reg.h" 12#include "mbox.h" 13#include "rvu.h" 14 15/* CPT PF device id */ 16#define PCI_DEVID_OTX2_CPT_PF 0xA0FD 17#define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2 18 19/* Length of initial context fetch in 128 byte words */ 20#define CPT_CTX_ILEN 1ULL 21 22/* Interrupt vector count of CPT RVU and RAS interrupts */ 23#define CPT_10K_AF_RVU_RAS_INT_VEC_CNT 2 24 25/* Default CPT_AF_RXC_CFG1:max_rxc_icb_cnt */ 26#define CPT_DFLT_MAX_RXC_ICB_CNT 0xC0ULL 27 28#define cpt_get_eng_sts(e_min, e_max, rsp, etype) \ 29({ \ 30 u64 free_sts = 0, busy_sts = 0; \ 31 typeof(rsp) _rsp = rsp; \ 32 u32 e, i; \ 33 \ 34 for (e = (e_min), i = 0; e < (e_max); e++, i++) { \ 35 reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \ 36 if (reg & 0x1) \ 37 busy_sts |= 1ULL << i; \ 38 \ 39 if (reg & 0x2) \ 40 free_sts |= 1ULL << i; \ 41 } \ 42 (_rsp)->busy_sts_##etype = busy_sts; \ 43 (_rsp)->free_sts_##etype = free_sts; \ 44}) 45 46#define MAX_AE GENMASK_ULL(47, 32) 47#define MAX_IE GENMASK_ULL(31, 16) 48#define MAX_SE GENMASK_ULL(15, 0) 49 50static u16 cpt_max_engines_get(struct rvu *rvu) 51{ 52 u16 max_ses, max_ies, max_aes; 53 u64 reg; 54 55 reg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS1); 56 max_ses = FIELD_GET(MAX_SE, reg); 57 max_ies = FIELD_GET(MAX_IE, reg); 58 max_aes = FIELD_GET(MAX_AE, reg); 59 60 return max_ses + max_ies + max_aes; 61} 62 63/* Number of flt interrupt vectors are depends on number of engines that the 64 * chip has. Each flt vector represents 64 engines. 65 */ 66static int cpt_10k_flt_nvecs_get(struct rvu *rvu, u16 max_engs) 67{ 68 int flt_vecs; 69 70 flt_vecs = DIV_ROUND_UP(max_engs, 64); 71 72 if (flt_vecs > CPT_10K_AF_INT_VEC_FLT_MAX) { 73 dev_warn_once(rvu->dev, "flt_vecs:%d exceeds the max vectors:%d\n", 74 flt_vecs, CPT_10K_AF_INT_VEC_FLT_MAX); 75 flt_vecs = CPT_10K_AF_INT_VEC_FLT_MAX; 76 } 77 78 return flt_vecs; 79} 80 81static irqreturn_t cpt_af_flt_intr_handler(int vec, void *ptr) 82{ 83 struct rvu_block *block = ptr; 84 struct rvu *rvu = block->rvu; 85 int blkaddr = block->addr; 86 u64 reg, val; 87 int i, eng; 88 u8 grp; 89 90 reg = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(vec)); 91 dev_err_ratelimited(rvu->dev, "Received CPTAF FLT%d irq : 0x%llx", vec, reg); 92 93 i = -1; 94 while ((i = find_next_bit((unsigned long *)®, 64, i + 1)) < 64) { 95 switch (vec) { 96 case 0: 97 eng = i; 98 break; 99 case 1: 100 eng = i + 64; 101 break; 102 case 2: 103 eng = i + 128; 104 break; 105 } 106 grp = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng)) & 0xFF; 107 /* Disable and enable the engine which triggers fault */ 108 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), 0x0); 109 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng)); 110 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val & ~1ULL); 111 112 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), grp); 113 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val | 1ULL); 114 115 spin_lock(&rvu->cpt_intr_lock); 116 block->cpt_flt_eng_map[vec] |= BIT_ULL(i); 117 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(eng)); 118 val = val & 0x3; 119 if (val == 0x1 || val == 0x2) 120 block->cpt_rcvrd_eng_map[vec] |= BIT_ULL(i); 121 spin_unlock(&rvu->cpt_intr_lock); 122 } 123 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(vec), reg); 124 125 return IRQ_HANDLED; 126} 127 128static irqreturn_t rvu_cpt_af_flt0_intr_handler(int irq, void *ptr) 129{ 130 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT0, ptr); 131} 132 133static irqreturn_t rvu_cpt_af_flt1_intr_handler(int irq, void *ptr) 134{ 135 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT1, ptr); 136} 137 138static irqreturn_t rvu_cpt_af_flt2_intr_handler(int irq, void *ptr) 139{ 140 return cpt_af_flt_intr_handler(CPT_10K_AF_INT_VEC_FLT2, ptr); 141} 142 143static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr) 144{ 145 struct rvu_block *block = ptr; 146 struct rvu *rvu = block->rvu; 147 int blkaddr = block->addr; 148 u64 reg; 149 150 reg = rvu_read64(rvu, blkaddr, CPT_AF_RVU_INT); 151 dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx", reg); 152 153 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT, reg); 154 return IRQ_HANDLED; 155} 156 157static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr) 158{ 159 struct rvu_block *block = ptr; 160 struct rvu *rvu = block->rvu; 161 int blkaddr = block->addr; 162 u64 reg; 163 164 reg = rvu_read64(rvu, blkaddr, CPT_AF_RAS_INT); 165 dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx", reg); 166 167 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT, reg); 168 return IRQ_HANDLED; 169} 170 171static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs, 172 irq_handler_t handler, 173 const char *name) 174{ 175 struct rvu *rvu = block->rvu; 176 int ret; 177 178 ret = request_irq(pci_irq_vector(rvu->pdev, irq_offs), handler, 0, 179 name, block); 180 if (ret) { 181 dev_err(rvu->dev, "RVUAF: %s irq registration failed", name); 182 return ret; 183 } 184 185 WARN_ON(rvu->irq_allocated[irq_offs]); 186 rvu->irq_allocated[irq_offs] = true; 187 return 0; 188} 189 190static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off) 191{ 192 struct rvu *rvu = block->rvu; 193 int blkaddr = block->addr; 194 int i, flt_vecs; 195 u16 max_engs; 196 u8 nr; 197 198 max_engs = cpt_max_engines_get(rvu); 199 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs); 200 201 /* Disable all CPT AF interrupts */ 202 for (i = CPT_10K_AF_INT_VEC_FLT0; i < flt_vecs; i++) { 203 nr = (max_engs > 64) ? 64 : max_engs; 204 max_engs -= nr; 205 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), 206 INTR_MASK(nr)); 207 } 208 209 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1); 210 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1); 211 212 /* CPT AF interrupt vectors are flt_int, rvu_int and ras_int. */ 213 for (i = 0; i < flt_vecs + CPT_10K_AF_RVU_RAS_INT_VEC_CNT; i++) 214 if (rvu->irq_allocated[off + i]) { 215 free_irq(pci_irq_vector(rvu->pdev, off + i), block); 216 rvu->irq_allocated[off + i] = false; 217 } 218} 219 220static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr) 221{ 222 struct rvu_hwinfo *hw = rvu->hw; 223 struct rvu_block *block; 224 int i, offs; 225 226 if (!is_block_implemented(rvu->hw, blkaddr)) 227 return; 228 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF; 229 if (!offs) { 230 dev_warn(rvu->dev, 231 "Failed to get CPT_AF_INT vector offsets\n"); 232 return; 233 } 234 block = &hw->block[blkaddr]; 235 if (!is_rvu_otx2(rvu)) 236 return cpt_10k_unregister_interrupts(block, offs); 237 238 /* Disable all CPT AF interrupts */ 239 for (i = 0; i < CPT_AF_INT_VEC_RVU; i++) 240 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), ~0ULL); 241 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1); 242 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1); 243 244 for (i = 0; i < CPT_AF_INT_VEC_CNT; i++) 245 if (rvu->irq_allocated[offs + i]) { 246 free_irq(pci_irq_vector(rvu->pdev, offs + i), block); 247 rvu->irq_allocated[offs + i] = false; 248 } 249} 250 251void rvu_cpt_unregister_interrupts(struct rvu *rvu) 252{ 253 cpt_unregister_interrupts(rvu, BLKADDR_CPT0); 254 cpt_unregister_interrupts(rvu, BLKADDR_CPT1); 255} 256 257static int cpt_10k_register_interrupts(struct rvu_block *block, int off) 258{ 259 int rvu_intr_vec, ras_intr_vec; 260 struct rvu *rvu = block->rvu; 261 int blkaddr = block->addr; 262 irq_handler_t flt_fn; 263 int i, ret, flt_vecs; 264 u16 max_engs; 265 u8 nr; 266 267 max_engs = cpt_max_engines_get(rvu); 268 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs); 269 270 for (i = CPT_10K_AF_INT_VEC_FLT0; i < flt_vecs; i++) { 271 sprintf(&rvu->irq_name[(off + i) * NAME_SIZE], "CPTAF FLT%d", i); 272 273 switch (i) { 274 case CPT_10K_AF_INT_VEC_FLT0: 275 flt_fn = rvu_cpt_af_flt0_intr_handler; 276 break; 277 case CPT_10K_AF_INT_VEC_FLT1: 278 flt_fn = rvu_cpt_af_flt1_intr_handler; 279 break; 280 case CPT_10K_AF_INT_VEC_FLT2: 281 flt_fn = rvu_cpt_af_flt2_intr_handler; 282 break; 283 } 284 ret = rvu_cpt_do_register_interrupt(block, off + i, 285 flt_fn, &rvu->irq_name[(off + i) * NAME_SIZE]); 286 if (ret) 287 goto err; 288 289 nr = (max_engs > 64) ? 64 : max_engs; 290 max_engs -= nr; 291 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), 292 INTR_MASK(nr)); 293 } 294 295 rvu_intr_vec = flt_vecs; 296 ras_intr_vec = rvu_intr_vec + 1; 297 298 ret = rvu_cpt_do_register_interrupt(block, off + rvu_intr_vec, 299 rvu_cpt_af_rvu_intr_handler, 300 "CPTAF RVU"); 301 if (ret) 302 goto err; 303 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1); 304 305 ret = rvu_cpt_do_register_interrupt(block, off + ras_intr_vec, 306 rvu_cpt_af_ras_intr_handler, 307 "CPTAF RAS"); 308 if (ret) 309 goto err; 310 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1); 311 312 return 0; 313err: 314 rvu_cpt_unregister_interrupts(rvu); 315 return ret; 316} 317 318static int cpt_register_interrupts(struct rvu *rvu, int blkaddr) 319{ 320 struct rvu_hwinfo *hw = rvu->hw; 321 struct rvu_block *block; 322 irq_handler_t flt_fn; 323 int i, offs, ret = 0; 324 325 if (!is_block_implemented(rvu->hw, blkaddr)) 326 return 0; 327 328 block = &hw->block[blkaddr]; 329 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF; 330 if (!offs) { 331 dev_warn(rvu->dev, 332 "Failed to get CPT_AF_INT vector offsets\n"); 333 return 0; 334 } 335 336 if (!is_rvu_otx2(rvu)) 337 return cpt_10k_register_interrupts(block, offs); 338 339 for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) { 340 sprintf(&rvu->irq_name[(offs + i) * NAME_SIZE], "CPTAF FLT%d", i); 341 switch (i) { 342 case CPT_AF_INT_VEC_FLT0: 343 flt_fn = rvu_cpt_af_flt0_intr_handler; 344 break; 345 case CPT_AF_INT_VEC_FLT1: 346 flt_fn = rvu_cpt_af_flt1_intr_handler; 347 break; 348 } 349 ret = rvu_cpt_do_register_interrupt(block, offs + i, 350 flt_fn, &rvu->irq_name[(offs + i) * NAME_SIZE]); 351 if (ret) 352 goto err; 353 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL); 354 } 355 356 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RVU, 357 rvu_cpt_af_rvu_intr_handler, 358 "CPTAF RVU"); 359 if (ret) 360 goto err; 361 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1); 362 363 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RAS, 364 rvu_cpt_af_ras_intr_handler, 365 "CPTAF RAS"); 366 if (ret) 367 goto err; 368 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1); 369 370 return 0; 371err: 372 rvu_cpt_unregister_interrupts(rvu); 373 return ret; 374} 375 376int rvu_cpt_register_interrupts(struct rvu *rvu) 377{ 378 int ret; 379 380 ret = cpt_register_interrupts(rvu, BLKADDR_CPT0); 381 if (ret) 382 return ret; 383 384 return cpt_register_interrupts(rvu, BLKADDR_CPT1); 385} 386 387static int get_cpt_pf_num(struct rvu *rvu) 388{ 389 int i, domain_nr, cpt_pf_num = -1; 390 struct pci_dev *pdev; 391 392 domain_nr = pci_domain_nr(rvu->pdev->bus); 393 for (i = 0; i < rvu->hw->total_pfs; i++) { 394 pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0); 395 if (!pdev) 396 continue; 397 398 if (pdev->device == PCI_DEVID_OTX2_CPT_PF || 399 pdev->device == PCI_DEVID_OTX2_CPT10K_PF) { 400 cpt_pf_num = i; 401 put_device(&pdev->dev); 402 break; 403 } 404 put_device(&pdev->dev); 405 } 406 return cpt_pf_num; 407} 408 409static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc) 410{ 411 int cpt_pf_num = rvu->cpt_pf_num; 412 413 if (rvu_get_pf(rvu->pdev, pcifunc) != cpt_pf_num) 414 return false; 415 if (pcifunc & RVU_PFVF_FUNC_MASK) 416 return false; 417 418 return true; 419} 420 421static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc) 422{ 423 int cpt_pf_num = rvu->cpt_pf_num; 424 425 if (rvu_get_pf(rvu->pdev, pcifunc) != cpt_pf_num) 426 return false; 427 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 428 return false; 429 430 return true; 431} 432 433static int validate_and_get_cpt_blkaddr(int req_blkaddr) 434{ 435 int blkaddr; 436 437 blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0; 438 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1) 439 return -EINVAL; 440 441 return blkaddr; 442} 443 444int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu, 445 struct cpt_lf_alloc_req_msg *req, 446 struct msg_rsp *rsp) 447{ 448 u16 pcifunc = req->hdr.pcifunc; 449 struct rvu_block *block; 450 int cptlf, blkaddr; 451 int num_lfs, slot; 452 u64 val; 453 454 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 455 if (blkaddr < 0) 456 return blkaddr; 457 458 if (req->eng_grpmsk == 0x0) 459 return CPT_AF_ERR_GRP_INVALID; 460 461 block = &rvu->hw->block[blkaddr]; 462 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 463 block->addr); 464 if (!num_lfs) 465 return CPT_AF_ERR_LF_INVALID; 466 467 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ 468 if (req->nix_pf_func) { 469 /* If default, use 'this' CPTLF's PFFUNC */ 470 if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC) 471 req->nix_pf_func = pcifunc; 472 if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX)) 473 return CPT_AF_ERR_NIX_PF_FUNC_INVALID; 474 } 475 476 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ 477 if (req->sso_pf_func) { 478 /* If default, use 'this' CPTLF's PFFUNC */ 479 if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC) 480 req->sso_pf_func = pcifunc; 481 if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO)) 482 return CPT_AF_ERR_SSO_PF_FUNC_INVALID; 483 } 484 485 for (slot = 0; slot < num_lfs; slot++) { 486 cptlf = rvu_get_lf(rvu, block, pcifunc, slot); 487 if (cptlf < 0) 488 return CPT_AF_ERR_LF_INVALID; 489 490 /* Set CPT LF group and priority */ 491 val = (u64)req->eng_grpmsk << 48 | 1; 492 if (!is_rvu_otx2(rvu)) { 493 if (req->ctx_ilen_valid) 494 val |= (req->ctx_ilen << 17); 495 else 496 val |= (CPT_CTX_ILEN << 17); 497 } 498 499 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 500 501 /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set 502 * on reset. 503 */ 504 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 505 val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32)); 506 val |= ((u64)req->nix_pf_func << 48 | 507 (u64)req->sso_pf_func << 32); 508 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 509 } 510 511 return 0; 512} 513 514static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr) 515{ 516 u16 pcifunc = req->hdr.pcifunc; 517 int num_lfs, cptlf, slot, err; 518 struct rvu_block *block; 519 520 block = &rvu->hw->block[blkaddr]; 521 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 522 block->addr); 523 if (!num_lfs) 524 return 0; 525 526 for (slot = 0; slot < num_lfs; slot++) { 527 cptlf = rvu_get_lf(rvu, block, pcifunc, slot); 528 if (cptlf < 0) 529 return CPT_AF_ERR_LF_INVALID; 530 531 /* Perform teardown */ 532 rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, cptlf, slot); 533 534 /* Reset LF */ 535 err = rvu_lf_reset(rvu, block, cptlf); 536 if (err) { 537 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n", 538 block->addr, cptlf); 539 } 540 } 541 542 return 0; 543} 544 545int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req, 546 struct msg_rsp *rsp) 547{ 548 int ret; 549 550 ret = cpt_lf_free(rvu, req, BLKADDR_CPT0); 551 if (ret) 552 return ret; 553 554 if (is_block_implemented(rvu->hw, BLKADDR_CPT1)) 555 ret = cpt_lf_free(rvu, req, BLKADDR_CPT1); 556 557 return ret; 558} 559 560static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf, 561 struct cpt_inline_ipsec_cfg_msg *req) 562{ 563 u16 sso_pf_func = req->sso_pf_func; 564 u8 nix_sel; 565 u64 val; 566 567 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 568 if (req->enable && (val & BIT_ULL(16))) { 569 /* IPSec inline outbound path is already enabled for a given 570 * CPT LF, HRM states that inline inbound & outbound paths 571 * must not be enabled at the same time for a given CPT LF 572 */ 573 return CPT_AF_ERR_INLINE_IPSEC_INB_ENA; 574 } 575 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ 576 if (sso_pf_func && !is_pffunc_map_valid(rvu, sso_pf_func, BLKTYPE_SSO)) 577 return CPT_AF_ERR_SSO_PF_FUNC_INVALID; 578 579 nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0; 580 /* Enable CPT LF for IPsec inline inbound operations */ 581 if (req->enable) 582 val |= BIT_ULL(9); 583 else 584 val &= ~BIT_ULL(9); 585 586 val |= (u64)nix_sel << 8; 587 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 588 589 if (sso_pf_func) { 590 /* Set SSO_PF_FUNC */ 591 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 592 val |= (u64)sso_pf_func << 32; 593 val |= (u64)req->nix_pf_func << 48; 594 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 595 } 596 if (req->sso_pf_func_ovrd) 597 /* Set SSO_PF_FUNC_OVRD for inline IPSec */ 598 rvu_write64(rvu, blkaddr, CPT_AF_ECO, 0x1); 599 600 /* Configure the X2P Link register with the cpt base channel number and 601 * range of channels it should propagate to X2P 602 */ 603 if (!is_rvu_otx2(rvu)) { 604 val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16); 605 val |= (u64)rvu->hw->cpt_chan_base; 606 607 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val); 608 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val); 609 } 610 611 return 0; 612} 613 614static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf, 615 struct cpt_inline_ipsec_cfg_msg *req) 616{ 617 u16 nix_pf_func = req->nix_pf_func; 618 int nix_blkaddr; 619 u8 nix_sel; 620 u64 val; 621 622 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 623 if (req->enable && (val & BIT_ULL(9))) { 624 /* IPSec inline inbound path is already enabled for a given 625 * CPT LF, HRM states that inline inbound & outbound paths 626 * must not be enabled at the same time for a given CPT LF 627 */ 628 return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA; 629 } 630 631 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ 632 if (nix_pf_func && !is_pffunc_map_valid(rvu, nix_pf_func, BLKTYPE_NIX)) 633 return CPT_AF_ERR_NIX_PF_FUNC_INVALID; 634 635 /* Enable CPT LF for IPsec inline outbound operations */ 636 if (req->enable) 637 val |= BIT_ULL(16); 638 else 639 val &= ~BIT_ULL(16); 640 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 641 642 if (nix_pf_func) { 643 /* Set NIX_PF_FUNC */ 644 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 645 val |= (u64)nix_pf_func << 48; 646 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 647 648 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, nix_pf_func); 649 nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1; 650 651 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 652 val |= (u64)nix_sel << 8; 653 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 654 } 655 656 return 0; 657} 658 659int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu, 660 struct cpt_inline_ipsec_cfg_msg *req, 661 struct msg_rsp *rsp) 662{ 663 u16 pcifunc = req->hdr.pcifunc; 664 struct rvu_block *block; 665 int cptlf, blkaddr, ret; 666 u16 actual_slot; 667 668 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc, 669 req->slot, &actual_slot); 670 if (blkaddr < 0) 671 return CPT_AF_ERR_LF_INVALID; 672 673 block = &rvu->hw->block[blkaddr]; 674 675 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot); 676 if (cptlf < 0) 677 return CPT_AF_ERR_LF_INVALID; 678 679 switch (req->dir) { 680 case CPT_INLINE_INBOUND: 681 ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req); 682 break; 683 684 case CPT_INLINE_OUTBOUND: 685 ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req); 686 break; 687 688 default: 689 return CPT_AF_ERR_PARAM; 690 } 691 692 return ret; 693} 694 695static bool validate_and_update_reg_offset(struct rvu *rvu, 696 struct cpt_rd_wr_reg_msg *req, 697 u64 *reg_offset) 698{ 699 u64 offset = req->reg_offset; 700 int blkaddr, num_lfs, lf; 701 struct rvu_block *block; 702 struct rvu_pfvf *pfvf; 703 704 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 705 if (blkaddr < 0) 706 return false; 707 708 /* Registers that can be accessed from PF/VF */ 709 if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) || 710 (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) { 711 if (offset & 7) 712 return false; 713 714 lf = (offset & 0xFFF) >> 3; 715 block = &rvu->hw->block[blkaddr]; 716 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 717 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr); 718 if (lf >= num_lfs) 719 /* Slot is not valid for that PF/VF */ 720 return false; 721 722 /* Translate local LF used by VFs to global CPT LF */ 723 lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr], 724 req->hdr.pcifunc, lf); 725 if (lf < 0) 726 return false; 727 728 /* Translate local LF's offset to global CPT LF's offset to 729 * access LFX register. 730 */ 731 *reg_offset = (req->reg_offset & 0xFF000) + (lf << 3); 732 733 return true; 734 } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) { 735 /* Registers that can be accessed from PF */ 736 switch (offset) { 737 case CPT_AF_DIAG: 738 case CPT_AF_CTL: 739 case CPT_AF_PF_FUNC: 740 case CPT_AF_BLK_RST: 741 case CPT_AF_CONSTANTS1: 742 case CPT_AF_CTX_FLUSH_TIMER: 743 case CPT_AF_RXC_CFG1: 744 return true; 745 } 746 747 switch (offset & 0xFF000) { 748 case CPT_AF_EXEX_STS(0): 749 case CPT_AF_EXEX_CTL(0): 750 case CPT_AF_EXEX_CTL2(0): 751 case CPT_AF_EXEX_UCODE_BASE(0): 752 if (offset & 7) 753 return false; 754 break; 755 default: 756 return false; 757 } 758 return true; 759 } 760 return false; 761} 762 763int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu, 764 struct cpt_rd_wr_reg_msg *req, 765 struct cpt_rd_wr_reg_msg *rsp) 766{ 767 u64 offset = req->reg_offset; 768 int blkaddr; 769 770 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 771 if (blkaddr < 0) 772 return blkaddr; 773 774 /* This message is accepted only if sent from CPT PF/VF */ 775 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 776 !is_cpt_vf(rvu, req->hdr.pcifunc)) 777 return CPT_AF_ERR_ACCESS_DENIED; 778 779 if (!validate_and_update_reg_offset(rvu, req, &offset)) 780 return CPT_AF_ERR_ACCESS_DENIED; 781 782 rsp->reg_offset = req->reg_offset; 783 rsp->ret_val = req->ret_val; 784 rsp->is_write = req->is_write; 785 786 if (req->is_write) 787 rvu_write64(rvu, blkaddr, offset, req->val); 788 else 789 rsp->val = rvu_read64(rvu, blkaddr, offset); 790 791 return 0; 792} 793 794static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) 795{ 796 struct rvu_hwinfo *hw = rvu->hw; 797 798 if (is_rvu_otx2(rvu)) 799 return; 800 801 rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC); 802 rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC); 803 rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC); 804 rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr, 805 CPT_AF_CTX_AOP_LATENCY_PC); 806 rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC); 807 rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr, 808 CPT_AF_CTX_IFETCH_LATENCY_PC); 809 rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 810 rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr, 811 CPT_AF_CTX_FFETCH_LATENCY_PC); 812 rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 813 rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr, 814 CPT_AF_CTX_FFETCH_LATENCY_PC); 815 rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 816 rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr, 817 CPT_AF_CTX_FFETCH_LATENCY_PC); 818 rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR); 819 rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID); 820 rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER); 821 rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0)); 822 rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1)); 823 824 if (!hw->cap.cpt_rxc) 825 return; 826 rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME); 827 rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG); 828 rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS); 829 rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS); 830 rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG); 831} 832 833static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) 834{ 835 u16 max_ses, max_ies, max_aes; 836 u32 e_min = 0, e_max = 0; 837 u64 reg; 838 839 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1); 840 max_ses = reg & 0xffff; 841 max_ies = (reg >> 16) & 0xffff; 842 max_aes = (reg >> 32) & 0xffff; 843 844 /* Get AE status */ 845 e_min = max_ses + max_ies; 846 e_max = max_ses + max_ies + max_aes; 847 cpt_get_eng_sts(e_min, e_max, rsp, ae); 848 /* Get SE status */ 849 e_min = 0; 850 e_max = max_ses; 851 cpt_get_eng_sts(e_min, e_max, rsp, se); 852 /* Get IE status */ 853 e_min = max_ses; 854 e_max = max_ses + max_ies; 855 cpt_get_eng_sts(e_min, e_max, rsp, ie); 856} 857 858int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req, 859 struct cpt_sts_rsp *rsp) 860{ 861 int blkaddr; 862 863 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 864 if (blkaddr < 0) 865 return blkaddr; 866 867 /* This message is accepted only if sent from CPT PF/VF */ 868 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 869 !is_cpt_vf(rvu, req->hdr.pcifunc)) 870 return CPT_AF_ERR_ACCESS_DENIED; 871 872 get_ctx_pc(rvu, rsp, blkaddr); 873 874 /* Get CPT engines status */ 875 get_eng_sts(rvu, rsp, blkaddr); 876 877 /* Read CPT instruction PC registers */ 878 rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC); 879 rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC); 880 rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC); 881 rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC); 882 rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC); 883 rsp->active_cycles_pc = rvu_read64(rvu, blkaddr, 884 CPT_AF_ACTIVE_CYCLES_PC); 885 rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO); 886 rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT); 887 rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG); 888 889 return 0; 890} 891 892#define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48) 893#define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32) 894#define RXC_ACTIVE_THRES GENMASK_ULL(27, 16) 895#define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0) 896#define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48) 897#define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48) 898 899static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req, 900 int blkaddr, struct cpt_rxc_time_cfg_req *save) 901{ 902 u64 dfrg_reg; 903 904 if (save) { 905 /* Save older config */ 906 dfrg_reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG); 907 save->zombie_thres = FIELD_GET(RXC_ZOMBIE_THRES, dfrg_reg); 908 save->zombie_limit = FIELD_GET(RXC_ZOMBIE_LIMIT, dfrg_reg); 909 save->active_thres = FIELD_GET(RXC_ACTIVE_THRES, dfrg_reg); 910 save->active_limit = FIELD_GET(RXC_ACTIVE_LIMIT, dfrg_reg); 911 912 save->step = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG); 913 } 914 915 dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres); 916 dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit); 917 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres); 918 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit); 919 920 rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step); 921 rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg); 922} 923 924int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu, 925 struct cpt_rxc_time_cfg_req *req, 926 struct msg_rsp *rsp) 927{ 928 int blkaddr; 929 930 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 931 if (blkaddr < 0) 932 return blkaddr; 933 934 /* This message is accepted only if sent from CPT PF/VF */ 935 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 936 !is_cpt_vf(rvu, req->hdr.pcifunc)) 937 return CPT_AF_ERR_ACCESS_DENIED; 938 939 cpt_rxc_time_cfg(rvu, req, blkaddr, NULL); 940 941 return 0; 942} 943 944int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req, 945 struct msg_rsp *rsp) 946{ 947 return rvu_cpt_ctx_flush(rvu, req->hdr.pcifunc); 948} 949 950int rvu_mbox_handler_cpt_lf_reset(struct rvu *rvu, struct cpt_lf_rst_req *req, 951 struct msg_rsp *rsp) 952{ 953 u16 pcifunc = req->hdr.pcifunc; 954 struct rvu_block *block; 955 int cptlf, blkaddr, ret; 956 u16 actual_slot; 957 u64 ctl, ctl2; 958 959 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc, 960 req->slot, &actual_slot); 961 if (blkaddr < 0) 962 return CPT_AF_ERR_LF_INVALID; 963 964 block = &rvu->hw->block[blkaddr]; 965 966 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot); 967 if (cptlf < 0) 968 return CPT_AF_ERR_LF_INVALID; 969 ctl = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 970 ctl2 = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 971 972 ret = rvu_lf_reset(rvu, block, cptlf); 973 if (ret) 974 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n", 975 block->addr, cptlf); 976 977 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), ctl); 978 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), ctl2); 979 980 return 0; 981} 982 983int rvu_mbox_handler_cpt_flt_eng_info(struct rvu *rvu, struct cpt_flt_eng_info_req *req, 984 struct cpt_flt_eng_info_rsp *rsp) 985{ 986 struct rvu_block *block; 987 unsigned long flags; 988 int blkaddr, vec; 989 int flt_vecs; 990 u16 max_engs; 991 992 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 993 if (blkaddr < 0) 994 return blkaddr; 995 996 block = &rvu->hw->block[blkaddr]; 997 max_engs = cpt_max_engines_get(rvu); 998 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs); 999 for (vec = 0; vec < flt_vecs; vec++) { 1000 spin_lock_irqsave(&rvu->cpt_intr_lock, flags); 1001 rsp->flt_eng_map[vec] = block->cpt_flt_eng_map[vec]; 1002 rsp->rcvrd_eng_map[vec] = block->cpt_rcvrd_eng_map[vec]; 1003 if (req->reset) { 1004 block->cpt_flt_eng_map[vec] = 0x0; 1005 block->cpt_rcvrd_eng_map[vec] = 0x0; 1006 } 1007 spin_unlock_irqrestore(&rvu->cpt_intr_lock, flags); 1008 } 1009 return 0; 1010} 1011 1012static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr) 1013{ 1014 struct cpt_rxc_time_cfg_req req, prev; 1015 struct rvu_hwinfo *hw = rvu->hw; 1016 int timeout = 2000; 1017 u64 reg; 1018 1019 if (!hw->cap.cpt_rxc) 1020 return; 1021 1022 /* Set time limit to minimum values, so that rxc entries will be 1023 * flushed out quickly. 1024 */ 1025 req.step = 1; 1026 req.zombie_thres = 1; 1027 req.zombie_limit = 1; 1028 req.active_thres = 1; 1029 req.active_limit = 1; 1030 1031 cpt_rxc_time_cfg(rvu, &req, blkaddr, &prev); 1032 1033 do { 1034 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS); 1035 udelay(1); 1036 if (FIELD_GET(RXC_ACTIVE_COUNT, reg)) 1037 timeout--; 1038 else 1039 break; 1040 } while (timeout); 1041 1042 if (timeout == 0) 1043 dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n"); 1044 1045 timeout = 2000; 1046 do { 1047 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS); 1048 udelay(1); 1049 if (FIELD_GET(RXC_ZOMBIE_COUNT, reg)) 1050 timeout--; 1051 else 1052 break; 1053 } while (timeout); 1054 1055 if (timeout == 0) 1056 dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n"); 1057 1058 /* Restore config */ 1059 cpt_rxc_time_cfg(rvu, &prev, blkaddr, NULL); 1060} 1061 1062#define INFLIGHT GENMASK_ULL(8, 0) 1063#define GRB_CNT GENMASK_ULL(39, 32) 1064#define GWB_CNT GENMASK_ULL(47, 40) 1065#define XQ_XOR GENMASK_ULL(63, 63) 1066#define DQPTR GENMASK_ULL(19, 0) 1067#define NQPTR GENMASK_ULL(51, 32) 1068 1069static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot) 1070{ 1071 int timeout = 1000000; 1072 u64 inprog, inst_ptr; 1073 u64 qsize, pending; 1074 int i = 0; 1075 1076 /* Disable instructions enqueuing */ 1077 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0); 1078 1079 inprog = rvu_read64(rvu, blkaddr, 1080 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 1081 inprog |= BIT_ULL(16); 1082 rvu_write64(rvu, blkaddr, 1083 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog); 1084 1085 qsize = rvu_read64(rvu, blkaddr, 1086 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_SIZE)) & 0x7FFF; 1087 do { 1088 inst_ptr = rvu_read64(rvu, blkaddr, 1089 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_INST_PTR)); 1090 pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) + 1091 FIELD_GET(NQPTR, inst_ptr) - 1092 FIELD_GET(DQPTR, inst_ptr); 1093 udelay(1); 1094 timeout--; 1095 } while ((pending != 0) && (timeout != 0)); 1096 1097 if (timeout == 0) 1098 dev_warn(rvu->dev, "TIMEOUT: CPT poll on pending instructions\n"); 1099 1100 timeout = 1000000; 1101 /* Wait for CPT queue to become execution-quiescent */ 1102 do { 1103 inprog = rvu_read64(rvu, blkaddr, 1104 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 1105 1106 if ((FIELD_GET(INFLIGHT, inprog) == 0) && 1107 (FIELD_GET(GRB_CNT, inprog) == 0)) { 1108 i++; 1109 } else { 1110 i = 0; 1111 timeout--; 1112 } 1113 } while ((timeout != 0) && (i < 10)); 1114 1115 if (timeout == 0) 1116 dev_warn(rvu->dev, "TIMEOUT: CPT poll on inflight count\n"); 1117 /* Wait for 2 us to flush all queue writes to memory */ 1118 udelay(2); 1119} 1120 1121int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot) 1122{ 1123 u64 reg; 1124 1125 if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc)) 1126 cpt_rxc_teardown(rvu, blkaddr); 1127 1128 mutex_lock(&rvu->alias_lock); 1129 /* Enable BAR2 ALIAS for this pcifunc. */ 1130 reg = BIT_ULL(16) | pcifunc; 1131 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg); 1132 1133 cpt_lf_disable_iqueue(rvu, blkaddr, slot); 1134 1135 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0); 1136 mutex_unlock(&rvu->alias_lock); 1137 1138 return 0; 1139} 1140 1141#define CPT_RES_LEN 16 1142#define CPT_SE_IE_EGRP 1ULL 1143 1144static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr, 1145 int nix_blkaddr) 1146{ 1147 int cpt_pf_num = rvu->cpt_pf_num; 1148 struct cpt_inst_lmtst_req *req; 1149 dma_addr_t res_daddr; 1150 int timeout = 3000; 1151 u8 cpt_idx; 1152 u64 *inst; 1153 u16 *res; 1154 int rc; 1155 1156 res = kzalloc(CPT_RES_LEN, GFP_KERNEL); 1157 if (!res) 1158 return -ENOMEM; 1159 1160 res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN, 1161 DMA_BIDIRECTIONAL); 1162 if (dma_mapping_error(rvu->dev, res_daddr)) { 1163 dev_err(rvu->dev, "DMA mapping failed for CPT result\n"); 1164 rc = -EFAULT; 1165 goto res_free; 1166 } 1167 *res = 0xFFFF; 1168 1169 /* Send mbox message to CPT PF */ 1170 req = (struct cpt_inst_lmtst_req *) 1171 otx2_mbox_alloc_msg_rsp(&rvu->afpf_wq_info.mbox_up, 1172 cpt_pf_num, sizeof(*req), 1173 sizeof(struct msg_rsp)); 1174 if (!req) { 1175 rc = -ENOMEM; 1176 goto res_daddr_unmap; 1177 } 1178 req->hdr.sig = OTX2_MBOX_REQ_SIG; 1179 req->hdr.id = MBOX_MSG_CPT_INST_LMTST; 1180 1181 inst = req->inst; 1182 /* Prepare CPT_INST_S */ 1183 inst[0] = 0; 1184 inst[1] = res_daddr; 1185 /* AF PF FUNC */ 1186 inst[2] = 0; 1187 /* Set QORD */ 1188 inst[3] = 1; 1189 inst[4] = 0; 1190 inst[5] = 0; 1191 inst[6] = 0; 1192 /* Set EGRP */ 1193 inst[7] = CPT_SE_IE_EGRP << 61; 1194 1195 /* Subtract 1 from the NIX-CPT credit count to preserve 1196 * credit counts. 1197 */ 1198 cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1; 1199 rvu_write64(rvu, nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 1200 BIT_ULL(22) - 1); 1201 1202 otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, cpt_pf_num); 1203 rc = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, cpt_pf_num); 1204 if (rc) 1205 dev_warn(rvu->dev, "notification to pf %d failed\n", 1206 cpt_pf_num); 1207 /* Wait for CPT instruction to be completed */ 1208 do { 1209 mdelay(1); 1210 if (*res == 0xFFFF) 1211 timeout--; 1212 else 1213 break; 1214 } while (timeout); 1215 1216 if (timeout == 0) 1217 dev_warn(rvu->dev, "Poll for result hits hard loop counter\n"); 1218 1219res_daddr_unmap: 1220 dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL); 1221res_free: 1222 kfree(res); 1223 1224 return 0; 1225} 1226 1227#define CTX_CAM_PF_FUNC GENMASK_ULL(61, 46) 1228#define CTX_CAM_CPTR GENMASK_ULL(45, 0) 1229 1230int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc) 1231{ 1232 int nix_blkaddr, blkaddr; 1233 u16 max_ctx_entries, i; 1234 int slot = 0, num_lfs; 1235 u64 reg, cam_data; 1236 int rc; 1237 1238 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1239 if (nix_blkaddr < 0) 1240 return -EINVAL; 1241 1242 if (is_rvu_otx2(rvu)) 1243 return 0; 1244 1245 blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0; 1246 1247 /* Submit CPT_INST_S to track when all packets have been 1248 * flushed through for the NIX PF FUNC in inline inbound case. 1249 */ 1250 rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr); 1251 if (rc) 1252 return rc; 1253 1254 /* Wait for rxc entries to be flushed out */ 1255 cpt_rxc_teardown(rvu, blkaddr); 1256 1257 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0); 1258 max_ctx_entries = (reg >> 48) & 0xFFF; 1259 1260 mutex_lock(&rvu->rsrc_lock); 1261 1262 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 1263 blkaddr); 1264 if (num_lfs == 0) { 1265 dev_warn(rvu->dev, "CPT LF is not configured\n"); 1266 goto unlock; 1267 } 1268 1269 /* Enable BAR2 ALIAS for this pcifunc. */ 1270 reg = BIT_ULL(16) | pcifunc; 1271 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg); 1272 1273 for (i = 0; i < max_ctx_entries; i++) { 1274 cam_data = rvu_read64(rvu, blkaddr, CPT_AF_CTX_CAM_DATA(i)); 1275 1276 if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) && 1277 FIELD_GET(CTX_CAM_CPTR, cam_data)) { 1278 reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data); 1279 rvu_write64(rvu, blkaddr, 1280 CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH), 1281 reg); 1282 } 1283 } 1284 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0); 1285 1286unlock: 1287 mutex_unlock(&rvu->rsrc_lock); 1288 1289 return 0; 1290} 1291 1292#define MAX_RXC_ICB_CNT GENMASK_ULL(40, 32) 1293 1294int rvu_cpt_init(struct rvu *rvu) 1295{ 1296 struct rvu_hwinfo *hw = rvu->hw; 1297 u64 reg_val; 1298 1299 /* Retrieve CPT PF number */ 1300 rvu->cpt_pf_num = get_cpt_pf_num(rvu); 1301 if (is_block_implemented(rvu->hw, BLKADDR_CPT0) && !is_rvu_otx2(rvu) && 1302 !is_cn10kb(rvu)) 1303 hw->cap.cpt_rxc = true; 1304 1305 if (hw->cap.cpt_rxc && !is_cn10ka_a0(rvu) && !is_cn10ka_a1(rvu)) { 1306 /* Set CPT_AF_RXC_CFG1:max_rxc_icb_cnt to 0xc0 to not effect 1307 * inline inbound peak performance 1308 */ 1309 reg_val = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_RXC_CFG1); 1310 reg_val &= ~MAX_RXC_ICB_CNT; 1311 reg_val |= FIELD_PREP(MAX_RXC_ICB_CNT, 1312 CPT_DFLT_MAX_RXC_ICB_CNT); 1313 rvu_write64(rvu, BLKADDR_CPT0, CPT_AF_RXC_CFG1, reg_val); 1314 } 1315 1316 spin_lock_init(&rvu->cpt_intr_lock); 1317 1318 return 0; 1319}
~/
Read-only file preview.