rvu_npa.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Marvell RVU Admin Function driver
  3 *
  4 * Copyright (C) 2018 Marvell.
  5 *
  6 */
  7#include <linux/bitfield.h>
  8#include <linux/module.h>
  9#include <linux/pci.h>
 10
 11#include "rvu_struct.h"
 12#include "rvu_reg.h"
 13#include "rvu.h"
 14
 15static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
 16			       struct npa_aq_inst_s *inst)
 17{
 18	struct admin_queue *aq = block->aq;
 19	struct npa_aq_res_s *result;
 20	int timeout = 1000;
 21	u64 reg, head;
 22
 23	result = (struct npa_aq_res_s *)aq->res->base;
 24
 25	/* Get current head pointer where to append this instruction */
 26	reg = rvu_read64(rvu, block->addr, NPA_AF_AQ_STATUS);
 27	head = (reg >> 4) & AQ_PTR_MASK;
 28
 29	memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
 30	       (void *)inst, aq->inst->entry_sz);
 31	memset(result, 0, sizeof(*result));
 32	/* sync into memory */
 33	wmb();
 34
 35	/* Ring the doorbell and wait for result */
 36	rvu_write64(rvu, block->addr, NPA_AF_AQ_DOOR, 1);
 37	while (result->compcode == NPA_AQ_COMP_NOTDONE) {
 38		cpu_relax();
 39		udelay(1);
 40		timeout--;
 41		if (!timeout)
 42			return -EBUSY;
 43	}
 44
 45	if (result->compcode != NPA_AQ_COMP_GOOD) {
 46		/* TODO: Replace this with some error code */
 47		if (result->compcode == NPA_AQ_COMP_CTX_FAULT ||
 48		    result->compcode == NPA_AQ_COMP_LOCKERR ||
 49		    result->compcode == NPA_AQ_COMP_CTX_POISON) {
 50			if (rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NPA0))
 51				dev_err(rvu->dev,
 52					"%s: Not able to unlock cachelines\n", __func__);
 53		}
 54
 55		return -EBUSY;
 56	}
 57
 58	return 0;
 59}
 60
 61int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req,
 62			struct npa_aq_enq_rsp *rsp)
 63{
 64	struct rvu_hwinfo *hw = rvu->hw;
 65	u16 pcifunc = req->hdr.pcifunc;
 66	int blkaddr, npalf, rc = 0;
 67	struct npa_aq_inst_s inst;
 68	struct rvu_block *block;
 69	struct admin_queue *aq;
 70	struct rvu_pfvf *pfvf;
 71	void *ctx, *mask;
 72	bool ena;
 73
 74	pfvf = rvu_get_pfvf(rvu, pcifunc);
 75	if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize)
 76		return NPA_AF_ERR_AQ_ENQUEUE;
 77
 78	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
 79	if (!pfvf->npalf || blkaddr < 0)
 80		return NPA_AF_ERR_AF_LF_INVALID;
 81
 82	block = &hw->block[blkaddr];
 83	aq = block->aq;
 84	if (!aq) {
 85		dev_warn(rvu->dev, "%s: NPA AQ not initialized\n", __func__);
 86		return NPA_AF_ERR_AQ_ENQUEUE;
 87	}
 88
 89	npalf = rvu_get_lf(rvu, block, pcifunc, 0);
 90	if (npalf < 0)
 91		return NPA_AF_ERR_AF_LF_INVALID;
 92
 93	memset(&inst, 0, sizeof(struct npa_aq_inst_s));
 94	inst.cindex = req->aura_id;
 95	inst.lf = npalf;
 96	inst.ctype = req->ctype;
 97	inst.op = req->op;
 98	/* Currently we are not supporting enqueuing multiple instructions,
 99	 * so always choose first entry in result memory.
100	 */
101	inst.res_addr = (u64)aq->res->iova;
102
103	/* Hardware uses same aq->res->base for updating result of
104	 * previous instruction hence wait here till it is done.
105	 */
106	spin_lock(&aq->lock);
107
108	/* Clean result + context memory */
109	memset(aq->res->base, 0, aq->res->entry_sz);
110	/* Context needs to be written at RES_ADDR + 128 */
111	ctx = aq->res->base + 128;
112	/* Mask needs to be written at RES_ADDR + 256 */
113	mask = aq->res->base + 256;
114
115	switch (req->op) {
116	case NPA_AQ_INSTOP_WRITE:
117		/* Copy context and write mask */
118		if (req->ctype == NPA_AQ_CTYPE_AURA) {
119			memcpy(mask, &req->aura_mask,
120			       sizeof(struct npa_aura_s));
121			memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
122		} else {
123			memcpy(mask, &req->pool_mask,
124			       sizeof(struct npa_pool_s));
125			memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
126		}
127		break;
128	case NPA_AQ_INSTOP_INIT:
129		if (req->ctype == NPA_AQ_CTYPE_AURA) {
130			if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) {
131				rc = NPA_AF_ERR_AQ_FULL;
132				break;
133			}
134			/* Set pool's context address */
135			req->aura.pool_addr = pfvf->pool_ctx->iova +
136			(req->aura.pool_addr * pfvf->pool_ctx->entry_sz);
137			memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
138		} else { /* POOL's context */
139			memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
140		}
141		break;
142	case NPA_AQ_INSTOP_NOP:
143	case NPA_AQ_INSTOP_READ:
144	case NPA_AQ_INSTOP_LOCK:
145	case NPA_AQ_INSTOP_UNLOCK:
146		break;
147	default:
148		rc = NPA_AF_ERR_AQ_FULL;
149		break;
150	}
151
152	if (rc) {
153		spin_unlock(&aq->lock);
154		return rc;
155	}
156
157	/* Submit the instruction to AQ */
158	rc = npa_aq_enqueue_wait(rvu, block, &inst);
159	if (rc) {
160		spin_unlock(&aq->lock);
161		return rc;
162	}
163
164	/* Set aura bitmap if aura hw context is enabled */
165	if (req->ctype == NPA_AQ_CTYPE_AURA) {
166		if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena)
167			__set_bit(req->aura_id, pfvf->aura_bmap);
168		if (req->op == NPA_AQ_INSTOP_WRITE) {
169			ena = (req->aura.ena & req->aura_mask.ena) |
170				(test_bit(req->aura_id, pfvf->aura_bmap) &
171				~req->aura_mask.ena);
172			if (ena)
173				__set_bit(req->aura_id, pfvf->aura_bmap);
174			else
175				__clear_bit(req->aura_id, pfvf->aura_bmap);
176		}
177	}
178
179	/* Set pool bitmap if pool hw context is enabled */
180	if (req->ctype == NPA_AQ_CTYPE_POOL) {
181		if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena)
182			__set_bit(req->aura_id, pfvf->pool_bmap);
183		if (req->op == NPA_AQ_INSTOP_WRITE) {
184			ena = (req->pool.ena & req->pool_mask.ena) |
185				(test_bit(req->aura_id, pfvf->pool_bmap) &
186				~req->pool_mask.ena);
187			if (ena)
188				__set_bit(req->aura_id, pfvf->pool_bmap);
189			else
190				__clear_bit(req->aura_id, pfvf->pool_bmap);
191		}
192	}
193	spin_unlock(&aq->lock);
194
195	if (rsp) {
196		/* Copy read context into mailbox */
197		if (req->op == NPA_AQ_INSTOP_READ) {
198			if (req->ctype == NPA_AQ_CTYPE_AURA)
199				memcpy(&rsp->aura, ctx,
200				       sizeof(struct npa_aura_s));
201			else
202				memcpy(&rsp->pool, ctx,
203				       sizeof(struct npa_pool_s));
204		}
205	}
206
207	return 0;
208}
209
210static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
211{
212	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
213	struct npa_aq_enq_req aq_req;
214	unsigned long *bmap;
215	int id, cnt = 0;
216	int err = 0, rc;
217
218	if (!pfvf->pool_ctx || !pfvf->aura_ctx)
219		return NPA_AF_ERR_AQ_ENQUEUE;
220
221	memset(&aq_req, 0, sizeof(struct npa_aq_enq_req));
222	aq_req.hdr.pcifunc = req->hdr.pcifunc;
223
224	if (req->ctype == NPA_AQ_CTYPE_POOL) {
225		aq_req.pool.ena = 0;
226		aq_req.pool_mask.ena = 1;
227		cnt = pfvf->pool_ctx->qsize;
228		bmap = pfvf->pool_bmap;
229	} else if (req->ctype == NPA_AQ_CTYPE_AURA) {
230		aq_req.aura.ena = 0;
231		aq_req.aura_mask.ena = 1;
232		aq_req.aura.bp_ena = 0;
233		aq_req.aura_mask.bp_ena = 1;
234		cnt = pfvf->aura_ctx->qsize;
235		bmap = pfvf->aura_bmap;
236	}
237
238	aq_req.ctype = req->ctype;
239	aq_req.op = NPA_AQ_INSTOP_WRITE;
240
241	for (id = 0; id < cnt; id++) {
242		if (!test_bit(id, bmap))
243			continue;
244		aq_req.aura_id = id;
245		rc = rvu_npa_aq_enq_inst(rvu, &aq_req, NULL);
246		if (rc) {
247			err = rc;
248			dev_err(rvu->dev, "Failed to disable %s:%d context\n",
249				(req->ctype == NPA_AQ_CTYPE_AURA) ?
250				"Aura" : "Pool", id);
251		}
252	}
253
254	return err;
255}
256
257#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
258static int npa_lf_hwctx_lockdown(struct rvu *rvu, struct npa_aq_enq_req *req)
259{
260	struct npa_aq_enq_req lock_ctx_req;
261	int err;
262
263	if (req->op != NPA_AQ_INSTOP_INIT)
264		return 0;
265
266	memset(&lock_ctx_req, 0, sizeof(struct npa_aq_enq_req));
267	lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
268	lock_ctx_req.ctype = req->ctype;
269	lock_ctx_req.op = NPA_AQ_INSTOP_LOCK;
270	lock_ctx_req.aura_id = req->aura_id;
271	err = rvu_npa_aq_enq_inst(rvu, &lock_ctx_req, NULL);
272	if (err)
273		dev_err(rvu->dev,
274			"PFUNC 0x%x: Failed to lock NPA context %s:%d\n",
275			req->hdr.pcifunc,
276			(req->ctype == NPA_AQ_CTYPE_AURA) ?
277			"Aura" : "Pool", req->aura_id);
278	return err;
279}
280
281int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
282				struct npa_aq_enq_req *req,
283				struct npa_aq_enq_rsp *rsp)
284{
285	int err;
286
287	err = rvu_npa_aq_enq_inst(rvu, req, rsp);
288	if (!err)
289		err = npa_lf_hwctx_lockdown(rvu, req);
290	return err;
291}
292#else
293
294int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
295				struct npa_aq_enq_req *req,
296				struct npa_aq_enq_rsp *rsp)
297{
298	return rvu_npa_aq_enq_inst(rvu, req, rsp);
299}
300#endif
301
302int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu,
303				       struct hwctx_disable_req *req,
304				       struct msg_rsp *rsp)
305{
306	return npa_lf_hwctx_disable(rvu, req);
307}
308
309static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
310{
311	kfree(pfvf->aura_bmap);
312	pfvf->aura_bmap = NULL;
313
314	qmem_free(rvu->dev, pfvf->aura_ctx);
315	pfvf->aura_ctx = NULL;
316
317	kfree(pfvf->pool_bmap);
318	pfvf->pool_bmap = NULL;
319
320	qmem_free(rvu->dev, pfvf->pool_ctx);
321	pfvf->pool_ctx = NULL;
322
323	qmem_free(rvu->dev, pfvf->npa_qints_ctx);
324	pfvf->npa_qints_ctx = NULL;
325}
326
327int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu,
328				  struct npa_lf_alloc_req *req,
329				  struct npa_lf_alloc_rsp *rsp)
330{
331	int npalf, qints, hwctx_size, err, rc = 0;
332	struct rvu_hwinfo *hw = rvu->hw;
333	u16 pcifunc = req->hdr.pcifunc;
334	struct rvu_block *block;
335	struct rvu_pfvf *pfvf;
336	u64 cfg, ctx_cfg;
337	int blkaddr;
338
339	if (req->aura_sz > NPA_AURA_SZ_MAX ||
340	    req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools)
341		return NPA_AF_ERR_PARAM;
342
343	if (req->way_mask)
344		req->way_mask &= 0xFFFF;
345
346	pfvf = rvu_get_pfvf(rvu, pcifunc);
347	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
348	if (!pfvf->npalf || blkaddr < 0)
349		return NPA_AF_ERR_AF_LF_INVALID;
350
351	block = &hw->block[blkaddr];
352	npalf = rvu_get_lf(rvu, block, pcifunc, 0);
353	if (npalf < 0)
354		return NPA_AF_ERR_AF_LF_INVALID;
355
356	/* Reset this NPA LF */
357	err = rvu_lf_reset(rvu, block, npalf);
358	if (err) {
359		dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
360		return NPA_AF_ERR_LF_RESET;
361	}
362
363	ctx_cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST1);
364
365	/* Alloc memory for aura HW contexts */
366	hwctx_size = 1UL << (ctx_cfg & 0xF);
367	err = qmem_alloc(rvu->dev, &pfvf->aura_ctx,
368			 NPA_AURA_COUNT(req->aura_sz), hwctx_size);
369	if (err)
370		goto free_mem;
371
372	pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
373				  GFP_KERNEL);
374	if (!pfvf->aura_bmap)
375		goto free_mem;
376
377	/* Alloc memory for pool HW contexts */
378	hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
379	err = qmem_alloc(rvu->dev, &pfvf->pool_ctx, req->nr_pools, hwctx_size);
380	if (err)
381		goto free_mem;
382
383	pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
384				  GFP_KERNEL);
385	if (!pfvf->pool_bmap)
386		goto free_mem;
387
388	/* Get no of queue interrupts supported */
389	cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
390	qints = (cfg >> 28) & 0xFFF;
391
392	/* Alloc memory for Qints HW contexts */
393	hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
394	err = qmem_alloc(rvu->dev, &pfvf->npa_qints_ctx, qints, hwctx_size);
395	if (err)
396		goto free_mem;
397
398	cfg = rvu_read64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf));
399	/* Clear way partition mask and set aura offset to '0' */
400	cfg &= ~(BIT_ULL(34) - 1);
401	/* Set aura size & enable caching of contexts */
402	cfg |= (req->aura_sz << 16) | BIT_ULL(34) | req->way_mask;
403
404	rvu_write64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), cfg);
405
406	/* Configure aura HW context's base */
407	rvu_write64(rvu, blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf),
408		    (u64)pfvf->aura_ctx->iova);
409
410	/* Enable caching of qints hw context */
411	rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_CFG(npalf),
412		    BIT_ULL(36) | req->way_mask << 20);
413	rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_BASE(npalf),
414		    (u64)pfvf->npa_qints_ctx->iova);
415
416	goto exit;
417
418free_mem:
419	npa_ctx_free(rvu, pfvf);
420	rc = -ENOMEM;
421
422exit:
423	/* set stack page info */
424	cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
425	rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF;
426	rsp->stack_pg_bytes = cfg & 0xFF;
427	rsp->qints = (cfg >> 28) & 0xFFF;
428	if (!is_rvu_otx2(rvu)) {
429		cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL);
430		rsp->cache_lines = (cfg >> 1) & 0x3F;
431	}
432	return rc;
433}
434
435int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req,
436				 struct msg_rsp *rsp)
437{
438	struct rvu_hwinfo *hw = rvu->hw;
439	u16 pcifunc = req->hdr.pcifunc;
440	struct rvu_block *block;
441	struct rvu_pfvf *pfvf;
442	int npalf, err;
443	int blkaddr;
444
445	pfvf = rvu_get_pfvf(rvu, pcifunc);
446	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
447	if (!pfvf->npalf || blkaddr < 0)
448		return NPA_AF_ERR_AF_LF_INVALID;
449
450	block = &hw->block[blkaddr];
451	npalf = rvu_get_lf(rvu, block, pcifunc, 0);
452	if (npalf < 0)
453		return NPA_AF_ERR_AF_LF_INVALID;
454
455	/* Reset this NPA LF */
456	err = rvu_lf_reset(rvu, block, npalf);
457	if (err) {
458		dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
459		return NPA_AF_ERR_LF_RESET;
460	}
461
462	npa_ctx_free(rvu, pfvf);
463
464	return 0;
465}
466
467static void npa_aq_ndc_config(struct rvu *rvu, struct rvu_block *block)
468{
469	u64 cfg;
470
471	if (is_cn20k(rvu->pdev)) /* NDC not applicable to cn20k */
472		return;
473
474	/* Do not bypass NDC cache */
475	cfg = rvu_read64(rvu, block->addr, NPA_AF_NDC_CFG);
476	cfg &= ~0x03DULL;
477#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
478	/* Disable caching of stack pages */
479	cfg |= 0x10ULL;
480#endif
481	rvu_write64(rvu, block->addr, NPA_AF_NDC_CFG, cfg);
482}
483
484static int npa_aq_init(struct rvu *rvu, struct rvu_block *block)
485{
486	u64 cfg;
487	int err;
488
489	/* Set admin queue endianness */
490	cfg = rvu_read64(rvu, block->addr, NPA_AF_GEN_CFG);
491#ifdef __BIG_ENDIAN
492	cfg |= BIT_ULL(1);
493	rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
494#else
495	cfg &= ~BIT_ULL(1);
496	rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
497#endif
498
499	npa_aq_ndc_config(rvu, block);
500
501	/* For CN10K NPA BATCH DMA set 35 cache lines */
502	if (!is_rvu_otx2(rvu)) {
503		cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL);
504		cfg &= ~0x7EULL;
505		cfg |= BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1);
506		rvu_write64(rvu, block->addr, NPA_AF_BATCH_CTL, cfg);
507	}
508	/* Result structure can be followed by Aura/Pool context at
509	 * RES + 128bytes and a write mask at RES + 256 bytes, depending on
510	 * operation type. Alloc sufficient result memory for all operations.
511	 */
512	err = rvu_aq_alloc(rvu, &block->aq,
513			   Q_COUNT(AQ_SIZE), sizeof(struct npa_aq_inst_s),
514			   ALIGN(sizeof(struct npa_aq_res_s), 128) + 256);
515	if (err)
516		return err;
517
518	rvu_write64(rvu, block->addr, NPA_AF_AQ_CFG, AQ_SIZE);
519	rvu_write64(rvu, block->addr,
520		    NPA_AF_AQ_BASE, (u64)block->aq->inst->iova);
521	return 0;
522}
523
524int rvu_npa_init(struct rvu *rvu)
525{
526	struct rvu_hwinfo *hw = rvu->hw;
527	int blkaddr;
528
529	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
530	if (blkaddr < 0)
531		return 0;
532
533	/* Initialize admin queue */
534	return npa_aq_init(rvu, &hw->block[blkaddr]);
535}
536
537void rvu_npa_freemem(struct rvu *rvu)
538{
539	struct rvu_hwinfo *hw = rvu->hw;
540	struct rvu_block *block;
541	int blkaddr;
542
543	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
544	if (blkaddr < 0)
545		return;
546
547	block = &hw->block[blkaddr];
548	rvu_aq_free(rvu, block->aq);
549}
550
551void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf)
552{
553	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
554	struct hwctx_disable_req ctx_req;
555
556	/* Disable all pools */
557	ctx_req.hdr.pcifunc = pcifunc;
558	ctx_req.ctype = NPA_AQ_CTYPE_POOL;
559	npa_lf_hwctx_disable(rvu, &ctx_req);
560
561	/* Disable all auras */
562	ctx_req.ctype = NPA_AQ_CTYPE_AURA;
563	npa_lf_hwctx_disable(rvu, &ctx_req);
564
565	npa_ctx_free(rvu, pfvf);
566}
567
568/* Due to an Hardware errata, in some corner cases, AQ context lock
569 * operations can result in a NDC way getting into an illegal state
570 * of not valid but locked.
571 *
572 * This API solves the problem by clearing the lock bit of the NDC block.
573 * The operation needs to be done for each line of all the NDC banks.
574 */
575int rvu_ndc_fix_locked_cacheline(struct rvu *rvu, int blkaddr)
576{
577	int bank, max_bank, line, max_line, err;
578	u64 reg, ndc_af_const;
579
580	if (is_cn20k(rvu->pdev)) /* NDC not applicable to cn20k */
581		return 0;
582
583	/* Set the ENABLE bit(63) to '0' */
584	reg = rvu_read64(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL);
585	rvu_write64(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL, reg & GENMASK_ULL(62, 0));
586
587	/* Poll until the BUSY bits(47:32) are set to '0' */
588	err = rvu_poll_reg(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL, GENMASK_ULL(47, 32), true);
589	if (err) {
590		dev_err(rvu->dev, "Timed out while polling for NDC CAM busy bits.\n");
591		return err;
592	}
593
594	ndc_af_const = rvu_read64(rvu, blkaddr, NDC_AF_CONST);
595	max_bank = FIELD_GET(NDC_AF_BANK_MASK, ndc_af_const);
596	max_line = FIELD_GET(NDC_AF_BANK_LINE_MASK, ndc_af_const);
597	for (bank = 0; bank < max_bank; bank++) {
598		for (line = 0; line < max_line; line++) {
599			/* Check if 'cache line valid bit(63)' is not set
600			 * but 'cache line lock bit(60)' is set and on
601			 * success, reset the lock bit(60).
602			 */
603			reg = rvu_read64(rvu, blkaddr,
604					 NDC_AF_BANKX_LINEX_METADATA(bank, line));
605			if (!(reg & BIT_ULL(63)) && (reg & BIT_ULL(60))) {
606				rvu_write64(rvu, blkaddr,
607					    NDC_AF_BANKX_LINEX_METADATA(bank, line),
608					    reg & ~BIT_ULL(60));
609			}
610		}
611	}
612
613	return 0;
614}