otx2_cptlf.h

  1/* SPDX-License-Identifier: GPL-2.0-only
  2 * Copyright (C) 2020 Marvell.
  3 */
  4#ifndef __OTX2_CPTLF_H
  5#define __OTX2_CPTLF_H
  6
  7#include <linux/soc/marvell/octeontx2/asm.h>
  8#include <linux/bitfield.h>
  9#include <mbox.h>
 10#include <rvu.h>
 11#include "otx2_cpt_common.h"
 12#include "otx2_cpt_reqmgr.h"
 13
 14/*
 15 * CPT instruction and pending queues user requested length in CPT_INST_S msgs
 16 */
 17#define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200
 18
 19/*
 20 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S
 21 * messages.
 22 */
 23#define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40)
 24
 25/*
 26 * CPT instruction and pending queues length in CPT_INST_S messages
 27 */
 28#define OTX2_CPT_INST_QLEN_MSGS	((OTX2_CPT_SIZE_DIV40 - 1) * 40)
 29
 30/*
 31 * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction
 32 * queue has less than 320 free entries. So, increase HW instruction queue
 33 * size by 320 and give 320 entries less for SW/NIX RX as a workaround.
 34 */
 35#define OTX2_CPT_INST_QLEN_EXTRA_BYTES  (320 * OTX2_CPT_INST_SIZE)
 36#define OTX2_CPT_EXTRA_SIZE_DIV40       (320/40)
 37
 38/* CPT instruction queue length in bytes */
 39#define OTX2_CPT_INST_QLEN_BYTES                                               \
 40		((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) +             \
 41		OTX2_CPT_INST_QLEN_EXTRA_BYTES)
 42
 43/* CPT instruction group queue length in bytes */
 44#define OTX2_CPT_INST_GRP_QLEN_BYTES                                           \
 45		((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16)
 46
 47/* CPT FC length in bytes */
 48#define OTX2_CPT_Q_FC_LEN 128
 49
 50/* CPT instruction queue alignment */
 51#define OTX2_CPT_INST_Q_ALIGNMENT  128
 52
 53/* Mask which selects all engine groups */
 54#define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF
 55
 56/* Maximum LFs supported in OcteonTX2 for CPT */
 57#define OTX2_CPT_MAX_LFS_NUM    64
 58
 59/* Queue priority */
 60#define OTX2_CPT_QUEUE_HI_PRIO  0x1
 61#define OTX2_CPT_QUEUE_LOW_PRIO 0x0
 62
 63enum otx2_cptlf_state {
 64	OTX2_CPTLF_IN_RESET,
 65	OTX2_CPTLF_STARTED,
 66};
 67
 68struct otx2_cpt_inst_queue {
 69	u8 *vaddr;
 70	u8 *real_vaddr;
 71	dma_addr_t dma_addr;
 72	dma_addr_t real_dma_addr;
 73	u32 size;
 74};
 75
 76struct otx2_cptlfs_info;
 77struct otx2_cptlf_wqe {
 78	struct tasklet_struct work;
 79	struct otx2_cptlfs_info *lfs;
 80	u8 lf_num;
 81};
 82
 83struct otx2_cptlf_info {
 84	struct otx2_cptlfs_info *lfs;           /* Ptr to cptlfs_info struct */
 85	void __iomem *lmtline;                  /* Address of LMTLINE */
 86	void __iomem *ioreg;                    /* LMTLINE send register */
 87	int msix_offset;                        /* MSI-X interrupts offset */
 88	cpumask_var_t affinity_mask;            /* IRQs affinity mask */
 89	u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */
 90	u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS];  /* Is interrupt registered */
 91	u8 slot;                                /* Slot number of this LF */
 92
 93	struct otx2_cpt_inst_queue iqueue;/* Instruction queue */
 94	struct otx2_cpt_pending_queue pqueue; /* Pending queue */
 95	struct otx2_cptlf_wqe *wqe;       /* Tasklet work info */
 96};
 97
 98struct cpt_hw_ops {
 99	void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
100			 struct otx2_cptlf_info *lf);
101	u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
102	u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
103	struct otx2_cpt_inst_info *
104	(*cpt_sg_info_create)(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
105			      gfp_t gfp);
106};
107
108#define LMTLINE_SIZE  128
109#define LMTLINE_ALIGN 128
110struct otx2_lmt_info {
111	void *base;
112	dma_addr_t iova;
113	u32 size;
114	u8 align;
115};
116
117struct otx2_cptlfs_info {
118	/* Registers start address of VF/PF LFs are attached to */
119	void __iomem *reg_base;
120	struct otx2_lmt_info lmt_info;
121	struct pci_dev *pdev;   /* Device LFs are attached to */
122	struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
123	struct otx2_mbox *mbox;
124	struct cpt_hw_ops *ops;
125	u8 are_lfs_attached;	/* Whether CPT LFs are attached */
126	u8 lfs_num;		/* Number of CPT LFs */
127	u8 kcrypto_se_eng_grp_num; /* Crypto symmetric engine group number */
128	u8 kcrypto_ae_eng_grp_num; /* Crypto asymmetric engine group number */
129	u8 kvf_limits;          /* Kernel crypto limits */
130	atomic_t state;         /* LF's state. started/reset */
131	int blkaddr;            /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
132	int global_slot;        /* Global slot across the blocks */
133	u8 ctx_ilen;
134	u8 ctx_ilen_ovrd;
135};
136
137static inline void otx2_cpt_free_instruction_queues(
138					struct otx2_cptlfs_info *lfs)
139{
140	struct otx2_cpt_inst_queue *iq;
141	int i;
142
143	for (i = 0; i < lfs->lfs_num; i++) {
144		iq = &lfs->lf[i].iqueue;
145		if (iq->real_vaddr)
146			dma_free_coherent(&lfs->pdev->dev,
147					  iq->size,
148					  iq->real_vaddr,
149					  iq->real_dma_addr);
150		iq->real_vaddr = NULL;
151		iq->vaddr = NULL;
152	}
153}
154
155static inline int otx2_cpt_alloc_instruction_queues(
156					struct otx2_cptlfs_info *lfs)
157{
158	struct otx2_cpt_inst_queue *iq;
159	int ret = 0, i;
160
161	if (!lfs->lfs_num)
162		return -EINVAL;
163
164	for (i = 0; i < lfs->lfs_num; i++) {
165		iq = &lfs->lf[i].iqueue;
166		iq->size = OTX2_CPT_INST_QLEN_BYTES +
167			   OTX2_CPT_Q_FC_LEN +
168			   OTX2_CPT_INST_GRP_QLEN_BYTES +
169			   OTX2_CPT_INST_Q_ALIGNMENT;
170		iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size,
171					&iq->real_dma_addr, GFP_KERNEL);
172		if (!iq->real_vaddr) {
173			ret = -ENOMEM;
174			goto error;
175		}
176		iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES;
177		iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES;
178
179		/* Align pointers */
180		iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT);
181		iq->dma_addr = PTR_ALIGN(iq->dma_addr,
182					 OTX2_CPT_INST_Q_ALIGNMENT);
183	}
184	return 0;
185
186error:
187	otx2_cpt_free_instruction_queues(lfs);
188	return ret;
189}
190
191static inline void otx2_cptlf_set_iqueues_base_addr(
192					struct otx2_cptlfs_info *lfs)
193{
194	union otx2_cptx_lf_q_base lf_q_base;
195	int slot;
196
197	for (slot = 0; slot < lfs->lfs_num; slot++) {
198		lf_q_base.u = lfs->lf[slot].iqueue.dma_addr;
199		otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot,
200				 OTX2_CPT_LF_Q_BASE, lf_q_base.u);
201	}
202}
203
204static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf)
205{
206	union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 };
207
208	lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 +
209				 OTX2_CPT_EXTRA_SIZE_DIV40;
210	otx2_cpt_write64(lf->lfs->reg_base, lf->lfs->blkaddr, lf->slot,
211			 OTX2_CPT_LF_Q_SIZE, lf_q_size.u);
212}
213
214static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs)
215{
216	int slot;
217
218	for (slot = 0; slot < lfs->lfs_num; slot++)
219		otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]);
220}
221
222#define INFLIGHT   GENMASK_ULL(8, 0)
223#define GRB_CNT    GENMASK_ULL(39, 32)
224#define GWB_CNT    GENMASK_ULL(47, 40)
225#define XQ_XOR     GENMASK_ULL(63, 63)
226#define DQPTR      GENMASK_ULL(19, 0)
227#define NQPTR      GENMASK_ULL(51, 32)
228
229static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf)
230{
231	void __iomem *reg_base = lf->lfs->reg_base;
232	struct pci_dev *pdev = lf->lfs->pdev;
233	u8 blkaddr = lf->lfs->blkaddr;
234	int timeout = 1000000;
235	u64 inprog, inst_ptr;
236	u64 slot = lf->slot;
237	u64 qsize, pending;
238	int i = 0;
239
240	/* Disable instructions enqueuing */
241	otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_CTL, 0x0);
242
243	inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG);
244	inprog |= BIT_ULL(16);
245	otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG, inprog);
246
247	qsize = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_SIZE) & 0x7FFF;
248	do {
249		inst_ptr = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_INST_PTR);
250		pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) +
251			  FIELD_GET(NQPTR, inst_ptr) - FIELD_GET(DQPTR, inst_ptr);
252		udelay(1);
253		timeout--;
254	} while ((pending != 0) && (timeout != 0));
255
256	if (timeout == 0)
257		dev_warn(&pdev->dev, "TIMEOUT: CPT poll on pending instructions\n");
258
259	timeout = 1000000;
260	/* Wait for CPT queue to become execution-quiescent */
261	do {
262		inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG);
263
264		if ((FIELD_GET(INFLIGHT, inprog) == 0) &&
265		    (FIELD_GET(GRB_CNT, inprog) == 0)) {
266			i++;
267		} else {
268			i = 0;
269			timeout--;
270		}
271	} while ((timeout != 0) && (i < 10));
272
273	if (timeout == 0)
274		dev_warn(&pdev->dev, "TIMEOUT: CPT poll on inflight count\n");
275	/* Wait for 2 us to flush all queue writes to memory */
276	udelay(2);
277}
278
279static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs)
280{
281	int slot;
282
283	for (slot = 0; slot < lfs->lfs_num; slot++) {
284		otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]);
285		otx2_cpt_lf_reset_msg(lfs, lfs->global_slot + slot);
286	}
287}
288
289static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf,
290					     bool enable)
291{
292	u8 blkaddr = lf->lfs->blkaddr;
293	union otx2_cptx_lf_ctl lf_ctl;
294
295	lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
296				   OTX2_CPT_LF_CTL);
297
298	/* Set iqueue's enqueuing */
299	lf_ctl.s.ena = enable ? 0x1 : 0x0;
300	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
301			 OTX2_CPT_LF_CTL, lf_ctl.u);
302}
303
304static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf)
305{
306	otx2_cptlf_set_iqueue_enq(lf, true);
307}
308
309static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
310					      bool enable)
311{
312	union otx2_cptx_lf_inprog lf_inprog;
313	u8 blkaddr = lf->lfs->blkaddr;
314
315	lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
316				      OTX2_CPT_LF_INPROG);
317
318	/* Set iqueue's execution */
319	lf_inprog.s.eena = enable ? 0x1 : 0x0;
320	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
321			 OTX2_CPT_LF_INPROG, lf_inprog.u);
322}
323
324static inline void otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info *lf)
325{
326	u8 blkaddr = lf->lfs->blkaddr;
327	u64 val;
328
329	val = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot,
330			      OTX2_CPT_LF_CTX_CTL);
331	val |= BIT_ULL(0);
332
333	otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot,
334			 OTX2_CPT_LF_CTX_CTL, val);
335}
336
337static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
338{
339	otx2_cptlf_set_iqueue_exec(lf, true);
340}
341
342static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf)
343{
344	otx2_cptlf_set_iqueue_exec(lf, false);
345}
346
347static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
348{
349	int slot;
350
351	for (slot = 0; slot < lfs->lfs_num; slot++) {
352		/* Enable flush on FLR for Errata */
353		if (is_dev_cn10kb(lfs->pdev))
354			otx2_cptlf_set_ctx_flr_flush(&lfs->lf[slot]);
355
356		otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
357		otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
358	}
359}
360
361static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst,
362				      struct otx2_cpt_iq_command *iq_cmd,
363				      u64 comp_baddr)
364{
365	cptinst->u[0] = 0x0;
366	cptinst->s.doneint = true;
367	cptinst->s.res_addr = comp_baddr;
368	cptinst->u[2] = 0x0;
369	cptinst->u[3] = 0x0;
370	cptinst->s.ei0 = iq_cmd->cmd.u;
371	cptinst->s.ei1 = iq_cmd->dptr;
372	cptinst->s.ei2 = iq_cmd->rptr;
373	cptinst->s.ei3 = iq_cmd->cptr.u;
374}
375
376/*
377 * On OcteonTX2 platform the parameter insts_num is used as a count of
378 * instructions to be enqueued. The valid values for insts_num are:
379 * 1 - 1 CPT instruction will be enqueued during LMTST operation
380 * 2 - 2 CPT instructions will be enqueued during LMTST operation
381 */
382static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst,
383				     u32 insts_num, struct otx2_cptlf_info *lf)
384{
385	void __iomem *lmtline = lf->lmtline;
386	long ret;
387
388	/*
389	 * Make sure memory areas pointed in CPT_INST_S
390	 * are flushed before the instruction is sent to CPT
391	 */
392	dma_wmb();
393
394	do {
395		/* Copy CPT command to LMTLINE */
396		memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
397
398		/*
399		 * LDEOR initiates atomic transfer to I/O device
400		 * The following will cause the LMTST to fail (the LDEOR
401		 * returns zero):
402		 * - No stores have been performed to the LMTLINE since it was
403		 * last invalidated.
404		 * - The bytes which have been stored to LMTLINE since it was
405		 * last invalidated form a pattern that is non-contiguous, does
406		 * not start at byte 0, or does not end on a 8-byte boundary.
407		 * (i.e.comprises a formation of other than 1–16 8-byte
408		 * words.)
409		 *
410		 * These rules are designed such that an operating system
411		 * context switch or hypervisor guest switch need have no
412		 * knowledge of the LMTST operations; the switch code does not
413		 * need to store to LMTCANCEL. Also note as LMTLINE data cannot
414		 * be read, there is no information leakage between processes.
415		 */
416		ret = otx2_lmt_flush(lf->ioreg);
417
418	} while (!ret);
419}
420
421static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs)
422{
423	return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED;
424}
425
426static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs,
427					   struct pci_dev *pdev,
428					   void __iomem *reg_base,
429					   struct otx2_mbox *mbox,
430					   int blkaddr)
431{
432	lfs->pdev = pdev;
433	lfs->reg_base = reg_base;
434	lfs->mbox = mbox;
435	lfs->blkaddr = blkaddr;
436}
437
438int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
439		    int lfs_num);
440void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
441int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs);
442int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs);
443void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs);
444void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs);
445void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
446int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
447
448#endif /* __OTX2_CPTLF_H */