cn10k_cpt.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2021 Marvell. */
  3
  4#include <linux/soc/marvell/octeontx2/asm.h>
  5#include "otx2_cptpf.h"
  6#include "otx2_cptvf.h"
  7#include "otx2_cptlf.h"
  8#include "cn10k_cpt.h"
  9#include "otx2_cpt_common.h"
 10
 11static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num,
 12			       struct otx2_cptlf_info *lf);
 13
 14static struct cpt_hw_ops otx2_hw_ops = {
 15	.send_cmd = otx2_cpt_send_cmd,
 16	.cpt_get_compcode = otx2_cpt_get_compcode,
 17	.cpt_get_uc_compcode = otx2_cpt_get_uc_compcode,
 18	.cpt_sg_info_create = otx2_sg_info_create,
 19};
 20
 21static struct cpt_hw_ops cn10k_hw_ops = {
 22	.send_cmd = cn10k_cpt_send_cmd,
 23	.cpt_get_compcode = cn10k_cpt_get_compcode,
 24	.cpt_get_uc_compcode = cn10k_cpt_get_uc_compcode,
 25	.cpt_sg_info_create = otx2_sg_info_create,
 26};
 27
 28static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num,
 29			       struct otx2_cptlf_info *lf)
 30{
 31	void *lmtline = lf->lfs->lmt_info.base + (lf->slot * LMTLINE_SIZE);
 32	u64 val = (lf->slot & 0x7FF);
 33	u64 tar_addr = 0;
 34
 35	/* tar_addr<6:4> = Size of first LMTST - 1 in units of 128b. */
 36	tar_addr |= (__force u64)lf->ioreg |
 37		    (((OTX2_CPT_INST_SIZE/16) - 1) & 0x7) << 4;
 38	/*
 39	 * Make sure memory areas pointed in CPT_INST_S
 40	 * are flushed before the instruction is sent to CPT
 41	 */
 42	dma_wmb();
 43
 44	/* Copy CPT command to LMTLINE */
 45	memcpy(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
 46	cn10k_lmt_flush(val, tar_addr);
 47}
 48
 49void cn10k_cpt_lmtst_free(struct pci_dev *pdev, struct otx2_cptlfs_info *lfs)
 50{
 51	struct otx2_lmt_info *lmt_info = &lfs->lmt_info;
 52
 53	if (!lmt_info->base)
 54		return;
 55
 56	dma_free_attrs(&pdev->dev, lmt_info->size,
 57		       lmt_info->base - lmt_info->align,
 58		       lmt_info->iova - lmt_info->align,
 59		       DMA_ATTR_FORCE_CONTIGUOUS);
 60}
 61EXPORT_SYMBOL_NS_GPL(cn10k_cpt_lmtst_free, "CRYPTO_DEV_OCTEONTX2_CPT");
 62
 63static int cn10k_cpt_lmtst_alloc(struct pci_dev *pdev,
 64				 struct otx2_cptlfs_info *lfs, u32 size)
 65{
 66	struct otx2_lmt_info *lmt_info = &lfs->lmt_info;
 67	dma_addr_t align_iova;
 68	dma_addr_t iova;
 69
 70	lmt_info->base = dma_alloc_attrs(&pdev->dev, size, &iova, GFP_KERNEL,
 71					 DMA_ATTR_FORCE_CONTIGUOUS);
 72	if (!lmt_info->base)
 73		return -ENOMEM;
 74
 75	align_iova = ALIGN((u64)iova, LMTLINE_ALIGN);
 76	lmt_info->iova = align_iova;
 77	lmt_info->align = align_iova - iova;
 78	lmt_info->size = size;
 79	lmt_info->base += lmt_info->align;
 80	return 0;
 81}
 82
 83int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf)
 84{
 85	struct pci_dev *pdev = cptpf->pdev;
 86	u32 size;
 87	int ret;
 88
 89	if (!test_bit(CN10K_LMTST, &cptpf->cap_flag)) {
 90		cptpf->lfs.ops = &otx2_hw_ops;
 91		return 0;
 92	}
 93
 94	cptpf->lfs.ops = &cn10k_hw_ops;
 95	size = OTX2_CPT_MAX_VFS_NUM * LMTLINE_SIZE + LMTLINE_ALIGN;
 96	ret = cn10k_cpt_lmtst_alloc(pdev, &cptpf->lfs, size);
 97	if (ret) {
 98		dev_err(&pdev->dev, "PF-%d LMTLINE memory allocation failed\n",
 99			cptpf->pf_id);
100		return ret;
101	}
102
103	ret = otx2_cpt_lmtst_tbl_setup_msg(&cptpf->lfs);
104	if (ret) {
105		dev_err(&pdev->dev, "PF-%d: LMTST Table setup failed\n",
106		cptpf->pf_id);
107		cn10k_cpt_lmtst_free(pdev, &cptpf->lfs);
108	}
109
110	return 0;
111}
112EXPORT_SYMBOL_NS_GPL(cn10k_cptpf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT");
113
114int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf)
115{
116	struct pci_dev *pdev = cptvf->pdev;
117	u32 size;
118	int ret;
119
120	if (!test_bit(CN10K_LMTST, &cptvf->cap_flag))
121		return 0;
122
123	size = cptvf->lfs.lfs_num * LMTLINE_SIZE + LMTLINE_ALIGN;
124	ret = cn10k_cpt_lmtst_alloc(pdev, &cptvf->lfs, size);
125	if (ret) {
126		dev_err(&pdev->dev, "VF-%d LMTLINE memory allocation failed\n",
127			cptvf->vf_id);
128		return ret;
129	}
130
131	ret = otx2_cpt_lmtst_tbl_setup_msg(&cptvf->lfs);
132	if (ret) {
133		dev_err(&pdev->dev, "VF-%d: LMTST Table setup failed\n",
134			cptvf->vf_id);
135		cn10k_cpt_lmtst_free(pdev, &cptvf->lfs);
136	}
137
138	return 0;
139}
140EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, "CRYPTO_DEV_OCTEONTX2_CPT");
141
142void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev,
143			    struct cn10k_cpt_errata_ctx *er_ctx)
144{
145	u64 cptr_dma;
146
147	if (!is_dev_cn10ka_ax(pdev))
148		return;
149
150	cptr_dma = er_ctx->cptr_dma & ~(BIT_ULL(60));
151	cn10k_cpt_ctx_flush(pdev, cptr_dma, true);
152	dma_unmap_single(&pdev->dev, cptr_dma, CN10K_CPT_HW_CTX_SIZE,
153			 DMA_BIDIRECTIONAL);
154	kfree(er_ctx->hw_ctx);
155}
156EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, "CRYPTO_DEV_OCTEONTX2_CPT");
157
158void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz)
159{
160	hctx->w0.aop_valid = 1;
161	hctx->w0.ctx_hdr_sz = 0;
162	hctx->w0.ctx_sz = ctx_sz;
163	hctx->w0.ctx_push_sz = 1;
164}
165EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, "CRYPTO_DEV_OCTEONTX2_CPT");
166
167int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev,
168			  struct cn10k_cpt_errata_ctx *er_ctx)
169{
170	union cn10k_cpt_hw_ctx *hctx;
171	u64 cptr_dma;
172
173	er_ctx->cptr_dma = 0;
174	er_ctx->hw_ctx = NULL;
175
176	if (!is_dev_cn10ka_ax(pdev))
177		return 0;
178
179	hctx = kmalloc(CN10K_CPT_HW_CTX_SIZE, GFP_KERNEL);
180	if (unlikely(!hctx))
181		return -ENOMEM;
182	cptr_dma = dma_map_single(&pdev->dev, hctx, CN10K_CPT_HW_CTX_SIZE,
183				  DMA_BIDIRECTIONAL);
184	if (dma_mapping_error(&pdev->dev, cptr_dma)) {
185		kfree(hctx);
186		return -ENOMEM;
187	}
188
189	cn10k_cpt_hw_ctx_set(hctx, 1);
190	er_ctx->hw_ctx = hctx;
191	er_ctx->cptr_dma = cptr_dma | BIT_ULL(60);
192
193	return 0;
194}
195EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, "CRYPTO_DEV_OCTEONTX2_CPT");
196
197void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval)
198{
199	struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
200	struct otx2_cptlfs_info *lfs = &cptvf->lfs;
201	u64 reg;
202
203	reg = (uintptr_t)cptr >> 7;
204	if (inval)
205		reg = reg | BIT_ULL(46);
206
207	otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot,
208			 OTX2_CPT_LF_CTX_FLUSH, reg);
209	/* Make sure that the FLUSH operation is complete */
210	wmb();
211	otx2_cpt_read64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot,
212			OTX2_CPT_LF_CTX_ERR);
213}
214EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, "CRYPTO_DEV_OCTEONTX2_CPT");
215
216void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf)
217{
218	if (test_bit(CN10K_LMTST, &cptvf->cap_flag))
219		cptvf->lfs.ops = &cn10k_hw_ops;
220	else
221		cptvf->lfs.ops = &otx2_hw_ops;
222}
223EXPORT_SYMBOL_NS_GPL(cptvf_hw_ops_get, "CRYPTO_DEV_OCTEONTX2_CPT");