otx2_cptvf_main.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2020 Marvell. */
  3
  4#include "otx2_cpt_common.h"
  5#include "otx2_cptvf.h"
  6#include "otx2_cptlf.h"
  7#include "otx2_cptvf_algs.h"
  8#include "cn10k_cpt.h"
  9#include <rvu_reg.h>
 10
 11#define OTX2_CPTVF_DRV_NAME "rvu_cptvf"
 12
 13static void cptvf_enable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf)
 14{
 15	/* Clear interrupt if any */
 16	otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT,
 17			 0x1ULL);
 18
 19	/* Enable PF-VF interrupt */
 20	otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0,
 21			 OTX2_RVU_VF_INT_ENA_W1S, 0x1ULL);
 22}
 23
 24static void cptvf_disable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf)
 25{
 26	/* Disable PF-VF interrupt */
 27	otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0,
 28			 OTX2_RVU_VF_INT_ENA_W1C, 0x1ULL);
 29
 30	/* Clear interrupt if any */
 31	otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT,
 32			 0x1ULL);
 33}
 34
 35static int cptvf_register_interrupts(struct otx2_cptvf_dev *cptvf)
 36{
 37	int ret, irq;
 38	int num_vec;
 39
 40	num_vec = pci_msix_vec_count(cptvf->pdev);
 41	if (num_vec <= 0)
 42		return -EINVAL;
 43
 44	/* Enable MSI-X */
 45	ret = pci_alloc_irq_vectors(cptvf->pdev, num_vec, num_vec,
 46				    PCI_IRQ_MSIX);
 47	if (ret < 0) {
 48		dev_err(&cptvf->pdev->dev,
 49			"Request for %d msix vectors failed\n", num_vec);
 50		return ret;
 51	}
 52	irq = pci_irq_vector(cptvf->pdev, OTX2_CPT_VF_INT_VEC_E_MBOX);
 53	/* Register VF<=>PF mailbox interrupt handler */
 54	ret = devm_request_irq(&cptvf->pdev->dev, irq,
 55			       otx2_cptvf_pfvf_mbox_intr, 0,
 56			       "CPTPFVF Mbox", cptvf);
 57	if (ret)
 58		return ret;
 59	/* Enable PF-VF mailbox interrupts */
 60	cptvf_enable_pfvf_mbox_intrs(cptvf);
 61
 62	ret = otx2_cpt_send_ready_msg(&cptvf->pfvf_mbox, cptvf->pdev);
 63	if (ret) {
 64		dev_warn(&cptvf->pdev->dev,
 65			 "PF not responding to mailbox, deferring probe\n");
 66		cptvf_disable_pfvf_mbox_intrs(cptvf);
 67		return -EPROBE_DEFER;
 68	}
 69	return 0;
 70}
 71
 72static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf)
 73{
 74	struct pci_dev *pdev = cptvf->pdev;
 75	resource_size_t offset, size;
 76	int ret;
 77
 78	cptvf->pfvf_mbox_wq =
 79		alloc_ordered_workqueue("cpt_pfvf_mailbox",
 80					WQ_HIGHPRI | WQ_MEM_RECLAIM);
 81	if (!cptvf->pfvf_mbox_wq)
 82		return -ENOMEM;
 83
 84	if (test_bit(CN10K_MBOX, &cptvf->cap_flag)) {
 85		/* For cn10k platform, VF mailbox region is in its BAR2
 86		 * register space
 87		 */
 88		cptvf->pfvf_mbox_base = cptvf->reg_base +
 89					CN10K_CPT_VF_MBOX_REGION;
 90	} else {
 91		offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM);
 92		size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM);
 93		/* Map PF-VF mailbox memory */
 94		cptvf->pfvf_mbox_base = devm_ioremap_wc(&pdev->dev, offset,
 95							size);
 96		if (!cptvf->pfvf_mbox_base) {
 97			dev_err(&pdev->dev, "Unable to map BAR4\n");
 98			ret = -ENOMEM;
 99			goto free_wqe;
100		}
101	}
102
103	ret = otx2_mbox_init(&cptvf->pfvf_mbox, cptvf->pfvf_mbox_base,
104			     pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1);
105	if (ret)
106		goto free_wqe;
107
108	ret = otx2_cpt_mbox_bbuf_init(cptvf, pdev);
109	if (ret)
110		goto destroy_mbox;
111
112	INIT_WORK(&cptvf->pfvf_mbox_work, otx2_cptvf_pfvf_mbox_handler);
113	return 0;
114
115destroy_mbox:
116	otx2_mbox_destroy(&cptvf->pfvf_mbox);
117free_wqe:
118	destroy_workqueue(cptvf->pfvf_mbox_wq);
119	return ret;
120}
121
122static void cptvf_pfvf_mbox_destroy(struct otx2_cptvf_dev *cptvf)
123{
124	destroy_workqueue(cptvf->pfvf_mbox_wq);
125	otx2_mbox_destroy(&cptvf->pfvf_mbox);
126}
127
128static void cptlf_work_handler(unsigned long data)
129{
130	otx2_cpt_post_process((struct otx2_cptlf_wqe *) data);
131}
132
133static void cleanup_tasklet_work(struct otx2_cptlfs_info *lfs)
134{
135	int i;
136
137	for (i = 0; i <  lfs->lfs_num; i++) {
138		if (!lfs->lf[i].wqe)
139			continue;
140
141		tasklet_kill(&lfs->lf[i].wqe->work);
142		kfree(lfs->lf[i].wqe);
143		lfs->lf[i].wqe = NULL;
144	}
145}
146
147static int init_tasklet_work(struct otx2_cptlfs_info *lfs)
148{
149	struct otx2_cptlf_wqe *wqe;
150	int i, ret = 0;
151
152	for (i = 0; i < lfs->lfs_num; i++) {
153		wqe = kzalloc(sizeof(struct otx2_cptlf_wqe), GFP_KERNEL);
154		if (!wqe) {
155			ret = -ENOMEM;
156			goto cleanup_tasklet;
157		}
158
159		tasklet_init(&wqe->work, cptlf_work_handler, (u64) wqe);
160		wqe->lfs = lfs;
161		wqe->lf_num = i;
162		lfs->lf[i].wqe = wqe;
163	}
164	return 0;
165
166cleanup_tasklet:
167	cleanup_tasklet_work(lfs);
168	return ret;
169}
170
171static void free_pending_queues(struct otx2_cptlfs_info *lfs)
172{
173	int i;
174
175	for (i = 0; i < lfs->lfs_num; i++) {
176		kfree(lfs->lf[i].pqueue.head);
177		lfs->lf[i].pqueue.head = NULL;
178	}
179}
180
181static int alloc_pending_queues(struct otx2_cptlfs_info *lfs)
182{
183	int size, ret, i;
184
185	if (!lfs->lfs_num)
186		return -EINVAL;
187
188	for (i = 0; i < lfs->lfs_num; i++) {
189		lfs->lf[i].pqueue.qlen = OTX2_CPT_INST_QLEN_MSGS;
190		size = lfs->lf[i].pqueue.qlen *
191		       sizeof(struct otx2_cpt_pending_entry);
192
193		lfs->lf[i].pqueue.head = kzalloc(size, GFP_KERNEL);
194		if (!lfs->lf[i].pqueue.head) {
195			ret = -ENOMEM;
196			goto error;
197		}
198
199		/* Initialize spin lock */
200		spin_lock_init(&lfs->lf[i].pqueue.lock);
201	}
202	return 0;
203
204error:
205	free_pending_queues(lfs);
206	return ret;
207}
208
209static void lf_sw_cleanup(struct otx2_cptlfs_info *lfs)
210{
211	cleanup_tasklet_work(lfs);
212	free_pending_queues(lfs);
213}
214
215static int lf_sw_init(struct otx2_cptlfs_info *lfs)
216{
217	int ret;
218
219	ret = alloc_pending_queues(lfs);
220	if (ret) {
221		dev_err(&lfs->pdev->dev,
222			"Allocating pending queues failed\n");
223		return ret;
224	}
225	ret = init_tasklet_work(lfs);
226	if (ret) {
227		dev_err(&lfs->pdev->dev,
228			"Tasklet work init failed\n");
229		goto pending_queues_free;
230	}
231	return 0;
232
233pending_queues_free:
234	free_pending_queues(lfs);
235	return ret;
236}
237
238static void cptvf_lf_shutdown(struct otx2_cptlfs_info *lfs)
239{
240	atomic_set(&lfs->state, OTX2_CPTLF_IN_RESET);
241
242	/* Remove interrupts affinity */
243	otx2_cptlf_free_irqs_affinity(lfs);
244	/* Disable instruction queue */
245	otx2_cptlf_disable_iqueues(lfs);
246	/* Unregister crypto algorithms */
247	otx2_cpt_crypto_exit(lfs->pdev, THIS_MODULE);
248	/* Unregister LFs interrupts */
249	otx2_cptlf_unregister_misc_interrupts(lfs);
250	otx2_cptlf_unregister_done_interrupts(lfs);
251	/* Cleanup LFs software side */
252	lf_sw_cleanup(lfs);
253	/* Free instruction queues */
254	otx2_cpt_free_instruction_queues(lfs);
255	/* Send request to detach LFs */
256	otx2_cpt_detach_rsrcs_msg(lfs);
257	lfs->lfs_num = 0;
258}
259
260static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf)
261{
262	struct otx2_cptlfs_info *lfs = &cptvf->lfs;
263	struct device *dev = &cptvf->pdev->dev;
264	int ret, lfs_num;
265	u8 eng_grp_msk;
266
267	/* Get engine group number for symmetric crypto */
268	cptvf->lfs.kcrypto_se_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP;
269	ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_SE_TYPES);
270	if (ret)
271		return ret;
272
273	if (cptvf->lfs.kcrypto_se_eng_grp_num ==
274		OTX2_CPT_INVALID_CRYPTO_ENG_GRP) {
275		dev_err(dev,
276			"Symmetric Engine group for crypto not available\n");
277		return -ENOENT;
278	}
279
280	/* Get engine group number for asymmetric crypto */
281	cptvf->lfs.kcrypto_ae_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP;
282	ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_AE_TYPES);
283	if (ret)
284		return ret;
285
286	if (cptvf->lfs.kcrypto_ae_eng_grp_num ==
287		OTX2_CPT_INVALID_CRYPTO_ENG_GRP) {
288		dev_err(dev,
289			"Asymmetric Engine group for crypto not available\n");
290		return -ENOENT;
291	}
292
293	eng_grp_msk = BIT(cptvf->lfs.kcrypto_se_eng_grp_num) |
294		      BIT(cptvf->lfs.kcrypto_ae_eng_grp_num);
295
296	ret = otx2_cptvf_send_kvf_limits_msg(cptvf);
297	if (ret)
298		return ret;
299
300	lfs_num = cptvf->lfs.kvf_limits;
301
302	ret = otx2_cptlf_init(lfs, eng_grp_msk, OTX2_CPT_QUEUE_HI_PRIO,
303			      lfs_num);
304	if (ret)
305		return ret;
306
307	/* Get msix offsets for attached LFs */
308	ret = otx2_cpt_msix_offset_msg(lfs);
309	if (ret)
310		goto cleanup_lf;
311
312	/* Initialize LFs software side */
313	ret = lf_sw_init(lfs);
314	if (ret)
315		goto cleanup_lf;
316
317	/* Register LFs interrupts */
318	ret = otx2_cptlf_register_misc_interrupts(lfs);
319	if (ret)
320		goto cleanup_lf_sw;
321
322	ret = otx2_cptlf_register_done_interrupts(lfs);
323	if (ret)
324		goto cleanup_lf_sw;
325
326	/* Set interrupts affinity */
327	ret = otx2_cptlf_set_irqs_affinity(lfs);
328	if (ret)
329		goto unregister_intr;
330
331	atomic_set(&lfs->state, OTX2_CPTLF_STARTED);
332	/* Register crypto algorithms */
333	ret = otx2_cpt_crypto_init(lfs->pdev, THIS_MODULE, lfs_num, 1);
334	if (ret) {
335		dev_err(&lfs->pdev->dev, "algorithms registration failed\n");
336		goto disable_irqs;
337	}
338	return 0;
339
340disable_irqs:
341	otx2_cptlf_free_irqs_affinity(lfs);
342unregister_intr:
343	otx2_cptlf_unregister_misc_interrupts(lfs);
344	otx2_cptlf_unregister_done_interrupts(lfs);
345cleanup_lf_sw:
346	lf_sw_cleanup(lfs);
347cleanup_lf:
348	otx2_cptlf_shutdown(lfs);
349
350	return ret;
351}
352
353static int otx2_cptvf_probe(struct pci_dev *pdev,
354			    const struct pci_device_id *ent)
355{
356	struct device *dev = &pdev->dev;
357	struct otx2_cptvf_dev *cptvf;
358	int ret;
359
360	cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
361	if (!cptvf)
362		return -ENOMEM;
363
364	ret = pcim_enable_device(pdev);
365	if (ret) {
366		dev_err(dev, "Failed to enable PCI device\n");
367		goto clear_drvdata;
368	}
369
370	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
371	if (ret) {
372		dev_err(dev, "Unable to get usable DMA configuration\n");
373		goto clear_drvdata;
374	}
375
376	ret = pcim_request_all_regions(pdev, OTX2_CPTVF_DRV_NAME);
377	if (ret) {
378		dev_err(dev, "Couldn't get PCI resources 0x%x\n", ret);
379		goto clear_drvdata;
380	}
381	pci_set_master(pdev);
382	pci_set_drvdata(pdev, cptvf);
383	cptvf->pdev = pdev;
384
385	/* Map VF's configuration registers */
386	cptvf->reg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
387	if (!cptvf->reg_base) {
388		ret = -ENOMEM;
389		dev_err(dev, "Couldn't ioremap PCI resource 0x%x\n", ret);
390		goto clear_drvdata;
391	}
392
393	otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag);
394
395	/* Initialize PF<=>VF mailbox */
396	ret = cptvf_pfvf_mbox_init(cptvf);
397	if (ret)
398		goto clear_drvdata;
399
400	/* Register interrupts */
401	ret = cptvf_register_interrupts(cptvf);
402	if (ret)
403		goto destroy_pfvf_mbox;
404
405	cptvf->blkaddr = BLKADDR_CPT0;
406
407	cptvf_hw_ops_get(cptvf);
408
409	otx2_cptlf_set_dev_info(&cptvf->lfs, cptvf->pdev, cptvf->reg_base,
410				&cptvf->pfvf_mbox, cptvf->blkaddr);
411
412	ret = otx2_cptvf_send_caps_msg(cptvf);
413	if (ret) {
414		dev_err(&pdev->dev, "Couldn't get CPT engine capabilities.\n");
415		goto unregister_interrupts;
416	}
417	if (cptvf->eng_caps[OTX2_CPT_SE_TYPES] & BIT_ULL(35))
418		cptvf->lfs.ops->cpt_sg_info_create = cn10k_sgv2_info_create;
419
420	ret = cn10k_cptvf_lmtst_init(cptvf);
421	if (ret)
422		goto unregister_interrupts;
423
424	/* Initialize CPT LFs */
425	ret = cptvf_lf_init(cptvf);
426	if (ret)
427		goto free_lmtst;
428
429	return 0;
430
431free_lmtst:
432	cn10k_cpt_lmtst_free(pdev, &cptvf->lfs);
433unregister_interrupts:
434	cptvf_disable_pfvf_mbox_intrs(cptvf);
435destroy_pfvf_mbox:
436	cptvf_pfvf_mbox_destroy(cptvf);
437clear_drvdata:
438	pci_set_drvdata(pdev, NULL);
439
440	return ret;
441}
442
443static void otx2_cptvf_remove(struct pci_dev *pdev)
444{
445	struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
446
447	if (!cptvf) {
448		dev_err(&pdev->dev, "Invalid CPT VF device.\n");
449		return;
450	}
451	cptvf_lf_shutdown(&cptvf->lfs);
452	/* Disable PF-VF mailbox interrupt */
453	cptvf_disable_pfvf_mbox_intrs(cptvf);
454	/* Destroy PF-VF mbox */
455	cptvf_pfvf_mbox_destroy(cptvf);
456	/* Free LMTST memory */
457	cn10k_cpt_lmtst_free(pdev, &cptvf->lfs);
458	pci_set_drvdata(pdev, NULL);
459}
460
461/* Supported devices */
462static const struct pci_device_id otx2_cptvf_id_table[] = {
463	{PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0},
464	{PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID), 0},
465	{ 0, }  /* end of table */
466};
467
468static struct pci_driver otx2_cptvf_pci_driver = {
469	.name = OTX2_CPTVF_DRV_NAME,
470	.id_table = otx2_cptvf_id_table,
471	.probe = otx2_cptvf_probe,
472	.remove = otx2_cptvf_remove,
473};
474
475module_pci_driver(otx2_cptvf_pci_driver);
476
477MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT");
478
479MODULE_AUTHOR("Marvell");
480MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver");
481MODULE_LICENSE("GPL v2");
482MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table);