cptvf_main.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2016 Cavium, Inc.
  4 */
  5
  6#include <linux/interrupt.h>
  7#include <linux/module.h>
  8
  9#include "cptvf.h"
 10
 11#define DRV_NAME	"thunder-cptvf"
 12#define DRV_VERSION	"1.0"
 13
 14struct cptvf_wqe {
 15	struct tasklet_struct twork;
 16	void *cptvf;
 17	u32 qno;
 18};
 19
 20struct cptvf_wqe_info {
 21	struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
 22};
 23
 24static void vq_work_handler(unsigned long data)
 25{
 26	struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
 27	struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
 28
 29	vq_post_process(cwqe->cptvf, cwqe->qno);
 30}
 31
 32static int init_worker_threads(struct cpt_vf *cptvf)
 33{
 34	struct pci_dev *pdev = cptvf->pdev;
 35	struct cptvf_wqe_info *cwqe_info;
 36	int i;
 37
 38	cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
 39	if (!cwqe_info)
 40		return -ENOMEM;
 41
 42	if (cptvf->nr_queues) {
 43		dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
 44			 cptvf->nr_queues);
 45	}
 46
 47	for (i = 0; i < cptvf->nr_queues; i++) {
 48		tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
 49			     (u64)cwqe_info);
 50		cwqe_info->vq_wqe[i].qno = i;
 51		cwqe_info->vq_wqe[i].cptvf = cptvf;
 52	}
 53
 54	cptvf->wqe_info = cwqe_info;
 55
 56	return 0;
 57}
 58
 59static void cleanup_worker_threads(struct cpt_vf *cptvf)
 60{
 61	struct cptvf_wqe_info *cwqe_info;
 62	struct pci_dev *pdev = cptvf->pdev;
 63	int i;
 64
 65	cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
 66	if (!cwqe_info)
 67		return;
 68
 69	if (cptvf->nr_queues) {
 70		dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
 71			 cptvf->nr_queues);
 72	}
 73
 74	for (i = 0; i < cptvf->nr_queues; i++)
 75		tasklet_kill(&cwqe_info->vq_wqe[i].twork);
 76
 77	kfree_sensitive(cwqe_info);
 78	cptvf->wqe_info = NULL;
 79}
 80
 81static void free_pending_queues(struct pending_qinfo *pqinfo)
 82{
 83	int i;
 84	struct pending_queue *queue;
 85
 86	for_each_pending_queue(pqinfo, queue, i) {
 87		if (!queue->head)
 88			continue;
 89
 90		/* free single queue */
 91		kfree_sensitive((queue->head));
 92
 93		queue->front = 0;
 94		queue->rear = 0;
 95
 96		return;
 97	}
 98
 99	pqinfo->qlen = 0;
100	pqinfo->nr_queues = 0;
101}
102
103static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
104				u32 nr_queues)
105{
106	u32 i;
107	int ret;
108	struct pending_queue *queue = NULL;
109
110	pqinfo->nr_queues = nr_queues;
111	pqinfo->qlen = qlen;
112
113	for_each_pending_queue(pqinfo, queue, i) {
114		queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
115		if (!queue->head) {
116			ret = -ENOMEM;
117			goto pending_qfail;
118		}
119
120		queue->front = 0;
121		queue->rear = 0;
122		atomic64_set((&queue->pending_count), (0));
123
124		/* init queue spin lock */
125		spin_lock_init(&queue->lock);
126	}
127
128	return 0;
129
130pending_qfail:
131	free_pending_queues(pqinfo);
132
133	return ret;
134}
135
136static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
137{
138	struct pci_dev *pdev = cptvf->pdev;
139	int ret;
140
141	if (!nr_queues)
142		return 0;
143
144	ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
145	if (ret) {
146		dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
147			nr_queues);
148		return ret;
149	}
150
151	return 0;
152}
153
154static void cleanup_pending_queues(struct cpt_vf *cptvf)
155{
156	struct pci_dev *pdev = cptvf->pdev;
157
158	if (!cptvf->nr_queues)
159		return;
160
161	dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
162		 cptvf->nr_queues);
163	free_pending_queues(&cptvf->pqinfo);
164}
165
166static void free_command_queues(struct cpt_vf *cptvf,
167				struct command_qinfo *cqinfo)
168{
169	int i;
170	struct command_queue *queue = NULL;
171	struct command_chunk *chunk = NULL;
172	struct pci_dev *pdev = cptvf->pdev;
173	struct hlist_node *node;
174
175	/* clean up for each queue */
176	for (i = 0; i < cptvf->nr_queues; i++) {
177		queue = &cqinfo->queue[i];
178		if (hlist_empty(&cqinfo->queue[i].chead))
179			continue;
180
181		hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
182					  nextchunk) {
183			dma_free_coherent(&pdev->dev,
184					  chunk->size + CPT_NEXT_CHUNK_PTR_SIZE,
185					  chunk->head,
186					  chunk->dma_addr);
187			chunk->head = NULL;
188			chunk->dma_addr = 0;
189			hlist_del(&chunk->nextchunk);
190			kfree_sensitive(chunk);
191		}
192
193		queue->nchunks = 0;
194		queue->idx = 0;
195	}
196
197	/* common cleanup */
198	cqinfo->cmd_size = 0;
199}
200
201static int alloc_command_queues(struct cpt_vf *cptvf,
202				struct command_qinfo *cqinfo, size_t cmd_size,
203				u32 qlen)
204{
205	int i;
206	size_t q_size;
207	struct command_queue *queue = NULL;
208	struct pci_dev *pdev = cptvf->pdev;
209
210	/* common init */
211	cqinfo->cmd_size = cmd_size;
212	/* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
213	cptvf->qsize = min(qlen, cqinfo->qchunksize) *
214			CPT_NEXT_CHUNK_PTR_SIZE + 1;
215	/* Qsize in bytes to create space for alignment */
216	q_size = qlen * cqinfo->cmd_size;
217
218	/* per queue initialization */
219	for (i = 0; i < cptvf->nr_queues; i++) {
220		size_t c_size = 0;
221		size_t rem_q_size = q_size;
222		struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
223		u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
224
225		queue = &cqinfo->queue[i];
226		INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
227		do {
228			curr = kzalloc(sizeof(*curr), GFP_KERNEL);
229			if (!curr)
230				goto cmd_qfail;
231
232			c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
233					rem_q_size;
234			curr->head = dma_alloc_coherent(&pdev->dev,
235							c_size + CPT_NEXT_CHUNK_PTR_SIZE,
236							&curr->dma_addr,
237							GFP_KERNEL);
238			if (!curr->head) {
239				dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
240					i, queue->nchunks);
241				kfree(curr);
242				goto cmd_qfail;
243			}
244
245			curr->size = c_size;
246			if (queue->nchunks == 0) {
247				hlist_add_head(&curr->nextchunk,
248					       &cqinfo->queue[i].chead);
249				first = curr;
250			} else {
251				hlist_add_behind(&curr->nextchunk,
252						 &last->nextchunk);
253			}
254
255			queue->nchunks++;
256			rem_q_size -= c_size;
257			if (last)
258				*((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
259
260			last = curr;
261		} while (rem_q_size);
262
263		/* Make the queue circular */
264		/* Tie back last chunk entry to head */
265		curr = first;
266		*((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
267		queue->qhead = curr;
268		spin_lock_init(&queue->lock);
269	}
270	return 0;
271
272cmd_qfail:
273	free_command_queues(cptvf, cqinfo);
274	return -ENOMEM;
275}
276
277static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
278{
279	struct pci_dev *pdev = cptvf->pdev;
280	int ret;
281
282	/* setup AE command queues */
283	ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
284				   qlen);
285	if (ret) {
286		dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
287			cptvf->nr_queues);
288		return ret;
289	}
290
291	return ret;
292}
293
294static void cleanup_command_queues(struct cpt_vf *cptvf)
295{
296	struct pci_dev *pdev = cptvf->pdev;
297
298	if (!cptvf->nr_queues)
299		return;
300
301	dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
302		 cptvf->nr_queues);
303	free_command_queues(cptvf, &cptvf->cqinfo);
304}
305
306static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
307{
308	cleanup_worker_threads(cptvf);
309	cleanup_pending_queues(cptvf);
310	cleanup_command_queues(cptvf);
311}
312
313static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
314{
315	struct pci_dev *pdev = cptvf->pdev;
316	int ret = 0;
317	u32 max_dev_queues = 0;
318
319	max_dev_queues = CPT_NUM_QS_PER_VF;
320	/* possible cpus */
321	nr_queues = min_t(u32, nr_queues, max_dev_queues);
322	cptvf->nr_queues = nr_queues;
323
324	ret = init_command_queues(cptvf, qlen);
325	if (ret) {
326		dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
327			nr_queues);
328		return ret;
329	}
330
331	ret = init_pending_queues(cptvf, qlen, nr_queues);
332	if (ret) {
333		dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
334			nr_queues);
335		goto setup_pqfail;
336	}
337
338	/* Create worker threads for BH processing */
339	ret = init_worker_threads(cptvf);
340	if (ret) {
341		dev_err(&pdev->dev, "Failed to setup worker threads\n");
342		goto init_work_fail;
343	}
344
345	return 0;
346
347init_work_fail:
348	cleanup_worker_threads(cptvf);
349	cleanup_pending_queues(cptvf);
350
351setup_pqfail:
352	cleanup_command_queues(cptvf);
353
354	return ret;
355}
356
357static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec)
358{
359	irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
360	free_cpumask_var(cptvf->affinity_mask[vec]);
361}
362
363static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
364{
365	union cptx_vqx_ctl vqx_ctl;
366
367	vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
368	vqx_ctl.s.ena = val;
369	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
370}
371
372void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
373{
374	union cptx_vqx_doorbell vqx_dbell;
375
376	vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
377				     CPTX_VQX_DOORBELL(0, 0));
378	vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
379	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
380			vqx_dbell.u);
381}
382
383static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
384{
385	union cptx_vqx_inprog vqx_inprg;
386
387	vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
388	vqx_inprg.s.inflight = val;
389	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
390}
391
392static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
393{
394	union cptx_vqx_done_wait vqx_dwait;
395
396	vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
397				     CPTX_VQX_DONE_WAIT(0, 0));
398	vqx_dwait.s.num_wait = val;
399	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
400			vqx_dwait.u);
401}
402
403static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
404{
405	union cptx_vqx_done_wait vqx_dwait;
406
407	vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
408				     CPTX_VQX_DONE_WAIT(0, 0));
409	vqx_dwait.s.time_wait = time;
410	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
411			vqx_dwait.u);
412}
413
414static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
415{
416	union cptx_vqx_misc_ena_w1s vqx_misc_ena;
417
418	vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
419					CPTX_VQX_MISC_ENA_W1S(0, 0));
420	/* Set mbox(0) interupts for the requested vf */
421	vqx_misc_ena.s.swerr = 1;
422	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
423			vqx_misc_ena.u);
424}
425
426static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
427{
428	union cptx_vqx_misc_ena_w1s vqx_misc_ena;
429
430	vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
431					CPTX_VQX_MISC_ENA_W1S(0, 0));
432	/* Set mbox(0) interupts for the requested vf */
433	vqx_misc_ena.s.mbox = 1;
434	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
435			vqx_misc_ena.u);
436}
437
438static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
439{
440	union cptx_vqx_done_ena_w1s vqx_done_ena;
441
442	vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
443					CPTX_VQX_DONE_ENA_W1S(0, 0));
444	/* Set DONE interrupt for the requested vf */
445	vqx_done_ena.s.done = 1;
446	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
447			vqx_done_ena.u);
448}
449
450static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
451{
452	union cptx_vqx_misc_int vqx_misc_int;
453
454	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
455					CPTX_VQX_MISC_INT(0, 0));
456	/* W1C for the VF */
457	vqx_misc_int.s.dovf = 1;
458	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
459			vqx_misc_int.u);
460}
461
462static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
463{
464	union cptx_vqx_misc_int vqx_misc_int;
465
466	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
467					CPTX_VQX_MISC_INT(0, 0));
468	/* W1C for the VF */
469	vqx_misc_int.s.irde = 1;
470	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
471			vqx_misc_int.u);
472}
473
474static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
475{
476	union cptx_vqx_misc_int vqx_misc_int;
477
478	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
479					CPTX_VQX_MISC_INT(0, 0));
480	/* W1C for the VF */
481	vqx_misc_int.s.nwrp = 1;
482	cpt_write_csr64(cptvf->reg_base,
483			CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
484}
485
486static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
487{
488	union cptx_vqx_misc_int vqx_misc_int;
489
490	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
491					CPTX_VQX_MISC_INT(0, 0));
492	/* W1C for the VF */
493	vqx_misc_int.s.mbox = 1;
494	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
495			vqx_misc_int.u);
496}
497
498static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
499{
500	union cptx_vqx_misc_int vqx_misc_int;
501
502	vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
503					CPTX_VQX_MISC_INT(0, 0));
504	/* W1C for the VF */
505	vqx_misc_int.s.swerr = 1;
506	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
507			vqx_misc_int.u);
508}
509
510static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
511{
512	return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
513}
514
515static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
516{
517	struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
518	struct pci_dev *pdev = cptvf->pdev;
519	u64 intr;
520
521	intr = cptvf_read_vf_misc_intr_status(cptvf);
522	/*Check for MISC interrupt types*/
523	if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
524		dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
525			intr, cptvf->vfid);
526		cptvf_handle_mbox_intr(cptvf);
527		cptvf_clear_mbox_intr(cptvf);
528	} else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
529		cptvf_clear_dovf_intr(cptvf);
530		/*Clear doorbell count*/
531		cptvf_write_vq_doorbell(cptvf, 0);
532		dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
533			intr, cptvf->vfid);
534	} else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
535		cptvf_clear_irde_intr(cptvf);
536		dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
537			intr, cptvf->vfid);
538	} else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
539		cptvf_clear_nwrp_intr(cptvf);
540		dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
541			intr, cptvf->vfid);
542	} else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
543		cptvf_clear_swerr_intr(cptvf);
544		dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
545			intr, cptvf->vfid);
546	} else {
547		dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
548			cptvf->vfid);
549	}
550
551	return IRQ_HANDLED;
552}
553
554static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
555						 int qno)
556{
557	struct cptvf_wqe_info *nwqe_info;
558
559	if (unlikely(qno >= cptvf->nr_queues))
560		return NULL;
561	nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
562
563	return &nwqe_info->vq_wqe[qno];
564}
565
566static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
567{
568	union cptx_vqx_done vqx_done;
569
570	vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
571	return vqx_done.s.done;
572}
573
574static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
575					   u32 ackcnt)
576{
577	union cptx_vqx_done_ack vqx_dack_cnt;
578
579	vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
580					CPTX_VQX_DONE_ACK(0, 0));
581	vqx_dack_cnt.s.done_ack = ackcnt;
582	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
583			vqx_dack_cnt.u);
584}
585
586static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
587{
588	struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
589	struct pci_dev *pdev = cptvf->pdev;
590	/* Read the number of completions */
591	u32 intr = cptvf_read_vq_done_count(cptvf);
592
593	if (intr) {
594		struct cptvf_wqe *wqe;
595
596		/* Acknowledge the number of
597		 * scheduled completions for processing
598		 */
599		cptvf_write_vq_done_ack(cptvf, intr);
600		wqe = get_cptvf_vq_wqe(cptvf, 0);
601		if (unlikely(!wqe)) {
602			dev_err(&pdev->dev, "No work to schedule for VF (%d)",
603				cptvf->vfid);
604			return IRQ_NONE;
605		}
606		tasklet_hi_schedule(&wqe->twork);
607	}
608
609	return IRQ_HANDLED;
610}
611
612static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec)
613{
614	struct pci_dev *pdev = cptvf->pdev;
615	int cpu;
616
617	if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
618				GFP_KERNEL)) {
619		dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
620			cptvf->vfid);
621		return;
622	}
623
624	cpu = cptvf->vfid % num_online_cpus();
625	cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
626			cptvf->affinity_mask[vec]);
627	irq_set_affinity_hint(pci_irq_vector(pdev, vec),
628			cptvf->affinity_mask[vec]);
629}
630
631static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
632{
633	union cptx_vqx_saddr vqx_saddr;
634
635	vqx_saddr.u = val;
636	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
637}
638
639static void cptvf_device_init(struct cpt_vf *cptvf)
640{
641	u64 base_addr = 0;
642
643	/* Disable the VQ */
644	cptvf_write_vq_ctl(cptvf, 0);
645	/* Reset the doorbell */
646	cptvf_write_vq_doorbell(cptvf, 0);
647	/* Clear inflight */
648	cptvf_write_vq_inprog(cptvf, 0);
649	/* Write VQ SADDR */
650	/* TODO: for now only one queue, so hard coded */
651	base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
652	cptvf_write_vq_saddr(cptvf, base_addr);
653	/* Configure timerhold / coalescence */
654	cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
655	cptvf_write_vq_done_numwait(cptvf, 1);
656	/* Enable the VQ */
657	cptvf_write_vq_ctl(cptvf, 1);
658	/* Flag the VF ready */
659	cptvf->flags |= CPT_FLAG_DEVICE_READY;
660}
661
662static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
663{
664	struct device *dev = &pdev->dev;
665	struct cpt_vf *cptvf;
666	int    err;
667
668	cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
669	if (!cptvf)
670		return -ENOMEM;
671
672	pci_set_drvdata(pdev, cptvf);
673	cptvf->pdev = pdev;
674	err = pci_enable_device(pdev);
675	if (err) {
676		dev_err(dev, "Failed to enable PCI device\n");
677		pci_set_drvdata(pdev, NULL);
678		return err;
679	}
680
681	err = pci_request_regions(pdev, DRV_NAME);
682	if (err) {
683		dev_err(dev, "PCI request regions failed 0x%x\n", err);
684		goto cptvf_err_disable_device;
685	}
686	/* Mark as VF driver */
687	cptvf->flags |= CPT_FLAG_VF_DRIVER;
688	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
689	if (err) {
690		dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
691		goto cptvf_err_release_regions;
692	}
693
694	/* MAP PF's configuration registers */
695	cptvf->reg_base = pcim_iomap(pdev, 0, 0);
696	if (!cptvf->reg_base) {
697		dev_err(dev, "Cannot map config register space, aborting\n");
698		err = -ENOMEM;
699		goto cptvf_err_release_regions;
700	}
701
702	cptvf->node = dev_to_node(&pdev->dev);
703	err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS,
704			CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
705	if (err < 0) {
706		dev_err(dev, "Request for #%d msix vectors failed\n",
707			CPT_VF_MSIX_VECTORS);
708		goto cptvf_err_release_regions;
709	}
710
711	err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
712			  cptvf_misc_intr_handler, 0, "CPT VF misc intr",
713			  cptvf);
714	if (err) {
715		dev_err(dev, "Request misc irq failed");
716		goto cptvf_free_vectors;
717	}
718
719	/* Enable mailbox interrupt */
720	cptvf_enable_mbox_interrupts(cptvf);
721	cptvf_enable_swerr_interrupts(cptvf);
722
723	/* Check ready with PF */
724	/* Gets chip ID / device Id from PF if ready */
725	err = cptvf_check_pf_ready(cptvf);
726	if (err) {
727		dev_err(dev, "PF not responding to READY msg");
728		goto cptvf_free_misc_irq;
729	}
730
731	/* CPT VF software resources initialization */
732	cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
733	err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
734	if (err) {
735		dev_err(dev, "cptvf_sw_init() failed");
736		goto cptvf_free_misc_irq;
737	}
738	/* Convey VQ LEN to PF */
739	err = cptvf_send_vq_size_msg(cptvf);
740	if (err) {
741		dev_err(dev, "PF not responding to QLEN msg");
742		goto cptvf_free_misc_irq;
743	}
744
745	/* CPT VF device initialization */
746	cptvf_device_init(cptvf);
747	/* Send msg to PF to assign currnet Q to required group */
748	cptvf->vfgrp = 1;
749	err = cptvf_send_vf_to_grp_msg(cptvf);
750	if (err) {
751		dev_err(dev, "PF not responding to VF_GRP msg");
752		goto cptvf_free_misc_irq;
753	}
754
755	cptvf->priority = 1;
756	err = cptvf_send_vf_priority_msg(cptvf);
757	if (err) {
758		dev_err(dev, "PF not responding to VF_PRIO msg");
759		goto cptvf_free_misc_irq;
760	}
761
762	err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
763			  cptvf_done_intr_handler, 0, "CPT VF done intr",
764			  cptvf);
765	if (err) {
766		dev_err(dev, "Request done irq failed\n");
767		goto cptvf_free_misc_irq;
768	}
769
770	/* Enable mailbox interrupt */
771	cptvf_enable_done_interrupts(cptvf);
772
773	/* Set irq affinity masks */
774	cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
775	cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
776
777	err = cptvf_send_vf_up(cptvf);
778	if (err) {
779		dev_err(dev, "PF not responding to UP msg");
780		goto cptvf_free_irq_affinity;
781	}
782	err = cvm_crypto_init(cptvf);
783	if (err) {
784		dev_err(dev, "Algorithm register failed\n");
785		goto cptvf_free_irq_affinity;
786	}
787	return 0;
788
789cptvf_free_irq_affinity:
790	cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
791	cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
792cptvf_free_misc_irq:
793	free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
794cptvf_free_vectors:
795	pci_free_irq_vectors(cptvf->pdev);
796cptvf_err_release_regions:
797	pci_release_regions(pdev);
798cptvf_err_disable_device:
799	pci_disable_device(pdev);
800	pci_set_drvdata(pdev, NULL);
801
802	return err;
803}
804
805static void cptvf_remove(struct pci_dev *pdev)
806{
807	struct cpt_vf *cptvf = pci_get_drvdata(pdev);
808
809	if (!cptvf) {
810		dev_err(&pdev->dev, "Invalid CPT-VF device\n");
811		return;
812	}
813
814	/* Convey DOWN to PF */
815	if (cptvf_send_vf_down(cptvf)) {
816		dev_err(&pdev->dev, "PF not responding to DOWN msg");
817	} else {
818		cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
819		cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
820		free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
821		free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
822		pci_free_irq_vectors(cptvf->pdev);
823		cptvf_sw_cleanup(cptvf);
824		pci_set_drvdata(pdev, NULL);
825		pci_release_regions(pdev);
826		pci_disable_device(pdev);
827		cvm_crypto_exit();
828	}
829}
830
831static void cptvf_shutdown(struct pci_dev *pdev)
832{
833	cptvf_remove(pdev);
834}
835
836/* Supported devices */
837static const struct pci_device_id cptvf_id_table[] = {
838	{PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
839	{ 0, }  /* end of table */
840};
841
842static struct pci_driver cptvf_pci_driver = {
843	.name = DRV_NAME,
844	.id_table = cptvf_id_table,
845	.probe = cptvf_probe,
846	.remove = cptvf_remove,
847	.shutdown = cptvf_shutdown,
848};
849
850module_pci_driver(cptvf_pci_driver);
851
852MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
853MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
854MODULE_LICENSE("GPL v2");
855MODULE_VERSION(DRV_VERSION);
856MODULE_DEVICE_TABLE(pci, cptvf_id_table);