octep_vdpa_main.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2024 Marvell. */
  3
  4#include <linux/interrupt.h>
  5#include <linux/io-64-nonatomic-lo-hi.h>
  6#include <linux/module.h>
  7#include <linux/iommu.h>
  8#include "octep_vdpa.h"
  9
 10#define OCTEP_VDPA_DRIVER_NAME "octep_vdpa"
 11
 12struct octep_pf {
 13	u8 __iomem *base[PCI_STD_NUM_BARS];
 14	struct pci_dev *pdev;
 15	struct resource res;
 16	u64 vf_base;
 17	int enabled_vfs;
 18	u32 vf_stride;
 19	u16 vf_devid;
 20};
 21
 22struct octep_vdpa {
 23	struct vdpa_device vdpa;
 24	struct octep_hw *oct_hw;
 25	struct pci_dev *pdev;
 26};
 27
 28struct octep_vdpa_mgmt_dev {
 29	struct vdpa_mgmt_dev mdev;
 30	struct octep_hw oct_hw;
 31	struct pci_dev *pdev;
 32	/* Work entry to handle device setup */
 33	struct work_struct setup_task;
 34	/* Device status */
 35	atomic_t status;
 36};
 37
 38static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev)
 39{
 40	struct octep_vdpa *oct_vdpa;
 41
 42	oct_vdpa = container_of(vdpa_dev, struct octep_vdpa, vdpa);
 43
 44	return oct_vdpa->oct_hw;
 45}
 46
 47static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
 48{
 49	struct octep_hw *oct_hw = data;
 50	int i;
 51
 52	/* Each device has multiple interrupts (nb_irqs) shared among rings
 53	 * (nr_vring). Device interrupts are mapped to the rings in a
 54	 * round-robin fashion.
 55	 *
 56	 * For example, if nb_irqs = 8 and nr_vring = 64:
 57	 * 0 -> 0, 8, 16, 24, 32, 40, 48, 56;
 58	 * 1 -> 1, 9, 17, 25, 33, 41, 49, 57;
 59	 * ...
 60	 * 7 -> 7, 15, 23, 31, 39, 47, 55, 63;
 61	 */
 62
 63	for (i = irq - oct_hw->irqs[0]; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) {
 64		if (ioread8(oct_hw->vqs[i].cb_notify_addr)) {
 65			/* Acknowledge the per ring notification to the device */
 66			iowrite8(0, oct_hw->vqs[i].cb_notify_addr);
 67
 68			if (likely(oct_hw->vqs[i].cb.callback))
 69				oct_hw->vqs[i].cb.callback(oct_hw->vqs[i].cb.private);
 70			break;
 71		}
 72	}
 73
 74	/* Check for config interrupt. Config uses the first interrupt */
 75	if (unlikely(irq == oct_hw->irqs[0] && ioread8(oct_hw->isr))) {
 76		iowrite8(0, oct_hw->isr);
 77
 78		if (oct_hw->config_cb.callback)
 79			oct_hw->config_cb.callback(oct_hw->config_cb.private);
 80	}
 81
 82	return IRQ_HANDLED;
 83}
 84
 85static void octep_free_irqs(struct octep_hw *oct_hw)
 86{
 87	struct pci_dev *pdev = oct_hw->pdev;
 88	int irq;
 89
 90	if (!oct_hw->irqs)
 91		return;
 92
 93	for (irq = 0; irq < oct_hw->nb_irqs; irq++) {
 94		if (!oct_hw->irqs[irq])
 95			break;
 96
 97		devm_free_irq(&pdev->dev, oct_hw->irqs[irq], oct_hw);
 98	}
 99
100	pci_free_irq_vectors(pdev);
101	devm_kfree(&pdev->dev, oct_hw->irqs);
102	oct_hw->irqs = NULL;
103}
104
105static int octep_request_irqs(struct octep_hw *oct_hw)
106{
107	struct pci_dev *pdev = oct_hw->pdev;
108	int ret, irq, idx;
109
110	oct_hw->irqs = devm_kcalloc(&pdev->dev, oct_hw->nb_irqs, sizeof(int), GFP_KERNEL);
111	if (!oct_hw->irqs)
112		return -ENOMEM;
113
114	ret = pci_alloc_irq_vectors(pdev, 1, oct_hw->nb_irqs, PCI_IRQ_MSIX);
115	if (ret < 0) {
116		dev_err(&pdev->dev, "Failed to alloc msix vector");
117		return ret;
118	}
119
120	for (idx = 0; idx < oct_hw->nb_irqs; idx++) {
121		irq = pci_irq_vector(pdev, idx);
122		ret = devm_request_irq(&pdev->dev, irq, octep_vdpa_intr_handler, 0,
123				       dev_name(&pdev->dev), oct_hw);
124		if (ret) {
125			dev_err(&pdev->dev, "Failed to register interrupt handler\n");
126			goto free_irqs;
127		}
128		oct_hw->irqs[idx] = irq;
129	}
130
131	return 0;
132
133free_irqs:
134	octep_free_irqs(oct_hw);
135	return ret;
136}
137
138static u64 octep_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
139{
140	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
141
142	return oct_hw->features;
143}
144
145static int octep_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
146{
147	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
148	int ret;
149
150	pr_debug("Driver Features: %llx\n", features);
151
152	ret = octep_verify_features(features);
153	if (ret) {
154		dev_warn(&oct_hw->pdev->dev,
155			 "Must negotiate minimum features 0x%llx for this device",
156			 BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) |
157			 BIT_ULL(VIRTIO_F_RING_PACKED));
158		return ret;
159	}
160	octep_hw_set_drv_features(oct_hw, features);
161
162	return 0;
163}
164
165static u64 octep_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
166{
167	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
168
169	return octep_hw_get_drv_features(oct_hw);
170}
171
172static u8 octep_vdpa_get_status(struct vdpa_device *vdpa_dev)
173{
174	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
175
176	return octep_hw_get_status(oct_hw);
177}
178
179static void octep_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
180{
181	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
182	u8 status_old;
183
184	status_old = octep_hw_get_status(oct_hw);
185
186	if (status_old == status)
187		return;
188
189	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
190	    !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
191		if (octep_request_irqs(oct_hw))
192			status = status_old | VIRTIO_CONFIG_S_FAILED;
193	}
194	octep_hw_set_status(oct_hw, status);
195}
196
197static int octep_vdpa_reset(struct vdpa_device *vdpa_dev)
198{
199	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
200	u8 status = octep_hw_get_status(oct_hw);
201	u16 qid;
202
203	if (status == 0)
204		return 0;
205
206	for (qid = 0; qid < oct_hw->nr_vring; qid++) {
207		oct_hw->vqs[qid].cb.callback = NULL;
208		oct_hw->vqs[qid].cb.private = NULL;
209		oct_hw->config_cb.callback = NULL;
210		oct_hw->config_cb.private = NULL;
211	}
212	octep_hw_reset(oct_hw);
213
214	if (status & VIRTIO_CONFIG_S_DRIVER_OK)
215		octep_free_irqs(oct_hw);
216
217	return 0;
218}
219
220static u16 octep_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
221{
222	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
223
224	return octep_get_vq_size(oct_hw);
225}
226
227static int octep_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
228				   struct vdpa_vq_state *state)
229{
230	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
231
232	return octep_get_vq_state(oct_hw, qid, state);
233}
234
235static int octep_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
236				   const struct vdpa_vq_state *state)
237{
238	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
239
240	return octep_set_vq_state(oct_hw, qid, state);
241}
242
243static void octep_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, struct vdpa_callback *cb)
244{
245	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
246
247	oct_hw->vqs[qid].cb = *cb;
248}
249
250static void octep_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
251{
252	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
253
254	octep_set_vq_ready(oct_hw, qid, ready);
255}
256
257static bool octep_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
258{
259	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
260
261	return octep_get_vq_ready(oct_hw, qid);
262}
263
264static void octep_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
265{
266	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
267
268	octep_set_vq_num(oct_hw, qid, num);
269}
270
271static int octep_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, u64 desc_area,
272				     u64 driver_area, u64 device_area)
273{
274	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
275
276	pr_debug("qid[%d]: desc_area: %llx\n", qid, desc_area);
277	pr_debug("qid[%d]: driver_area: %llx\n", qid, driver_area);
278	pr_debug("qid[%d]: device_area: %llx\n\n", qid, device_area);
279
280	return octep_set_vq_address(oct_hw, qid, desc_area, driver_area, device_area);
281}
282
283static void octep_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
284{
285	/* Not supported */
286}
287
288static void octep_vdpa_kick_vq_with_data(struct vdpa_device *vdpa_dev, u32 data)
289{
290	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
291	u16 idx = data & 0xFFFF;
292
293	vp_iowrite32(data, oct_hw->vqs[idx].notify_addr);
294}
295
296static u32 octep_vdpa_get_generation(struct vdpa_device *vdpa_dev)
297{
298	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
299
300	return vp_ioread8(&oct_hw->common_cfg->config_generation);
301}
302
303static u32 octep_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
304{
305	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
306
307	return oct_hw->dev_id;
308}
309
310static u32 octep_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
311{
312	return PCI_VENDOR_ID_CAVIUM;
313}
314
315static u32 octep_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
316{
317	return PAGE_SIZE;
318}
319
320static size_t octep_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
321{
322	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
323
324	return oct_hw->config_size;
325}
326
327static void octep_vdpa_get_config(struct vdpa_device *vdpa_dev, unsigned int offset, void *buf,
328				  unsigned int len)
329{
330	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
331
332	octep_read_dev_config(oct_hw, offset, buf, len);
333}
334
335static void octep_vdpa_set_config(struct vdpa_device *vdpa_dev, unsigned int offset,
336				  const void *buf, unsigned int len)
337{
338	/* Not supported */
339}
340
341static void octep_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, struct vdpa_callback *cb)
342{
343	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
344
345	oct_hw->config_cb.callback = cb->callback;
346	oct_hw->config_cb.private = cb->private;
347}
348
349static struct vdpa_notification_area octep_get_vq_notification(struct vdpa_device *vdpa_dev,
350							       u16 idx)
351{
352	struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
353	struct vdpa_notification_area area;
354
355	area.addr = oct_hw->vqs[idx].notify_pa;
356	area.size = PAGE_SIZE;
357
358	return area;
359}
360
361static struct vdpa_config_ops octep_vdpa_ops = {
362	.get_device_features = octep_vdpa_get_device_features,
363	.set_driver_features = octep_vdpa_set_driver_features,
364	.get_driver_features = octep_vdpa_get_driver_features,
365	.get_status	= octep_vdpa_get_status,
366	.set_status	= octep_vdpa_set_status,
367	.reset		= octep_vdpa_reset,
368	.get_vq_num_max	= octep_vdpa_get_vq_num_max,
369	.get_vq_state	= octep_vdpa_get_vq_state,
370	.set_vq_state	= octep_vdpa_set_vq_state,
371	.set_vq_cb	= octep_vdpa_set_vq_cb,
372	.set_vq_ready	= octep_vdpa_set_vq_ready,
373	.get_vq_ready	= octep_vdpa_get_vq_ready,
374	.set_vq_num	= octep_vdpa_set_vq_num,
375	.set_vq_address	= octep_vdpa_set_vq_address,
376	.get_vq_irq	= NULL,
377	.kick_vq	= octep_vdpa_kick_vq,
378	.kick_vq_with_data	= octep_vdpa_kick_vq_with_data,
379	.get_generation	= octep_vdpa_get_generation,
380	.get_device_id	= octep_vdpa_get_device_id,
381	.get_vendor_id	= octep_vdpa_get_vendor_id,
382	.get_vq_align	= octep_vdpa_get_vq_align,
383	.get_config_size	= octep_vdpa_get_config_size,
384	.get_config	= octep_vdpa_get_config,
385	.set_config	= octep_vdpa_set_config,
386	.set_config_cb  = octep_vdpa_set_config_cb,
387	.get_vq_notification = octep_get_vq_notification,
388};
389
390static int octep_iomap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar)
391{
392	int ret;
393
394	ret = pci_request_region(pdev, bar, OCTEP_VDPA_DRIVER_NAME);
395	if (ret) {
396		dev_err(&pdev->dev, "Failed to request BAR:%u region\n", bar);
397		return ret;
398	}
399
400	tbl[bar] = pci_iomap(pdev, bar, pci_resource_len(pdev, bar));
401	if (!tbl[bar]) {
402		dev_err(&pdev->dev, "Failed to iomap BAR:%u\n", bar);
403		pci_release_region(pdev, bar);
404		ret = -ENOMEM;
405	}
406
407	return ret;
408}
409
410static void octep_iounmap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar)
411{
412	pci_iounmap(pdev, tbl[bar]);
413	pci_release_region(pdev, bar);
414}
415
416static void octep_vdpa_pf_bar_shrink(struct octep_pf *octpf)
417{
418	struct pci_dev *pf_dev = octpf->pdev;
419	struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;
420	struct pci_bus_region bus_region;
421
422	octpf->res.start = res->start;
423	octpf->res.end = res->end;
424	octpf->vf_base = res->start;
425
426	bus_region.start = res->start;
427	bus_region.end = res->start - 1;
428
429	pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);
430}
431
432static void octep_vdpa_pf_bar_expand(struct octep_pf *octpf)
433{
434	struct pci_dev *pf_dev = octpf->pdev;
435	struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;
436	struct pci_bus_region bus_region;
437
438	bus_region.start = octpf->res.start;
439	bus_region.end = octpf->res.end;
440
441	pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);
442}
443
444static void octep_vdpa_remove_pf(struct pci_dev *pdev)
445{
446	struct octep_pf *octpf = pci_get_drvdata(pdev);
447
448	pci_disable_sriov(pdev);
449
450	if (octpf->base[OCTEP_HW_CAPS_BAR])
451		octep_iounmap_region(pdev, octpf->base, OCTEP_HW_CAPS_BAR);
452
453	if (octpf->base[OCTEP_HW_MBOX_BAR])
454		octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
455
456	octep_vdpa_pf_bar_expand(octpf);
457
458	/* The pf version does not use managed PCI. */
459	pci_disable_device(pdev);
460}
461
462static void octep_vdpa_vf_bar_shrink(struct pci_dev *pdev)
463{
464	struct resource *vf_res = pdev->resource + PCI_STD_RESOURCES + 4;
465
466	memset(vf_res, 0, sizeof(*vf_res));
467}
468
469static void octep_vdpa_remove_vf(struct pci_dev *pdev)
470{
471	struct octep_vdpa_mgmt_dev *mgmt_dev = pci_get_drvdata(pdev);
472	struct octep_hw *oct_hw;
473	int status;
474
475	oct_hw = &mgmt_dev->oct_hw;
476	status = atomic_read(&mgmt_dev->status);
477	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT);
478
479	cancel_work_sync(&mgmt_dev->setup_task);
480	if (status == OCTEP_VDPA_DEV_STATUS_READY)
481		vdpa_mgmtdev_unregister(&mgmt_dev->mdev);
482
483	if (oct_hw->base[OCTEP_HW_CAPS_BAR])
484		octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
485
486	if (oct_hw->base[OCTEP_HW_MBOX_BAR])
487		octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR);
488
489	octep_vdpa_vf_bar_shrink(pdev);
490}
491
492static void octep_vdpa_remove(struct pci_dev *pdev)
493{
494	if (pdev->is_virtfn)
495		octep_vdpa_remove_vf(pdev);
496	else
497		octep_vdpa_remove_pf(pdev);
498}
499
500static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
501			      const struct vdpa_dev_set_config *config)
502{
503	struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev);
504	struct octep_hw *oct_hw = &mgmt_dev->oct_hw;
505	struct pci_dev *pdev = oct_hw->pdev;
506	struct vdpa_device *vdpa_dev;
507	struct octep_vdpa *oct_vdpa;
508	u64 device_features;
509	int ret;
510
511	oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops,
512				     NULL, 1, 1, NULL, false);
513	if (IS_ERR(oct_vdpa)) {
514		dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device");
515		return PTR_ERR(oct_vdpa);
516	}
517
518	oct_vdpa->pdev = pdev;
519	oct_vdpa->vdpa.vmap.dma_dev = &pdev->dev;
520	oct_vdpa->vdpa.mdev = mdev;
521	oct_vdpa->oct_hw = oct_hw;
522	vdpa_dev = &oct_vdpa->vdpa;
523
524	device_features = oct_hw->features;
525	if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
526		if (config->device_features & ~device_features) {
527			dev_err(&pdev->dev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n",
528				config->device_features, device_features);
529			ret = -EINVAL;
530			goto vdpa_dev_put;
531		}
532		device_features &= config->device_features;
533	}
534
535	oct_hw->features = device_features;
536	dev_info(&pdev->dev, "Vdpa management device features : %llx\n", device_features);
537
538	ret = octep_verify_features(device_features);
539	if (ret) {
540		dev_warn(mdev->device,
541			 "Must provision minimum features 0x%llx for this device",
542			 BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) |
543			 BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | BIT_ULL(VIRTIO_F_RING_PACKED));
544		goto vdpa_dev_put;
545	}
546	if (name)
547		ret = dev_set_name(&vdpa_dev->dev, "%s", name);
548	else
549		ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);
550
551	ret = _vdpa_register_device(&oct_vdpa->vdpa, oct_hw->nr_vring);
552	if (ret) {
553		dev_err(&pdev->dev, "Failed to register to vDPA bus");
554		goto vdpa_dev_put;
555	}
556	return 0;
557
558vdpa_dev_put:
559	put_device(&oct_vdpa->vdpa.dev);
560	return ret;
561}
562
563static void octep_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *vdpa_dev)
564{
565	_vdpa_unregister_device(vdpa_dev);
566}
567
568static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = {
569	.dev_add = octep_vdpa_dev_add,
570	.dev_del = octep_vdpa_dev_del
571};
572
573static bool get_device_ready_status(u8 __iomem *addr)
574{
575	u64 signature = readq(addr + OCTEP_VF_MBOX_DATA(0));
576
577	if (signature == OCTEP_DEV_READY_SIGNATURE) {
578		writeq(0, addr + OCTEP_VF_MBOX_DATA(0));
579		return true;
580	}
581
582	return false;
583}
584
585static struct virtio_device_id id_table[] = {
586	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
587	{ 0 },
588};
589
590static void octep_vdpa_setup_task(struct work_struct *work)
591{
592	struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(work, struct octep_vdpa_mgmt_dev,
593							    setup_task);
594	struct pci_dev *pdev = mgmt_dev->pdev;
595	struct device *dev = &pdev->dev;
596	struct octep_hw *oct_hw;
597	unsigned long timeout;
598	u64 val;
599	int ret;
600
601	oct_hw = &mgmt_dev->oct_hw;
602
603	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT);
604
605	/* Wait for a maximum of 5 sec */
606	timeout = jiffies + msecs_to_jiffies(5000);
607	while (!time_after(jiffies, timeout)) {
608		if (get_device_ready_status(oct_hw->base[OCTEP_HW_MBOX_BAR])) {
609			atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_INIT);
610			break;
611		}
612
613		if (atomic_read(&mgmt_dev->status) >= OCTEP_VDPA_DEV_STATUS_READY) {
614			dev_info(dev, "Stopping vDPA setup task.\n");
615			return;
616		}
617
618		usleep_range(1000, 1500);
619	}
620
621	if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_INIT) {
622		dev_err(dev, "BAR initialization is timed out\n");
623		return;
624	}
625
626	ret = octep_iomap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
627	if (ret)
628		return;
629
630	val = readq(oct_hw->base[OCTEP_HW_MBOX_BAR] + OCTEP_VF_IN_CTRL(0));
631	oct_hw->nb_irqs = OCTEP_VF_IN_CTRL_RPVF(val);
632	if (!oct_hw->nb_irqs || oct_hw->nb_irqs > OCTEP_MAX_CB_INTR) {
633		dev_err(dev, "Invalid number of interrupts %d\n", oct_hw->nb_irqs);
634		goto unmap_region;
635	}
636
637	ret = octep_hw_caps_read(oct_hw, pdev);
638	if (ret < 0)
639		goto unmap_region;
640
641	mgmt_dev->mdev.ops = &octep_vdpa_mgmt_dev_ops;
642	mgmt_dev->mdev.id_table = id_table;
643	mgmt_dev->mdev.max_supported_vqs = oct_hw->nr_vring;
644	mgmt_dev->mdev.supported_features = oct_hw->features;
645	mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);
646	mgmt_dev->mdev.device = dev;
647
648	ret = vdpa_mgmtdev_register(&mgmt_dev->mdev);
649	if (ret) {
650		dev_err(dev, "Failed to register vdpa management interface\n");
651		goto unmap_region;
652	}
653
654	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY);
655
656	return;
657
658unmap_region:
659	octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
660	oct_hw->base[OCTEP_HW_CAPS_BAR] = NULL;
661}
662
663static int octep_vdpa_probe_vf(struct pci_dev *pdev)
664{
665	struct octep_vdpa_mgmt_dev *mgmt_dev;
666	struct device *dev = &pdev->dev;
667	int ret;
668
669	ret = pcim_enable_device(pdev);
670	if (ret) {
671		dev_err(dev, "Failed to enable device\n");
672		return ret;
673	}
674
675	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
676	if (ret) {
677		dev_err(dev, "No usable DMA configuration\n");
678		return ret;
679	}
680	pci_set_master(pdev);
681
682	mgmt_dev = devm_kzalloc(dev, sizeof(struct octep_vdpa_mgmt_dev), GFP_KERNEL);
683	if (!mgmt_dev)
684		return -ENOMEM;
685
686	ret = octep_iomap_region(pdev, mgmt_dev->oct_hw.base, OCTEP_HW_MBOX_BAR);
687	if (ret)
688		return ret;
689
690	mgmt_dev->pdev = pdev;
691	pci_set_drvdata(pdev, mgmt_dev);
692
693	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ALLOC);
694	INIT_WORK(&mgmt_dev->setup_task, octep_vdpa_setup_task);
695	schedule_work(&mgmt_dev->setup_task);
696	dev_info(&pdev->dev, "octep vdpa mgmt device setup task is queued\n");
697
698	return 0;
699}
700
701static void octep_vdpa_assign_barspace(struct pci_dev *vf_dev, struct pci_dev *pf_dev, u8 idx)
702{
703	struct resource *vf_res = vf_dev->resource + PCI_STD_RESOURCES + 4;
704	struct resource *pf_res = pf_dev->resource + PCI_STD_RESOURCES + 4;
705	struct octep_pf *pf = pci_get_drvdata(pf_dev);
706	struct pci_bus_region bus_region;
707
708	vf_res->name = pci_name(vf_dev);
709	vf_res->flags = pf_res->flags;
710	vf_res->parent = (pf_dev->resource + PCI_STD_RESOURCES)->parent;
711
712	bus_region.start = pf->vf_base + idx * pf->vf_stride;
713	bus_region.end = bus_region.start + pf->vf_stride - 1;
714	pcibios_bus_to_resource(vf_dev->bus, vf_res, &bus_region);
715}
716
717static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs)
718{
719	struct octep_pf *pf = pci_get_drvdata(pdev);
720	u8 __iomem *addr = pf->base[OCTEP_HW_MBOX_BAR];
721	struct pci_dev *vf_pdev = NULL;
722	bool done = false;
723	int index = 0;
724	int ret, i;
725
726	ret = pci_enable_sriov(pdev, num_vfs);
727	if (ret)
728		return ret;
729
730	pf->enabled_vfs = num_vfs;
731
732	while ((vf_pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, vf_pdev))) {
733		if (vf_pdev->device != pf->vf_devid)
734			continue;
735
736		octep_vdpa_assign_barspace(vf_pdev, pdev, index);
737		if (++index == num_vfs) {
738			done = true;
739			pci_dev_put(vf_pdev);
740			break;
741		}
742	}
743
744	if (done) {
745		for (i = 0; i < pf->enabled_vfs; i++)
746			writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i));
747	}
748
749	return num_vfs;
750}
751
752static int octep_sriov_disable(struct pci_dev *pdev)
753{
754	struct octep_pf *pf = pci_get_drvdata(pdev);
755
756	if (!pci_num_vf(pdev))
757		return 0;
758
759	pci_disable_sriov(pdev);
760	pf->enabled_vfs = 0;
761
762	return 0;
763}
764
765static int octep_vdpa_sriov_configure(struct pci_dev *pdev, int num_vfs)
766{
767	if (num_vfs > 0)
768		return octep_sriov_enable(pdev, num_vfs);
769	else
770		return octep_sriov_disable(pdev);
771}
772
773static u16 octep_get_vf_devid(struct pci_dev *pdev)
774{
775	u16 did;
776
777	switch (pdev->device) {
778	case OCTEP_VDPA_DEVID_CN106K_PF:
779		did = OCTEP_VDPA_DEVID_CN106K_VF;
780		break;
781	case OCTEP_VDPA_DEVID_CN105K_PF:
782		did = OCTEP_VDPA_DEVID_CN105K_VF;
783		break;
784	case OCTEP_VDPA_DEVID_CN103K_PF:
785		did = OCTEP_VDPA_DEVID_CN103K_VF;
786		break;
787	default:
788		did = 0xFFFF;
789		break;
790	}
791
792	return did;
793}
794
795static int octep_vdpa_pf_setup(struct octep_pf *octpf)
796{
797	u8 __iomem *addr = octpf->base[OCTEP_HW_MBOX_BAR];
798	struct pci_dev *pdev = octpf->pdev;
799	int totalvfs;
800	size_t len;
801	u64 val;
802
803	totalvfs = pci_sriov_get_totalvfs(pdev);
804	if (unlikely(!totalvfs)) {
805		dev_info(&pdev->dev, "Total VFs are %d in PF sriov configuration\n", totalvfs);
806		return 0;
807	}
808
809	addr = octpf->base[OCTEP_HW_MBOX_BAR];
810	val = readq(addr + OCTEP_EPF_RINFO(0));
811	if (val == 0) {
812		dev_err(&pdev->dev, "Invalid device configuration\n");
813		return -EINVAL;
814	}
815
816	len = pci_resource_len(pdev, OCTEP_HW_CAPS_BAR);
817
818	octpf->vf_stride = len / totalvfs;
819	octpf->vf_devid = octep_get_vf_devid(pdev);
820
821	octep_vdpa_pf_bar_shrink(octpf);
822
823	return 0;
824}
825
826static int octep_vdpa_probe_pf(struct pci_dev *pdev)
827{
828	struct device *dev = &pdev->dev;
829	struct octep_pf *octpf;
830	int ret;
831
832	ret = pci_enable_device(pdev);
833	if (ret) {
834		dev_err(dev, "Failed to enable device\n");
835		return ret;
836	}
837
838	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
839	if (ret) {
840		dev_err(dev, "No usable DMA configuration\n");
841		goto disable_pci;
842	}
843	octpf = devm_kzalloc(dev, sizeof(*octpf), GFP_KERNEL);
844	if (!octpf) {
845		ret = -ENOMEM;
846		goto disable_pci;
847	}
848
849	ret = octep_iomap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
850	if (ret)
851		goto disable_pci;
852
853	pci_set_master(pdev);
854	pci_set_drvdata(pdev, octpf);
855	octpf->pdev = pdev;
856
857	ret = octep_vdpa_pf_setup(octpf);
858	if (ret)
859		goto unmap_region;
860
861	return 0;
862
863unmap_region:
864	octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
865disable_pci:
866	pci_disable_device(pdev);
867	return ret;
868}
869
870static int octep_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
871{
872	if (pdev->is_virtfn)
873		return octep_vdpa_probe_vf(pdev);
874	else
875		return octep_vdpa_probe_pf(pdev);
876}
877
878static struct pci_device_id octep_pci_vdpa_map[] = {
879	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_PF) },
880	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_VF) },
881	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_PF) },
882	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_VF) },
883	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_PF) },
884	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_VF) },
885	{ 0 },
886};
887
888static struct pci_driver octep_pci_vdpa = {
889	.name     = OCTEP_VDPA_DRIVER_NAME,
890	.id_table = octep_pci_vdpa_map,
891	.probe    = octep_vdpa_probe,
892	.remove   = octep_vdpa_remove,
893	.sriov_configure = octep_vdpa_sriov_configure
894};
895
896module_pci_driver(octep_pci_vdpa);
897
898MODULE_AUTHOR("Marvell");
899MODULE_DESCRIPTION("Marvell Octeon PCIe endpoint vDPA driver");
900MODULE_LICENSE("GPL");