vdpa_dev.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2023 Advanced Micro Devices, Inc */
  3
  4#include <linux/pci.h>
  5#include <linux/vdpa.h>
  6#include <uapi/linux/vdpa.h>
  7#include <linux/virtio_pci_modern.h>
  8
  9#include <linux/pds/pds_common.h>
 10#include <linux/pds/pds_core_if.h>
 11#include <linux/pds/pds_adminq.h>
 12#include <linux/pds/pds_auxbus.h>
 13
 14#include "vdpa_dev.h"
 15#include "aux_drv.h"
 16#include "cmds.h"
 17#include "debugfs.h"
 18
 19static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev);
 20
 21static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev)
 22{
 23	return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
 24}
 25
 26static int pds_vdpa_notify_handler(struct notifier_block *nb,
 27				   unsigned long ecode,
 28				   void *data)
 29{
 30	struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb);
 31	struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
 32
 33	dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
 34
 35	if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
 36		if (pdsv->config_cb.callback)
 37			pdsv->config_cb.callback(pdsv->config_cb.private);
 38	}
 39
 40	return 0;
 41}
 42
 43static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
 44{
 45	struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
 46	struct notifier_block *nb = &pdsv->nb;
 47	int err;
 48
 49	if (!nb->notifier_call) {
 50		nb->notifier_call = pds_vdpa_notify_handler;
 51		err = pdsc_register_notify(nb);
 52		if (err) {
 53			nb->notifier_call = NULL;
 54			dev_err(dev, "failed to register pds event handler: %pe\n",
 55				ERR_PTR(err));
 56			return -EINVAL;
 57		}
 58		dev_dbg(dev, "pds event handler registered\n");
 59	}
 60
 61	return 0;
 62}
 63
 64static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
 65{
 66	if (pdsv->nb.notifier_call) {
 67		pdsc_unregister_notify(&pdsv->nb);
 68		pdsv->nb.notifier_call = NULL;
 69	}
 70}
 71
 72static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
 73				   u64 desc_addr, u64 driver_addr, u64 device_addr)
 74{
 75	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 76
 77	pdsv->vqs[qid].desc_addr = desc_addr;
 78	pdsv->vqs[qid].avail_addr = driver_addr;
 79	pdsv->vqs[qid].used_addr = device_addr;
 80
 81	return 0;
 82}
 83
 84static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
 85{
 86	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 87
 88	pdsv->vqs[qid].q_len = num;
 89}
 90
 91static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
 92{
 93	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
 94
 95	iowrite16(qid, pdsv->vqs[qid].notify);
 96}
 97
 98static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
 99			       struct vdpa_callback *cb)
100{
101	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
102
103	pdsv->vqs[qid].event_cb = *cb;
104}
105
106static irqreturn_t pds_vdpa_isr(int irq, void *data)
107{
108	struct pds_vdpa_vq_info *vq;
109
110	vq = data;
111	if (vq->event_cb.callback)
112		vq->event_cb.callback(vq->event_cb.private);
113
114	return IRQ_HANDLED;
115}
116
117static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid)
118{
119	if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR)
120		return;
121
122	free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]);
123	pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
124}
125
126static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
127{
128	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
129	struct device *dev = &pdsv->vdpa_dev.dev;
130	u64 driver_features;
131	u16 invert_idx = 0;
132	int err;
133
134	dev_dbg(dev, "%s: qid %d ready %d => %d\n",
135		__func__, qid, pdsv->vqs[qid].ready, ready);
136	if (ready == pdsv->vqs[qid].ready)
137		return;
138
139	driver_features = pds_vdpa_get_driver_features(vdpa_dev);
140	if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
141		invert_idx = PDS_VDPA_PACKED_INVERT_IDX;
142
143	if (ready) {
144		/* Pass vq setup info to DSC using adminq to gather up and
145		 * send all info at once so FW can do its full set up in
146		 * one easy operation
147		 */
148		err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
149		if (err) {
150			dev_err(dev, "Failed to init vq %d: %pe\n",
151				qid, ERR_PTR(err));
152			ready = false;
153		}
154	} else {
155		err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
156		if (err)
157			dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
158				__func__, qid, ERR_PTR(err));
159	}
160
161	pdsv->vqs[qid].ready = ready;
162}
163
164static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
165{
166	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
167
168	return pdsv->vqs[qid].ready;
169}
170
171static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
172				 const struct vdpa_vq_state *state)
173{
174	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
175	struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
176	struct device *dev = &padev->aux_dev.dev;
177	u64 driver_features;
178	u16 avail;
179	u16 used;
180
181	if (pdsv->vqs[qid].ready) {
182		dev_err(dev, "Setting device position is denied while vq is enabled\n");
183		return -EINVAL;
184	}
185
186	driver_features = pds_vdpa_get_driver_features(vdpa_dev);
187	if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
188		avail = state->packed.last_avail_idx |
189			(state->packed.last_avail_counter << 15);
190		used = state->packed.last_used_idx |
191		       (state->packed.last_used_counter << 15);
192
193		/* The avail and used index are stored with the packed wrap
194		 * counter bit inverted.  This way, in case set_vq_state is
195		 * not called, the initial value can be set to zero prior to
196		 * feature negotiation, and it is good for both packed and
197		 * split vq.
198		 */
199		avail ^= PDS_VDPA_PACKED_INVERT_IDX;
200		used ^= PDS_VDPA_PACKED_INVERT_IDX;
201	} else {
202		avail = state->split.avail_index;
203		/* state->split does not provide a used_index:
204		 * the vq will be set to "empty" here, and the vq will read
205		 * the current used index the next time the vq is kicked.
206		 */
207		used = avail;
208	}
209
210	if (used != avail) {
211		dev_dbg(dev, "Setting used equal to avail, for interoperability\n");
212		used = avail;
213	}
214
215	pdsv->vqs[qid].avail_idx = avail;
216	pdsv->vqs[qid].used_idx = used;
217
218	return 0;
219}
220
221static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
222				 struct vdpa_vq_state *state)
223{
224	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
225	struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
226	struct device *dev = &padev->aux_dev.dev;
227	u64 driver_features;
228	u16 avail;
229	u16 used;
230
231	if (pdsv->vqs[qid].ready) {
232		dev_err(dev, "Getting device position is denied while vq is enabled\n");
233		return -EINVAL;
234	}
235
236	avail = pdsv->vqs[qid].avail_idx;
237	used = pdsv->vqs[qid].used_idx;
238
239	driver_features = pds_vdpa_get_driver_features(vdpa_dev);
240	if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
241		avail ^= PDS_VDPA_PACKED_INVERT_IDX;
242		used ^= PDS_VDPA_PACKED_INVERT_IDX;
243
244		state->packed.last_avail_idx = avail & 0x7fff;
245		state->packed.last_avail_counter = avail >> 15;
246		state->packed.last_used_idx = used & 0x7fff;
247		state->packed.last_used_counter = used >> 15;
248	} else {
249		state->split.avail_index = avail;
250		/* state->split does not provide a used_index. */
251	}
252
253	return 0;
254}
255
256static struct vdpa_notification_area
257pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid)
258{
259	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
260	struct virtio_pci_modern_device *vd_mdev;
261	struct vdpa_notification_area area;
262
263	area.addr = pdsv->vqs[qid].notify_pa;
264
265	vd_mdev = &pdsv->vdpa_aux->vd_mdev;
266	if (!vd_mdev->notify_offset_multiplier)
267		area.size = PDS_PAGE_SIZE;
268	else
269		area.size = vd_mdev->notify_offset_multiplier;
270
271	return area;
272}
273
274static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid)
275{
276	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
277
278	return pdsv->vqs[qid].irq;
279}
280
281static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
282{
283	return PDS_PAGE_SIZE;
284}
285
286static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx)
287{
288	return 0;
289}
290
291static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
292{
293	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
294
295	return pdsv->supported_features;
296}
297
298static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
299{
300	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
301	struct device *dev = &pdsv->vdpa_dev.dev;
302	u64 driver_features;
303	u64 nego_features;
304	u64 hw_features;
305	u64 missing;
306
307	if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {
308		dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n");
309		return -EOPNOTSUPP;
310	}
311
312	/* Check for valid feature bits */
313	nego_features = features & pdsv->supported_features;
314	missing = features & ~nego_features;
315	if (missing) {
316		dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n",
317			features, missing);
318		return -EOPNOTSUPP;
319	}
320
321	driver_features = pds_vdpa_get_driver_features(vdpa_dev);
322	pdsv->negotiated_features = nego_features;
323	dev_dbg(dev, "%s: %#llx => %#llx\n",
324		__func__, driver_features, nego_features);
325
326	/* if we're faking the F_MAC, strip it before writing to device */
327	hw_features = le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
328	if (!(hw_features & BIT_ULL(VIRTIO_NET_F_MAC)))
329		nego_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);
330
331	if (driver_features == nego_features)
332		return 0;
333
334	vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features);
335
336	return 0;
337}
338
339static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
340{
341	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
342
343	return pdsv->negotiated_features;
344}
345
346static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
347				   struct vdpa_callback *cb)
348{
349	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
350
351	pdsv->config_cb.callback = cb->callback;
352	pdsv->config_cb.private = cb->private;
353}
354
355static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
356{
357	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
358
359	/* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */
360	return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen)));
361}
362
363static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
364{
365	return VIRTIO_ID_NET;
366}
367
368static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
369{
370	return PCI_VENDOR_ID_PENSANDO;
371}
372
373static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)
374{
375	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
376
377	return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev);
378}
379
380static int pds_vdpa_request_irqs(struct pds_vdpa_device *pdsv)
381{
382	struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
383	struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
384	struct device *dev = &pdsv->vdpa_dev.dev;
385	int max_vq, nintrs, qid, err;
386
387	max_vq = vdpa_aux->vdpa_mdev.max_supported_vqs;
388
389	nintrs = pci_alloc_irq_vectors(pdev, max_vq, max_vq, PCI_IRQ_MSIX);
390	if (nintrs < 0) {
391		dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
392			max_vq, ERR_PTR(nintrs));
393		return nintrs;
394	}
395
396	for (qid = 0; qid < pdsv->num_vqs; ++qid) {
397		int irq = pci_irq_vector(pdev, qid);
398
399		snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name),
400			 "vdpa-%s-%d", dev_name(dev), qid);
401
402		err = request_irq(irq, pds_vdpa_isr, 0,
403				  pdsv->vqs[qid].irq_name,
404				  &pdsv->vqs[qid]);
405		if (err) {
406			dev_err(dev, "%s: no irq for qid %d: %pe\n",
407				__func__, qid, ERR_PTR(err));
408			goto err_release;
409		}
410
411		pdsv->vqs[qid].irq = irq;
412	}
413
414	vdpa_aux->nintrs = nintrs;
415
416	return 0;
417
418err_release:
419	while (qid--)
420		pds_vdpa_release_irq(pdsv, qid);
421
422	pci_free_irq_vectors(pdev);
423
424	vdpa_aux->nintrs = 0;
425
426	return err;
427}
428
429void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
430{
431	struct pds_vdpa_aux *vdpa_aux;
432	struct pci_dev *pdev;
433	int qid;
434
435	if (!pdsv)
436		return;
437
438	pdev = pdsv->vdpa_aux->padev->vf_pdev;
439	vdpa_aux = pdsv->vdpa_aux;
440
441	if (!vdpa_aux->nintrs)
442		return;
443
444	for (qid = 0; qid < pdsv->num_vqs; qid++)
445		pds_vdpa_release_irq(pdsv, qid);
446
447	pci_free_irq_vectors(pdev);
448
449	vdpa_aux->nintrs = 0;
450}
451
452static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
453{
454	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
455	struct device *dev = &pdsv->vdpa_dev.dev;
456	u8 old_status;
457	int i;
458
459	old_status = pds_vdpa_get_status(vdpa_dev);
460	dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status);
461
462	if (status & ~old_status & VIRTIO_CONFIG_S_DRIVER_OK) {
463		if (pds_vdpa_request_irqs(pdsv))
464			status = old_status | VIRTIO_CONFIG_S_FAILED;
465	}
466
467	pds_vdpa_cmd_set_status(pdsv, status);
468
469	if (status == 0) {
470		struct vdpa_callback null_cb = { };
471
472		pds_vdpa_set_config_cb(vdpa_dev, &null_cb);
473		pds_vdpa_cmd_reset(pdsv);
474
475		for (i = 0; i < pdsv->num_vqs; i++) {
476			pdsv->vqs[i].avail_idx = 0;
477			pdsv->vqs[i].used_idx = 0;
478		}
479
480		pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
481	}
482
483	if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
484		for (i = 0; i < pdsv->num_vqs; i++) {
485			pdsv->vqs[i].notify =
486				vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
487							i, &pdsv->vqs[i].notify_pa);
488		}
489	}
490
491	if (old_status & ~status & VIRTIO_CONFIG_S_DRIVER_OK)
492		pds_vdpa_release_irqs(pdsv);
493}
494
495static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid,
496				    void __iomem *notify)
497{
498	memset(&pdsv->vqs[qid], 0, sizeof(pdsv->vqs[0]));
499	pdsv->vqs[qid].qid = qid;
500	pdsv->vqs[qid].pdsv = pdsv;
501	pdsv->vqs[qid].ready = false;
502	pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
503	pdsv->vqs[qid].notify = notify;
504}
505
506static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
507{
508	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
509	struct device *dev;
510	int err = 0;
511	u8 status;
512	int i;
513
514	dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
515	status = pds_vdpa_get_status(vdpa_dev);
516
517	if (status == 0)
518		return 0;
519
520	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
521		/* Reset the vqs */
522		for (i = 0; i < pdsv->num_vqs && !err; i++) {
523			err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]);
524			if (err)
525				dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
526					__func__, i, ERR_PTR(err));
527		}
528	}
529
530	pds_vdpa_set_status(vdpa_dev, 0);
531
532	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
533		/* Reset the vq info */
534		for (i = 0; i < pdsv->num_vqs && !err; i++)
535			pds_vdpa_init_vqs_entry(pdsv, i, pdsv->vqs[i].notify);
536	}
537
538	return 0;
539}
540
541static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
542{
543	return sizeof(struct virtio_net_config);
544}
545
546static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev,
547				unsigned int offset,
548				void *buf, unsigned int len)
549{
550	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
551	void __iomem *device;
552
553	if (offset + len > sizeof(struct virtio_net_config)) {
554		WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
555		return;
556	}
557
558	device = pdsv->vdpa_aux->vd_mdev.device;
559	memcpy_fromio(buf, device + offset, len);
560}
561
562static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev,
563				unsigned int offset, const void *buf,
564				unsigned int len)
565{
566	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
567	void __iomem *device;
568
569	if (offset + len > sizeof(struct virtio_net_config)) {
570		WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
571		return;
572	}
573
574	device = pdsv->vdpa_aux->vd_mdev.device;
575	memcpy_toio(device + offset, buf, len);
576}
577
578static const struct vdpa_config_ops pds_vdpa_ops = {
579	.set_vq_address		= pds_vdpa_set_vq_address,
580	.set_vq_num		= pds_vdpa_set_vq_num,
581	.kick_vq		= pds_vdpa_kick_vq,
582	.set_vq_cb		= pds_vdpa_set_vq_cb,
583	.set_vq_ready		= pds_vdpa_set_vq_ready,
584	.get_vq_ready		= pds_vdpa_get_vq_ready,
585	.set_vq_state		= pds_vdpa_set_vq_state,
586	.get_vq_state		= pds_vdpa_get_vq_state,
587	.get_vq_notification	= pds_vdpa_get_vq_notification,
588	.get_vq_irq		= pds_vdpa_get_vq_irq,
589	.get_vq_align		= pds_vdpa_get_vq_align,
590	.get_vq_group		= pds_vdpa_get_vq_group,
591
592	.get_device_features	= pds_vdpa_get_device_features,
593	.set_driver_features	= pds_vdpa_set_driver_features,
594	.get_driver_features	= pds_vdpa_get_driver_features,
595	.set_config_cb		= pds_vdpa_set_config_cb,
596	.get_vq_num_max		= pds_vdpa_get_vq_num_max,
597	.get_device_id		= pds_vdpa_get_device_id,
598	.get_vendor_id		= pds_vdpa_get_vendor_id,
599	.get_status		= pds_vdpa_get_status,
600	.set_status		= pds_vdpa_set_status,
601	.reset			= pds_vdpa_reset,
602	.get_config_size	= pds_vdpa_get_config_size,
603	.get_config		= pds_vdpa_get_config,
604	.set_config		= pds_vdpa_set_config,
605};
606static struct virtio_device_id pds_vdpa_id_table[] = {
607	{VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
608	{0},
609};
610
611static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
612			    const struct vdpa_dev_set_config *add_config)
613{
614	struct pds_vdpa_aux *vdpa_aux;
615	struct pds_vdpa_device *pdsv;
616	struct vdpa_mgmt_dev *mgmt;
617	u16 fw_max_vqs, vq_pairs;
618	struct device *dma_dev;
619	struct pci_dev *pdev;
620	struct device *dev;
621	u8 status;
622	int err;
623	int i;
624
625	vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
626	dev = &vdpa_aux->padev->aux_dev.dev;
627	mgmt = &vdpa_aux->vdpa_mdev;
628
629	if (vdpa_aux->pdsv) {
630		dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n");
631		return -EOPNOTSUPP;
632	}
633
634	pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev,
635				 dev, &pds_vdpa_ops, NULL,
636				 1, 1, name, false);
637	if (IS_ERR(pdsv)) {
638		dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv);
639		return PTR_ERR(pdsv);
640	}
641
642	vdpa_aux->pdsv = pdsv;
643	pdsv->vdpa_aux = vdpa_aux;
644
645	pdev = vdpa_aux->padev->vf_pdev;
646	dma_dev = &pdev->dev;
647	pdsv->vdpa_dev.vmap.dma_dev = dma_dev;
648
649	status = pds_vdpa_get_status(&pdsv->vdpa_dev);
650	if (status == 0xff) {
651		dev_err(dev, "Broken PCI - status %#x\n", status);
652		err = -ENXIO;
653		goto err_unmap;
654	}
655
656	pdsv->supported_features = mgmt->supported_features;
657
658	if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
659		u64 unsupp_features =
660			add_config->device_features & ~pdsv->supported_features;
661
662		if (unsupp_features) {
663			dev_err(dev, "Unsupported features: %#llx\n", unsupp_features);
664			err = -EOPNOTSUPP;
665			goto err_unmap;
666		}
667
668		pdsv->supported_features = add_config->device_features;
669	}
670
671	err = pds_vdpa_cmd_reset(pdsv);
672	if (err) {
673		dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err));
674		goto err_unmap;
675	}
676
677	err = pds_vdpa_init_hw(pdsv);
678	if (err) {
679		dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err));
680		goto err_unmap;
681	}
682
683	fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs);
684	vq_pairs = fw_max_vqs / 2;
685
686	/* Make sure we have the queues being requested */
687	if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
688		vq_pairs = add_config->net.max_vq_pairs;
689
690	pdsv->num_vqs = 2 * vq_pairs;
691	if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))
692		pdsv->num_vqs++;
693
694	if (pdsv->num_vqs > fw_max_vqs) {
695		dev_err(dev, "%s: queue count requested %u greater than max %u\n",
696			__func__, pdsv->num_vqs, fw_max_vqs);
697		err = -ENOSPC;
698		goto err_unmap;
699	}
700
701	if (pdsv->num_vqs != fw_max_vqs) {
702		err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs);
703		if (err) {
704			dev_err(dev, "Failed to set max_vq_pairs: %pe\n",
705				ERR_PTR(err));
706			goto err_unmap;
707		}
708	}
709
710	/* Set a mac, either from the user config if provided
711	 * or use the device's mac if not 00:..:00
712	 * or set a random mac
713	 */
714	if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
715		ether_addr_copy(pdsv->mac, add_config->net.mac);
716	} else {
717		struct virtio_net_config __iomem *vc;
718
719		vc = pdsv->vdpa_aux->vd_mdev.device;
720		memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
721		if (is_zero_ether_addr(pdsv->mac) &&
722		    (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_MAC))) {
723			eth_random_addr(pdsv->mac);
724			dev_info(dev, "setting random mac %pM\n", pdsv->mac);
725		}
726	}
727	pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
728
729	for (i = 0; i < pdsv->num_vqs; i++) {
730		void __iomem *notify;
731
732		notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
733						 i, &pdsv->vqs[i].notify_pa);
734		pds_vdpa_init_vqs_entry(pdsv, i, notify);
735	}
736
737	pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev;
738
739	err = pds_vdpa_register_event_handler(pdsv);
740	if (err) {
741		dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err));
742		goto err_unmap;
743	}
744
745	/* We use the _vdpa_register_device() call rather than the
746	 * vdpa_register_device() to avoid a deadlock because our
747	 * dev_add() is called with the vdpa_dev_lock already set
748	 * by vdpa_nl_cmd_dev_add_set_doit()
749	 */
750	err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs);
751	if (err) {
752		dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err));
753		goto err_unevent;
754	}
755
756	pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
757
758	return 0;
759
760err_unevent:
761	pds_vdpa_unregister_event_handler(pdsv);
762err_unmap:
763	put_device(&pdsv->vdpa_dev.dev);
764	vdpa_aux->pdsv = NULL;
765	return err;
766}
767
768static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
769			     struct vdpa_device *vdpa_dev)
770{
771	struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
772	struct pds_vdpa_aux *vdpa_aux;
773
774	pds_vdpa_unregister_event_handler(pdsv);
775
776	vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
777	_vdpa_unregister_device(vdpa_dev);
778
779	pds_vdpa_cmd_reset(vdpa_aux->pdsv);
780	pds_vdpa_debugfs_reset_vdpadev(vdpa_aux);
781
782	vdpa_aux->pdsv = NULL;
783
784	dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n");
785}
786
787static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
788	.dev_add = pds_vdpa_dev_add,
789	.dev_del = pds_vdpa_dev_del
790};
791
792int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
793{
794	union pds_core_adminq_cmd cmd = {
795		.vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
796		.vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
797	};
798	union pds_core_adminq_comp comp = {};
799	struct vdpa_mgmt_dev *mgmt;
800	struct pci_dev *pf_pdev;
801	struct device *pf_dev;
802	struct pci_dev *pdev;
803	dma_addr_t ident_pa;
804	struct device *dev;
805	u16 dev_intrs;
806	u16 max_vqs;
807	int err;
808
809	dev = &vdpa_aux->padev->aux_dev.dev;
810	pdev = vdpa_aux->padev->vf_pdev;
811	mgmt = &vdpa_aux->vdpa_mdev;
812
813	/* Get resource info through the PF's adminq.  It is a block of info,
814	 * so we need to map some memory for PF to make available to the
815	 * firmware for writing the data.
816	 */
817	pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev);
818	pf_dev = &pf_pdev->dev;
819	ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident,
820				  sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
821	if (dma_mapping_error(pf_dev, ident_pa)) {
822		dev_err(dev, "Failed to map ident space\n");
823		return -ENOMEM;
824	}
825
826	cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa);
827	cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident));
828	err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd,
829				    sizeof(cmd.vdpa_ident), &comp, 0);
830	dma_unmap_single(pf_dev, ident_pa,
831			 sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
832	if (err) {
833		dev_err(dev, "Failed to ident hw, status %d: %pe\n",
834			comp.status, ERR_PTR(err));
835		return err;
836	}
837
838	max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs);
839	dev_intrs = pci_msix_vec_count(pdev);
840	dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs);
841
842	max_vqs = min_t(u16, dev_intrs, max_vqs);
843	mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs);
844	vdpa_aux->nintrs = 0;
845
846	mgmt->ops = &pds_vdpa_mgmt_dev_ops;
847	mgmt->id_table = pds_vdpa_id_table;
848	mgmt->device = dev;
849	mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
850
851	/* advertise F_MAC even if the device doesn't */
852	mgmt->supported_features |= BIT_ULL(VIRTIO_NET_F_MAC);
853
854	mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
855	mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
856	mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
857
858	return 0;
859}