virtio_vdpa.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * VIRTIO based driver for vDPA device
  4 *
  5 * Copyright (c) 2020, Red Hat. All rights reserved.
  6 *     Author: Jason Wang <jasowang@redhat.com>
  7 *
  8 */
  9
 10#include <linux/init.h>
 11#include <linux/module.h>
 12#include <linux/device.h>
 13#include <linux/kernel.h>
 14#include <linux/slab.h>
 15#include <linux/uuid.h>
 16#include <linux/group_cpus.h>
 17#include <linux/virtio.h>
 18#include <linux/vdpa.h>
 19#include <linux/virtio_config.h>
 20#include <linux/virtio_ring.h>
 21
 22#define MOD_VERSION  "0.1"
 23#define MOD_AUTHOR   "Jason Wang <jasowang@redhat.com>"
 24#define MOD_DESC     "vDPA bus driver for virtio devices"
 25#define MOD_LICENSE  "GPL v2"
 26
 27struct virtio_vdpa_device {
 28	struct virtio_device vdev;
 29	struct vdpa_device *vdpa;
 30	u64 features;
 31};
 32
 33static inline struct virtio_vdpa_device *
 34to_virtio_vdpa_device(struct virtio_device *dev)
 35{
 36	return container_of(dev, struct virtio_vdpa_device, vdev);
 37}
 38
 39static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
 40{
 41	return to_virtio_vdpa_device(vdev)->vdpa;
 42}
 43
 44static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
 45			    void *buf, unsigned int len)
 46{
 47	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 48
 49	vdpa_get_config(vdpa, offset, buf, len);
 50}
 51
 52static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
 53			    const void *buf, unsigned int len)
 54{
 55	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 56
 57	vdpa_set_config(vdpa, offset, buf, len);
 58}
 59
 60static u32 virtio_vdpa_generation(struct virtio_device *vdev)
 61{
 62	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 63	const struct vdpa_config_ops *ops = vdpa->config;
 64
 65	if (ops->get_generation)
 66		return ops->get_generation(vdpa);
 67
 68	return 0;
 69}
 70
 71static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
 72{
 73	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 74	const struct vdpa_config_ops *ops = vdpa->config;
 75
 76	return ops->get_status(vdpa);
 77}
 78
 79static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
 80{
 81	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 82
 83	vdpa_set_status(vdpa, status);
 84}
 85
 86static void virtio_vdpa_reset(struct virtio_device *vdev)
 87{
 88	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
 89
 90	vdpa_reset(vdpa, 0);
 91}
 92
 93static bool virtio_vdpa_notify(struct virtqueue *vq)
 94{
 95	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
 96	const struct vdpa_config_ops *ops = vdpa->config;
 97
 98	ops->kick_vq(vdpa, vq->index);
 99
100	return true;
101}
102
103static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
104{
105	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
106	const struct vdpa_config_ops *ops = vdpa->config;
107	u32 data = vring_notification_data(vq);
108
109	ops->kick_vq_with_data(vdpa, data);
110
111	return true;
112}
113
114static irqreturn_t virtio_vdpa_config_cb(void *private)
115{
116	struct virtio_vdpa_device *vd_dev = private;
117
118	virtio_config_changed(&vd_dev->vdev);
119
120	return IRQ_HANDLED;
121}
122
123static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
124{
125	struct virtqueue *vq = private;
126
127	return vring_interrupt(0, vq);
128}
129
130static struct virtqueue *
131virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
132		     void (*callback)(struct virtqueue *vq),
133		     const char *name, bool ctx)
134{
135	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
136	const struct vdpa_config_ops *ops = vdpa->config;
137	bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
138	struct vdpa_callback cb;
139	struct virtqueue *vq;
140	u64 desc_addr, driver_addr, device_addr;
141	union virtio_map map = {0};
142	/* Assume split virtqueue, switch to packed if necessary */
143	struct vdpa_vq_state state = {0};
144	u32 align, max_num, min_num = 1;
145	bool may_reduce_num = true;
146	int err;
147
148	if (!name)
149		return NULL;
150
151	if (index >= vdpa->nvqs)
152		return ERR_PTR(-ENOENT);
153
154	/* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
155	if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
156		if (ops->kick_vq_with_data)
157			notify = virtio_vdpa_notify_with_data;
158		else
159			__virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
160	}
161
162	/* Queue shouldn't already be set up. */
163	if (ops->get_vq_ready(vdpa, index))
164		return ERR_PTR(-ENOENT);
165
166	if (ops->get_vq_size)
167		max_num = ops->get_vq_size(vdpa, index);
168	else
169		max_num = ops->get_vq_num_max(vdpa);
170
171	if (max_num == 0) {
172		err = -ENOENT;
173		goto error_new_virtqueue;
174	}
175
176	if (ops->get_vq_num_min)
177		min_num = ops->get_vq_num_min(vdpa);
178
179	may_reduce_num = (max_num != min_num);
180
181	/* Create the vring */
182	align = ops->get_vq_align(vdpa);
183
184	if (ops->get_vq_map)
185		map = ops->get_vq_map(vdpa, index);
186	else
187		map = vdpa_get_map(vdpa);
188
189	vq = vring_create_virtqueue_map(index, max_num, align, vdev,
190					true, may_reduce_num, ctx,
191					notify, callback, name, map);
192	if (!vq) {
193		err = -ENOMEM;
194		goto error_new_virtqueue;
195	}
196
197	if (index == 0)
198		vdev->vmap = map;
199
200	vq->num_max = max_num;
201
202	/* Setup virtqueue callback */
203	cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
204	cb.private = vq;
205	cb.trigger = NULL;
206	ops->set_vq_cb(vdpa, index, &cb);
207	ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
208
209	desc_addr = virtqueue_get_desc_addr(vq);
210	driver_addr = virtqueue_get_avail_addr(vq);
211	device_addr = virtqueue_get_used_addr(vq);
212
213	if (ops->set_vq_address(vdpa, index,
214				desc_addr, driver_addr,
215				device_addr)) {
216		err = -EINVAL;
217		goto err_vq;
218	}
219
220	/* reset virtqueue state index */
221	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
222		struct vdpa_vq_state_packed *s = &state.packed;
223
224		s->last_avail_counter = 1;
225		s->last_avail_idx = 0;
226		s->last_used_counter = 1;
227		s->last_used_idx = 0;
228	}
229	err = ops->set_vq_state(vdpa, index, &state);
230	if (err)
231		goto err_vq;
232
233	ops->set_vq_ready(vdpa, index, 1);
234
235	return vq;
236
237err_vq:
238	vring_del_virtqueue(vq);
239error_new_virtqueue:
240	ops->set_vq_ready(vdpa, index, 0);
241	/* VDPA driver should make sure vq is stopeed here */
242	WARN_ON(ops->get_vq_ready(vdpa, index));
243	return ERR_PTR(err);
244}
245
246static void virtio_vdpa_del_vq(struct virtqueue *vq)
247{
248	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
249	struct vdpa_device *vdpa = vd_dev->vdpa;
250	const struct vdpa_config_ops *ops = vdpa->config;
251	unsigned int index = vq->index;
252
253	/* Select and deactivate the queue (best effort) */
254	ops->set_vq_ready(vdpa, index, 0);
255
256	vring_del_virtqueue(vq);
257}
258
259static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
260{
261	struct virtqueue *vq, *n;
262
263	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
264		virtio_vdpa_del_vq(vq);
265}
266
267static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
268{
269	affd->nr_sets = 1;
270	affd->set_size[0] = affvecs;
271}
272
273static struct cpumask *
274create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
275{
276	unsigned int affvecs = 0, curvec, usedvecs, i;
277	struct cpumask *masks = NULL;
278
279	if (nvecs > affd->pre_vectors + affd->post_vectors)
280		affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
281
282	if (!affd->calc_sets)
283		affd->calc_sets = default_calc_sets;
284
285	affd->calc_sets(affd, affvecs);
286
287	if (!affvecs)
288		return NULL;
289
290	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
291	if (!masks)
292		return NULL;
293
294	/* Fill out vectors at the beginning that don't need affinity */
295	for (curvec = 0; curvec < affd->pre_vectors; curvec++)
296		cpumask_setall(&masks[curvec]);
297
298	for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
299		unsigned int this_vecs = affd->set_size[i];
300		unsigned int nr_masks;
301		int j;
302		struct cpumask *result = group_cpus_evenly(this_vecs, &nr_masks);
303
304		if (!result) {
305			kfree(masks);
306			return NULL;
307		}
308
309		for (j = 0; j < nr_masks; j++)
310			cpumask_copy(&masks[curvec + j], &result[j]);
311		kfree(result);
312
313		curvec += nr_masks;
314		usedvecs += nr_masks;
315	}
316
317	/* Fill out vectors at the end that don't need affinity */
318	if (usedvecs >= affvecs)
319		curvec = affd->pre_vectors + affvecs;
320	else
321		curvec = affd->pre_vectors + usedvecs;
322	for (; curvec < nvecs; curvec++)
323		cpumask_setall(&masks[curvec]);
324
325	return masks;
326}
327
328static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
329				struct virtqueue *vqs[],
330				struct virtqueue_info vqs_info[],
331				struct irq_affinity *desc)
332{
333	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
334	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
335	const struct vdpa_config_ops *ops = vdpa->config;
336	struct cpumask *masks;
337	struct vdpa_callback cb;
338	bool has_affinity = desc && ops->set_vq_affinity;
339	int i, err, queue_idx = 0;
340
341	if (has_affinity) {
342		masks = create_affinity_masks(nvqs, desc);
343		if (!masks)
344			return -ENOMEM;
345	}
346
347	for (i = 0; i < nvqs; ++i) {
348		struct virtqueue_info *vqi = &vqs_info[i];
349
350		if (!vqi->name) {
351			vqs[i] = NULL;
352			continue;
353		}
354
355		vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback,
356					      vqi->name, vqi->ctx);
357		if (IS_ERR(vqs[i])) {
358			err = PTR_ERR(vqs[i]);
359			goto err_setup_vq;
360		}
361
362		if (has_affinity)
363			ops->set_vq_affinity(vdpa, i, &masks[i]);
364	}
365
366	cb.callback = virtio_vdpa_config_cb;
367	cb.private = vd_dev;
368	ops->set_config_cb(vdpa, &cb);
369	if (has_affinity)
370		kfree(masks);
371
372	return 0;
373
374err_setup_vq:
375	virtio_vdpa_del_vqs(vdev);
376	if (has_affinity)
377		kfree(masks);
378	return err;
379}
380
381static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
382{
383	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
384	const struct vdpa_config_ops *ops = vdpa->config;
385
386	return ops->get_device_features(vdpa);
387}
388
389static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
390{
391	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
392
393	/* Give virtio_ring a chance to accept features. */
394	vring_transport_features(vdev);
395
396	return vdpa_set_features(vdpa, vdev->features);
397}
398
399static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
400{
401	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
402	struct vdpa_device *vdpa = vd_dev->vdpa;
403
404	return dev_name(&vdpa->dev);
405}
406
407static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
408				       const struct cpumask *cpu_mask)
409{
410	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
411	struct vdpa_device *vdpa = vd_dev->vdpa;
412	const struct vdpa_config_ops *ops = vdpa->config;
413	unsigned int index = vq->index;
414
415	if (ops->set_vq_affinity)
416		return ops->set_vq_affinity(vdpa, index, cpu_mask);
417
418	return 0;
419}
420
421static const struct cpumask *
422virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
423{
424	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
425	const struct vdpa_config_ops *ops = vdpa->config;
426
427	if (ops->get_vq_affinity)
428		return ops->get_vq_affinity(vdpa, index);
429
430	return NULL;
431}
432
433static const struct virtio_config_ops virtio_vdpa_config_ops = {
434	.get		= virtio_vdpa_get,
435	.set		= virtio_vdpa_set,
436	.generation	= virtio_vdpa_generation,
437	.get_status	= virtio_vdpa_get_status,
438	.set_status	= virtio_vdpa_set_status,
439	.reset		= virtio_vdpa_reset,
440	.find_vqs	= virtio_vdpa_find_vqs,
441	.del_vqs	= virtio_vdpa_del_vqs,
442	.get_features	= virtio_vdpa_get_features,
443	.finalize_features = virtio_vdpa_finalize_features,
444	.bus_name	= virtio_vdpa_bus_name,
445	.set_vq_affinity = virtio_vdpa_set_vq_affinity,
446	.get_vq_affinity = virtio_vdpa_get_vq_affinity,
447};
448
449static void virtio_vdpa_release_dev(struct device *_d)
450{
451	struct virtio_device *vdev =
452	       container_of(_d, struct virtio_device, dev);
453	struct virtio_vdpa_device *vd_dev =
454	       container_of(vdev, struct virtio_vdpa_device, vdev);
455
456	kfree(vd_dev);
457}
458
459static int virtio_vdpa_probe(struct vdpa_device *vdpa)
460{
461	const struct vdpa_config_ops *ops = vdpa->config;
462	struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
463	int ret = -EINVAL;
464
465	vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
466	if (!vd_dev)
467		return -ENOMEM;
468
469	vd_dev->vdev.dev.parent = vdpa->map ? &vdpa->dev :
470				  vdpa_get_map(vdpa).dma_dev;
471	vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
472	vd_dev->vdev.config = &virtio_vdpa_config_ops;
473	vd_dev->vdev.map = vdpa->map;
474	vd_dev->vdpa = vdpa;
475
476	vd_dev->vdev.id.device = ops->get_device_id(vdpa);
477	if (vd_dev->vdev.id.device == 0)
478		goto err;
479
480	vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
481	ret = register_virtio_device(&vd_dev->vdev);
482	reg_dev = vd_dev;
483	if (ret)
484		goto err;
485
486	vdpa_set_drvdata(vdpa, vd_dev);
487
488	return 0;
489
490err:
491	if (reg_dev)
492		put_device(&vd_dev->vdev.dev);
493	else
494		kfree(vd_dev);
495	return ret;
496}
497
498static void virtio_vdpa_remove(struct vdpa_device *vdpa)
499{
500	struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
501
502	unregister_virtio_device(&vd_dev->vdev);
503}
504
505static struct vdpa_driver virtio_vdpa_driver = {
506	.driver = {
507		.name	= "virtio_vdpa",
508	},
509	.probe	= virtio_vdpa_probe,
510	.remove = virtio_vdpa_remove,
511};
512
513module_vdpa_driver(virtio_vdpa_driver);
514
515MODULE_VERSION(MOD_VERSION);
516MODULE_LICENSE(MOD_LICENSE);
517MODULE_AUTHOR(MOD_AUTHOR);
518MODULE_DESCRIPTION(MOD_DESC);