virtio.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2#include <linux/virtio.h>
  3#include <linux/spinlock.h>
  4#include <linux/virtio_config.h>
  5#include <linux/virtio_anchor.h>
  6#include <linux/module.h>
  7#include <linux/idr.h>
  8#include <linux/of.h>
  9#include <uapi/linux/virtio_ids.h>
 10
 11/* Unique numbering for virtio devices. */
 12static DEFINE_IDA(virtio_index_ida);
 13
 14static ssize_t device_show(struct device *_d,
 15			   struct device_attribute *attr, char *buf)
 16{
 17	struct virtio_device *dev = dev_to_virtio(_d);
 18	return sysfs_emit(buf, "0x%04x\n", dev->id.device);
 19}
 20static DEVICE_ATTR_RO(device);
 21
 22static ssize_t vendor_show(struct device *_d,
 23			   struct device_attribute *attr, char *buf)
 24{
 25	struct virtio_device *dev = dev_to_virtio(_d);
 26	return sysfs_emit(buf, "0x%04x\n", dev->id.vendor);
 27}
 28static DEVICE_ATTR_RO(vendor);
 29
 30static ssize_t status_show(struct device *_d,
 31			   struct device_attribute *attr, char *buf)
 32{
 33	struct virtio_device *dev = dev_to_virtio(_d);
 34	return sysfs_emit(buf, "0x%08x\n", dev->config->get_status(dev));
 35}
 36static DEVICE_ATTR_RO(status);
 37
 38static ssize_t modalias_show(struct device *_d,
 39			     struct device_attribute *attr, char *buf)
 40{
 41	struct virtio_device *dev = dev_to_virtio(_d);
 42	return sysfs_emit(buf, "virtio:d%08Xv%08X\n",
 43		       dev->id.device, dev->id.vendor);
 44}
 45static DEVICE_ATTR_RO(modalias);
 46
 47static ssize_t features_show(struct device *_d,
 48			     struct device_attribute *attr, char *buf)
 49{
 50	struct virtio_device *dev = dev_to_virtio(_d);
 51	unsigned int i;
 52	ssize_t len = 0;
 53
 54	/* We actually represent this as a bitstring, as it could be
 55	 * arbitrary length in future. */
 56	for (i = 0; i < VIRTIO_FEATURES_BITS; i++)
 57		len += sysfs_emit_at(buf, len, "%c",
 58			       __virtio_test_bit(dev, i) ? '1' : '0');
 59	len += sysfs_emit_at(buf, len, "\n");
 60	return len;
 61}
 62static DEVICE_ATTR_RO(features);
 63
 64static struct attribute *virtio_dev_attrs[] = {
 65	&dev_attr_device.attr,
 66	&dev_attr_vendor.attr,
 67	&dev_attr_status.attr,
 68	&dev_attr_modalias.attr,
 69	&dev_attr_features.attr,
 70	NULL,
 71};
 72ATTRIBUTE_GROUPS(virtio_dev);
 73
 74static inline int virtio_id_match(const struct virtio_device *dev,
 75				  const struct virtio_device_id *id)
 76{
 77	if (id->device != dev->id.device && id->device != VIRTIO_DEV_ANY_ID)
 78		return 0;
 79
 80	return id->vendor == VIRTIO_DEV_ANY_ID || id->vendor == dev->id.vendor;
 81}
 82
 83/* This looks through all the IDs a driver claims to support.  If any of them
 84 * match, we return 1 and the kernel will call virtio_dev_probe(). */
 85static int virtio_dev_match(struct device *_dv, const struct device_driver *_dr)
 86{
 87	unsigned int i;
 88	struct virtio_device *dev = dev_to_virtio(_dv);
 89	const struct virtio_device_id *ids;
 90
 91	ids = drv_to_virtio(_dr)->id_table;
 92	for (i = 0; ids[i].device; i++)
 93		if (virtio_id_match(dev, &ids[i]))
 94			return 1;
 95	return 0;
 96}
 97
 98static int virtio_uevent(const struct device *_dv, struct kobj_uevent_env *env)
 99{
100	const struct virtio_device *dev = dev_to_virtio(_dv);
101
102	return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",
103			      dev->id.device, dev->id.vendor);
104}
105
106void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
107					 unsigned int fbit)
108{
109	unsigned int i;
110	struct virtio_driver *drv = drv_to_virtio(vdev->dev.driver);
111
112	for (i = 0; i < drv->feature_table_size; i++)
113		if (drv->feature_table[i] == fbit)
114			return;
115
116	if (drv->feature_table_legacy) {
117		for (i = 0; i < drv->feature_table_size_legacy; i++)
118			if (drv->feature_table_legacy[i] == fbit)
119				return;
120	}
121
122	BUG();
123}
124EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);
125
126static void __virtio_config_changed(struct virtio_device *dev)
127{
128	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
129
130	if (!dev->config_core_enabled || dev->config_driver_disabled)
131		dev->config_change_pending = true;
132	else if (drv && drv->config_changed) {
133		drv->config_changed(dev);
134		dev->config_change_pending = false;
135	}
136}
137
138void virtio_config_changed(struct virtio_device *dev)
139{
140	unsigned long flags;
141
142	spin_lock_irqsave(&dev->config_lock, flags);
143	__virtio_config_changed(dev);
144	spin_unlock_irqrestore(&dev->config_lock, flags);
145}
146EXPORT_SYMBOL_GPL(virtio_config_changed);
147
148/**
149 * virtio_config_driver_disable - disable config change reporting by drivers
150 * @dev: the device to disable
151 *
152 * This is only allowed to be called by a driver and disabling can't
153 * be nested.
154 */
155void virtio_config_driver_disable(struct virtio_device *dev)
156{
157	spin_lock_irq(&dev->config_lock);
158	dev->config_driver_disabled = true;
159	spin_unlock_irq(&dev->config_lock);
160}
161EXPORT_SYMBOL_GPL(virtio_config_driver_disable);
162
163/**
164 * virtio_config_driver_enable - enable config change reporting by drivers
165 * @dev: the device to enable
166 *
167 * This is only allowed to be called by a driver and enabling can't
168 * be nested.
169 */
170void virtio_config_driver_enable(struct virtio_device *dev)
171{
172	spin_lock_irq(&dev->config_lock);
173	dev->config_driver_disabled = false;
174	if (dev->config_change_pending)
175		__virtio_config_changed(dev);
176	spin_unlock_irq(&dev->config_lock);
177}
178EXPORT_SYMBOL_GPL(virtio_config_driver_enable);
179
180static void virtio_config_core_disable(struct virtio_device *dev)
181{
182	spin_lock_irq(&dev->config_lock);
183	dev->config_core_enabled = false;
184	spin_unlock_irq(&dev->config_lock);
185}
186
187static void virtio_config_core_enable(struct virtio_device *dev)
188{
189	spin_lock_irq(&dev->config_lock);
190	dev->config_core_enabled = true;
191	if (dev->config_change_pending)
192		__virtio_config_changed(dev);
193	spin_unlock_irq(&dev->config_lock);
194}
195
196void virtio_add_status(struct virtio_device *dev, unsigned int status)
197{
198	might_sleep();
199	dev->config->set_status(dev, dev->config->get_status(dev) | status);
200}
201EXPORT_SYMBOL_GPL(virtio_add_status);
202
203/* Do some validation, then set FEATURES_OK */
204static int virtio_features_ok(struct virtio_device *dev)
205{
206	unsigned int status;
207
208	might_sleep();
209
210	if (virtio_check_mem_acc_cb(dev)) {
211		if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) {
212			dev_warn(&dev->dev,
213				 "device must provide VIRTIO_F_VERSION_1\n");
214			return -ENODEV;
215		}
216
217		if (!virtio_has_feature(dev, VIRTIO_F_ACCESS_PLATFORM)) {
218			dev_warn(&dev->dev,
219				 "device must provide VIRTIO_F_ACCESS_PLATFORM\n");
220			return -ENODEV;
221		}
222	}
223
224	if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))
225		return 0;
226
227	virtio_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
228	status = dev->config->get_status(dev);
229	if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
230		dev_err(&dev->dev, "virtio: device refuses features: %x\n",
231			status);
232		return -ENODEV;
233	}
234	return 0;
235}
236
237/**
238 * virtio_reset_device - quiesce device for removal
239 * @dev: the device to reset
240 *
241 * Prevents device from sending interrupts and accessing memory.
242 *
243 * Generally used for cleanup during driver / device removal.
244 *
245 * Once this has been invoked, caller must ensure that
246 * virtqueue_notify / virtqueue_kick are not in progress.
247 *
248 * Note: this guarantees that vq callbacks are not in progress, however caller
249 * is responsible for preventing access from other contexts, such as a system
250 * call/workqueue/bh.  Invoking virtio_break_device then flushing any such
251 * contexts is one way to handle that.
252 * */
253void virtio_reset_device(struct virtio_device *dev)
254{
255#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
256	/*
257	 * The below virtio_synchronize_cbs() guarantees that any
258	 * interrupt for this line arriving after
259	 * virtio_synchronize_vqs() has completed is guaranteed to see
260	 * vq->broken as true.
261	 */
262	virtio_break_device(dev);
263	virtio_synchronize_cbs(dev);
264#endif
265
266	dev->config->reset(dev);
267}
268EXPORT_SYMBOL_GPL(virtio_reset_device);
269
270static int virtio_dev_probe(struct device *_d)
271{
272	int err, i;
273	struct virtio_device *dev = dev_to_virtio(_d);
274	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
275	u64 device_features[VIRTIO_FEATURES_U64S];
276	u64 driver_features[VIRTIO_FEATURES_U64S];
277	u64 driver_features_legacy;
278
279	/* We have a driver! */
280	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
281
282	/* Figure out what features the device supports. */
283	virtio_get_features(dev, device_features);
284
285	/* Figure out what features the driver supports. */
286	virtio_features_zero(driver_features);
287	for (i = 0; i < drv->feature_table_size; i++) {
288		unsigned int f = drv->feature_table[i];
289		if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_BITS))
290			virtio_features_set_bit(driver_features, f);
291	}
292
293	/* Some drivers have a separate feature table for virtio v1.0 */
294	if (drv->feature_table_legacy) {
295		driver_features_legacy = 0;
296		for (i = 0; i < drv->feature_table_size_legacy; i++) {
297			unsigned int f = drv->feature_table_legacy[i];
298			if (!WARN_ON_ONCE(f >= 64))
299				driver_features_legacy |= (1ULL << f);
300		}
301	} else {
302		driver_features_legacy = driver_features[0];
303	}
304
305	if (virtio_features_test_bit(device_features, VIRTIO_F_VERSION_1)) {
306		for (i = 0; i < VIRTIO_FEATURES_U64S; ++i)
307			dev->features_array[i] = driver_features[i] &
308						 device_features[i];
309	} else {
310		virtio_features_from_u64(dev->features_array,
311					 driver_features_legacy &
312					 device_features[0]);
313	}
314
315	/* When debugging, user may filter some features by hand. */
316	virtio_debug_device_filter_features(dev);
317
318	/* Transport features always preserved to pass to finalize_features. */
319	for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++)
320		if (virtio_features_test_bit(device_features, i))
321			__virtio_set_bit(dev, i);
322
323	err = dev->config->finalize_features(dev);
324	if (err)
325		goto err;
326
327	if (drv->validate) {
328		u64 features[VIRTIO_FEATURES_U64S];
329
330		virtio_features_copy(features, dev->features_array);
331		err = drv->validate(dev);
332		if (err)
333			goto err;
334
335		/* Did validation change any features? Then write them again. */
336		if (!virtio_features_equal(features, dev->features_array)) {
337			err = dev->config->finalize_features(dev);
338			if (err)
339				goto err;
340		}
341	}
342
343	err = virtio_features_ok(dev);
344	if (err)
345		goto err;
346
347	err = drv->probe(dev);
348	if (err)
349		goto err;
350
351	/* If probe didn't do it, mark device DRIVER_OK ourselves. */
352	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
353		virtio_device_ready(dev);
354
355	if (drv->scan)
356		drv->scan(dev);
357
358	virtio_config_core_enable(dev);
359
360	return 0;
361
362err:
363	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
364	return err;
365
366}
367
368static void virtio_dev_remove(struct device *_d)
369{
370	struct virtio_device *dev = dev_to_virtio(_d);
371	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
372
373	virtio_config_core_disable(dev);
374
375	drv->remove(dev);
376
377	/* Driver should have reset device. */
378	WARN_ON_ONCE(dev->config->get_status(dev));
379
380	/* Acknowledge the device's existence again. */
381	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
382
383	of_node_put(dev->dev.of_node);
384}
385
386/*
387 * virtio_irq_get_affinity - get IRQ affinity mask for device
388 * @_d: ptr to dev structure
389 * @irq_vec: interrupt vector number
390 *
391 * Return the CPU affinity mask for @_d and @irq_vec.
392 */
393static const struct cpumask *virtio_irq_get_affinity(struct device *_d,
394						     unsigned int irq_vec)
395{
396	struct virtio_device *dev = dev_to_virtio(_d);
397
398	if (!dev->config->get_vq_affinity)
399		return NULL;
400
401	return dev->config->get_vq_affinity(dev, irq_vec);
402}
403
404static void virtio_dev_shutdown(struct device *_d)
405{
406	struct virtio_device *dev = dev_to_virtio(_d);
407	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
408
409	/*
410	 * Stop accesses to or from the device.
411	 * We only need to do it if there's a driver - no accesses otherwise.
412	 */
413	if (!drv)
414		return;
415
416	/* If the driver has its own shutdown method, use that. */
417	if (drv->shutdown) {
418		drv->shutdown(dev);
419		return;
420	}
421
422	/*
423	 * Some devices get wedged if you kick them after they are
424	 * reset. Mark all vqs as broken to make sure we don't.
425	 */
426	virtio_break_device(dev);
427	/*
428	 * Guarantee that any callback will see vq->broken as true.
429	 */
430	virtio_synchronize_cbs(dev);
431	/*
432	 * As IOMMUs are reset on shutdown, this will block device access to memory.
433	 * Some devices get wedged if this happens, so reset to make sure it does not.
434	 */
435	dev->config->reset(dev);
436}
437
438static const struct bus_type virtio_bus = {
439	.name  = "virtio",
440	.match = virtio_dev_match,
441	.dev_groups = virtio_dev_groups,
442	.uevent = virtio_uevent,
443	.probe = virtio_dev_probe,
444	.remove = virtio_dev_remove,
445	.irq_get_affinity = virtio_irq_get_affinity,
446	.shutdown = virtio_dev_shutdown,
447};
448
449int __register_virtio_driver(struct virtio_driver *driver, struct module *owner)
450{
451	/* Catch this early. */
452	BUG_ON(driver->feature_table_size && !driver->feature_table);
453	driver->driver.bus = &virtio_bus;
454	driver->driver.owner = owner;
455
456	return driver_register(&driver->driver);
457}
458EXPORT_SYMBOL_GPL(__register_virtio_driver);
459
460void unregister_virtio_driver(struct virtio_driver *driver)
461{
462	driver_unregister(&driver->driver);
463}
464EXPORT_SYMBOL_GPL(unregister_virtio_driver);
465
466static int virtio_device_of_init(struct virtio_device *dev)
467{
468	struct device_node *np, *pnode = dev_of_node(dev->dev.parent);
469	char compat[] = "virtio,deviceXXXXXXXX";
470	int ret, count;
471
472	if (!pnode)
473		return 0;
474
475	count = of_get_available_child_count(pnode);
476	if (!count)
477		return 0;
478
479	/* There can be only 1 child node */
480	if (WARN_ON(count > 1))
481		return -EINVAL;
482
483	np = of_get_next_available_child(pnode, NULL);
484	if (WARN_ON(!np))
485		return -ENODEV;
486
487	ret = snprintf(compat, sizeof(compat), "virtio,device%x", dev->id.device);
488	BUG_ON(ret >= sizeof(compat));
489
490	/*
491	 * On powerpc/pseries virtio devices are PCI devices so PCI
492	 * vendor/device ids play the role of the "compatible" property.
493	 * Simply don't init of_node in this case.
494	 */
495	if (!of_device_is_compatible(np, compat)) {
496		ret = 0;
497		goto out;
498	}
499
500	dev->dev.of_node = np;
501	return 0;
502
503out:
504	of_node_put(np);
505	return ret;
506}
507
508/**
509 * register_virtio_device - register virtio device
510 * @dev        : virtio device to be registered
511 *
512 * On error, the caller must call put_device on &@dev->dev (and not kfree),
513 * as another code path may have obtained a reference to @dev.
514 *
515 * Returns: 0 on success, -error on failure
516 */
517int register_virtio_device(struct virtio_device *dev)
518{
519	int err;
520
521	dev->dev.bus = &virtio_bus;
522	device_initialize(&dev->dev);
523
524	/* Assign a unique device index and hence name. */
525	err = ida_alloc(&virtio_index_ida, GFP_KERNEL);
526	if (err < 0)
527		goto out;
528
529	dev->index = err;
530	err = dev_set_name(&dev->dev, "virtio%u", dev->index);
531	if (err)
532		goto out_ida_remove;
533
534	err = virtio_device_of_init(dev);
535	if (err)
536		goto out_ida_remove;
537
538	spin_lock_init(&dev->config_lock);
539	dev->config_driver_disabled = false;
540	dev->config_core_enabled = false;
541	dev->config_change_pending = false;
542
543	INIT_LIST_HEAD(&dev->vqs);
544	spin_lock_init(&dev->vqs_list_lock);
545
546	/* We always start by resetting the device, in case a previous
547	 * driver messed it up.  This also tests that code path a little. */
548	virtio_reset_device(dev);
549
550	/* Acknowledge that we've seen the device. */
551	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
552
553	virtio_debug_device_init(dev);
554
555	/*
556	 * device_add() causes the bus infrastructure to look for a matching
557	 * driver.
558	 */
559	err = device_add(&dev->dev);
560	if (err)
561		goto out_of_node_put;
562
563	return 0;
564
565out_of_node_put:
566	of_node_put(dev->dev.of_node);
567out_ida_remove:
568	ida_free(&virtio_index_ida, dev->index);
569out:
570	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
571	return err;
572}
573EXPORT_SYMBOL_GPL(register_virtio_device);
574
575bool is_virtio_device(struct device *dev)
576{
577	return dev->bus == &virtio_bus;
578}
579EXPORT_SYMBOL_GPL(is_virtio_device);
580
581void unregister_virtio_device(struct virtio_device *dev)
582{
583	int index = dev->index; /* save for after device release */
584
585	device_unregister(&dev->dev);
586	virtio_debug_device_exit(dev);
587	ida_free(&virtio_index_ida, index);
588}
589EXPORT_SYMBOL_GPL(unregister_virtio_device);
590
591static int virtio_device_restore_priv(struct virtio_device *dev, bool restore)
592{
593	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
594	int ret;
595
596	/* We always start by resetting the device, in case a previous
597	 * driver messed it up. */
598	virtio_reset_device(dev);
599
600	/* Acknowledge that we've seen the device. */
601	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
602
603	/* Maybe driver failed before freeze.
604	 * Restore the failed status, for debugging. */
605	if (dev->failed)
606		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
607
608	if (!drv)
609		return 0;
610
611	/* We have a driver! */
612	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
613
614	ret = dev->config->finalize_features(dev);
615	if (ret)
616		goto err;
617
618	ret = virtio_features_ok(dev);
619	if (ret)
620		goto err;
621
622	if (restore) {
623		if (drv->restore) {
624			ret = drv->restore(dev);
625			if (ret)
626				goto err;
627		}
628	} else {
629		ret = drv->reset_done(dev);
630		if (ret)
631			goto err;
632	}
633
634	/* If restore didn't do it, mark device DRIVER_OK ourselves. */
635	if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
636		virtio_device_ready(dev);
637
638	virtio_config_core_enable(dev);
639
640	return 0;
641
642err:
643	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
644	return ret;
645}
646
647#ifdef CONFIG_PM_SLEEP
648int virtio_device_freeze(struct virtio_device *dev)
649{
650	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
651	int ret;
652
653	virtio_config_core_disable(dev);
654
655	dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
656
657	if (drv && drv->freeze) {
658		ret = drv->freeze(dev);
659		if (ret) {
660			virtio_config_core_enable(dev);
661			return ret;
662		}
663	}
664
665	return 0;
666}
667EXPORT_SYMBOL_GPL(virtio_device_freeze);
668
669int virtio_device_restore(struct virtio_device *dev)
670{
671	return virtio_device_restore_priv(dev, true);
672}
673EXPORT_SYMBOL_GPL(virtio_device_restore);
674#endif
675
676int virtio_device_reset_prepare(struct virtio_device *dev)
677{
678	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
679	int ret;
680
681	if (!drv || !drv->reset_prepare)
682		return -EOPNOTSUPP;
683
684	virtio_config_core_disable(dev);
685
686	dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
687
688	ret = drv->reset_prepare(dev);
689	if (ret) {
690		virtio_config_core_enable(dev);
691		return ret;
692	}
693
694	return 0;
695}
696EXPORT_SYMBOL_GPL(virtio_device_reset_prepare);
697
698int virtio_device_reset_done(struct virtio_device *dev)
699{
700	struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
701
702	if (!drv || !drv->reset_done)
703		return -EOPNOTSUPP;
704
705	return virtio_device_restore_priv(dev, false);
706}
707EXPORT_SYMBOL_GPL(virtio_device_reset_done);
708
709static int virtio_init(void)
710{
711	BUILD_BUG_ON(offsetof(struct virtio_device, features) !=
712		     offsetof(struct virtio_device, features_array[0]));
713
714	if (bus_register(&virtio_bus) != 0)
715		panic("virtio bus registration failed");
716	virtio_debug_init();
717	return 0;
718}
719
720static void __exit virtio_exit(void)
721{
722	virtio_debug_exit();
723	bus_unregister(&virtio_bus);
724	ida_destroy(&virtio_index_ida);
725}
726core_initcall(virtio_init);
727module_exit(virtio_exit);
728
729MODULE_DESCRIPTION("Virtio core interface");
730MODULE_LICENSE("GPL");