virtio_crypto_core.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2 /* Driver for Virtio crypto device.
  3  *
  4  * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
  5  */
  6
  7#include <linux/err.h>
  8#include <linux/module.h>
  9#include <linux/virtio_config.h>
 10#include <linux/cpu.h>
 11
 12#include <uapi/linux/virtio_crypto.h>
 13#include "virtio_crypto_common.h"
 14
 15
 16void
 17virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
 18{
 19	if (vc_req) {
 20		kfree_sensitive(vc_req->req_data);
 21		kfree(vc_req->sgs);
 22	}
 23}
 24
 25static void virtio_crypto_ctrlq_callback(struct virtio_crypto_ctrl_request *vc_ctrl_req)
 26{
 27	complete(&vc_ctrl_req->compl);
 28}
 29
 30static void virtcrypto_ctrlq_callback(struct virtqueue *vq)
 31{
 32	struct virtio_crypto *vcrypto = vq->vdev->priv;
 33	struct virtio_crypto_ctrl_request *vc_ctrl_req;
 34	unsigned long flags;
 35	unsigned int len;
 36
 37	spin_lock_irqsave(&vcrypto->ctrl_lock, flags);
 38	do {
 39		virtqueue_disable_cb(vq);
 40		while ((vc_ctrl_req = virtqueue_get_buf(vq, &len)) != NULL) {
 41			spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
 42			virtio_crypto_ctrlq_callback(vc_ctrl_req);
 43			spin_lock_irqsave(&vcrypto->ctrl_lock, flags);
 44		}
 45	} while (!virtqueue_enable_cb(vq));
 46	spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
 47}
 48
 49int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[],
 50		unsigned int out_sgs, unsigned int in_sgs,
 51		struct virtio_crypto_ctrl_request *vc_ctrl_req)
 52{
 53	int err;
 54	unsigned long flags;
 55
 56	init_completion(&vc_ctrl_req->compl);
 57
 58	spin_lock_irqsave(&vcrypto->ctrl_lock, flags);
 59	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, out_sgs, in_sgs, vc_ctrl_req, GFP_ATOMIC);
 60	if (err < 0) {
 61		spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
 62		return err;
 63	}
 64
 65	virtqueue_kick(vcrypto->ctrl_vq);
 66	spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags);
 67
 68	wait_for_completion(&vc_ctrl_req->compl);
 69
 70	return 0;
 71}
 72
 73static void virtcrypto_done_task(unsigned long data)
 74{
 75	struct data_queue *data_vq = (struct data_queue *)data;
 76	struct virtqueue *vq = data_vq->vq;
 77	struct virtio_crypto_request *vc_req;
 78	unsigned long flags;
 79	unsigned int len;
 80
 81	spin_lock_irqsave(&data_vq->lock, flags);
 82	do {
 83		virtqueue_disable_cb(vq);
 84		while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
 85			spin_unlock_irqrestore(&data_vq->lock, flags);
 86			if (vc_req->alg_cb)
 87				vc_req->alg_cb(vc_req, len);
 88			spin_lock_irqsave(&data_vq->lock, flags);
 89		}
 90	} while (!virtqueue_enable_cb(vq));
 91	spin_unlock_irqrestore(&data_vq->lock, flags);
 92}
 93
 94static void virtcrypto_dataq_callback(struct virtqueue *vq)
 95{
 96	struct virtio_crypto *vcrypto = vq->vdev->priv;
 97	struct data_queue *dq = &vcrypto->data_vq[vq->index];
 98
 99	tasklet_schedule(&dq->done_task);
100}
101
102static int virtcrypto_find_vqs(struct virtio_crypto *vi)
103{
104	struct virtqueue_info *vqs_info;
105	struct virtqueue **vqs;
106	int ret = -ENOMEM;
107	int i, total_vqs;
108	struct device *dev = &vi->vdev->dev;
109
110	/*
111	 * We expect 1 data virtqueue, followed by
112	 * possible N-1 data queues used in multiqueue mode,
113	 * followed by control vq.
114	 */
115	total_vqs = vi->max_data_queues + 1;
116
117	/* Allocate space for find_vqs parameters */
118	vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
119	if (!vqs)
120		goto err_vq;
121	vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL);
122	if (!vqs_info)
123		goto err_vqs_info;
124
125	/* Parameters for control virtqueue */
126	vqs_info[total_vqs - 1].callback = virtcrypto_ctrlq_callback;
127	vqs_info[total_vqs - 1].name = "controlq";
128
129	/* Allocate/initialize parameters for data virtqueues */
130	for (i = 0; i < vi->max_data_queues; i++) {
131		vqs_info[i].callback = virtcrypto_dataq_callback;
132		snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name),
133				"dataq.%d", i);
134		vqs_info[i].name = vi->data_vq[i].name;
135	}
136
137	ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL);
138	if (ret)
139		goto err_find;
140
141	vi->ctrl_vq = vqs[total_vqs - 1];
142
143	for (i = 0; i < vi->max_data_queues; i++) {
144		spin_lock_init(&vi->data_vq[i].lock);
145		vi->data_vq[i].vq = vqs[i];
146		/* Initialize crypto engine */
147		vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, true,
148						virtqueue_get_vring_size(vqs[i]));
149		if (!vi->data_vq[i].engine) {
150			ret = -ENOMEM;
151			goto err_engine;
152		}
153		tasklet_init(&vi->data_vq[i].done_task, virtcrypto_done_task,
154				(unsigned long)&vi->data_vq[i]);
155	}
156
157	kfree(vqs_info);
158	kfree(vqs);
159
160	return 0;
161
162err_engine:
163err_find:
164	kfree(vqs_info);
165err_vqs_info:
166	kfree(vqs);
167err_vq:
168	return ret;
169}
170
171static int virtcrypto_alloc_queues(struct virtio_crypto *vi)
172{
173	vi->data_vq = kcalloc(vi->max_data_queues, sizeof(*vi->data_vq),
174				GFP_KERNEL);
175	if (!vi->data_vq)
176		return -ENOMEM;
177
178	return 0;
179}
180
181static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu)
182{
183	int i;
184
185	if (vi->affinity_hint_set) {
186		for (i = 0; i < vi->max_data_queues; i++)
187			virtqueue_set_affinity(vi->data_vq[i].vq, NULL);
188
189		vi->affinity_hint_set = false;
190	}
191}
192
193static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto)
194{
195	int i = 0;
196	int cpu;
197
198	/*
199	 * In single queue mode, we don't set the cpu affinity.
200	 */
201	if (vcrypto->curr_queue == 1 || vcrypto->max_data_queues == 1) {
202		virtcrypto_clean_affinity(vcrypto, -1);
203		return;
204	}
205
206	/*
207	 * In multiqueue mode, we let the queue to be private to one cpu
208	 * by setting the affinity hint to eliminate the contention.
209	 *
210	 * TODO: adds cpu hotplug support by register cpu notifier.
211	 *
212	 */
213	for_each_online_cpu(cpu) {
214		virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpumask_of(cpu));
215		if (++i >= vcrypto->max_data_queues)
216			break;
217	}
218
219	vcrypto->affinity_hint_set = true;
220}
221
222static void virtcrypto_free_queues(struct virtio_crypto *vi)
223{
224	kfree(vi->data_vq);
225}
226
227static int virtcrypto_init_vqs(struct virtio_crypto *vi)
228{
229	int ret;
230
231	/* Allocate send & receive queues */
232	ret = virtcrypto_alloc_queues(vi);
233	if (ret)
234		goto err;
235
236	ret = virtcrypto_find_vqs(vi);
237	if (ret)
238		goto err_free;
239
240	cpus_read_lock();
241	virtcrypto_set_affinity(vi);
242	cpus_read_unlock();
243
244	return 0;
245
246err_free:
247	virtcrypto_free_queues(vi);
248err:
249	return ret;
250}
251
252static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
253{
254	u32 status;
255	int err;
256
257	virtio_cread_le(vcrypto->vdev,
258			struct virtio_crypto_config, status, &status);
259
260	/*
261	 * Unknown status bits would be a host error and the driver
262	 * should consider the device to be broken.
263	 */
264	if (status & (~VIRTIO_CRYPTO_S_HW_READY)) {
265		dev_warn(&vcrypto->vdev->dev,
266				"Unknown status bits: 0x%x\n", status);
267
268		virtio_break_device(vcrypto->vdev);
269		return -EPERM;
270	}
271
272	if (vcrypto->status == status)
273		return 0;
274
275	vcrypto->status = status;
276
277	if (vcrypto->status & VIRTIO_CRYPTO_S_HW_READY) {
278		err = virtcrypto_dev_start(vcrypto);
279		if (err) {
280			dev_err(&vcrypto->vdev->dev,
281				"Failed to start virtio crypto device.\n");
282
283			return -EPERM;
284		}
285		dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n");
286	} else {
287		virtcrypto_dev_stop(vcrypto);
288		dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
289	}
290
291	return 0;
292}
293
294static int virtcrypto_start_crypto_engines(struct virtio_crypto *vcrypto)
295{
296	int32_t i;
297	int ret;
298
299	for (i = 0; i < vcrypto->max_data_queues; i++) {
300		if (vcrypto->data_vq[i].engine) {
301			ret = crypto_engine_start(vcrypto->data_vq[i].engine);
302			if (ret)
303				goto err;
304		}
305	}
306
307	return 0;
308
309err:
310	while (--i >= 0)
311		if (vcrypto->data_vq[i].engine)
312			crypto_engine_exit(vcrypto->data_vq[i].engine);
313
314	return ret;
315}
316
317static void virtcrypto_clear_crypto_engines(struct virtio_crypto *vcrypto)
318{
319	u32 i;
320
321	for (i = 0; i < vcrypto->max_data_queues; i++)
322		if (vcrypto->data_vq[i].engine)
323			crypto_engine_exit(vcrypto->data_vq[i].engine);
324}
325
326static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto)
327{
328	struct virtio_device *vdev = vcrypto->vdev;
329
330	virtcrypto_clean_affinity(vcrypto, -1);
331
332	vdev->config->del_vqs(vdev);
333
334	virtcrypto_free_queues(vcrypto);
335}
336
337static void vcrypto_config_changed_work(struct work_struct *work)
338{
339	struct virtio_crypto *vcrypto =
340		container_of(work, struct virtio_crypto, config_work);
341
342	virtcrypto_update_status(vcrypto);
343}
344
345static int virtcrypto_probe(struct virtio_device *vdev)
346{
347	int err = -EFAULT;
348	struct virtio_crypto *vcrypto;
349	u32 max_data_queues = 0, max_cipher_key_len = 0;
350	u32 max_auth_key_len = 0;
351	u64 max_size = 0;
352	u32 cipher_algo_l = 0;
353	u32 cipher_algo_h = 0;
354	u32 hash_algo = 0;
355	u32 mac_algo_l = 0;
356	u32 mac_algo_h = 0;
357	u32 aead_algo = 0;
358	u32 akcipher_algo = 0;
359	u32 crypto_services = 0;
360
361	if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
362		return -ENODEV;
363
364	if (!vdev->config->get) {
365		dev_err(&vdev->dev, "%s failure: config access disabled\n",
366			__func__);
367		return -EINVAL;
368	}
369
370	if (num_possible_nodes() > 1 && dev_to_node(&vdev->dev) < 0) {
371		/*
372		 * If the accelerator is connected to a node with no memory
373		 * there is no point in using the accelerator since the remote
374		 * memory transaction will be very slow.
375		 */
376		dev_err(&vdev->dev, "Invalid NUMA configuration.\n");
377		return -EINVAL;
378	}
379
380	vcrypto = kzalloc_node(sizeof(*vcrypto), GFP_KERNEL,
381					dev_to_node(&vdev->dev));
382	if (!vcrypto)
383		return -ENOMEM;
384
385	virtio_cread_le(vdev, struct virtio_crypto_config,
386			max_dataqueues, &max_data_queues);
387	if (max_data_queues < 1)
388		max_data_queues = 1;
389
390	virtio_cread_le(vdev, struct virtio_crypto_config,
391			max_cipher_key_len, &max_cipher_key_len);
392	virtio_cread_le(vdev, struct virtio_crypto_config,
393			max_auth_key_len, &max_auth_key_len);
394	virtio_cread_le(vdev, struct virtio_crypto_config,
395			max_size, &max_size);
396	virtio_cread_le(vdev, struct virtio_crypto_config,
397			crypto_services, &crypto_services);
398	virtio_cread_le(vdev, struct virtio_crypto_config,
399			cipher_algo_l, &cipher_algo_l);
400	virtio_cread_le(vdev, struct virtio_crypto_config,
401			cipher_algo_h, &cipher_algo_h);
402	virtio_cread_le(vdev, struct virtio_crypto_config,
403			hash_algo, &hash_algo);
404	virtio_cread_le(vdev, struct virtio_crypto_config,
405			mac_algo_l, &mac_algo_l);
406	virtio_cread_le(vdev, struct virtio_crypto_config,
407			mac_algo_h, &mac_algo_h);
408	virtio_cread_le(vdev, struct virtio_crypto_config,
409			aead_algo, &aead_algo);
410	if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER))
411		virtio_cread_le(vdev, struct virtio_crypto_config,
412				akcipher_algo, &akcipher_algo);
413
414	/* Add virtio crypto device to global table */
415	err = virtcrypto_devmgr_add_dev(vcrypto);
416	if (err) {
417		dev_err(&vdev->dev, "Failed to add new virtio crypto device.\n");
418		goto free;
419	}
420	vcrypto->owner = THIS_MODULE;
421	vcrypto = vdev->priv = vcrypto;
422	vcrypto->vdev = vdev;
423
424	spin_lock_init(&vcrypto->ctrl_lock);
425
426	/* Use single data queue as default */
427	vcrypto->curr_queue = 1;
428	vcrypto->max_data_queues = max_data_queues;
429	vcrypto->max_cipher_key_len = max_cipher_key_len;
430	vcrypto->max_auth_key_len = max_auth_key_len;
431	vcrypto->max_size = max_size;
432	vcrypto->crypto_services = crypto_services;
433	vcrypto->cipher_algo_l = cipher_algo_l;
434	vcrypto->cipher_algo_h = cipher_algo_h;
435	vcrypto->mac_algo_l = mac_algo_l;
436	vcrypto->mac_algo_h = mac_algo_h;
437	vcrypto->hash_algo = hash_algo;
438	vcrypto->aead_algo = aead_algo;
439	vcrypto->akcipher_algo = akcipher_algo;
440
441	dev_info(&vdev->dev,
442		"max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
443		vcrypto->max_data_queues,
444		vcrypto->max_cipher_key_len,
445		vcrypto->max_auth_key_len,
446		vcrypto->max_size);
447
448	err = virtcrypto_init_vqs(vcrypto);
449	if (err) {
450		dev_err(&vdev->dev, "Failed to initialize vqs.\n");
451		goto free_dev;
452	}
453
454	err = virtcrypto_start_crypto_engines(vcrypto);
455	if (err)
456		goto free_vqs;
457
458	virtio_device_ready(vdev);
459
460	err = virtcrypto_update_status(vcrypto);
461	if (err)
462		goto free_engines;
463
464	INIT_WORK(&vcrypto->config_work, vcrypto_config_changed_work);
465
466	return 0;
467
468free_engines:
469	virtcrypto_clear_crypto_engines(vcrypto);
470free_vqs:
471	virtio_reset_device(vdev);
472	virtcrypto_del_vqs(vcrypto);
473free_dev:
474	virtcrypto_devmgr_rm_dev(vcrypto);
475free:
476	kfree(vcrypto);
477	return err;
478}
479
480static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto)
481{
482	struct virtio_crypto_request *vc_req;
483	int i;
484	struct virtqueue *vq;
485
486	for (i = 0; i < vcrypto->max_data_queues; i++) {
487		vq = vcrypto->data_vq[i].vq;
488		while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL)
489			virtcrypto_clear_request(vc_req);
490		cond_resched();
491	}
492}
493
494static void virtcrypto_remove(struct virtio_device *vdev)
495{
496	struct virtio_crypto *vcrypto = vdev->priv;
497	int i;
498
499	dev_info(&vdev->dev, "Start virtcrypto_remove.\n");
500
501	flush_work(&vcrypto->config_work);
502	if (virtcrypto_dev_started(vcrypto))
503		virtcrypto_dev_stop(vcrypto);
504	for (i = 0; i < vcrypto->max_data_queues; i++)
505		tasklet_kill(&vcrypto->data_vq[i].done_task);
506	virtio_reset_device(vdev);
507	virtcrypto_free_unused_reqs(vcrypto);
508	virtcrypto_clear_crypto_engines(vcrypto);
509	virtcrypto_del_vqs(vcrypto);
510	virtcrypto_devmgr_rm_dev(vcrypto);
511	kfree(vcrypto);
512}
513
514static void virtcrypto_config_changed(struct virtio_device *vdev)
515{
516	struct virtio_crypto *vcrypto = vdev->priv;
517
518	schedule_work(&vcrypto->config_work);
519}
520
521#ifdef CONFIG_PM_SLEEP
522static int virtcrypto_freeze(struct virtio_device *vdev)
523{
524	struct virtio_crypto *vcrypto = vdev->priv;
525
526	flush_work(&vcrypto->config_work);
527	virtio_reset_device(vdev);
528	virtcrypto_free_unused_reqs(vcrypto);
529	if (virtcrypto_dev_started(vcrypto))
530		virtcrypto_dev_stop(vcrypto);
531
532	virtcrypto_clear_crypto_engines(vcrypto);
533	virtcrypto_del_vqs(vcrypto);
534	return 0;
535}
536
537static int virtcrypto_restore(struct virtio_device *vdev)
538{
539	struct virtio_crypto *vcrypto = vdev->priv;
540	int err;
541
542	err = virtcrypto_init_vqs(vcrypto);
543	if (err)
544		return err;
545
546	err = virtcrypto_start_crypto_engines(vcrypto);
547	if (err)
548		goto free_vqs;
549
550	virtio_device_ready(vdev);
551
552	err = virtcrypto_dev_start(vcrypto);
553	if (err) {
554		dev_err(&vdev->dev, "Failed to start virtio crypto device.\n");
555		goto free_engines;
556	}
557
558	return 0;
559
560free_engines:
561	virtcrypto_clear_crypto_engines(vcrypto);
562free_vqs:
563	virtio_reset_device(vdev);
564	virtcrypto_del_vqs(vcrypto);
565	return err;
566}
567#endif
568
569static const unsigned int features[] = {
570	/* none */
571};
572
573static const struct virtio_device_id id_table[] = {
574	{ VIRTIO_ID_CRYPTO, VIRTIO_DEV_ANY_ID },
575	{ 0 },
576};
577
578static struct virtio_driver virtio_crypto_driver = {
579	.driver.name         = KBUILD_MODNAME,
580	.feature_table       = features,
581	.feature_table_size  = ARRAY_SIZE(features),
582	.id_table            = id_table,
583	.probe               = virtcrypto_probe,
584	.remove              = virtcrypto_remove,
585	.config_changed = virtcrypto_config_changed,
586#ifdef CONFIG_PM_SLEEP
587	.freeze = virtcrypto_freeze,
588	.restore = virtcrypto_restore,
589#endif
590};
591
592module_virtio_driver(virtio_crypto_driver);
593
594MODULE_DEVICE_TABLE(virtio, id_table);
595MODULE_DESCRIPTION("virtio crypto device driver");
596MODULE_LICENSE("GPL");
597MODULE_AUTHOR("Gonglei <arei.gonglei@huawei.com>");