adf_ctl_drv.c

  1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
  2/* Copyright(c) 2014 - 2020 Intel Corporation */
  3
  4#include <crypto/algapi.h>
  5#include <linux/module.h>
  6#include <linux/mutex.h>
  7#include <linux/slab.h>
  8#include <linux/fs.h>
  9#include <linux/bitops.h>
 10#include <linux/pci.h>
 11#include <linux/cdev.h>
 12#include <linux/uaccess.h>
 13
 14#include "adf_accel_devices.h"
 15#include "adf_common_drv.h"
 16#include "adf_cfg.h"
 17#include "adf_cfg_common.h"
 18#include "adf_cfg_user.h"
 19
 20#define ADF_CFG_MAX_SECTION 512
 21#define ADF_CFG_MAX_KEY_VAL 256
 22
 23#define DEVICE_NAME "qat_adf_ctl"
 24
 25static DEFINE_MUTEX(adf_ctl_lock);
 26static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg);
 27
 28static const struct file_operations adf_ctl_ops = {
 29	.owner = THIS_MODULE,
 30	.unlocked_ioctl = adf_ctl_ioctl,
 31	.compat_ioctl = compat_ptr_ioctl,
 32};
 33
 34static const struct class adf_ctl_class = {
 35	.name = DEVICE_NAME,
 36};
 37
 38struct adf_ctl_drv_info {
 39	unsigned int major;
 40	struct cdev drv_cdev;
 41};
 42
 43static struct adf_ctl_drv_info adf_ctl_drv;
 44
 45static void adf_chr_drv_destroy(void)
 46{
 47	device_destroy(&adf_ctl_class, MKDEV(adf_ctl_drv.major, 0));
 48	cdev_del(&adf_ctl_drv.drv_cdev);
 49	class_unregister(&adf_ctl_class);
 50	unregister_chrdev_region(MKDEV(adf_ctl_drv.major, 0), 1);
 51}
 52
 53static int adf_chr_drv_create(void)
 54{
 55	dev_t dev_id;
 56	struct device *drv_device;
 57	int ret;
 58
 59	if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) {
 60		pr_err("QAT: unable to allocate chrdev region\n");
 61		return -EFAULT;
 62	}
 63
 64	ret = class_register(&adf_ctl_class);
 65	if (ret)
 66		goto err_chrdev_unreg;
 67
 68	adf_ctl_drv.major = MAJOR(dev_id);
 69	cdev_init(&adf_ctl_drv.drv_cdev, &adf_ctl_ops);
 70	if (cdev_add(&adf_ctl_drv.drv_cdev, dev_id, 1)) {
 71		pr_err("QAT: cdev add failed\n");
 72		goto err_class_destr;
 73	}
 74
 75	drv_device = device_create(&adf_ctl_class, NULL,
 76				   MKDEV(adf_ctl_drv.major, 0),
 77				   NULL, DEVICE_NAME);
 78	if (IS_ERR(drv_device)) {
 79		pr_err("QAT: failed to create device\n");
 80		goto err_cdev_del;
 81	}
 82	return 0;
 83err_cdev_del:
 84	cdev_del(&adf_ctl_drv.drv_cdev);
 85err_class_destr:
 86	class_unregister(&adf_ctl_class);
 87err_chrdev_unreg:
 88	unregister_chrdev_region(dev_id, 1);
 89	return -EFAULT;
 90}
 91
 92static struct adf_user_cfg_ctl_data *adf_ctl_alloc_resources(unsigned long arg)
 93{
 94	struct adf_user_cfg_ctl_data *cfg_data;
 95
 96	cfg_data = memdup_user((void __user *)arg, sizeof(*cfg_data));
 97	if (IS_ERR(cfg_data))
 98		pr_err("QAT: failed to copy from user cfg_data.\n");
 99	return cfg_data;
100}
101
102static int adf_add_key_value_data(struct adf_accel_dev *accel_dev,
103				  const char *section,
104				  const struct adf_user_cfg_key_val *key_val)
105{
106	if (key_val->type == ADF_HEX) {
107		long *ptr = (long *)key_val->val;
108		long val = *ptr;
109
110		if (adf_cfg_add_key_value_param(accel_dev, section,
111						key_val->key, (void *)val,
112						key_val->type)) {
113			dev_err(&GET_DEV(accel_dev),
114				"failed to add hex keyvalue.\n");
115			return -EFAULT;
116		}
117	} else {
118		if (adf_cfg_add_key_value_param(accel_dev, section,
119						key_val->key, key_val->val,
120						key_val->type)) {
121			dev_err(&GET_DEV(accel_dev),
122				"failed to add keyvalue.\n");
123			return -EFAULT;
124		}
125	}
126	return 0;
127}
128
129static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev,
130				   struct adf_user_cfg_ctl_data *ctl_data)
131{
132	struct adf_user_cfg_key_val key_val;
133	struct adf_user_cfg_key_val *params_head;
134	struct adf_user_cfg_section section, *section_head;
135	int i, j;
136
137	section_head = ctl_data->config_section;
138
139	for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) {
140		if (copy_from_user(&section, (void __user *)section_head,
141				   sizeof(*section_head))) {
142			dev_err(&GET_DEV(accel_dev),
143				"failed to copy section info\n");
144			goto out_err;
145		}
146
147		if (adf_cfg_section_add(accel_dev, section.name)) {
148			dev_err(&GET_DEV(accel_dev),
149				"failed to add section.\n");
150			goto out_err;
151		}
152
153		params_head = section.params;
154
155		for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) {
156			if (copy_from_user(&key_val, (void __user *)params_head,
157					   sizeof(key_val))) {
158				dev_err(&GET_DEV(accel_dev),
159					"Failed to copy keyvalue.\n");
160				goto out_err;
161			}
162			if (adf_add_key_value_data(accel_dev, section.name,
163						   &key_val)) {
164				goto out_err;
165			}
166			params_head = key_val.next;
167		}
168		section_head = section.next;
169	}
170	return 0;
171out_err:
172	adf_cfg_del_all(accel_dev);
173	return -EFAULT;
174}
175
176static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd,
177				    unsigned long arg)
178{
179	struct adf_user_cfg_ctl_data *ctl_data;
180	struct adf_accel_dev *accel_dev;
181	int ret = 0;
182
183	ctl_data = adf_ctl_alloc_resources(arg);
184	if (IS_ERR(ctl_data))
185		return PTR_ERR(ctl_data);
186
187	accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id);
188	if (!accel_dev) {
189		ret = -EFAULT;
190		goto out;
191	}
192
193	if (adf_dev_started(accel_dev)) {
194		ret = -EFAULT;
195		goto out;
196	}
197
198	if (adf_copy_key_value_data(accel_dev, ctl_data)) {
199		ret = -EFAULT;
200		goto out;
201	}
202	set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
203out:
204	kfree(ctl_data);
205	return ret;
206}
207
208static int adf_ctl_is_device_in_use(int id)
209{
210	struct adf_accel_dev *dev;
211
212	list_for_each_entry(dev, adf_devmgr_get_head(), list) {
213		if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
214			if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) {
215				dev_info(&GET_DEV(dev),
216					 "device qat_dev%d is busy\n",
217					 dev->accel_id);
218				return -EBUSY;
219			}
220		}
221	}
222	return 0;
223}
224
225static void adf_ctl_stop_devices(u32 id)
226{
227	struct adf_accel_dev *accel_dev;
228
229	list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) {
230		if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
231			if (!adf_dev_started(accel_dev))
232				continue;
233
234			/* First stop all VFs */
235			if (!accel_dev->is_vf)
236				continue;
237
238			adf_dev_down(accel_dev);
239		}
240	}
241
242	list_for_each_entry(accel_dev, adf_devmgr_get_head(), list) {
243		if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
244			if (!adf_dev_started(accel_dev))
245				continue;
246
247			adf_dev_down(accel_dev);
248		}
249	}
250}
251
252static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd,
253				  unsigned long arg)
254{
255	int ret;
256	struct adf_user_cfg_ctl_data *ctl_data;
257
258	ctl_data = adf_ctl_alloc_resources(arg);
259	if (IS_ERR(ctl_data))
260		return PTR_ERR(ctl_data);
261
262	if (adf_devmgr_verify_id(ctl_data->device_id)) {
263		pr_err("QAT: Device %d not found\n", ctl_data->device_id);
264		ret = -ENODEV;
265		goto out;
266	}
267
268	ret = adf_ctl_is_device_in_use(ctl_data->device_id);
269	if (ret)
270		goto out;
271
272	if (ctl_data->device_id == ADF_CFG_ALL_DEVICES)
273		pr_info("QAT: Stopping all acceleration devices.\n");
274	else
275		pr_info("QAT: Stopping acceleration device qat_dev%d.\n",
276			ctl_data->device_id);
277
278	adf_ctl_stop_devices(ctl_data->device_id);
279
280out:
281	kfree(ctl_data);
282	return ret;
283}
284
285static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd,
286				   unsigned long arg)
287{
288	int ret;
289	struct adf_user_cfg_ctl_data *ctl_data;
290	struct adf_accel_dev *accel_dev;
291
292	ctl_data = adf_ctl_alloc_resources(arg);
293	if (IS_ERR(ctl_data))
294		return PTR_ERR(ctl_data);
295
296	ret = -ENODEV;
297	accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id);
298	if (!accel_dev)
299		goto out;
300
301	dev_info(&GET_DEV(accel_dev),
302		 "Starting acceleration device qat_dev%d.\n",
303		 ctl_data->device_id);
304
305	ret = adf_dev_up(accel_dev, false);
306
307	if (ret) {
308		dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n",
309			ctl_data->device_id);
310		adf_dev_down(accel_dev);
311	}
312out:
313	kfree(ctl_data);
314	return ret;
315}
316
317static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd,
318					 unsigned long arg)
319{
320	u32 num_devices = 0;
321
322	adf_devmgr_get_num_dev(&num_devices);
323	if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices)))
324		return -EFAULT;
325
326	return 0;
327}
328
329static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd,
330				    unsigned long arg)
331{
332	struct adf_hw_device_data *hw_data;
333	struct adf_dev_status_info dev_info;
334	struct adf_accel_dev *accel_dev;
335
336	if (copy_from_user(&dev_info, (void __user *)arg,
337			   sizeof(struct adf_dev_status_info))) {
338		pr_err("QAT: failed to copy from user.\n");
339		return -EFAULT;
340	}
341
342	accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id);
343	if (!accel_dev)
344		return -ENODEV;
345
346	hw_data = accel_dev->hw_device;
347	dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN;
348	dev_info.num_ae = hw_data->get_num_aes(hw_data);
349	dev_info.num_accel = hw_data->get_num_accels(hw_data);
350	dev_info.num_logical_accel = hw_data->num_logical_accel;
351	dev_info.banks_per_accel = hw_data->num_banks
352					/ hw_data->num_logical_accel;
353	strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name));
354	dev_info.instance_id = hw_data->instance_id;
355	dev_info.type = hw_data->dev_class->type;
356	dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number;
357	dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn);
358	dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn);
359
360	if (copy_to_user((void __user *)arg, &dev_info,
361			 sizeof(struct adf_dev_status_info))) {
362		dev_err(&GET_DEV(accel_dev), "failed to copy status.\n");
363		return -EFAULT;
364	}
365	return 0;
366}
367
368static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
369{
370	int ret;
371
372	if (mutex_lock_interruptible(&adf_ctl_lock))
373		return -EFAULT;
374
375	switch (cmd) {
376	case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS:
377		ret = adf_ctl_ioctl_dev_config(fp, cmd, arg);
378		break;
379
380	case IOCTL_STOP_ACCEL_DEV:
381		ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg);
382		break;
383
384	case IOCTL_START_ACCEL_DEV:
385		ret = adf_ctl_ioctl_dev_start(fp, cmd, arg);
386		break;
387
388	case IOCTL_GET_NUM_DEVICES:
389		ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg);
390		break;
391
392	case IOCTL_STATUS_ACCEL_DEV:
393		ret = adf_ctl_ioctl_get_status(fp, cmd, arg);
394		break;
395	default:
396		pr_err_ratelimited("QAT: Invalid ioctl %d\n", cmd);
397		ret = -EFAULT;
398		break;
399	}
400	mutex_unlock(&adf_ctl_lock);
401	return ret;
402}
403
404static int __init adf_register_ctl_device_driver(void)
405{
406	if (adf_chr_drv_create())
407		goto err_chr_dev;
408
409	if (adf_init_misc_wq())
410		goto err_misc_wq;
411
412	if (adf_init_aer())
413		goto err_aer;
414
415	if (adf_init_pf_wq())
416		goto err_pf_wq;
417
418	if (adf_init_vf_wq())
419		goto err_vf_wq;
420
421	if (qat_crypto_register())
422		goto err_crypto_register;
423
424	if (qat_compression_register())
425		goto err_compression_register;
426
427	return 0;
428
429err_compression_register:
430	qat_crypto_unregister();
431err_crypto_register:
432	adf_exit_vf_wq();
433err_vf_wq:
434	adf_exit_pf_wq();
435err_pf_wq:
436	adf_exit_aer();
437err_aer:
438	adf_exit_misc_wq();
439err_misc_wq:
440	adf_chr_drv_destroy();
441err_chr_dev:
442	mutex_destroy(&adf_ctl_lock);
443	return -EFAULT;
444}
445
446static void __exit adf_unregister_ctl_device_driver(void)
447{
448	adf_chr_drv_destroy();
449	adf_exit_misc_wq();
450	adf_exit_aer();
451	adf_exit_vf_wq();
452	adf_exit_pf_wq();
453	qat_crypto_unregister();
454	qat_compression_unregister();
455	adf_clean_vf_map(false);
456	mutex_destroy(&adf_ctl_lock);
457}
458
459module_init(adf_register_ctl_device_driver);
460module_exit(adf_unregister_ctl_device_driver);
461MODULE_LICENSE("Dual BSD/GPL");
462MODULE_AUTHOR("Intel");
463MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
464MODULE_ALIAS_CRYPTO("intel_qat");
465MODULE_VERSION(ADF_DRV_VERSION);
466MODULE_IMPORT_NS("CRYPTO_INTERNAL");