uacce.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/compat.h>
  3#include <linux/dma-mapping.h>
  4#include <linux/iommu.h>
  5#include <linux/module.h>
  6#include <linux/poll.h>
  7#include <linux/slab.h>
  8#include <linux/uacce.h>
  9
 10static dev_t uacce_devt;
 11static DEFINE_XARRAY_ALLOC(uacce_xa);
 12
 13static const struct class uacce_class = {
 14	.name = UACCE_NAME,
 15};
 16
 17/*
 18 * If the parent driver or the device disappears, the queue state is invalid and
 19 * ops are not usable anymore.
 20 */
 21static bool uacce_queue_is_valid(struct uacce_queue *q)
 22{
 23	return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;
 24}
 25
 26static int uacce_start_queue(struct uacce_queue *q)
 27{
 28	int ret;
 29
 30	if (q->state != UACCE_Q_INIT)
 31		return -EINVAL;
 32
 33	if (q->uacce->ops->start_queue) {
 34		ret = q->uacce->ops->start_queue(q);
 35		if (ret < 0)
 36			return ret;
 37	}
 38
 39	q->state = UACCE_Q_STARTED;
 40	return 0;
 41}
 42
 43static int uacce_stop_queue(struct uacce_queue *q)
 44{
 45	struct uacce_device *uacce = q->uacce;
 46
 47	if (q->state != UACCE_Q_STARTED)
 48		return 0;
 49
 50	if (uacce->ops->stop_queue)
 51		uacce->ops->stop_queue(q);
 52
 53	q->state = UACCE_Q_INIT;
 54
 55	return 0;
 56}
 57
 58static void uacce_put_queue(struct uacce_queue *q)
 59{
 60	struct uacce_device *uacce = q->uacce;
 61
 62	uacce_stop_queue(q);
 63
 64	if (q->state != UACCE_Q_INIT)
 65		return;
 66
 67	if (uacce->ops->put_queue)
 68		uacce->ops->put_queue(q);
 69
 70	q->state = UACCE_Q_ZOMBIE;
 71}
 72
 73static long uacce_fops_unl_ioctl(struct file *filep,
 74				 unsigned int cmd, unsigned long arg)
 75{
 76	struct uacce_queue *q = filep->private_data;
 77	struct uacce_device *uacce = q->uacce;
 78	long ret = -ENXIO;
 79
 80	/*
 81	 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from
 82	 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which
 83	 * gets called with mmap_lock held, by taking uacce->mutex instead of
 84	 * q->mutex. Doing this in uacce_fops_mmap() is not possible because
 85	 * uacce_fops_open() calls iommu_sva_bind_device(), which takes
 86	 * mmap_lock, while holding uacce->mutex.
 87	 */
 88	mutex_lock(&uacce->mutex);
 89	if (!uacce_queue_is_valid(q))
 90		goto out_unlock;
 91
 92	switch (cmd) {
 93	case UACCE_CMD_START_Q:
 94		ret = uacce_start_queue(q);
 95		break;
 96	case UACCE_CMD_PUT_Q:
 97		ret = uacce_stop_queue(q);
 98		break;
 99	default:
100		if (uacce->ops->ioctl)
101			ret = uacce->ops->ioctl(q, cmd, arg);
102		else
103			ret = -EINVAL;
104	}
105out_unlock:
106	mutex_unlock(&uacce->mutex);
107	return ret;
108}
109
110#ifdef CONFIG_COMPAT
111static long uacce_fops_compat_ioctl(struct file *filep,
112				   unsigned int cmd, unsigned long arg)
113{
114	arg = (unsigned long)compat_ptr(arg);
115
116	return uacce_fops_unl_ioctl(filep, cmd, arg);
117}
118#endif
119
120static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
121{
122	u32 pasid;
123	struct iommu_sva *handle;
124
125	if (!(uacce->flags & UACCE_DEV_SVA))
126		return 0;
127
128	handle = iommu_sva_bind_device(uacce->parent, current->mm);
129	if (IS_ERR(handle))
130		return PTR_ERR(handle);
131
132	pasid = iommu_sva_get_pasid(handle);
133	if (pasid == IOMMU_PASID_INVALID) {
134		iommu_sva_unbind_device(handle);
135		return -ENODEV;
136	}
137
138	q->handle = handle;
139	q->pasid = pasid;
140	return 0;
141}
142
143static void uacce_unbind_queue(struct uacce_queue *q)
144{
145	if (!q->handle)
146		return;
147	iommu_sva_unbind_device(q->handle);
148	q->handle = NULL;
149}
150
151static int uacce_fops_open(struct inode *inode, struct file *filep)
152{
153	struct uacce_device *uacce;
154	struct uacce_queue *q;
155	int ret;
156
157	uacce = xa_load(&uacce_xa, iminor(inode));
158	if (!uacce)
159		return -ENODEV;
160
161	q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL);
162	if (!q)
163		return -ENOMEM;
164
165	mutex_lock(&uacce->mutex);
166
167	if (!uacce->parent) {
168		ret = -EINVAL;
169		goto out_with_mem;
170	}
171
172	ret = uacce_bind_queue(uacce, q);
173	if (ret)
174		goto out_with_mem;
175
176	q->uacce = uacce;
177
178	if (uacce->ops->get_queue) {
179		ret = uacce->ops->get_queue(uacce, q->pasid, q);
180		if (ret < 0)
181			goto out_with_bond;
182	}
183
184	init_waitqueue_head(&q->wait);
185	filep->private_data = q;
186	q->state = UACCE_Q_INIT;
187	q->mapping = filep->f_mapping;
188	mutex_init(&q->mutex);
189	list_add(&q->list, &uacce->queues);
190	mutex_unlock(&uacce->mutex);
191
192	return 0;
193
194out_with_bond:
195	uacce_unbind_queue(q);
196out_with_mem:
197	kfree(q);
198	mutex_unlock(&uacce->mutex);
199	return ret;
200}
201
202static int uacce_fops_release(struct inode *inode, struct file *filep)
203{
204	struct uacce_queue *q = filep->private_data;
205	struct uacce_device *uacce = q->uacce;
206
207	mutex_lock(&uacce->mutex);
208	uacce_put_queue(q);
209	uacce_unbind_queue(q);
210	list_del(&q->list);
211	mutex_unlock(&uacce->mutex);
212	kfree(q);
213
214	return 0;
215}
216
217static void uacce_vma_close(struct vm_area_struct *vma)
218{
219	struct uacce_queue *q = vma->vm_private_data;
220
221	if (vma->vm_pgoff < UACCE_MAX_REGION) {
222		struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff];
223
224		mutex_lock(&q->mutex);
225		q->qfrs[vma->vm_pgoff] = NULL;
226		mutex_unlock(&q->mutex);
227		kfree(qfr);
228	}
229}
230
231static int uacce_vma_mremap(struct vm_area_struct *area)
232{
233	return -EPERM;
234}
235
236static const struct vm_operations_struct uacce_vm_ops = {
237	.close = uacce_vma_close,
238	.mremap = uacce_vma_mremap,
239};
240
241static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)
242{
243	struct uacce_queue *q = filep->private_data;
244	struct uacce_device *uacce = q->uacce;
245	struct uacce_qfile_region *qfr;
246	enum uacce_qfrt type = UACCE_MAX_REGION;
247	int ret = 0;
248
249	if (vma->vm_pgoff < UACCE_MAX_REGION)
250		type = vma->vm_pgoff;
251	else
252		return -EINVAL;
253
254	qfr = kzalloc(sizeof(*qfr), GFP_KERNEL);
255	if (!qfr)
256		return -ENOMEM;
257
258	vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK);
259	vma->vm_ops = &uacce_vm_ops;
260	vma->vm_private_data = q;
261	qfr->type = type;
262
263	mutex_lock(&q->mutex);
264	if (!uacce_queue_is_valid(q)) {
265		ret = -ENXIO;
266		goto out_with_lock;
267	}
268
269	if (q->qfrs[type]) {
270		ret = -EEXIST;
271		goto out_with_lock;
272	}
273
274	switch (type) {
275	case UACCE_QFRT_MMIO:
276	case UACCE_QFRT_DUS:
277		if (!uacce->ops->mmap) {
278			ret = -EINVAL;
279			goto out_with_lock;
280		}
281
282		ret = uacce->ops->mmap(q, vma, qfr);
283		if (ret)
284			goto out_with_lock;
285		break;
286
287	default:
288		ret = -EINVAL;
289		goto out_with_lock;
290	}
291
292	q->qfrs[type] = qfr;
293	mutex_unlock(&q->mutex);
294
295	return ret;
296
297out_with_lock:
298	mutex_unlock(&q->mutex);
299	kfree(qfr);
300	return ret;
301}
302
303static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)
304{
305	struct uacce_queue *q = file->private_data;
306	struct uacce_device *uacce = q->uacce;
307	__poll_t ret = 0;
308
309	mutex_lock(&q->mutex);
310	if (!uacce_queue_is_valid(q))
311		goto out_unlock;
312
313	poll_wait(file, &q->wait, wait);
314
315	if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))
316		ret = EPOLLIN | EPOLLRDNORM;
317
318out_unlock:
319	mutex_unlock(&q->mutex);
320	return ret;
321}
322
323static const struct file_operations uacce_fops = {
324	.owner		= THIS_MODULE,
325	.open		= uacce_fops_open,
326	.release	= uacce_fops_release,
327	.unlocked_ioctl	= uacce_fops_unl_ioctl,
328#ifdef CONFIG_COMPAT
329	.compat_ioctl	= uacce_fops_compat_ioctl,
330#endif
331	.mmap		= uacce_fops_mmap,
332	.poll		= uacce_fops_poll,
333};
334
335#define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
336
337static ssize_t api_show(struct device *dev,
338			struct device_attribute *attr, char *buf)
339{
340	struct uacce_device *uacce = to_uacce_device(dev);
341
342	return sysfs_emit(buf, "%s\n", uacce->api_ver);
343}
344
345static ssize_t flags_show(struct device *dev,
346			  struct device_attribute *attr, char *buf)
347{
348	struct uacce_device *uacce = to_uacce_device(dev);
349
350	return sysfs_emit(buf, "%u\n", uacce->flags);
351}
352
353static ssize_t available_instances_show(struct device *dev,
354					struct device_attribute *attr,
355					char *buf)
356{
357	struct uacce_device *uacce = to_uacce_device(dev);
358
359	if (!uacce->ops->get_available_instances)
360		return -ENODEV;
361
362	return sysfs_emit(buf, "%d\n",
363		       uacce->ops->get_available_instances(uacce));
364}
365
366static ssize_t algorithms_show(struct device *dev,
367			       struct device_attribute *attr, char *buf)
368{
369	struct uacce_device *uacce = to_uacce_device(dev);
370
371	return sysfs_emit(buf, "%s\n", uacce->algs);
372}
373
374static ssize_t region_mmio_size_show(struct device *dev,
375				     struct device_attribute *attr, char *buf)
376{
377	struct uacce_device *uacce = to_uacce_device(dev);
378
379	return sysfs_emit(buf, "%lu\n",
380		       uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);
381}
382
383static ssize_t region_dus_size_show(struct device *dev,
384				    struct device_attribute *attr, char *buf)
385{
386	struct uacce_device *uacce = to_uacce_device(dev);
387
388	return sysfs_emit(buf, "%lu\n",
389		       uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);
390}
391
392static ssize_t isolate_show(struct device *dev,
393			    struct device_attribute *attr, char *buf)
394{
395	struct uacce_device *uacce = to_uacce_device(dev);
396
397	return sysfs_emit(buf, "%d\n", uacce->ops->get_isolate_state(uacce));
398}
399
400static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute *attr, char *buf)
401{
402	struct uacce_device *uacce = to_uacce_device(dev);
403	u32 val;
404
405	if (!uacce->ops->isolate_err_threshold_read)
406		return -ENOENT;
407
408	val = uacce->ops->isolate_err_threshold_read(uacce);
409
410	return sysfs_emit(buf, "%u\n", val);
411}
412
413static ssize_t isolate_strategy_store(struct device *dev, struct device_attribute *attr,
414				   const char *buf, size_t count)
415{
416	struct uacce_device *uacce = to_uacce_device(dev);
417	unsigned long val;
418	int ret;
419
420	if (!uacce->ops->isolate_err_threshold_write)
421		return -ENOENT;
422
423	if (kstrtoul(buf, 0, &val) < 0)
424		return -EINVAL;
425
426	if (val > UACCE_MAX_ERR_THRESHOLD)
427		return -EINVAL;
428
429	ret = uacce->ops->isolate_err_threshold_write(uacce, val);
430	if (ret)
431		return ret;
432
433	return count;
434}
435
436static DEVICE_ATTR_RO(api);
437static DEVICE_ATTR_RO(flags);
438static DEVICE_ATTR_RO(available_instances);
439static DEVICE_ATTR_RO(algorithms);
440static DEVICE_ATTR_RO(region_mmio_size);
441static DEVICE_ATTR_RO(region_dus_size);
442static DEVICE_ATTR_RO(isolate);
443static DEVICE_ATTR_RW(isolate_strategy);
444
445static struct attribute *uacce_dev_attrs[] = {
446	&dev_attr_api.attr,
447	&dev_attr_flags.attr,
448	&dev_attr_available_instances.attr,
449	&dev_attr_algorithms.attr,
450	&dev_attr_region_mmio_size.attr,
451	&dev_attr_region_dus_size.attr,
452	&dev_attr_isolate.attr,
453	&dev_attr_isolate_strategy.attr,
454	NULL,
455};
456
457static umode_t uacce_dev_is_visible(struct kobject *kobj,
458				    struct attribute *attr, int n)
459{
460	struct device *dev = kobj_to_dev(kobj);
461	struct uacce_device *uacce = to_uacce_device(dev);
462
463	if (((attr == &dev_attr_region_mmio_size.attr) &&
464	    (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||
465	    ((attr == &dev_attr_region_dus_size.attr) &&
466	    (!uacce->qf_pg_num[UACCE_QFRT_DUS])))
467		return 0;
468
469	if (attr == &dev_attr_isolate_strategy.attr &&
470	    (!uacce->ops->isolate_err_threshold_read &&
471	     !uacce->ops->isolate_err_threshold_write))
472		return 0;
473
474	if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state)
475		return 0;
476
477	return attr->mode;
478}
479
480static struct attribute_group uacce_dev_group = {
481	.is_visible	= uacce_dev_is_visible,
482	.attrs		= uacce_dev_attrs,
483};
484
485__ATTRIBUTE_GROUPS(uacce_dev);
486
487static void uacce_release(struct device *dev)
488{
489	struct uacce_device *uacce = to_uacce_device(dev);
490
491	kfree(uacce);
492}
493
494/**
495 * uacce_alloc() - alloc an accelerator
496 * @parent: pointer of uacce parent device
497 * @interface: pointer of uacce_interface for register
498 *
499 * Returns uacce pointer if success and ERR_PTR if not
500 * Need check returned negotiated uacce->flags
501 */
502struct uacce_device *uacce_alloc(struct device *parent,
503				 struct uacce_interface *interface)
504{
505	unsigned int flags = interface->flags;
506	struct uacce_device *uacce;
507	int ret;
508
509	uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL);
510	if (!uacce)
511		return ERR_PTR(-ENOMEM);
512
513	uacce->parent = parent;
514	uacce->flags = flags;
515	uacce->ops = interface->ops;
516
517	ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,
518		       GFP_KERNEL);
519	if (ret < 0)
520		goto err_with_uacce;
521
522	INIT_LIST_HEAD(&uacce->queues);
523	mutex_init(&uacce->mutex);
524	device_initialize(&uacce->dev);
525	uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
526	uacce->dev.class = &uacce_class;
527	uacce->dev.groups = uacce_dev_groups;
528	uacce->dev.parent = uacce->parent;
529	uacce->dev.release = uacce_release;
530	dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);
531
532	return uacce;
533
534err_with_uacce:
535	kfree(uacce);
536	return ERR_PTR(ret);
537}
538EXPORT_SYMBOL_GPL(uacce_alloc);
539
540/**
541 * uacce_register() - add the accelerator to cdev and export to user space
542 * @uacce: The initialized uacce device
543 *
544 * Return 0 if register succeeded, or an error.
545 */
546int uacce_register(struct uacce_device *uacce)
547{
548	int ret;
549
550	if (!uacce)
551		return -ENODEV;
552
553	uacce->cdev = cdev_alloc();
554	if (!uacce->cdev)
555		return -ENOMEM;
556
557	uacce->cdev->ops = &uacce_fops;
558	uacce->cdev->owner = THIS_MODULE;
559
560	ret = cdev_device_add(uacce->cdev, &uacce->dev);
561	if (ret)
562		uacce->cdev = NULL;
563
564	return ret;
565}
566EXPORT_SYMBOL_GPL(uacce_register);
567
568/**
569 * uacce_remove() - remove the accelerator
570 * @uacce: the accelerator to remove
571 */
572void uacce_remove(struct uacce_device *uacce)
573{
574	struct uacce_queue *q, *next_q;
575
576	if (!uacce)
577		return;
578
579	/*
580	 * uacce_fops_open() may be running concurrently, even after we remove
581	 * the cdev. Holding uacce->mutex ensures that open() does not obtain a
582	 * removed uacce device.
583	 */
584	mutex_lock(&uacce->mutex);
585	/* ensure no open queue remains */
586	list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
587		/*
588		 * Taking q->mutex ensures that fops do not use the defunct
589		 * uacce->ops after the queue is disabled.
590		 */
591		mutex_lock(&q->mutex);
592		uacce_put_queue(q);
593		mutex_unlock(&q->mutex);
594		uacce_unbind_queue(q);
595
596		/*
597		 * unmap remaining mapping from user space, preventing user still
598		 * access the mmaped area while parent device is already removed
599		 */
600		unmap_mapping_range(q->mapping, 0, 0, 1);
601	}
602
603	if (uacce->cdev)
604		cdev_device_del(uacce->cdev, &uacce->dev);
605	xa_erase(&uacce_xa, uacce->dev_id);
606	/*
607	 * uacce exists as long as there are open fds, but ops will be freed
608	 * now. Ensure that bugs cause NULL deref rather than use-after-free.
609	 */
610	uacce->ops = NULL;
611	uacce->parent = NULL;
612	mutex_unlock(&uacce->mutex);
613	put_device(&uacce->dev);
614}
615EXPORT_SYMBOL_GPL(uacce_remove);
616
617static int __init uacce_init(void)
618{
619	int ret;
620
621	ret = class_register(&uacce_class);
622	if (ret)
623		return ret;
624
625	ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);
626	if (ret)
627		class_unregister(&uacce_class);
628
629	return ret;
630}
631
632static __exit void uacce_exit(void)
633{
634	unregister_chrdev_region(uacce_devt, MINORMASK);
635	class_unregister(&uacce_class);
636}
637
638subsys_initcall(uacce_init);
639module_exit(uacce_exit);
640
641MODULE_LICENSE("GPL");
642MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");
643MODULE_DESCRIPTION("Accelerator interface for Userland applications");