udmabuf.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/cred.h>
  3#include <linux/device.h>
  4#include <linux/dma-buf.h>
  5#include <linux/dma-resv.h>
  6#include <linux/highmem.h>
  7#include <linux/init.h>
  8#include <linux/kernel.h>
  9#include <linux/memfd.h>
 10#include <linux/miscdevice.h>
 11#include <linux/module.h>
 12#include <linux/shmem_fs.h>
 13#include <linux/hugetlb.h>
 14#include <linux/slab.h>
 15#include <linux/udmabuf.h>
 16#include <linux/vmalloc.h>
 17#include <linux/iosys-map.h>
 18
 19static int list_limit = 1024;
 20module_param(list_limit, int, 0644);
 21MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024.");
 22
 23static int size_limit_mb = 64;
 24module_param(size_limit_mb, int, 0644);
 25MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64.");
 26
 27struct udmabuf {
 28	pgoff_t pagecount;
 29	struct folio **folios;
 30
 31	/**
 32	 * Unlike folios, pinned_folios is only used for unpin.
 33	 * So, nr_pinned is not the same to pagecount, the pinned_folios
 34	 * only set each folio which already pinned when udmabuf_create.
 35	 * Note that, since a folio may be pinned multiple times, each folio
 36	 * can be added to pinned_folios multiple times, depending on how many
 37	 * times the folio has been pinned when create.
 38	 */
 39	pgoff_t nr_pinned;
 40	struct folio **pinned_folios;
 41
 42	struct sg_table *sg;
 43	struct miscdevice *device;
 44	pgoff_t *offsets;
 45};
 46
 47static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
 48{
 49	struct vm_area_struct *vma = vmf->vma;
 50	struct udmabuf *ubuf = vma->vm_private_data;
 51	pgoff_t pgoff = vmf->pgoff;
 52	unsigned long addr, pfn;
 53	vm_fault_t ret;
 54
 55	if (pgoff >= ubuf->pagecount)
 56		return VM_FAULT_SIGBUS;
 57
 58	pfn = folio_pfn(ubuf->folios[pgoff]);
 59	pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT;
 60
 61	ret = vmf_insert_pfn(vma, vmf->address, pfn);
 62	if (ret & VM_FAULT_ERROR)
 63		return ret;
 64
 65	/* pre fault */
 66	pgoff = vma->vm_pgoff;
 67	addr = vma->vm_start;
 68
 69	for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) {
 70		if (addr == vmf->address)
 71			continue;
 72
 73		if (WARN_ON(pgoff >= ubuf->pagecount))
 74			break;
 75
 76		pfn = folio_pfn(ubuf->folios[pgoff]);
 77		pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT;
 78
 79		/**
 80		 * If the below vmf_insert_pfn() fails, we do not return an
 81		 * error here during this pre-fault step. However, an error
 82		 * will be returned if the failure occurs when the addr is
 83		 * truly accessed.
 84		 */
 85		if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR)
 86			break;
 87	}
 88
 89	return ret;
 90}
 91
 92static const struct vm_operations_struct udmabuf_vm_ops = {
 93	.fault = udmabuf_vm_fault,
 94};
 95
 96static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma)
 97{
 98	struct udmabuf *ubuf = buf->priv;
 99
100	if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
101		return -EINVAL;
102
103	vma->vm_ops = &udmabuf_vm_ops;
104	vma->vm_private_data = ubuf;
105	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
106	return 0;
107}
108
109static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)
110{
111	struct udmabuf *ubuf = buf->priv;
112	struct page **pages;
113	void *vaddr;
114	pgoff_t pg;
115
116	dma_resv_assert_held(buf->resv);
117
118	pages = kvmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL);
119	if (!pages)
120		return -ENOMEM;
121
122	for (pg = 0; pg < ubuf->pagecount; pg++)
123		pages[pg] = folio_page(ubuf->folios[pg],
124				       ubuf->offsets[pg] >> PAGE_SHIFT);
125
126	vaddr = vm_map_ram(pages, ubuf->pagecount, -1);
127	kvfree(pages);
128	if (!vaddr)
129		return -EINVAL;
130
131	iosys_map_set_vaddr(map, vaddr);
132	return 0;
133}
134
135static void vunmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)
136{
137	struct udmabuf *ubuf = buf->priv;
138
139	dma_resv_assert_held(buf->resv);
140
141	vm_unmap_ram(map->vaddr, ubuf->pagecount);
142}
143
144static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
145				     enum dma_data_direction direction)
146{
147	struct udmabuf *ubuf = buf->priv;
148	struct sg_table *sg;
149	struct scatterlist *sgl;
150	unsigned int i = 0;
151	int ret;
152
153	sg = kzalloc(sizeof(*sg), GFP_KERNEL);
154	if (!sg)
155		return ERR_PTR(-ENOMEM);
156
157	ret = sg_alloc_table(sg, ubuf->pagecount, GFP_KERNEL);
158	if (ret < 0)
159		goto err_alloc;
160
161	for_each_sg(sg->sgl, sgl, ubuf->pagecount, i)
162		sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE,
163			     ubuf->offsets[i]);
164
165	ret = dma_map_sgtable(dev, sg, direction, 0);
166	if (ret < 0)
167		goto err_map;
168	return sg;
169
170err_map:
171	sg_free_table(sg);
172err_alloc:
173	kfree(sg);
174	return ERR_PTR(ret);
175}
176
177static void put_sg_table(struct device *dev, struct sg_table *sg,
178			 enum dma_data_direction direction)
179{
180	dma_unmap_sgtable(dev, sg, direction, 0);
181	sg_free_table(sg);
182	kfree(sg);
183}
184
185static struct sg_table *map_udmabuf(struct dma_buf_attachment *at,
186				    enum dma_data_direction direction)
187{
188	return get_sg_table(at->dev, at->dmabuf, direction);
189}
190
191static void unmap_udmabuf(struct dma_buf_attachment *at,
192			  struct sg_table *sg,
193			  enum dma_data_direction direction)
194{
195	return put_sg_table(at->dev, sg, direction);
196}
197
198static void unpin_all_folios(struct udmabuf *ubuf)
199{
200	pgoff_t i;
201
202	for (i = 0; i < ubuf->nr_pinned; ++i)
203		unpin_folio(ubuf->pinned_folios[i]);
204
205	kvfree(ubuf->pinned_folios);
206}
207
208static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt)
209{
210	ubuf->folios = kvmalloc_array(pgcnt, sizeof(*ubuf->folios), GFP_KERNEL);
211	if (!ubuf->folios)
212		return -ENOMEM;
213
214	ubuf->offsets = kvcalloc(pgcnt, sizeof(*ubuf->offsets), GFP_KERNEL);
215	if (!ubuf->offsets)
216		return -ENOMEM;
217
218	ubuf->pinned_folios = kvmalloc_array(pgcnt,
219					     sizeof(*ubuf->pinned_folios),
220					     GFP_KERNEL);
221	if (!ubuf->pinned_folios)
222		return -ENOMEM;
223
224	return 0;
225}
226
227static __always_inline void deinit_udmabuf(struct udmabuf *ubuf)
228{
229	unpin_all_folios(ubuf);
230	kvfree(ubuf->offsets);
231	kvfree(ubuf->folios);
232}
233
234static void release_udmabuf(struct dma_buf *buf)
235{
236	struct udmabuf *ubuf = buf->priv;
237	struct device *dev = ubuf->device->this_device;
238
239	if (ubuf->sg)
240		put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);
241
242	deinit_udmabuf(ubuf);
243	kfree(ubuf);
244}
245
246static int begin_cpu_udmabuf(struct dma_buf *buf,
247			     enum dma_data_direction direction)
248{
249	struct udmabuf *ubuf = buf->priv;
250	struct device *dev = ubuf->device->this_device;
251	int ret = 0;
252
253	if (!ubuf->sg) {
254		ubuf->sg = get_sg_table(dev, buf, direction);
255		if (IS_ERR(ubuf->sg)) {
256			ret = PTR_ERR(ubuf->sg);
257			ubuf->sg = NULL;
258		}
259	} else {
260		dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction);
261	}
262
263	return ret;
264}
265
266static int end_cpu_udmabuf(struct dma_buf *buf,
267			   enum dma_data_direction direction)
268{
269	struct udmabuf *ubuf = buf->priv;
270	struct device *dev = ubuf->device->this_device;
271
272	if (!ubuf->sg)
273		return -EINVAL;
274
275	dma_sync_sgtable_for_device(dev, ubuf->sg, direction);
276	return 0;
277}
278
279static const struct dma_buf_ops udmabuf_ops = {
280	.map_dma_buf	   = map_udmabuf,
281	.unmap_dma_buf	   = unmap_udmabuf,
282	.release	   = release_udmabuf,
283	.mmap		   = mmap_udmabuf,
284	.vmap		   = vmap_udmabuf,
285	.vunmap		   = vunmap_udmabuf,
286	.begin_cpu_access  = begin_cpu_udmabuf,
287	.end_cpu_access    = end_cpu_udmabuf,
288};
289
290#define SEALS_WANTED (F_SEAL_SHRINK)
291#define SEALS_DENIED (F_SEAL_WRITE|F_SEAL_FUTURE_WRITE)
292
293static int check_memfd_seals(struct file *memfd)
294{
295	int seals;
296
297	if (!shmem_file(memfd) && !is_file_hugepages(memfd))
298		return -EBADFD;
299
300	seals = memfd_fcntl(memfd, F_GET_SEALS, 0);
301	if (seals == -EINVAL)
302		return -EBADFD;
303
304	if ((seals & SEALS_WANTED) != SEALS_WANTED ||
305	    (seals & SEALS_DENIED) != 0)
306		return -EINVAL;
307
308	return 0;
309}
310
311static struct dma_buf *export_udmabuf(struct udmabuf *ubuf,
312				      struct miscdevice *device)
313{
314	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
315
316	ubuf->device = device;
317	exp_info.ops  = &udmabuf_ops;
318	exp_info.size = ubuf->pagecount << PAGE_SHIFT;
319	exp_info.priv = ubuf;
320	exp_info.flags = O_RDWR;
321
322	return dma_buf_export(&exp_info);
323}
324
325static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd,
326			       loff_t start, loff_t size, struct folio **folios)
327{
328	pgoff_t nr_pinned = ubuf->nr_pinned;
329	pgoff_t upgcnt = ubuf->pagecount;
330	u32 cur_folio, cur_pgcnt;
331	pgoff_t pgoff, pgcnt;
332	long nr_folios;
333	loff_t end;
334
335	pgcnt = size >> PAGE_SHIFT;
336	end = start + (pgcnt << PAGE_SHIFT) - 1;
337	nr_folios = memfd_pin_folios(memfd, start, end, folios, pgcnt, &pgoff);
338	if (nr_folios <= 0)
339		return nr_folios ? nr_folios : -EINVAL;
340
341	cur_pgcnt = 0;
342	for (cur_folio = 0; cur_folio < nr_folios; ++cur_folio) {
343		pgoff_t subpgoff = pgoff;
344		size_t fsize = folio_size(folios[cur_folio]);
345
346		ubuf->pinned_folios[nr_pinned++] = folios[cur_folio];
347
348		for (; subpgoff < fsize; subpgoff += PAGE_SIZE) {
349			ubuf->folios[upgcnt] = folios[cur_folio];
350			ubuf->offsets[upgcnt] = subpgoff;
351			++upgcnt;
352
353			if (++cur_pgcnt >= pgcnt)
354				goto end;
355		}
356
357		/**
358		 * In a given range, only the first subpage of the first folio
359		 * has an offset, that is returned by memfd_pin_folios().
360		 * The first subpages of other folios (in the range) have an
361		 * offset of 0.
362		 */
363		pgoff = 0;
364	}
365end:
366	ubuf->pagecount = upgcnt;
367	ubuf->nr_pinned = nr_pinned;
368	return 0;
369}
370
371static long udmabuf_create(struct miscdevice *device,
372			   struct udmabuf_create_list *head,
373			   struct udmabuf_create_item *list)
374{
375	unsigned long max_nr_folios = 0;
376	struct folio **folios = NULL;
377	pgoff_t pgcnt = 0, pglimit;
378	struct udmabuf *ubuf;
379	struct dma_buf *dmabuf;
380	long ret = -EINVAL;
381	u32 i, flags;
382
383	ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
384	if (!ubuf)
385		return -ENOMEM;
386
387	pglimit = ((u64)size_limit_mb * 1024 * 1024) >> PAGE_SHIFT;
388	for (i = 0; i < head->count; i++) {
389		pgoff_t subpgcnt;
390
391		if (!PAGE_ALIGNED(list[i].offset))
392			goto err_noinit;
393		if (!PAGE_ALIGNED(list[i].size))
394			goto err_noinit;
395
396		subpgcnt = list[i].size >> PAGE_SHIFT;
397		pgcnt += subpgcnt;
398		if (pgcnt > pglimit)
399			goto err_noinit;
400
401		max_nr_folios = max_t(unsigned long, subpgcnt, max_nr_folios);
402	}
403
404	if (!pgcnt)
405		goto err_noinit;
406
407	ret = init_udmabuf(ubuf, pgcnt);
408	if (ret)
409		goto err;
410
411	folios = kvmalloc_array(max_nr_folios, sizeof(*folios), GFP_KERNEL);
412	if (!folios) {
413		ret = -ENOMEM;
414		goto err;
415	}
416
417	for (i = 0; i < head->count; i++) {
418		struct file *memfd = fget(list[i].memfd);
419
420		if (!memfd) {
421			ret = -EBADFD;
422			goto err;
423		}
424
425		/*
426		 * Take the inode lock to protect against concurrent
427		 * memfd_add_seals(), which takes this lock in write mode.
428		 */
429		inode_lock_shared(file_inode(memfd));
430		ret = check_memfd_seals(memfd);
431		if (ret)
432			goto out_unlock;
433
434		ret = udmabuf_pin_folios(ubuf, memfd, list[i].offset,
435					 list[i].size, folios);
436out_unlock:
437		inode_unlock_shared(file_inode(memfd));
438		fput(memfd);
439		if (ret)
440			goto err;
441	}
442
443	flags = head->flags & UDMABUF_FLAGS_CLOEXEC ? O_CLOEXEC : 0;
444	dmabuf = export_udmabuf(ubuf, device);
445	if (IS_ERR(dmabuf)) {
446		ret = PTR_ERR(dmabuf);
447		goto err;
448	}
449	/*
450	 * Ownership of ubuf is held by the dmabuf from here.
451	 * If the following dma_buf_fd() fails, dma_buf_put() cleans up both the
452	 * dmabuf and the ubuf (through udmabuf_ops.release).
453	 */
454
455	ret = dma_buf_fd(dmabuf, flags);
456	if (ret < 0)
457		dma_buf_put(dmabuf);
458
459	kvfree(folios);
460	return ret;
461
462err:
463	deinit_udmabuf(ubuf);
464err_noinit:
465	kfree(ubuf);
466	kvfree(folios);
467	return ret;
468}
469
470static long udmabuf_ioctl_create(struct file *filp, unsigned long arg)
471{
472	struct udmabuf_create create;
473	struct udmabuf_create_list head;
474	struct udmabuf_create_item list;
475
476	if (copy_from_user(&create, (void __user *)arg,
477			   sizeof(create)))
478		return -EFAULT;
479
480	head.flags  = create.flags;
481	head.count  = 1;
482	list.memfd  = create.memfd;
483	list.offset = create.offset;
484	list.size   = create.size;
485
486	return udmabuf_create(filp->private_data, &head, &list);
487}
488
489static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg)
490{
491	struct udmabuf_create_list head;
492	struct udmabuf_create_item *list;
493	int ret = -EINVAL;
494	u32 lsize;
495
496	if (copy_from_user(&head, (void __user *)arg, sizeof(head)))
497		return -EFAULT;
498	if (head.count > list_limit)
499		return -EINVAL;
500	lsize = sizeof(struct udmabuf_create_item) * head.count;
501	list = memdup_user((void __user *)(arg + sizeof(head)), lsize);
502	if (IS_ERR(list))
503		return PTR_ERR(list);
504
505	ret = udmabuf_create(filp->private_data, &head, list);
506	kfree(list);
507	return ret;
508}
509
510static long udmabuf_ioctl(struct file *filp, unsigned int ioctl,
511			  unsigned long arg)
512{
513	long ret;
514
515	switch (ioctl) {
516	case UDMABUF_CREATE:
517		ret = udmabuf_ioctl_create(filp, arg);
518		break;
519	case UDMABUF_CREATE_LIST:
520		ret = udmabuf_ioctl_create_list(filp, arg);
521		break;
522	default:
523		ret = -ENOTTY;
524		break;
525	}
526	return ret;
527}
528
529static const struct file_operations udmabuf_fops = {
530	.owner		= THIS_MODULE,
531	.unlocked_ioctl = udmabuf_ioctl,
532#ifdef CONFIG_COMPAT
533	.compat_ioctl   = udmabuf_ioctl,
534#endif
535};
536
537static struct miscdevice udmabuf_misc = {
538	.minor          = MISC_DYNAMIC_MINOR,
539	.name           = "udmabuf",
540	.fops           = &udmabuf_fops,
541};
542
543static int __init udmabuf_dev_init(void)
544{
545	int ret;
546
547	ret = misc_register(&udmabuf_misc);
548	if (ret < 0) {
549		pr_err("Could not initialize udmabuf device\n");
550		return ret;
551	}
552
553	ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device,
554					   DMA_BIT_MASK(64));
555	if (ret < 0) {
556		pr_err("Could not setup DMA mask for udmabuf device\n");
557		misc_deregister(&udmabuf_misc);
558		return ret;
559	}
560
561	return 0;
562}
563
564static void __exit udmabuf_dev_exit(void)
565{
566	misc_deregister(&udmabuf_misc);
567}
568
569module_init(udmabuf_dev_init)
570module_exit(udmabuf_dev_exit)
571
572MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>");