dma-buf.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Framework for buffer objects that can be shared across devices/subsystems.
   4 *
   5 * Copyright(C) 2011 Linaro Limited. All rights reserved.
   6 * Author: Sumit Semwal <sumit.semwal@ti.com>
   7 *
   8 * Many thanks to linaro-mm-sig list, and specially
   9 * Arnd Bergmann <arnd@arndb.de>, Rob Clark <rob@ti.com> and
  10 * Daniel Vetter <daniel@ffwll.ch> for their support in creation and
  11 * refining of this idea.
  12 */
  13
  14#include <linux/fs.h>
  15#include <linux/slab.h>
  16#include <linux/dma-buf.h>
  17#include <linux/dma-fence.h>
  18#include <linux/dma-fence-unwrap.h>
  19#include <linux/anon_inodes.h>
  20#include <linux/export.h>
  21#include <linux/debugfs.h>
  22#include <linux/list.h>
  23#include <linux/module.h>
  24#include <linux/mutex.h>
  25#include <linux/seq_file.h>
  26#include <linux/sync_file.h>
  27#include <linux/poll.h>
  28#include <linux/dma-resv.h>
  29#include <linux/mm.h>
  30#include <linux/mount.h>
  31#include <linux/pseudo_fs.h>
  32
  33#include <uapi/linux/dma-buf.h>
  34#include <uapi/linux/magic.h>
  35
  36#include "dma-buf-sysfs-stats.h"
  37
  38static inline int is_dma_buf_file(struct file *);
  39
  40static DEFINE_MUTEX(dmabuf_list_mutex);
  41static LIST_HEAD(dmabuf_list);
  42
  43static void __dma_buf_list_add(struct dma_buf *dmabuf)
  44{
  45	mutex_lock(&dmabuf_list_mutex);
  46	list_add(&dmabuf->list_node, &dmabuf_list);
  47	mutex_unlock(&dmabuf_list_mutex);
  48}
  49
  50static void __dma_buf_list_del(struct dma_buf *dmabuf)
  51{
  52	if (!dmabuf)
  53		return;
  54
  55	mutex_lock(&dmabuf_list_mutex);
  56	list_del(&dmabuf->list_node);
  57	mutex_unlock(&dmabuf_list_mutex);
  58}
  59
  60/**
  61 * dma_buf_iter_begin - begin iteration through global list of all DMA buffers
  62 *
  63 * Returns the first buffer in the global list of DMA-bufs that's not in the
  64 * process of being destroyed. Increments that buffer's reference count to
  65 * prevent buffer destruction. Callers must release the reference, either by
  66 * continuing iteration with dma_buf_iter_next(), or with dma_buf_put().
  67 *
  68 * Return:
  69 * * First buffer from global list, with refcount elevated
  70 * * NULL if no active buffers are present
  71 */
  72struct dma_buf *dma_buf_iter_begin(void)
  73{
  74	struct dma_buf *ret = NULL, *dmabuf;
  75
  76	/*
  77	 * The list mutex does not protect a dmabuf's refcount, so it can be
  78	 * zeroed while we are iterating. We cannot call get_dma_buf() since the
  79	 * caller may not already own a reference to the buffer.
  80	 */
  81	mutex_lock(&dmabuf_list_mutex);
  82	list_for_each_entry(dmabuf, &dmabuf_list, list_node) {
  83		if (file_ref_get(&dmabuf->file->f_ref)) {
  84			ret = dmabuf;
  85			break;
  86		}
  87	}
  88	mutex_unlock(&dmabuf_list_mutex);
  89	return ret;
  90}
  91
  92/**
  93 * dma_buf_iter_next - continue iteration through global list of all DMA buffers
  94 * @dmabuf:	[in]	pointer to dma_buf
  95 *
  96 * Decrements the reference count on the provided buffer. Returns the next
  97 * buffer from the remainder of the global list of DMA-bufs with its reference
  98 * count incremented. Callers must release the reference, either by continuing
  99 * iteration with dma_buf_iter_next(), or with dma_buf_put().
 100 *
 101 * Return:
 102 * * Next buffer from global list, with refcount elevated
 103 * * NULL if no additional active buffers are present
 104 */
 105struct dma_buf *dma_buf_iter_next(struct dma_buf *dmabuf)
 106{
 107	struct dma_buf *ret = NULL;
 108
 109	/*
 110	 * The list mutex does not protect a dmabuf's refcount, so it can be
 111	 * zeroed while we are iterating. We cannot call get_dma_buf() since the
 112	 * caller may not already own a reference to the buffer.
 113	 */
 114	mutex_lock(&dmabuf_list_mutex);
 115	dma_buf_put(dmabuf);
 116	list_for_each_entry_continue(dmabuf, &dmabuf_list, list_node) {
 117		if (file_ref_get(&dmabuf->file->f_ref)) {
 118			ret = dmabuf;
 119			break;
 120		}
 121	}
 122	mutex_unlock(&dmabuf_list_mutex);
 123	return ret;
 124}
 125
 126static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
 127{
 128	struct dma_buf *dmabuf;
 129	char name[DMA_BUF_NAME_LEN];
 130	ssize_t ret = 0;
 131
 132	dmabuf = dentry->d_fsdata;
 133	spin_lock(&dmabuf->name_lock);
 134	if (dmabuf->name)
 135		ret = strscpy(name, dmabuf->name, sizeof(name));
 136	spin_unlock(&dmabuf->name_lock);
 137
 138	return dynamic_dname(buffer, buflen, "/%s:%s",
 139			     dentry->d_name.name, ret > 0 ? name : "");
 140}
 141
 142static void dma_buf_release(struct dentry *dentry)
 143{
 144	struct dma_buf *dmabuf;
 145
 146	dmabuf = dentry->d_fsdata;
 147	if (unlikely(!dmabuf))
 148		return;
 149
 150	BUG_ON(dmabuf->vmapping_counter);
 151
 152	/*
 153	 * If you hit this BUG() it could mean:
 154	 * * There's a file reference imbalance in dma_buf_poll / dma_buf_poll_cb or somewhere else
 155	 * * dmabuf->cb_in/out.active are non-0 despite no pending fence callback
 156	 */
 157	BUG_ON(dmabuf->cb_in.active || dmabuf->cb_out.active);
 158
 159	dma_buf_stats_teardown(dmabuf);
 160	dmabuf->ops->release(dmabuf);
 161
 162	if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])
 163		dma_resv_fini(dmabuf->resv);
 164
 165	WARN_ON(!list_empty(&dmabuf->attachments));
 166	module_put(dmabuf->owner);
 167	kfree(dmabuf->name);
 168	kfree(dmabuf);
 169}
 170
 171static int dma_buf_file_release(struct inode *inode, struct file *file)
 172{
 173	if (!is_dma_buf_file(file))
 174		return -EINVAL;
 175
 176	__dma_buf_list_del(file->private_data);
 177
 178	return 0;
 179}
 180
 181static const struct dentry_operations dma_buf_dentry_ops = {
 182	.d_dname = dmabuffs_dname,
 183	.d_release = dma_buf_release,
 184};
 185
 186static struct vfsmount *dma_buf_mnt;
 187
 188static int dma_buf_fs_init_context(struct fs_context *fc)
 189{
 190	struct pseudo_fs_context *ctx;
 191
 192	ctx = init_pseudo(fc, DMA_BUF_MAGIC);
 193	if (!ctx)
 194		return -ENOMEM;
 195	ctx->dops = &dma_buf_dentry_ops;
 196	return 0;
 197}
 198
 199static struct file_system_type dma_buf_fs_type = {
 200	.name = "dmabuf",
 201	.init_fs_context = dma_buf_fs_init_context,
 202	.kill_sb = kill_anon_super,
 203};
 204
 205static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
 206{
 207	struct dma_buf *dmabuf;
 208
 209	if (!is_dma_buf_file(file))
 210		return -EINVAL;
 211
 212	dmabuf = file->private_data;
 213
 214	/* check if buffer supports mmap */
 215	if (!dmabuf->ops->mmap)
 216		return -EINVAL;
 217
 218	/* check for overflowing the buffer's size */
 219	if (vma->vm_pgoff + vma_pages(vma) >
 220	    dmabuf->size >> PAGE_SHIFT)
 221		return -EINVAL;
 222
 223	return dmabuf->ops->mmap(dmabuf, vma);
 224}
 225
 226static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
 227{
 228	struct dma_buf *dmabuf;
 229	loff_t base;
 230
 231	if (!is_dma_buf_file(file))
 232		return -EBADF;
 233
 234	dmabuf = file->private_data;
 235
 236	/* only support discovering the end of the buffer,
 237	 * but also allow SEEK_SET to maintain the idiomatic
 238	 * SEEK_END(0), SEEK_CUR(0) pattern.
 239	 */
 240	if (whence == SEEK_END)
 241		base = dmabuf->size;
 242	else if (whence == SEEK_SET)
 243		base = 0;
 244	else
 245		return -EINVAL;
 246
 247	if (offset != 0)
 248		return -EINVAL;
 249
 250	return base + offset;
 251}
 252
 253/**
 254 * DOC: implicit fence polling
 255 *
 256 * To support cross-device and cross-driver synchronization of buffer access
 257 * implicit fences (represented internally in the kernel with &struct dma_fence)
 258 * can be attached to a &dma_buf. The glue for that and a few related things are
 259 * provided in the &dma_resv structure.
 260 *
 261 * Userspace can query the state of these implicitly tracked fences using poll()
 262 * and related system calls:
 263 *
 264 * - Checking for EPOLLIN, i.e. read access, can be use to query the state of the
 265 *   most recent write or exclusive fence.
 266 *
 267 * - Checking for EPOLLOUT, i.e. write access, can be used to query the state of
 268 *   all attached fences, shared and exclusive ones.
 269 *
 270 * Note that this only signals the completion of the respective fences, i.e. the
 271 * DMA transfers are complete. Cache flushing and any other necessary
 272 * preparations before CPU access can begin still need to happen.
 273 *
 274 * As an alternative to poll(), the set of fences on DMA buffer can be
 275 * exported as a &sync_file using &dma_buf_sync_file_export.
 276 */
 277
 278static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 279{
 280	struct dma_buf_poll_cb_t *dcb = (struct dma_buf_poll_cb_t *)cb;
 281	struct dma_buf *dmabuf = container_of(dcb->poll, struct dma_buf, poll);
 282	unsigned long flags;
 283
 284	spin_lock_irqsave(&dcb->poll->lock, flags);
 285	wake_up_locked_poll(dcb->poll, dcb->active);
 286	dcb->active = 0;
 287	spin_unlock_irqrestore(&dcb->poll->lock, flags);
 288	dma_fence_put(fence);
 289	/* Paired with get_file in dma_buf_poll */
 290	fput(dmabuf->file);
 291}
 292
 293static bool dma_buf_poll_add_cb(struct dma_resv *resv, bool write,
 294				struct dma_buf_poll_cb_t *dcb)
 295{
 296	struct dma_resv_iter cursor;
 297	struct dma_fence *fence;
 298	int r;
 299
 300	dma_resv_for_each_fence(&cursor, resv, dma_resv_usage_rw(write),
 301				fence) {
 302		dma_fence_get(fence);
 303		r = dma_fence_add_callback(fence, &dcb->cb, dma_buf_poll_cb);
 304		if (!r)
 305			return true;
 306		dma_fence_put(fence);
 307	}
 308
 309	return false;
 310}
 311
 312static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
 313{
 314	struct dma_buf *dmabuf;
 315	struct dma_resv *resv;
 316	__poll_t events;
 317
 318	dmabuf = file->private_data;
 319	if (!dmabuf || !dmabuf->resv)
 320		return EPOLLERR;
 321
 322	resv = dmabuf->resv;
 323
 324	poll_wait(file, &dmabuf->poll, poll);
 325
 326	events = poll_requested_events(poll) & (EPOLLIN | EPOLLOUT);
 327	if (!events)
 328		return 0;
 329
 330	dma_resv_lock(resv, NULL);
 331
 332	if (events & EPOLLOUT) {
 333		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_out;
 334
 335		/* Check that callback isn't busy */
 336		spin_lock_irq(&dmabuf->poll.lock);
 337		if (dcb->active)
 338			events &= ~EPOLLOUT;
 339		else
 340			dcb->active = EPOLLOUT;
 341		spin_unlock_irq(&dmabuf->poll.lock);
 342
 343		if (events & EPOLLOUT) {
 344			/* Paired with fput in dma_buf_poll_cb */
 345			get_file(dmabuf->file);
 346
 347			if (!dma_buf_poll_add_cb(resv, true, dcb))
 348				/* No callback queued, wake up any other waiters */
 349				dma_buf_poll_cb(NULL, &dcb->cb);
 350			else
 351				events &= ~EPOLLOUT;
 352		}
 353	}
 354
 355	if (events & EPOLLIN) {
 356		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_in;
 357
 358		/* Check that callback isn't busy */
 359		spin_lock_irq(&dmabuf->poll.lock);
 360		if (dcb->active)
 361			events &= ~EPOLLIN;
 362		else
 363			dcb->active = EPOLLIN;
 364		spin_unlock_irq(&dmabuf->poll.lock);
 365
 366		if (events & EPOLLIN) {
 367			/* Paired with fput in dma_buf_poll_cb */
 368			get_file(dmabuf->file);
 369
 370			if (!dma_buf_poll_add_cb(resv, false, dcb))
 371				/* No callback queued, wake up any other waiters */
 372				dma_buf_poll_cb(NULL, &dcb->cb);
 373			else
 374				events &= ~EPOLLIN;
 375		}
 376	}
 377
 378	dma_resv_unlock(resv);
 379	return events;
 380}
 381
 382/**
 383 * dma_buf_set_name - Set a name to a specific dma_buf to track the usage.
 384 * It could support changing the name of the dma-buf if the same
 385 * piece of memory is used for multiple purpose between different devices.
 386 *
 387 * @dmabuf: [in]     dmabuf buffer that will be renamed.
 388 * @buf:    [in]     A piece of userspace memory that contains the name of
 389 *                   the dma-buf.
 390 *
 391 * Returns 0 on success. If the dma-buf buffer is already attached to
 392 * devices, return -EBUSY.
 393 *
 394 */
 395static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf)
 396{
 397	char *name = strndup_user(buf, DMA_BUF_NAME_LEN);
 398
 399	if (IS_ERR(name))
 400		return PTR_ERR(name);
 401
 402	spin_lock(&dmabuf->name_lock);
 403	kfree(dmabuf->name);
 404	dmabuf->name = name;
 405	spin_unlock(&dmabuf->name_lock);
 406
 407	return 0;
 408}
 409
 410#if IS_ENABLED(CONFIG_SYNC_FILE)
 411static long dma_buf_export_sync_file(struct dma_buf *dmabuf,
 412				     void __user *user_data)
 413{
 414	struct dma_buf_export_sync_file arg;
 415	enum dma_resv_usage usage;
 416	struct dma_fence *fence = NULL;
 417	struct sync_file *sync_file;
 418	int fd, ret;
 419
 420	if (copy_from_user(&arg, user_data, sizeof(arg)))
 421		return -EFAULT;
 422
 423	if (arg.flags & ~DMA_BUF_SYNC_RW)
 424		return -EINVAL;
 425
 426	if ((arg.flags & DMA_BUF_SYNC_RW) == 0)
 427		return -EINVAL;
 428
 429	fd = get_unused_fd_flags(O_CLOEXEC);
 430	if (fd < 0)
 431		return fd;
 432
 433	usage = dma_resv_usage_rw(arg.flags & DMA_BUF_SYNC_WRITE);
 434	ret = dma_resv_get_singleton(dmabuf->resv, usage, &fence);
 435	if (ret)
 436		goto err_put_fd;
 437
 438	if (!fence)
 439		fence = dma_fence_get_stub();
 440
 441	sync_file = sync_file_create(fence);
 442
 443	dma_fence_put(fence);
 444
 445	if (!sync_file) {
 446		ret = -ENOMEM;
 447		goto err_put_fd;
 448	}
 449
 450	arg.fd = fd;
 451	if (copy_to_user(user_data, &arg, sizeof(arg))) {
 452		ret = -EFAULT;
 453		goto err_put_file;
 454	}
 455
 456	fd_install(fd, sync_file->file);
 457
 458	return 0;
 459
 460err_put_file:
 461	fput(sync_file->file);
 462err_put_fd:
 463	put_unused_fd(fd);
 464	return ret;
 465}
 466
 467static long dma_buf_import_sync_file(struct dma_buf *dmabuf,
 468				     const void __user *user_data)
 469{
 470	struct dma_buf_import_sync_file arg;
 471	struct dma_fence *fence, *f;
 472	enum dma_resv_usage usage;
 473	struct dma_fence_unwrap iter;
 474	unsigned int num_fences;
 475	int ret = 0;
 476
 477	if (copy_from_user(&arg, user_data, sizeof(arg)))
 478		return -EFAULT;
 479
 480	if (arg.flags & ~DMA_BUF_SYNC_RW)
 481		return -EINVAL;
 482
 483	if ((arg.flags & DMA_BUF_SYNC_RW) == 0)
 484		return -EINVAL;
 485
 486	fence = sync_file_get_fence(arg.fd);
 487	if (!fence)
 488		return -EINVAL;
 489
 490	usage = (arg.flags & DMA_BUF_SYNC_WRITE) ? DMA_RESV_USAGE_WRITE :
 491						   DMA_RESV_USAGE_READ;
 492
 493	num_fences = 0;
 494	dma_fence_unwrap_for_each(f, &iter, fence)
 495		++num_fences;
 496
 497	if (num_fences > 0) {
 498		dma_resv_lock(dmabuf->resv, NULL);
 499
 500		ret = dma_resv_reserve_fences(dmabuf->resv, num_fences);
 501		if (!ret) {
 502			dma_fence_unwrap_for_each(f, &iter, fence)
 503				dma_resv_add_fence(dmabuf->resv, f, usage);
 504		}
 505
 506		dma_resv_unlock(dmabuf->resv);
 507	}
 508
 509	dma_fence_put(fence);
 510
 511	return ret;
 512}
 513#endif
 514
 515static long dma_buf_ioctl(struct file *file,
 516			  unsigned int cmd, unsigned long arg)
 517{
 518	struct dma_buf *dmabuf;
 519	struct dma_buf_sync sync;
 520	enum dma_data_direction direction;
 521	int ret;
 522
 523	dmabuf = file->private_data;
 524
 525	switch (cmd) {
 526	case DMA_BUF_IOCTL_SYNC:
 527		if (copy_from_user(&sync, (void __user *) arg, sizeof(sync)))
 528			return -EFAULT;
 529
 530		if (sync.flags & ~DMA_BUF_SYNC_VALID_FLAGS_MASK)
 531			return -EINVAL;
 532
 533		switch (sync.flags & DMA_BUF_SYNC_RW) {
 534		case DMA_BUF_SYNC_READ:
 535			direction = DMA_FROM_DEVICE;
 536			break;
 537		case DMA_BUF_SYNC_WRITE:
 538			direction = DMA_TO_DEVICE;
 539			break;
 540		case DMA_BUF_SYNC_RW:
 541			direction = DMA_BIDIRECTIONAL;
 542			break;
 543		default:
 544			return -EINVAL;
 545		}
 546
 547		if (sync.flags & DMA_BUF_SYNC_END)
 548			ret = dma_buf_end_cpu_access(dmabuf, direction);
 549		else
 550			ret = dma_buf_begin_cpu_access(dmabuf, direction);
 551
 552		return ret;
 553
 554	case DMA_BUF_SET_NAME_A:
 555	case DMA_BUF_SET_NAME_B:
 556		return dma_buf_set_name(dmabuf, (const char __user *)arg);
 557
 558#if IS_ENABLED(CONFIG_SYNC_FILE)
 559	case DMA_BUF_IOCTL_EXPORT_SYNC_FILE:
 560		return dma_buf_export_sync_file(dmabuf, (void __user *)arg);
 561	case DMA_BUF_IOCTL_IMPORT_SYNC_FILE:
 562		return dma_buf_import_sync_file(dmabuf, (const void __user *)arg);
 563#endif
 564
 565	default:
 566		return -ENOTTY;
 567	}
 568}
 569
 570static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file)
 571{
 572	struct dma_buf *dmabuf = file->private_data;
 573
 574	seq_printf(m, "size:\t%zu\n", dmabuf->size);
 575	/* Don't count the temporary reference taken inside procfs seq_show */
 576	seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1);
 577	seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name);
 578	spin_lock(&dmabuf->name_lock);
 579	if (dmabuf->name)
 580		seq_printf(m, "name:\t%s\n", dmabuf->name);
 581	spin_unlock(&dmabuf->name_lock);
 582}
 583
 584static const struct file_operations dma_buf_fops = {
 585	.release	= dma_buf_file_release,
 586	.mmap		= dma_buf_mmap_internal,
 587	.llseek		= dma_buf_llseek,
 588	.poll		= dma_buf_poll,
 589	.unlocked_ioctl	= dma_buf_ioctl,
 590	.compat_ioctl	= compat_ptr_ioctl,
 591	.show_fdinfo	= dma_buf_show_fdinfo,
 592};
 593
 594/*
 595 * is_dma_buf_file - Check if struct file* is associated with dma_buf
 596 */
 597static inline int is_dma_buf_file(struct file *file)
 598{
 599	return file->f_op == &dma_buf_fops;
 600}
 601
 602static struct file *dma_buf_getfile(size_t size, int flags)
 603{
 604	static atomic64_t dmabuf_inode = ATOMIC64_INIT(0);
 605	struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb);
 606	struct file *file;
 607
 608	if (IS_ERR(inode))
 609		return ERR_CAST(inode);
 610
 611	inode->i_size = size;
 612	inode_set_bytes(inode, size);
 613
 614	/*
 615	 * The ->i_ino acquired from get_next_ino() is not unique thus
 616	 * not suitable for using it as dentry name by dmabuf stats.
 617	 * Override ->i_ino with the unique and dmabuffs specific
 618	 * value.
 619	 */
 620	inode->i_ino = atomic64_inc_return(&dmabuf_inode);
 621	flags &= O_ACCMODE | O_NONBLOCK;
 622	file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
 623				 flags, &dma_buf_fops);
 624	if (IS_ERR(file))
 625		goto err_alloc_file;
 626
 627	return file;
 628
 629err_alloc_file:
 630	iput(inode);
 631	return file;
 632}
 633
 634/**
 635 * DOC: dma buf device access
 636 *
 637 * For device DMA access to a shared DMA buffer the usual sequence of operations
 638 * is fairly simple:
 639 *
 640 * 1. The exporter defines his exporter instance using
 641 *    DEFINE_DMA_BUF_EXPORT_INFO() and calls dma_buf_export() to wrap a private
 642 *    buffer object into a &dma_buf. It then exports that &dma_buf to userspace
 643 *    as a file descriptor by calling dma_buf_fd().
 644 *
 645 * 2. Userspace passes this file-descriptors to all drivers it wants this buffer
 646 *    to share with: First the file descriptor is converted to a &dma_buf using
 647 *    dma_buf_get(). Then the buffer is attached to the device using
 648 *    dma_buf_attach().
 649 *
 650 *    Up to this stage the exporter is still free to migrate or reallocate the
 651 *    backing storage.
 652 *
 653 * 3. Once the buffer is attached to all devices userspace can initiate DMA
 654 *    access to the shared buffer. In the kernel this is done by calling
 655 *    dma_buf_map_attachment() and dma_buf_unmap_attachment().
 656 *
 657 * 4. Once a driver is done with a shared buffer it needs to call
 658 *    dma_buf_detach() (after cleaning up any mappings) and then release the
 659 *    reference acquired with dma_buf_get() by calling dma_buf_put().
 660 *
 661 * For the detailed semantics exporters are expected to implement see
 662 * &dma_buf_ops.
 663 */
 664
 665/**
 666 * dma_buf_export - Creates a new dma_buf, and associates an anon file
 667 * with this buffer, so it can be exported.
 668 * Also connect the allocator specific data and ops to the buffer.
 669 * Additionally, provide a name string for exporter; useful in debugging.
 670 *
 671 * @exp_info:	[in]	holds all the export related information provided
 672 *			by the exporter. see &struct dma_buf_export_info
 673 *			for further details.
 674 *
 675 * Returns, on success, a newly created struct dma_buf object, which wraps the
 676 * supplied private data and operations for struct dma_buf_ops. On either
 677 * missing ops, or error in allocating struct dma_buf, will return negative
 678 * error.
 679 *
 680 * For most cases the easiest way to create @exp_info is through the
 681 * %DEFINE_DMA_BUF_EXPORT_INFO macro.
 682 */
 683struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
 684{
 685	struct dma_buf *dmabuf;
 686	struct dma_resv *resv = exp_info->resv;
 687	struct file *file;
 688	size_t alloc_size = sizeof(struct dma_buf);
 689	int ret;
 690
 691	if (WARN_ON(!exp_info->priv || !exp_info->ops
 692		    || !exp_info->ops->map_dma_buf
 693		    || !exp_info->ops->unmap_dma_buf
 694		    || !exp_info->ops->release))
 695		return ERR_PTR(-EINVAL);
 696
 697	if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin))
 698		return ERR_PTR(-EINVAL);
 699
 700	if (!try_module_get(exp_info->owner))
 701		return ERR_PTR(-ENOENT);
 702
 703	file = dma_buf_getfile(exp_info->size, exp_info->flags);
 704	if (IS_ERR(file)) {
 705		ret = PTR_ERR(file);
 706		goto err_module;
 707	}
 708
 709	if (!exp_info->resv)
 710		alloc_size += sizeof(struct dma_resv);
 711	else
 712		/* prevent &dma_buf[1] == dma_buf->resv */
 713		alloc_size += 1;
 714	dmabuf = kzalloc(alloc_size, GFP_KERNEL);
 715	if (!dmabuf) {
 716		ret = -ENOMEM;
 717		goto err_file;
 718	}
 719
 720	dmabuf->priv = exp_info->priv;
 721	dmabuf->ops = exp_info->ops;
 722	dmabuf->size = exp_info->size;
 723	dmabuf->exp_name = exp_info->exp_name;
 724	dmabuf->owner = exp_info->owner;
 725	spin_lock_init(&dmabuf->name_lock);
 726	init_waitqueue_head(&dmabuf->poll);
 727	dmabuf->cb_in.poll = dmabuf->cb_out.poll = &dmabuf->poll;
 728	dmabuf->cb_in.active = dmabuf->cb_out.active = 0;
 729	INIT_LIST_HEAD(&dmabuf->attachments);
 730
 731	if (!resv) {
 732		dmabuf->resv = (struct dma_resv *)&dmabuf[1];
 733		dma_resv_init(dmabuf->resv);
 734	} else {
 735		dmabuf->resv = resv;
 736	}
 737
 738	ret = dma_buf_stats_setup(dmabuf, file);
 739	if (ret)
 740		goto err_dmabuf;
 741
 742	file->private_data = dmabuf;
 743	file->f_path.dentry->d_fsdata = dmabuf;
 744	dmabuf->file = file;
 745
 746	__dma_buf_list_add(dmabuf);
 747
 748	return dmabuf;
 749
 750err_dmabuf:
 751	if (!resv)
 752		dma_resv_fini(dmabuf->resv);
 753	kfree(dmabuf);
 754err_file:
 755	fput(file);
 756err_module:
 757	module_put(exp_info->owner);
 758	return ERR_PTR(ret);
 759}
 760EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF");
 761
 762/**
 763 * dma_buf_fd - returns a file descriptor for the given struct dma_buf
 764 * @dmabuf:	[in]	pointer to dma_buf for which fd is required.
 765 * @flags:      [in]    flags to give to fd
 766 *
 767 * On success, returns an associated 'fd'. Else, returns error.
 768 */
 769int dma_buf_fd(struct dma_buf *dmabuf, int flags)
 770{
 771	if (!dmabuf || !dmabuf->file)
 772		return -EINVAL;
 773
 774	return FD_ADD(flags, dmabuf->file);
 775}
 776EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF");
 777
 778/**
 779 * dma_buf_get - returns the struct dma_buf related to an fd
 780 * @fd:	[in]	fd associated with the struct dma_buf to be returned
 781 *
 782 * On success, returns the struct dma_buf associated with an fd; uses
 783 * file's refcounting done by fget to increase refcount. returns ERR_PTR
 784 * otherwise.
 785 */
 786struct dma_buf *dma_buf_get(int fd)
 787{
 788	struct file *file;
 789
 790	file = fget(fd);
 791
 792	if (!file)
 793		return ERR_PTR(-EBADF);
 794
 795	if (!is_dma_buf_file(file)) {
 796		fput(file);
 797		return ERR_PTR(-EINVAL);
 798	}
 799
 800	return file->private_data;
 801}
 802EXPORT_SYMBOL_NS_GPL(dma_buf_get, "DMA_BUF");
 803
 804/**
 805 * dma_buf_put - decreases refcount of the buffer
 806 * @dmabuf:	[in]	buffer to reduce refcount of
 807 *
 808 * Uses file's refcounting done implicitly by fput().
 809 *
 810 * If, as a result of this call, the refcount becomes 0, the 'release' file
 811 * operation related to this fd is called. It calls &dma_buf_ops.release vfunc
 812 * in turn, and frees the memory allocated for dmabuf when exported.
 813 */
 814void dma_buf_put(struct dma_buf *dmabuf)
 815{
 816	if (WARN_ON(!dmabuf || !dmabuf->file))
 817		return;
 818
 819	fput(dmabuf->file);
 820}
 821EXPORT_SYMBOL_NS_GPL(dma_buf_put, "DMA_BUF");
 822
 823static void mangle_sg_table(struct sg_table *sg_table)
 824{
 825#ifdef CONFIG_DMABUF_DEBUG
 826	int i;
 827	struct scatterlist *sg;
 828
 829	/* To catch abuse of the underlying struct page by importers mix
 830	 * up the bits, but take care to preserve the low SG_ bits to
 831	 * not corrupt the sgt. The mixing is undone on unmap
 832	 * before passing the sgt back to the exporter.
 833	 */
 834	for_each_sgtable_sg(sg_table, sg, i)
 835		sg->page_link ^= ~0xffUL;
 836#endif
 837
 838}
 839
 840static inline bool
 841dma_buf_attachment_is_dynamic(struct dma_buf_attachment *attach)
 842{
 843	return !!attach->importer_ops;
 844}
 845
 846static bool
 847dma_buf_pin_on_map(struct dma_buf_attachment *attach)
 848{
 849	return attach->dmabuf->ops->pin &&
 850		(!dma_buf_attachment_is_dynamic(attach) ||
 851		 !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY));
 852}
 853
 854/**
 855 * DOC: locking convention
 856 *
 857 * In order to avoid deadlock situations between dma-buf exports and importers,
 858 * all dma-buf API users must follow the common dma-buf locking convention.
 859 *
 860 * Convention for importers
 861 *
 862 * 1. Importers must hold the dma-buf reservation lock when calling these
 863 *    functions:
 864 *
 865 *     - dma_buf_pin()
 866 *     - dma_buf_unpin()
 867 *     - dma_buf_map_attachment()
 868 *     - dma_buf_unmap_attachment()
 869 *     - dma_buf_vmap()
 870 *     - dma_buf_vunmap()
 871 *
 872 * 2. Importers must not hold the dma-buf reservation lock when calling these
 873 *    functions:
 874 *
 875 *     - dma_buf_attach()
 876 *     - dma_buf_dynamic_attach()
 877 *     - dma_buf_detach()
 878 *     - dma_buf_export()
 879 *     - dma_buf_fd()
 880 *     - dma_buf_get()
 881 *     - dma_buf_put()
 882 *     - dma_buf_mmap()
 883 *     - dma_buf_begin_cpu_access()
 884 *     - dma_buf_end_cpu_access()
 885 *     - dma_buf_map_attachment_unlocked()
 886 *     - dma_buf_unmap_attachment_unlocked()
 887 *     - dma_buf_vmap_unlocked()
 888 *     - dma_buf_vunmap_unlocked()
 889 *
 890 * Convention for exporters
 891 *
 892 * 1. These &dma_buf_ops callbacks are invoked with unlocked dma-buf
 893 *    reservation and exporter can take the lock:
 894 *
 895 *     - &dma_buf_ops.attach()
 896 *     - &dma_buf_ops.detach()
 897 *     - &dma_buf_ops.release()
 898 *     - &dma_buf_ops.begin_cpu_access()
 899 *     - &dma_buf_ops.end_cpu_access()
 900 *     - &dma_buf_ops.mmap()
 901 *
 902 * 2. These &dma_buf_ops callbacks are invoked with locked dma-buf
 903 *    reservation and exporter can't take the lock:
 904 *
 905 *     - &dma_buf_ops.pin()
 906 *     - &dma_buf_ops.unpin()
 907 *     - &dma_buf_ops.map_dma_buf()
 908 *     - &dma_buf_ops.unmap_dma_buf()
 909 *     - &dma_buf_ops.vmap()
 910 *     - &dma_buf_ops.vunmap()
 911 *
 912 * 3. Exporters must hold the dma-buf reservation lock when calling these
 913 *    functions:
 914 *
 915 *     - dma_buf_move_notify()
 916 */
 917
 918/**
 919 * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list
 920 * @dmabuf:		[in]	buffer to attach device to.
 921 * @dev:		[in]	device to be attached.
 922 * @importer_ops:	[in]	importer operations for the attachment
 923 * @importer_priv:	[in]	importer private pointer for the attachment
 924 *
 925 * Returns struct dma_buf_attachment pointer for this attachment. Attachments
 926 * must be cleaned up by calling dma_buf_detach().
 927 *
 928 * Optionally this calls &dma_buf_ops.attach to allow device-specific attach
 929 * functionality.
 930 *
 931 * Returns:
 932 *
 933 * A pointer to newly created &dma_buf_attachment on success, or a negative
 934 * error code wrapped into a pointer on failure.
 935 *
 936 * Note that this can fail if the backing storage of @dmabuf is in a place not
 937 * accessible to @dev, and cannot be moved to a more suitable place. This is
 938 * indicated with the error code -EBUSY.
 939 */
 940struct dma_buf_attachment *
 941dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 942		       const struct dma_buf_attach_ops *importer_ops,
 943		       void *importer_priv)
 944{
 945	struct dma_buf_attachment *attach;
 946	int ret;
 947
 948	if (WARN_ON(!dmabuf || !dev))
 949		return ERR_PTR(-EINVAL);
 950
 951	if (WARN_ON(importer_ops && !importer_ops->move_notify))
 952		return ERR_PTR(-EINVAL);
 953
 954	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
 955	if (!attach)
 956		return ERR_PTR(-ENOMEM);
 957
 958	attach->dev = dev;
 959	attach->dmabuf = dmabuf;
 960	if (importer_ops)
 961		attach->peer2peer = importer_ops->allow_peer2peer;
 962	attach->importer_ops = importer_ops;
 963	attach->importer_priv = importer_priv;
 964
 965	if (dmabuf->ops->attach) {
 966		ret = dmabuf->ops->attach(dmabuf, attach);
 967		if (ret)
 968			goto err_attach;
 969	}
 970	dma_resv_lock(dmabuf->resv, NULL);
 971	list_add(&attach->node, &dmabuf->attachments);
 972	dma_resv_unlock(dmabuf->resv);
 973
 974	return attach;
 975
 976err_attach:
 977	kfree(attach);
 978	return ERR_PTR(ret);
 979}
 980EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, "DMA_BUF");
 981
 982/**
 983 * dma_buf_attach - Wrapper for dma_buf_dynamic_attach
 984 * @dmabuf:	[in]	buffer to attach device to.
 985 * @dev:	[in]	device to be attached.
 986 *
 987 * Wrapper to call dma_buf_dynamic_attach() for drivers which still use a static
 988 * mapping.
 989 */
 990struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
 991					  struct device *dev)
 992{
 993	return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);
 994}
 995EXPORT_SYMBOL_NS_GPL(dma_buf_attach, "DMA_BUF");
 996
 997/**
 998 * dma_buf_detach - Remove the given attachment from dmabuf's attachments list
 999 * @dmabuf:	[in]	buffer to detach from.
1000 * @attach:	[in]	attachment to be detached; is free'd after this call.
1001 *
1002 * Clean up a device attachment obtained by calling dma_buf_attach().
1003 *
1004 * Optionally this calls &dma_buf_ops.detach for device-specific detach.
1005 */
1006void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
1007{
1008	if (WARN_ON(!dmabuf || !attach || dmabuf != attach->dmabuf))
1009		return;
1010
1011	dma_resv_lock(dmabuf->resv, NULL);
1012	list_del(&attach->node);
1013	dma_resv_unlock(dmabuf->resv);
1014
1015	if (dmabuf->ops->detach)
1016		dmabuf->ops->detach(dmabuf, attach);
1017
1018	kfree(attach);
1019}
1020EXPORT_SYMBOL_NS_GPL(dma_buf_detach, "DMA_BUF");
1021
1022/**
1023 * dma_buf_pin - Lock down the DMA-buf
1024 * @attach:	[in]	attachment which should be pinned
1025 *
1026 * Only dynamic importers (who set up @attach with dma_buf_dynamic_attach()) may
1027 * call this, and only for limited use cases like scanout and not for temporary
1028 * pin operations. It is not permitted to allow userspace to pin arbitrary
1029 * amounts of buffers through this interface.
1030 *
1031 * Buffers must be unpinned by calling dma_buf_unpin().
1032 *
1033 * Returns:
1034 * 0 on success, negative error code on failure.
1035 */
1036int dma_buf_pin(struct dma_buf_attachment *attach)
1037{
1038	struct dma_buf *dmabuf = attach->dmabuf;
1039	int ret = 0;
1040
1041	WARN_ON(!attach->importer_ops);
1042
1043	dma_resv_assert_held(dmabuf->resv);
1044
1045	if (dmabuf->ops->pin)
1046		ret = dmabuf->ops->pin(attach);
1047
1048	return ret;
1049}
1050EXPORT_SYMBOL_NS_GPL(dma_buf_pin, "DMA_BUF");
1051
1052/**
1053 * dma_buf_unpin - Unpin a DMA-buf
1054 * @attach:	[in]	attachment which should be unpinned
1055 *
1056 * This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move
1057 * any mapping of @attach again and inform the importer through
1058 * &dma_buf_attach_ops.move_notify.
1059 */
1060void dma_buf_unpin(struct dma_buf_attachment *attach)
1061{
1062	struct dma_buf *dmabuf = attach->dmabuf;
1063
1064	WARN_ON(!attach->importer_ops);
1065
1066	dma_resv_assert_held(dmabuf->resv);
1067
1068	if (dmabuf->ops->unpin)
1069		dmabuf->ops->unpin(attach);
1070}
1071EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, "DMA_BUF");
1072
1073/**
1074 * dma_buf_map_attachment - Returns the scatterlist table of the attachment;
1075 * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
1076 * dma_buf_ops.
1077 * @attach:	[in]	attachment whose scatterlist is to be returned
1078 * @direction:	[in]	direction of DMA transfer
1079 *
1080 * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
1081 * on error. May return -EINTR if it is interrupted by a signal.
1082 *
1083 * On success, the DMA addresses and lengths in the returned scatterlist are
1084 * PAGE_SIZE aligned.
1085 *
1086 * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
1087 * the underlying backing storage is pinned for as long as a mapping exists,
1088 * therefore users/importers should not hold onto a mapping for undue amounts of
1089 * time.
1090 *
1091 * Important: Dynamic importers must wait for the exclusive fence of the struct
1092 * dma_resv attached to the DMA-BUF first.
1093 */
1094struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
1095					enum dma_data_direction direction)
1096{
1097	struct sg_table *sg_table;
1098	signed long ret;
1099
1100	might_sleep();
1101
1102	if (WARN_ON(!attach || !attach->dmabuf))
1103		return ERR_PTR(-EINVAL);
1104
1105	dma_resv_assert_held(attach->dmabuf->resv);
1106
1107	if (dma_buf_pin_on_map(attach)) {
1108		ret = attach->dmabuf->ops->pin(attach);
1109		/*
1110		 * Catch exporters making buffers inaccessible even when
1111		 * attachments preventing that exist.
1112		 */
1113		WARN_ON_ONCE(ret == -EBUSY);
1114		if (ret)
1115			return ERR_PTR(ret);
1116	}
1117
1118	sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);
1119	if (!sg_table)
1120		sg_table = ERR_PTR(-ENOMEM);
1121	if (IS_ERR(sg_table))
1122		goto error_unpin;
1123
1124	/*
1125	 * Importers with static attachments don't wait for fences.
1126	 */
1127	if (!dma_buf_attachment_is_dynamic(attach)) {
1128		ret = dma_resv_wait_timeout(attach->dmabuf->resv,
1129					    DMA_RESV_USAGE_KERNEL, true,
1130					    MAX_SCHEDULE_TIMEOUT);
1131		if (ret < 0)
1132			goto error_unmap;
1133	}
1134	mangle_sg_table(sg_table);
1135
1136#ifdef CONFIG_DMA_API_DEBUG
1137	{
1138		struct scatterlist *sg;
1139		u64 addr;
1140		int len;
1141		int i;
1142
1143		for_each_sgtable_dma_sg(sg_table, sg, i) {
1144			addr = sg_dma_address(sg);
1145			len = sg_dma_len(sg);
1146			if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(len)) {
1147				pr_debug("%s: addr %llx or len %x is not page aligned!\n",
1148					 __func__, addr, len);
1149			}
1150		}
1151	}
1152#endif /* CONFIG_DMA_API_DEBUG */
1153	return sg_table;
1154
1155error_unmap:
1156	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
1157	sg_table = ERR_PTR(ret);
1158
1159error_unpin:
1160	if (dma_buf_pin_on_map(attach))
1161		attach->dmabuf->ops->unpin(attach);
1162
1163	return sg_table;
1164}
1165EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment, "DMA_BUF");
1166
1167/**
1168 * dma_buf_map_attachment_unlocked - Returns the scatterlist table of the attachment;
1169 * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
1170 * dma_buf_ops.
1171 * @attach:	[in]	attachment whose scatterlist is to be returned
1172 * @direction:	[in]	direction of DMA transfer
1173 *
1174 * Unlocked variant of dma_buf_map_attachment().
1175 */
1176struct sg_table *
1177dma_buf_map_attachment_unlocked(struct dma_buf_attachment *attach,
1178				enum dma_data_direction direction)
1179{
1180	struct sg_table *sg_table;
1181
1182	might_sleep();
1183
1184	if (WARN_ON(!attach || !attach->dmabuf))
1185		return ERR_PTR(-EINVAL);
1186
1187	dma_resv_lock(attach->dmabuf->resv, NULL);
1188	sg_table = dma_buf_map_attachment(attach, direction);
1189	dma_resv_unlock(attach->dmabuf->resv);
1190
1191	return sg_table;
1192}
1193EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment_unlocked, "DMA_BUF");
1194
1195/**
1196 * dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might
1197 * deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of
1198 * dma_buf_ops.
1199 * @attach:	[in]	attachment to unmap buffer from
1200 * @sg_table:	[in]	scatterlist info of the buffer to unmap
1201 * @direction:  [in]    direction of DMA transfer
1202 *
1203 * This unmaps a DMA mapping for @attached obtained by dma_buf_map_attachment().
1204 */
1205void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
1206				struct sg_table *sg_table,
1207				enum dma_data_direction direction)
1208{
1209	might_sleep();
1210
1211	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
1212		return;
1213
1214	dma_resv_assert_held(attach->dmabuf->resv);
1215
1216	mangle_sg_table(sg_table);
1217	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
1218
1219	if (dma_buf_pin_on_map(attach))
1220		attach->dmabuf->ops->unpin(attach);
1221}
1222EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment, "DMA_BUF");
1223
1224/**
1225 * dma_buf_unmap_attachment_unlocked - unmaps and decreases usecount of the buffer;might
1226 * deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of
1227 * dma_buf_ops.
1228 * @attach:	[in]	attachment to unmap buffer from
1229 * @sg_table:	[in]	scatterlist info of the buffer to unmap
1230 * @direction:	[in]	direction of DMA transfer
1231 *
1232 * Unlocked variant of dma_buf_unmap_attachment().
1233 */
1234void dma_buf_unmap_attachment_unlocked(struct dma_buf_attachment *attach,
1235				       struct sg_table *sg_table,
1236				       enum dma_data_direction direction)
1237{
1238	might_sleep();
1239
1240	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
1241		return;
1242
1243	dma_resv_lock(attach->dmabuf->resv, NULL);
1244	dma_buf_unmap_attachment(attach, sg_table, direction);
1245	dma_resv_unlock(attach->dmabuf->resv);
1246}
1247EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment_unlocked, "DMA_BUF");
1248
1249/**
1250 * dma_buf_move_notify - notify attachments that DMA-buf is moving
1251 *
1252 * @dmabuf:	[in]	buffer which is moving
1253 *
1254 * Informs all attachments that they need to destroy and recreate all their
1255 * mappings.
1256 */
1257void dma_buf_move_notify(struct dma_buf *dmabuf)
1258{
1259	struct dma_buf_attachment *attach;
1260
1261	dma_resv_assert_held(dmabuf->resv);
1262
1263	list_for_each_entry(attach, &dmabuf->attachments, node)
1264		if (attach->importer_ops)
1265			attach->importer_ops->move_notify(attach);
1266}
1267EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, "DMA_BUF");
1268
1269/**
1270 * DOC: cpu access
1271 *
1272 * There are multiple reasons for supporting CPU access to a dma buffer object:
1273 *
1274 * - Fallback operations in the kernel, for example when a device is connected
1275 *   over USB and the kernel needs to shuffle the data around first before
1276 *   sending it away. Cache coherency is handled by bracketing any transactions
1277 *   with calls to dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
1278 *   access.
1279 *
1280 *   Since for most kernel internal dma-buf accesses need the entire buffer, a
1281 *   vmap interface is introduced. Note that on very old 32-bit architectures
1282 *   vmalloc space might be limited and result in vmap calls failing.
1283 *
1284 *   Interfaces:
1285 *
1286 *   .. code-block:: c
1287 *
1288 *     void *dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
1289 *     void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
1290 *
1291 *   The vmap call can fail if there is no vmap support in the exporter, or if
1292 *   it runs out of vmalloc space. Note that the dma-buf layer keeps a reference
1293 *   count for all vmap access and calls down into the exporter's vmap function
1294 *   only when no vmapping exists, and only unmaps it once. Protection against
1295 *   concurrent vmap/vunmap calls is provided by taking the &dma_buf.lock mutex.
1296 *
1297 * - For full compatibility on the importer side with existing userspace
1298 *   interfaces, which might already support mmap'ing buffers. This is needed in
1299 *   many processing pipelines (e.g. feeding a software rendered image into a
1300 *   hardware pipeline, thumbnail creation, snapshots, ...). Also, Android's ION
1301 *   framework already supported this and for DMA buffer file descriptors to
1302 *   replace ION buffers mmap support was needed.
1303 *
1304 *   There is no special interfaces, userspace simply calls mmap on the dma-buf
1305 *   fd. But like for CPU access there's a need to bracket the actual access,
1306 *   which is handled by the ioctl (DMA_BUF_IOCTL_SYNC). Note that
1307 *   DMA_BUF_IOCTL_SYNC can fail with -EAGAIN or -EINTR, in which case it must
1308 *   be restarted.
1309 *
1310 *   Some systems might need some sort of cache coherency management e.g. when
1311 *   CPU and GPU domains are being accessed through dma-buf at the same time.
1312 *   To circumvent this problem there are begin/end coherency markers, that
1313 *   forward directly to existing dma-buf device drivers vfunc hooks. Userspace
1314 *   can make use of those markers through the DMA_BUF_IOCTL_SYNC ioctl. The
1315 *   sequence would be used like following:
1316 *
1317 *     - mmap dma-buf fd
1318 *     - for each drawing/upload cycle in CPU 1. SYNC_START ioctl, 2. read/write
1319 *       to mmap area 3. SYNC_END ioctl. This can be repeated as often as you
1320 *       want (with the new data being consumed by say the GPU or the scanout
1321 *       device)
1322 *     - munmap once you don't need the buffer any more
1323 *
1324 *    For correctness and optimal performance, it is always required to use
1325 *    SYNC_START and SYNC_END before and after, respectively, when accessing the
1326 *    mapped address. Userspace cannot rely on coherent access, even when there
1327 *    are systems where it just works without calling these ioctls.
1328 *
1329 * - And as a CPU fallback in userspace processing pipelines.
1330 *
1331 *   Similar to the motivation for kernel cpu access it is again important that
1332 *   the userspace code of a given importing subsystem can use the same
1333 *   interfaces with a imported dma-buf buffer object as with a native buffer
1334 *   object. This is especially important for drm where the userspace part of
1335 *   contemporary OpenGL, X, and other drivers is huge, and reworking them to
1336 *   use a different way to mmap a buffer rather invasive.
1337 *
1338 *   The assumption in the current dma-buf interfaces is that redirecting the
1339 *   initial mmap is all that's needed. A survey of some of the existing
1340 *   subsystems shows that no driver seems to do any nefarious thing like
1341 *   syncing up with outstanding asynchronous processing on the device or
1342 *   allocating special resources at fault time. So hopefully this is good
1343 *   enough, since adding interfaces to intercept pagefaults and allow pte
1344 *   shootdowns would increase the complexity quite a bit.
1345 *
1346 *   Interface:
1347 *
1348 *   .. code-block:: c
1349 *
1350 *     int dma_buf_mmap(struct dma_buf *, struct vm_area_struct *, unsigned long);
1351 *
1352 *   If the importing subsystem simply provides a special-purpose mmap call to
1353 *   set up a mapping in userspace, calling do_mmap with &dma_buf.file will
1354 *   equally achieve that for a dma-buf object.
1355 */
1356
1357static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1358				      enum dma_data_direction direction)
1359{
1360	bool write = (direction == DMA_BIDIRECTIONAL ||
1361		      direction == DMA_TO_DEVICE);
1362	struct dma_resv *resv = dmabuf->resv;
1363	long ret;
1364
1365	/* Wait on any implicit rendering fences */
1366	ret = dma_resv_wait_timeout(resv, dma_resv_usage_rw(write),
1367				    true, MAX_SCHEDULE_TIMEOUT);
1368	if (ret < 0)
1369		return ret;
1370
1371	return 0;
1372}
1373
1374/**
1375 * dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the
1376 * cpu in the kernel context. Calls begin_cpu_access to allow exporter-specific
1377 * preparations. Coherency is only guaranteed in the specified range for the
1378 * specified access direction.
1379 * @dmabuf:	[in]	buffer to prepare cpu access for.
1380 * @direction:	[in]	direction of access.
1381 *
1382 * After the cpu access is complete the caller should call
1383 * dma_buf_end_cpu_access(). Only when cpu access is bracketed by both calls is
1384 * it guaranteed to be coherent with other DMA access.
1385 *
1386 * This function will also wait for any DMA transactions tracked through
1387 * implicit synchronization in &dma_buf.resv. For DMA transactions with explicit
1388 * synchronization this function will only ensure cache coherency, callers must
1389 * ensure synchronization with such DMA transactions on their own.
1390 *
1391 * Can return negative error values, returns 0 on success.
1392 */
1393int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1394			     enum dma_data_direction direction)
1395{
1396	int ret = 0;
1397
1398	if (WARN_ON(!dmabuf))
1399		return -EINVAL;
1400
1401	might_lock(&dmabuf->resv->lock.base);
1402
1403	if (dmabuf->ops->begin_cpu_access)
1404		ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);
1405
1406	/* Ensure that all fences are waited upon - but we first allow
1407	 * the native handler the chance to do so more efficiently if it
1408	 * chooses. A double invocation here will be reasonably cheap no-op.
1409	 */
1410	if (ret == 0)
1411		ret = __dma_buf_begin_cpu_access(dmabuf, direction);
1412
1413	return ret;
1414}
1415EXPORT_SYMBOL_NS_GPL(dma_buf_begin_cpu_access, "DMA_BUF");
1416
1417/**
1418 * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the
1419 * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific
1420 * actions. Coherency is only guaranteed in the specified range for the
1421 * specified access direction.
1422 * @dmabuf:	[in]	buffer to complete cpu access for.
1423 * @direction:	[in]	direction of access.
1424 *
1425 * This terminates CPU access started with dma_buf_begin_cpu_access().
1426 *
1427 * Can return negative error values, returns 0 on success.
1428 */
1429int dma_buf_end_cpu_access(struct dma_buf *dmabuf,
1430			   enum dma_data_direction direction)
1431{
1432	int ret = 0;
1433
1434	WARN_ON(!dmabuf);
1435
1436	might_lock(&dmabuf->resv->lock.base);
1437
1438	if (dmabuf->ops->end_cpu_access)
1439		ret = dmabuf->ops->end_cpu_access(dmabuf, direction);
1440
1441	return ret;
1442}
1443EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF");
1444
1445
1446/**
1447 * dma_buf_mmap - Setup up a userspace mmap with the given vma
1448 * @dmabuf:	[in]	buffer that should back the vma
1449 * @vma:	[in]	vma for the mmap
1450 * @pgoff:	[in]	offset in pages where this mmap should start within the
1451 *			dma-buf buffer.
1452 *
1453 * This function adjusts the passed in vma so that it points at the file of the
1454 * dma_buf operation. It also adjusts the starting pgoff and does bounds
1455 * checking on the size of the vma. Then it calls the exporters mmap function to
1456 * set up the mapping.
1457 *
1458 * Can return negative error values, returns 0 on success.
1459 */
1460int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
1461		 unsigned long pgoff)
1462{
1463	if (WARN_ON(!dmabuf || !vma))
1464		return -EINVAL;
1465
1466	/* check if buffer supports mmap */
1467	if (!dmabuf->ops->mmap)
1468		return -EINVAL;
1469
1470	/* check for offset overflow */
1471	if (pgoff + vma_pages(vma) < pgoff)
1472		return -EOVERFLOW;
1473
1474	/* check for overflowing the buffer's size */
1475	if (pgoff + vma_pages(vma) >
1476	    dmabuf->size >> PAGE_SHIFT)
1477		return -EINVAL;
1478
1479	/* readjust the vma */
1480	vma_set_file(vma, dmabuf->file);
1481	vma->vm_pgoff = pgoff;
1482
1483	return dmabuf->ops->mmap(dmabuf, vma);
1484}
1485EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, "DMA_BUF");
1486
1487/**
1488 * dma_buf_vmap - Create virtual mapping for the buffer object into kernel
1489 * address space. Same restrictions as for vmap and friends apply.
1490 * @dmabuf:	[in]	buffer to vmap
1491 * @map:	[out]	returns the vmap pointer
1492 *
1493 * This call may fail due to lack of virtual mapping address space.
1494 * These calls are optional in drivers. The intended use for them
1495 * is for mapping objects linear in kernel space for high use objects.
1496 *
1497 * To ensure coherency users must call dma_buf_begin_cpu_access() and
1498 * dma_buf_end_cpu_access() around any cpu access performed through this
1499 * mapping.
1500 *
1501 * Returns 0 on success, or a negative errno code otherwise.
1502 */
1503int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
1504{
1505	struct iosys_map ptr;
1506	int ret;
1507
1508	iosys_map_clear(map);
1509
1510	if (WARN_ON(!dmabuf))
1511		return -EINVAL;
1512
1513	dma_resv_assert_held(dmabuf->resv);
1514
1515	if (!dmabuf->ops->vmap)
1516		return -EINVAL;
1517
1518	if (dmabuf->vmapping_counter) {
1519		dmabuf->vmapping_counter++;
1520		BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));
1521		*map = dmabuf->vmap_ptr;
1522		return 0;
1523	}
1524
1525	BUG_ON(iosys_map_is_set(&dmabuf->vmap_ptr));
1526
1527	ret = dmabuf->ops->vmap(dmabuf, &ptr);
1528	if (WARN_ON_ONCE(ret))
1529		return ret;
1530
1531	dmabuf->vmap_ptr = ptr;
1532	dmabuf->vmapping_counter = 1;
1533
1534	*map = dmabuf->vmap_ptr;
1535
1536	return 0;
1537}
1538EXPORT_SYMBOL_NS_GPL(dma_buf_vmap, "DMA_BUF");
1539
1540/**
1541 * dma_buf_vmap_unlocked - Create virtual mapping for the buffer object into kernel
1542 * address space. Same restrictions as for vmap and friends apply.
1543 * @dmabuf:	[in]	buffer to vmap
1544 * @map:	[out]	returns the vmap pointer
1545 *
1546 * Unlocked version of dma_buf_vmap()
1547 *
1548 * Returns 0 on success, or a negative errno code otherwise.
1549 */
1550int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)
1551{
1552	int ret;
1553
1554	iosys_map_clear(map);
1555
1556	if (WARN_ON(!dmabuf))
1557		return -EINVAL;
1558
1559	dma_resv_lock(dmabuf->resv, NULL);
1560	ret = dma_buf_vmap(dmabuf, map);
1561	dma_resv_unlock(dmabuf->resv);
1562
1563	return ret;
1564}
1565EXPORT_SYMBOL_NS_GPL(dma_buf_vmap_unlocked, "DMA_BUF");
1566
1567/**
1568 * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap.
1569 * @dmabuf:	[in]	buffer to vunmap
1570 * @map:	[in]	vmap pointer to vunmap
1571 */
1572void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
1573{
1574	if (WARN_ON(!dmabuf))
1575		return;
1576
1577	dma_resv_assert_held(dmabuf->resv);
1578
1579	BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));
1580	BUG_ON(dmabuf->vmapping_counter == 0);
1581	BUG_ON(!iosys_map_is_equal(&dmabuf->vmap_ptr, map));
1582
1583	if (--dmabuf->vmapping_counter == 0) {
1584		if (dmabuf->ops->vunmap)
1585			dmabuf->ops->vunmap(dmabuf, map);
1586		iosys_map_clear(&dmabuf->vmap_ptr);
1587	}
1588}
1589EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap, "DMA_BUF");
1590
1591/**
1592 * dma_buf_vunmap_unlocked - Unmap a vmap obtained by dma_buf_vmap.
1593 * @dmabuf:	[in]	buffer to vunmap
1594 * @map:	[in]	vmap pointer to vunmap
1595 */
1596void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)
1597{
1598	if (WARN_ON(!dmabuf))
1599		return;
1600
1601	dma_resv_lock(dmabuf->resv, NULL);
1602	dma_buf_vunmap(dmabuf, map);
1603	dma_resv_unlock(dmabuf->resv);
1604}
1605EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap_unlocked, "DMA_BUF");
1606
1607#ifdef CONFIG_DEBUG_FS
1608static int dma_buf_debug_show(struct seq_file *s, void *unused)
1609{
1610	struct dma_buf *buf_obj;
1611	struct dma_buf_attachment *attach_obj;
1612	int count = 0, attach_count;
1613	size_t size = 0;
1614	int ret;
1615
1616	ret = mutex_lock_interruptible(&dmabuf_list_mutex);
1617
1618	if (ret)
1619		return ret;
1620
1621	seq_puts(s, "\nDma-buf Objects:\n");
1622	seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\tname\n",
1623		   "size", "flags", "mode", "count", "ino");
1624
1625	list_for_each_entry(buf_obj, &dmabuf_list, list_node) {
1626
1627		ret = dma_resv_lock_interruptible(buf_obj->resv, NULL);
1628		if (ret)
1629			goto error_unlock;
1630
1631
1632		spin_lock(&buf_obj->name_lock);
1633		seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n",
1634				buf_obj->size,
1635				buf_obj->file->f_flags, buf_obj->file->f_mode,
1636				file_count(buf_obj->file),
1637				buf_obj->exp_name,
1638				file_inode(buf_obj->file)->i_ino,
1639				buf_obj->name ?: "<none>");
1640		spin_unlock(&buf_obj->name_lock);
1641
1642		dma_resv_describe(buf_obj->resv, s);
1643
1644		seq_puts(s, "\tAttached Devices:\n");
1645		attach_count = 0;
1646
1647		list_for_each_entry(attach_obj, &buf_obj->attachments, node) {
1648			seq_printf(s, "\t%s\n", dev_name(attach_obj->dev));
1649			attach_count++;
1650		}
1651		dma_resv_unlock(buf_obj->resv);
1652
1653		seq_printf(s, "Total %d devices attached\n\n",
1654				attach_count);
1655
1656		count++;
1657		size += buf_obj->size;
1658	}
1659
1660	seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size);
1661
1662	mutex_unlock(&dmabuf_list_mutex);
1663	return 0;
1664
1665error_unlock:
1666	mutex_unlock(&dmabuf_list_mutex);
1667	return ret;
1668}
1669
1670DEFINE_SHOW_ATTRIBUTE(dma_buf_debug);
1671
1672static struct dentry *dma_buf_debugfs_dir;
1673
1674static int dma_buf_init_debugfs(void)
1675{
1676	struct dentry *d;
1677	int err = 0;
1678
1679	d = debugfs_create_dir("dma_buf", NULL);
1680	if (IS_ERR(d))
1681		return PTR_ERR(d);
1682
1683	dma_buf_debugfs_dir = d;
1684
1685	d = debugfs_create_file("bufinfo", 0444, dma_buf_debugfs_dir,
1686				NULL, &dma_buf_debug_fops);
1687	if (IS_ERR(d)) {
1688		pr_debug("dma_buf: debugfs: failed to create node bufinfo\n");
1689		debugfs_remove_recursive(dma_buf_debugfs_dir);
1690		dma_buf_debugfs_dir = NULL;
1691		err = PTR_ERR(d);
1692	}
1693
1694	return err;
1695}
1696
1697static void dma_buf_uninit_debugfs(void)
1698{
1699	debugfs_remove_recursive(dma_buf_debugfs_dir);
1700}
1701#else
1702static inline int dma_buf_init_debugfs(void)
1703{
1704	return 0;
1705}
1706static inline void dma_buf_uninit_debugfs(void)
1707{
1708}
1709#endif
1710
1711static int __init dma_buf_init(void)
1712{
1713	int ret;
1714
1715	ret = dma_buf_init_sysfs_statistics();
1716	if (ret)
1717		return ret;
1718
1719	dma_buf_mnt = kern_mount(&dma_buf_fs_type);
1720	if (IS_ERR(dma_buf_mnt))
1721		return PTR_ERR(dma_buf_mnt);
1722
1723	dma_buf_init_debugfs();
1724	return 0;
1725}
1726subsys_initcall(dma_buf_init);
1727
1728static void __exit dma_buf_deinit(void)
1729{
1730	dma_buf_uninit_debugfs();
1731	kern_unmount(dma_buf_mnt);
1732	dma_buf_uninit_sysfs_statistics();
1733}
1734__exitcall(dma_buf_deinit);