f_fs.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * f_fs.c -- user mode file system API for USB composite function controllers
   4 *
   5 * Copyright (C) 2010 Samsung Electronics
   6 * Author: Michal Nazarewicz <mina86@mina86.com>
   7 *
   8 * Based on inode.c (GadgetFS) which was:
   9 * Copyright (C) 2003-2004 David Brownell
  10 * Copyright (C) 2003 Agilent Technologies
  11 */
  12
  13
  14/* #define DEBUG */
  15/* #define VERBOSE_DEBUG */
  16
  17#include <linux/blkdev.h>
  18#include <linux/dma-buf.h>
  19#include <linux/dma-fence.h>
  20#include <linux/dma-resv.h>
  21#include <linux/pagemap.h>
  22#include <linux/export.h>
  23#include <linux/fs_parser.h>
  24#include <linux/hid.h>
  25#include <linux/mm.h>
  26#include <linux/module.h>
  27#include <linux/scatterlist.h>
  28#include <linux/sched/signal.h>
  29#include <linux/uio.h>
  30#include <linux/vmalloc.h>
  31#include <linux/unaligned.h>
  32
  33#include <linux/usb/ccid.h>
  34#include <linux/usb/composite.h>
  35#include <linux/usb/functionfs.h>
  36#include <linux/usb/func_utils.h>
  37
  38#include <linux/aio.h>
  39#include <linux/kthread.h>
  40#include <linux/poll.h>
  41#include <linux/eventfd.h>
  42
  43#include "u_fs.h"
  44#include "u_os_desc.h"
  45#include "configfs.h"
  46
  47#define FUNCTIONFS_MAGIC	0xa647361 /* Chosen by a honest dice roll ;) */
  48#define MAX_ALT_SETTINGS	2		  /* Allow up to 2 alt settings to be set. */
  49
  50#define DMABUF_ENQUEUE_TIMEOUT_MS 5000
  51
  52MODULE_IMPORT_NS("DMA_BUF");
  53
  54/* Reference counter handling */
  55static void ffs_data_get(struct ffs_data *ffs);
  56static void ffs_data_put(struct ffs_data *ffs);
  57/* Creates new ffs_data object. */
  58static struct ffs_data *__must_check ffs_data_new(const char *dev_name)
  59	__attribute__((malloc));
  60
  61/* Opened counter handling. */
  62static void ffs_data_closed(struct ffs_data *ffs);
  63
  64/* Called with ffs->mutex held; take over ownership of data. */
  65static int __must_check
  66__ffs_data_got_descs(struct ffs_data *ffs, char *data, size_t len);
  67static int __must_check
  68__ffs_data_got_strings(struct ffs_data *ffs, char *data, size_t len);
  69
  70
  71/* The function structure ***************************************************/
  72
  73struct ffs_ep;
  74
  75struct ffs_function {
  76	struct usb_configuration	*conf;
  77	struct usb_gadget		*gadget;
  78	struct ffs_data			*ffs;
  79
  80	struct ffs_ep			*eps;
  81	u8				eps_revmap[16];
  82	short				*interfaces_nums;
  83
  84	struct usb_function		function;
  85	int				cur_alt[MAX_CONFIG_INTERFACES];
  86};
  87
  88
  89static struct ffs_function *ffs_func_from_usb(struct usb_function *f)
  90{
  91	return container_of(f, struct ffs_function, function);
  92}
  93
  94
  95static inline enum ffs_setup_state
  96ffs_setup_state_clear_cancelled(struct ffs_data *ffs)
  97{
  98	return (enum ffs_setup_state)
  99		cmpxchg(&ffs->setup_state, FFS_SETUP_CANCELLED, FFS_NO_SETUP);
 100}
 101
 102
 103static void ffs_func_eps_disable(struct ffs_function *func);
 104static int __must_check ffs_func_eps_enable(struct ffs_function *func);
 105
 106static int ffs_func_bind(struct usb_configuration *,
 107			 struct usb_function *);
 108static int ffs_func_set_alt(struct usb_function *, unsigned, unsigned);
 109static int ffs_func_get_alt(struct usb_function *f, unsigned int intf);
 110static void ffs_func_disable(struct usb_function *);
 111static int ffs_func_setup(struct usb_function *,
 112			  const struct usb_ctrlrequest *);
 113static bool ffs_func_req_match(struct usb_function *,
 114			       const struct usb_ctrlrequest *,
 115			       bool config0);
 116static void ffs_func_suspend(struct usb_function *);
 117static void ffs_func_resume(struct usb_function *);
 118
 119
 120static int ffs_func_revmap_ep(struct ffs_function *func, u8 num);
 121static int ffs_func_revmap_intf(struct ffs_function *func, u8 intf);
 122
 123
 124/* The endpoints structures *************************************************/
 125
 126struct ffs_ep {
 127	struct usb_ep			*ep;	/* P: ffs->eps_lock */
 128	struct usb_request		*req;	/* P: epfile->mutex */
 129
 130	/* [0]: full speed, [1]: high speed, [2]: super speed */
 131	struct usb_endpoint_descriptor	*descs[3];
 132
 133	u8				num;
 134};
 135
 136struct ffs_dmabuf_priv {
 137	struct list_head entry;
 138	struct kref ref;
 139	struct ffs_data *ffs;
 140	struct dma_buf_attachment *attach;
 141	struct sg_table *sgt;
 142	enum dma_data_direction dir;
 143	spinlock_t lock;
 144	u64 context;
 145	struct usb_request *req;	/* P: ffs->eps_lock */
 146	struct usb_ep *ep;		/* P: ffs->eps_lock */
 147};
 148
 149struct ffs_dma_fence {
 150	struct dma_fence base;
 151	struct ffs_dmabuf_priv *priv;
 152	struct work_struct work;
 153};
 154
 155struct ffs_epfile {
 156	/* Protects ep->ep and ep->req. */
 157	struct mutex			mutex;
 158
 159	struct ffs_data			*ffs;
 160	struct ffs_ep			*ep;	/* P: ffs->eps_lock */
 161
 162	/*
 163	 * Buffer for holding data from partial reads which may happen since
 164	 * we’re rounding user read requests to a multiple of a max packet size.
 165	 *
 166	 * The pointer is initialised with NULL value and may be set by
 167	 * __ffs_epfile_read_data function to point to a temporary buffer.
 168	 *
 169	 * In normal operation, calls to __ffs_epfile_read_buffered will consume
 170	 * data from said buffer and eventually free it.  Importantly, while the
 171	 * function is using the buffer, it sets the pointer to NULL.  This is
 172	 * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered
 173	 * can never run concurrently (they are synchronised by epfile->mutex)
 174	 * so the latter will not assign a new value to the pointer.
 175	 *
 176	 * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is
 177	 * valid) and sets the pointer to READ_BUFFER_DROP value.  This special
 178	 * value is crux of the synchronisation between ffs_func_eps_disable and
 179	 * __ffs_epfile_read_data.
 180	 *
 181	 * Once __ffs_epfile_read_data is about to finish it will try to set the
 182	 * pointer back to its old value (as described above), but seeing as the
 183	 * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free
 184	 * the buffer.
 185	 *
 186	 * == State transitions ==
 187	 *
 188	 * • ptr == NULL:  (initial state)
 189	 *   ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP
 190	 *   ◦ __ffs_epfile_read_buffered:    nop
 191	 *   ◦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf
 192	 *   ◦ reading finishes:              n/a, not in ‘and reading’ state
 193	 * • ptr == DROP:
 194	 *   ◦ __ffs_epfile_read_buffer_free: nop
 195	 *   ◦ __ffs_epfile_read_buffered:    go to ptr == NULL
 196	 *   ◦ __ffs_epfile_read_data allocates temp buffer: free buf, nop
 197	 *   ◦ reading finishes:              n/a, not in ‘and reading’ state
 198	 * • ptr == buf:
 199	 *   ◦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP
 200	 *   ◦ __ffs_epfile_read_buffered:    go to ptr == NULL and reading
 201	 *   ◦ __ffs_epfile_read_data:        n/a, __ffs_epfile_read_buffered
 202	 *                                    is always called first
 203	 *   ◦ reading finishes:              n/a, not in ‘and reading’ state
 204	 * • ptr == NULL and reading:
 205	 *   ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading
 206	 *   ◦ __ffs_epfile_read_buffered:    n/a, mutex is held
 207	 *   ◦ __ffs_epfile_read_data:        n/a, mutex is held
 208	 *   ◦ reading finishes and …
 209	 *     … all data read:               free buf, go to ptr == NULL
 210	 *     … otherwise:                   go to ptr == buf and reading
 211	 * • ptr == DROP and reading:
 212	 *   ◦ __ffs_epfile_read_buffer_free: nop
 213	 *   ◦ __ffs_epfile_read_buffered:    n/a, mutex is held
 214	 *   ◦ __ffs_epfile_read_data:        n/a, mutex is held
 215	 *   ◦ reading finishes:              free buf, go to ptr == DROP
 216	 */
 217	struct ffs_buffer		*read_buffer;
 218#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN))
 219
 220	char				name[5];
 221
 222	unsigned char			in;	/* P: ffs->eps_lock */
 223	unsigned char			isoc;	/* P: ffs->eps_lock */
 224
 225	unsigned char			_pad;
 226
 227	/* Protects dmabufs */
 228	struct mutex			dmabufs_mutex;
 229	struct list_head		dmabufs; /* P: dmabufs_mutex */
 230	atomic_t			seqno;
 231};
 232
 233struct ffs_buffer {
 234	size_t length;
 235	char *data;
 236	char storage[] __counted_by(length);
 237};
 238
 239/*  ffs_io_data structure ***************************************************/
 240
 241struct ffs_io_data {
 242	bool aio;
 243	bool read;
 244
 245	struct kiocb *kiocb;
 246	struct iov_iter data;
 247	const void *to_free;
 248	char *buf;
 249
 250	struct mm_struct *mm;
 251	struct work_struct work;
 252
 253	struct usb_ep *ep;
 254	struct usb_request *req;
 255	struct sg_table sgt;
 256	bool use_sg;
 257
 258	struct ffs_data *ffs;
 259
 260	int status;
 261	struct completion done;
 262};
 263
 264struct ffs_desc_helper {
 265	struct ffs_data *ffs;
 266	unsigned interfaces_count;
 267	unsigned eps_count;
 268};
 269
 270static int  __must_check ffs_epfiles_create(struct ffs_data *ffs);
 271static void ffs_epfiles_destroy(struct super_block *sb,
 272				struct ffs_epfile *epfiles, unsigned count);
 273
 274static int ffs_sb_create_file(struct super_block *sb, const char *name,
 275			      void *data, const struct file_operations *fops);
 276
 277/* Devices management *******************************************************/
 278
 279DEFINE_MUTEX(ffs_lock);
 280EXPORT_SYMBOL_GPL(ffs_lock);
 281
 282static struct ffs_dev *_ffs_find_dev(const char *name);
 283static struct ffs_dev *_ffs_alloc_dev(void);
 284static void _ffs_free_dev(struct ffs_dev *dev);
 285static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data);
 286static void ffs_release_dev(struct ffs_dev *ffs_dev);
 287static int ffs_ready(struct ffs_data *ffs);
 288static void ffs_closed(struct ffs_data *ffs);
 289
 290/* Misc helper functions ****************************************************/
 291
 292static int ffs_mutex_lock(struct mutex *mutex, unsigned nonblock)
 293	__attribute__((warn_unused_result, nonnull));
 294static char *ffs_prepare_buffer(const char __user *buf, size_t len)
 295	__attribute__((warn_unused_result, nonnull));
 296
 297
 298/* Control file aka ep0 *****************************************************/
 299
 300static void ffs_ep0_complete(struct usb_ep *ep, struct usb_request *req)
 301{
 302	struct ffs_data *ffs = req->context;
 303
 304	complete(&ffs->ep0req_completion);
 305}
 306
 307static int __ffs_ep0_queue_wait(struct ffs_data *ffs, char *data, size_t len)
 308	__releases(&ffs->ev.waitq.lock)
 309{
 310	struct usb_request *req = ffs->ep0req;
 311	int ret;
 312
 313	if (!req) {
 314		spin_unlock_irq(&ffs->ev.waitq.lock);
 315		return -EINVAL;
 316	}
 317
 318	req->zero     = len < le16_to_cpu(ffs->ev.setup.wLength);
 319
 320	spin_unlock_irq(&ffs->ev.waitq.lock);
 321
 322	req->buf      = data;
 323	req->length   = len;
 324
 325	/*
 326	 * UDC layer requires to provide a buffer even for ZLP, but should
 327	 * not use it at all. Let's provide some poisoned pointer to catch
 328	 * possible bug in the driver.
 329	 */
 330	if (req->buf == NULL)
 331		req->buf = (void *)0xDEADBABE;
 332
 333	reinit_completion(&ffs->ep0req_completion);
 334
 335	ret = usb_ep_queue(ffs->gadget->ep0, req, GFP_ATOMIC);
 336	if (ret < 0)
 337		return ret;
 338
 339	ret = wait_for_completion_interruptible(&ffs->ep0req_completion);
 340	if (ret) {
 341		usb_ep_dequeue(ffs->gadget->ep0, req);
 342		return -EINTR;
 343	}
 344
 345	ffs->setup_state = FFS_NO_SETUP;
 346	return req->status ? req->status : req->actual;
 347}
 348
 349static int __ffs_ep0_stall(struct ffs_data *ffs)
 350{
 351	if (ffs->ev.can_stall) {
 352		pr_vdebug("ep0 stall\n");
 353		usb_ep_set_halt(ffs->gadget->ep0);
 354		ffs->setup_state = FFS_NO_SETUP;
 355		return -EL2HLT;
 356	} else {
 357		pr_debug("bogus ep0 stall!\n");
 358		return -ESRCH;
 359	}
 360}
 361
 362static ssize_t ffs_ep0_write(struct file *file, const char __user *buf,
 363			     size_t len, loff_t *ptr)
 364{
 365	struct ffs_data *ffs = file->private_data;
 366	ssize_t ret;
 367	char *data;
 368
 369	/* Fast check if setup was canceled */
 370	if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED)
 371		return -EIDRM;
 372
 373	/* Acquire mutex */
 374	ret = ffs_mutex_lock(&ffs->mutex, file->f_flags & O_NONBLOCK);
 375	if (ret < 0)
 376		return ret;
 377
 378	/* Check state */
 379	switch (ffs->state) {
 380	case FFS_READ_DESCRIPTORS:
 381	case FFS_READ_STRINGS:
 382		/* Copy data */
 383		if (len < 16) {
 384			ret = -EINVAL;
 385			break;
 386		}
 387
 388		data = ffs_prepare_buffer(buf, len);
 389		if (IS_ERR(data)) {
 390			ret = PTR_ERR(data);
 391			break;
 392		}
 393
 394		/* Handle data */
 395		if (ffs->state == FFS_READ_DESCRIPTORS) {
 396			pr_info("read descriptors\n");
 397			ret = __ffs_data_got_descs(ffs, data, len);
 398			if (ret < 0)
 399				break;
 400
 401			ffs->state = FFS_READ_STRINGS;
 402			ret = len;
 403		} else {
 404			pr_info("read strings\n");
 405			ret = __ffs_data_got_strings(ffs, data, len);
 406			if (ret < 0)
 407				break;
 408
 409			ret = ffs_epfiles_create(ffs);
 410			if (ret) {
 411				ffs->state = FFS_CLOSING;
 412				break;
 413			}
 414
 415			ffs->state = FFS_ACTIVE;
 416			mutex_unlock(&ffs->mutex);
 417
 418			ret = ffs_ready(ffs);
 419			if (ret < 0) {
 420				ffs->state = FFS_CLOSING;
 421				return ret;
 422			}
 423
 424			return len;
 425		}
 426		break;
 427
 428	case FFS_ACTIVE:
 429		data = NULL;
 430		/*
 431		 * We're called from user space, we can use _irq
 432		 * rather then _irqsave
 433		 */
 434		spin_lock_irq(&ffs->ev.waitq.lock);
 435		switch (ffs_setup_state_clear_cancelled(ffs)) {
 436		case FFS_SETUP_CANCELLED:
 437			ret = -EIDRM;
 438			goto done_spin;
 439
 440		case FFS_NO_SETUP:
 441			ret = -ESRCH;
 442			goto done_spin;
 443
 444		case FFS_SETUP_PENDING:
 445			break;
 446		}
 447
 448		/* FFS_SETUP_PENDING */
 449		if (!(ffs->ev.setup.bRequestType & USB_DIR_IN)) {
 450			spin_unlock_irq(&ffs->ev.waitq.lock);
 451			ret = __ffs_ep0_stall(ffs);
 452			break;
 453		}
 454
 455		/* FFS_SETUP_PENDING and not stall */
 456		len = min_t(size_t, len, le16_to_cpu(ffs->ev.setup.wLength));
 457
 458		spin_unlock_irq(&ffs->ev.waitq.lock);
 459
 460		data = ffs_prepare_buffer(buf, len);
 461		if (IS_ERR(data)) {
 462			ret = PTR_ERR(data);
 463			break;
 464		}
 465
 466		spin_lock_irq(&ffs->ev.waitq.lock);
 467
 468		/*
 469		 * We are guaranteed to be still in FFS_ACTIVE state
 470		 * but the state of setup could have changed from
 471		 * FFS_SETUP_PENDING to FFS_SETUP_CANCELLED so we need
 472		 * to check for that.  If that happened we copied data
 473		 * from user space in vain but it's unlikely.
 474		 *
 475		 * For sure we are not in FFS_NO_SETUP since this is
 476		 * the only place FFS_SETUP_PENDING -> FFS_NO_SETUP
 477		 * transition can be performed and it's protected by
 478		 * mutex.
 479		 */
 480		if (ffs_setup_state_clear_cancelled(ffs) ==
 481		    FFS_SETUP_CANCELLED) {
 482			ret = -EIDRM;
 483done_spin:
 484			spin_unlock_irq(&ffs->ev.waitq.lock);
 485		} else {
 486			/* unlocks spinlock */
 487			ret = __ffs_ep0_queue_wait(ffs, data, len);
 488		}
 489		kfree(data);
 490		break;
 491
 492	default:
 493		ret = -EBADFD;
 494		break;
 495	}
 496
 497	mutex_unlock(&ffs->mutex);
 498	return ret;
 499}
 500
 501/* Called with ffs->ev.waitq.lock and ffs->mutex held, both released on exit. */
 502static ssize_t __ffs_ep0_read_events(struct ffs_data *ffs, char __user *buf,
 503				     size_t n)
 504	__releases(&ffs->ev.waitq.lock)
 505{
 506	/*
 507	 * n cannot be bigger than ffs->ev.count, which cannot be bigger than
 508	 * size of ffs->ev.types array (which is four) so that's how much space
 509	 * we reserve.
 510	 */
 511	struct usb_functionfs_event events[ARRAY_SIZE(ffs->ev.types)];
 512	const size_t size = n * sizeof *events;
 513	unsigned i = 0;
 514
 515	memset(events, 0, size);
 516
 517	do {
 518		events[i].type = ffs->ev.types[i];
 519		if (events[i].type == FUNCTIONFS_SETUP) {
 520			events[i].u.setup = ffs->ev.setup;
 521			ffs->setup_state = FFS_SETUP_PENDING;
 522		}
 523	} while (++i < n);
 524
 525	ffs->ev.count -= n;
 526	if (ffs->ev.count)
 527		memmove(ffs->ev.types, ffs->ev.types + n,
 528			ffs->ev.count * sizeof *ffs->ev.types);
 529
 530	spin_unlock_irq(&ffs->ev.waitq.lock);
 531	mutex_unlock(&ffs->mutex);
 532
 533	return copy_to_user(buf, events, size) ? -EFAULT : size;
 534}
 535
 536static ssize_t ffs_ep0_read(struct file *file, char __user *buf,
 537			    size_t len, loff_t *ptr)
 538{
 539	struct ffs_data *ffs = file->private_data;
 540	char *data = NULL;
 541	size_t n;
 542	int ret;
 543
 544	/* Fast check if setup was canceled */
 545	if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED)
 546		return -EIDRM;
 547
 548	/* Acquire mutex */
 549	ret = ffs_mutex_lock(&ffs->mutex, file->f_flags & O_NONBLOCK);
 550	if (ret < 0)
 551		return ret;
 552
 553	/* Check state */
 554	if (ffs->state != FFS_ACTIVE) {
 555		ret = -EBADFD;
 556		goto done_mutex;
 557	}
 558
 559	/*
 560	 * We're called from user space, we can use _irq rather then
 561	 * _irqsave
 562	 */
 563	spin_lock_irq(&ffs->ev.waitq.lock);
 564
 565	switch (ffs_setup_state_clear_cancelled(ffs)) {
 566	case FFS_SETUP_CANCELLED:
 567		ret = -EIDRM;
 568		break;
 569
 570	case FFS_NO_SETUP:
 571		n = len / sizeof(struct usb_functionfs_event);
 572		if (!n) {
 573			ret = -EINVAL;
 574			break;
 575		}
 576
 577		if ((file->f_flags & O_NONBLOCK) && !ffs->ev.count) {
 578			ret = -EAGAIN;
 579			break;
 580		}
 581
 582		if (wait_event_interruptible_exclusive_locked_irq(ffs->ev.waitq,
 583							ffs->ev.count)) {
 584			ret = -EINTR;
 585			break;
 586		}
 587
 588		/* unlocks spinlock */
 589		return __ffs_ep0_read_events(ffs, buf,
 590					     min_t(size_t, n, ffs->ev.count));
 591
 592	case FFS_SETUP_PENDING:
 593		if (ffs->ev.setup.bRequestType & USB_DIR_IN) {
 594			spin_unlock_irq(&ffs->ev.waitq.lock);
 595			ret = __ffs_ep0_stall(ffs);
 596			goto done_mutex;
 597		}
 598
 599		len = min_t(size_t, len, le16_to_cpu(ffs->ev.setup.wLength));
 600
 601		spin_unlock_irq(&ffs->ev.waitq.lock);
 602
 603		if (len) {
 604			data = kmalloc(len, GFP_KERNEL);
 605			if (!data) {
 606				ret = -ENOMEM;
 607				goto done_mutex;
 608			}
 609		}
 610
 611		spin_lock_irq(&ffs->ev.waitq.lock);
 612
 613		/* See ffs_ep0_write() */
 614		if (ffs_setup_state_clear_cancelled(ffs) ==
 615		    FFS_SETUP_CANCELLED) {
 616			ret = -EIDRM;
 617			break;
 618		}
 619
 620		/* unlocks spinlock */
 621		ret = __ffs_ep0_queue_wait(ffs, data, len);
 622		if ((ret > 0) && (copy_to_user(buf, data, len)))
 623			ret = -EFAULT;
 624		goto done_mutex;
 625
 626	default:
 627		ret = -EBADFD;
 628		break;
 629	}
 630
 631	spin_unlock_irq(&ffs->ev.waitq.lock);
 632done_mutex:
 633	mutex_unlock(&ffs->mutex);
 634	kfree(data);
 635	return ret;
 636}
 637
 638
 639static void ffs_data_reset(struct ffs_data *ffs);
 640
 641static int ffs_ep0_open(struct inode *inode, struct file *file)
 642{
 643	struct ffs_data *ffs = inode->i_sb->s_fs_info;
 644
 645	spin_lock_irq(&ffs->eps_lock);
 646	if (ffs->state == FFS_CLOSING) {
 647		spin_unlock_irq(&ffs->eps_lock);
 648		return -EBUSY;
 649	}
 650	if (!ffs->opened++ && ffs->state == FFS_DEACTIVATED) {
 651		ffs->state = FFS_CLOSING;
 652		spin_unlock_irq(&ffs->eps_lock);
 653		ffs_data_reset(ffs);
 654	} else {
 655		spin_unlock_irq(&ffs->eps_lock);
 656	}
 657	file->private_data = ffs;
 658
 659	return stream_open(inode, file);
 660}
 661
 662static int ffs_ep0_release(struct inode *inode, struct file *file)
 663{
 664	struct ffs_data *ffs = file->private_data;
 665
 666	ffs_data_closed(ffs);
 667
 668	return 0;
 669}
 670
 671static long ffs_ep0_ioctl(struct file *file, unsigned code, unsigned long value)
 672{
 673	struct ffs_data *ffs = file->private_data;
 674	struct usb_gadget *gadget = ffs->gadget;
 675	long ret;
 676
 677	if (code == FUNCTIONFS_INTERFACE_REVMAP) {
 678		struct ffs_function *func = ffs->func;
 679		ret = func ? ffs_func_revmap_intf(func, value) : -ENODEV;
 680	} else if (gadget && gadget->ops->ioctl) {
 681		ret = gadget->ops->ioctl(gadget, code, value);
 682	} else {
 683		ret = -ENOTTY;
 684	}
 685
 686	return ret;
 687}
 688
 689static __poll_t ffs_ep0_poll(struct file *file, poll_table *wait)
 690{
 691	struct ffs_data *ffs = file->private_data;
 692	__poll_t mask = EPOLLWRNORM;
 693	int ret;
 694
 695	poll_wait(file, &ffs->ev.waitq, wait);
 696
 697	ret = ffs_mutex_lock(&ffs->mutex, file->f_flags & O_NONBLOCK);
 698	if (ret < 0)
 699		return mask;
 700
 701	switch (ffs->state) {
 702	case FFS_READ_DESCRIPTORS:
 703	case FFS_READ_STRINGS:
 704		mask |= EPOLLOUT;
 705		break;
 706
 707	case FFS_ACTIVE:
 708		switch (ffs->setup_state) {
 709		case FFS_NO_SETUP:
 710			if (ffs->ev.count)
 711				mask |= EPOLLIN;
 712			break;
 713
 714		case FFS_SETUP_PENDING:
 715		case FFS_SETUP_CANCELLED:
 716			mask |= (EPOLLIN | EPOLLOUT);
 717			break;
 718		}
 719		break;
 720
 721	case FFS_CLOSING:
 722		break;
 723	case FFS_DEACTIVATED:
 724		break;
 725	}
 726
 727	mutex_unlock(&ffs->mutex);
 728
 729	return mask;
 730}
 731
 732static const struct file_operations ffs_ep0_operations = {
 733
 734	.open =		ffs_ep0_open,
 735	.write =	ffs_ep0_write,
 736	.read =		ffs_ep0_read,
 737	.release =	ffs_ep0_release,
 738	.unlocked_ioctl =	ffs_ep0_ioctl,
 739	.poll =		ffs_ep0_poll,
 740};
 741
 742
 743/* "Normal" endpoints operations ********************************************/
 744
 745static void ffs_epfile_io_complete(struct usb_ep *_ep, struct usb_request *req)
 746{
 747	struct ffs_io_data *io_data = req->context;
 748
 749	if (req->status)
 750		io_data->status = req->status;
 751	else
 752		io_data->status = req->actual;
 753
 754	complete(&io_data->done);
 755}
 756
 757static ssize_t ffs_copy_to_iter(void *data, int data_len, struct iov_iter *iter)
 758{
 759	ssize_t ret = copy_to_iter(data, data_len, iter);
 760	if (ret == data_len)
 761		return ret;
 762
 763	if (iov_iter_count(iter))
 764		return -EFAULT;
 765
 766	/*
 767	 * Dear user space developer!
 768	 *
 769	 * TL;DR: To stop getting below error message in your kernel log, change
 770	 * user space code using functionfs to align read buffers to a max
 771	 * packet size.
 772	 *
 773	 * Some UDCs (e.g. dwc3) require request sizes to be a multiple of a max
 774	 * packet size.  When unaligned buffer is passed to functionfs, it
 775	 * internally uses a larger, aligned buffer so that such UDCs are happy.
 776	 *
 777	 * Unfortunately, this means that host may send more data than was
 778	 * requested in read(2) system call.  f_fs doesn’t know what to do with
 779	 * that excess data so it simply drops it.
 780	 *
 781	 * Was the buffer aligned in the first place, no such problem would
 782	 * happen.
 783	 *
 784	 * Data may be dropped only in AIO reads.  Synchronous reads are handled
 785	 * by splitting a request into multiple parts.  This splitting may still
 786	 * be a problem though so it’s likely best to align the buffer
 787	 * regardless of it being AIO or not..
 788	 *
 789	 * This only affects OUT endpoints, i.e. reading data with a read(2),
 790	 * aio_read(2) etc. system calls.  Writing data to an IN endpoint is not
 791	 * affected.
 792	 */
 793	pr_err("functionfs read size %d > requested size %zd, dropping excess data. "
 794	       "Align read buffer size to max packet size to avoid the problem.\n",
 795	       data_len, ret);
 796
 797	return ret;
 798}
 799
 800/*
 801 * allocate a virtually contiguous buffer and create a scatterlist describing it
 802 * @sg_table	- pointer to a place to be filled with sg_table contents
 803 * @size	- required buffer size
 804 */
 805static void *ffs_build_sg_list(struct sg_table *sgt, size_t sz)
 806{
 807	struct page **pages;
 808	void *vaddr, *ptr;
 809	unsigned int n_pages;
 810	int i;
 811
 812	vaddr = vmalloc(sz);
 813	if (!vaddr)
 814		return NULL;
 815
 816	n_pages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
 817	pages = kvmalloc_array(n_pages, sizeof(struct page *), GFP_KERNEL);
 818	if (!pages) {
 819		vfree(vaddr);
 820
 821		return NULL;
 822	}
 823	for (i = 0, ptr = vaddr; i < n_pages; ++i, ptr += PAGE_SIZE)
 824		pages[i] = vmalloc_to_page(ptr);
 825
 826	if (sg_alloc_table_from_pages(sgt, pages, n_pages, 0, sz, GFP_KERNEL)) {
 827		kvfree(pages);
 828		vfree(vaddr);
 829
 830		return NULL;
 831	}
 832	kvfree(pages);
 833
 834	return vaddr;
 835}
 836
 837static inline void *ffs_alloc_buffer(struct ffs_io_data *io_data,
 838	size_t data_len)
 839{
 840	if (io_data->use_sg)
 841		return ffs_build_sg_list(&io_data->sgt, data_len);
 842
 843	return kmalloc(data_len, GFP_KERNEL);
 844}
 845
 846static inline void ffs_free_buffer(struct ffs_io_data *io_data)
 847{
 848	if (!io_data->buf)
 849		return;
 850
 851	if (io_data->use_sg) {
 852		sg_free_table(&io_data->sgt);
 853		vfree(io_data->buf);
 854	} else {
 855		kfree(io_data->buf);
 856	}
 857}
 858
 859static void ffs_user_copy_worker(struct work_struct *work)
 860{
 861	struct ffs_io_data *io_data = container_of(work, struct ffs_io_data,
 862						   work);
 863	int ret = io_data->status;
 864	bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD;
 865
 866	if (io_data->read && ret > 0) {
 867		kthread_use_mm(io_data->mm);
 868		ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data);
 869		kthread_unuse_mm(io_data->mm);
 870	}
 871
 872	io_data->kiocb->ki_complete(io_data->kiocb, ret);
 873
 874	if (io_data->ffs->ffs_eventfd && !kiocb_has_eventfd)
 875		eventfd_signal(io_data->ffs->ffs_eventfd);
 876
 877	usb_ep_free_request(io_data->ep, io_data->req);
 878
 879	if (io_data->read)
 880		kfree(io_data->to_free);
 881	ffs_free_buffer(io_data);
 882	kfree(io_data);
 883}
 884
 885static void ffs_epfile_async_io_complete(struct usb_ep *_ep,
 886					 struct usb_request *req)
 887{
 888	struct ffs_io_data *io_data = req->context;
 889	struct ffs_data *ffs = io_data->ffs;
 890
 891	io_data->status = req->status ? req->status : req->actual;
 892
 893	INIT_WORK(&io_data->work, ffs_user_copy_worker);
 894	queue_work(ffs->io_completion_wq, &io_data->work);
 895}
 896
 897static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile)
 898{
 899	/*
 900	 * See comment in struct ffs_epfile for full read_buffer pointer
 901	 * synchronisation story.
 902	 */
 903	struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP);
 904	if (buf && buf != READ_BUFFER_DROP)
 905		kfree(buf);
 906}
 907
 908/* Assumes epfile->mutex is held. */
 909static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile,
 910					  struct iov_iter *iter)
 911{
 912	/*
 913	 * Null out epfile->read_buffer so ffs_func_eps_disable does not free
 914	 * the buffer while we are using it.  See comment in struct ffs_epfile
 915	 * for full read_buffer pointer synchronisation story.
 916	 */
 917	struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL);
 918	ssize_t ret;
 919	if (!buf || buf == READ_BUFFER_DROP)
 920		return 0;
 921
 922	ret = copy_to_iter(buf->data, buf->length, iter);
 923	if (buf->length == ret) {
 924		kfree(buf);
 925		return ret;
 926	}
 927
 928	if (iov_iter_count(iter)) {
 929		ret = -EFAULT;
 930	} else {
 931		buf->length -= ret;
 932		buf->data += ret;
 933	}
 934
 935	if (cmpxchg(&epfile->read_buffer, NULL, buf))
 936		kfree(buf);
 937
 938	return ret;
 939}
 940
 941/* Assumes epfile->mutex is held. */
 942static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile,
 943				      void *data, int data_len,
 944				      struct iov_iter *iter)
 945{
 946	struct ffs_buffer *buf;
 947
 948	ssize_t ret = copy_to_iter(data, data_len, iter);
 949	if (data_len == ret)
 950		return ret;
 951
 952	if (iov_iter_count(iter))
 953		return -EFAULT;
 954
 955	/* See ffs_copy_to_iter for more context. */
 956	pr_warn("functionfs read size %d > requested size %zd, splitting request into multiple reads.",
 957		data_len, ret);
 958
 959	data_len -= ret;
 960	buf = kmalloc(struct_size(buf, storage, data_len), GFP_KERNEL);
 961	if (!buf)
 962		return -ENOMEM;
 963	buf->length = data_len;
 964	buf->data = buf->storage;
 965	memcpy(buf->storage, data + ret, flex_array_size(buf, storage, data_len));
 966
 967	/*
 968	 * At this point read_buffer is NULL or READ_BUFFER_DROP (if
 969	 * ffs_func_eps_disable has been called in the meanwhile).  See comment
 970	 * in struct ffs_epfile for full read_buffer pointer synchronisation
 971	 * story.
 972	 */
 973	if (cmpxchg(&epfile->read_buffer, NULL, buf))
 974		kfree(buf);
 975
 976	return ret;
 977}
 978
 979static struct ffs_ep *ffs_epfile_wait_ep(struct file *file)
 980{
 981	struct ffs_epfile *epfile = file->private_data;
 982	struct ffs_ep *ep;
 983	int ret;
 984
 985	/* Wait for endpoint to be enabled */
 986	ep = epfile->ep;
 987	if (!ep) {
 988		if (file->f_flags & O_NONBLOCK)
 989			return ERR_PTR(-EAGAIN);
 990
 991		ret = wait_event_interruptible(
 992				epfile->ffs->wait, (ep = epfile->ep));
 993		if (ret)
 994			return ERR_PTR(-EINTR);
 995	}
 996
 997	return ep;
 998}
 999
1000static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)
1001{
1002	struct ffs_epfile *epfile = file->private_data;
1003	struct usb_request *req;
1004	struct ffs_ep *ep;
1005	char *data = NULL;
1006	ssize_t ret, data_len = -EINVAL;
1007	int halt;
1008
1009	/* Are we still active? */
1010	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
1011		return -ENODEV;
1012
1013	ep = ffs_epfile_wait_ep(file);
1014	if (IS_ERR(ep))
1015		return PTR_ERR(ep);
1016
1017	/* Do we halt? */
1018	halt = (!io_data->read == !epfile->in);
1019	if (halt && epfile->isoc)
1020		return -EINVAL;
1021
1022	/* We will be using request and read_buffer */
1023	ret = ffs_mutex_lock(&epfile->mutex, file->f_flags & O_NONBLOCK);
1024	if (ret)
1025		goto error;
1026
1027	/* Allocate & copy */
1028	if (!halt) {
1029		struct usb_gadget *gadget;
1030
1031		/*
1032		 * Do we have buffered data from previous partial read?  Check
1033		 * that for synchronous case only because we do not have
1034		 * facility to ‘wake up’ a pending asynchronous read and push
1035		 * buffered data to it which we would need to make things behave
1036		 * consistently.
1037		 */
1038		if (!io_data->aio && io_data->read) {
1039			ret = __ffs_epfile_read_buffered(epfile, &io_data->data);
1040			if (ret)
1041				goto error_mutex;
1042		}
1043
1044		/*
1045		 * if we _do_ wait above, the epfile->ffs->gadget might be NULL
1046		 * before the waiting completes, so do not assign to 'gadget'
1047		 * earlier
1048		 */
1049		gadget = epfile->ffs->gadget;
1050
1051		spin_lock_irq(&epfile->ffs->eps_lock);
1052		/* In the meantime, endpoint got disabled or changed. */
1053		if (epfile->ep != ep) {
1054			ret = -ESHUTDOWN;
1055			goto error_lock;
1056		}
1057		data_len = iov_iter_count(&io_data->data);
1058		/*
1059		 * Controller may require buffer size to be aligned to
1060		 * maxpacketsize of an out endpoint.
1061		 */
1062		if (io_data->read)
1063			data_len = usb_ep_align_maybe(gadget, ep->ep, data_len);
1064
1065		io_data->use_sg = gadget->sg_supported && data_len > PAGE_SIZE;
1066		spin_unlock_irq(&epfile->ffs->eps_lock);
1067
1068		data = ffs_alloc_buffer(io_data, data_len);
1069		if (!data) {
1070			ret = -ENOMEM;
1071			goto error_mutex;
1072		}
1073		if (!io_data->read &&
1074		    !copy_from_iter_full(data, data_len, &io_data->data)) {
1075			ret = -EFAULT;
1076			goto error_mutex;
1077		}
1078	}
1079
1080	spin_lock_irq(&epfile->ffs->eps_lock);
1081
1082	if (epfile->ep != ep) {
1083		/* In the meantime, endpoint got disabled or changed. */
1084		ret = -ESHUTDOWN;
1085	} else if (halt) {
1086		ret = usb_ep_set_halt(ep->ep);
1087		if (!ret)
1088			ret = -EBADMSG;
1089	} else if (data_len == -EINVAL) {
1090		/*
1091		 * Sanity Check: even though data_len can't be used
1092		 * uninitialized at the time I write this comment, some
1093		 * compilers complain about this situation.
1094		 * In order to keep the code clean from warnings, data_len is
1095		 * being initialized to -EINVAL during its declaration, which
1096		 * means we can't rely on compiler anymore to warn no future
1097		 * changes won't result in data_len being used uninitialized.
1098		 * For such reason, we're adding this redundant sanity check
1099		 * here.
1100		 */
1101		WARN(1, "%s: data_len == -EINVAL\n", __func__);
1102		ret = -EINVAL;
1103	} else if (!io_data->aio) {
1104		bool interrupted = false;
1105
1106		req = ep->req;
1107		if (io_data->use_sg) {
1108			req->buf = NULL;
1109			req->sg	= io_data->sgt.sgl;
1110			req->num_sgs = io_data->sgt.nents;
1111		} else {
1112			req->buf = data;
1113			req->num_sgs = 0;
1114		}
1115		req->length = data_len;
1116
1117		io_data->buf = data;
1118
1119		init_completion(&io_data->done);
1120		req->context  = io_data;
1121		req->complete = ffs_epfile_io_complete;
1122
1123		ret = usb_ep_queue(ep->ep, req, GFP_ATOMIC);
1124		if (ret < 0)
1125			goto error_lock;
1126
1127		spin_unlock_irq(&epfile->ffs->eps_lock);
1128
1129		if (wait_for_completion_interruptible(&io_data->done)) {
1130			spin_lock_irq(&epfile->ffs->eps_lock);
1131			if (epfile->ep != ep) {
1132				ret = -ESHUTDOWN;
1133				goto error_lock;
1134			}
1135			/*
1136			 * To avoid race condition with ffs_epfile_io_complete,
1137			 * dequeue the request first then check
1138			 * status. usb_ep_dequeue API should guarantee no race
1139			 * condition with req->complete callback.
1140			 */
1141			usb_ep_dequeue(ep->ep, req);
1142			spin_unlock_irq(&epfile->ffs->eps_lock);
1143			wait_for_completion(&io_data->done);
1144			interrupted = io_data->status < 0;
1145		}
1146
1147		if (interrupted)
1148			ret = -EINTR;
1149		else if (io_data->read && io_data->status > 0)
1150			ret = __ffs_epfile_read_data(epfile, data, io_data->status,
1151						     &io_data->data);
1152		else
1153			ret = io_data->status;
1154		goto error_mutex;
1155	} else if (!(req = usb_ep_alloc_request(ep->ep, GFP_ATOMIC))) {
1156		ret = -ENOMEM;
1157	} else {
1158		if (io_data->use_sg) {
1159			req->buf = NULL;
1160			req->sg	= io_data->sgt.sgl;
1161			req->num_sgs = io_data->sgt.nents;
1162		} else {
1163			req->buf = data;
1164			req->num_sgs = 0;
1165		}
1166		req->length = data_len;
1167
1168		io_data->buf = data;
1169		io_data->ep = ep->ep;
1170		io_data->req = req;
1171		io_data->ffs = epfile->ffs;
1172
1173		req->context  = io_data;
1174		req->complete = ffs_epfile_async_io_complete;
1175
1176		ret = usb_ep_queue(ep->ep, req, GFP_ATOMIC);
1177		if (ret) {
1178			io_data->req = NULL;
1179			usb_ep_free_request(ep->ep, req);
1180			goto error_lock;
1181		}
1182
1183		ret = -EIOCBQUEUED;
1184		/*
1185		 * Do not kfree the buffer in this function.  It will be freed
1186		 * by ffs_user_copy_worker.
1187		 */
1188		data = NULL;
1189	}
1190
1191error_lock:
1192	spin_unlock_irq(&epfile->ffs->eps_lock);
1193error_mutex:
1194	mutex_unlock(&epfile->mutex);
1195error:
1196	if (ret != -EIOCBQUEUED) /* don't free if there is iocb queued */
1197		ffs_free_buffer(io_data);
1198	return ret;
1199}
1200
1201static int
1202ffs_epfile_open(struct inode *inode, struct file *file)
1203{
1204	struct ffs_data *ffs = inode->i_sb->s_fs_info;
1205	struct ffs_epfile *epfile;
1206
1207	spin_lock_irq(&ffs->eps_lock);
1208	if (!ffs->opened) {
1209		spin_unlock_irq(&ffs->eps_lock);
1210		return -ENODEV;
1211	}
1212	/*
1213	 * we want the state to be FFS_ACTIVE; FFS_ACTIVE alone is
1214	 * not enough, though - we might have been through FFS_CLOSING
1215	 * and back to FFS_ACTIVE, with our file already removed.
1216	 */
1217	epfile = smp_load_acquire(&inode->i_private);
1218	if (unlikely(ffs->state != FFS_ACTIVE || !epfile)) {
1219		spin_unlock_irq(&ffs->eps_lock);
1220		return -ENODEV;
1221	}
1222	ffs->opened++;
1223	spin_unlock_irq(&ffs->eps_lock);
1224
1225	file->private_data = epfile;
1226	return stream_open(inode, file);
1227}
1228
1229static int ffs_aio_cancel(struct kiocb *kiocb)
1230{
1231	struct ffs_io_data *io_data = kiocb->private;
1232	int value;
1233
1234	if (io_data && io_data->ep && io_data->req)
1235		value = usb_ep_dequeue(io_data->ep, io_data->req);
1236	else
1237		value = -EINVAL;
1238
1239	return value;
1240}
1241
1242static ssize_t ffs_epfile_write_iter(struct kiocb *kiocb, struct iov_iter *from)
1243{
1244	struct ffs_io_data io_data, *p = &io_data;
1245	ssize_t res;
1246
1247	if (!is_sync_kiocb(kiocb)) {
1248		p = kzalloc(sizeof(io_data), GFP_KERNEL);
1249		if (!p)
1250			return -ENOMEM;
1251		p->aio = true;
1252	} else {
1253		memset(p, 0, sizeof(*p));
1254		p->aio = false;
1255	}
1256
1257	p->read = false;
1258	p->kiocb = kiocb;
1259	p->data = *from;
1260	p->mm = current->mm;
1261
1262	kiocb->private = p;
1263
1264	if (p->aio)
1265		kiocb_set_cancel_fn(kiocb, ffs_aio_cancel);
1266
1267	res = ffs_epfile_io(kiocb->ki_filp, p);
1268	if (res == -EIOCBQUEUED)
1269		return res;
1270	if (p->aio)
1271		kfree(p);
1272	else
1273		*from = p->data;
1274	return res;
1275}
1276
1277static ssize_t ffs_epfile_read_iter(struct kiocb *kiocb, struct iov_iter *to)
1278{
1279	struct ffs_io_data io_data, *p = &io_data;
1280	ssize_t res;
1281
1282	if (!is_sync_kiocb(kiocb)) {
1283		p = kzalloc(sizeof(io_data), GFP_KERNEL);
1284		if (!p)
1285			return -ENOMEM;
1286		p->aio = true;
1287	} else {
1288		memset(p, 0, sizeof(*p));
1289		p->aio = false;
1290	}
1291
1292	p->read = true;
1293	p->kiocb = kiocb;
1294	if (p->aio) {
1295		p->to_free = dup_iter(&p->data, to, GFP_KERNEL);
1296		if (!iter_is_ubuf(&p->data) && !p->to_free) {
1297			kfree(p);
1298			return -ENOMEM;
1299		}
1300	} else {
1301		p->data = *to;
1302		p->to_free = NULL;
1303	}
1304	p->mm = current->mm;
1305
1306	kiocb->private = p;
1307
1308	if (p->aio)
1309		kiocb_set_cancel_fn(kiocb, ffs_aio_cancel);
1310
1311	res = ffs_epfile_io(kiocb->ki_filp, p);
1312	if (res == -EIOCBQUEUED)
1313		return res;
1314
1315	if (p->aio) {
1316		kfree(p->to_free);
1317		kfree(p);
1318	} else {
1319		*to = p->data;
1320	}
1321	return res;
1322}
1323
1324static void ffs_dmabuf_release(struct kref *ref)
1325{
1326	struct ffs_dmabuf_priv *priv = container_of(ref, struct ffs_dmabuf_priv, ref);
1327	struct dma_buf_attachment *attach = priv->attach;
1328	struct dma_buf *dmabuf = attach->dmabuf;
1329
1330	pr_vdebug("FFS DMABUF release\n");
1331	dma_buf_unmap_attachment_unlocked(attach, priv->sgt, priv->dir);
1332
1333	dma_buf_detach(attach->dmabuf, attach);
1334	dma_buf_put(dmabuf);
1335	kfree(priv);
1336}
1337
1338static void ffs_dmabuf_get(struct dma_buf_attachment *attach)
1339{
1340	struct ffs_dmabuf_priv *priv = attach->importer_priv;
1341
1342	kref_get(&priv->ref);
1343}
1344
1345static void ffs_dmabuf_put(struct dma_buf_attachment *attach)
1346{
1347	struct ffs_dmabuf_priv *priv = attach->importer_priv;
1348
1349	kref_put(&priv->ref, ffs_dmabuf_release);
1350}
1351
1352static int
1353ffs_epfile_release(struct inode *inode, struct file *file)
1354{
1355	struct ffs_epfile *epfile = file->private_data;
1356	struct ffs_dmabuf_priv *priv, *tmp;
1357	struct ffs_data *ffs = epfile->ffs;
1358
1359	mutex_lock(&epfile->dmabufs_mutex);
1360
1361	/* Close all attached DMABUFs */
1362	list_for_each_entry_safe(priv, tmp, &epfile->dmabufs, entry) {
1363		/* Cancel any pending transfer */
1364		spin_lock_irq(&ffs->eps_lock);
1365		if (priv->ep && priv->req)
1366			usb_ep_dequeue(priv->ep, priv->req);
1367		spin_unlock_irq(&ffs->eps_lock);
1368
1369		list_del(&priv->entry);
1370		ffs_dmabuf_put(priv->attach);
1371	}
1372
1373	mutex_unlock(&epfile->dmabufs_mutex);
1374
1375	__ffs_epfile_read_buffer_free(epfile);
1376	ffs_data_closed(epfile->ffs);
1377
1378	return 0;
1379}
1380
1381static void ffs_dmabuf_cleanup(struct work_struct *work)
1382{
1383	struct ffs_dma_fence *dma_fence =
1384		container_of(work, struct ffs_dma_fence, work);
1385	struct ffs_dmabuf_priv *priv = dma_fence->priv;
1386	struct dma_buf_attachment *attach = priv->attach;
1387	struct dma_fence *fence = &dma_fence->base;
1388
1389	ffs_dmabuf_put(attach);
1390	dma_fence_put(fence);
1391}
1392
1393static void ffs_dmabuf_signal_done(struct ffs_dma_fence *dma_fence, int ret)
1394{
1395	struct ffs_dmabuf_priv *priv = dma_fence->priv;
1396	struct dma_fence *fence = &dma_fence->base;
1397	bool cookie = dma_fence_begin_signalling();
1398
1399	dma_fence_get(fence);
1400	fence->error = ret;
1401	dma_fence_signal(fence);
1402	dma_fence_end_signalling(cookie);
1403
1404	/*
1405	 * The fence will be unref'd in ffs_dmabuf_cleanup.
1406	 * It can't be done here, as the unref functions might try to lock
1407	 * the resv object, which would deadlock.
1408	 */
1409	INIT_WORK(&dma_fence->work, ffs_dmabuf_cleanup);
1410	queue_work(priv->ffs->io_completion_wq, &dma_fence->work);
1411}
1412
1413static void ffs_epfile_dmabuf_io_complete(struct usb_ep *ep,
1414					  struct usb_request *req)
1415{
1416	pr_vdebug("FFS: DMABUF transfer complete, status=%d\n", req->status);
1417	ffs_dmabuf_signal_done(req->context, req->status);
1418	usb_ep_free_request(ep, req);
1419}
1420
1421static const char *ffs_dmabuf_get_driver_name(struct dma_fence *fence)
1422{
1423	return "functionfs";
1424}
1425
1426static const char *ffs_dmabuf_get_timeline_name(struct dma_fence *fence)
1427{
1428	return "";
1429}
1430
1431static void ffs_dmabuf_fence_release(struct dma_fence *fence)
1432{
1433	struct ffs_dma_fence *dma_fence =
1434		container_of(fence, struct ffs_dma_fence, base);
1435
1436	kfree(dma_fence);
1437}
1438
1439static const struct dma_fence_ops ffs_dmabuf_fence_ops = {
1440	.get_driver_name	= ffs_dmabuf_get_driver_name,
1441	.get_timeline_name	= ffs_dmabuf_get_timeline_name,
1442	.release		= ffs_dmabuf_fence_release,
1443};
1444
1445static int ffs_dma_resv_lock(struct dma_buf *dmabuf, bool nonblock)
1446{
1447	if (!nonblock)
1448		return dma_resv_lock_interruptible(dmabuf->resv, NULL);
1449
1450	if (!dma_resv_trylock(dmabuf->resv))
1451		return -EBUSY;
1452
1453	return 0;
1454}
1455
1456static struct dma_buf_attachment *
1457ffs_dmabuf_find_attachment(struct ffs_epfile *epfile, struct dma_buf *dmabuf)
1458{
1459	struct device *dev = epfile->ffs->gadget->dev.parent;
1460	struct dma_buf_attachment *attach = NULL;
1461	struct ffs_dmabuf_priv *priv;
1462
1463	mutex_lock(&epfile->dmabufs_mutex);
1464
1465	list_for_each_entry(priv, &epfile->dmabufs, entry) {
1466		if (priv->attach->dev == dev
1467		    && priv->attach->dmabuf == dmabuf) {
1468			attach = priv->attach;
1469			break;
1470		}
1471	}
1472
1473	if (attach)
1474		ffs_dmabuf_get(attach);
1475
1476	mutex_unlock(&epfile->dmabufs_mutex);
1477
1478	return attach ?: ERR_PTR(-EPERM);
1479}
1480
1481static int ffs_dmabuf_attach(struct file *file, int fd)
1482{
1483	bool nonblock = file->f_flags & O_NONBLOCK;
1484	struct ffs_epfile *epfile = file->private_data;
1485	struct usb_gadget *gadget = epfile->ffs->gadget;
1486	struct dma_buf_attachment *attach;
1487	struct ffs_dmabuf_priv *priv;
1488	enum dma_data_direction dir;
1489	struct sg_table *sg_table;
1490	struct dma_buf *dmabuf;
1491	int err;
1492
1493	if (!gadget || !gadget->sg_supported)
1494		return -EPERM;
1495
1496	dmabuf = dma_buf_get(fd);
1497	if (IS_ERR(dmabuf))
1498		return PTR_ERR(dmabuf);
1499
1500	attach = dma_buf_attach(dmabuf, gadget->dev.parent);
1501	if (IS_ERR(attach)) {
1502		err = PTR_ERR(attach);
1503		goto err_dmabuf_put;
1504	}
1505
1506	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1507	if (!priv) {
1508		err = -ENOMEM;
1509		goto err_dmabuf_detach;
1510	}
1511
1512	dir = epfile->in ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
1513
1514	err = ffs_dma_resv_lock(dmabuf, nonblock);
1515	if (err)
1516		goto err_free_priv;
1517
1518	sg_table = dma_buf_map_attachment(attach, dir);
1519	dma_resv_unlock(dmabuf->resv);
1520
1521	if (IS_ERR(sg_table)) {
1522		err = PTR_ERR(sg_table);
1523		goto err_free_priv;
1524	}
1525
1526	attach->importer_priv = priv;
1527
1528	priv->sgt = sg_table;
1529	priv->dir = dir;
1530	priv->ffs = epfile->ffs;
1531	priv->attach = attach;
1532	spin_lock_init(&priv->lock);
1533	kref_init(&priv->ref);
1534	priv->context = dma_fence_context_alloc(1);
1535
1536	mutex_lock(&epfile->dmabufs_mutex);
1537	list_add(&priv->entry, &epfile->dmabufs);
1538	mutex_unlock(&epfile->dmabufs_mutex);
1539
1540	return 0;
1541
1542err_free_priv:
1543	kfree(priv);
1544err_dmabuf_detach:
1545	dma_buf_detach(dmabuf, attach);
1546err_dmabuf_put:
1547	dma_buf_put(dmabuf);
1548
1549	return err;
1550}
1551
1552static int ffs_dmabuf_detach(struct file *file, int fd)
1553{
1554	struct ffs_epfile *epfile = file->private_data;
1555	struct ffs_data *ffs = epfile->ffs;
1556	struct device *dev = ffs->gadget->dev.parent;
1557	struct ffs_dmabuf_priv *priv, *tmp;
1558	struct dma_buf *dmabuf;
1559	int ret = -EPERM;
1560
1561	dmabuf = dma_buf_get(fd);
1562	if (IS_ERR(dmabuf))
1563		return PTR_ERR(dmabuf);
1564
1565	mutex_lock(&epfile->dmabufs_mutex);
1566
1567	list_for_each_entry_safe(priv, tmp, &epfile->dmabufs, entry) {
1568		if (priv->attach->dev == dev
1569		    && priv->attach->dmabuf == dmabuf) {
1570			/* Cancel any pending transfer */
1571			spin_lock_irq(&ffs->eps_lock);
1572			if (priv->ep && priv->req)
1573				usb_ep_dequeue(priv->ep, priv->req);
1574			spin_unlock_irq(&ffs->eps_lock);
1575
1576			list_del(&priv->entry);
1577
1578			/* Unref the reference from ffs_dmabuf_attach() */
1579			ffs_dmabuf_put(priv->attach);
1580			ret = 0;
1581			break;
1582		}
1583	}
1584
1585	mutex_unlock(&epfile->dmabufs_mutex);
1586	dma_buf_put(dmabuf);
1587
1588	return ret;
1589}
1590
1591static int ffs_dmabuf_transfer(struct file *file,
1592			       const struct usb_ffs_dmabuf_transfer_req *req)
1593{
1594	bool nonblock = file->f_flags & O_NONBLOCK;
1595	struct ffs_epfile *epfile = file->private_data;
1596	struct dma_buf_attachment *attach;
1597	struct ffs_dmabuf_priv *priv;
1598	struct ffs_dma_fence *fence;
1599	struct usb_request *usb_req;
1600	enum dma_resv_usage resv_dir;
1601	struct dma_buf *dmabuf;
1602	unsigned long timeout;
1603	struct ffs_ep *ep;
1604	bool cookie;
1605	u32 seqno;
1606	long retl;
1607	int ret;
1608
1609	if (req->flags & ~USB_FFS_DMABUF_TRANSFER_MASK)
1610		return -EINVAL;
1611
1612	dmabuf = dma_buf_get(req->fd);
1613	if (IS_ERR(dmabuf))
1614		return PTR_ERR(dmabuf);
1615
1616	if (req->length > dmabuf->size || req->length == 0) {
1617		ret = -EINVAL;
1618		goto err_dmabuf_put;
1619	}
1620
1621	attach = ffs_dmabuf_find_attachment(epfile, dmabuf);
1622	if (IS_ERR(attach)) {
1623		ret = PTR_ERR(attach);
1624		goto err_dmabuf_put;
1625	}
1626
1627	priv = attach->importer_priv;
1628
1629	ep = ffs_epfile_wait_ep(file);
1630	if (IS_ERR(ep)) {
1631		ret = PTR_ERR(ep);
1632		goto err_attachment_put;
1633	}
1634
1635	ret = ffs_dma_resv_lock(dmabuf, nonblock);
1636	if (ret)
1637		goto err_attachment_put;
1638
1639	/* Make sure we don't have writers */
1640	timeout = nonblock ? 0 : msecs_to_jiffies(DMABUF_ENQUEUE_TIMEOUT_MS);
1641	retl = dma_resv_wait_timeout(dmabuf->resv,
1642				     dma_resv_usage_rw(!epfile->in),
1643				     true, timeout);
1644	if (retl == 0)
1645		retl = -EBUSY;
1646	if (retl < 0) {
1647		ret = (int)retl;
1648		goto err_resv_unlock;
1649	}
1650
1651	ret = dma_resv_reserve_fences(dmabuf->resv, 1);
1652	if (ret)
1653		goto err_resv_unlock;
1654
1655	fence = kmalloc(sizeof(*fence), GFP_KERNEL);
1656	if (!fence) {
1657		ret = -ENOMEM;
1658		goto err_resv_unlock;
1659	}
1660
1661	fence->priv = priv;
1662
1663	spin_lock_irq(&epfile->ffs->eps_lock);
1664
1665	/* In the meantime, endpoint got disabled or changed. */
1666	if (epfile->ep != ep) {
1667		ret = -ESHUTDOWN;
1668		goto err_fence_put;
1669	}
1670
1671	usb_req = usb_ep_alloc_request(ep->ep, GFP_ATOMIC);
1672	if (!usb_req) {
1673		ret = -ENOMEM;
1674		goto err_fence_put;
1675	}
1676
1677	/*
1678	 * usb_ep_queue() guarantees that all transfers are processed in the
1679	 * order they are enqueued, so we can use a simple incrementing
1680	 * sequence number for the dma_fence.
1681	 */
1682	seqno = atomic_add_return(1, &epfile->seqno);
1683
1684	dma_fence_init(&fence->base, &ffs_dmabuf_fence_ops,
1685		       &priv->lock, priv->context, seqno);
1686
1687	resv_dir = epfile->in ? DMA_RESV_USAGE_READ : DMA_RESV_USAGE_WRITE;
1688
1689	dma_resv_add_fence(dmabuf->resv, &fence->base, resv_dir);
1690	dma_resv_unlock(dmabuf->resv);
1691
1692	/* Now that the dma_fence is in place, queue the transfer. */
1693
1694	usb_req->length = req->length;
1695	usb_req->buf = NULL;
1696	usb_req->sg = priv->sgt->sgl;
1697	usb_req->num_sgs = sg_nents_for_len(priv->sgt->sgl, req->length);
1698	usb_req->sg_was_mapped = true;
1699	usb_req->context  = fence;
1700	usb_req->complete = ffs_epfile_dmabuf_io_complete;
1701
1702	cookie = dma_fence_begin_signalling();
1703	ret = usb_ep_queue(ep->ep, usb_req, GFP_ATOMIC);
1704	dma_fence_end_signalling(cookie);
1705	if (!ret) {
1706		priv->req = usb_req;
1707		priv->ep = ep->ep;
1708	} else {
1709		pr_warn("FFS: Failed to queue DMABUF: %d\n", ret);
1710		ffs_dmabuf_signal_done(fence, ret);
1711		usb_ep_free_request(ep->ep, usb_req);
1712	}
1713
1714	spin_unlock_irq(&epfile->ffs->eps_lock);
1715	dma_buf_put(dmabuf);
1716
1717	return ret;
1718
1719err_fence_put:
1720	spin_unlock_irq(&epfile->ffs->eps_lock);
1721	dma_fence_put(&fence->base);
1722err_resv_unlock:
1723	dma_resv_unlock(dmabuf->resv);
1724err_attachment_put:
1725	ffs_dmabuf_put(attach);
1726err_dmabuf_put:
1727	dma_buf_put(dmabuf);
1728
1729	return ret;
1730}
1731
1732static long ffs_epfile_ioctl(struct file *file, unsigned code,
1733			     unsigned long value)
1734{
1735	struct ffs_epfile *epfile = file->private_data;
1736	struct ffs_ep *ep;
1737	int ret;
1738
1739	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
1740		return -ENODEV;
1741
1742	switch (code) {
1743	case FUNCTIONFS_DMABUF_ATTACH:
1744	{
1745		int fd;
1746
1747		if (copy_from_user(&fd, (void __user *)value, sizeof(fd)))
1748			return -EFAULT;
1749
1750		return ffs_dmabuf_attach(file, fd);
1751	}
1752	case FUNCTIONFS_DMABUF_DETACH:
1753	{
1754		int fd;
1755
1756		if (copy_from_user(&fd, (void __user *)value, sizeof(fd)))
1757			return -EFAULT;
1758
1759		return ffs_dmabuf_detach(file, fd);
1760	}
1761	case FUNCTIONFS_DMABUF_TRANSFER:
1762	{
1763		struct usb_ffs_dmabuf_transfer_req req;
1764
1765		if (copy_from_user(&req, (void __user *)value, sizeof(req)))
1766			return -EFAULT;
1767
1768		return ffs_dmabuf_transfer(file, &req);
1769	}
1770	default:
1771		break;
1772	}
1773
1774	/* Wait for endpoint to be enabled */
1775	ep = ffs_epfile_wait_ep(file);
1776	if (IS_ERR(ep))
1777		return PTR_ERR(ep);
1778
1779	spin_lock_irq(&epfile->ffs->eps_lock);
1780
1781	/* In the meantime, endpoint got disabled or changed. */
1782	if (epfile->ep != ep) {
1783		spin_unlock_irq(&epfile->ffs->eps_lock);
1784		return -ESHUTDOWN;
1785	}
1786
1787	switch (code) {
1788	case FUNCTIONFS_FIFO_STATUS:
1789		ret = usb_ep_fifo_status(epfile->ep->ep);
1790		break;
1791	case FUNCTIONFS_FIFO_FLUSH:
1792		usb_ep_fifo_flush(epfile->ep->ep);
1793		ret = 0;
1794		break;
1795	case FUNCTIONFS_CLEAR_HALT:
1796		ret = usb_ep_clear_halt(epfile->ep->ep);
1797		break;
1798	case FUNCTIONFS_ENDPOINT_REVMAP:
1799		ret = epfile->ep->num;
1800		break;
1801	case FUNCTIONFS_ENDPOINT_DESC:
1802	{
1803		int desc_idx;
1804		struct usb_endpoint_descriptor desc1, *desc;
1805
1806		switch (epfile->ffs->gadget->speed) {
1807		case USB_SPEED_SUPER:
1808		case USB_SPEED_SUPER_PLUS:
1809			desc_idx = 2;
1810			break;
1811		case USB_SPEED_HIGH:
1812			desc_idx = 1;
1813			break;
1814		default:
1815			desc_idx = 0;
1816		}
1817
1818		desc = epfile->ep->descs[desc_idx];
1819		memcpy(&desc1, desc, desc->bLength);
1820
1821		spin_unlock_irq(&epfile->ffs->eps_lock);
1822		ret = copy_to_user((void __user *)value, &desc1, desc1.bLength);
1823		if (ret)
1824			ret = -EFAULT;
1825		return ret;
1826	}
1827	default:
1828		ret = -ENOTTY;
1829	}
1830	spin_unlock_irq(&epfile->ffs->eps_lock);
1831
1832	return ret;
1833}
1834
1835static const struct file_operations ffs_epfile_operations = {
1836
1837	.open =		ffs_epfile_open,
1838	.write_iter =	ffs_epfile_write_iter,
1839	.read_iter =	ffs_epfile_read_iter,
1840	.release =	ffs_epfile_release,
1841	.unlocked_ioctl =	ffs_epfile_ioctl,
1842	.compat_ioctl = compat_ptr_ioctl,
1843};
1844
1845
1846/* File system and super block operations ***********************************/
1847
1848/*
1849 * Mounting the file system creates a controller file, used first for
1850 * function configuration then later for event monitoring.
1851 */
1852
1853static struct inode *__must_check
1854ffs_sb_make_inode(struct super_block *sb, void *data,
1855		  const struct file_operations *fops,
1856		  const struct inode_operations *iops,
1857		  struct ffs_file_perms *perms)
1858{
1859	struct inode *inode;
1860
1861	inode = new_inode(sb);
1862
1863	if (inode) {
1864		struct timespec64 ts = inode_set_ctime_current(inode);
1865
1866		inode->i_ino	 = get_next_ino();
1867		inode->i_mode    = perms->mode;
1868		inode->i_uid     = perms->uid;
1869		inode->i_gid     = perms->gid;
1870		inode_set_atime_to_ts(inode, ts);
1871		inode_set_mtime_to_ts(inode, ts);
1872		inode->i_private = data;
1873		if (fops)
1874			inode->i_fop = fops;
1875		if (iops)
1876			inode->i_op  = iops;
1877	}
1878
1879	return inode;
1880}
1881
1882/* Create "regular" file */
1883static int ffs_sb_create_file(struct super_block *sb, const char *name,
1884			      void *data, const struct file_operations *fops)
1885{
1886	struct ffs_data	*ffs = sb->s_fs_info;
1887	struct dentry	*dentry;
1888	struct inode	*inode;
1889
1890	inode = ffs_sb_make_inode(sb, data, fops, NULL, &ffs->file_perms);
1891	if (!inode)
1892		return -ENOMEM;
1893	dentry = simple_start_creating(sb->s_root, name);
1894	if (IS_ERR(dentry)) {
1895		iput(inode);
1896		return PTR_ERR(dentry);
1897	}
1898
1899	d_make_persistent(dentry, inode);
1900
1901	simple_done_creating(dentry);
1902	return 0;
1903}
1904
1905/* Super block */
1906static const struct super_operations ffs_sb_operations = {
1907	.statfs =	simple_statfs,
1908	.drop_inode =	inode_just_drop,
1909};
1910
1911struct ffs_sb_fill_data {
1912	struct ffs_file_perms perms;
1913	umode_t root_mode;
1914	const char *dev_name;
1915	bool no_disconnect;
1916	struct ffs_data *ffs_data;
1917};
1918
1919static int ffs_sb_fill(struct super_block *sb, struct fs_context *fc)
1920{
1921	struct ffs_sb_fill_data *data = fc->fs_private;
1922	struct inode	*inode;
1923	struct ffs_data	*ffs = data->ffs_data;
1924
1925	ffs->sb              = sb;
1926	data->ffs_data       = NULL;
1927	sb->s_fs_info        = ffs;
1928	sb->s_blocksize      = PAGE_SIZE;
1929	sb->s_blocksize_bits = PAGE_SHIFT;
1930	sb->s_magic          = FUNCTIONFS_MAGIC;
1931	sb->s_op             = &ffs_sb_operations;
1932	sb->s_time_gran      = 1;
1933
1934	/* Root inode */
1935	data->perms.mode = data->root_mode;
1936	inode = ffs_sb_make_inode(sb, NULL,
1937				  &simple_dir_operations,
1938				  &simple_dir_inode_operations,
1939				  &data->perms);
1940	sb->s_root = d_make_root(inode);
1941	if (!sb->s_root)
1942		return -ENOMEM;
1943
1944	/* EP0 file */
1945	return ffs_sb_create_file(sb, "ep0", ffs, &ffs_ep0_operations);
1946}
1947
1948enum {
1949	Opt_no_disconnect,
1950	Opt_rmode,
1951	Opt_fmode,
1952	Opt_mode,
1953	Opt_uid,
1954	Opt_gid,
1955};
1956
1957static const struct fs_parameter_spec ffs_fs_fs_parameters[] = {
1958	fsparam_bool	("no_disconnect",	Opt_no_disconnect),
1959	fsparam_u32	("rmode",		Opt_rmode),
1960	fsparam_u32	("fmode",		Opt_fmode),
1961	fsparam_u32	("mode",		Opt_mode),
1962	fsparam_u32	("uid",			Opt_uid),
1963	fsparam_u32	("gid",			Opt_gid),
1964	{}
1965};
1966
1967static int ffs_fs_parse_param(struct fs_context *fc, struct fs_parameter *param)
1968{
1969	struct ffs_sb_fill_data *data = fc->fs_private;
1970	struct fs_parse_result result;
1971	int opt;
1972
1973	opt = fs_parse(fc, ffs_fs_fs_parameters, param, &result);
1974	if (opt < 0)
1975		return opt;
1976
1977	switch (opt) {
1978	case Opt_no_disconnect:
1979		data->no_disconnect = result.boolean;
1980		break;
1981	case Opt_rmode:
1982		data->root_mode  = (result.uint_32 & 0555) | S_IFDIR;
1983		break;
1984	case Opt_fmode:
1985		data->perms.mode = (result.uint_32 & 0666) | S_IFREG;
1986		break;
1987	case Opt_mode:
1988		data->root_mode  = (result.uint_32 & 0555) | S_IFDIR;
1989		data->perms.mode = (result.uint_32 & 0666) | S_IFREG;
1990		break;
1991
1992	case Opt_uid:
1993		data->perms.uid = make_kuid(current_user_ns(), result.uint_32);
1994		if (!uid_valid(data->perms.uid))
1995			goto unmapped_value;
1996		break;
1997	case Opt_gid:
1998		data->perms.gid = make_kgid(current_user_ns(), result.uint_32);
1999		if (!gid_valid(data->perms.gid))
2000			goto unmapped_value;
2001		break;
2002
2003	default:
2004		return -ENOPARAM;
2005	}
2006
2007	return 0;
2008
2009unmapped_value:
2010	return invalf(fc, "%s: unmapped value: %u", param->key, result.uint_32);
2011}
2012
2013/*
2014 * Set up the superblock for a mount.
2015 */
2016static int ffs_fs_get_tree(struct fs_context *fc)
2017{
2018	struct ffs_sb_fill_data *ctx = fc->fs_private;
2019	struct ffs_data	*ffs;
2020	int ret;
2021
2022	if (!fc->source)
2023		return invalf(fc, "No source specified");
2024
2025	ffs = ffs_data_new(fc->source);
2026	if (!ffs)
2027		return -ENOMEM;
2028	ffs->file_perms = ctx->perms;
2029	ffs->no_disconnect = ctx->no_disconnect;
2030
2031	ffs->dev_name = kstrdup(fc->source, GFP_KERNEL);
2032	if (!ffs->dev_name) {
2033		ffs_data_put(ffs);
2034		return -ENOMEM;
2035	}
2036
2037	ret = ffs_acquire_dev(ffs->dev_name, ffs);
2038	if (ret) {
2039		ffs_data_put(ffs);
2040		return ret;
2041	}
2042
2043	ctx->ffs_data = ffs;
2044	return get_tree_nodev(fc, ffs_sb_fill);
2045}
2046
2047static void ffs_fs_free_fc(struct fs_context *fc)
2048{
2049	struct ffs_sb_fill_data *ctx = fc->fs_private;
2050
2051	if (ctx) {
2052		if (ctx->ffs_data) {
2053			ffs_data_put(ctx->ffs_data);
2054		}
2055
2056		kfree(ctx);
2057	}
2058}
2059
2060static const struct fs_context_operations ffs_fs_context_ops = {
2061	.free		= ffs_fs_free_fc,
2062	.parse_param	= ffs_fs_parse_param,
2063	.get_tree	= ffs_fs_get_tree,
2064};
2065
2066static int ffs_fs_init_fs_context(struct fs_context *fc)
2067{
2068	struct ffs_sb_fill_data *ctx;
2069
2070	ctx = kzalloc(sizeof(struct ffs_sb_fill_data), GFP_KERNEL);
2071	if (!ctx)
2072		return -ENOMEM;
2073
2074	ctx->perms.mode = S_IFREG | 0600;
2075	ctx->perms.uid = GLOBAL_ROOT_UID;
2076	ctx->perms.gid = GLOBAL_ROOT_GID;
2077	ctx->root_mode = S_IFDIR | 0500;
2078	ctx->no_disconnect = false;
2079
2080	fc->fs_private = ctx;
2081	fc->ops = &ffs_fs_context_ops;
2082	return 0;
2083}
2084
2085static void
2086ffs_fs_kill_sb(struct super_block *sb)
2087{
2088	kill_anon_super(sb);
2089	if (sb->s_fs_info) {
2090		struct ffs_data *ffs = sb->s_fs_info;
2091		ffs->state = FFS_CLOSING;
2092		ffs_data_reset(ffs);
2093		// no configfs accesses from that point on,
2094		// so no further schedule_work() is possible
2095		cancel_work_sync(&ffs->reset_work);
2096		ffs_data_put(ffs);
2097	}
2098}
2099
2100static struct file_system_type ffs_fs_type = {
2101	.owner		= THIS_MODULE,
2102	.name		= "functionfs",
2103	.init_fs_context = ffs_fs_init_fs_context,
2104	.parameters	= ffs_fs_fs_parameters,
2105	.kill_sb	= ffs_fs_kill_sb,
2106};
2107MODULE_ALIAS_FS("functionfs");
2108
2109
2110/* Driver's main init/cleanup functions *************************************/
2111
2112static int functionfs_init(void)
2113{
2114	int ret;
2115
2116	ret = register_filesystem(&ffs_fs_type);
2117	if (!ret)
2118		pr_info("file system registered\n");
2119	else
2120		pr_err("failed registering file system (%d)\n", ret);
2121
2122	return ret;
2123}
2124
2125static void functionfs_cleanup(void)
2126{
2127	pr_info("unloading\n");
2128	unregister_filesystem(&ffs_fs_type);
2129}
2130
2131
2132/* ffs_data and ffs_function construction and destruction code **************/
2133
2134static void ffs_data_clear(struct ffs_data *ffs);
2135
2136static void ffs_data_get(struct ffs_data *ffs)
2137{
2138	refcount_inc(&ffs->ref);
2139}
2140
2141static void ffs_data_put(struct ffs_data *ffs)
2142{
2143	if (refcount_dec_and_test(&ffs->ref)) {
2144		pr_info("%s(): freeing\n", __func__);
2145		ffs_data_clear(ffs);
2146		ffs_release_dev(ffs->private_data);
2147		BUG_ON(waitqueue_active(&ffs->ev.waitq) ||
2148		       swait_active(&ffs->ep0req_completion.wait) ||
2149		       waitqueue_active(&ffs->wait));
2150		destroy_workqueue(ffs->io_completion_wq);
2151		kfree(ffs->dev_name);
2152		kfree(ffs);
2153	}
2154}
2155
2156static void ffs_data_closed(struct ffs_data *ffs)
2157{
2158	spin_lock_irq(&ffs->eps_lock);
2159	if (--ffs->opened) {	// not the last opener?
2160		spin_unlock_irq(&ffs->eps_lock);
2161		return;
2162	}
2163	if (ffs->no_disconnect) {
2164		struct ffs_epfile *epfiles;
2165
2166		ffs->state = FFS_DEACTIVATED;
2167		epfiles = ffs->epfiles;
2168		ffs->epfiles = NULL;
2169		spin_unlock_irq(&ffs->eps_lock);
2170
2171		if (epfiles)
2172			ffs_epfiles_destroy(ffs->sb, epfiles,
2173					 ffs->eps_count);
2174
2175		if (ffs->setup_state == FFS_SETUP_PENDING)
2176			__ffs_ep0_stall(ffs);
2177	} else {
2178		ffs->state = FFS_CLOSING;
2179		spin_unlock_irq(&ffs->eps_lock);
2180		ffs_data_reset(ffs);
2181	}
2182}
2183
2184static struct ffs_data *ffs_data_new(const char *dev_name)
2185{
2186	struct ffs_data *ffs = kzalloc(sizeof *ffs, GFP_KERNEL);
2187	if (!ffs)
2188		return NULL;
2189
2190	ffs->io_completion_wq = alloc_ordered_workqueue("%s", 0, dev_name);
2191	if (!ffs->io_completion_wq) {
2192		kfree(ffs);
2193		return NULL;
2194	}
2195
2196	refcount_set(&ffs->ref, 1);
2197	ffs->opened = 0;
2198	ffs->state = FFS_READ_DESCRIPTORS;
2199	mutex_init(&ffs->mutex);
2200	spin_lock_init(&ffs->eps_lock);
2201	init_waitqueue_head(&ffs->ev.waitq);
2202	init_waitqueue_head(&ffs->wait);
2203	init_completion(&ffs->ep0req_completion);
2204
2205	/* XXX REVISIT need to update it in some places, or do we? */
2206	ffs->ev.can_stall = 1;
2207
2208	return ffs;
2209}
2210
2211static void ffs_data_clear(struct ffs_data *ffs)
2212{
2213	struct ffs_epfile *epfiles;
2214	unsigned long flags;
2215
2216	ffs_closed(ffs);
2217
2218	BUG_ON(ffs->gadget);
2219
2220	spin_lock_irqsave(&ffs->eps_lock, flags);
2221	epfiles = ffs->epfiles;
2222	ffs->epfiles = NULL;
2223	spin_unlock_irqrestore(&ffs->eps_lock, flags);
2224
2225	/*
2226	 * potential race possible between ffs_func_eps_disable
2227	 * & ffs_epfile_release therefore maintaining a local
2228	 * copy of epfile will save us from use-after-free.
2229	 */
2230	if (epfiles) {
2231		ffs_epfiles_destroy(ffs->sb, epfiles, ffs->eps_count);
2232		ffs->epfiles = NULL;
2233	}
2234
2235	if (ffs->ffs_eventfd) {
2236		eventfd_ctx_put(ffs->ffs_eventfd);
2237		ffs->ffs_eventfd = NULL;
2238	}
2239
2240	kfree(ffs->raw_descs_data);
2241	kfree(ffs->raw_strings);
2242	kfree(ffs->stringtabs);
2243}
2244
2245static void ffs_data_reset(struct ffs_data *ffs)
2246{
2247	ffs_data_clear(ffs);
2248
2249	spin_lock_irq(&ffs->eps_lock);
2250	ffs->raw_descs_data = NULL;
2251	ffs->raw_descs = NULL;
2252	ffs->raw_strings = NULL;
2253	ffs->stringtabs = NULL;
2254
2255	ffs->raw_descs_length = 0;
2256	ffs->fs_descs_count = 0;
2257	ffs->hs_descs_count = 0;
2258	ffs->ss_descs_count = 0;
2259
2260	ffs->strings_count = 0;
2261	ffs->interfaces_count = 0;
2262	ffs->eps_count = 0;
2263
2264	ffs->ev.count = 0;
2265
2266	ffs->state = FFS_READ_DESCRIPTORS;
2267	ffs->setup_state = FFS_NO_SETUP;
2268	ffs->flags = 0;
2269
2270	ffs->ms_os_descs_ext_prop_count = 0;
2271	ffs->ms_os_descs_ext_prop_name_len = 0;
2272	ffs->ms_os_descs_ext_prop_data_len = 0;
2273	spin_unlock_irq(&ffs->eps_lock);
2274}
2275
2276
2277static int functionfs_bind(struct ffs_data *ffs, struct usb_composite_dev *cdev)
2278{
2279	struct usb_gadget_strings **lang;
2280	int first_id;
2281
2282	if ((ffs->state != FFS_ACTIVE
2283		 || test_and_set_bit(FFS_FL_BOUND, &ffs->flags)))
2284		return -EBADFD;
2285
2286	first_id = usb_string_ids_n(cdev, ffs->strings_count);
2287	if (first_id < 0)
2288		return first_id;
2289
2290	ffs->ep0req = usb_ep_alloc_request(cdev->gadget->ep0, GFP_KERNEL);
2291	if (!ffs->ep0req)
2292		return -ENOMEM;
2293	ffs->ep0req->complete = ffs_ep0_complete;
2294	ffs->ep0req->context = ffs;
2295
2296	lang = ffs->stringtabs;
2297	if (lang) {
2298		for (; *lang; ++lang) {
2299			struct usb_string *str = (*lang)->strings;
2300			int id = first_id;
2301			for (; str->s; ++id, ++str)
2302				str->id = id;
2303		}
2304	}
2305
2306	ffs->gadget = cdev->gadget;
2307	ffs_data_get(ffs);
2308	return 0;
2309}
2310
2311static void functionfs_unbind(struct ffs_data *ffs)
2312{
2313	if (!WARN_ON(!ffs->gadget)) {
2314		/* dequeue before freeing ep0req */
2315		usb_ep_dequeue(ffs->gadget->ep0, ffs->ep0req);
2316		mutex_lock(&ffs->mutex);
2317		usb_ep_free_request(ffs->gadget->ep0, ffs->ep0req);
2318		ffs->ep0req = NULL;
2319		ffs->gadget = NULL;
2320		clear_bit(FFS_FL_BOUND, &ffs->flags);
2321		mutex_unlock(&ffs->mutex);
2322		ffs_data_put(ffs);
2323	}
2324}
2325
2326static int ffs_epfiles_create(struct ffs_data *ffs)
2327{
2328	struct ffs_epfile *epfile, *epfiles;
2329	unsigned i, count;
2330	int err;
2331
2332	count = ffs->eps_count;
2333	epfiles = kcalloc(count, sizeof(*epfiles), GFP_KERNEL);
2334	if (!epfiles)
2335		return -ENOMEM;
2336
2337	epfile = epfiles;
2338	for (i = 1; i <= count; ++i, ++epfile) {
2339		epfile->ffs = ffs;
2340		mutex_init(&epfile->mutex);
2341		mutex_init(&epfile->dmabufs_mutex);
2342		INIT_LIST_HEAD(&epfile->dmabufs);
2343		if (ffs->user_flags & FUNCTIONFS_VIRTUAL_ADDR)
2344			sprintf(epfile->name, "ep%02x", ffs->eps_addrmap[i]);
2345		else
2346			sprintf(epfile->name, "ep%u", i);
2347		err = ffs_sb_create_file(ffs->sb, epfile->name,
2348					 epfile, &ffs_epfile_operations);
2349		if (err) {
2350			ffs_epfiles_destroy(ffs->sb, epfiles, i - 1);
2351			return err;
2352		}
2353	}
2354
2355	ffs->epfiles = epfiles;
2356	return 0;
2357}
2358
2359static void clear_one(struct dentry *dentry)
2360{
2361	smp_store_release(&dentry->d_inode->i_private, NULL);
2362}
2363
2364static void ffs_epfiles_destroy(struct super_block *sb,
2365				struct ffs_epfile *epfiles, unsigned count)
2366{
2367	struct ffs_epfile *epfile = epfiles;
2368	struct dentry *root = sb->s_root;
2369
2370	for (; count; --count, ++epfile) {
2371		BUG_ON(mutex_is_locked(&epfile->mutex));
2372		simple_remove_by_name(root, epfile->name, clear_one);
2373	}
2374
2375	kfree(epfiles);
2376}
2377
2378static void ffs_func_eps_disable(struct ffs_function *func)
2379{
2380	struct ffs_ep *ep;
2381	struct ffs_epfile *epfile;
2382	unsigned short count;
2383	unsigned long flags;
2384
2385	spin_lock_irqsave(&func->ffs->eps_lock, flags);
2386	count = func->ffs->eps_count;
2387	epfile = func->ffs->epfiles;
2388	ep = func->eps;
2389	while (count--) {
2390		/* pending requests get nuked */
2391		if (ep->ep)
2392			usb_ep_disable(ep->ep);
2393		++ep;
2394
2395		if (epfile) {
2396			epfile->ep = NULL;
2397			__ffs_epfile_read_buffer_free(epfile);
2398			++epfile;
2399		}
2400	}
2401	spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
2402}
2403
2404static int ffs_func_eps_enable(struct ffs_function *func)
2405{
2406	struct ffs_data *ffs;
2407	struct ffs_ep *ep;
2408	struct ffs_epfile *epfile;
2409	unsigned short count;
2410	unsigned long flags;
2411	int ret = 0;
2412
2413	spin_lock_irqsave(&func->ffs->eps_lock, flags);
2414	ffs = func->ffs;
2415	ep = func->eps;
2416	epfile = ffs->epfiles;
2417	count = ffs->eps_count;
2418	if (!epfile) {
2419		ret = -ENOMEM;
2420		goto done;
2421	}
2422
2423	while (count--) {
2424		ep->ep->driver_data = ep;
2425
2426		ret = config_ep_by_speed(func->gadget, &func->function, ep->ep);
2427		if (ret) {
2428			pr_err("%s: config_ep_by_speed(%s) returned %d\n",
2429					__func__, ep->ep->name, ret);
2430			break;
2431		}
2432
2433		ret = usb_ep_enable(ep->ep);
2434		if (!ret) {
2435			epfile->ep = ep;
2436			epfile->in = usb_endpoint_dir_in(ep->ep->desc);
2437			epfile->isoc = usb_endpoint_xfer_isoc(ep->ep->desc);
2438		} else {
2439			break;
2440		}
2441
2442		++ep;
2443		++epfile;
2444	}
2445
2446	wake_up_interruptible(&ffs->wait);
2447done:
2448	spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
2449
2450	return ret;
2451}
2452
2453
2454/* Parsing and building descriptors and strings *****************************/
2455
2456/*
2457 * This validates if data pointed by data is a valid USB descriptor as
2458 * well as record how many interfaces, endpoints and strings are
2459 * required by given configuration.  Returns address after the
2460 * descriptor or NULL if data is invalid.
2461 */
2462
2463enum ffs_entity_type {
2464	FFS_DESCRIPTOR, FFS_INTERFACE, FFS_STRING, FFS_ENDPOINT
2465};
2466
2467enum ffs_os_desc_type {
2468	FFS_OS_DESC, FFS_OS_DESC_EXT_COMPAT, FFS_OS_DESC_EXT_PROP
2469};
2470
2471typedef int (*ffs_entity_callback)(enum ffs_entity_type entity,
2472				   u8 *valuep,
2473				   struct usb_descriptor_header *desc,
2474				   void *priv);
2475
2476typedef int (*ffs_os_desc_callback)(enum ffs_os_desc_type entity,
2477				    struct usb_os_desc_header *h, void *data,
2478				    unsigned len, void *priv);
2479
2480static int __must_check ffs_do_single_desc(char *data, unsigned len,
2481					   ffs_entity_callback entity,
2482					   void *priv, int *current_class, int *current_subclass)
2483{
2484	struct usb_descriptor_header *_ds = (void *)data;
2485	u8 length;
2486	int ret;
2487
2488	/* At least two bytes are required: length and type */
2489	if (len < 2) {
2490		pr_vdebug("descriptor too short\n");
2491		return -EINVAL;
2492	}
2493
2494	/* If we have at least as many bytes as the descriptor takes? */
2495	length = _ds->bLength;
2496	if (len < length) {
2497		pr_vdebug("descriptor longer then available data\n");
2498		return -EINVAL;
2499	}
2500
2501#define __entity_check_INTERFACE(val)  1
2502#define __entity_check_STRING(val)     (val)
2503#define __entity_check_ENDPOINT(val)   ((val) & USB_ENDPOINT_NUMBER_MASK)
2504#define __entity(type, val) do {					\
2505		pr_vdebug("entity " #type "(%02x)\n", (val));		\
2506		if (!__entity_check_ ##type(val)) {			\
2507			pr_vdebug("invalid entity's value\n");		\
2508			return -EINVAL;					\
2509		}							\
2510		ret = entity(FFS_ ##type, &val, _ds, priv);		\
2511		if (ret < 0) {						\
2512			pr_debug("entity " #type "(%02x); ret = %d\n",	\
2513				 (val), ret);				\
2514			return ret;					\
2515		}							\
2516	} while (0)
2517
2518	/* Parse descriptor depending on type. */
2519	switch (_ds->bDescriptorType) {
2520	case USB_DT_DEVICE:
2521	case USB_DT_CONFIG:
2522	case USB_DT_STRING:
2523	case USB_DT_DEVICE_QUALIFIER:
2524		/* function can't have any of those */
2525		pr_vdebug("descriptor reserved for gadget: %d\n",
2526		      _ds->bDescriptorType);
2527		return -EINVAL;
2528
2529	case USB_DT_INTERFACE: {
2530		struct usb_interface_descriptor *ds = (void *)_ds;
2531		pr_vdebug("interface descriptor\n");
2532		if (length != sizeof *ds)
2533			goto inv_length;
2534
2535		__entity(INTERFACE, ds->bInterfaceNumber);
2536		if (ds->iInterface)
2537			__entity(STRING, ds->iInterface);
2538		*current_class = ds->bInterfaceClass;
2539		*current_subclass = ds->bInterfaceSubClass;
2540	}
2541		break;
2542
2543	case USB_DT_ENDPOINT: {
2544		struct usb_endpoint_descriptor *ds = (void *)_ds;
2545		pr_vdebug("endpoint descriptor\n");
2546		if (length != USB_DT_ENDPOINT_SIZE &&
2547		    length != USB_DT_ENDPOINT_AUDIO_SIZE)
2548			goto inv_length;
2549		__entity(ENDPOINT, ds->bEndpointAddress);
2550	}
2551		break;
2552
2553	case USB_TYPE_CLASS | 0x01:
2554		if (*current_class == USB_INTERFACE_CLASS_HID) {
2555			pr_vdebug("hid descriptor\n");
2556			if (length != sizeof(struct hid_descriptor))
2557				goto inv_length;
2558			break;
2559		} else if (*current_class == USB_INTERFACE_CLASS_CCID) {
2560			pr_vdebug("ccid descriptor\n");
2561			if (length != sizeof(struct ccid_descriptor))
2562				goto inv_length;
2563			break;
2564		} else if (*current_class == USB_CLASS_APP_SPEC &&
2565			   *current_subclass == USB_SUBCLASS_DFU) {
2566			pr_vdebug("dfu functional descriptor\n");
2567			if (length != sizeof(struct usb_dfu_functional_descriptor))
2568				goto inv_length;
2569			break;
2570		} else {
2571			pr_vdebug("unknown descriptor: %d for class %d\n",
2572			      _ds->bDescriptorType, *current_class);
2573			return -EINVAL;
2574		}
2575
2576	case USB_DT_OTG:
2577		if (length != sizeof(struct usb_otg_descriptor))
2578			goto inv_length;
2579		break;
2580
2581	case USB_DT_INTERFACE_ASSOCIATION: {
2582		struct usb_interface_assoc_descriptor *ds = (void *)_ds;
2583		pr_vdebug("interface association descriptor\n");
2584		if (length != sizeof *ds)
2585			goto inv_length;
2586		if (ds->iFunction)
2587			__entity(STRING, ds->iFunction);
2588	}
2589		break;
2590
2591	case USB_DT_SS_ENDPOINT_COMP:
2592		pr_vdebug("EP SS companion descriptor\n");
2593		if (length != sizeof(struct usb_ss_ep_comp_descriptor))
2594			goto inv_length;
2595		break;
2596
2597	case USB_DT_OTHER_SPEED_CONFIG:
2598	case USB_DT_INTERFACE_POWER:
2599	case USB_DT_DEBUG:
2600	case USB_DT_SECURITY:
2601	case USB_DT_CS_RADIO_CONTROL:
2602		/* TODO */
2603		pr_vdebug("unimplemented descriptor: %d\n", _ds->bDescriptorType);
2604		return -EINVAL;
2605
2606	default:
2607		/* We should never be here */
2608		pr_vdebug("unknown descriptor: %d\n", _ds->bDescriptorType);
2609		return -EINVAL;
2610
2611inv_length:
2612		pr_vdebug("invalid length: %d (descriptor %d)\n",
2613			  _ds->bLength, _ds->bDescriptorType);
2614		return -EINVAL;
2615	}
2616
2617#undef __entity
2618#undef __entity_check_DESCRIPTOR
2619#undef __entity_check_INTERFACE
2620#undef __entity_check_STRING
2621#undef __entity_check_ENDPOINT
2622
2623	return length;
2624}
2625
2626static int __must_check ffs_do_descs(unsigned count, char *data, unsigned len,
2627				     ffs_entity_callback entity, void *priv)
2628{
2629	const unsigned _len = len;
2630	unsigned long num = 0;
2631	int current_class = -1;
2632	int current_subclass = -1;
2633
2634	for (;;) {
2635		int ret;
2636
2637		if (num == count)
2638			data = NULL;
2639
2640		/* Record "descriptor" entity */
2641		ret = entity(FFS_DESCRIPTOR, (u8 *)num, (void *)data, priv);
2642		if (ret < 0) {
2643			pr_debug("entity DESCRIPTOR(%02lx); ret = %d\n",
2644				 num, ret);
2645			return ret;
2646		}
2647
2648		if (!data)
2649			return _len - len;
2650
2651		ret = ffs_do_single_desc(data, len, entity, priv,
2652			&current_class, &current_subclass);
2653		if (ret < 0) {
2654			pr_debug("%s returns %d\n", __func__, ret);
2655			return ret;
2656		}
2657
2658		len -= ret;
2659		data += ret;
2660		++num;
2661	}
2662}
2663
2664static int __ffs_data_do_entity(enum ffs_entity_type type,
2665				u8 *valuep, struct usb_descriptor_header *desc,
2666				void *priv)
2667{
2668	struct ffs_desc_helper *helper = priv;
2669	struct usb_endpoint_descriptor *d;
2670
2671	switch (type) {
2672	case FFS_DESCRIPTOR:
2673		break;
2674
2675	case FFS_INTERFACE:
2676		/*
2677		 * Interfaces are indexed from zero so if we
2678		 * encountered interface "n" then there are at least
2679		 * "n+1" interfaces.
2680		 */
2681		if (*valuep >= helper->interfaces_count)
2682			helper->interfaces_count = *valuep + 1;
2683		break;
2684
2685	case FFS_STRING:
2686		/*
2687		 * Strings are indexed from 1 (0 is reserved
2688		 * for languages list)
2689		 */
2690		if (*valuep > helper->ffs->strings_count)
2691			helper->ffs->strings_count = *valuep;
2692		break;
2693
2694	case FFS_ENDPOINT:
2695		d = (void *)desc;
2696		helper->eps_count++;
2697		if (helper->eps_count >= FFS_MAX_EPS_COUNT)
2698			return -EINVAL;
2699		/* Check if descriptors for any speed were already parsed */
2700		if (!helper->ffs->eps_count && !helper->ffs->interfaces_count)
2701			helper->ffs->eps_addrmap[helper->eps_count] =
2702				d->bEndpointAddress;
2703		else if (helper->ffs->eps_addrmap[helper->eps_count] !=
2704				d->bEndpointAddress)
2705			return -EINVAL;
2706		break;
2707	}
2708
2709	return 0;
2710}
2711
2712static int __ffs_do_os_desc_header(enum ffs_os_desc_type *next_type,
2713				   struct usb_os_desc_header *desc)
2714{
2715	u16 bcd_version = le16_to_cpu(desc->bcdVersion);
2716	u16 w_index = le16_to_cpu(desc->wIndex);
2717
2718	if (bcd_version == 0x1) {
2719		pr_warn("bcdVersion must be 0x0100, stored in Little Endian order. "
2720			"Userspace driver should be fixed, accepting 0x0001 for compatibility.\n");
2721	} else if (bcd_version != 0x100) {
2722		pr_vdebug("unsupported os descriptors version: 0x%x\n",
2723			  bcd_version);
2724		return -EINVAL;
2725	}
2726	switch (w_index) {
2727	case 0x4:
2728		*next_type = FFS_OS_DESC_EXT_COMPAT;
2729		break;
2730	case 0x5:
2731		*next_type = FFS_OS_DESC_EXT_PROP;
2732		break;
2733	default:
2734		pr_vdebug("unsupported os descriptor type: %d", w_index);
2735		return -EINVAL;
2736	}
2737
2738	return sizeof(*desc);
2739}
2740
2741/*
2742 * Process all extended compatibility/extended property descriptors
2743 * of a feature descriptor
2744 */
2745static int __must_check ffs_do_single_os_desc(char *data, unsigned len,
2746					      enum ffs_os_desc_type type,
2747					      u16 feature_count,
2748					      ffs_os_desc_callback entity,
2749					      void *priv,
2750					      struct usb_os_desc_header *h)
2751{
2752	int ret;
2753	const unsigned _len = len;
2754
2755	/* loop over all ext compat/ext prop descriptors */
2756	while (feature_count--) {
2757		ret = entity(type, h, data, len, priv);
2758		if (ret < 0) {
2759			pr_debug("bad OS descriptor, type: %d\n", type);
2760			return ret;
2761		}
2762		data += ret;
2763		len -= ret;
2764	}
2765	return _len - len;
2766}
2767
2768/* Process a number of complete Feature Descriptors (Ext Compat or Ext Prop) */
2769static int __must_check ffs_do_os_descs(unsigned count,
2770					char *data, unsigned len,
2771					ffs_os_desc_callback entity, void *priv)
2772{
2773	const unsigned _len = len;
2774	unsigned long num = 0;
2775
2776	for (num = 0; num < count; ++num) {
2777		int ret;
2778		enum ffs_os_desc_type type;
2779		u16 feature_count;
2780		struct usb_os_desc_header *desc = (void *)data;
2781
2782		if (len < sizeof(*desc))
2783			return -EINVAL;
2784
2785		/*
2786		 * Record "descriptor" entity.
2787		 * Process dwLength, bcdVersion, wIndex, get b/wCount.
2788		 * Move the data pointer to the beginning of extended
2789		 * compatibilities proper or extended properties proper
2790		 * portions of the data
2791		 */
2792		if (le32_to_cpu(desc->dwLength) > len)
2793			return -EINVAL;
2794
2795		ret = __ffs_do_os_desc_header(&type, desc);
2796		if (ret < 0) {
2797			pr_debug("entity OS_DESCRIPTOR(%02lx); ret = %d\n",
2798				 num, ret);
2799			return ret;
2800		}
2801		/*
2802		 * 16-bit hex "?? 00" Little Endian looks like 8-bit hex "??"
2803		 */
2804		feature_count = le16_to_cpu(desc->wCount);
2805		if (type == FFS_OS_DESC_EXT_COMPAT &&
2806		    (feature_count > 255 || desc->Reserved))
2807				return -EINVAL;
2808		len -= ret;
2809		data += ret;
2810
2811		/*
2812		 * Process all function/property descriptors
2813		 * of this Feature Descriptor
2814		 */
2815		ret = ffs_do_single_os_desc(data, len, type,
2816					    feature_count, entity, priv, desc);
2817		if (ret < 0) {
2818			pr_debug("%s returns %d\n", __func__, ret);
2819			return ret;
2820		}
2821
2822		len -= ret;
2823		data += ret;
2824	}
2825	return _len - len;
2826}
2827
2828/*
2829 * Validate contents of the buffer from userspace related to OS descriptors.
2830 */
2831static int __ffs_data_do_os_desc(enum ffs_os_desc_type type,
2832				 struct usb_os_desc_header *h, void *data,
2833				 unsigned len, void *priv)
2834{
2835	struct ffs_data *ffs = priv;
2836	u8 length;
2837
2838	switch (type) {
2839	case FFS_OS_DESC_EXT_COMPAT: {
2840		struct usb_ext_compat_desc *d = data;
2841		int i;
2842
2843		if (len < sizeof(*d) ||
2844		    d->bFirstInterfaceNumber >= ffs->interfaces_count)
2845			return -EINVAL;
2846		if (d->Reserved1 != 1) {
2847			/*
2848			 * According to the spec, Reserved1 must be set to 1
2849			 * but older kernels incorrectly rejected non-zero
2850			 * values.  We fix it here to avoid returning EINVAL
2851			 * in response to values we used to accept.
2852			 */
2853			pr_debug("usb_ext_compat_desc::Reserved1 forced to 1\n");
2854			d->Reserved1 = 1;
2855		}
2856		for (i = 0; i < ARRAY_SIZE(d->Reserved2); ++i)
2857			if (d->Reserved2[i])
2858				return -EINVAL;
2859
2860		length = sizeof(struct usb_ext_compat_desc);
2861	}
2862		break;
2863	case FFS_OS_DESC_EXT_PROP: {
2864		struct usb_ext_prop_desc *d = data;
2865		u32 type, pdl;
2866		u16 pnl;
2867
2868		if (len < sizeof(*d) || h->interface >= ffs->interfaces_count)
2869			return -EINVAL;
2870		length = le32_to_cpu(d->dwSize);
2871		if (len < length)
2872			return -EINVAL;
2873		type = le32_to_cpu(d->dwPropertyDataType);
2874		if (type < USB_EXT_PROP_UNICODE ||
2875		    type > USB_EXT_PROP_UNICODE_MULTI) {
2876			pr_vdebug("unsupported os descriptor property type: %d",
2877				  type);
2878			return -EINVAL;
2879		}
2880		pnl = le16_to_cpu(d->wPropertyNameLength);
2881		if (length < 14 + pnl) {
2882			pr_vdebug("invalid os descriptor length: %d pnl:%d (descriptor %d)\n",
2883				  length, pnl, type);
2884			return -EINVAL;
2885		}
2886		pdl = le32_to_cpu(*(__le32 *)((u8 *)data + 10 + pnl));
2887		if (length != 14 + pnl + pdl) {
2888			pr_vdebug("invalid os descriptor length: %d pnl:%d pdl:%d (descriptor %d)\n",
2889				  length, pnl, pdl, type);
2890			return -EINVAL;
2891		}
2892		++ffs->ms_os_descs_ext_prop_count;
2893		/* property name reported to the host as "WCHAR"s */
2894		ffs->ms_os_descs_ext_prop_name_len += pnl * 2;
2895		ffs->ms_os_descs_ext_prop_data_len += pdl;
2896	}
2897		break;
2898	default:
2899		pr_vdebug("unknown descriptor: %d\n", type);
2900		return -EINVAL;
2901	}
2902	return length;
2903}
2904
2905static int __ffs_data_got_descs(struct ffs_data *ffs,
2906				char *const _data, size_t len)
2907{
2908	char *data = _data, *raw_descs;
2909	unsigned os_descs_count = 0, counts[3], flags;
2910	int ret = -EINVAL, i;
2911	struct ffs_desc_helper helper;
2912
2913	if (get_unaligned_le32(data + 4) != len)
2914		goto error;
2915
2916	switch (get_unaligned_le32(data)) {
2917	case FUNCTIONFS_DESCRIPTORS_MAGIC:
2918		flags = FUNCTIONFS_HAS_FS_DESC | FUNCTIONFS_HAS_HS_DESC;
2919		data += 8;
2920		len  -= 8;
2921		break;
2922	case FUNCTIONFS_DESCRIPTORS_MAGIC_V2:
2923		flags = get_unaligned_le32(data + 8);
2924		ffs->user_flags = flags;
2925		if (flags & ~(FUNCTIONFS_HAS_FS_DESC |
2926			      FUNCTIONFS_HAS_HS_DESC |
2927			      FUNCTIONFS_HAS_SS_DESC |
2928			      FUNCTIONFS_HAS_MS_OS_DESC |
2929			      FUNCTIONFS_VIRTUAL_ADDR |
2930			      FUNCTIONFS_EVENTFD |
2931			      FUNCTIONFS_ALL_CTRL_RECIP |
2932			      FUNCTIONFS_CONFIG0_SETUP)) {
2933			ret = -ENOSYS;
2934			goto error;
2935		}
2936		data += 12;
2937		len  -= 12;
2938		break;
2939	default:
2940		goto error;
2941	}
2942
2943	if (flags & FUNCTIONFS_EVENTFD) {
2944		if (len < 4)
2945			goto error;
2946		ffs->ffs_eventfd =
2947			eventfd_ctx_fdget((int)get_unaligned_le32(data));
2948		if (IS_ERR(ffs->ffs_eventfd)) {
2949			ret = PTR_ERR(ffs->ffs_eventfd);
2950			ffs->ffs_eventfd = NULL;
2951			goto error;
2952		}
2953		data += 4;
2954		len  -= 4;
2955	}
2956
2957	/* Read fs_count, hs_count and ss_count (if present) */
2958	for (i = 0; i < 3; ++i) {
2959		if (!(flags & (1 << i))) {
2960			counts[i] = 0;
2961		} else if (len < 4) {
2962			goto error;
2963		} else {
2964			counts[i] = get_unaligned_le32(data);
2965			data += 4;
2966			len  -= 4;
2967		}
2968	}
2969	if (flags & (1 << i)) {
2970		if (len < 4) {
2971			goto error;
2972		}
2973		os_descs_count = get_unaligned_le32(data);
2974		data += 4;
2975		len -= 4;
2976	}
2977
2978	/* Read descriptors */
2979	raw_descs = data;
2980	helper.ffs = ffs;
2981	for (i = 0; i < 3; ++i) {
2982		if (!counts[i])
2983			continue;
2984		helper.interfaces_count = 0;
2985		helper.eps_count = 0;
2986		ret = ffs_do_descs(counts[i], data, len,
2987				   __ffs_data_do_entity, &helper);
2988		if (ret < 0)
2989			goto error;
2990		if (!ffs->eps_count && !ffs->interfaces_count) {
2991			ffs->eps_count = helper.eps_count;
2992			ffs->interfaces_count = helper.interfaces_count;
2993		} else {
2994			if (ffs->eps_count != helper.eps_count) {
2995				ret = -EINVAL;
2996				goto error;
2997			}
2998			if (ffs->interfaces_count != helper.interfaces_count) {
2999				ret = -EINVAL;
3000				goto error;
3001			}
3002		}
3003		data += ret;
3004		len  -= ret;
3005	}
3006	if (os_descs_count) {
3007		ret = ffs_do_os_descs(os_descs_count, data, len,
3008				      __ffs_data_do_os_desc, ffs);
3009		if (ret < 0)
3010			goto error;
3011		data += ret;
3012		len -= ret;
3013	}
3014
3015	if (raw_descs == data || len) {
3016		ret = -EINVAL;
3017		goto error;
3018	}
3019
3020	ffs->raw_descs_data	= _data;
3021	ffs->raw_descs		= raw_descs;
3022	ffs->raw_descs_length	= data - raw_descs;
3023	ffs->fs_descs_count	= counts[0];
3024	ffs->hs_descs_count	= counts[1];
3025	ffs->ss_descs_count	= counts[2];
3026	ffs->ms_os_descs_count	= os_descs_count;
3027
3028	return 0;
3029
3030error:
3031	kfree(_data);
3032	return ret;
3033}
3034
3035static int __ffs_data_got_strings(struct ffs_data *ffs,
3036				  char *const _data, size_t len)
3037{
3038	u32 str_count, needed_count, lang_count;
3039	struct usb_gadget_strings **stringtabs, *t;
3040	const char *data = _data;
3041	struct usb_string *s;
3042
3043	if (len < 16 ||
3044	    get_unaligned_le32(data) != FUNCTIONFS_STRINGS_MAGIC ||
3045	    get_unaligned_le32(data + 4) != len)
3046		goto error;
3047	str_count  = get_unaligned_le32(data + 8);
3048	lang_count = get_unaligned_le32(data + 12);
3049
3050	/* if one is zero the other must be zero */
3051	if (!str_count != !lang_count)
3052		goto error;
3053
3054	/* Do we have at least as many strings as descriptors need? */
3055	needed_count = ffs->strings_count;
3056	if (str_count < needed_count)
3057		goto error;
3058
3059	/*
3060	 * If we don't need any strings just return and free all
3061	 * memory.
3062	 */
3063	if (!needed_count) {
3064		kfree(_data);
3065		return 0;
3066	}
3067
3068	/* Allocate everything in one chunk so there's less maintenance. */
3069	{
3070		unsigned i = 0;
3071		vla_group(d);
3072		vla_item(d, struct usb_gadget_strings *, stringtabs,
3073			size_add(lang_count, 1));
3074		vla_item(d, struct usb_gadget_strings, stringtab, lang_count);
3075		vla_item(d, struct usb_string, strings,
3076			size_mul(lang_count, (needed_count + 1)));
3077
3078		char *vlabuf = kmalloc(vla_group_size(d), GFP_KERNEL);
3079
3080		if (!vlabuf) {
3081			kfree(_data);
3082			return -ENOMEM;
3083		}
3084
3085		/* Initialize the VLA pointers */
3086		stringtabs = vla_ptr(vlabuf, d, stringtabs);
3087		t = vla_ptr(vlabuf, d, stringtab);
3088		i = lang_count;
3089		do {
3090			*stringtabs++ = t++;
3091		} while (--i);
3092		*stringtabs = NULL;
3093
3094		/* stringtabs = vlabuf = d_stringtabs for later kfree */
3095		stringtabs = vla_ptr(vlabuf, d, stringtabs);
3096		t = vla_ptr(vlabuf, d, stringtab);
3097		s = vla_ptr(vlabuf, d, strings);
3098	}
3099
3100	/* For each language */
3101	data += 16;
3102	len -= 16;
3103
3104	do { /* lang_count > 0 so we can use do-while */
3105		unsigned needed = needed_count;
3106		u32 str_per_lang = str_count;
3107
3108		if (len < 3)
3109			goto error_free;
3110		t->language = get_unaligned_le16(data);
3111		t->strings  = s;
3112		++t;
3113
3114		data += 2;
3115		len -= 2;
3116
3117		/* For each string */
3118		do { /* str_count > 0 so we can use do-while */
3119			size_t length = strnlen(data, len);
3120
3121			if (length == len)
3122				goto error_free;
3123
3124			/*
3125			 * User may provide more strings then we need,
3126			 * if that's the case we simply ignore the
3127			 * rest
3128			 */
3129			if (needed) {
3130				/*
3131				 * s->id will be set while adding
3132				 * function to configuration so for
3133				 * now just leave garbage here.
3134				 */
3135				s->s = data;
3136				--needed;
3137				++s;
3138			}
3139
3140			data += length + 1;
3141			len -= length + 1;
3142		} while (--str_per_lang);
3143
3144		s->id = 0;   /* terminator */
3145		s->s = NULL;
3146		++s;
3147
3148	} while (--lang_count);
3149
3150	/* Some garbage left? */
3151	if (len)
3152		goto error_free;
3153
3154	/* Done! */
3155	ffs->stringtabs = stringtabs;
3156	ffs->raw_strings = _data;
3157
3158	return 0;
3159
3160error_free:
3161	kfree(stringtabs);
3162error:
3163	kfree(_data);
3164	return -EINVAL;
3165}
3166
3167
3168/* Events handling and management *******************************************/
3169
3170static void __ffs_event_add(struct ffs_data *ffs,
3171			    enum usb_functionfs_event_type type)
3172{
3173	enum usb_functionfs_event_type rem_type1, rem_type2 = type;
3174	int neg = 0;
3175
3176	/*
3177	 * Abort any unhandled setup
3178	 *
3179	 * We do not need to worry about some cmpxchg() changing value
3180	 * of ffs->setup_state without holding the lock because when
3181	 * state is FFS_SETUP_PENDING cmpxchg() in several places in
3182	 * the source does nothing.
3183	 */
3184	if (ffs->setup_state == FFS_SETUP_PENDING)
3185		ffs->setup_state = FFS_SETUP_CANCELLED;
3186
3187	/*
3188	 * Logic of this function guarantees that there are at most four pending
3189	 * evens on ffs->ev.types queue.  This is important because the queue
3190	 * has space for four elements only and __ffs_ep0_read_events function
3191	 * depends on that limit as well.  If more event types are added, those
3192	 * limits have to be revisited or guaranteed to still hold.
3193	 */
3194	switch (type) {
3195	case FUNCTIONFS_RESUME:
3196		rem_type2 = FUNCTIONFS_SUSPEND;
3197		fallthrough;
3198	case FUNCTIONFS_SUSPEND:
3199	case FUNCTIONFS_SETUP:
3200		rem_type1 = type;
3201		/* Discard all similar events */
3202		break;
3203
3204	case FUNCTIONFS_BIND:
3205	case FUNCTIONFS_UNBIND:
3206	case FUNCTIONFS_DISABLE:
3207	case FUNCTIONFS_ENABLE:
3208		/* Discard everything other then power management. */
3209		rem_type1 = FUNCTIONFS_SUSPEND;
3210		rem_type2 = FUNCTIONFS_RESUME;
3211		neg = 1;
3212		break;
3213
3214	default:
3215		WARN(1, "%d: unknown event, this should not happen\n", type);
3216		return;
3217	}
3218
3219	{
3220		u8 *ev  = ffs->ev.types, *out = ev;
3221		unsigned n = ffs->ev.count;
3222		for (; n; --n, ++ev)
3223			if ((*ev == rem_type1 || *ev == rem_type2) == neg)
3224				*out++ = *ev;
3225			else
3226				pr_vdebug("purging event %d\n", *ev);
3227		ffs->ev.count = out - ffs->ev.types;
3228	}
3229
3230	pr_vdebug("adding event %d\n", type);
3231	ffs->ev.types[ffs->ev.count++] = type;
3232	wake_up_locked(&ffs->ev.waitq);
3233	if (ffs->ffs_eventfd)
3234		eventfd_signal(ffs->ffs_eventfd);
3235}
3236
3237static void ffs_event_add(struct ffs_data *ffs,
3238			  enum usb_functionfs_event_type type)
3239{
3240	unsigned long flags;
3241	spin_lock_irqsave(&ffs->ev.waitq.lock, flags);
3242	__ffs_event_add(ffs, type);
3243	spin_unlock_irqrestore(&ffs->ev.waitq.lock, flags);
3244}
3245
3246/* Bind/unbind USB function hooks *******************************************/
3247
3248static int ffs_ep_addr2idx(struct ffs_data *ffs, u8 endpoint_address)
3249{
3250	int i;
3251
3252	for (i = 1; i < ARRAY_SIZE(ffs->eps_addrmap); ++i)
3253		if (ffs->eps_addrmap[i] == endpoint_address)
3254			return i;
3255	return -ENOENT;
3256}
3257
3258static int __ffs_func_bind_do_descs(enum ffs_entity_type type, u8 *valuep,
3259				    struct usb_descriptor_header *desc,
3260				    void *priv)
3261{
3262	struct usb_endpoint_descriptor *ds = (void *)desc;
3263	struct ffs_function *func = priv;
3264	struct ffs_ep *ffs_ep;
3265	unsigned ep_desc_id;
3266	int idx;
3267	static const char *speed_names[] = { "full", "high", "super" };
3268
3269	if (type != FFS_DESCRIPTOR)
3270		return 0;
3271
3272	/*
3273	 * If ss_descriptors is not NULL, we are reading super speed
3274	 * descriptors; if hs_descriptors is not NULL, we are reading high
3275	 * speed descriptors; otherwise, we are reading full speed
3276	 * descriptors.
3277	 */
3278	if (func->function.ss_descriptors) {
3279		ep_desc_id = 2;
3280		func->function.ss_descriptors[(long)valuep] = desc;
3281	} else if (func->function.hs_descriptors) {
3282		ep_desc_id = 1;
3283		func->function.hs_descriptors[(long)valuep] = desc;
3284	} else {
3285		ep_desc_id = 0;
3286		func->function.fs_descriptors[(long)valuep]    = desc;
3287	}
3288
3289	if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT)
3290		return 0;
3291
3292	idx = ffs_ep_addr2idx(func->ffs, ds->bEndpointAddress) - 1;
3293	if (idx < 0)
3294		return idx;
3295
3296	ffs_ep = func->eps + idx;
3297
3298	if (ffs_ep->descs[ep_desc_id]) {
3299		pr_err("two %sspeed descriptors for EP %d\n",
3300			  speed_names[ep_desc_id],
3301			  usb_endpoint_num(ds));
3302		return -EINVAL;
3303	}
3304	ffs_ep->descs[ep_desc_id] = ds;
3305
3306	ffs_dump_mem(": Original  ep desc", ds, ds->bLength);
3307	if (ffs_ep->ep) {
3308		ds->bEndpointAddress = ffs_ep->descs[0]->bEndpointAddress;
3309		if (!ds->wMaxPacketSize)
3310			ds->wMaxPacketSize = ffs_ep->descs[0]->wMaxPacketSize;
3311	} else {
3312		struct usb_request *req;
3313		struct usb_ep *ep;
3314		u8 bEndpointAddress;
3315		u16 wMaxPacketSize;
3316
3317		/*
3318		 * We back up bEndpointAddress because autoconfig overwrites
3319		 * it with physical endpoint address.
3320		 */
3321		bEndpointAddress = ds->bEndpointAddress;
3322		/*
3323		 * We back up wMaxPacketSize because autoconfig treats
3324		 * endpoint descriptors as if they were full speed.
3325		 */
3326		wMaxPacketSize = ds->wMaxPacketSize;
3327		pr_vdebug("autoconfig\n");
3328		ep = usb_ep_autoconfig(func->gadget, ds);
3329		if (!ep)
3330			return -ENOTSUPP;
3331		ep->driver_data = func->eps + idx;
3332
3333		req = usb_ep_alloc_request(ep, GFP_KERNEL);
3334		if (!req)
3335			return -ENOMEM;
3336
3337		ffs_ep->ep  = ep;
3338		ffs_ep->req = req;
3339		func->eps_revmap[ds->bEndpointAddress &
3340				 USB_ENDPOINT_NUMBER_MASK] = idx + 1;
3341		/*
3342		 * If we use virtual address mapping, we restore
3343		 * original bEndpointAddress value.
3344		 */
3345		if (func->ffs->user_flags & FUNCTIONFS_VIRTUAL_ADDR)
3346			ds->bEndpointAddress = bEndpointAddress;
3347		/*
3348		 * Restore wMaxPacketSize which was potentially
3349		 * overwritten by autoconfig.
3350		 */
3351		ds->wMaxPacketSize = wMaxPacketSize;
3352	}
3353	ffs_dump_mem(": Rewritten ep desc", ds, ds->bLength);
3354
3355	return 0;
3356}
3357
3358static int __ffs_func_bind_do_nums(enum ffs_entity_type type, u8 *valuep,
3359				   struct usb_descriptor_header *desc,
3360				   void *priv)
3361{
3362	struct ffs_function *func = priv;
3363	unsigned idx;
3364	u8 newValue;
3365
3366	switch (type) {
3367	default:
3368	case FFS_DESCRIPTOR:
3369		/* Handled in previous pass by __ffs_func_bind_do_descs() */
3370		return 0;
3371
3372	case FFS_INTERFACE:
3373		idx = *valuep;
3374		if (func->interfaces_nums[idx] < 0) {
3375			int id = usb_interface_id(func->conf, &func->function);
3376			if (id < 0)
3377				return id;
3378			func->interfaces_nums[idx] = id;
3379		}
3380		newValue = func->interfaces_nums[idx];
3381		break;
3382
3383	case FFS_STRING:
3384		/* String' IDs are allocated when fsf_data is bound to cdev */
3385		newValue = func->ffs->stringtabs[0]->strings[*valuep - 1].id;
3386		break;
3387
3388	case FFS_ENDPOINT:
3389		/*
3390		 * USB_DT_ENDPOINT are handled in
3391		 * __ffs_func_bind_do_descs().
3392		 */
3393		if (desc->bDescriptorType == USB_DT_ENDPOINT)
3394			return 0;
3395
3396		idx = (*valuep & USB_ENDPOINT_NUMBER_MASK) - 1;
3397		if (!func->eps[idx].ep)
3398			return -EINVAL;
3399
3400		{
3401			struct usb_endpoint_descriptor **descs;
3402			descs = func->eps[idx].descs;
3403			newValue = descs[descs[0] ? 0 : 1]->bEndpointAddress;
3404		}
3405		break;
3406	}
3407
3408	pr_vdebug("%02x -> %02x\n", *valuep, newValue);
3409	*valuep = newValue;
3410	return 0;
3411}
3412
3413static int __ffs_func_bind_do_os_desc(enum ffs_os_desc_type type,
3414				      struct usb_os_desc_header *h, void *data,
3415				      unsigned len, void *priv)
3416{
3417	struct ffs_function *func = priv;
3418	u8 length = 0;
3419
3420	switch (type) {
3421	case FFS_OS_DESC_EXT_COMPAT: {
3422		struct usb_ext_compat_desc *desc = data;
3423		struct usb_os_desc_table *t;
3424
3425		t = &func->function.os_desc_table[desc->bFirstInterfaceNumber];
3426		t->if_id = func->interfaces_nums[desc->bFirstInterfaceNumber];
3427		memcpy(t->os_desc->ext_compat_id, &desc->IDs,
3428		       sizeof_field(struct usb_ext_compat_desc, IDs));
3429		length = sizeof(*desc);
3430	}
3431		break;
3432	case FFS_OS_DESC_EXT_PROP: {
3433		struct usb_ext_prop_desc *desc = data;
3434		struct usb_os_desc_table *t;
3435		struct usb_os_desc_ext_prop *ext_prop;
3436		char *ext_prop_name;
3437		char *ext_prop_data;
3438
3439		t = &func->function.os_desc_table[h->interface];
3440		t->if_id = func->interfaces_nums[h->interface];
3441
3442		ext_prop = func->ffs->ms_os_descs_ext_prop_avail;
3443		func->ffs->ms_os_descs_ext_prop_avail += sizeof(*ext_prop);
3444
3445		ext_prop->type = le32_to_cpu(desc->dwPropertyDataType);
3446		ext_prop->name_len = le16_to_cpu(desc->wPropertyNameLength);
3447		ext_prop->data_len = le32_to_cpu(*(__le32 *)
3448			usb_ext_prop_data_len_ptr(data, ext_prop->name_len));
3449		length = ext_prop->name_len + ext_prop->data_len + 14;
3450
3451		ext_prop_name = func->ffs->ms_os_descs_ext_prop_name_avail;
3452		func->ffs->ms_os_descs_ext_prop_name_avail +=
3453			ext_prop->name_len;
3454
3455		ext_prop_data = func->ffs->ms_os_descs_ext_prop_data_avail;
3456		func->ffs->ms_os_descs_ext_prop_data_avail +=
3457			ext_prop->data_len;
3458		memcpy(ext_prop_data,
3459		       usb_ext_prop_data_ptr(data, ext_prop->name_len),
3460		       ext_prop->data_len);
3461		/* unicode data reported to the host as "WCHAR"s */
3462		switch (ext_prop->type) {
3463		case USB_EXT_PROP_UNICODE:
3464		case USB_EXT_PROP_UNICODE_ENV:
3465		case USB_EXT_PROP_UNICODE_LINK:
3466		case USB_EXT_PROP_UNICODE_MULTI:
3467			ext_prop->data_len *= 2;
3468			break;
3469		}
3470		ext_prop->data = ext_prop_data;
3471
3472		memcpy(ext_prop_name, usb_ext_prop_name_ptr(data),
3473		       ext_prop->name_len);
3474		/* property name reported to the host as "WCHAR"s */
3475		ext_prop->name_len *= 2;
3476		ext_prop->name = ext_prop_name;
3477
3478		t->os_desc->ext_prop_len +=
3479			ext_prop->name_len + ext_prop->data_len + 14;
3480		++t->os_desc->ext_prop_count;
3481		list_add_tail(&ext_prop->entry, &t->os_desc->ext_prop);
3482	}
3483		break;
3484	default:
3485		pr_vdebug("unknown descriptor: %d\n", type);
3486	}
3487
3488	return length;
3489}
3490
3491static inline struct f_fs_opts *ffs_do_functionfs_bind(struct usb_function *f,
3492						struct usb_configuration *c)
3493{
3494	struct ffs_function *func = ffs_func_from_usb(f);
3495	struct f_fs_opts *ffs_opts =
3496		container_of(f->fi, struct f_fs_opts, func_inst);
3497	struct ffs_data *ffs_data;
3498	int ret;
3499
3500	/*
3501	 * Legacy gadget triggers binding in functionfs_ready_callback,
3502	 * which already uses locking; taking the same lock here would
3503	 * cause a deadlock.
3504	 *
3505	 * Configfs-enabled gadgets however do need ffs_dev_lock.
3506	 */
3507	if (!ffs_opts->no_configfs)
3508		ffs_dev_lock();
3509	ret = ffs_opts->dev->desc_ready ? 0 : -ENODEV;
3510	ffs_data = ffs_opts->dev->ffs_data;
3511	if (!ffs_opts->no_configfs)
3512		ffs_dev_unlock();
3513	if (ret)
3514		return ERR_PTR(ret);
3515
3516	func->ffs = ffs_data;
3517	func->conf = c;
3518	func->gadget = c->cdev->gadget;
3519
3520	/*
3521	 * in drivers/usb/gadget/configfs.c:configfs_composite_bind()
3522	 * configurations are bound in sequence with list_for_each_entry,
3523	 * in each configuration its functions are bound in sequence
3524	 * with list_for_each_entry, so we assume no race condition
3525	 * with regard to ffs_opts->bound access
3526	 */
3527	if (!ffs_opts->refcnt) {
3528		ret = functionfs_bind(func->ffs, c->cdev);
3529		if (ret)
3530			return ERR_PTR(ret);
3531	}
3532	ffs_opts->refcnt++;
3533	func->function.strings = func->ffs->stringtabs;
3534
3535	return ffs_opts;
3536}
3537
3538static int _ffs_func_bind(struct usb_configuration *c,
3539			  struct usb_function *f)
3540{
3541	struct ffs_function *func = ffs_func_from_usb(f);
3542	struct ffs_data *ffs = func->ffs;
3543
3544	const int full = !!func->ffs->fs_descs_count;
3545	const int high = !!func->ffs->hs_descs_count;
3546	const int super = !!func->ffs->ss_descs_count;
3547
3548	int fs_len, hs_len, ss_len, ret, i;
3549	struct ffs_ep *eps_ptr;
3550
3551	/* Make it a single chunk, less management later on */
3552	vla_group(d);
3553	vla_item_with_sz(d, struct ffs_ep, eps, ffs->eps_count);
3554	vla_item_with_sz(d, struct usb_descriptor_header *, fs_descs,
3555		full ? ffs->fs_descs_count + 1 : 0);
3556	vla_item_with_sz(d, struct usb_descriptor_header *, hs_descs,
3557		high ? ffs->hs_descs_count + 1 : 0);
3558	vla_item_with_sz(d, struct usb_descriptor_header *, ss_descs,
3559		super ? ffs->ss_descs_count + 1 : 0);
3560	vla_item_with_sz(d, short, inums, ffs->interfaces_count);
3561	vla_item_with_sz(d, struct usb_os_desc_table, os_desc_table,
3562			 c->cdev->use_os_string ? ffs->interfaces_count : 0);
3563	vla_item_with_sz(d, char[16], ext_compat,
3564			 c->cdev->use_os_string ? ffs->interfaces_count : 0);
3565	vla_item_with_sz(d, struct usb_os_desc, os_desc,
3566			 c->cdev->use_os_string ? ffs->interfaces_count : 0);
3567	vla_item_with_sz(d, struct usb_os_desc_ext_prop, ext_prop,
3568			 ffs->ms_os_descs_ext_prop_count);
3569	vla_item_with_sz(d, char, ext_prop_name,
3570			 ffs->ms_os_descs_ext_prop_name_len);
3571	vla_item_with_sz(d, char, ext_prop_data,
3572			 ffs->ms_os_descs_ext_prop_data_len);
3573	vla_item_with_sz(d, char, raw_descs, ffs->raw_descs_length);
3574	char *vlabuf;
3575
3576	/* Has descriptors only for speeds gadget does not support */
3577	if (!(full | high | super))
3578		return -ENOTSUPP;
3579
3580	/* Allocate a single chunk, less management later on */
3581	vlabuf = kzalloc(vla_group_size(d), GFP_KERNEL);
3582	if (!vlabuf)
3583		return -ENOMEM;
3584
3585	ffs->ms_os_descs_ext_prop_avail = vla_ptr(vlabuf, d, ext_prop);
3586	ffs->ms_os_descs_ext_prop_name_avail =
3587		vla_ptr(vlabuf, d, ext_prop_name);
3588	ffs->ms_os_descs_ext_prop_data_avail =
3589		vla_ptr(vlabuf, d, ext_prop_data);
3590
3591	/* Copy descriptors  */
3592	memcpy(vla_ptr(vlabuf, d, raw_descs), ffs->raw_descs,
3593	       ffs->raw_descs_length);
3594
3595	memset(vla_ptr(vlabuf, d, inums), 0xff, d_inums__sz);
3596	eps_ptr = vla_ptr(vlabuf, d, eps);
3597	for (i = 0; i < ffs->eps_count; i++)
3598		eps_ptr[i].num = -1;
3599
3600	/* Save pointers
3601	 * d_eps == vlabuf, func->eps used to kfree vlabuf later
3602	*/
3603	func->eps             = vla_ptr(vlabuf, d, eps);
3604	func->interfaces_nums = vla_ptr(vlabuf, d, inums);
3605
3606	/*
3607	 * Go through all the endpoint descriptors and allocate
3608	 * endpoints first, so that later we can rewrite the endpoint
3609	 * numbers without worrying that it may be described later on.
3610	 */
3611	if (full) {
3612		func->function.fs_descriptors = vla_ptr(vlabuf, d, fs_descs);
3613		fs_len = ffs_do_descs(ffs->fs_descs_count,
3614				      vla_ptr(vlabuf, d, raw_descs),
3615				      d_raw_descs__sz,
3616				      __ffs_func_bind_do_descs, func);
3617		if (fs_len < 0) {
3618			ret = fs_len;
3619			goto error;
3620		}
3621	} else {
3622		fs_len = 0;
3623	}
3624
3625	if (high) {
3626		func->function.hs_descriptors = vla_ptr(vlabuf, d, hs_descs);
3627		hs_len = ffs_do_descs(ffs->hs_descs_count,
3628				      vla_ptr(vlabuf, d, raw_descs) + fs_len,
3629				      d_raw_descs__sz - fs_len,
3630				      __ffs_func_bind_do_descs, func);
3631		if (hs_len < 0) {
3632			ret = hs_len;
3633			goto error;
3634		}
3635	} else {
3636		hs_len = 0;
3637	}
3638
3639	if (super) {
3640		func->function.ss_descriptors = func->function.ssp_descriptors =
3641			vla_ptr(vlabuf, d, ss_descs);
3642		ss_len = ffs_do_descs(ffs->ss_descs_count,
3643				vla_ptr(vlabuf, d, raw_descs) + fs_len + hs_len,
3644				d_raw_descs__sz - fs_len - hs_len,
3645				__ffs_func_bind_do_descs, func);
3646		if (ss_len < 0) {
3647			ret = ss_len;
3648			goto error;
3649		}
3650	} else {
3651		ss_len = 0;
3652	}
3653
3654	/*
3655	 * Now handle interface numbers allocation and interface and
3656	 * endpoint numbers rewriting.  We can do that in one go
3657	 * now.
3658	 */
3659	ret = ffs_do_descs(ffs->fs_descs_count +
3660			   (high ? ffs->hs_descs_count : 0) +
3661			   (super ? ffs->ss_descs_count : 0),
3662			   vla_ptr(vlabuf, d, raw_descs), d_raw_descs__sz,
3663			   __ffs_func_bind_do_nums, func);
3664	if (ret < 0)
3665		goto error;
3666
3667	func->function.os_desc_table = vla_ptr(vlabuf, d, os_desc_table);
3668	if (c->cdev->use_os_string) {
3669		for (i = 0; i < ffs->interfaces_count; ++i) {
3670			struct usb_os_desc *desc;
3671
3672			desc = func->function.os_desc_table[i].os_desc =
3673				vla_ptr(vlabuf, d, os_desc) +
3674				i * sizeof(struct usb_os_desc);
3675			desc->ext_compat_id =
3676				vla_ptr(vlabuf, d, ext_compat) + i * 16;
3677			INIT_LIST_HEAD(&desc->ext_prop);
3678		}
3679		ret = ffs_do_os_descs(ffs->ms_os_descs_count,
3680				      vla_ptr(vlabuf, d, raw_descs) +
3681				      fs_len + hs_len + ss_len,
3682				      d_raw_descs__sz - fs_len - hs_len -
3683				      ss_len,
3684				      __ffs_func_bind_do_os_desc, func);
3685		if (ret < 0)
3686			goto error;
3687	}
3688	func->function.os_desc_n =
3689		c->cdev->use_os_string ? ffs->interfaces_count : 0;
3690
3691	/* And we're done */
3692	ffs_event_add(ffs, FUNCTIONFS_BIND);
3693	return 0;
3694
3695error:
3696	/* XXX Do we need to release all claimed endpoints here? */
3697	return ret;
3698}
3699
3700static int ffs_func_bind(struct usb_configuration *c,
3701			 struct usb_function *f)
3702{
3703	struct f_fs_opts *ffs_opts = ffs_do_functionfs_bind(f, c);
3704	struct ffs_function *func = ffs_func_from_usb(f);
3705	int ret;
3706
3707	if (IS_ERR(ffs_opts))
3708		return PTR_ERR(ffs_opts);
3709
3710	ret = _ffs_func_bind(c, f);
3711	if (ret && !--ffs_opts->refcnt)
3712		functionfs_unbind(func->ffs);
3713
3714	return ret;
3715}
3716
3717
3718/* Other USB function hooks *************************************************/
3719
3720static void ffs_reset_work(struct work_struct *work)
3721{
3722	struct ffs_data *ffs = container_of(work,
3723		struct ffs_data, reset_work);
3724	ffs_data_reset(ffs);
3725}
3726
3727static int ffs_func_get_alt(struct usb_function *f,
3728			    unsigned int interface)
3729{
3730	struct ffs_function *func = ffs_func_from_usb(f);
3731	int intf = ffs_func_revmap_intf(func, interface);
3732
3733	return (intf < 0) ? intf : func->cur_alt[interface];
3734}
3735
3736static int ffs_func_set_alt(struct usb_function *f,
3737			    unsigned interface, unsigned alt)
3738{
3739	struct ffs_function *func = ffs_func_from_usb(f);
3740	struct ffs_data *ffs = func->ffs;
3741	unsigned long flags;
3742	int ret = 0, intf;
3743
3744	if (alt > MAX_ALT_SETTINGS)
3745		return -EINVAL;
3746
3747	intf = ffs_func_revmap_intf(func, interface);
3748	if (intf < 0)
3749		return intf;
3750
3751	if (ffs->func)
3752		ffs_func_eps_disable(ffs->func);
3753
3754	spin_lock_irqsave(&ffs->eps_lock, flags);
3755	if (ffs->state == FFS_DEACTIVATED) {
3756		ffs->state = FFS_CLOSING;
3757		spin_unlock_irqrestore(&ffs->eps_lock, flags);
3758		INIT_WORK(&ffs->reset_work, ffs_reset_work);
3759		schedule_work(&ffs->reset_work);
3760		return -ENODEV;
3761	}
3762	spin_unlock_irqrestore(&ffs->eps_lock, flags);
3763
3764	if (ffs->state != FFS_ACTIVE)
3765		return -ENODEV;
3766
3767	ffs->func = func;
3768	ret = ffs_func_eps_enable(func);
3769	if (ret >= 0) {
3770		ffs_event_add(ffs, FUNCTIONFS_ENABLE);
3771		func->cur_alt[interface] = alt;
3772	}
3773	return ret;
3774}
3775
3776static void ffs_func_disable(struct usb_function *f)
3777{
3778	struct ffs_function *func = ffs_func_from_usb(f);
3779	struct ffs_data *ffs = func->ffs;
3780	unsigned long flags;
3781
3782	if (ffs->func)
3783		ffs_func_eps_disable(ffs->func);
3784
3785	spin_lock_irqsave(&ffs->eps_lock, flags);
3786	if (ffs->state == FFS_DEACTIVATED) {
3787		ffs->state = FFS_CLOSING;
3788		spin_unlock_irqrestore(&ffs->eps_lock, flags);
3789		INIT_WORK(&ffs->reset_work, ffs_reset_work);
3790		schedule_work(&ffs->reset_work);
3791		return;
3792	}
3793	spin_unlock_irqrestore(&ffs->eps_lock, flags);
3794
3795	if (ffs->state == FFS_ACTIVE) {
3796		ffs->func = NULL;
3797		ffs_event_add(ffs, FUNCTIONFS_DISABLE);
3798	}
3799}
3800
3801static int ffs_func_setup(struct usb_function *f,
3802			  const struct usb_ctrlrequest *creq)
3803{
3804	struct ffs_function *func = ffs_func_from_usb(f);
3805	struct ffs_data *ffs = func->ffs;
3806	unsigned long flags;
3807	int ret;
3808
3809	pr_vdebug("creq->bRequestType = %02x\n", creq->bRequestType);
3810	pr_vdebug("creq->bRequest     = %02x\n", creq->bRequest);
3811	pr_vdebug("creq->wValue       = %04x\n", le16_to_cpu(creq->wValue));
3812	pr_vdebug("creq->wIndex       = %04x\n", le16_to_cpu(creq->wIndex));
3813	pr_vdebug("creq->wLength      = %04x\n", le16_to_cpu(creq->wLength));
3814
3815	/*
3816	 * Most requests directed to interface go through here
3817	 * (notable exceptions are set/get interface) so we need to
3818	 * handle them.  All other either handled by composite or
3819	 * passed to usb_configuration->setup() (if one is set).  No
3820	 * matter, we will handle requests directed to endpoint here
3821	 * as well (as it's straightforward).  Other request recipient
3822	 * types are only handled when the user flag FUNCTIONFS_ALL_CTRL_RECIP
3823	 * is being used.
3824	 */
3825	if (ffs->state != FFS_ACTIVE)
3826		return -ENODEV;
3827
3828	switch (creq->bRequestType & USB_RECIP_MASK) {
3829	case USB_RECIP_INTERFACE:
3830		ret = ffs_func_revmap_intf(func, le16_to_cpu(creq->wIndex));
3831		if (ret < 0)
3832			return ret;
3833		break;
3834
3835	case USB_RECIP_ENDPOINT:
3836		ret = ffs_func_revmap_ep(func, le16_to_cpu(creq->wIndex));
3837		if (ret < 0)
3838			return ret;
3839		if (func->ffs->user_flags & FUNCTIONFS_VIRTUAL_ADDR)
3840			ret = func->ffs->eps_addrmap[ret];
3841		break;
3842
3843	default:
3844		if (func->ffs->user_flags & FUNCTIONFS_ALL_CTRL_RECIP)
3845			ret = le16_to_cpu(creq->wIndex);
3846		else
3847			return -EOPNOTSUPP;
3848	}
3849
3850	spin_lock_irqsave(&ffs->ev.waitq.lock, flags);
3851	ffs->ev.setup = *creq;
3852	ffs->ev.setup.wIndex = cpu_to_le16(ret);
3853	__ffs_event_add(ffs, FUNCTIONFS_SETUP);
3854	spin_unlock_irqrestore(&ffs->ev.waitq.lock, flags);
3855
3856	return ffs->ev.setup.wLength == 0 ? USB_GADGET_DELAYED_STATUS : 0;
3857}
3858
3859static bool ffs_func_req_match(struct usb_function *f,
3860			       const struct usb_ctrlrequest *creq,
3861			       bool config0)
3862{
3863	struct ffs_function *func = ffs_func_from_usb(f);
3864
3865	if (config0 && !(func->ffs->user_flags & FUNCTIONFS_CONFIG0_SETUP))
3866		return false;
3867
3868	switch (creq->bRequestType & USB_RECIP_MASK) {
3869	case USB_RECIP_INTERFACE:
3870		return (ffs_func_revmap_intf(func,
3871					     le16_to_cpu(creq->wIndex)) >= 0);
3872	case USB_RECIP_ENDPOINT:
3873		return (ffs_func_revmap_ep(func,
3874					   le16_to_cpu(creq->wIndex)) >= 0);
3875	default:
3876		return (bool) (func->ffs->user_flags &
3877			       FUNCTIONFS_ALL_CTRL_RECIP);
3878	}
3879}
3880
3881static void ffs_func_suspend(struct usb_function *f)
3882{
3883	ffs_event_add(ffs_func_from_usb(f)->ffs, FUNCTIONFS_SUSPEND);
3884}
3885
3886static void ffs_func_resume(struct usb_function *f)
3887{
3888	ffs_event_add(ffs_func_from_usb(f)->ffs, FUNCTIONFS_RESUME);
3889}
3890
3891
3892/* Endpoint and interface numbers reverse mapping ***************************/
3893
3894static int ffs_func_revmap_ep(struct ffs_function *func, u8 num)
3895{
3896	num = func->eps_revmap[num & USB_ENDPOINT_NUMBER_MASK];
3897	return num ? num : -EDOM;
3898}
3899
3900static int ffs_func_revmap_intf(struct ffs_function *func, u8 intf)
3901{
3902	short *nums = func->interfaces_nums;
3903	unsigned count = func->ffs->interfaces_count;
3904
3905	for (; count; --count, ++nums) {
3906		if (*nums >= 0 && *nums == intf)
3907			return nums - func->interfaces_nums;
3908	}
3909
3910	return -EDOM;
3911}
3912
3913
3914/* Devices management *******************************************************/
3915
3916static LIST_HEAD(ffs_devices);
3917
3918static struct ffs_dev *_ffs_do_find_dev(const char *name)
3919{
3920	struct ffs_dev *dev;
3921
3922	if (!name)
3923		return NULL;
3924
3925	list_for_each_entry(dev, &ffs_devices, entry) {
3926		if (strcmp(dev->name, name) == 0)
3927			return dev;
3928	}
3929
3930	return NULL;
3931}
3932
3933/*
3934 * ffs_lock must be taken by the caller of this function
3935 */
3936static struct ffs_dev *_ffs_get_single_dev(void)
3937{
3938	struct ffs_dev *dev;
3939
3940	if (list_is_singular(&ffs_devices)) {
3941		dev = list_first_entry(&ffs_devices, struct ffs_dev, entry);
3942		if (dev->single)
3943			return dev;
3944	}
3945
3946	return NULL;
3947}
3948
3949/*
3950 * ffs_lock must be taken by the caller of this function
3951 */
3952static struct ffs_dev *_ffs_find_dev(const char *name)
3953{
3954	struct ffs_dev *dev;
3955
3956	dev = _ffs_get_single_dev();
3957	if (dev)
3958		return dev;
3959
3960	return _ffs_do_find_dev(name);
3961}
3962
3963/* Configfs support *********************************************************/
3964
3965static inline struct f_fs_opts *to_ffs_opts(struct config_item *item)
3966{
3967	return container_of(to_config_group(item), struct f_fs_opts,
3968			    func_inst.group);
3969}
3970
3971static ssize_t f_fs_opts_ready_show(struct config_item *item, char *page)
3972{
3973	struct f_fs_opts *opts = to_ffs_opts(item);
3974	int ready;
3975
3976	ffs_dev_lock();
3977	ready = opts->dev->desc_ready;
3978	ffs_dev_unlock();
3979
3980	return sprintf(page, "%d\n", ready);
3981}
3982
3983CONFIGFS_ATTR_RO(f_fs_opts_, ready);
3984
3985static struct configfs_attribute *ffs_attrs[] = {
3986	&f_fs_opts_attr_ready,
3987	NULL,
3988};
3989
3990static void ffs_attr_release(struct config_item *item)
3991{
3992	struct f_fs_opts *opts = to_ffs_opts(item);
3993
3994	usb_put_function_instance(&opts->func_inst);
3995}
3996
3997static struct configfs_item_operations ffs_item_ops = {
3998	.release	= ffs_attr_release,
3999};
4000
4001static const struct config_item_type ffs_func_type = {
4002	.ct_item_ops	= &ffs_item_ops,
4003	.ct_attrs	= ffs_attrs,
4004	.ct_owner	= THIS_MODULE,
4005};
4006
4007
4008/* Function registration interface ******************************************/
4009
4010static void ffs_free_inst(struct usb_function_instance *f)
4011{
4012	struct f_fs_opts *opts;
4013
4014	opts = to_f_fs_opts(f);
4015	ffs_release_dev(opts->dev);
4016	ffs_dev_lock();
4017	_ffs_free_dev(opts->dev);
4018	ffs_dev_unlock();
4019	kfree(opts);
4020}
4021
4022static int ffs_set_inst_name(struct usb_function_instance *fi, const char *name)
4023{
4024	if (strlen(name) >= sizeof_field(struct ffs_dev, name))
4025		return -ENAMETOOLONG;
4026	return ffs_name_dev(to_f_fs_opts(fi)->dev, name);
4027}
4028
4029static struct usb_function_instance *ffs_alloc_inst(void)
4030{
4031	struct f_fs_opts *opts;
4032	struct ffs_dev *dev;
4033
4034	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
4035	if (!opts)
4036		return ERR_PTR(-ENOMEM);
4037
4038	opts->func_inst.set_inst_name = ffs_set_inst_name;
4039	opts->func_inst.free_func_inst = ffs_free_inst;
4040	ffs_dev_lock();
4041	dev = _ffs_alloc_dev();
4042	ffs_dev_unlock();
4043	if (IS_ERR(dev)) {
4044		kfree(opts);
4045		return ERR_CAST(dev);
4046	}
4047	opts->dev = dev;
4048	dev->opts = opts;
4049
4050	config_group_init_type_name(&opts->func_inst.group, "",
4051				    &ffs_func_type);
4052	return &opts->func_inst;
4053}
4054
4055static void ffs_free(struct usb_function *f)
4056{
4057	kfree(ffs_func_from_usb(f));
4058}
4059
4060static void ffs_func_unbind(struct usb_configuration *c,
4061			    struct usb_function *f)
4062{
4063	struct ffs_function *func = ffs_func_from_usb(f);
4064	struct ffs_data *ffs = func->ffs;
4065	struct f_fs_opts *opts =
4066		container_of(f->fi, struct f_fs_opts, func_inst);
4067	struct ffs_ep *ep = func->eps;
4068	unsigned count = ffs->eps_count;
4069	unsigned long flags;
4070
4071	if (ffs->func == func) {
4072		ffs_func_eps_disable(func);
4073		ffs->func = NULL;
4074	}
4075
4076	/* Drain any pending AIO completions */
4077	drain_workqueue(ffs->io_completion_wq);
4078
4079	ffs_event_add(ffs, FUNCTIONFS_UNBIND);
4080	if (!--opts->refcnt)
4081		functionfs_unbind(ffs);
4082
4083	/* cleanup after autoconfig */
4084	spin_lock_irqsave(&func->ffs->eps_lock, flags);
4085	while (count--) {
4086		if (ep->ep && ep->req)
4087			usb_ep_free_request(ep->ep, ep->req);
4088		ep->req = NULL;
4089		++ep;
4090	}
4091	spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
4092	kfree(func->eps);
4093	func->eps = NULL;
4094	/*
4095	 * eps, descriptors and interfaces_nums are allocated in the
4096	 * same chunk so only one free is required.
4097	 */
4098	func->function.fs_descriptors = NULL;
4099	func->function.hs_descriptors = NULL;
4100	func->function.ss_descriptors = NULL;
4101	func->function.ssp_descriptors = NULL;
4102	func->interfaces_nums = NULL;
4103
4104}
4105
4106static struct usb_function *ffs_alloc(struct usb_function_instance *fi)
4107{
4108	struct ffs_function *func;
4109
4110	func = kzalloc(sizeof(*func), GFP_KERNEL);
4111	if (!func)
4112		return ERR_PTR(-ENOMEM);
4113
4114	func->function.name    = "Function FS Gadget";
4115
4116	func->function.bind    = ffs_func_bind;
4117	func->function.unbind  = ffs_func_unbind;
4118	func->function.set_alt = ffs_func_set_alt;
4119	func->function.get_alt = ffs_func_get_alt;
4120	func->function.disable = ffs_func_disable;
4121	func->function.setup   = ffs_func_setup;
4122	func->function.req_match = ffs_func_req_match;
4123	func->function.suspend = ffs_func_suspend;
4124	func->function.resume  = ffs_func_resume;
4125	func->function.free_func = ffs_free;
4126
4127	return &func->function;
4128}
4129
4130/*
4131 * ffs_lock must be taken by the caller of this function
4132 */
4133static struct ffs_dev *_ffs_alloc_dev(void)
4134{
4135	struct ffs_dev *dev;
4136	int ret;
4137
4138	if (_ffs_get_single_dev())
4139			return ERR_PTR(-EBUSY);
4140
4141	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
4142	if (!dev)
4143		return ERR_PTR(-ENOMEM);
4144
4145	if (list_empty(&ffs_devices)) {
4146		ret = functionfs_init();
4147		if (ret) {
4148			kfree(dev);
4149			return ERR_PTR(ret);
4150		}
4151	}
4152
4153	list_add(&dev->entry, &ffs_devices);
4154
4155	return dev;
4156}
4157
4158int ffs_name_dev(struct ffs_dev *dev, const char *name)
4159{
4160	struct ffs_dev *existing;
4161	int ret = 0;
4162
4163	ffs_dev_lock();
4164
4165	existing = _ffs_do_find_dev(name);
4166	if (!existing)
4167		strscpy(dev->name, name, ARRAY_SIZE(dev->name));
4168	else if (existing != dev)
4169		ret = -EBUSY;
4170
4171	ffs_dev_unlock();
4172
4173	return ret;
4174}
4175EXPORT_SYMBOL_GPL(ffs_name_dev);
4176
4177int ffs_single_dev(struct ffs_dev *dev)
4178{
4179	int ret;
4180
4181	ret = 0;
4182	ffs_dev_lock();
4183
4184	if (!list_is_singular(&ffs_devices))
4185		ret = -EBUSY;
4186	else
4187		dev->single = true;
4188
4189	ffs_dev_unlock();
4190	return ret;
4191}
4192EXPORT_SYMBOL_GPL(ffs_single_dev);
4193
4194/*
4195 * ffs_lock must be taken by the caller of this function
4196 */
4197static void _ffs_free_dev(struct ffs_dev *dev)
4198{
4199	list_del(&dev->entry);
4200
4201	kfree(dev);
4202	if (list_empty(&ffs_devices))
4203		functionfs_cleanup();
4204}
4205
4206static int ffs_acquire_dev(const char *dev_name, struct ffs_data *ffs_data)
4207{
4208	int ret = 0;
4209	struct ffs_dev *ffs_dev;
4210
4211	ffs_dev_lock();
4212
4213	ffs_dev = _ffs_find_dev(dev_name);
4214	if (!ffs_dev) {
4215		ret = -ENOENT;
4216	} else if (ffs_dev->mounted) {
4217		ret = -EBUSY;
4218	} else if (ffs_dev->ffs_acquire_dev_callback &&
4219		   ffs_dev->ffs_acquire_dev_callback(ffs_dev)) {
4220		ret = -ENOENT;
4221	} else {
4222		ffs_dev->mounted = true;
4223		ffs_dev->ffs_data = ffs_data;
4224		ffs_data->private_data = ffs_dev;
4225	}
4226
4227	ffs_dev_unlock();
4228	return ret;
4229}
4230
4231static void ffs_release_dev(struct ffs_dev *ffs_dev)
4232{
4233	ffs_dev_lock();
4234
4235	if (ffs_dev && ffs_dev->mounted) {
4236		ffs_dev->mounted = false;
4237		if (ffs_dev->ffs_data) {
4238			ffs_dev->ffs_data->private_data = NULL;
4239			ffs_dev->ffs_data = NULL;
4240		}
4241
4242		if (ffs_dev->ffs_release_dev_callback)
4243			ffs_dev->ffs_release_dev_callback(ffs_dev);
4244	}
4245
4246	ffs_dev_unlock();
4247}
4248
4249static int ffs_ready(struct ffs_data *ffs)
4250{
4251	struct ffs_dev *ffs_obj;
4252	int ret = 0;
4253
4254	ffs_dev_lock();
4255
4256	ffs_obj = ffs->private_data;
4257	if (!ffs_obj) {
4258		ret = -EINVAL;
4259		goto done;
4260	}
4261	if (WARN_ON(ffs_obj->desc_ready)) {
4262		ret = -EBUSY;
4263		goto done;
4264	}
4265
4266	ffs_obj->desc_ready = true;
4267
4268	if (ffs_obj->ffs_ready_callback) {
4269		ret = ffs_obj->ffs_ready_callback(ffs);
4270		if (ret)
4271			goto done;
4272	}
4273
4274	set_bit(FFS_FL_CALL_CLOSED_CALLBACK, &ffs->flags);
4275done:
4276	ffs_dev_unlock();
4277	return ret;
4278}
4279
4280static void ffs_closed(struct ffs_data *ffs)
4281{
4282	struct ffs_dev *ffs_obj;
4283	struct f_fs_opts *opts;
4284	struct config_item *ci;
4285
4286	ffs_dev_lock();
4287
4288	ffs_obj = ffs->private_data;
4289	if (!ffs_obj)
4290		goto done;
4291
4292	ffs_obj->desc_ready = false;
4293
4294	if (test_and_clear_bit(FFS_FL_CALL_CLOSED_CALLBACK, &ffs->flags) &&
4295	    ffs_obj->ffs_closed_callback)
4296		ffs_obj->ffs_closed_callback(ffs);
4297
4298	if (ffs_obj->opts)
4299		opts = ffs_obj->opts;
4300	else
4301		goto done;
4302
4303	if (opts->no_configfs || !opts->func_inst.group.cg_item.ci_parent
4304	    || !kref_read(&opts->func_inst.group.cg_item.ci_kref))
4305		goto done;
4306
4307	ci = opts->func_inst.group.cg_item.ci_parent->ci_parent;
4308	ffs_dev_unlock();
4309
4310	if (test_bit(FFS_FL_BOUND, &ffs->flags))
4311		unregister_gadget_item(ci);
4312	return;
4313done:
4314	ffs_dev_unlock();
4315}
4316
4317/* Misc helper functions ****************************************************/
4318
4319static int ffs_mutex_lock(struct mutex *mutex, unsigned nonblock)
4320{
4321	return nonblock
4322		? mutex_trylock(mutex) ? 0 : -EAGAIN
4323		: mutex_lock_interruptible(mutex);
4324}
4325
4326static char *ffs_prepare_buffer(const char __user *buf, size_t len)
4327{
4328	char *data;
4329
4330	if (!len)
4331		return NULL;
4332
4333	data = memdup_user(buf, len);
4334	if (IS_ERR(data))
4335		return data;
4336
4337	pr_vdebug("Buffer from user space:\n");
4338	ffs_dump_mem("", data, len);
4339
4340	return data;
4341}
4342
4343DECLARE_USB_FUNCTION_INIT(ffs, ffs_alloc_inst, ffs_alloc);
4344MODULE_DESCRIPTION("user mode file system API for USB composite function controllers");
4345MODULE_LICENSE("GPL");
4346MODULE_AUTHOR("Michal Nazarewicz");