xillyusb.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright 2020 Xillybus Ltd, http://xillybus.com
   4 *
   5 * Driver for the XillyUSB FPGA/host framework.
   6 *
   7 * This driver interfaces with a special IP core in an FPGA, setting up
   8 * a pipe between a hardware FIFO in the programmable logic and a device
   9 * file in the host. The number of such pipes and their attributes are
  10 * set up on the logic. This driver detects these automatically and
  11 * creates the device files accordingly.
  12 */
  13
  14#include <linux/types.h>
  15#include <linux/slab.h>
  16#include <linux/list.h>
  17#include <linux/device.h>
  18#include <linux/module.h>
  19#include <asm/byteorder.h>
  20#include <linux/io.h>
  21#include <linux/interrupt.h>
  22#include <linux/sched.h>
  23#include <linux/fs.h>
  24#include <linux/spinlock.h>
  25#include <linux/mutex.h>
  26#include <linux/workqueue.h>
  27#include <linux/crc32.h>
  28#include <linux/poll.h>
  29#include <linux/delay.h>
  30#include <linux/usb.h>
  31
  32#include "xillybus_class.h"
  33
  34MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core");
  35MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  36MODULE_ALIAS("xillyusb");
  37MODULE_LICENSE("GPL v2");
  38
  39#define XILLY_RX_TIMEOUT		(10 * HZ / 1000)
  40#define XILLY_RESPONSE_TIMEOUT		(500 * HZ / 1000)
  41
  42#define BUF_SIZE_ORDER			4
  43#define BUFNUM				8
  44#define LOG2_IDT_FIFO_SIZE		16
  45#define LOG2_INITIAL_FIFO_BUF_SIZE	16
  46
  47#define MSG_EP_NUM			1
  48#define IN_EP_NUM			1
  49
  50static const char xillyname[] = "xillyusb";
  51
  52static unsigned int fifo_buf_order;
  53static struct workqueue_struct *wakeup_wq;
  54
  55#define USB_VENDOR_ID_XILINX		0x03fd
  56#define USB_VENDOR_ID_ALTERA		0x09fb
  57
  58#define USB_PRODUCT_ID_XILLYUSB		0xebbe
  59
  60static const struct usb_device_id xillyusb_table[] = {
  61	{ USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) },
  62	{ USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) },
  63	{ }
  64};
  65
  66MODULE_DEVICE_TABLE(usb, xillyusb_table);
  67
  68struct xillyusb_dev;
  69
  70struct xillyfifo {
  71	unsigned int bufsize; /* In bytes, always a power of 2 */
  72	unsigned int bufnum;
  73	unsigned int size; /* Lazy: Equals bufsize * bufnum */
  74	unsigned int buf_order;
  75
  76	int fill; /* Number of bytes in the FIFO */
  77	spinlock_t lock;
  78	wait_queue_head_t waitq;
  79
  80	unsigned int readpos;
  81	unsigned int readbuf;
  82	unsigned int writepos;
  83	unsigned int writebuf;
  84	char **mem;
  85};
  86
  87struct xillyusb_channel;
  88
  89struct xillyusb_endpoint {
  90	struct xillyusb_dev *xdev;
  91
  92	struct mutex ep_mutex; /* serialize operations on endpoint */
  93
  94	struct list_head buffers;
  95	struct list_head filled_buffers;
  96	spinlock_t buffers_lock; /* protect these two lists */
  97
  98	unsigned int order;
  99	unsigned int buffer_size;
 100
 101	unsigned int fill_mask;
 102
 103	int outstanding_urbs;
 104
 105	struct usb_anchor anchor;
 106
 107	struct xillyfifo fifo;
 108
 109	struct work_struct workitem;
 110
 111	bool shutting_down;
 112	bool drained;
 113	bool wake_on_drain;
 114
 115	u8 ep_num;
 116};
 117
 118struct xillyusb_channel {
 119	struct xillyusb_dev *xdev;
 120
 121	struct xillyfifo *in_fifo;
 122	struct xillyusb_endpoint *out_ep;
 123	struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */
 124
 125	struct mutex in_mutex; /* serialize fops on FPGA to host stream */
 126	struct mutex out_mutex; /* serialize fops on host to FPGA stream */
 127	wait_queue_head_t flushq;
 128
 129	int chan_idx;
 130
 131	u32 in_consumed_bytes;
 132	u32 in_current_checkpoint;
 133	u32 out_bytes;
 134
 135	unsigned int in_log2_element_size;
 136	unsigned int out_log2_element_size;
 137	unsigned int in_log2_fifo_size;
 138	unsigned int out_log2_fifo_size;
 139
 140	unsigned int read_data_ok; /* EOF not arrived (yet) */
 141	unsigned int poll_used;
 142	unsigned int flushing;
 143	unsigned int flushed;
 144	unsigned int canceled;
 145
 146	/* Bit fields protected by @lock except for initialization */
 147	unsigned readable:1;
 148	unsigned writable:1;
 149	unsigned open_for_read:1;
 150	unsigned open_for_write:1;
 151	unsigned in_synchronous:1;
 152	unsigned out_synchronous:1;
 153	unsigned in_seekable:1;
 154	unsigned out_seekable:1;
 155};
 156
 157struct xillybuffer {
 158	struct list_head entry;
 159	struct xillyusb_endpoint *ep;
 160	void *buf;
 161	unsigned int len;
 162};
 163
 164struct xillyusb_dev {
 165	struct xillyusb_channel *channels;
 166
 167	struct usb_device	*udev;
 168	struct device		*dev; /* For dev_err() and such */
 169	struct kref		kref;
 170	struct workqueue_struct	*workq;
 171
 172	int error;
 173	spinlock_t error_lock; /* protect @error */
 174	struct work_struct wakeup_workitem;
 175
 176	int num_channels;
 177
 178	struct xillyusb_endpoint *msg_ep;
 179	struct xillyusb_endpoint *in_ep;
 180
 181	struct mutex msg_mutex; /* serialize opcode transmission */
 182	int in_bytes_left;
 183	int leftover_chan_num;
 184	unsigned int in_counter;
 185	struct mutex process_in_mutex; /* synchronize wakeup_all() */
 186};
 187
 188/*
 189 * kref_mutex is used in xillyusb_open() to prevent the xillyusb_dev
 190 * struct from being freed during the gap between being found by
 191 * xillybus_find_inode() and having its reference count incremented.
 192 */
 193
 194static DEFINE_MUTEX(kref_mutex);
 195
 196/* FPGA to host opcodes */
 197enum {
 198	OPCODE_DATA = 0,
 199	OPCODE_QUIESCE_ACK = 1,
 200	OPCODE_EOF = 2,
 201	OPCODE_REACHED_CHECKPOINT = 3,
 202	OPCODE_CANCELED_CHECKPOINT = 4,
 203};
 204
 205/* Host to FPGA opcodes */
 206enum {
 207	OPCODE_QUIESCE = 0,
 208	OPCODE_REQ_IDT = 1,
 209	OPCODE_SET_CHECKPOINT = 2,
 210	OPCODE_CLOSE = 3,
 211	OPCODE_SET_PUSH = 4,
 212	OPCODE_UPDATE_PUSH = 5,
 213	OPCODE_CANCEL_CHECKPOINT = 6,
 214	OPCODE_SET_ADDR = 7,
 215};
 216
 217/*
 218 * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple
 219 * calls to each on the same FIFO is not allowed) however it's OK to have
 220 * threads calling each of the two functions once on the same FIFO, and
 221 * at the same time.
 222 */
 223
 224static int fifo_write(struct xillyfifo *fifo,
 225		      const void *data, unsigned int len,
 226		      int (*copier)(void *, const void *, int))
 227{
 228	unsigned int done = 0;
 229	unsigned int todo = len;
 230	unsigned int nmax;
 231	unsigned int writepos = fifo->writepos;
 232	unsigned int writebuf = fifo->writebuf;
 233	unsigned long flags;
 234	int rc;
 235
 236	nmax = fifo->size - READ_ONCE(fifo->fill);
 237
 238	while (1) {
 239		unsigned int nrail = fifo->bufsize - writepos;
 240		unsigned int n = min(todo, nmax);
 241
 242		if (n == 0) {
 243			spin_lock_irqsave(&fifo->lock, flags);
 244			fifo->fill += done;
 245			spin_unlock_irqrestore(&fifo->lock, flags);
 246
 247			fifo->writepos = writepos;
 248			fifo->writebuf = writebuf;
 249
 250			return done;
 251		}
 252
 253		if (n > nrail)
 254			n = nrail;
 255
 256		rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n);
 257
 258		if (rc)
 259			return rc;
 260
 261		done += n;
 262		todo -= n;
 263
 264		writepos += n;
 265		nmax -= n;
 266
 267		if (writepos == fifo->bufsize) {
 268			writepos = 0;
 269			writebuf++;
 270
 271			if (writebuf == fifo->bufnum)
 272				writebuf = 0;
 273		}
 274	}
 275}
 276
 277static int fifo_read(struct xillyfifo *fifo,
 278		     void *data, unsigned int len,
 279		     int (*copier)(void *, const void *, int))
 280{
 281	unsigned int done = 0;
 282	unsigned int todo = len;
 283	unsigned int fill;
 284	unsigned int readpos = fifo->readpos;
 285	unsigned int readbuf = fifo->readbuf;
 286	unsigned long flags;
 287	int rc;
 288
 289	/*
 290	 * The spinlock here is necessary, because otherwise fifo->fill
 291	 * could have been increased by fifo_write() after writing data
 292	 * to the buffer, but this data would potentially not have been
 293	 * visible on this thread at the time the updated fifo->fill was.
 294	 * That could lead to reading invalid data.
 295	 */
 296
 297	spin_lock_irqsave(&fifo->lock, flags);
 298	fill = fifo->fill;
 299	spin_unlock_irqrestore(&fifo->lock, flags);
 300
 301	while (1) {
 302		unsigned int nrail = fifo->bufsize - readpos;
 303		unsigned int n = min(todo, fill);
 304
 305		if (n == 0) {
 306			spin_lock_irqsave(&fifo->lock, flags);
 307			fifo->fill -= done;
 308			spin_unlock_irqrestore(&fifo->lock, flags);
 309
 310			fifo->readpos = readpos;
 311			fifo->readbuf = readbuf;
 312
 313			return done;
 314		}
 315
 316		if (n > nrail)
 317			n = nrail;
 318
 319		rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n);
 320
 321		if (rc)
 322			return rc;
 323
 324		done += n;
 325		todo -= n;
 326
 327		readpos += n;
 328		fill -= n;
 329
 330		if (readpos == fifo->bufsize) {
 331			readpos = 0;
 332			readbuf++;
 333
 334			if (readbuf == fifo->bufnum)
 335				readbuf = 0;
 336		}
 337	}
 338}
 339
 340/*
 341 * These three wrapper functions are used as the @copier argument to
 342 * fifo_write() and fifo_read(), so that they can work directly with
 343 * user memory as well.
 344 */
 345
 346static int xilly_copy_from_user(void *dst, const void *src, int n)
 347{
 348	if (copy_from_user(dst, (const void __user *)src, n))
 349		return -EFAULT;
 350
 351	return 0;
 352}
 353
 354static int xilly_copy_to_user(void *dst, const void *src, int n)
 355{
 356	if (copy_to_user((void __user *)dst, src, n))
 357		return -EFAULT;
 358
 359	return 0;
 360}
 361
 362static int xilly_memcpy(void *dst, const void *src, int n)
 363{
 364	memcpy(dst, src, n);
 365
 366	return 0;
 367}
 368
 369static int fifo_init(struct xillyfifo *fifo,
 370		     unsigned int log2_size)
 371{
 372	unsigned int log2_bufnum;
 373	unsigned int buf_order;
 374	int i;
 375
 376	unsigned int log2_fifo_buf_size;
 377
 378retry:
 379	log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT;
 380
 381	if (log2_size > log2_fifo_buf_size) {
 382		log2_bufnum = log2_size - log2_fifo_buf_size;
 383		buf_order = fifo_buf_order;
 384		fifo->bufsize = 1 << log2_fifo_buf_size;
 385	} else {
 386		log2_bufnum = 0;
 387		buf_order = (log2_size > PAGE_SHIFT) ?
 388			log2_size - PAGE_SHIFT : 0;
 389		fifo->bufsize = 1 << log2_size;
 390	}
 391
 392	fifo->bufnum = 1 << log2_bufnum;
 393	fifo->size = fifo->bufnum * fifo->bufsize;
 394	fifo->buf_order = buf_order;
 395
 396	fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL);
 397
 398	if (!fifo->mem)
 399		return -ENOMEM;
 400
 401	for (i = 0; i < fifo->bufnum; i++) {
 402		fifo->mem[i] = (void *)
 403			__get_free_pages(GFP_KERNEL, buf_order);
 404
 405		if (!fifo->mem[i])
 406			goto memfail;
 407	}
 408
 409	fifo->fill = 0;
 410	fifo->readpos = 0;
 411	fifo->readbuf = 0;
 412	fifo->writepos = 0;
 413	fifo->writebuf = 0;
 414	spin_lock_init(&fifo->lock);
 415	init_waitqueue_head(&fifo->waitq);
 416	return 0;
 417
 418memfail:
 419	for (i--; i >= 0; i--)
 420		free_pages((unsigned long)fifo->mem[i], buf_order);
 421
 422	kfree(fifo->mem);
 423	fifo->mem = NULL;
 424
 425	if (fifo_buf_order) {
 426		fifo_buf_order--;
 427		goto retry;
 428	} else {
 429		return -ENOMEM;
 430	}
 431}
 432
 433static void fifo_mem_release(struct xillyfifo *fifo)
 434{
 435	int i;
 436
 437	if (!fifo->mem)
 438		return;
 439
 440	for (i = 0; i < fifo->bufnum; i++)
 441		free_pages((unsigned long)fifo->mem[i], fifo->buf_order);
 442
 443	kfree(fifo->mem);
 444}
 445
 446/*
 447 * When endpoint_quiesce() returns, the endpoint has no URBs submitted,
 448 * won't accept any new URB submissions, and its related work item doesn't
 449 * and won't run anymore.
 450 */
 451
 452static void endpoint_quiesce(struct xillyusb_endpoint *ep)
 453{
 454	mutex_lock(&ep->ep_mutex);
 455	ep->shutting_down = true;
 456	mutex_unlock(&ep->ep_mutex);
 457
 458	usb_kill_anchored_urbs(&ep->anchor);
 459	cancel_work_sync(&ep->workitem);
 460}
 461
 462/*
 463 * Note that endpoint_dealloc() also frees fifo memory (if allocated), even
 464 * though endpoint_alloc doesn't allocate that memory.
 465 */
 466
 467static void endpoint_dealloc(struct xillyusb_endpoint *ep)
 468{
 469	struct list_head *this, *next;
 470
 471	fifo_mem_release(&ep->fifo);
 472
 473	/* Join @filled_buffers with @buffers to free these entries too */
 474	list_splice(&ep->filled_buffers, &ep->buffers);
 475
 476	list_for_each_safe(this, next, &ep->buffers) {
 477		struct xillybuffer *xb =
 478			list_entry(this, struct xillybuffer, entry);
 479
 480		free_pages((unsigned long)xb->buf, ep->order);
 481		kfree(xb);
 482	}
 483
 484	kfree(ep);
 485}
 486
 487static struct xillyusb_endpoint
 488*endpoint_alloc(struct xillyusb_dev *xdev,
 489		u8 ep_num,
 490		void (*work)(struct work_struct *),
 491		unsigned int order,
 492		int bufnum)
 493{
 494	int i;
 495
 496	struct xillyusb_endpoint *ep;
 497
 498	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
 499
 500	if (!ep)
 501		return NULL;
 502
 503	INIT_LIST_HEAD(&ep->buffers);
 504	INIT_LIST_HEAD(&ep->filled_buffers);
 505
 506	spin_lock_init(&ep->buffers_lock);
 507	mutex_init(&ep->ep_mutex);
 508
 509	init_usb_anchor(&ep->anchor);
 510	INIT_WORK(&ep->workitem, work);
 511
 512	ep->order = order;
 513	ep->buffer_size =  1 << (PAGE_SHIFT + order);
 514	ep->outstanding_urbs = 0;
 515	ep->drained = true;
 516	ep->wake_on_drain = false;
 517	ep->xdev = xdev;
 518	ep->ep_num = ep_num;
 519	ep->shutting_down = false;
 520
 521	for (i = 0; i < bufnum; i++) {
 522		struct xillybuffer *xb;
 523		unsigned long addr;
 524
 525		xb = kzalloc(sizeof(*xb), GFP_KERNEL);
 526
 527		if (!xb) {
 528			endpoint_dealloc(ep);
 529			return NULL;
 530		}
 531
 532		addr = __get_free_pages(GFP_KERNEL, order);
 533
 534		if (!addr) {
 535			kfree(xb);
 536			endpoint_dealloc(ep);
 537			return NULL;
 538		}
 539
 540		xb->buf = (void *)addr;
 541		xb->ep = ep;
 542		list_add_tail(&xb->entry, &ep->buffers);
 543	}
 544	return ep;
 545}
 546
 547static void cleanup_dev(struct kref *kref)
 548{
 549	struct xillyusb_dev *xdev =
 550		container_of(kref, struct xillyusb_dev, kref);
 551
 552	if (xdev->in_ep)
 553		endpoint_dealloc(xdev->in_ep);
 554
 555	if (xdev->msg_ep)
 556		endpoint_dealloc(xdev->msg_ep);
 557
 558	if (xdev->workq)
 559		destroy_workqueue(xdev->workq);
 560
 561	usb_put_dev(xdev->udev);
 562	kfree(xdev->channels); /* Argument may be NULL, and that's fine */
 563	kfree(xdev);
 564}
 565
 566/*
 567 * @process_in_mutex is taken to ensure that bulk_in_work() won't call
 568 * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all
 569 * @read_data_ok entries, which will make process_bulk_in() report false
 570 * errors if executed. The mechanism relies on that xdev->error is assigned
 571 * a non-zero value by report_io_error() prior to queueing wakeup_all(),
 572 * which prevents bulk_in_work() from calling process_bulk_in().
 573 */
 574
 575static void wakeup_all(struct work_struct *work)
 576{
 577	int i;
 578	struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev,
 579						 wakeup_workitem);
 580
 581	mutex_lock(&xdev->process_in_mutex);
 582
 583	for (i = 0; i < xdev->num_channels; i++) {
 584		struct xillyusb_channel *chan = &xdev->channels[i];
 585
 586		mutex_lock(&chan->lock);
 587
 588		if (chan->in_fifo) {
 589			/*
 590			 * Fake an EOF: Even if such arrives, it won't be
 591			 * processed.
 592			 */
 593			chan->read_data_ok = 0;
 594			wake_up_interruptible(&chan->in_fifo->waitq);
 595		}
 596
 597		if (chan->out_ep)
 598			wake_up_interruptible(&chan->out_ep->fifo.waitq);
 599
 600		mutex_unlock(&chan->lock);
 601
 602		wake_up_interruptible(&chan->flushq);
 603	}
 604
 605	mutex_unlock(&xdev->process_in_mutex);
 606
 607	wake_up_interruptible(&xdev->msg_ep->fifo.waitq);
 608
 609	kref_put(&xdev->kref, cleanup_dev);
 610}
 611
 612static void report_io_error(struct xillyusb_dev *xdev,
 613			    int errcode)
 614{
 615	unsigned long flags;
 616	bool do_once = false;
 617
 618	spin_lock_irqsave(&xdev->error_lock, flags);
 619	if (!xdev->error) {
 620		xdev->error = errcode;
 621		do_once = true;
 622	}
 623	spin_unlock_irqrestore(&xdev->error_lock, flags);
 624
 625	if (do_once) {
 626		kref_get(&xdev->kref); /* xdev is used by work item */
 627		queue_work(wakeup_wq, &xdev->wakeup_workitem);
 628	}
 629}
 630
 631/*
 632 * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures
 633 * the previous pointer is never used after its return.
 634 */
 635
 636static void safely_assign_in_fifo(struct xillyusb_channel *chan,
 637				  struct xillyfifo *fifo)
 638{
 639	mutex_lock(&chan->lock);
 640	chan->in_fifo = fifo;
 641	mutex_unlock(&chan->lock);
 642
 643	flush_work(&chan->xdev->in_ep->workitem);
 644}
 645
 646static void bulk_in_completer(struct urb *urb)
 647{
 648	struct xillybuffer *xb = urb->context;
 649	struct xillyusb_endpoint *ep = xb->ep;
 650	unsigned long flags;
 651
 652	if (urb->status) {
 653		if (!(urb->status == -ENOENT ||
 654		      urb->status == -ECONNRESET ||
 655		      urb->status == -ESHUTDOWN))
 656			report_io_error(ep->xdev, -EIO);
 657
 658		spin_lock_irqsave(&ep->buffers_lock, flags);
 659		list_add_tail(&xb->entry, &ep->buffers);
 660		ep->outstanding_urbs--;
 661		spin_unlock_irqrestore(&ep->buffers_lock, flags);
 662
 663		return;
 664	}
 665
 666	xb->len = urb->actual_length;
 667
 668	spin_lock_irqsave(&ep->buffers_lock, flags);
 669	list_add_tail(&xb->entry, &ep->filled_buffers);
 670	spin_unlock_irqrestore(&ep->buffers_lock, flags);
 671
 672	if (!ep->shutting_down)
 673		queue_work(ep->xdev->workq, &ep->workitem);
 674}
 675
 676static void bulk_out_completer(struct urb *urb)
 677{
 678	struct xillybuffer *xb = urb->context;
 679	struct xillyusb_endpoint *ep = xb->ep;
 680	unsigned long flags;
 681
 682	if (urb->status &&
 683	    (!(urb->status == -ENOENT ||
 684	       urb->status == -ECONNRESET ||
 685	       urb->status == -ESHUTDOWN)))
 686		report_io_error(ep->xdev, -EIO);
 687
 688	spin_lock_irqsave(&ep->buffers_lock, flags);
 689	list_add_tail(&xb->entry, &ep->buffers);
 690	ep->outstanding_urbs--;
 691	spin_unlock_irqrestore(&ep->buffers_lock, flags);
 692
 693	if (!ep->shutting_down)
 694		queue_work(ep->xdev->workq, &ep->workitem);
 695}
 696
 697static void try_queue_bulk_in(struct xillyusb_endpoint *ep)
 698{
 699	struct xillyusb_dev *xdev = ep->xdev;
 700	struct xillybuffer *xb;
 701	struct urb *urb;
 702
 703	int rc;
 704	unsigned long flags;
 705	unsigned int bufsize = ep->buffer_size;
 706
 707	mutex_lock(&ep->ep_mutex);
 708
 709	if (ep->shutting_down || xdev->error)
 710		goto done;
 711
 712	while (1) {
 713		spin_lock_irqsave(&ep->buffers_lock, flags);
 714
 715		if (list_empty(&ep->buffers)) {
 716			spin_unlock_irqrestore(&ep->buffers_lock, flags);
 717			goto done;
 718		}
 719
 720		xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
 721		list_del(&xb->entry);
 722		ep->outstanding_urbs++;
 723
 724		spin_unlock_irqrestore(&ep->buffers_lock, flags);
 725
 726		urb = usb_alloc_urb(0, GFP_KERNEL);
 727		if (!urb) {
 728			report_io_error(xdev, -ENOMEM);
 729			goto relist;
 730		}
 731
 732		usb_fill_bulk_urb(urb, xdev->udev,
 733				  usb_rcvbulkpipe(xdev->udev, ep->ep_num),
 734				  xb->buf, bufsize, bulk_in_completer, xb);
 735
 736		usb_anchor_urb(urb, &ep->anchor);
 737
 738		rc = usb_submit_urb(urb, GFP_KERNEL);
 739
 740		if (rc) {
 741			report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
 742					-EIO);
 743			goto unanchor;
 744		}
 745
 746		usb_free_urb(urb); /* This just decrements reference count */
 747	}
 748
 749unanchor:
 750	usb_unanchor_urb(urb);
 751	usb_free_urb(urb);
 752
 753relist:
 754	spin_lock_irqsave(&ep->buffers_lock, flags);
 755	list_add_tail(&xb->entry, &ep->buffers);
 756	ep->outstanding_urbs--;
 757	spin_unlock_irqrestore(&ep->buffers_lock, flags);
 758
 759done:
 760	mutex_unlock(&ep->ep_mutex);
 761}
 762
 763static void try_queue_bulk_out(struct xillyusb_endpoint *ep)
 764{
 765	struct xillyfifo *fifo = &ep->fifo;
 766	struct xillyusb_dev *xdev = ep->xdev;
 767	struct xillybuffer *xb;
 768	struct urb *urb;
 769
 770	int rc;
 771	unsigned int fill;
 772	unsigned long flags;
 773	bool do_wake = false;
 774
 775	mutex_lock(&ep->ep_mutex);
 776
 777	if (ep->shutting_down || xdev->error)
 778		goto done;
 779
 780	fill = READ_ONCE(fifo->fill) & ep->fill_mask;
 781
 782	while (1) {
 783		int count;
 784		unsigned int max_read;
 785
 786		spin_lock_irqsave(&ep->buffers_lock, flags);
 787
 788		/*
 789		 * Race conditions might have the FIFO filled while the
 790		 * endpoint is marked as drained here. That doesn't matter,
 791		 * because the sole purpose of @drained is to ensure that
 792		 * certain data has been sent on the USB channel before
 793		 * shutting it down. Hence knowing that the FIFO appears
 794		 * to be empty with no outstanding URBs at some moment
 795		 * is good enough.
 796		 */
 797
 798		if (!fill) {
 799			ep->drained = !ep->outstanding_urbs;
 800			if (ep->drained && ep->wake_on_drain)
 801				do_wake = true;
 802
 803			spin_unlock_irqrestore(&ep->buffers_lock, flags);
 804			goto done;
 805		}
 806
 807		ep->drained = false;
 808
 809		if ((fill < ep->buffer_size && ep->outstanding_urbs) ||
 810		    list_empty(&ep->buffers)) {
 811			spin_unlock_irqrestore(&ep->buffers_lock, flags);
 812			goto done;
 813		}
 814
 815		xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);
 816		list_del(&xb->entry);
 817		ep->outstanding_urbs++;
 818
 819		spin_unlock_irqrestore(&ep->buffers_lock, flags);
 820
 821		max_read = min(fill, ep->buffer_size);
 822
 823		count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy);
 824
 825		/*
 826		 * xilly_memcpy always returns 0 => fifo_read can't fail =>
 827		 * count > 0
 828		 */
 829
 830		urb = usb_alloc_urb(0, GFP_KERNEL);
 831		if (!urb) {
 832			report_io_error(xdev, -ENOMEM);
 833			goto relist;
 834		}
 835
 836		usb_fill_bulk_urb(urb, xdev->udev,
 837				  usb_sndbulkpipe(xdev->udev, ep->ep_num),
 838				  xb->buf, count, bulk_out_completer, xb);
 839
 840		usb_anchor_urb(urb, &ep->anchor);
 841
 842		rc = usb_submit_urb(urb, GFP_KERNEL);
 843
 844		if (rc) {
 845			report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :
 846					-EIO);
 847			goto unanchor;
 848		}
 849
 850		usb_free_urb(urb); /* This just decrements reference count */
 851
 852		fill -= count;
 853		do_wake = true;
 854	}
 855
 856unanchor:
 857	usb_unanchor_urb(urb);
 858	usb_free_urb(urb);
 859
 860relist:
 861	spin_lock_irqsave(&ep->buffers_lock, flags);
 862	list_add_tail(&xb->entry, &ep->buffers);
 863	ep->outstanding_urbs--;
 864	spin_unlock_irqrestore(&ep->buffers_lock, flags);
 865
 866done:
 867	mutex_unlock(&ep->ep_mutex);
 868
 869	if (do_wake)
 870		wake_up_interruptible(&fifo->waitq);
 871}
 872
 873static void bulk_out_work(struct work_struct *work)
 874{
 875	struct xillyusb_endpoint *ep = container_of(work,
 876						    struct xillyusb_endpoint,
 877						    workitem);
 878	try_queue_bulk_out(ep);
 879}
 880
 881static int process_in_opcode(struct xillyusb_dev *xdev,
 882			     int opcode,
 883			     int chan_num)
 884{
 885	struct xillyusb_channel *chan;
 886	struct device *dev = xdev->dev;
 887	int chan_idx = chan_num >> 1;
 888
 889	if (chan_idx >= xdev->num_channels) {
 890		dev_err(dev, "Received illegal channel ID %d from FPGA\n",
 891			chan_num);
 892		return -EIO;
 893	}
 894
 895	chan = &xdev->channels[chan_idx];
 896
 897	switch (opcode) {
 898	case OPCODE_EOF:
 899		if (!chan->read_data_ok) {
 900			dev_err(dev, "Received unexpected EOF for channel %d\n",
 901				chan_num);
 902			return -EIO;
 903		}
 904
 905		/*
 906		 * A write memory barrier ensures that the FIFO's fill level
 907		 * is visible before read_data_ok turns zero, so the data in
 908		 * the FIFO isn't missed by the consumer.
 909		 */
 910		smp_wmb();
 911		WRITE_ONCE(chan->read_data_ok, 0);
 912		wake_up_interruptible(&chan->in_fifo->waitq);
 913		break;
 914
 915	case OPCODE_REACHED_CHECKPOINT:
 916		chan->flushing = 0;
 917		wake_up_interruptible(&chan->flushq);
 918		break;
 919
 920	case OPCODE_CANCELED_CHECKPOINT:
 921		chan->canceled = 1;
 922		wake_up_interruptible(&chan->flushq);
 923		break;
 924
 925	default:
 926		dev_err(dev, "Received illegal opcode %d from FPGA\n",
 927			opcode);
 928		return -EIO;
 929	}
 930
 931	return 0;
 932}
 933
 934static int process_bulk_in(struct xillybuffer *xb)
 935{
 936	struct xillyusb_endpoint *ep = xb->ep;
 937	struct xillyusb_dev *xdev = ep->xdev;
 938	struct device *dev = xdev->dev;
 939	int dws = xb->len >> 2;
 940	__le32 *p = xb->buf;
 941	u32 ctrlword;
 942	struct xillyusb_channel *chan;
 943	struct xillyfifo *fifo;
 944	int chan_num = 0, opcode;
 945	int chan_idx;
 946	int bytes, count, dwconsume;
 947	int in_bytes_left = 0;
 948	int rc;
 949
 950	if ((dws << 2) != xb->len) {
 951		dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n",
 952			xb->len);
 953		return -EIO;
 954	}
 955
 956	if (xdev->in_bytes_left) {
 957		bytes = min(xdev->in_bytes_left, dws << 2);
 958		in_bytes_left = xdev->in_bytes_left - bytes;
 959		chan_num = xdev->leftover_chan_num;
 960		goto resume_leftovers;
 961	}
 962
 963	while (dws) {
 964		ctrlword = le32_to_cpu(*p++);
 965		dws--;
 966
 967		chan_num = ctrlword & 0xfff;
 968		count = (ctrlword >> 12) & 0x3ff;
 969		opcode = (ctrlword >> 24) & 0xf;
 970
 971		if (opcode != OPCODE_DATA) {
 972			unsigned int in_counter = xdev->in_counter++ & 0x3ff;
 973
 974			if (count != in_counter) {
 975				dev_err(dev, "Expected opcode counter %d, got %d\n",
 976					in_counter, count);
 977				return -EIO;
 978			}
 979
 980			rc = process_in_opcode(xdev, opcode, chan_num);
 981
 982			if (rc)
 983				return rc;
 984
 985			continue;
 986		}
 987
 988		bytes = min(count + 1, dws << 2);
 989		in_bytes_left = count + 1 - bytes;
 990
 991resume_leftovers:
 992		chan_idx = chan_num >> 1;
 993
 994		if (!(chan_num & 1) || chan_idx >= xdev->num_channels ||
 995		    !xdev->channels[chan_idx].read_data_ok) {
 996			dev_err(dev, "Received illegal channel ID %d from FPGA\n",
 997				chan_num);
 998			return -EIO;
 999		}
1000		chan = &xdev->channels[chan_idx];
1001
1002		fifo = chan->in_fifo;
1003
1004		if (unlikely(!fifo))
1005			return -EIO; /* We got really unexpected data */
1006
1007		if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) {
1008			dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n");
1009			return -EIO;
1010		}
1011
1012		wake_up_interruptible(&fifo->waitq);
1013
1014		dwconsume = (bytes + 3) >> 2;
1015		dws -= dwconsume;
1016		p += dwconsume;
1017	}
1018
1019	xdev->in_bytes_left = in_bytes_left;
1020	xdev->leftover_chan_num = chan_num;
1021	return 0;
1022}
1023
1024static void bulk_in_work(struct work_struct *work)
1025{
1026	struct xillyusb_endpoint *ep =
1027		container_of(work, struct xillyusb_endpoint, workitem);
1028	struct xillyusb_dev *xdev = ep->xdev;
1029	unsigned long flags;
1030	struct xillybuffer *xb;
1031	bool consumed = false;
1032	int rc = 0;
1033
1034	mutex_lock(&xdev->process_in_mutex);
1035
1036	spin_lock_irqsave(&ep->buffers_lock, flags);
1037
1038	while (1) {
1039		if (rc || list_empty(&ep->filled_buffers)) {
1040			spin_unlock_irqrestore(&ep->buffers_lock, flags);
1041			mutex_unlock(&xdev->process_in_mutex);
1042
1043			if (rc)
1044				report_io_error(xdev, rc);
1045			else if (consumed)
1046				try_queue_bulk_in(ep);
1047
1048			return;
1049		}
1050
1051		xb = list_first_entry(&ep->filled_buffers, struct xillybuffer,
1052				      entry);
1053		list_del(&xb->entry);
1054
1055		spin_unlock_irqrestore(&ep->buffers_lock, flags);
1056
1057		consumed = true;
1058
1059		if (!xdev->error)
1060			rc = process_bulk_in(xb);
1061
1062		spin_lock_irqsave(&ep->buffers_lock, flags);
1063		list_add_tail(&xb->entry, &ep->buffers);
1064		ep->outstanding_urbs--;
1065	}
1066}
1067
1068static int xillyusb_send_opcode(struct xillyusb_dev *xdev,
1069				int chan_num, char opcode, u32 data)
1070{
1071	struct xillyusb_endpoint *ep = xdev->msg_ep;
1072	struct xillyfifo *fifo = &ep->fifo;
1073	__le32 msg[2];
1074
1075	int rc = 0;
1076
1077	msg[0] = cpu_to_le32((chan_num & 0xfff) |
1078			     ((opcode & 0xf) << 24));
1079	msg[1] = cpu_to_le32(data);
1080
1081	mutex_lock(&xdev->msg_mutex);
1082
1083	/*
1084	 * The wait queue is woken with the interruptible variant, so the
1085	 * wait function matches, however returning because of an interrupt
1086	 * will mess things up considerably, in particular when the caller is
1087	 * the release method. And the xdev->error part prevents being stuck
1088	 * forever in the event of a bizarre hardware bug: Pull the USB plug.
1089	 */
1090
1091	while (wait_event_interruptible(fifo->waitq,
1092					fifo->fill <= (fifo->size - 8) ||
1093					xdev->error))
1094		; /* Empty loop */
1095
1096	if (xdev->error) {
1097		rc = xdev->error;
1098		goto unlock_done;
1099	}
1100
1101	fifo_write(fifo, (void *)msg, 8, xilly_memcpy);
1102
1103	try_queue_bulk_out(ep);
1104
1105unlock_done:
1106	mutex_unlock(&xdev->msg_mutex);
1107
1108	return rc;
1109}
1110
1111/*
1112 * Note that flush_downstream() merely waits for the data to arrive to
1113 * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart,
1114 * it does nothing to make it happen (and neither is it necessary).
1115 *
1116 * This function is not reentrant for the same @chan, but this is covered
1117 * by the fact that for any given @chan, it's called either by the open,
1118 * write, llseek and flush fops methods, which can't run in parallel (and the
1119 * write + flush and llseek method handlers are protected with out_mutex).
1120 *
1121 * chan->flushed is there to avoid multiple flushes at the same position,
1122 * in particular as a result of programs that close the file descriptor
1123 * e.g. after a dup2() for redirection.
1124 */
1125
1126static int flush_downstream(struct xillyusb_channel *chan,
1127			    long timeout,
1128			    bool interruptible)
1129{
1130	struct xillyusb_dev *xdev = chan->xdev;
1131	int chan_num = chan->chan_idx << 1;
1132	long deadline, left_to_sleep;
1133	int rc;
1134
1135	if (chan->flushed)
1136		return 0;
1137
1138	deadline = jiffies + 1 + timeout;
1139
1140	if (chan->flushing) {
1141		long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT;
1142
1143		chan->canceled = 0;
1144		rc = xillyusb_send_opcode(xdev, chan_num,
1145					  OPCODE_CANCEL_CHECKPOINT, 0);
1146
1147		if (rc)
1148			return rc; /* Only real error, never -EINTR */
1149
1150		/* Ignoring interrupts. Cancellation must be handled */
1151		while (!chan->canceled) {
1152			left_to_sleep = cancel_deadline - ((long)jiffies);
1153
1154			if (left_to_sleep <= 0) {
1155				report_io_error(xdev, -EIO);
1156				return -EIO;
1157			}
1158
1159			rc = wait_event_interruptible_timeout(chan->flushq,
1160							      chan->canceled ||
1161							      xdev->error,
1162							      left_to_sleep);
1163
1164			if (xdev->error)
1165				return xdev->error;
1166		}
1167	}
1168
1169	chan->flushing = 1;
1170
1171	/*
1172	 * The checkpoint is given in terms of data elements, not bytes. As
1173	 * a result, if less than an element's worth of data is stored in the
1174	 * FIFO, it's not flushed, including the flush before closing, which
1175	 * means that such data is lost. This is consistent with PCIe Xillybus.
1176	 */
1177
1178	rc = xillyusb_send_opcode(xdev, chan_num,
1179				  OPCODE_SET_CHECKPOINT,
1180				  chan->out_bytes >>
1181				  chan->out_log2_element_size);
1182
1183	if (rc)
1184		return rc; /* Only real error, never -EINTR */
1185
1186	if (!timeout) {
1187		while (chan->flushing) {
1188			rc = wait_event_interruptible(chan->flushq,
1189						      !chan->flushing ||
1190						      xdev->error);
1191			if (xdev->error)
1192				return xdev->error;
1193
1194			if (interruptible && rc)
1195				return -EINTR;
1196		}
1197
1198		goto done;
1199	}
1200
1201	while (chan->flushing) {
1202		left_to_sleep = deadline - ((long)jiffies);
1203
1204		if (left_to_sleep <= 0)
1205			return -ETIMEDOUT;
1206
1207		rc = wait_event_interruptible_timeout(chan->flushq,
1208						      !chan->flushing ||
1209						      xdev->error,
1210						      left_to_sleep);
1211
1212		if (xdev->error)
1213			return xdev->error;
1214
1215		if (interruptible && rc < 0)
1216			return -EINTR;
1217	}
1218
1219done:
1220	chan->flushed = 1;
1221	return 0;
1222}
1223
1224/* request_read_anything(): Ask the FPGA for any little amount of data */
1225static int request_read_anything(struct xillyusb_channel *chan,
1226				 char opcode)
1227{
1228	struct xillyusb_dev *xdev = chan->xdev;
1229	unsigned int sh = chan->in_log2_element_size;
1230	int chan_num = (chan->chan_idx << 1) | 1;
1231	u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1;
1232
1233	return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh);
1234}
1235
1236static int xillyusb_open(struct inode *inode, struct file *filp)
1237{
1238	struct xillyusb_dev *xdev;
1239	struct xillyusb_channel *chan;
1240	struct xillyfifo *in_fifo = NULL;
1241	struct xillyusb_endpoint *out_ep = NULL;
1242	int rc;
1243	int index;
1244
1245	mutex_lock(&kref_mutex);
1246
1247	rc = xillybus_find_inode(inode, (void **)&xdev, &index);
1248	if (rc) {
1249		mutex_unlock(&kref_mutex);
1250		return rc;
1251	}
1252
1253	kref_get(&xdev->kref);
1254	mutex_unlock(&kref_mutex);
1255
1256	chan = &xdev->channels[index];
1257	filp->private_data = chan;
1258
1259	mutex_lock(&chan->lock);
1260
1261	rc = -ENODEV;
1262
1263	if (xdev->error)
1264		goto unmutex_fail;
1265
1266	if (((filp->f_mode & FMODE_READ) && !chan->readable) ||
1267	    ((filp->f_mode & FMODE_WRITE) && !chan->writable))
1268		goto unmutex_fail;
1269
1270	if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) &&
1271	    chan->in_synchronous) {
1272		dev_err(xdev->dev,
1273			"open() failed: O_NONBLOCK not allowed for read on this device\n");
1274		goto unmutex_fail;
1275	}
1276
1277	if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) &&
1278	    chan->out_synchronous) {
1279		dev_err(xdev->dev,
1280			"open() failed: O_NONBLOCK not allowed for write on this device\n");
1281		goto unmutex_fail;
1282	}
1283
1284	rc = -EBUSY;
1285
1286	if (((filp->f_mode & FMODE_READ) && chan->open_for_read) ||
1287	    ((filp->f_mode & FMODE_WRITE) && chan->open_for_write))
1288		goto unmutex_fail;
1289
1290	if (filp->f_mode & FMODE_READ)
1291		chan->open_for_read = 1;
1292
1293	if (filp->f_mode & FMODE_WRITE)
1294		chan->open_for_write = 1;
1295
1296	mutex_unlock(&chan->lock);
1297
1298	if (filp->f_mode & FMODE_WRITE) {
1299		out_ep = endpoint_alloc(xdev,
1300					(chan->chan_idx + 2) | USB_DIR_OUT,
1301					bulk_out_work, BUF_SIZE_ORDER, BUFNUM);
1302
1303		if (!out_ep) {
1304			rc = -ENOMEM;
1305			goto unopen;
1306		}
1307
1308		rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size);
1309
1310		if (rc)
1311			goto late_unopen;
1312
1313		out_ep->fill_mask = -(1 << chan->out_log2_element_size);
1314		chan->out_bytes = 0;
1315		chan->flushed = 0;
1316
1317		/*
1318		 * Sending a flush request to a previously closed stream
1319		 * effectively opens it, and also waits until the command is
1320		 * confirmed by the FPGA. The latter is necessary because the
1321		 * data is sent through a separate BULK OUT endpoint, and the
1322		 * xHCI controller is free to reorder transmissions.
1323		 *
1324		 * This can't go wrong unless there's a serious hardware error
1325		 * (or the computer is stuck for 500 ms?)
1326		 */
1327		rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false);
1328
1329		if (rc == -ETIMEDOUT) {
1330			rc = -EIO;
1331			report_io_error(xdev, rc);
1332		}
1333
1334		if (rc)
1335			goto late_unopen;
1336	}
1337
1338	if (filp->f_mode & FMODE_READ) {
1339		in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL);
1340
1341		if (!in_fifo) {
1342			rc = -ENOMEM;
1343			goto late_unopen;
1344		}
1345
1346		rc = fifo_init(in_fifo, chan->in_log2_fifo_size);
1347
1348		if (rc) {
1349			kfree(in_fifo);
1350			goto late_unopen;
1351		}
1352	}
1353
1354	mutex_lock(&chan->lock);
1355	if (in_fifo) {
1356		chan->in_fifo = in_fifo;
1357		chan->read_data_ok = 1;
1358	}
1359	if (out_ep)
1360		chan->out_ep = out_ep;
1361	mutex_unlock(&chan->lock);
1362
1363	if (in_fifo) {
1364		u32 in_checkpoint = 0;
1365
1366		if (!chan->in_synchronous)
1367			in_checkpoint = in_fifo->size >>
1368				chan->in_log2_element_size;
1369
1370		chan->in_consumed_bytes = 0;
1371		chan->poll_used = 0;
1372		chan->in_current_checkpoint = in_checkpoint;
1373		rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
1374					  OPCODE_SET_CHECKPOINT,
1375					  in_checkpoint);
1376
1377		if (rc) /* Failure guarantees that opcode wasn't sent */
1378			goto unfifo;
1379
1380		/*
1381		 * In non-blocking mode, request the FPGA to send any data it
1382		 * has right away. Otherwise, the first read() will always
1383		 * return -EAGAIN, which is OK strictly speaking, but ugly.
1384		 * Checking and unrolling if this fails isn't worth the
1385		 * effort -- the error is propagated to the first read()
1386		 * anyhow.
1387		 */
1388		if (filp->f_flags & O_NONBLOCK)
1389			request_read_anything(chan, OPCODE_SET_PUSH);
1390	}
1391
1392	return 0;
1393
1394unfifo:
1395	chan->read_data_ok = 0;
1396	safely_assign_in_fifo(chan, NULL);
1397	fifo_mem_release(in_fifo);
1398	kfree(in_fifo);
1399
1400	if (out_ep) {
1401		mutex_lock(&chan->lock);
1402		chan->out_ep = NULL;
1403		mutex_unlock(&chan->lock);
1404	}
1405
1406late_unopen:
1407	if (out_ep)
1408		endpoint_dealloc(out_ep);
1409
1410unopen:
1411	mutex_lock(&chan->lock);
1412
1413	if (filp->f_mode & FMODE_READ)
1414		chan->open_for_read = 0;
1415
1416	if (filp->f_mode & FMODE_WRITE)
1417		chan->open_for_write = 0;
1418
1419	mutex_unlock(&chan->lock);
1420
1421	kref_put(&xdev->kref, cleanup_dev);
1422
1423	return rc;
1424
1425unmutex_fail:
1426	kref_put(&xdev->kref, cleanup_dev);
1427	mutex_unlock(&chan->lock);
1428	return rc;
1429}
1430
1431static ssize_t xillyusb_read(struct file *filp, char __user *userbuf,
1432			     size_t count, loff_t *f_pos)
1433{
1434	struct xillyusb_channel *chan = filp->private_data;
1435	struct xillyusb_dev *xdev = chan->xdev;
1436	struct xillyfifo *fifo = chan->in_fifo;
1437	int chan_num = (chan->chan_idx << 1) | 1;
1438
1439	long deadline, left_to_sleep;
1440	int bytes_done = 0;
1441	bool sent_set_push = false;
1442	int rc;
1443
1444	deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
1445
1446	rc = mutex_lock_interruptible(&chan->in_mutex);
1447
1448	if (rc)
1449		return rc;
1450
1451	while (1) {
1452		u32 fifo_checkpoint_bytes, complete_checkpoint_bytes;
1453		u32 complete_checkpoint, fifo_checkpoint;
1454		u32 checkpoint;
1455		s32 diff, leap;
1456		unsigned int sh = chan->in_log2_element_size;
1457		bool checkpoint_for_complete;
1458
1459		rc = fifo_read(fifo, (__force void *)userbuf + bytes_done,
1460			       count - bytes_done, xilly_copy_to_user);
1461
1462		if (rc < 0)
1463			break;
1464
1465		bytes_done += rc;
1466		chan->in_consumed_bytes += rc;
1467
1468		left_to_sleep = deadline - ((long)jiffies);
1469
1470		/*
1471		 * Some 32-bit arithmetic that may wrap. Note that
1472		 * complete_checkpoint is rounded up to the closest element
1473		 * boundary, because the read() can't be completed otherwise.
1474		 * fifo_checkpoint_bytes is rounded down, because it protects
1475		 * in_fifo from overflowing.
1476		 */
1477
1478		fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size;
1479		complete_checkpoint_bytes =
1480			chan->in_consumed_bytes + count - bytes_done;
1481
1482		fifo_checkpoint = fifo_checkpoint_bytes >> sh;
1483		complete_checkpoint =
1484			(complete_checkpoint_bytes + (1 << sh) - 1) >> sh;
1485
1486		diff = (fifo_checkpoint - complete_checkpoint) << sh;
1487
1488		if (chan->in_synchronous && diff >= 0) {
1489			checkpoint = complete_checkpoint;
1490			checkpoint_for_complete = true;
1491		} else {
1492			checkpoint = fifo_checkpoint;
1493			checkpoint_for_complete = false;
1494		}
1495
1496		leap = (checkpoint - chan->in_current_checkpoint) << sh;
1497
1498		/*
1499		 * To prevent flooding of OPCODE_SET_CHECKPOINT commands as
1500		 * data is consumed, it's issued only if it moves the
1501		 * checkpoint by at least an 8th of the FIFO's size, or if
1502		 * it's necessary to complete the number of bytes requested by
1503		 * the read() call.
1504		 *
1505		 * chan->read_data_ok is checked to spare an unnecessary
1506		 * submission after receiving EOF, however it's harmless if
1507		 * such slips away.
1508		 */
1509
1510		if (chan->read_data_ok &&
1511		    (leap > (fifo->size >> 3) ||
1512		     (checkpoint_for_complete && leap > 0))) {
1513			chan->in_current_checkpoint = checkpoint;
1514			rc = xillyusb_send_opcode(xdev, chan_num,
1515						  OPCODE_SET_CHECKPOINT,
1516						  checkpoint);
1517
1518			if (rc)
1519				break;
1520		}
1521
1522		if (bytes_done == count ||
1523		    (left_to_sleep <= 0 && bytes_done))
1524			break;
1525
1526		/*
1527		 * Reaching here means that the FIFO was empty when
1528		 * fifo_read() returned, but not necessarily right now. Error
1529		 * and EOF are checked and reported only now, so that no data
1530		 * that managed its way to the FIFO is lost.
1531		 */
1532
1533		if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */
1534			/* Has data slipped into the FIFO since fifo_read()? */
1535			smp_rmb();
1536			if (READ_ONCE(fifo->fill))
1537				continue;
1538
1539			rc = 0;
1540			break;
1541		}
1542
1543		if (xdev->error) {
1544			rc = xdev->error;
1545			break;
1546		}
1547
1548		if (filp->f_flags & O_NONBLOCK) {
1549			rc = -EAGAIN;
1550			break;
1551		}
1552
1553		if (!sent_set_push) {
1554			rc = xillyusb_send_opcode(xdev, chan_num,
1555						  OPCODE_SET_PUSH,
1556						  complete_checkpoint);
1557
1558			if (rc)
1559				break;
1560
1561			sent_set_push = true;
1562		}
1563
1564		if (left_to_sleep > 0) {
1565			/*
1566			 * Note that when xdev->error is set (e.g. when the
1567			 * device is unplugged), read_data_ok turns zero and
1568			 * fifo->waitq is awaken.
1569			 * Therefore no special attention to xdev->error.
1570			 */
1571
1572			rc = wait_event_interruptible_timeout
1573				(fifo->waitq,
1574				 fifo->fill || !chan->read_data_ok,
1575				 left_to_sleep);
1576		} else { /* bytes_done == 0 */
1577			/* Tell FPGA to send anything it has */
1578			rc = request_read_anything(chan, OPCODE_UPDATE_PUSH);
1579
1580			if (rc)
1581				break;
1582
1583			rc = wait_event_interruptible
1584				(fifo->waitq,
1585				 fifo->fill || !chan->read_data_ok);
1586		}
1587
1588		if (rc < 0) {
1589			rc = -EINTR;
1590			break;
1591		}
1592	}
1593
1594	if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) &&
1595	    !READ_ONCE(fifo->fill))
1596		request_read_anything(chan, OPCODE_SET_PUSH);
1597
1598	mutex_unlock(&chan->in_mutex);
1599
1600	if (bytes_done)
1601		return bytes_done;
1602
1603	return rc;
1604}
1605
1606static int xillyusb_flush(struct file *filp, fl_owner_t id)
1607{
1608	struct xillyusb_channel *chan = filp->private_data;
1609	int rc;
1610
1611	if (!(filp->f_mode & FMODE_WRITE))
1612		return 0;
1613
1614	rc = mutex_lock_interruptible(&chan->out_mutex);
1615
1616	if (rc)
1617		return rc;
1618
1619	/*
1620	 * One second's timeout on flushing. Interrupts are ignored, because if
1621	 * the user pressed CTRL-C, that interrupt will still be in flight by
1622	 * the time we reach here, and the opportunity to flush is lost.
1623	 */
1624	rc = flush_downstream(chan, HZ, false);
1625
1626	mutex_unlock(&chan->out_mutex);
1627
1628	if (rc == -ETIMEDOUT) {
1629		/* The things you do to use dev_warn() and not pr_warn() */
1630		struct xillyusb_dev *xdev = chan->xdev;
1631
1632		mutex_lock(&chan->lock);
1633		if (!xdev->error)
1634			dev_warn(xdev->dev,
1635				 "Timed out while flushing. Output data may be lost.\n");
1636		mutex_unlock(&chan->lock);
1637	}
1638
1639	return rc;
1640}
1641
1642static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf,
1643			      size_t count, loff_t *f_pos)
1644{
1645	struct xillyusb_channel *chan = filp->private_data;
1646	struct xillyusb_dev *xdev = chan->xdev;
1647	struct xillyfifo *fifo = &chan->out_ep->fifo;
1648	int rc;
1649
1650	rc = mutex_lock_interruptible(&chan->out_mutex);
1651
1652	if (rc)
1653		return rc;
1654
1655	while (1) {
1656		if (xdev->error) {
1657			rc = xdev->error;
1658			break;
1659		}
1660
1661		if (count == 0)
1662			break;
1663
1664		rc = fifo_write(fifo, (__force void *)userbuf, count,
1665				xilly_copy_from_user);
1666
1667		if (rc != 0)
1668			break;
1669
1670		if (filp->f_flags & O_NONBLOCK) {
1671			rc = -EAGAIN;
1672			break;
1673		}
1674
1675		if (wait_event_interruptible
1676		    (fifo->waitq,
1677		     fifo->fill != fifo->size || xdev->error)) {
1678			rc = -EINTR;
1679			break;
1680		}
1681	}
1682
1683	if (rc < 0)
1684		goto done;
1685
1686	chan->out_bytes += rc;
1687
1688	if (rc) {
1689		try_queue_bulk_out(chan->out_ep);
1690		chan->flushed = 0;
1691	}
1692
1693	if (chan->out_synchronous) {
1694		int flush_rc = flush_downstream(chan, 0, true);
1695
1696		if (flush_rc && !rc)
1697			rc = flush_rc;
1698	}
1699
1700done:
1701	mutex_unlock(&chan->out_mutex);
1702
1703	return rc;
1704}
1705
1706static int xillyusb_release(struct inode *inode, struct file *filp)
1707{
1708	struct xillyusb_channel *chan = filp->private_data;
1709	struct xillyusb_dev *xdev = chan->xdev;
1710	int rc_read = 0, rc_write = 0;
1711
1712	if (filp->f_mode & FMODE_READ) {
1713		struct xillyfifo *in_fifo = chan->in_fifo;
1714
1715		rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,
1716					       OPCODE_CLOSE, 0);
1717		/*
1718		 * If rc_read is nonzero, xdev->error indicates a global
1719		 * device error. The error is reported later, so that
1720		 * resources are freed.
1721		 *
1722		 * Looping on wait_event_interruptible() kinda breaks the idea
1723		 * of being interruptible, and this should have been
1724		 * wait_event(). Only it's being waken with
1725		 * wake_up_interruptible() for the sake of other uses. If
1726		 * there's a global device error, chan->read_data_ok is
1727		 * deasserted and the wait queue is awaken, so this is covered.
1728		 */
1729
1730		while (wait_event_interruptible(in_fifo->waitq,
1731						!chan->read_data_ok))
1732			; /* Empty loop */
1733
1734		safely_assign_in_fifo(chan, NULL);
1735		fifo_mem_release(in_fifo);
1736		kfree(in_fifo);
1737
1738		mutex_lock(&chan->lock);
1739		chan->open_for_read = 0;
1740		mutex_unlock(&chan->lock);
1741	}
1742
1743	if (filp->f_mode & FMODE_WRITE) {
1744		struct xillyusb_endpoint *ep = chan->out_ep;
1745		/*
1746		 * chan->flushing isn't zeroed. If the pre-release flush timed
1747		 * out, a cancel request will be sent before the next
1748		 * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again).
1749		 * This is despite that the FPGA forgets about the checkpoint
1750		 * request as the file closes. Still, in an exceptional race
1751		 * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT
1752		 * just before closing that would reach the host after the
1753		 * file has re-opened.
1754		 */
1755
1756		mutex_lock(&chan->lock);
1757		chan->out_ep = NULL;
1758		mutex_unlock(&chan->lock);
1759
1760		endpoint_quiesce(ep);
1761		endpoint_dealloc(ep);
1762
1763		/* See comments on rc_read above */
1764		rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
1765						OPCODE_CLOSE, 0);
1766
1767		mutex_lock(&chan->lock);
1768		chan->open_for_write = 0;
1769		mutex_unlock(&chan->lock);
1770	}
1771
1772	kref_put(&xdev->kref, cleanup_dev);
1773
1774	return rc_read ? rc_read : rc_write;
1775}
1776
1777/*
1778 * Xillybus' API allows device nodes to be seekable, giving the user
1779 * application access to a RAM array on the FPGA (or logic emulating it).
1780 */
1781
1782static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence)
1783{
1784	struct xillyusb_channel *chan = filp->private_data;
1785	struct xillyusb_dev *xdev = chan->xdev;
1786	loff_t pos = filp->f_pos;
1787	int rc = 0;
1788	unsigned int log2_element_size = chan->readable ?
1789		chan->in_log2_element_size : chan->out_log2_element_size;
1790
1791	/*
1792	 * Take both mutexes not allowing interrupts, since it seems like
1793	 * common applications don't expect an -EINTR here. Besides, multiple
1794	 * access to a single file descriptor on seekable devices is a mess
1795	 * anyhow.
1796	 */
1797
1798	mutex_lock(&chan->out_mutex);
1799	mutex_lock(&chan->in_mutex);
1800
1801	switch (whence) {
1802	case SEEK_SET:
1803		pos = offset;
1804		break;
1805	case SEEK_CUR:
1806		pos += offset;
1807		break;
1808	case SEEK_END:
1809		pos = offset; /* Going to the end => to the beginning */
1810		break;
1811	default:
1812		rc = -EINVAL;
1813		goto end;
1814	}
1815
1816	/* In any case, we must finish on an element boundary */
1817	if (pos & ((1 << log2_element_size) - 1)) {
1818		rc = -EINVAL;
1819		goto end;
1820	}
1821
1822	rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1,
1823				  OPCODE_SET_ADDR,
1824				  pos >> log2_element_size);
1825
1826	if (rc)
1827		goto end;
1828
1829	if (chan->writable) {
1830		chan->flushed = 0;
1831		rc = flush_downstream(chan, HZ, false);
1832	}
1833
1834end:
1835	mutex_unlock(&chan->out_mutex);
1836	mutex_unlock(&chan->in_mutex);
1837
1838	if (rc) /* Return error after releasing mutexes */
1839		return rc;
1840
1841	filp->f_pos = pos;
1842
1843	return pos;
1844}
1845
1846static __poll_t xillyusb_poll(struct file *filp, poll_table *wait)
1847{
1848	struct xillyusb_channel *chan = filp->private_data;
1849	__poll_t mask = 0;
1850
1851	if (chan->in_fifo)
1852		poll_wait(filp, &chan->in_fifo->waitq, wait);
1853
1854	if (chan->out_ep)
1855		poll_wait(filp, &chan->out_ep->fifo.waitq, wait);
1856
1857	/*
1858	 * If this is the first time poll() is called, and the file is
1859	 * readable, set the relevant flag. Also tell the FPGA to send all it
1860	 * has, to kickstart the mechanism that ensures there's always some
1861	 * data in in_fifo unless the stream is dry end-to-end. Note that the
1862	 * first poll() may not return a EPOLLIN, even if there's data on the
1863	 * FPGA. Rather, the data will arrive soon, and trigger the relevant
1864	 * wait queue.
1865	 */
1866
1867	if (!chan->poll_used && chan->in_fifo) {
1868		chan->poll_used = 1;
1869		request_read_anything(chan, OPCODE_SET_PUSH);
1870	}
1871
1872	/*
1873	 * poll() won't play ball regarding read() channels which
1874	 * are synchronous. Allowing that will create situations where data has
1875	 * been delivered at the FPGA, and users expecting select() to wake up,
1876	 * which it may not. So make it never work.
1877	 */
1878
1879	if (chan->in_fifo && !chan->in_synchronous &&
1880	    (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok))
1881		mask |= EPOLLIN | EPOLLRDNORM;
1882
1883	if (chan->out_ep &&
1884	    (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size))
1885		mask |= EPOLLOUT | EPOLLWRNORM;
1886
1887	if (chan->xdev->error)
1888		mask |= EPOLLERR;
1889
1890	return mask;
1891}
1892
1893static const struct file_operations xillyusb_fops = {
1894	.owner      = THIS_MODULE,
1895	.read       = xillyusb_read,
1896	.write      = xillyusb_write,
1897	.open       = xillyusb_open,
1898	.flush      = xillyusb_flush,
1899	.release    = xillyusb_release,
1900	.llseek     = xillyusb_llseek,
1901	.poll       = xillyusb_poll,
1902};
1903
1904static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev)
1905{
1906	struct usb_device *udev = xdev->udev;
1907
1908	/* Verify that device has the two fundamental bulk in/out endpoints */
1909	if (usb_pipe_type_check(udev, usb_sndbulkpipe(udev, MSG_EP_NUM)) ||
1910	    usb_pipe_type_check(udev, usb_rcvbulkpipe(udev, IN_EP_NUM)))
1911		return -ENODEV;
1912
1913	xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT,
1914				      bulk_out_work, 1, 2);
1915	if (!xdev->msg_ep)
1916		return -ENOMEM;
1917
1918	if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */
1919		goto dealloc;
1920
1921	xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */
1922
1923	xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN,
1924				     bulk_in_work, BUF_SIZE_ORDER, BUFNUM);
1925	if (!xdev->in_ep)
1926		goto dealloc;
1927
1928	try_queue_bulk_in(xdev->in_ep);
1929
1930	return 0;
1931
1932dealloc:
1933	endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */
1934	xdev->msg_ep = NULL;
1935	return -ENOMEM;
1936}
1937
1938static int setup_channels(struct xillyusb_dev *xdev,
1939			  __le16 *chandesc,
1940			  int num_channels)
1941{
1942	struct usb_device *udev = xdev->udev;
1943	struct xillyusb_channel *chan, *new_channels;
1944	int i;
1945
1946	chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL);
1947	if (!chan)
1948		return -ENOMEM;
1949
1950	new_channels = chan;
1951
1952	for (i = 0; i < num_channels; i++, chan++) {
1953		unsigned int in_desc = le16_to_cpu(*chandesc++);
1954		unsigned int out_desc = le16_to_cpu(*chandesc++);
1955
1956		chan->xdev = xdev;
1957		mutex_init(&chan->in_mutex);
1958		mutex_init(&chan->out_mutex);
1959		mutex_init(&chan->lock);
1960		init_waitqueue_head(&chan->flushq);
1961
1962		chan->chan_idx = i;
1963
1964		if (in_desc & 0x80) { /* Entry is valid */
1965			chan->readable = 1;
1966			chan->in_synchronous = !!(in_desc & 0x40);
1967			chan->in_seekable = !!(in_desc & 0x20);
1968			chan->in_log2_element_size = in_desc & 0x0f;
1969			chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16;
1970		}
1971
1972		/*
1973		 * A downstream channel should never exist above index 13,
1974		 * as it would request a nonexistent BULK endpoint > 15.
1975		 * In the peculiar case that it does, it's ignored silently.
1976		 */
1977
1978		if ((out_desc & 0x80) && i < 14) { /* Entry is valid */
1979			if (usb_pipe_type_check(udev,
1980						usb_sndbulkpipe(udev, i + 2))) {
1981				dev_err(xdev->dev,
1982					"Missing BULK OUT endpoint %d\n",
1983					i + 2);
1984				kfree(new_channels);
1985				return -ENODEV;
1986			}
1987
1988			chan->writable = 1;
1989			chan->out_synchronous = !!(out_desc & 0x40);
1990			chan->out_seekable = !!(out_desc & 0x20);
1991			chan->out_log2_element_size = out_desc & 0x0f;
1992			chan->out_log2_fifo_size =
1993				((out_desc >> 8) & 0x1f) + 16;
1994		}
1995	}
1996
1997	xdev->channels = new_channels;
1998	return 0;
1999}
2000
2001static int xillyusb_discovery(struct usb_interface *interface)
2002{
2003	int rc;
2004	struct xillyusb_dev *xdev = usb_get_intfdata(interface);
2005	__le16 bogus_chandesc[2];
2006	struct xillyfifo idt_fifo;
2007	struct xillyusb_channel *chan;
2008	unsigned int idt_len, names_offset;
2009	unsigned char *idt;
2010	int num_channels;
2011
2012	rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
2013
2014	if (rc) {
2015		dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n");
2016		return rc;
2017	}
2018
2019	/* Phase I: Set up one fake upstream channel and obtain IDT */
2020
2021	/* Set up a fake IDT with one async IN stream */
2022	bogus_chandesc[0] = cpu_to_le16(0x80);
2023	bogus_chandesc[1] = cpu_to_le16(0);
2024
2025	rc = setup_channels(xdev, bogus_chandesc, 1);
2026
2027	if (rc)
2028		return rc;
2029
2030	rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE);
2031
2032	if (rc)
2033		return rc;
2034
2035	chan = xdev->channels;
2036
2037	chan->in_fifo = &idt_fifo;
2038	chan->read_data_ok = 1;
2039
2040	xdev->num_channels = 1;
2041
2042	rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0);
2043
2044	if (rc) {
2045		dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n");
2046		goto unfifo;
2047	}
2048
2049	rc = wait_event_interruptible_timeout(idt_fifo.waitq,
2050					      !chan->read_data_ok,
2051					      XILLY_RESPONSE_TIMEOUT);
2052
2053	if (xdev->error) {
2054		rc = xdev->error;
2055		goto unfifo;
2056	}
2057
2058	if (rc < 0) {
2059		rc = -EINTR; /* Interrupt on probe method? Interesting. */
2060		goto unfifo;
2061	}
2062
2063	if (chan->read_data_ok) {
2064		rc = -ETIMEDOUT;
2065		dev_err(&interface->dev, "No response from FPGA. Aborting.\n");
2066		goto unfifo;
2067	}
2068
2069	idt_len = READ_ONCE(idt_fifo.fill);
2070	idt = kmalloc(idt_len, GFP_KERNEL);
2071
2072	if (!idt) {
2073		rc = -ENOMEM;
2074		goto unfifo;
2075	}
2076
2077	fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy);
2078
2079	if (crc32_le(~0, idt, idt_len) != 0) {
2080		dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n");
2081		rc = -ENODEV;
2082		goto unidt;
2083	}
2084
2085	if (*idt > 0x90) {
2086		dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n",
2087			(int)*idt);
2088		rc = -ENODEV;
2089		goto unidt;
2090	}
2091
2092	/* Phase II: Set up the streams as defined in IDT */
2093
2094	num_channels = le16_to_cpu(*((__le16 *)(idt + 1)));
2095	names_offset = 3 + num_channels * 4;
2096	idt_len -= 4; /* Exclude CRC */
2097
2098	if (idt_len < names_offset) {
2099		dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n");
2100		rc = -ENODEV;
2101		goto unidt;
2102	}
2103
2104	rc = setup_channels(xdev, (void *)idt + 3, num_channels);
2105
2106	if (rc)
2107		goto unidt;
2108
2109	/*
2110	 * Except for wildly misbehaving hardware, or if it was disconnected
2111	 * just after responding with the IDT, there is no reason for any
2112	 * work item to be running now. To be sure that xdev->channels
2113	 * is updated on anything that might run in parallel, flush the
2114	 * device's workqueue and the wakeup work item. This rarely
2115	 * does anything.
2116	 */
2117	flush_workqueue(xdev->workq);
2118	flush_work(&xdev->wakeup_workitem);
2119
2120	xdev->num_channels = num_channels;
2121
2122	fifo_mem_release(&idt_fifo);
2123	kfree(chan);
2124
2125	rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops,
2126				  THIS_MODULE, xdev,
2127				  idt + names_offset,
2128				  idt_len - names_offset,
2129				  num_channels,
2130				  xillyname, true);
2131
2132	kfree(idt);
2133
2134	return rc;
2135
2136unidt:
2137	kfree(idt);
2138
2139unfifo:
2140	safely_assign_in_fifo(chan, NULL);
2141	fifo_mem_release(&idt_fifo);
2142
2143	return rc;
2144}
2145
2146static int xillyusb_probe(struct usb_interface *interface,
2147			  const struct usb_device_id *id)
2148{
2149	struct xillyusb_dev *xdev;
2150	int rc;
2151
2152	xdev = kzalloc(sizeof(*xdev), GFP_KERNEL);
2153	if (!xdev)
2154		return -ENOMEM;
2155
2156	kref_init(&xdev->kref);
2157	mutex_init(&xdev->process_in_mutex);
2158	mutex_init(&xdev->msg_mutex);
2159
2160	xdev->udev = usb_get_dev(interface_to_usbdev(interface));
2161	xdev->dev = &interface->dev;
2162	xdev->error = 0;
2163	spin_lock_init(&xdev->error_lock);
2164	xdev->in_counter = 0;
2165	xdev->in_bytes_left = 0;
2166	xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI | WQ_UNBOUND, 0);
2167
2168	if (!xdev->workq) {
2169		dev_err(&interface->dev, "Failed to allocate work queue\n");
2170		rc = -ENOMEM;
2171		goto fail;
2172	}
2173
2174	INIT_WORK(&xdev->wakeup_workitem, wakeup_all);
2175
2176	usb_set_intfdata(interface, xdev);
2177
2178	rc = xillyusb_setup_base_eps(xdev);
2179	if (rc)
2180		goto fail;
2181
2182	rc = xillyusb_discovery(interface);
2183	if (rc)
2184		goto latefail;
2185
2186	return 0;
2187
2188latefail:
2189	endpoint_quiesce(xdev->in_ep);
2190	endpoint_quiesce(xdev->msg_ep);
2191
2192fail:
2193	usb_set_intfdata(interface, NULL);
2194	kref_put(&xdev->kref, cleanup_dev);
2195	return rc;
2196}
2197
2198static void xillyusb_disconnect(struct usb_interface *interface)
2199{
2200	struct xillyusb_dev *xdev = usb_get_intfdata(interface);
2201	struct xillyusb_endpoint *msg_ep = xdev->msg_ep;
2202	struct xillyfifo *fifo = &msg_ep->fifo;
2203	int rc;
2204	int i;
2205
2206	xillybus_cleanup_chrdev(xdev, &interface->dev);
2207
2208	/*
2209	 * Try to send OPCODE_QUIESCE, which will fail silently if the device
2210	 * was disconnected, but makes sense on module unload.
2211	 */
2212
2213	msg_ep->wake_on_drain = true;
2214	xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);
2215
2216	/*
2217	 * If the device has been disconnected, sending the opcode causes
2218	 * a global device error with xdev->error, if such error didn't
2219	 * occur earlier. Hence timing out means that the USB link is fine,
2220	 * but somehow the message wasn't sent. Should never happen.
2221	 */
2222
2223	rc = wait_event_interruptible_timeout(fifo->waitq,
2224					      msg_ep->drained || xdev->error,
2225					      XILLY_RESPONSE_TIMEOUT);
2226
2227	if (!rc)
2228		dev_err(&interface->dev,
2229			"Weird timeout condition on sending quiesce request.\n");
2230
2231	report_io_error(xdev, -ENODEV); /* Discourage further activity */
2232
2233	/*
2234	 * This device driver is declared with soft_unbind set, or else
2235	 * sending OPCODE_QUIESCE above would always fail. The price is
2236	 * that the USB framework didn't kill outstanding URBs, so it has
2237	 * to be done explicitly before returning from this call.
2238	 */
2239
2240	for (i = 0; i < xdev->num_channels; i++) {
2241		struct xillyusb_channel *chan = &xdev->channels[i];
2242
2243		/*
2244		 * Lock taken to prevent chan->out_ep from changing. It also
2245		 * ensures xillyusb_open() and xillyusb_flush() don't access
2246		 * xdev->dev after being nullified below.
2247		 */
2248		mutex_lock(&chan->lock);
2249		if (chan->out_ep)
2250			endpoint_quiesce(chan->out_ep);
2251		mutex_unlock(&chan->lock);
2252	}
2253
2254	endpoint_quiesce(xdev->in_ep);
2255	endpoint_quiesce(xdev->msg_ep);
2256
2257	usb_set_intfdata(interface, NULL);
2258
2259	xdev->dev = NULL;
2260
2261	mutex_lock(&kref_mutex);
2262	kref_put(&xdev->kref, cleanup_dev);
2263	mutex_unlock(&kref_mutex);
2264}
2265
2266static struct usb_driver xillyusb_driver = {
2267	.name = xillyname,
2268	.id_table = xillyusb_table,
2269	.probe = xillyusb_probe,
2270	.disconnect = xillyusb_disconnect,
2271	.soft_unbind = 1,
2272};
2273
2274static int __init xillyusb_init(void)
2275{
2276	int rc = 0;
2277
2278	wakeup_wq = alloc_workqueue(xillyname, WQ_UNBOUND, 0);
2279	if (!wakeup_wq)
2280		return -ENOMEM;
2281
2282	if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT)
2283		fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT;
2284	else
2285		fifo_buf_order = 0;
2286
2287	rc = usb_register(&xillyusb_driver);
2288
2289	if (rc)
2290		destroy_workqueue(wakeup_wq);
2291
2292	return rc;
2293}
2294
2295static void __exit xillyusb_exit(void)
2296{
2297	usb_deregister(&xillyusb_driver);
2298
2299	destroy_workqueue(wakeup_wq);
2300}
2301
2302module_init(xillyusb_init);
2303module_exit(xillyusb_exit);