xillybus_core.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * linux/drivers/misc/xillybus_core.c
   4 *
   5 * Copyright 2011 Xillybus Ltd, http://xillybus.com
   6 *
   7 * Driver for the Xillybus FPGA/host framework.
   8 *
   9 * This driver interfaces with a special IP core in an FPGA, setting up
  10 * a pipe between a hardware FIFO in the programmable logic and a device
  11 * file in the host. The number of such pipes and their attributes are
  12 * set up on the logic. This driver detects these automatically and
  13 * creates the device files accordingly.
  14 */
  15
  16#include <linux/list.h>
  17#include <linux/device.h>
  18#include <linux/module.h>
  19#include <linux/io.h>
  20#include <linux/dma-mapping.h>
  21#include <linux/interrupt.h>
  22#include <linux/sched.h>
  23#include <linux/fs.h>
  24#include <linux/spinlock.h>
  25#include <linux/mutex.h>
  26#include <linux/crc32.h>
  27#include <linux/poll.h>
  28#include <linux/delay.h>
  29#include <linux/slab.h>
  30#include <linux/workqueue.h>
  31#include "xillybus.h"
  32#include "xillybus_class.h"
  33
  34MODULE_DESCRIPTION("Xillybus core functions");
  35MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  36MODULE_ALIAS("xillybus_core");
  37MODULE_LICENSE("GPL v2");
  38
  39/* General timeout is 100 ms, rx timeout is 10 ms */
  40#define XILLY_RX_TIMEOUT (10*HZ/1000)
  41#define XILLY_TIMEOUT (100*HZ/1000)
  42
  43#define fpga_msg_ctrl_reg              0x0008
  44#define fpga_dma_control_reg           0x0020
  45#define fpga_dma_bufno_reg             0x0024
  46#define fpga_dma_bufaddr_lowaddr_reg   0x0028
  47#define fpga_dma_bufaddr_highaddr_reg  0x002c
  48#define fpga_buf_ctrl_reg              0x0030
  49#define fpga_buf_offset_reg            0x0034
  50#define fpga_endian_reg                0x0040
  51
  52#define XILLYMSG_OPCODE_RELEASEBUF 1
  53#define XILLYMSG_OPCODE_QUIESCEACK 2
  54#define XILLYMSG_OPCODE_FIFOEOF 3
  55#define XILLYMSG_OPCODE_FATAL_ERROR 4
  56#define XILLYMSG_OPCODE_NONEMPTY 5
  57
  58static const char xillyname[] = "xillybus";
  59
  60static struct workqueue_struct *xillybus_wq;
  61
  62/*
  63 * Locking scheme: Mutexes protect invocations of character device methods.
  64 * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  65 *
  66 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  67 * buffers' end_offset fields against changes made by IRQ handler (and in
  68 * theory, other file request handlers, but the mutex handles that). Nothing
  69 * else.
  70 * They are held for short direct memory manipulations. Needless to say,
  71 * no mutex locking is allowed when a spinlock is held.
  72 *
  73 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  74 *
  75 * register_mutex is endpoint-specific, and is held when non-atomic
  76 * register operations are performed. wr_mutex and rd_mutex may be
  77 * held when register_mutex is taken, but none of the spinlocks. Note that
  78 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  79 * which are unrelated to buf_offset_reg, since they are harmless.
  80 *
  81 * Blocking on the wait queues is allowed with mutexes held, but not with
  82 * spinlocks.
  83 *
  84 * Only interruptible blocking is allowed on mutexes and wait queues.
  85 *
  86 * All in all, the locking order goes (with skips allowed, of course):
  87 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
  88 */
  89
  90static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
  91{
  92	int opcode;
  93	int msg_channel, msg_bufno, msg_data, msg_dir;
  94
  95	opcode = (buf[0] >> 24) & 0xff;
  96	msg_dir = buf[0] & 1;
  97	msg_channel = (buf[0] >> 1) & 0x7ff;
  98	msg_bufno = (buf[0] >> 12) & 0x3ff;
  99	msg_data = buf[1] & 0xfffffff;
 100
 101	dev_warn(endpoint->dev,
 102		 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
 103		 opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 104}
 105
 106/*
 107 * xillybus_isr assumes the interrupt is allocated exclusively to it,
 108 * which is the natural case MSI and several other hardware-oriented
 109 * interrupts. Sharing is not allowed.
 110 */
 111
 112irqreturn_t xillybus_isr(int irq, void *data)
 113{
 114	struct xilly_endpoint *ep = data;
 115	u32 *buf;
 116	unsigned int buf_size;
 117	int i;
 118	int opcode;
 119	unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
 120	struct xilly_channel *channel;
 121
 122	buf = ep->msgbuf_addr;
 123	buf_size = ep->msg_buf_size/sizeof(u32);
 124
 125	dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr,
 126				ep->msg_buf_size, DMA_FROM_DEVICE);
 127
 128	for (i = 0; i < buf_size; i += 2) {
 129		if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
 130			malformed_message(ep, &buf[i]);
 131			dev_warn(ep->dev,
 132				 "Sending a NACK on counter %x (instead of %x) on entry %d\n",
 133				 ((buf[i+1] >> 28) & 0xf),
 134				 ep->msg_counter,
 135				 i/2);
 136
 137			if (++ep->failed_messages > 10) {
 138				dev_err(ep->dev,
 139					"Lost sync with interrupt messages. Stopping.\n");
 140			} else {
 141				dma_sync_single_for_device(ep->dev,
 142							   ep->msgbuf_dma_addr,
 143							   ep->msg_buf_size,
 144							   DMA_FROM_DEVICE);
 145
 146				iowrite32(0x01,  /* Message NACK */
 147					  ep->registers + fpga_msg_ctrl_reg);
 148			}
 149			return IRQ_HANDLED;
 150		} else if (buf[i] & (1 << 22)) /* Last message */
 151			break;
 152	}
 153
 154	if (i >= buf_size) {
 155		dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
 156		return IRQ_HANDLED;
 157	}
 158
 159	buf_size = i + 2;
 160
 161	for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
 162		opcode = (buf[i] >> 24) & 0xff;
 163
 164		msg_dir = buf[i] & 1;
 165		msg_channel = (buf[i] >> 1) & 0x7ff;
 166		msg_bufno = (buf[i] >> 12) & 0x3ff;
 167		msg_data = buf[i+1] & 0xfffffff;
 168
 169		switch (opcode) {
 170		case XILLYMSG_OPCODE_RELEASEBUF:
 171			if ((msg_channel > ep->num_channels) ||
 172			    (msg_channel == 0)) {
 173				malformed_message(ep, &buf[i]);
 174				break;
 175			}
 176
 177			channel = ep->channels[msg_channel];
 178
 179			if (msg_dir) { /* Write channel */
 180				if (msg_bufno >= channel->num_wr_buffers) {
 181					malformed_message(ep, &buf[i]);
 182					break;
 183				}
 184				spin_lock(&channel->wr_spinlock);
 185				channel->wr_buffers[msg_bufno]->end_offset =
 186					msg_data;
 187				channel->wr_fpga_buf_idx = msg_bufno;
 188				channel->wr_empty = 0;
 189				channel->wr_sleepy = 0;
 190				spin_unlock(&channel->wr_spinlock);
 191
 192				wake_up_interruptible(&channel->wr_wait);
 193
 194			} else {
 195				/* Read channel */
 196
 197				if (msg_bufno >= channel->num_rd_buffers) {
 198					malformed_message(ep, &buf[i]);
 199					break;
 200				}
 201
 202				spin_lock(&channel->rd_spinlock);
 203				channel->rd_fpga_buf_idx = msg_bufno;
 204				channel->rd_full = 0;
 205				spin_unlock(&channel->rd_spinlock);
 206
 207				wake_up_interruptible(&channel->rd_wait);
 208				if (!channel->rd_synchronous)
 209					queue_delayed_work(
 210						xillybus_wq,
 211						&channel->rd_workitem,
 212						XILLY_RX_TIMEOUT);
 213			}
 214
 215			break;
 216		case XILLYMSG_OPCODE_NONEMPTY:
 217			if ((msg_channel > ep->num_channels) ||
 218			    (msg_channel == 0) || (!msg_dir) ||
 219			    !ep->channels[msg_channel]->wr_supports_nonempty) {
 220				malformed_message(ep, &buf[i]);
 221				break;
 222			}
 223
 224			channel = ep->channels[msg_channel];
 225
 226			if (msg_bufno >= channel->num_wr_buffers) {
 227				malformed_message(ep, &buf[i]);
 228				break;
 229			}
 230			spin_lock(&channel->wr_spinlock);
 231			if (msg_bufno == channel->wr_host_buf_idx)
 232				channel->wr_ready = 1;
 233			spin_unlock(&channel->wr_spinlock);
 234
 235			wake_up_interruptible(&channel->wr_ready_wait);
 236
 237			break;
 238		case XILLYMSG_OPCODE_QUIESCEACK:
 239			ep->idtlen = msg_data;
 240			wake_up_interruptible(&ep->ep_wait);
 241
 242			break;
 243		case XILLYMSG_OPCODE_FIFOEOF:
 244			if ((msg_channel > ep->num_channels) ||
 245			    (msg_channel == 0) || (!msg_dir) ||
 246			    !ep->channels[msg_channel]->num_wr_buffers) {
 247				malformed_message(ep, &buf[i]);
 248				break;
 249			}
 250			channel = ep->channels[msg_channel];
 251			spin_lock(&channel->wr_spinlock);
 252			channel->wr_eof = msg_bufno;
 253			channel->wr_sleepy = 0;
 254
 255			channel->wr_hangup = channel->wr_empty &&
 256				(channel->wr_host_buf_idx == msg_bufno);
 257
 258			spin_unlock(&channel->wr_spinlock);
 259
 260			wake_up_interruptible(&channel->wr_wait);
 261
 262			break;
 263		case XILLYMSG_OPCODE_FATAL_ERROR:
 264			ep->fatal_error = 1;
 265			wake_up_interruptible(&ep->ep_wait); /* For select() */
 266			dev_err(ep->dev,
 267				"FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
 268			break;
 269		default:
 270			malformed_message(ep, &buf[i]);
 271			break;
 272		}
 273	}
 274
 275	dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr,
 276				   ep->msg_buf_size, DMA_FROM_DEVICE);
 277
 278	ep->msg_counter = (ep->msg_counter + 1) & 0xf;
 279	ep->failed_messages = 0;
 280	iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
 281
 282	return IRQ_HANDLED;
 283}
 284EXPORT_SYMBOL(xillybus_isr);
 285
 286/*
 287 * A few trivial memory management functions.
 288 * NOTE: These functions are used only on probe and remove, and therefore
 289 * no locks are applied!
 290 */
 291
 292static void xillybus_autoflush(struct work_struct *work);
 293
 294struct xilly_alloc_state {
 295	void *salami;
 296	int left_of_salami;
 297	int nbuffer;
 298	enum dma_data_direction direction;
 299	u32 regdirection;
 300};
 301
 302static void xilly_unmap(void *ptr)
 303{
 304	struct xilly_mapping *data = ptr;
 305
 306	dma_unmap_single(data->device, data->dma_addr,
 307			 data->size, data->direction);
 308
 309	kfree(ptr);
 310}
 311
 312static int xilly_map_single(struct xilly_endpoint *ep,
 313			    void *ptr,
 314			    size_t size,
 315			    int direction,
 316			    dma_addr_t *ret_dma_handle
 317	)
 318{
 319	dma_addr_t addr;
 320	struct xilly_mapping *this;
 321
 322	this = kzalloc(sizeof(*this), GFP_KERNEL);
 323	if (!this)
 324		return -ENOMEM;
 325
 326	addr = dma_map_single(ep->dev, ptr, size, direction);
 327
 328	if (dma_mapping_error(ep->dev, addr)) {
 329		kfree(this);
 330		return -ENODEV;
 331	}
 332
 333	this->device = ep->dev;
 334	this->dma_addr = addr;
 335	this->size = size;
 336	this->direction = direction;
 337
 338	*ret_dma_handle = addr;
 339
 340	return devm_add_action_or_reset(ep->dev, xilly_unmap, this);
 341}
 342
 343static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
 344				 struct xilly_alloc_state *s,
 345				 struct xilly_buffer **buffers,
 346				 int bufnum, int bytebufsize)
 347{
 348	int i, rc;
 349	dma_addr_t dma_addr;
 350	struct device *dev = ep->dev;
 351	struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 352
 353	if (buffers) { /* Not the message buffer */
 354		this_buffer = devm_kcalloc(dev, bufnum,
 355					   sizeof(struct xilly_buffer),
 356					   GFP_KERNEL);
 357		if (!this_buffer)
 358			return -ENOMEM;
 359	}
 360
 361	for (i = 0; i < bufnum; i++) {
 362		/*
 363		 * Buffers are expected in descending size order, so there
 364		 * is either enough space for this buffer or none at all.
 365		 */
 366
 367		if ((s->left_of_salami < bytebufsize) &&
 368		    (s->left_of_salami > 0)) {
 369			dev_err(ep->dev,
 370				"Corrupt buffer allocation in IDT. Aborting.\n");
 371			return -ENODEV;
 372		}
 373
 374		if (s->left_of_salami == 0) {
 375			int allocorder, allocsize;
 376
 377			allocsize = PAGE_SIZE;
 378			allocorder = 0;
 379			while (bytebufsize > allocsize) {
 380				allocsize *= 2;
 381				allocorder++;
 382			}
 383
 384			s->salami = (void *) devm_get_free_pages(
 385				dev,
 386				GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
 387				allocorder);
 388			if (!s->salami)
 389				return -ENOMEM;
 390
 391			s->left_of_salami = allocsize;
 392		}
 393
 394		rc = xilly_map_single(ep, s->salami,
 395				      bytebufsize, s->direction,
 396				      &dma_addr);
 397		if (rc)
 398			return rc;
 399
 400		iowrite32((u32) (dma_addr & 0xffffffff),
 401			  ep->registers + fpga_dma_bufaddr_lowaddr_reg);
 402		iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
 403			  ep->registers + fpga_dma_bufaddr_highaddr_reg);
 404
 405		if (buffers) { /* Not the message buffer */
 406			this_buffer->addr = s->salami;
 407			this_buffer->dma_addr = dma_addr;
 408			buffers[i] = this_buffer++;
 409
 410			iowrite32(s->regdirection | s->nbuffer++,
 411				  ep->registers + fpga_dma_bufno_reg);
 412		} else {
 413			ep->msgbuf_addr = s->salami;
 414			ep->msgbuf_dma_addr = dma_addr;
 415			ep->msg_buf_size = bytebufsize;
 416
 417			iowrite32(s->regdirection,
 418				  ep->registers + fpga_dma_bufno_reg);
 419		}
 420
 421		s->left_of_salami -= bytebufsize;
 422		s->salami += bytebufsize;
 423	}
 424	return 0;
 425}
 426
 427static int xilly_setupchannels(struct xilly_endpoint *ep,
 428			       unsigned char *chandesc,
 429			       int entries)
 430{
 431	struct device *dev = ep->dev;
 432	int i, entry, rc;
 433	struct xilly_channel *channel;
 434	int channelnum, bufnum, bufsize, format, is_writebuf;
 435	int bytebufsize;
 436	int synchronous, allowpartial, exclusive_open, seekable;
 437	int supports_nonempty;
 438	int msg_buf_done = 0;
 439
 440	struct xilly_alloc_state rd_alloc = {
 441		.salami = NULL,
 442		.left_of_salami = 0,
 443		.nbuffer = 1,
 444		.direction = DMA_TO_DEVICE,
 445		.regdirection = 0,
 446	};
 447
 448	struct xilly_alloc_state wr_alloc = {
 449		.salami = NULL,
 450		.left_of_salami = 0,
 451		.nbuffer = 1,
 452		.direction = DMA_FROM_DEVICE,
 453		.regdirection = 0x80000000,
 454	};
 455
 456	channel = devm_kcalloc(dev, ep->num_channels,
 457			       sizeof(struct xilly_channel), GFP_KERNEL);
 458	if (!channel)
 459		return -ENOMEM;
 460
 461	ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
 462				    sizeof(struct xilly_channel *),
 463				    GFP_KERNEL);
 464	if (!ep->channels)
 465		return -ENOMEM;
 466
 467	ep->channels[0] = NULL; /* Channel 0 is message buf. */
 468
 469	/* Initialize all channels with defaults */
 470
 471	for (i = 1; i <= ep->num_channels; i++) {
 472		channel->wr_buffers = NULL;
 473		channel->rd_buffers = NULL;
 474		channel->num_wr_buffers = 0;
 475		channel->num_rd_buffers = 0;
 476		channel->wr_fpga_buf_idx = -1;
 477		channel->wr_host_buf_idx = 0;
 478		channel->wr_host_buf_pos = 0;
 479		channel->wr_empty = 1;
 480		channel->wr_ready = 0;
 481		channel->wr_sleepy = 1;
 482		channel->rd_fpga_buf_idx = 0;
 483		channel->rd_host_buf_idx = 0;
 484		channel->rd_host_buf_pos = 0;
 485		channel->rd_full = 0;
 486		channel->wr_ref_count = 0;
 487		channel->rd_ref_count = 0;
 488
 489		spin_lock_init(&channel->wr_spinlock);
 490		spin_lock_init(&channel->rd_spinlock);
 491		mutex_init(&channel->wr_mutex);
 492		mutex_init(&channel->rd_mutex);
 493		init_waitqueue_head(&channel->rd_wait);
 494		init_waitqueue_head(&channel->wr_wait);
 495		init_waitqueue_head(&channel->wr_ready_wait);
 496
 497		INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 498
 499		channel->endpoint = ep;
 500		channel->chan_num = i;
 501
 502		channel->log2_element_size = 0;
 503
 504		ep->channels[i] = channel++;
 505	}
 506
 507	for (entry = 0; entry < entries; entry++, chandesc += 4) {
 508		struct xilly_buffer **buffers = NULL;
 509
 510		is_writebuf = chandesc[0] & 0x01;
 511		channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
 512		format = (chandesc[1] >> 4) & 0x03;
 513		allowpartial = (chandesc[1] >> 6) & 0x01;
 514		synchronous = (chandesc[1] >> 7) & 0x01;
 515		bufsize = 1 << (chandesc[2] & 0x1f);
 516		bufnum = 1 << (chandesc[3] & 0x0f);
 517		exclusive_open = (chandesc[2] >> 7) & 0x01;
 518		seekable = (chandesc[2] >> 6) & 0x01;
 519		supports_nonempty = (chandesc[2] >> 5) & 0x01;
 520
 521		if ((channelnum > ep->num_channels) ||
 522		    ((channelnum == 0) && !is_writebuf)) {
 523			dev_err(ep->dev,
 524				"IDT requests channel out of range. Aborting.\n");
 525			return -ENODEV;
 526		}
 527
 528		channel = ep->channels[channelnum]; /* NULL for msg channel */
 529
 530		if (!is_writebuf || channelnum > 0) {
 531			channel->log2_element_size = ((format > 2) ?
 532						      2 : format);
 533
 534			bytebufsize = bufsize *
 535				(1 << channel->log2_element_size);
 536
 537			buffers = devm_kcalloc(dev, bufnum,
 538					       sizeof(struct xilly_buffer *),
 539					       GFP_KERNEL);
 540			if (!buffers)
 541				return -ENOMEM;
 542		} else {
 543			bytebufsize = bufsize << 2;
 544		}
 545
 546		if (!is_writebuf) {
 547			channel->num_rd_buffers = bufnum;
 548			channel->rd_buf_size = bytebufsize;
 549			channel->rd_allow_partial = allowpartial;
 550			channel->rd_synchronous = synchronous;
 551			channel->rd_exclusive_open = exclusive_open;
 552			channel->seekable = seekable;
 553
 554			channel->rd_buffers = buffers;
 555			rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
 556						   bufnum, bytebufsize);
 557		} else if (channelnum > 0) {
 558			channel->num_wr_buffers = bufnum;
 559			channel->wr_buf_size = bytebufsize;
 560
 561			channel->seekable = seekable;
 562			channel->wr_supports_nonempty = supports_nonempty;
 563
 564			channel->wr_allow_partial = allowpartial;
 565			channel->wr_synchronous = synchronous;
 566			channel->wr_exclusive_open = exclusive_open;
 567
 568			channel->wr_buffers = buffers;
 569			rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
 570						   bufnum, bytebufsize);
 571		} else {
 572			rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
 573						   bufnum, bytebufsize);
 574			msg_buf_done++;
 575		}
 576
 577		if (rc)
 578			return -ENOMEM;
 579	}
 580
 581	if (!msg_buf_done) {
 582		dev_err(ep->dev,
 583			"Corrupt IDT: No message buffer. Aborting.\n");
 584		return -ENODEV;
 585	}
 586	return 0;
 587}
 588
 589static int xilly_scan_idt(struct xilly_endpoint *endpoint,
 590			  struct xilly_idt_handle *idt_handle)
 591{
 592	int count = 0;
 593	unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
 594	unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
 595	unsigned char *scan;
 596	int len;
 597
 598	scan = idt + 1;
 599	idt_handle->names = scan;
 600
 601	while ((scan <= end_of_idt) && *scan) {
 602		while ((scan <= end_of_idt) && *scan++)
 603			/* Do nothing, just scan thru string */;
 604		count++;
 605	}
 606
 607	idt_handle->names_len = scan - idt_handle->names;
 608
 609	scan++;
 610
 611	if (scan > end_of_idt) {
 612		dev_err(endpoint->dev,
 613			"IDT device name list overflow. Aborting.\n");
 614		return -ENODEV;
 615	}
 616	idt_handle->chandesc = scan;
 617
 618	len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 619
 620	if (len & 0x03) {
 621		dev_err(endpoint->dev,
 622			"Corrupt IDT device name list. Aborting.\n");
 623		return -ENODEV;
 624	}
 625
 626	idt_handle->entries = len >> 2;
 627	endpoint->num_channels = count;
 628
 629	return 0;
 630}
 631
 632static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 633{
 634	struct xilly_channel *channel;
 635	unsigned char *version;
 636	long t;
 637
 638	channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 639
 640	channel->wr_sleepy = 1;
 641
 642	iowrite32(1 |
 643		  (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
 644		  endpoint->registers + fpga_buf_ctrl_reg);
 645
 646	t = wait_event_interruptible_timeout(channel->wr_wait,
 647					     (!channel->wr_sleepy),
 648					     XILLY_TIMEOUT);
 649
 650	if (t <= 0) {
 651		dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 652
 653		if (endpoint->fatal_error)
 654			return -EIO;
 655
 656		return -ENODEV;
 657	}
 658
 659	dma_sync_single_for_cpu(channel->endpoint->dev,
 660				channel->wr_buffers[0]->dma_addr,
 661				channel->wr_buf_size,
 662				DMA_FROM_DEVICE);
 663
 664	if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
 665		dev_err(endpoint->dev,
 666			"IDT length mismatch (%d != %d). Aborting.\n",
 667			channel->wr_buffers[0]->end_offset, endpoint->idtlen);
 668		return -ENODEV;
 669	}
 670
 671	if (crc32_le(~0, channel->wr_buffers[0]->addr,
 672		     endpoint->idtlen+1) != 0) {
 673		dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
 674		return -ENODEV;
 675	}
 676
 677	version = channel->wr_buffers[0]->addr;
 678
 679	/* Check version number. Reject anything above 0x82. */
 680	if (*version > 0x82) {
 681		dev_err(endpoint->dev,
 682			"No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
 683			*version);
 684		return -ENODEV;
 685	}
 686
 687	return 0;
 688}
 689
 690static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
 691			     size_t count, loff_t *f_pos)
 692{
 693	ssize_t rc;
 694	unsigned long flags;
 695	int bytes_done = 0;
 696	int no_time_left = 0;
 697	long deadline, left_to_sleep;
 698	struct xilly_channel *channel = filp->private_data;
 699
 700	int empty, reached_eof, exhausted, ready;
 701	/* Initializations are there only to silence warnings */
 702
 703	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
 704	int waiting_bufidx;
 705
 706	if (channel->endpoint->fatal_error)
 707		return -EIO;
 708
 709	deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 710
 711	rc = mutex_lock_interruptible(&channel->wr_mutex);
 712	if (rc)
 713		return rc;
 714
 715	while (1) { /* Note that we may drop mutex within this loop */
 716		int bytes_to_do = count - bytes_done;
 717
 718		spin_lock_irqsave(&channel->wr_spinlock, flags);
 719
 720		empty = channel->wr_empty;
 721		ready = !empty || channel->wr_ready;
 722
 723		if (!empty) {
 724			bufidx = channel->wr_host_buf_idx;
 725			bufpos = channel->wr_host_buf_pos;
 726			howmany = ((channel->wr_buffers[bufidx]->end_offset
 727				    + 1) << channel->log2_element_size)
 728				- bufpos;
 729
 730			/* Update wr_host_* to its post-operation state */
 731			if (howmany > bytes_to_do) {
 732				bufferdone = 0;
 733
 734				howmany = bytes_to_do;
 735				channel->wr_host_buf_pos += howmany;
 736			} else {
 737				bufferdone = 1;
 738
 739				channel->wr_host_buf_pos = 0;
 740
 741				if (bufidx == channel->wr_fpga_buf_idx) {
 742					channel->wr_empty = 1;
 743					channel->wr_sleepy = 1;
 744					channel->wr_ready = 0;
 745				}
 746
 747				if (bufidx >= (channel->num_wr_buffers - 1))
 748					channel->wr_host_buf_idx = 0;
 749				else
 750					channel->wr_host_buf_idx++;
 751			}
 752		}
 753
 754		/*
 755		 * Marking our situation after the possible changes above,
 756		 * for use after releasing the spinlock.
 757		 *
 758		 * empty = empty before change
 759		 * exhasted = empty after possible change
 760		 */
 761
 762		reached_eof = channel->wr_empty &&
 763			(channel->wr_host_buf_idx == channel->wr_eof);
 764		channel->wr_hangup = reached_eof;
 765		exhausted = channel->wr_empty;
 766		waiting_bufidx = channel->wr_host_buf_idx;
 767
 768		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 769
 770		if (!empty) { /* Go on, now without the spinlock */
 771
 772			if (bufpos == 0) /* Position zero means it's virgin */
 773				dma_sync_single_for_cpu(channel->endpoint->dev,
 774							channel->wr_buffers[bufidx]->dma_addr,
 775							channel->wr_buf_size,
 776							DMA_FROM_DEVICE);
 777
 778			if (copy_to_user(
 779				    userbuf,
 780				    channel->wr_buffers[bufidx]->addr
 781				    + bufpos, howmany))
 782				rc = -EFAULT;
 783
 784			userbuf += howmany;
 785			bytes_done += howmany;
 786
 787			if (bufferdone) {
 788				dma_sync_single_for_device(channel->endpoint->dev,
 789							   channel->wr_buffers[bufidx]->dma_addr,
 790							   channel->wr_buf_size,
 791							   DMA_FROM_DEVICE);
 792
 793				/*
 794				 * Tell FPGA the buffer is done with. It's an
 795				 * atomic operation to the FPGA, so what
 796				 * happens with other channels doesn't matter,
 797				 * and the certain channel is protected with
 798				 * the channel-specific mutex.
 799				 */
 800
 801				iowrite32(1 | (channel->chan_num << 1) |
 802					  (bufidx << 12),
 803					  channel->endpoint->registers +
 804					  fpga_buf_ctrl_reg);
 805			}
 806
 807			if (rc) {
 808				mutex_unlock(&channel->wr_mutex);
 809				return rc;
 810			}
 811		}
 812
 813		/* This includes a zero-count return = EOF */
 814		if ((bytes_done >= count) || reached_eof)
 815			break;
 816
 817		if (!exhausted)
 818			continue; /* More in RAM buffer(s)? Just go on. */
 819
 820		if ((bytes_done > 0) &&
 821		    (no_time_left ||
 822		     (channel->wr_synchronous && channel->wr_allow_partial)))
 823			break;
 824
 825		/*
 826		 * Nonblocking read: The "ready" flag tells us that the FPGA
 827		 * has data to send. In non-blocking mode, if it isn't on,
 828		 * just return. But if there is, we jump directly to the point
 829		 * where we ask for the FPGA to send all it has, and wait
 830		 * until that data arrives. So in a sense, we *do* block in
 831		 * nonblocking mode, but only for a very short time.
 832		 */
 833
 834		if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
 835			if (bytes_done > 0)
 836				break;
 837
 838			if (ready)
 839				goto desperate;
 840
 841			rc = -EAGAIN;
 842			break;
 843		}
 844
 845		if (!no_time_left || (bytes_done > 0)) {
 846			/*
 847			 * Note that in case of an element-misaligned read
 848			 * request, offsetlimit will include the last element,
 849			 * which will be partially read from.
 850			 */
 851			int offsetlimit = ((count - bytes_done) - 1) >>
 852				channel->log2_element_size;
 853			int buf_elements = channel->wr_buf_size >>
 854				channel->log2_element_size;
 855
 856			/*
 857			 * In synchronous mode, always send an offset limit.
 858			 * Just don't send a value too big.
 859			 */
 860
 861			if (channel->wr_synchronous) {
 862				/* Don't request more than one buffer */
 863				if (channel->wr_allow_partial &&
 864				    (offsetlimit >= buf_elements))
 865					offsetlimit = buf_elements - 1;
 866
 867				/* Don't request more than all buffers */
 868				if (!channel->wr_allow_partial &&
 869				    (offsetlimit >=
 870				     (buf_elements * channel->num_wr_buffers)))
 871					offsetlimit = buf_elements *
 872						channel->num_wr_buffers - 1;
 873			}
 874
 875			/*
 876			 * In asynchronous mode, force early flush of a buffer
 877			 * only if that will allow returning a full count. The
 878			 * "offsetlimit < ( ... )" rather than "<=" excludes
 879			 * requesting a full buffer, which would obviously
 880			 * cause a buffer transmission anyhow
 881			 */
 882
 883			if (channel->wr_synchronous ||
 884			    (offsetlimit < (buf_elements - 1))) {
 885				mutex_lock(&channel->endpoint->register_mutex);
 886
 887				iowrite32(offsetlimit,
 888					  channel->endpoint->registers +
 889					  fpga_buf_offset_reg);
 890
 891				iowrite32(1 | (channel->chan_num << 1) |
 892					  (2 << 24) |  /* 2 = offset limit */
 893					  (waiting_bufidx << 12),
 894					  channel->endpoint->registers +
 895					  fpga_buf_ctrl_reg);
 896
 897				mutex_unlock(&channel->endpoint->
 898					     register_mutex);
 899			}
 900		}
 901
 902		/*
 903		 * If partial completion is disallowed, there is no point in
 904		 * timeout sleeping. Neither if no_time_left is set and
 905		 * there's no data.
 906		 */
 907
 908		if (!channel->wr_allow_partial ||
 909		    (no_time_left && (bytes_done == 0))) {
 910			/*
 911			 * This do-loop will run more than once if another
 912			 * thread reasserted wr_sleepy before we got the mutex
 913			 * back, so we try again.
 914			 */
 915
 916			do {
 917				mutex_unlock(&channel->wr_mutex);
 918
 919				if (wait_event_interruptible(
 920					    channel->wr_wait,
 921					    (!channel->wr_sleepy)))
 922					goto interrupted;
 923
 924				if (mutex_lock_interruptible(
 925					    &channel->wr_mutex))
 926					goto interrupted;
 927			} while (channel->wr_sleepy);
 928
 929			continue;
 930
 931interrupted: /* Mutex is not held if got here */
 932			if (channel->endpoint->fatal_error)
 933				return -EIO;
 934			if (bytes_done)
 935				return bytes_done;
 936			if (filp->f_flags & O_NONBLOCK)
 937				return -EAGAIN; /* Don't admit snoozing */
 938			return -EINTR;
 939		}
 940
 941		left_to_sleep = deadline - ((long) jiffies);
 942
 943		/*
 944		 * If our time is out, skip the waiting. We may miss wr_sleepy
 945		 * being deasserted but hey, almost missing the train is like
 946		 * missing it.
 947		 */
 948
 949		if (left_to_sleep > 0) {
 950			left_to_sleep =
 951				wait_event_interruptible_timeout(
 952					channel->wr_wait,
 953					(!channel->wr_sleepy),
 954					left_to_sleep);
 955
 956			if (left_to_sleep > 0) /* wr_sleepy deasserted */
 957				continue;
 958
 959			if (left_to_sleep < 0) { /* Interrupt */
 960				mutex_unlock(&channel->wr_mutex);
 961				if (channel->endpoint->fatal_error)
 962					return -EIO;
 963				if (bytes_done)
 964					return bytes_done;
 965				return -EINTR;
 966			}
 967		}
 968
 969desperate:
 970		no_time_left = 1; /* We're out of sleeping time. Desperate! */
 971
 972		if (bytes_done == 0) {
 973			/*
 974			 * Reaching here means that we allow partial return,
 975			 * that we've run out of time, and that we have
 976			 * nothing to return.
 977			 * So tell the FPGA to send anything it has or gets.
 978			 */
 979
 980			iowrite32(1 | (channel->chan_num << 1) |
 981				  (3 << 24) |  /* Opcode 3, flush it all! */
 982				  (waiting_bufidx << 12),
 983				  channel->endpoint->registers +
 984				  fpga_buf_ctrl_reg);
 985		}
 986
 987		/*
 988		 * Reaching here means that we *do* have data in the buffer,
 989		 * but the "partial" flag disallows returning less than
 990		 * required. And we don't have as much. So loop again,
 991		 * which is likely to end up blocking indefinitely until
 992		 * enough data has arrived.
 993		 */
 994	}
 995
 996	mutex_unlock(&channel->wr_mutex);
 997
 998	if (channel->endpoint->fatal_error)
 999		return -EIO;
1000
1001	if (rc)
1002		return rc;
1003
1004	return bytes_done;
1005}
1006
1007/*
1008 * The timeout argument takes values as follows:
1009 *  >0 : Flush with timeout
1010 * ==0 : Flush, and wait idefinitely for the flush to complete
1011 *  <0 : Autoflush: Flush only if there's a single buffer occupied
1012 */
1013
1014static int xillybus_myflush(struct xilly_channel *channel, long timeout)
1015{
1016	int rc;
1017	unsigned long flags;
1018
1019	int end_offset_plus1;
1020	int bufidx, bufidx_minus1;
1021	int i;
1022	int empty;
1023	int new_rd_host_buf_pos;
1024
1025	if (channel->endpoint->fatal_error)
1026		return -EIO;
1027	rc = mutex_lock_interruptible(&channel->rd_mutex);
1028	if (rc)
1029		return rc;
1030
1031	/*
1032	 * Don't flush a closed channel. This can happen when the work queued
1033	 * autoflush thread fires off after the file has closed. This is not
1034	 * an error, just something to dismiss.
1035	 */
1036
1037	if (!channel->rd_ref_count)
1038		goto done;
1039
1040	bufidx = channel->rd_host_buf_idx;
1041
1042	bufidx_minus1 = (bufidx == 0) ?
1043		channel->num_rd_buffers - 1 :
1044		bufidx - 1;
1045
1046	end_offset_plus1 = channel->rd_host_buf_pos >>
1047		channel->log2_element_size;
1048
1049	new_rd_host_buf_pos = channel->rd_host_buf_pos -
1050		(end_offset_plus1 << channel->log2_element_size);
1051
1052	/* Submit the current buffer if it's nonempty */
1053	if (end_offset_plus1) {
1054		unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1055			(end_offset_plus1 << channel->log2_element_size);
1056
1057		/* Copy  unflushed data, so we can put it in next buffer */
1058		for (i = 0; i < new_rd_host_buf_pos; i++)
1059			channel->rd_leftovers[i] = *tail++;
1060
1061		spin_lock_irqsave(&channel->rd_spinlock, flags);
1062
1063		/* Autoflush only if a single buffer is occupied */
1064
1065		if ((timeout < 0) &&
1066		    (channel->rd_full ||
1067		     (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1068			spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1069			/*
1070			 * A new work item may be queued by the ISR exactly
1071			 * now, since the execution of a work item allows the
1072			 * queuing of a new one while it's running.
1073			 */
1074			goto done;
1075		}
1076
1077		/* The 4th element is never needed for data, so it's a flag */
1078		channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1079
1080		/* Set up rd_full to reflect a certain moment's state */
1081
1082		if (bufidx == channel->rd_fpga_buf_idx)
1083			channel->rd_full = 1;
1084		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1085
1086		if (bufidx >= (channel->num_rd_buffers - 1))
1087			channel->rd_host_buf_idx = 0;
1088		else
1089			channel->rd_host_buf_idx++;
1090
1091		dma_sync_single_for_device(channel->endpoint->dev,
1092					   channel->rd_buffers[bufidx]->dma_addr,
1093					   channel->rd_buf_size,
1094					   DMA_TO_DEVICE);
1095
1096		mutex_lock(&channel->endpoint->register_mutex);
1097
1098		iowrite32(end_offset_plus1 - 1,
1099			  channel->endpoint->registers + fpga_buf_offset_reg);
1100
1101		iowrite32((channel->chan_num << 1) | /* Channel ID */
1102			  (2 << 24) |  /* Opcode 2, submit buffer */
1103			  (bufidx << 12),
1104			  channel->endpoint->registers + fpga_buf_ctrl_reg);
1105
1106		mutex_unlock(&channel->endpoint->register_mutex);
1107	} else if (bufidx == 0) {
1108		bufidx = channel->num_rd_buffers - 1;
1109	} else {
1110		bufidx--;
1111	}
1112
1113	channel->rd_host_buf_pos = new_rd_host_buf_pos;
1114
1115	if (timeout < 0)
1116		goto done; /* Autoflush */
1117
1118	/*
1119	 * bufidx is now the last buffer written to (or equal to
1120	 * rd_fpga_buf_idx if buffer was never written to), and
1121	 * channel->rd_host_buf_idx the one after it.
1122	 *
1123	 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1124	 */
1125
1126	while (1) { /* Loop waiting for draining of buffers */
1127		spin_lock_irqsave(&channel->rd_spinlock, flags);
1128
1129		if (bufidx != channel->rd_fpga_buf_idx)
1130			channel->rd_full = 1; /*
1131					       * Not really full,
1132					       * but needs waiting.
1133					       */
1134
1135		empty = !channel->rd_full;
1136
1137		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1138
1139		if (empty)
1140			break;
1141
1142		/*
1143		 * Indefinite sleep with mutex taken. With data waiting for
1144		 * flushing user should not be surprised if open() for write
1145		 * sleeps.
1146		 */
1147		if (timeout == 0)
1148			wait_event_interruptible(channel->rd_wait,
1149						 (!channel->rd_full));
1150
1151		else if (wait_event_interruptible_timeout(
1152				 channel->rd_wait,
1153				 (!channel->rd_full),
1154				 timeout) == 0) {
1155			dev_warn(channel->endpoint->dev,
1156				 "Timed out while flushing. Output data may be lost.\n");
1157
1158			rc = -ETIMEDOUT;
1159			break;
1160		}
1161
1162		if (channel->rd_full) {
1163			rc = -EINTR;
1164			break;
1165		}
1166	}
1167
1168done:
1169	mutex_unlock(&channel->rd_mutex);
1170
1171	if (channel->endpoint->fatal_error)
1172		return -EIO;
1173
1174	return rc;
1175}
1176
1177static int xillybus_flush(struct file *filp, fl_owner_t id)
1178{
1179	if (!(filp->f_mode & FMODE_WRITE))
1180		return 0;
1181
1182	return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1183}
1184
1185static void xillybus_autoflush(struct work_struct *work)
1186{
1187	struct delayed_work *workitem = to_delayed_work(work);
1188	struct xilly_channel *channel = container_of(
1189		workitem, struct xilly_channel, rd_workitem);
1190	int rc;
1191
1192	rc = xillybus_myflush(channel, -1);
1193	if (rc == -EINTR)
1194		dev_warn(channel->endpoint->dev,
1195			 "Autoflush failed because work queue thread got a signal.\n");
1196	else if (rc)
1197		dev_err(channel->endpoint->dev,
1198			"Autoflush failed under weird circumstances.\n");
1199}
1200
1201static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1202			      size_t count, loff_t *f_pos)
1203{
1204	ssize_t rc;
1205	unsigned long flags;
1206	int bytes_done = 0;
1207	struct xilly_channel *channel = filp->private_data;
1208
1209	int full, exhausted;
1210	/* Initializations are there only to silence warnings */
1211
1212	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1213	int end_offset_plus1 = 0;
1214
1215	if (channel->endpoint->fatal_error)
1216		return -EIO;
1217
1218	rc = mutex_lock_interruptible(&channel->rd_mutex);
1219	if (rc)
1220		return rc;
1221
1222	while (1) {
1223		int bytes_to_do = count - bytes_done;
1224
1225		spin_lock_irqsave(&channel->rd_spinlock, flags);
1226
1227		full = channel->rd_full;
1228
1229		if (!full) {
1230			bufidx = channel->rd_host_buf_idx;
1231			bufpos = channel->rd_host_buf_pos;
1232			howmany = channel->rd_buf_size - bufpos;
1233
1234			/*
1235			 * Update rd_host_* to its state after this operation.
1236			 * count=0 means committing the buffer immediately,
1237			 * which is like flushing, but not necessarily block.
1238			 */
1239
1240			if ((howmany > bytes_to_do) &&
1241			    (count ||
1242			     ((bufpos >> channel->log2_element_size) == 0))) {
1243				bufferdone = 0;
1244
1245				howmany = bytes_to_do;
1246				channel->rd_host_buf_pos += howmany;
1247			} else {
1248				bufferdone = 1;
1249
1250				if (count) {
1251					end_offset_plus1 =
1252						channel->rd_buf_size >>
1253						channel->log2_element_size;
1254					channel->rd_host_buf_pos = 0;
1255				} else {
1256					unsigned char *tail;
1257					int i;
1258
1259					howmany = 0;
1260
1261					end_offset_plus1 = bufpos >>
1262						channel->log2_element_size;
1263
1264					channel->rd_host_buf_pos -=
1265						end_offset_plus1 <<
1266						channel->log2_element_size;
1267
1268					tail = channel->
1269						rd_buffers[bufidx]->addr +
1270						(end_offset_plus1 <<
1271						 channel->log2_element_size);
1272
1273					for (i = 0;
1274					     i < channel->rd_host_buf_pos;
1275					     i++)
1276						channel->rd_leftovers[i] =
1277							*tail++;
1278				}
1279
1280				if (bufidx == channel->rd_fpga_buf_idx)
1281					channel->rd_full = 1;
1282
1283				if (bufidx >= (channel->num_rd_buffers - 1))
1284					channel->rd_host_buf_idx = 0;
1285				else
1286					channel->rd_host_buf_idx++;
1287			}
1288		}
1289
1290		/*
1291		 * Marking our situation after the possible changes above,
1292		 * for use  after releasing the spinlock.
1293		 *
1294		 * full = full before change
1295		 * exhasted = full after possible change
1296		 */
1297
1298		exhausted = channel->rd_full;
1299
1300		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1301
1302		if (!full) { /* Go on, now without the spinlock */
1303			unsigned char *head =
1304				channel->rd_buffers[bufidx]->addr;
1305			int i;
1306
1307			if ((bufpos == 0) || /* Zero means it's virgin */
1308			    (channel->rd_leftovers[3] != 0)) {
1309				dma_sync_single_for_cpu(channel->endpoint->dev,
1310							channel->rd_buffers[bufidx]->dma_addr,
1311							channel->rd_buf_size,
1312							DMA_TO_DEVICE);
1313
1314				/* Virgin, but leftovers are due */
1315				for (i = 0; i < bufpos; i++)
1316					*head++ = channel->rd_leftovers[i];
1317
1318				channel->rd_leftovers[3] = 0; /* Clear flag */
1319			}
1320
1321			if (copy_from_user(
1322				    channel->rd_buffers[bufidx]->addr + bufpos,
1323				    userbuf, howmany))
1324				rc = -EFAULT;
1325
1326			userbuf += howmany;
1327			bytes_done += howmany;
1328
1329			if (bufferdone) {
1330				dma_sync_single_for_device(channel->endpoint->dev,
1331							   channel->rd_buffers[bufidx]->dma_addr,
1332							   channel->rd_buf_size,
1333							   DMA_TO_DEVICE);
1334
1335				mutex_lock(&channel->endpoint->register_mutex);
1336
1337				iowrite32(end_offset_plus1 - 1,
1338					  channel->endpoint->registers +
1339					  fpga_buf_offset_reg);
1340
1341				iowrite32((channel->chan_num << 1) |
1342					  (2 << 24) |  /* 2 = submit buffer */
1343					  (bufidx << 12),
1344					  channel->endpoint->registers +
1345					  fpga_buf_ctrl_reg);
1346
1347				mutex_unlock(&channel->endpoint->
1348					     register_mutex);
1349
1350				channel->rd_leftovers[3] =
1351					(channel->rd_host_buf_pos != 0);
1352			}
1353
1354			if (rc) {
1355				mutex_unlock(&channel->rd_mutex);
1356
1357				if (channel->endpoint->fatal_error)
1358					return -EIO;
1359
1360				if (!channel->rd_synchronous)
1361					queue_delayed_work(
1362						xillybus_wq,
1363						&channel->rd_workitem,
1364						XILLY_RX_TIMEOUT);
1365
1366				return rc;
1367			}
1368		}
1369
1370		if (bytes_done >= count)
1371			break;
1372
1373		if (!exhausted)
1374			continue; /* If there's more space, just go on */
1375
1376		if ((bytes_done > 0) && channel->rd_allow_partial)
1377			break;
1378
1379		/*
1380		 * Indefinite sleep with mutex taken. With data waiting for
1381		 * flushing, user should not be surprised if open() for write
1382		 * sleeps.
1383		 */
1384
1385		if (filp->f_flags & O_NONBLOCK) {
1386			rc = -EAGAIN;
1387			break;
1388		}
1389
1390		if (wait_event_interruptible(channel->rd_wait,
1391					     (!channel->rd_full))) {
1392			mutex_unlock(&channel->rd_mutex);
1393
1394			if (channel->endpoint->fatal_error)
1395				return -EIO;
1396
1397			if (bytes_done)
1398				return bytes_done;
1399			return -EINTR;
1400		}
1401	}
1402
1403	mutex_unlock(&channel->rd_mutex);
1404
1405	if (!channel->rd_synchronous)
1406		queue_delayed_work(xillybus_wq,
1407				   &channel->rd_workitem,
1408				   XILLY_RX_TIMEOUT);
1409
1410	if (channel->endpoint->fatal_error)
1411		return -EIO;
1412
1413	if (rc)
1414		return rc;
1415
1416	if ((channel->rd_synchronous) && (bytes_done > 0)) {
1417		rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1418
1419		if (rc && (rc != -EINTR))
1420			return rc;
1421	}
1422
1423	return bytes_done;
1424}
1425
1426static int xillybus_open(struct inode *inode, struct file *filp)
1427{
1428	int rc;
1429	unsigned long flags;
1430	struct xilly_endpoint *endpoint;
1431	struct xilly_channel *channel;
1432	int index;
1433
1434	rc = xillybus_find_inode(inode, (void **)&endpoint, &index);
1435	if (rc)
1436		return rc;
1437
1438	if (endpoint->fatal_error)
1439		return -EIO;
1440
1441	channel = endpoint->channels[1 + index];
1442	filp->private_data = channel;
1443
1444	/*
1445	 * It gets complicated because:
1446	 * 1. We don't want to take a mutex we don't have to
1447	 * 2. We don't want to open one direction if the other will fail.
1448	 */
1449
1450	if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1451		return -ENODEV;
1452
1453	if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1454		return -ENODEV;
1455
1456	if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1457	    (channel->wr_synchronous || !channel->wr_allow_partial ||
1458	     !channel->wr_supports_nonempty)) {
1459		dev_err(endpoint->dev,
1460			"open() failed: O_NONBLOCK not allowed for read on this device\n");
1461		return -ENODEV;
1462	}
1463
1464	if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1465	    (channel->rd_synchronous || !channel->rd_allow_partial)) {
1466		dev_err(endpoint->dev,
1467			"open() failed: O_NONBLOCK not allowed for write on this device\n");
1468		return -ENODEV;
1469	}
1470
1471	/*
1472	 * Note: open() may block on getting mutexes despite O_NONBLOCK.
1473	 * This shouldn't occur normally, since multiple open of the same
1474	 * file descriptor is almost always prohibited anyhow
1475	 * (*_exclusive_open is normally set in real-life systems).
1476	 */
1477
1478	if (filp->f_mode & FMODE_READ) {
1479		rc = mutex_lock_interruptible(&channel->wr_mutex);
1480		if (rc)
1481			return rc;
1482	}
1483
1484	if (filp->f_mode & FMODE_WRITE) {
1485		rc = mutex_lock_interruptible(&channel->rd_mutex);
1486		if (rc)
1487			goto unlock_wr;
1488	}
1489
1490	if ((filp->f_mode & FMODE_READ) &&
1491	    (channel->wr_ref_count != 0) &&
1492	    (channel->wr_exclusive_open)) {
1493		rc = -EBUSY;
1494		goto unlock;
1495	}
1496
1497	if ((filp->f_mode & FMODE_WRITE) &&
1498	    (channel->rd_ref_count != 0) &&
1499	    (channel->rd_exclusive_open)) {
1500		rc = -EBUSY;
1501		goto unlock;
1502	}
1503
1504	if (filp->f_mode & FMODE_READ) {
1505		if (channel->wr_ref_count == 0) { /* First open of file */
1506			/* Move the host to first buffer */
1507			spin_lock_irqsave(&channel->wr_spinlock, flags);
1508			channel->wr_host_buf_idx = 0;
1509			channel->wr_host_buf_pos = 0;
1510			channel->wr_fpga_buf_idx = -1;
1511			channel->wr_empty = 1;
1512			channel->wr_ready = 0;
1513			channel->wr_sleepy = 1;
1514			channel->wr_eof = -1;
1515			channel->wr_hangup = 0;
1516
1517			spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1518
1519			iowrite32(1 | (channel->chan_num << 1) |
1520				  (4 << 24) |  /* Opcode 4, open channel */
1521				  ((channel->wr_synchronous & 1) << 23),
1522				  channel->endpoint->registers +
1523				  fpga_buf_ctrl_reg);
1524		}
1525
1526		channel->wr_ref_count++;
1527	}
1528
1529	if (filp->f_mode & FMODE_WRITE) {
1530		if (channel->rd_ref_count == 0) { /* First open of file */
1531			/* Move the host to first buffer */
1532			spin_lock_irqsave(&channel->rd_spinlock, flags);
1533			channel->rd_host_buf_idx = 0;
1534			channel->rd_host_buf_pos = 0;
1535			channel->rd_leftovers[3] = 0; /* No leftovers. */
1536			channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1537			channel->rd_full = 0;
1538
1539			spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1540
1541			iowrite32((channel->chan_num << 1) |
1542				  (4 << 24),   /* Opcode 4, open channel */
1543				  channel->endpoint->registers +
1544				  fpga_buf_ctrl_reg);
1545		}
1546
1547		channel->rd_ref_count++;
1548	}
1549
1550unlock:
1551	if (filp->f_mode & FMODE_WRITE)
1552		mutex_unlock(&channel->rd_mutex);
1553unlock_wr:
1554	if (filp->f_mode & FMODE_READ)
1555		mutex_unlock(&channel->wr_mutex);
1556
1557	if (!rc && (!channel->seekable))
1558		return nonseekable_open(inode, filp);
1559
1560	return rc;
1561}
1562
1563static int xillybus_release(struct inode *inode, struct file *filp)
1564{
1565	unsigned long flags;
1566	struct xilly_channel *channel = filp->private_data;
1567
1568	int buf_idx;
1569	int eof;
1570
1571	if (channel->endpoint->fatal_error)
1572		return -EIO;
1573
1574	if (filp->f_mode & FMODE_WRITE) {
1575		mutex_lock(&channel->rd_mutex);
1576
1577		channel->rd_ref_count--;
1578
1579		if (channel->rd_ref_count == 0) {
1580			/*
1581			 * We rely on the kernel calling flush()
1582			 * before we get here.
1583			 */
1584
1585			iowrite32((channel->chan_num << 1) | /* Channel ID */
1586				  (5 << 24),  /* Opcode 5, close channel */
1587				  channel->endpoint->registers +
1588				  fpga_buf_ctrl_reg);
1589		}
1590		mutex_unlock(&channel->rd_mutex);
1591	}
1592
1593	if (filp->f_mode & FMODE_READ) {
1594		mutex_lock(&channel->wr_mutex);
1595
1596		channel->wr_ref_count--;
1597
1598		if (channel->wr_ref_count == 0) {
1599			iowrite32(1 | (channel->chan_num << 1) |
1600				  (5 << 24),  /* Opcode 5, close channel */
1601				  channel->endpoint->registers +
1602				  fpga_buf_ctrl_reg);
1603
1604			/*
1605			 * This is crazily cautious: We make sure that not
1606			 * only that we got an EOF (be it because we closed
1607			 * the channel or because of a user's EOF), but verify
1608			 * that it's one beyond the last buffer arrived, so
1609			 * we have no leftover buffers pending before wrapping
1610			 * up (which can only happen in asynchronous channels,
1611			 * BTW)
1612			 */
1613
1614			while (1) {
1615				spin_lock_irqsave(&channel->wr_spinlock,
1616						  flags);
1617				buf_idx = channel->wr_fpga_buf_idx;
1618				eof = channel->wr_eof;
1619				channel->wr_sleepy = 1;
1620				spin_unlock_irqrestore(&channel->wr_spinlock,
1621						       flags);
1622
1623				/*
1624				 * Check if eof points at the buffer after
1625				 * the last one the FPGA submitted. Note that
1626				 * no EOF is marked by negative eof.
1627				 */
1628
1629				buf_idx++;
1630				if (buf_idx == channel->num_wr_buffers)
1631					buf_idx = 0;
1632
1633				if (buf_idx == eof)
1634					break;
1635
1636				/*
1637				 * Steal extra 100 ms if awaken by interrupt.
1638				 * This is a simple workaround for an
1639				 * interrupt pending when entering, which would
1640				 * otherwise result in declaring the hardware
1641				 * non-responsive.
1642				 */
1643
1644				if (wait_event_interruptible(
1645					    channel->wr_wait,
1646					    (!channel->wr_sleepy)))
1647					msleep(100);
1648
1649				if (channel->wr_sleepy) {
1650					mutex_unlock(&channel->wr_mutex);
1651					dev_warn(channel->endpoint->dev,
1652						 "Hardware failed to respond to close command, therefore left in messy state.\n");
1653					return -EINTR;
1654				}
1655			}
1656		}
1657
1658		mutex_unlock(&channel->wr_mutex);
1659	}
1660
1661	return 0;
1662}
1663
1664static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1665{
1666	struct xilly_channel *channel = filp->private_data;
1667	loff_t pos = filp->f_pos;
1668	int rc = 0;
1669
1670	/*
1671	 * Take both mutexes not allowing interrupts, since it seems like
1672	 * common applications don't expect an -EINTR here. Besides, multiple
1673	 * access to a single file descriptor on seekable devices is a mess
1674	 * anyhow.
1675	 */
1676
1677	if (channel->endpoint->fatal_error)
1678		return -EIO;
1679
1680	mutex_lock(&channel->wr_mutex);
1681	mutex_lock(&channel->rd_mutex);
1682
1683	switch (whence) {
1684	case SEEK_SET:
1685		pos = offset;
1686		break;
1687	case SEEK_CUR:
1688		pos += offset;
1689		break;
1690	case SEEK_END:
1691		pos = offset; /* Going to the end => to the beginning */
1692		break;
1693	default:
1694		rc = -EINVAL;
1695		goto end;
1696	}
1697
1698	/* In any case, we must finish on an element boundary */
1699	if (pos & ((1 << channel->log2_element_size) - 1)) {
1700		rc = -EINVAL;
1701		goto end;
1702	}
1703
1704	mutex_lock(&channel->endpoint->register_mutex);
1705
1706	iowrite32(pos >> channel->log2_element_size,
1707		  channel->endpoint->registers + fpga_buf_offset_reg);
1708
1709	iowrite32((channel->chan_num << 1) |
1710		  (6 << 24),  /* Opcode 6, set address */
1711		  channel->endpoint->registers + fpga_buf_ctrl_reg);
1712
1713	mutex_unlock(&channel->endpoint->register_mutex);
1714
1715end:
1716	mutex_unlock(&channel->rd_mutex);
1717	mutex_unlock(&channel->wr_mutex);
1718
1719	if (rc) /* Return error after releasing mutexes */
1720		return rc;
1721
1722	filp->f_pos = pos;
1723
1724	/*
1725	 * Since seekable devices are allowed only when the channel is
1726	 * synchronous, we assume that there is no data pending in either
1727	 * direction (which holds true as long as no concurrent access on the
1728	 * file descriptor takes place).
1729	 * The only thing we may need to throw away is leftovers from partial
1730	 * write() flush.
1731	 */
1732
1733	channel->rd_leftovers[3] = 0;
1734
1735	return pos;
1736}
1737
1738static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
1739{
1740	struct xilly_channel *channel = filp->private_data;
1741	__poll_t mask = 0;
1742	unsigned long flags;
1743
1744	poll_wait(filp, &channel->endpoint->ep_wait, wait);
1745
1746	/*
1747	 * poll() won't play ball regarding read() channels which
1748	 * aren't asynchronous and support the nonempty message. Allowing
1749	 * that will create situations where data has been delivered at
1750	 * the FPGA, and users expecting select() to wake up, which it may
1751	 * not.
1752	 */
1753
1754	if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1755		poll_wait(filp, &channel->wr_wait, wait);
1756		poll_wait(filp, &channel->wr_ready_wait, wait);
1757
1758		spin_lock_irqsave(&channel->wr_spinlock, flags);
1759		if (!channel->wr_empty || channel->wr_ready)
1760			mask |= EPOLLIN | EPOLLRDNORM;
1761
1762		if (channel->wr_hangup)
1763			/*
1764			 * Not EPOLLHUP, because its behavior is in the
1765			 * mist, and EPOLLIN does what we want: Wake up
1766			 * the read file descriptor so it sees EOF.
1767			 */
1768			mask |=  EPOLLIN | EPOLLRDNORM;
1769		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1770	}
1771
1772	/*
1773	 * If partial data write is disallowed on a write() channel,
1774	 * it's pointless to ever signal OK to write, because is could
1775	 * block despite some space being available.
1776	 */
1777
1778	if (channel->rd_allow_partial) {
1779		poll_wait(filp, &channel->rd_wait, wait);
1780
1781		spin_lock_irqsave(&channel->rd_spinlock, flags);
1782		if (!channel->rd_full)
1783			mask |= EPOLLOUT | EPOLLWRNORM;
1784		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1785	}
1786
1787	if (channel->endpoint->fatal_error)
1788		mask |= EPOLLERR;
1789
1790	return mask;
1791}
1792
1793static const struct file_operations xillybus_fops = {
1794	.owner      = THIS_MODULE,
1795	.read       = xillybus_read,
1796	.write      = xillybus_write,
1797	.open       = xillybus_open,
1798	.flush      = xillybus_flush,
1799	.release    = xillybus_release,
1800	.llseek     = xillybus_llseek,
1801	.poll       = xillybus_poll,
1802};
1803
1804struct xilly_endpoint *xillybus_init_endpoint(struct device *dev)
1805{
1806	struct xilly_endpoint *endpoint;
1807
1808	endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1809	if (!endpoint)
1810		return NULL;
1811
1812	endpoint->dev = dev;
1813	endpoint->msg_counter = 0x0b;
1814	endpoint->failed_messages = 0;
1815	endpoint->fatal_error = 0;
1816
1817	init_waitqueue_head(&endpoint->ep_wait);
1818	mutex_init(&endpoint->register_mutex);
1819
1820	return endpoint;
1821}
1822EXPORT_SYMBOL(xillybus_init_endpoint);
1823
1824static int xilly_quiesce(struct xilly_endpoint *endpoint)
1825{
1826	long t;
1827
1828	endpoint->idtlen = -1;
1829
1830	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1831		  endpoint->registers + fpga_dma_control_reg);
1832
1833	t = wait_event_interruptible_timeout(endpoint->ep_wait,
1834					     (endpoint->idtlen >= 0),
1835					     XILLY_TIMEOUT);
1836	if (t <= 0) {
1837		dev_err(endpoint->dev,
1838			"Failed to quiesce the device on exit.\n");
1839		return -ENODEV;
1840	}
1841	return 0;
1842}
1843
1844int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1845{
1846	int rc;
1847	long t;
1848
1849	void *bootstrap_resources;
1850	int idtbuffersize = (1 << PAGE_SHIFT);
1851	struct device *dev = endpoint->dev;
1852
1853	/*
1854	 * The bogus IDT is used during bootstrap for allocating the initial
1855	 * message buffer, and then the message buffer and space for the IDT
1856	 * itself. The initial message buffer is of a single page's size, but
1857	 * it's soon replaced with a more modest one (and memory is freed).
1858	 */
1859
1860	unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1861				       3, 192, PAGE_SHIFT, 0 };
1862	struct xilly_idt_handle idt_handle;
1863
1864	/*
1865	 * Writing the value 0x00000001 to Endianness register signals which
1866	 * endianness this processor is using, so the FPGA can swap words as
1867	 * necessary.
1868	 */
1869
1870	iowrite32(1, endpoint->registers + fpga_endian_reg);
1871
1872	/* Bootstrap phase I: Allocate temporary message buffer */
1873
1874	bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1875	if (!bootstrap_resources)
1876		return -ENOMEM;
1877
1878	endpoint->num_channels = 0;
1879
1880	rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1881	if (rc)
1882		return rc;
1883
1884	/* Clear the message subsystem (and counter in particular) */
1885	iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1886
1887	endpoint->idtlen = -1;
1888
1889	/*
1890	 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1891	 * buffer size.
1892	 */
1893	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1894		  endpoint->registers + fpga_dma_control_reg);
1895
1896	t = wait_event_interruptible_timeout(endpoint->ep_wait,
1897					     (endpoint->idtlen >= 0),
1898					     XILLY_TIMEOUT);
1899	if (t <= 0) {
1900		dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1901		return -ENODEV;
1902	}
1903
1904	/* Enable DMA */
1905	iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
1906		  endpoint->registers + fpga_dma_control_reg);
1907
1908	/* Bootstrap phase II: Allocate buffer for IDT and obtain it */
1909	while (endpoint->idtlen >= idtbuffersize) {
1910		idtbuffersize *= 2;
1911		bogus_idt[6]++;
1912	}
1913
1914	endpoint->num_channels = 1;
1915
1916	rc = xilly_setupchannels(endpoint, bogus_idt, 2);
1917	if (rc)
1918		goto failed_idt;
1919
1920	rc = xilly_obtain_idt(endpoint);
1921	if (rc)
1922		goto failed_idt;
1923
1924	rc = xilly_scan_idt(endpoint, &idt_handle);
1925	if (rc)
1926		goto failed_idt;
1927
1928	devres_close_group(dev, bootstrap_resources);
1929
1930	/* Bootstrap phase III: Allocate buffers according to IDT */
1931
1932	rc = xilly_setupchannels(endpoint,
1933				 idt_handle.chandesc,
1934				 idt_handle.entries);
1935	if (rc)
1936		goto failed_idt;
1937
1938	rc = xillybus_init_chrdev(dev, &xillybus_fops,
1939				  endpoint->owner, endpoint,
1940				  idt_handle.names,
1941				  idt_handle.names_len,
1942				  endpoint->num_channels,
1943				  xillyname, false);
1944
1945	if (rc)
1946		goto failed_idt;
1947
1948	devres_release_group(dev, bootstrap_resources);
1949
1950	return 0;
1951
1952failed_idt:
1953	xilly_quiesce(endpoint);
1954	flush_workqueue(xillybus_wq);
1955
1956	return rc;
1957}
1958EXPORT_SYMBOL(xillybus_endpoint_discovery);
1959
1960void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
1961{
1962	xillybus_cleanup_chrdev(endpoint, endpoint->dev);
1963
1964	xilly_quiesce(endpoint);
1965
1966	/*
1967	 * Flushing is done upon endpoint release to prevent access to memory
1968	 * just about to be released. This makes the quiesce complete.
1969	 */
1970	flush_workqueue(xillybus_wq);
1971}
1972EXPORT_SYMBOL(xillybus_endpoint_remove);
1973
1974static int __init xillybus_init(void)
1975{
1976	xillybus_wq = alloc_workqueue(xillyname, WQ_UNBOUND, 0);
1977	if (!xillybus_wq)
1978		return -ENOMEM;
1979
1980	return 0;
1981}
1982
1983static void __exit xillybus_exit(void)
1984{
1985	/* flush_workqueue() was called for each endpoint released */
1986	destroy_workqueue(xillybus_wq);
1987}
1988
1989module_init(xillybus_init);
1990module_exit(xillybus_exit);