at91_udc.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * at91_udc -- driver for at91-series USB peripheral controller
   4 *
   5 * Copyright (C) 2004 by Thomas Rathbone
   6 * Copyright (C) 2005 by HP Labs
   7 * Copyright (C) 2005 by David Brownell
   8 */
   9
  10#undef	VERBOSE_DEBUG
  11#undef	PACKET_TRACE
  12
  13#include <linux/kernel.h>
  14#include <linux/module.h>
  15#include <linux/platform_device.h>
  16#include <linux/delay.h>
  17#include <linux/ioport.h>
  18#include <linux/slab.h>
  19#include <linux/string_choices.h>
  20#include <linux/errno.h>
  21#include <linux/list.h>
  22#include <linux/interrupt.h>
  23#include <linux/proc_fs.h>
  24#include <linux/prefetch.h>
  25#include <linux/clk.h>
  26#include <linux/usb/ch9.h>
  27#include <linux/usb/gadget.h>
  28#include <linux/of.h>
  29#include <linux/gpio/consumer.h>
  30#include <linux/platform_data/atmel.h>
  31#include <linux/regmap.h>
  32#include <linux/mfd/syscon.h>
  33#include <linux/mfd/syscon/atmel-matrix.h>
  34
  35#include "at91_udc.h"
  36
  37
  38/*
  39 * This controller is simple and PIO-only.  It's used in many AT91-series
  40 * full speed USB controllers, including the at91rm9200 (arm920T, with MMU),
  41 * at91sam926x (arm926ejs, with MMU), and several no-mmu versions.
  42 *
  43 * This driver expects the board has been wired with two GPIOs supporting
  44 * a VBUS sensing IRQ, and a D+ pullup.  (They may be omitted, but the
  45 * testing hasn't covered such cases.)
  46 *
  47 * The pullup is most important (so it's integrated on sam926x parts).  It
  48 * provides software control over whether the host enumerates the device.
  49 *
  50 * The VBUS sensing helps during enumeration, and allows both USB clocks
  51 * (and the transceiver) to stay gated off until they're necessary, saving
  52 * power.  During USB suspend, the 48 MHz clock is gated off in hardware;
  53 * it may also be gated off by software during some Linux sleep states.
  54 */
  55
  56#define	DRIVER_VERSION	"3 May 2006"
  57
  58static const char driver_name [] = "at91_udc";
  59
  60static const struct {
  61	const char *name;
  62	const struct usb_ep_caps caps;
  63} ep_info[] = {
  64#define EP_INFO(_name, _caps) \
  65	{ \
  66		.name = _name, \
  67		.caps = _caps, \
  68	}
  69
  70	EP_INFO("ep0",
  71		USB_EP_CAPS(USB_EP_CAPS_TYPE_CONTROL, USB_EP_CAPS_DIR_ALL)),
  72	EP_INFO("ep1",
  73		USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  74	EP_INFO("ep2",
  75		USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  76	EP_INFO("ep3-int",
  77		USB_EP_CAPS(USB_EP_CAPS_TYPE_INT, USB_EP_CAPS_DIR_ALL)),
  78	EP_INFO("ep4",
  79		USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  80	EP_INFO("ep5",
  81		USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  82
  83#undef EP_INFO
  84};
  85
  86#define ep0name		ep_info[0].name
  87
  88#define VBUS_POLL_TIMEOUT	msecs_to_jiffies(1000)
  89
  90#define at91_udp_read(udc, reg) \
  91	__raw_readl((udc)->udp_baseaddr + (reg))
  92#define at91_udp_write(udc, reg, val) \
  93	__raw_writel((val), (udc)->udp_baseaddr + (reg))
  94
  95/*-------------------------------------------------------------------------*/
  96
  97#ifdef CONFIG_USB_GADGET_DEBUG_FILES
  98
  99#include <linux/seq_file.h>
 100
 101static const char debug_filename[] = "driver/udc";
 102
 103#define FOURBITS "%s%s%s%s"
 104#define EIGHTBITS FOURBITS FOURBITS
 105
 106static void proc_ep_show(struct seq_file *s, struct at91_ep *ep)
 107{
 108	static char		*types[] = {
 109		"control", "out-iso", "out-bulk", "out-int",
 110		"BOGUS",   "in-iso",  "in-bulk",  "in-int"};
 111
 112	u32			csr;
 113	struct at91_request	*req;
 114	unsigned long	flags;
 115	struct at91_udc	*udc = ep->udc;
 116
 117	spin_lock_irqsave(&udc->lock, flags);
 118
 119	csr = __raw_readl(ep->creg);
 120
 121	/* NOTE:  not collecting per-endpoint irq statistics... */
 122
 123	seq_printf(s, "\n");
 124	seq_printf(s, "%s, maxpacket %d %s%s %s%s\n",
 125			ep->ep.name, ep->ep.maxpacket,
 126			ep->is_in ? "in" : "out",
 127			ep->is_iso ? " iso" : "",
 128			ep->is_pingpong
 129				? (ep->fifo_bank ? "pong" : "ping")
 130				: "",
 131			ep->stopped ? " stopped" : "");
 132	seq_printf(s, "csr %08x rxbytes=%d %s %s %s" EIGHTBITS "\n",
 133		csr,
 134		(csr & 0x07ff0000) >> 16,
 135		str_enabled_disabled(csr & (1 << 15)),
 136		(csr & (1 << 11)) ? "DATA1" : "DATA0",
 137		types[(csr & 0x700) >> 8],
 138
 139		/* iff type is control then print current direction */
 140		(!(csr & 0x700))
 141			? ((csr & (1 << 7)) ? " IN" : " OUT")
 142			: "",
 143		(csr & (1 << 6)) ? " rxdatabk1" : "",
 144		(csr & (1 << 5)) ? " forcestall" : "",
 145		(csr & (1 << 4)) ? " txpktrdy" : "",
 146
 147		(csr & (1 << 3)) ? " stallsent" : "",
 148		(csr & (1 << 2)) ? " rxsetup" : "",
 149		(csr & (1 << 1)) ? " rxdatabk0" : "",
 150		(csr & (1 << 0)) ? " txcomp" : "");
 151	if (list_empty (&ep->queue))
 152		seq_printf(s, "\t(queue empty)\n");
 153
 154	else list_for_each_entry (req, &ep->queue, queue) {
 155		unsigned	length = req->req.actual;
 156
 157		seq_printf(s, "\treq %p len %d/%d buf %p\n",
 158				&req->req, length,
 159				req->req.length, req->req.buf);
 160	}
 161	spin_unlock_irqrestore(&udc->lock, flags);
 162}
 163
 164static void proc_irq_show(struct seq_file *s, const char *label, u32 mask)
 165{
 166	int i;
 167
 168	seq_printf(s, "%s %04x:%s%s" FOURBITS, label, mask,
 169		(mask & (1 << 13)) ? " wakeup" : "",
 170		(mask & (1 << 12)) ? " endbusres" : "",
 171
 172		(mask & (1 << 11)) ? " sofint" : "",
 173		(mask & (1 << 10)) ? " extrsm" : "",
 174		(mask & (1 << 9)) ? " rxrsm" : "",
 175		(mask & (1 << 8)) ? " rxsusp" : "");
 176	for (i = 0; i < 8; i++) {
 177		if (mask & (1 << i))
 178			seq_printf(s, " ep%d", i);
 179	}
 180	seq_printf(s, "\n");
 181}
 182
 183static int proc_udc_show(struct seq_file *s, void *unused)
 184{
 185	struct at91_udc	*udc = s->private;
 186	struct at91_ep	*ep;
 187	u32		tmp;
 188
 189	seq_printf(s, "%s: version %s\n", driver_name, DRIVER_VERSION);
 190
 191	seq_printf(s, "vbus %s, pullup %s, %s powered%s, gadget %s\n\n",
 192		udc->vbus ? "present" : "off",
 193		udc->enabled
 194			? (udc->vbus ? "active" : "enabled")
 195			: "disabled",
 196		udc->gadget.is_selfpowered ? "self" : "VBUS",
 197		udc->suspended ? ", suspended" : "",
 198		udc->driver ? udc->driver->driver.name : "(none)");
 199
 200	/* don't access registers when interface isn't clocked */
 201	if (!udc->clocked) {
 202		seq_printf(s, "(not clocked)\n");
 203		return 0;
 204	}
 205
 206	tmp = at91_udp_read(udc, AT91_UDP_FRM_NUM);
 207	seq_printf(s, "frame %05x:%s%s frame=%d\n", tmp,
 208		(tmp & AT91_UDP_FRM_OK) ? " ok" : "",
 209		(tmp & AT91_UDP_FRM_ERR) ? " err" : "",
 210		(tmp & AT91_UDP_NUM));
 211
 212	tmp = at91_udp_read(udc, AT91_UDP_GLB_STAT);
 213	seq_printf(s, "glbstate %02x:%s" FOURBITS "\n", tmp,
 214		(tmp & AT91_UDP_RMWUPE) ? " rmwupe" : "",
 215		(tmp & AT91_UDP_RSMINPR) ? " rsminpr" : "",
 216		(tmp & AT91_UDP_ESR) ? " esr" : "",
 217		(tmp & AT91_UDP_CONFG) ? " confg" : "",
 218		(tmp & AT91_UDP_FADDEN) ? " fadden" : "");
 219
 220	tmp = at91_udp_read(udc, AT91_UDP_FADDR);
 221	seq_printf(s, "faddr   %03x:%s fadd=%d\n", tmp,
 222		(tmp & AT91_UDP_FEN) ? " fen" : "",
 223		(tmp & AT91_UDP_FADD));
 224
 225	proc_irq_show(s, "imr   ", at91_udp_read(udc, AT91_UDP_IMR));
 226	proc_irq_show(s, "isr   ", at91_udp_read(udc, AT91_UDP_ISR));
 227
 228	if (udc->enabled && udc->vbus) {
 229		proc_ep_show(s, &udc->ep[0]);
 230		list_for_each_entry (ep, &udc->gadget.ep_list, ep.ep_list) {
 231			if (ep->ep.desc)
 232				proc_ep_show(s, ep);
 233		}
 234	}
 235	return 0;
 236}
 237
 238static void create_debug_file(struct at91_udc *udc)
 239{
 240	udc->pde = proc_create_single_data(debug_filename, 0, NULL,
 241			proc_udc_show, udc);
 242}
 243
 244static void remove_debug_file(struct at91_udc *udc)
 245{
 246	if (udc->pde)
 247		remove_proc_entry(debug_filename, NULL);
 248}
 249
 250#else
 251
 252static inline void create_debug_file(struct at91_udc *udc) {}
 253static inline void remove_debug_file(struct at91_udc *udc) {}
 254
 255#endif
 256
 257
 258/*-------------------------------------------------------------------------*/
 259
 260static void done(struct at91_ep *ep, struct at91_request *req, int status)
 261{
 262	unsigned	stopped = ep->stopped;
 263	struct at91_udc	*udc = ep->udc;
 264
 265	list_del_init(&req->queue);
 266	if (req->req.status == -EINPROGRESS)
 267		req->req.status = status;
 268	else
 269		status = req->req.status;
 270	if (status && status != -ESHUTDOWN)
 271		VDBG("%s done %p, status %d\n", ep->ep.name, req, status);
 272
 273	ep->stopped = 1;
 274	spin_unlock(&udc->lock);
 275	usb_gadget_giveback_request(&ep->ep, &req->req);
 276	spin_lock(&udc->lock);
 277	ep->stopped = stopped;
 278
 279	/* ep0 is always ready; other endpoints need a non-empty queue */
 280	if (list_empty(&ep->queue) && ep->int_mask != (1 << 0))
 281		at91_udp_write(udc, AT91_UDP_IDR, ep->int_mask);
 282}
 283
 284/*-------------------------------------------------------------------------*/
 285
 286/* bits indicating OUT fifo has data ready */
 287#define	RX_DATA_READY	(AT91_UDP_RX_DATA_BK0 | AT91_UDP_RX_DATA_BK1)
 288
 289/*
 290 * Endpoint FIFO CSR bits have a mix of bits, making it unsafe to just write
 291 * back most of the value you just read (because of side effects, including
 292 * bits that may change after reading and before writing).
 293 *
 294 * Except when changing a specific bit, always write values which:
 295 *  - clear SET_FX bits (setting them could change something)
 296 *  - set CLR_FX bits (clearing them could change something)
 297 *
 298 * There are also state bits like FORCESTALL, EPEDS, DIR, and EPTYPE
 299 * that shouldn't normally be changed.
 300 *
 301 * NOTE at91sam9260 docs mention synch between UDPCK and MCK clock domains,
 302 * implying a need to wait for one write to complete (test relevant bits)
 303 * before starting the next write.  This shouldn't be an issue given how
 304 * infrequently we write, except maybe for write-then-read idioms.
 305 */
 306#define	SET_FX	(AT91_UDP_TXPKTRDY)
 307#define	CLR_FX	(RX_DATA_READY | AT91_UDP_RXSETUP \
 308		| AT91_UDP_STALLSENT | AT91_UDP_TXCOMP)
 309
 310/* pull OUT packet data from the endpoint's fifo */
 311static int read_fifo (struct at91_ep *ep, struct at91_request *req)
 312{
 313	u32 __iomem	*creg = ep->creg;
 314	u8 __iomem	*dreg = ep->creg + (AT91_UDP_FDR(0) - AT91_UDP_CSR(0));
 315	u32		csr;
 316	u8		*buf;
 317	unsigned int	count, bufferspace, is_done;
 318
 319	buf = req->req.buf + req->req.actual;
 320	bufferspace = req->req.length - req->req.actual;
 321
 322	/*
 323	 * there might be nothing to read if ep_queue() calls us,
 324	 * or if we already emptied both pingpong buffers
 325	 */
 326rescan:
 327	csr = __raw_readl(creg);
 328	if ((csr & RX_DATA_READY) == 0)
 329		return 0;
 330
 331	count = (csr & AT91_UDP_RXBYTECNT) >> 16;
 332	if (count > ep->ep.maxpacket)
 333		count = ep->ep.maxpacket;
 334	if (count > bufferspace) {
 335		DBG("%s buffer overflow\n", ep->ep.name);
 336		req->req.status = -EOVERFLOW;
 337		count = bufferspace;
 338	}
 339	__raw_readsb(dreg, buf, count);
 340
 341	/* release and swap pingpong mem bank */
 342	csr |= CLR_FX;
 343	if (ep->is_pingpong) {
 344		if (ep->fifo_bank == 0) {
 345			csr &= ~(SET_FX | AT91_UDP_RX_DATA_BK0);
 346			ep->fifo_bank = 1;
 347		} else {
 348			csr &= ~(SET_FX | AT91_UDP_RX_DATA_BK1);
 349			ep->fifo_bank = 0;
 350		}
 351	} else
 352		csr &= ~(SET_FX | AT91_UDP_RX_DATA_BK0);
 353	__raw_writel(csr, creg);
 354
 355	req->req.actual += count;
 356	is_done = (count < ep->ep.maxpacket);
 357	if (count == bufferspace)
 358		is_done = 1;
 359
 360	PACKET("%s %p out/%d%s\n", ep->ep.name, &req->req, count,
 361			is_done ? " (done)" : "");
 362
 363	/*
 364	 * avoid extra trips through IRQ logic for packets already in
 365	 * the fifo ... maybe preventing an extra (expensive) OUT-NAK
 366	 */
 367	if (is_done)
 368		done(ep, req, 0);
 369	else if (ep->is_pingpong) {
 370		/*
 371		 * One dummy read to delay the code because of a HW glitch:
 372		 * CSR returns bad RXCOUNT when read too soon after updating
 373		 * RX_DATA_BK flags.
 374		 */
 375		csr = __raw_readl(creg);
 376
 377		bufferspace -= count;
 378		buf += count;
 379		goto rescan;
 380	}
 381
 382	return is_done;
 383}
 384
 385/* load fifo for an IN packet */
 386static int write_fifo(struct at91_ep *ep, struct at91_request *req)
 387{
 388	u32 __iomem	*creg = ep->creg;
 389	u32		csr = __raw_readl(creg);
 390	u8 __iomem	*dreg = ep->creg + (AT91_UDP_FDR(0) - AT91_UDP_CSR(0));
 391	unsigned	total, count, is_last;
 392	u8		*buf;
 393
 394	/*
 395	 * TODO: allow for writing two packets to the fifo ... that'll
 396	 * reduce the amount of IN-NAKing, but probably won't affect
 397	 * throughput much.  (Unlike preventing OUT-NAKing!)
 398	 */
 399
 400	/*
 401	 * If ep_queue() calls us, the queue is empty and possibly in
 402	 * odd states like TXCOMP not yet cleared (we do it, saving at
 403	 * least one IRQ) or the fifo not yet being free.  Those aren't
 404	 * issues normally (IRQ handler fast path).
 405	 */
 406	if (unlikely(csr & (AT91_UDP_TXCOMP | AT91_UDP_TXPKTRDY))) {
 407		if (csr & AT91_UDP_TXCOMP) {
 408			csr |= CLR_FX;
 409			csr &= ~(SET_FX | AT91_UDP_TXCOMP);
 410			__raw_writel(csr, creg);
 411			csr = __raw_readl(creg);
 412		}
 413		if (csr & AT91_UDP_TXPKTRDY)
 414			return 0;
 415	}
 416
 417	buf = req->req.buf + req->req.actual;
 418	prefetch(buf);
 419	total = req->req.length - req->req.actual;
 420	if (ep->ep.maxpacket < total) {
 421		count = ep->ep.maxpacket;
 422		is_last = 0;
 423	} else {
 424		count = total;
 425		is_last = (count < ep->ep.maxpacket) || !req->req.zero;
 426	}
 427
 428	/*
 429	 * Write the packet, maybe it's a ZLP.
 430	 *
 431	 * NOTE:  incrementing req->actual before we receive the ACK means
 432	 * gadget driver IN bytecounts can be wrong in fault cases.  That's
 433	 * fixable with PIO drivers like this one (save "count" here, and
 434	 * do the increment later on TX irq), but not for most DMA hardware.
 435	 *
 436	 * So all gadget drivers must accept that potential error.  Some
 437	 * hardware supports precise fifo status reporting, letting them
 438	 * recover when the actual bytecount matters (e.g. for USB Test
 439	 * and Measurement Class devices).
 440	 */
 441	__raw_writesb(dreg, buf, count);
 442	csr &= ~SET_FX;
 443	csr |= CLR_FX | AT91_UDP_TXPKTRDY;
 444	__raw_writel(csr, creg);
 445	req->req.actual += count;
 446
 447	PACKET("%s %p in/%d%s\n", ep->ep.name, &req->req, count,
 448			is_last ? " (done)" : "");
 449	if (is_last)
 450		done(ep, req, 0);
 451	return is_last;
 452}
 453
 454static void nuke(struct at91_ep *ep, int status)
 455{
 456	struct at91_request *req;
 457
 458	/* terminate any request in the queue */
 459	ep->stopped = 1;
 460	if (list_empty(&ep->queue))
 461		return;
 462
 463	VDBG("%s %s\n", __func__, ep->ep.name);
 464	while (!list_empty(&ep->queue)) {
 465		req = list_entry(ep->queue.next, struct at91_request, queue);
 466		done(ep, req, status);
 467	}
 468}
 469
 470/*-------------------------------------------------------------------------*/
 471
 472static int at91_ep_enable(struct usb_ep *_ep,
 473				const struct usb_endpoint_descriptor *desc)
 474{
 475	struct at91_ep	*ep = container_of(_ep, struct at91_ep, ep);
 476	struct at91_udc *udc;
 477	u16		maxpacket;
 478	u32		tmp;
 479	unsigned long	flags;
 480
 481	if (!_ep || !ep
 482			|| !desc || _ep->name == ep0name
 483			|| desc->bDescriptorType != USB_DT_ENDPOINT
 484			|| (maxpacket = usb_endpoint_maxp(desc)) == 0
 485			|| maxpacket > ep->maxpacket) {
 486		DBG("bad ep or descriptor\n");
 487		return -EINVAL;
 488	}
 489
 490	udc = ep->udc;
 491	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN) {
 492		DBG("bogus device state\n");
 493		return -ESHUTDOWN;
 494	}
 495
 496	tmp = usb_endpoint_type(desc);
 497	switch (tmp) {
 498	case USB_ENDPOINT_XFER_CONTROL:
 499		DBG("only one control endpoint\n");
 500		return -EINVAL;
 501	case USB_ENDPOINT_XFER_INT:
 502		if (maxpacket > 64)
 503			goto bogus_max;
 504		break;
 505	case USB_ENDPOINT_XFER_BULK:
 506		switch (maxpacket) {
 507		case 8:
 508		case 16:
 509		case 32:
 510		case 64:
 511			goto ok;
 512		}
 513bogus_max:
 514		DBG("bogus maxpacket %d\n", maxpacket);
 515		return -EINVAL;
 516	case USB_ENDPOINT_XFER_ISOC:
 517		if (!ep->is_pingpong) {
 518			DBG("iso requires double buffering\n");
 519			return -EINVAL;
 520		}
 521		break;
 522	}
 523
 524ok:
 525	spin_lock_irqsave(&udc->lock, flags);
 526
 527	/* initialize endpoint to match this descriptor */
 528	ep->is_in = usb_endpoint_dir_in(desc);
 529	ep->is_iso = (tmp == USB_ENDPOINT_XFER_ISOC);
 530	ep->stopped = 0;
 531	if (ep->is_in)
 532		tmp |= 0x04;
 533	tmp <<= 8;
 534	tmp |= AT91_UDP_EPEDS;
 535	__raw_writel(tmp, ep->creg);
 536
 537	ep->ep.maxpacket = maxpacket;
 538
 539	/*
 540	 * reset/init endpoint fifo.  NOTE:  leaves fifo_bank alone,
 541	 * since endpoint resets don't reset hw pingpong state.
 542	 */
 543	at91_udp_write(udc, AT91_UDP_RST_EP, ep->int_mask);
 544	at91_udp_write(udc, AT91_UDP_RST_EP, 0);
 545
 546	spin_unlock_irqrestore(&udc->lock, flags);
 547	return 0;
 548}
 549
 550static int at91_ep_disable (struct usb_ep * _ep)
 551{
 552	struct at91_ep	*ep = container_of(_ep, struct at91_ep, ep);
 553	struct at91_udc	*udc = ep->udc;
 554	unsigned long	flags;
 555
 556	if (ep == &ep->udc->ep[0])
 557		return -EINVAL;
 558
 559	spin_lock_irqsave(&udc->lock, flags);
 560
 561	nuke(ep, -ESHUTDOWN);
 562
 563	/* restore the endpoint's pristine config */
 564	ep->ep.desc = NULL;
 565	ep->ep.maxpacket = ep->maxpacket;
 566
 567	/* reset fifos and endpoint */
 568	if (ep->udc->clocked) {
 569		at91_udp_write(udc, AT91_UDP_RST_EP, ep->int_mask);
 570		at91_udp_write(udc, AT91_UDP_RST_EP, 0);
 571		__raw_writel(0, ep->creg);
 572	}
 573
 574	spin_unlock_irqrestore(&udc->lock, flags);
 575	return 0;
 576}
 577
 578/*
 579 * this is a PIO-only driver, so there's nothing
 580 * interesting for request or buffer allocation.
 581 */
 582
 583static struct usb_request *
 584at91_ep_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 585{
 586	struct at91_request *req;
 587
 588	req = kzalloc(sizeof (struct at91_request), gfp_flags);
 589	if (!req)
 590		return NULL;
 591
 592	INIT_LIST_HEAD(&req->queue);
 593	return &req->req;
 594}
 595
 596static void at91_ep_free_request(struct usb_ep *_ep, struct usb_request *_req)
 597{
 598	struct at91_request *req;
 599
 600	req = container_of(_req, struct at91_request, req);
 601	BUG_ON(!list_empty(&req->queue));
 602	kfree(req);
 603}
 604
 605static int at91_ep_queue(struct usb_ep *_ep,
 606			struct usb_request *_req, gfp_t gfp_flags)
 607{
 608	struct at91_request	*req;
 609	struct at91_ep		*ep;
 610	struct at91_udc		*udc;
 611	int			status;
 612	unsigned long		flags;
 613
 614	req = container_of(_req, struct at91_request, req);
 615	ep = container_of(_ep, struct at91_ep, ep);
 616
 617	if (!_req || !_req->complete
 618			|| !_req->buf || !list_empty(&req->queue)) {
 619		DBG("invalid request\n");
 620		return -EINVAL;
 621	}
 622
 623	if (!_ep || (!ep->ep.desc && ep->ep.name != ep0name)) {
 624		DBG("invalid ep\n");
 625		return -EINVAL;
 626	}
 627
 628	udc = ep->udc;
 629
 630	if (!udc || !udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN) {
 631		DBG("invalid device\n");
 632		return -EINVAL;
 633	}
 634
 635	_req->status = -EINPROGRESS;
 636	_req->actual = 0;
 637
 638	spin_lock_irqsave(&udc->lock, flags);
 639
 640	/* try to kickstart any empty and idle queue */
 641	if (list_empty(&ep->queue) && !ep->stopped) {
 642		int	is_ep0;
 643
 644		/*
 645		 * If this control request has a non-empty DATA stage, this
 646		 * will start that stage.  It works just like a non-control
 647		 * request (until the status stage starts, maybe early).
 648		 *
 649		 * If the data stage is empty, then this starts a successful
 650		 * IN/STATUS stage.  (Unsuccessful ones use set_halt.)
 651		 */
 652		is_ep0 = (ep->ep.name == ep0name);
 653		if (is_ep0) {
 654			u32	tmp;
 655
 656			if (!udc->req_pending) {
 657				status = -EINVAL;
 658				goto done;
 659			}
 660
 661			/*
 662			 * defer changing CONFG until after the gadget driver
 663			 * reconfigures the endpoints.
 664			 */
 665			if (udc->wait_for_config_ack) {
 666				tmp = at91_udp_read(udc, AT91_UDP_GLB_STAT);
 667				tmp ^= AT91_UDP_CONFG;
 668				VDBG("toggle config\n");
 669				at91_udp_write(udc, AT91_UDP_GLB_STAT, tmp);
 670			}
 671			if (req->req.length == 0) {
 672ep0_in_status:
 673				PACKET("ep0 in/status\n");
 674				status = 0;
 675				tmp = __raw_readl(ep->creg);
 676				tmp &= ~SET_FX;
 677				tmp |= CLR_FX | AT91_UDP_TXPKTRDY;
 678				__raw_writel(tmp, ep->creg);
 679				udc->req_pending = 0;
 680				goto done;
 681			}
 682		}
 683
 684		if (ep->is_in)
 685			status = write_fifo(ep, req);
 686		else {
 687			status = read_fifo(ep, req);
 688
 689			/* IN/STATUS stage is otherwise triggered by irq */
 690			if (status && is_ep0)
 691				goto ep0_in_status;
 692		}
 693	} else
 694		status = 0;
 695
 696	if (req && !status) {
 697		list_add_tail (&req->queue, &ep->queue);
 698		at91_udp_write(udc, AT91_UDP_IER, ep->int_mask);
 699	}
 700done:
 701	spin_unlock_irqrestore(&udc->lock, flags);
 702	return (status < 0) ? status : 0;
 703}
 704
 705static int at91_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
 706{
 707	struct at91_ep		*ep;
 708	struct at91_request	*req = NULL, *iter;
 709	unsigned long		flags;
 710	struct at91_udc		*udc;
 711
 712	ep = container_of(_ep, struct at91_ep, ep);
 713	if (!_ep || ep->ep.name == ep0name)
 714		return -EINVAL;
 715
 716	udc = ep->udc;
 717
 718	spin_lock_irqsave(&udc->lock, flags);
 719
 720	/* make sure it's actually queued on this endpoint */
 721	list_for_each_entry(iter, &ep->queue, queue) {
 722		if (&iter->req != _req)
 723			continue;
 724		req = iter;
 725		break;
 726	}
 727	if (!req) {
 728		spin_unlock_irqrestore(&udc->lock, flags);
 729		return -EINVAL;
 730	}
 731
 732	done(ep, req, -ECONNRESET);
 733	spin_unlock_irqrestore(&udc->lock, flags);
 734	return 0;
 735}
 736
 737static int at91_ep_set_halt(struct usb_ep *_ep, int value)
 738{
 739	struct at91_ep	*ep = container_of(_ep, struct at91_ep, ep);
 740	struct at91_udc	*udc = ep->udc;
 741	u32 __iomem	*creg;
 742	u32		csr;
 743	unsigned long	flags;
 744	int		status = 0;
 745
 746	if (!_ep || ep->is_iso || !ep->udc->clocked)
 747		return -EINVAL;
 748
 749	creg = ep->creg;
 750	spin_lock_irqsave(&udc->lock, flags);
 751
 752	csr = __raw_readl(creg);
 753
 754	/*
 755	 * fail with still-busy IN endpoints, ensuring correct sequencing
 756	 * of data tx then stall.  note that the fifo rx bytecount isn't
 757	 * completely accurate as a tx bytecount.
 758	 */
 759	if (ep->is_in && (!list_empty(&ep->queue) || (csr >> 16) != 0))
 760		status = -EAGAIN;
 761	else {
 762		csr |= CLR_FX;
 763		csr &= ~SET_FX;
 764		if (value) {
 765			csr |= AT91_UDP_FORCESTALL;
 766			VDBG("halt %s\n", ep->ep.name);
 767		} else {
 768			at91_udp_write(udc, AT91_UDP_RST_EP, ep->int_mask);
 769			at91_udp_write(udc, AT91_UDP_RST_EP, 0);
 770			csr &= ~AT91_UDP_FORCESTALL;
 771		}
 772		__raw_writel(csr, creg);
 773	}
 774
 775	spin_unlock_irqrestore(&udc->lock, flags);
 776	return status;
 777}
 778
 779static const struct usb_ep_ops at91_ep_ops = {
 780	.enable		= at91_ep_enable,
 781	.disable	= at91_ep_disable,
 782	.alloc_request	= at91_ep_alloc_request,
 783	.free_request	= at91_ep_free_request,
 784	.queue		= at91_ep_queue,
 785	.dequeue	= at91_ep_dequeue,
 786	.set_halt	= at91_ep_set_halt,
 787	/* there's only imprecise fifo status reporting */
 788};
 789
 790/*-------------------------------------------------------------------------*/
 791
 792static int at91_get_frame(struct usb_gadget *gadget)
 793{
 794	struct at91_udc *udc = to_udc(gadget);
 795
 796	if (!to_udc(gadget)->clocked)
 797		return -EINVAL;
 798	return at91_udp_read(udc, AT91_UDP_FRM_NUM) & AT91_UDP_NUM;
 799}
 800
 801static int at91_wakeup(struct usb_gadget *gadget)
 802{
 803	struct at91_udc	*udc = to_udc(gadget);
 804	u32		glbstate;
 805	unsigned long	flags;
 806
 807	DBG("%s\n", __func__ );
 808	spin_lock_irqsave(&udc->lock, flags);
 809
 810	if (!udc->clocked || !udc->suspended)
 811		goto done;
 812
 813	/* NOTE:  some "early versions" handle ESR differently ... */
 814
 815	glbstate = at91_udp_read(udc, AT91_UDP_GLB_STAT);
 816	if (!(glbstate & AT91_UDP_ESR))
 817		goto done;
 818	glbstate |= AT91_UDP_ESR;
 819	at91_udp_write(udc, AT91_UDP_GLB_STAT, glbstate);
 820
 821done:
 822	spin_unlock_irqrestore(&udc->lock, flags);
 823	return 0;
 824}
 825
 826/* reinit == restore initial software state */
 827static void udc_reinit(struct at91_udc *udc)
 828{
 829	u32 i;
 830
 831	INIT_LIST_HEAD(&udc->gadget.ep_list);
 832	INIT_LIST_HEAD(&udc->gadget.ep0->ep_list);
 833	udc->gadget.quirk_stall_not_supp = 1;
 834
 835	for (i = 0; i < NUM_ENDPOINTS; i++) {
 836		struct at91_ep *ep = &udc->ep[i];
 837
 838		if (i != 0)
 839			list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
 840		ep->ep.desc = NULL;
 841		ep->stopped = 0;
 842		ep->fifo_bank = 0;
 843		usb_ep_set_maxpacket_limit(&ep->ep, ep->maxpacket);
 844		ep->creg = (void __iomem *) udc->udp_baseaddr + AT91_UDP_CSR(i);
 845		/* initialize one queue per endpoint */
 846		INIT_LIST_HEAD(&ep->queue);
 847	}
 848}
 849
 850static void reset_gadget(struct at91_udc *udc)
 851{
 852	struct usb_gadget_driver *driver = udc->driver;
 853	int i;
 854
 855	if (udc->gadget.speed == USB_SPEED_UNKNOWN)
 856		driver = NULL;
 857	udc->gadget.speed = USB_SPEED_UNKNOWN;
 858	udc->suspended = 0;
 859
 860	for (i = 0; i < NUM_ENDPOINTS; i++) {
 861		struct at91_ep *ep = &udc->ep[i];
 862
 863		ep->stopped = 1;
 864		nuke(ep, -ESHUTDOWN);
 865	}
 866	if (driver) {
 867		spin_unlock(&udc->lock);
 868		usb_gadget_udc_reset(&udc->gadget, driver);
 869		spin_lock(&udc->lock);
 870	}
 871
 872	udc_reinit(udc);
 873}
 874
 875static void stop_activity(struct at91_udc *udc)
 876{
 877	struct usb_gadget_driver *driver = udc->driver;
 878	int i;
 879
 880	if (udc->gadget.speed == USB_SPEED_UNKNOWN)
 881		driver = NULL;
 882	udc->gadget.speed = USB_SPEED_UNKNOWN;
 883	udc->suspended = 0;
 884
 885	for (i = 0; i < NUM_ENDPOINTS; i++) {
 886		struct at91_ep *ep = &udc->ep[i];
 887		ep->stopped = 1;
 888		nuke(ep, -ESHUTDOWN);
 889	}
 890	if (driver) {
 891		spin_unlock(&udc->lock);
 892		driver->disconnect(&udc->gadget);
 893		spin_lock(&udc->lock);
 894	}
 895
 896	udc_reinit(udc);
 897}
 898
 899static void clk_on(struct at91_udc *udc)
 900{
 901	if (udc->clocked)
 902		return;
 903	udc->clocked = 1;
 904
 905	clk_enable(udc->iclk);
 906	clk_enable(udc->fclk);
 907}
 908
 909static void clk_off(struct at91_udc *udc)
 910{
 911	if (!udc->clocked)
 912		return;
 913	udc->clocked = 0;
 914	udc->gadget.speed = USB_SPEED_UNKNOWN;
 915	clk_disable(udc->fclk);
 916	clk_disable(udc->iclk);
 917}
 918
 919/*
 920 * activate/deactivate link with host; minimize power usage for
 921 * inactive links by cutting clocks and transceiver power.
 922 */
 923static void pullup(struct at91_udc *udc, int is_on)
 924{
 925	if (!udc->enabled || !udc->vbus)
 926		is_on = 0;
 927	DBG("%sactive\n", is_on ? "" : "in");
 928
 929	if (is_on) {
 930		clk_on(udc);
 931		at91_udp_write(udc, AT91_UDP_ICR, AT91_UDP_RXRSM);
 932		at91_udp_write(udc, AT91_UDP_TXVC, 0);
 933	} else {
 934		stop_activity(udc);
 935		at91_udp_write(udc, AT91_UDP_IDR, AT91_UDP_RXRSM);
 936		at91_udp_write(udc, AT91_UDP_TXVC, AT91_UDP_TXVC_TXVDIS);
 937		clk_off(udc);
 938	}
 939
 940	if (udc->caps && udc->caps->pullup)
 941		udc->caps->pullup(udc, is_on);
 942}
 943
 944/* vbus is here!  turn everything on that's ready */
 945static int at91_vbus_session(struct usb_gadget *gadget, int is_active)
 946{
 947	struct at91_udc	*udc = to_udc(gadget);
 948	unsigned long	flags;
 949
 950	/* VDBG("vbus %s\n", is_active ? "on" : "off"); */
 951	spin_lock_irqsave(&udc->lock, flags);
 952	udc->vbus = (is_active != 0);
 953	if (udc->driver)
 954		pullup(udc, is_active);
 955	else
 956		pullup(udc, 0);
 957	spin_unlock_irqrestore(&udc->lock, flags);
 958	return 0;
 959}
 960
 961static int at91_pullup(struct usb_gadget *gadget, int is_on)
 962{
 963	struct at91_udc	*udc = to_udc(gadget);
 964	unsigned long	flags;
 965
 966	spin_lock_irqsave(&udc->lock, flags);
 967	udc->enabled = is_on = !!is_on;
 968	pullup(udc, is_on);
 969	spin_unlock_irqrestore(&udc->lock, flags);
 970	return 0;
 971}
 972
 973static int at91_set_selfpowered(struct usb_gadget *gadget, int is_on)
 974{
 975	struct at91_udc	*udc = to_udc(gadget);
 976	unsigned long	flags;
 977
 978	spin_lock_irqsave(&udc->lock, flags);
 979	gadget->is_selfpowered = (is_on != 0);
 980	spin_unlock_irqrestore(&udc->lock, flags);
 981	return 0;
 982}
 983
 984static int at91_start(struct usb_gadget *gadget,
 985		struct usb_gadget_driver *driver);
 986static int at91_stop(struct usb_gadget *gadget);
 987
 988static const struct usb_gadget_ops at91_udc_ops = {
 989	.get_frame		= at91_get_frame,
 990	.wakeup			= at91_wakeup,
 991	.set_selfpowered	= at91_set_selfpowered,
 992	.vbus_session		= at91_vbus_session,
 993	.pullup			= at91_pullup,
 994	.udc_start		= at91_start,
 995	.udc_stop		= at91_stop,
 996
 997	/*
 998	 * VBUS-powered devices may also want to support bigger
 999	 * power budgets after an appropriate SET_CONFIGURATION.
1000	 */
1001	/* .vbus_power		= at91_vbus_power, */
1002};
1003
1004/*-------------------------------------------------------------------------*/
1005
1006static int handle_ep(struct at91_ep *ep)
1007{
1008	struct at91_request	*req;
1009	u32 __iomem		*creg = ep->creg;
1010	u32			csr = __raw_readl(creg);
1011
1012	if (!list_empty(&ep->queue))
1013		req = list_entry(ep->queue.next,
1014			struct at91_request, queue);
1015	else
1016		req = NULL;
1017
1018	if (ep->is_in) {
1019		if (csr & (AT91_UDP_STALLSENT | AT91_UDP_TXCOMP)) {
1020			csr |= CLR_FX;
1021			csr &= ~(SET_FX | AT91_UDP_STALLSENT | AT91_UDP_TXCOMP);
1022			__raw_writel(csr, creg);
1023		}
1024		if (req)
1025			return write_fifo(ep, req);
1026
1027	} else {
1028		if (csr & AT91_UDP_STALLSENT) {
1029			/* STALLSENT bit == ISOERR */
1030			if (ep->is_iso && req)
1031				req->req.status = -EILSEQ;
1032			csr |= CLR_FX;
1033			csr &= ~(SET_FX | AT91_UDP_STALLSENT);
1034			__raw_writel(csr, creg);
1035			csr = __raw_readl(creg);
1036		}
1037		if (req && (csr & RX_DATA_READY))
1038			return read_fifo(ep, req);
1039	}
1040	return 0;
1041}
1042
1043union setup {
1044	u8			raw[8];
1045	struct usb_ctrlrequest	r;
1046};
1047
1048static void handle_setup(struct at91_udc *udc, struct at91_ep *ep, u32 csr)
1049{
1050	u32 __iomem	*creg = ep->creg;
1051	u8 __iomem	*dreg = ep->creg + (AT91_UDP_FDR(0) - AT91_UDP_CSR(0));
1052	unsigned	rxcount, i = 0;
1053	u32		tmp;
1054	union setup	pkt;
1055	int		status = 0;
1056
1057	/* read and ack SETUP; hard-fail for bogus packets */
1058	rxcount = (csr & AT91_UDP_RXBYTECNT) >> 16;
1059	if (likely(rxcount == 8)) {
1060		while (rxcount--)
1061			pkt.raw[i++] = __raw_readb(dreg);
1062		if (pkt.r.bRequestType & USB_DIR_IN) {
1063			csr |= AT91_UDP_DIR;
1064			ep->is_in = 1;
1065		} else {
1066			csr &= ~AT91_UDP_DIR;
1067			ep->is_in = 0;
1068		}
1069	} else {
1070		/* REVISIT this happens sometimes under load; why?? */
1071		ERR("SETUP len %d, csr %08x\n", rxcount, csr);
1072		status = -EINVAL;
1073	}
1074	csr |= CLR_FX;
1075	csr &= ~(SET_FX | AT91_UDP_RXSETUP);
1076	__raw_writel(csr, creg);
1077	udc->wait_for_addr_ack = 0;
1078	udc->wait_for_config_ack = 0;
1079	ep->stopped = 0;
1080	if (unlikely(status != 0))
1081		goto stall;
1082
1083#define w_index		le16_to_cpu(pkt.r.wIndex)
1084#define w_value		le16_to_cpu(pkt.r.wValue)
1085#define w_length	le16_to_cpu(pkt.r.wLength)
1086
1087	VDBG("SETUP %02x.%02x v%04x i%04x l%04x\n",
1088			pkt.r.bRequestType, pkt.r.bRequest,
1089			w_value, w_index, w_length);
1090
1091	/*
1092	 * A few standard requests get handled here, ones that touch
1093	 * hardware ... notably for device and endpoint features.
1094	 */
1095	udc->req_pending = 1;
1096	csr = __raw_readl(creg);
1097	csr |= CLR_FX;
1098	csr &= ~SET_FX;
1099	switch ((pkt.r.bRequestType << 8) | pkt.r.bRequest) {
1100
1101	case ((USB_TYPE_STANDARD|USB_RECIP_DEVICE) << 8)
1102			| USB_REQ_SET_ADDRESS:
1103		__raw_writel(csr | AT91_UDP_TXPKTRDY, creg);
1104		udc->addr = w_value;
1105		udc->wait_for_addr_ack = 1;
1106		udc->req_pending = 0;
1107		/* FADDR is set later, when we ack host STATUS */
1108		return;
1109
1110	case ((USB_TYPE_STANDARD|USB_RECIP_DEVICE) << 8)
1111			| USB_REQ_SET_CONFIGURATION:
1112		tmp = at91_udp_read(udc, AT91_UDP_GLB_STAT) & AT91_UDP_CONFG;
1113		if (pkt.r.wValue)
1114			udc->wait_for_config_ack = (tmp == 0);
1115		else
1116			udc->wait_for_config_ack = (tmp != 0);
1117		if (udc->wait_for_config_ack)
1118			VDBG("wait for config\n");
1119		/* CONFG is toggled later, if gadget driver succeeds */
1120		break;
1121
1122	/*
1123	 * Hosts may set or clear remote wakeup status, and
1124	 * devices may report they're VBUS powered.
1125	 */
1126	case ((USB_DIR_IN|USB_TYPE_STANDARD|USB_RECIP_DEVICE) << 8)
1127			| USB_REQ_GET_STATUS:
1128		tmp = (udc->gadget.is_selfpowered << USB_DEVICE_SELF_POWERED);
1129		if (at91_udp_read(udc, AT91_UDP_GLB_STAT) & AT91_UDP_ESR)
1130			tmp |= (1 << USB_DEVICE_REMOTE_WAKEUP);
1131		PACKET("get device status\n");
1132		__raw_writeb(tmp, dreg);
1133		__raw_writeb(0, dreg);
1134		goto write_in;
1135		/* then STATUS starts later, automatically */
1136	case ((USB_TYPE_STANDARD|USB_RECIP_DEVICE) << 8)
1137			| USB_REQ_SET_FEATURE:
1138		if (w_value != USB_DEVICE_REMOTE_WAKEUP)
1139			goto stall;
1140		tmp = at91_udp_read(udc, AT91_UDP_GLB_STAT);
1141		tmp |= AT91_UDP_ESR;
1142		at91_udp_write(udc, AT91_UDP_GLB_STAT, tmp);
1143		goto succeed;
1144	case ((USB_TYPE_STANDARD|USB_RECIP_DEVICE) << 8)
1145			| USB_REQ_CLEAR_FEATURE:
1146		if (w_value != USB_DEVICE_REMOTE_WAKEUP)
1147			goto stall;
1148		tmp = at91_udp_read(udc, AT91_UDP_GLB_STAT);
1149		tmp &= ~AT91_UDP_ESR;
1150		at91_udp_write(udc, AT91_UDP_GLB_STAT, tmp);
1151		goto succeed;
1152
1153	/*
1154	 * Interfaces have no feature settings; this is pretty useless.
1155	 * we won't even insist the interface exists...
1156	 */
1157	case ((USB_DIR_IN|USB_TYPE_STANDARD|USB_RECIP_INTERFACE) << 8)
1158			| USB_REQ_GET_STATUS:
1159		PACKET("get interface status\n");
1160		__raw_writeb(0, dreg);
1161		__raw_writeb(0, dreg);
1162		goto write_in;
1163		/* then STATUS starts later, automatically */
1164	case ((USB_TYPE_STANDARD|USB_RECIP_INTERFACE) << 8)
1165			| USB_REQ_SET_FEATURE:
1166	case ((USB_TYPE_STANDARD|USB_RECIP_INTERFACE) << 8)
1167			| USB_REQ_CLEAR_FEATURE:
1168		goto stall;
1169
1170	/*
1171	 * Hosts may clear bulk/intr endpoint halt after the gadget
1172	 * driver sets it (not widely used); or set it (for testing)
1173	 */
1174	case ((USB_DIR_IN|USB_TYPE_STANDARD|USB_RECIP_ENDPOINT) << 8)
1175			| USB_REQ_GET_STATUS:
1176		tmp = w_index & USB_ENDPOINT_NUMBER_MASK;
1177		ep = &udc->ep[tmp];
1178		if (tmp >= NUM_ENDPOINTS || (tmp && !ep->ep.desc))
1179			goto stall;
1180
1181		if (tmp) {
1182			if ((w_index & USB_DIR_IN)) {
1183				if (!ep->is_in)
1184					goto stall;
1185			} else if (ep->is_in)
1186				goto stall;
1187		}
1188		PACKET("get %s status\n", ep->ep.name);
1189		if (__raw_readl(ep->creg) & AT91_UDP_FORCESTALL)
1190			tmp = (1 << USB_ENDPOINT_HALT);
1191		else
1192			tmp = 0;
1193		__raw_writeb(tmp, dreg);
1194		__raw_writeb(0, dreg);
1195		goto write_in;
1196		/* then STATUS starts later, automatically */
1197	case ((USB_TYPE_STANDARD|USB_RECIP_ENDPOINT) << 8)
1198			| USB_REQ_SET_FEATURE:
1199		tmp = w_index & USB_ENDPOINT_NUMBER_MASK;
1200		ep = &udc->ep[tmp];
1201		if (w_value != USB_ENDPOINT_HALT || tmp >= NUM_ENDPOINTS)
1202			goto stall;
1203		if (!ep->ep.desc || ep->is_iso)
1204			goto stall;
1205		if ((w_index & USB_DIR_IN)) {
1206			if (!ep->is_in)
1207				goto stall;
1208		} else if (ep->is_in)
1209			goto stall;
1210
1211		tmp = __raw_readl(ep->creg);
1212		tmp &= ~SET_FX;
1213		tmp |= CLR_FX | AT91_UDP_FORCESTALL;
1214		__raw_writel(tmp, ep->creg);
1215		goto succeed;
1216	case ((USB_TYPE_STANDARD|USB_RECIP_ENDPOINT) << 8)
1217			| USB_REQ_CLEAR_FEATURE:
1218		tmp = w_index & USB_ENDPOINT_NUMBER_MASK;
1219		ep = &udc->ep[tmp];
1220		if (w_value != USB_ENDPOINT_HALT || tmp >= NUM_ENDPOINTS)
1221			goto stall;
1222		if (tmp == 0)
1223			goto succeed;
1224		if (!ep->ep.desc || ep->is_iso)
1225			goto stall;
1226		if ((w_index & USB_DIR_IN)) {
1227			if (!ep->is_in)
1228				goto stall;
1229		} else if (ep->is_in)
1230			goto stall;
1231
1232		at91_udp_write(udc, AT91_UDP_RST_EP, ep->int_mask);
1233		at91_udp_write(udc, AT91_UDP_RST_EP, 0);
1234		tmp = __raw_readl(ep->creg);
1235		tmp |= CLR_FX;
1236		tmp &= ~(SET_FX | AT91_UDP_FORCESTALL);
1237		__raw_writel(tmp, ep->creg);
1238		if (!list_empty(&ep->queue))
1239			handle_ep(ep);
1240		goto succeed;
1241	}
1242
1243#undef w_value
1244#undef w_index
1245#undef w_length
1246
1247	/* pass request up to the gadget driver */
1248	if (udc->driver) {
1249		spin_unlock(&udc->lock);
1250		status = udc->driver->setup(&udc->gadget, &pkt.r);
1251		spin_lock(&udc->lock);
1252	}
1253	else
1254		status = -ENODEV;
1255	if (status < 0) {
1256stall:
1257		VDBG("req %02x.%02x protocol STALL; stat %d\n",
1258				pkt.r.bRequestType, pkt.r.bRequest, status);
1259		csr |= AT91_UDP_FORCESTALL;
1260		__raw_writel(csr, creg);
1261		udc->req_pending = 0;
1262	}
1263	return;
1264
1265succeed:
1266	/* immediate successful (IN) STATUS after zero length DATA */
1267	PACKET("ep0 in/status\n");
1268write_in:
1269	csr |= AT91_UDP_TXPKTRDY;
1270	__raw_writel(csr, creg);
1271	udc->req_pending = 0;
1272}
1273
1274static void handle_ep0(struct at91_udc *udc)
1275{
1276	struct at91_ep		*ep0 = &udc->ep[0];
1277	u32 __iomem		*creg = ep0->creg;
1278	u32			csr = __raw_readl(creg);
1279	struct at91_request	*req;
1280
1281	if (unlikely(csr & AT91_UDP_STALLSENT)) {
1282		nuke(ep0, -EPROTO);
1283		udc->req_pending = 0;
1284		csr |= CLR_FX;
1285		csr &= ~(SET_FX | AT91_UDP_STALLSENT | AT91_UDP_FORCESTALL);
1286		__raw_writel(csr, creg);
1287		VDBG("ep0 stalled\n");
1288		csr = __raw_readl(creg);
1289	}
1290	if (csr & AT91_UDP_RXSETUP) {
1291		nuke(ep0, 0);
1292		udc->req_pending = 0;
1293		handle_setup(udc, ep0, csr);
1294		return;
1295	}
1296
1297	if (list_empty(&ep0->queue))
1298		req = NULL;
1299	else
1300		req = list_entry(ep0->queue.next, struct at91_request, queue);
1301
1302	/* host ACKed an IN packet that we sent */
1303	if (csr & AT91_UDP_TXCOMP) {
1304		csr |= CLR_FX;
1305		csr &= ~(SET_FX | AT91_UDP_TXCOMP);
1306
1307		/* write more IN DATA? */
1308		if (req && ep0->is_in) {
1309			if (handle_ep(ep0))
1310				udc->req_pending = 0;
1311
1312		/*
1313		 * Ack after:
1314		 *  - last IN DATA packet (including GET_STATUS)
1315		 *  - IN/STATUS for OUT DATA
1316		 *  - IN/STATUS for any zero-length DATA stage
1317		 * except for the IN DATA case, the host should send
1318		 * an OUT status later, which we'll ack.
1319		 */
1320		} else {
1321			udc->req_pending = 0;
1322			__raw_writel(csr, creg);
1323
1324			/*
1325			 * SET_ADDRESS takes effect only after the STATUS
1326			 * (to the original address) gets acked.
1327			 */
1328			if (udc->wait_for_addr_ack) {
1329				u32	tmp;
1330
1331				at91_udp_write(udc, AT91_UDP_FADDR,
1332						AT91_UDP_FEN | udc->addr);
1333				tmp = at91_udp_read(udc, AT91_UDP_GLB_STAT);
1334				tmp &= ~AT91_UDP_FADDEN;
1335				if (udc->addr)
1336					tmp |= AT91_UDP_FADDEN;
1337				at91_udp_write(udc, AT91_UDP_GLB_STAT, tmp);
1338
1339				udc->wait_for_addr_ack = 0;
1340				VDBG("address %d\n", udc->addr);
1341			}
1342		}
1343	}
1344
1345	/* OUT packet arrived ... */
1346	else if (csr & AT91_UDP_RX_DATA_BK0) {
1347		csr |= CLR_FX;
1348		csr &= ~(SET_FX | AT91_UDP_RX_DATA_BK0);
1349
1350		/* OUT DATA stage */
1351		if (!ep0->is_in) {
1352			if (req) {
1353				if (handle_ep(ep0)) {
1354					/* send IN/STATUS */
1355					PACKET("ep0 in/status\n");
1356					csr = __raw_readl(creg);
1357					csr &= ~SET_FX;
1358					csr |= CLR_FX | AT91_UDP_TXPKTRDY;
1359					__raw_writel(csr, creg);
1360					udc->req_pending = 0;
1361				}
1362			} else if (udc->req_pending) {
1363				/*
1364				 * AT91 hardware has a hard time with this
1365				 * "deferred response" mode for control-OUT
1366				 * transfers.  (For control-IN it's fine.)
1367				 *
1368				 * The normal solution leaves OUT data in the
1369				 * fifo until the gadget driver is ready.
1370				 * We couldn't do that here without disabling
1371				 * the IRQ that tells about SETUP packets,
1372				 * e.g. when the host gets impatient...
1373				 *
1374				 * Working around it by copying into a buffer
1375				 * would almost be a non-deferred response,
1376				 * except that it wouldn't permit reliable
1377				 * stalling of the request.  Instead, demand
1378				 * that gadget drivers not use this mode.
1379				 */
1380				DBG("no control-OUT deferred responses!\n");
1381				__raw_writel(csr | AT91_UDP_FORCESTALL, creg);
1382				udc->req_pending = 0;
1383			}
1384
1385		/* STATUS stage for control-IN; ack.  */
1386		} else {
1387			PACKET("ep0 out/status ACK\n");
1388			__raw_writel(csr, creg);
1389
1390			/* "early" status stage */
1391			if (req)
1392				done(ep0, req, 0);
1393		}
1394	}
1395}
1396
1397static irqreturn_t at91_udc_irq (int irq, void *_udc)
1398{
1399	struct at91_udc		*udc = _udc;
1400	u32			rescans = 5;
1401	int			disable_clock = 0;
1402	unsigned long		flags;
1403
1404	spin_lock_irqsave(&udc->lock, flags);
1405
1406	if (!udc->clocked) {
1407		clk_on(udc);
1408		disable_clock = 1;
1409	}
1410
1411	while (rescans--) {
1412		u32 status;
1413
1414		status = at91_udp_read(udc, AT91_UDP_ISR)
1415			& at91_udp_read(udc, AT91_UDP_IMR);
1416		if (!status)
1417			break;
1418
1419		/* USB reset irq:  not maskable */
1420		if (status & AT91_UDP_ENDBUSRES) {
1421			at91_udp_write(udc, AT91_UDP_IDR, ~MINIMUS_INTERRUPTUS);
1422			at91_udp_write(udc, AT91_UDP_IER, MINIMUS_INTERRUPTUS);
1423			/* Atmel code clears this irq twice */
1424			at91_udp_write(udc, AT91_UDP_ICR, AT91_UDP_ENDBUSRES);
1425			at91_udp_write(udc, AT91_UDP_ICR, AT91_UDP_ENDBUSRES);
1426			VDBG("end bus reset\n");
1427			udc->addr = 0;
1428			reset_gadget(udc);
1429
1430			/* enable ep0 */
1431			at91_udp_write(udc, AT91_UDP_CSR(0),
1432					AT91_UDP_EPEDS | AT91_UDP_EPTYPE_CTRL);
1433			udc->gadget.speed = USB_SPEED_FULL;
1434			udc->suspended = 0;
1435			at91_udp_write(udc, AT91_UDP_IER, AT91_UDP_EP(0));
1436
1437			/*
1438			 * NOTE:  this driver keeps clocks off unless the
1439			 * USB host is present.  That saves power, but for
1440			 * boards that don't support VBUS detection, both
1441			 * clocks need to be active most of the time.
1442			 */
1443
1444		/* host initiated suspend (3+ms bus idle) */
1445		} else if (status & AT91_UDP_RXSUSP) {
1446			at91_udp_write(udc, AT91_UDP_IDR, AT91_UDP_RXSUSP);
1447			at91_udp_write(udc, AT91_UDP_IER, AT91_UDP_RXRSM);
1448			at91_udp_write(udc, AT91_UDP_ICR, AT91_UDP_RXSUSP);
1449			/* VDBG("bus suspend\n"); */
1450			if (udc->suspended)
1451				continue;
1452			udc->suspended = 1;
1453
1454			/*
1455			 * NOTE:  when suspending a VBUS-powered device, the
1456			 * gadget driver should switch into slow clock mode
1457			 * and then into standby to avoid drawing more than
1458			 * 500uA power (2500uA for some high-power configs).
1459			 */
1460			if (udc->driver && udc->driver->suspend) {
1461				spin_unlock(&udc->lock);
1462				udc->driver->suspend(&udc->gadget);
1463				spin_lock(&udc->lock);
1464			}
1465
1466		/* host initiated resume */
1467		} else if (status & AT91_UDP_RXRSM) {
1468			at91_udp_write(udc, AT91_UDP_IDR, AT91_UDP_RXRSM);
1469			at91_udp_write(udc, AT91_UDP_IER, AT91_UDP_RXSUSP);
1470			at91_udp_write(udc, AT91_UDP_ICR, AT91_UDP_RXRSM);
1471			/* VDBG("bus resume\n"); */
1472			if (!udc->suspended)
1473				continue;
1474			udc->suspended = 0;
1475
1476			/*
1477			 * NOTE:  for a VBUS-powered device, the gadget driver
1478			 * would normally want to switch out of slow clock
1479			 * mode into normal mode.
1480			 */
1481			if (udc->driver && udc->driver->resume) {
1482				spin_unlock(&udc->lock);
1483				udc->driver->resume(&udc->gadget);
1484				spin_lock(&udc->lock);
1485			}
1486
1487		/* endpoint IRQs are cleared by handling them */
1488		} else {
1489			int		i;
1490			unsigned	mask = 1;
1491			struct at91_ep	*ep = &udc->ep[1];
1492
1493			if (status & mask)
1494				handle_ep0(udc);
1495			for (i = 1; i < NUM_ENDPOINTS; i++) {
1496				mask <<= 1;
1497				if (status & mask)
1498					handle_ep(ep);
1499				ep++;
1500			}
1501		}
1502	}
1503
1504	if (disable_clock)
1505		clk_off(udc);
1506
1507	spin_unlock_irqrestore(&udc->lock, flags);
1508
1509	return IRQ_HANDLED;
1510}
1511
1512/*-------------------------------------------------------------------------*/
1513
1514static void at91_vbus_update(struct at91_udc *udc, unsigned value)
1515{
1516	if (value != udc->vbus)
1517		at91_vbus_session(&udc->gadget, value);
1518}
1519
1520static irqreturn_t at91_vbus_irq(int irq, void *_udc)
1521{
1522	struct at91_udc	*udc = _udc;
1523
1524	/* vbus needs at least brief debouncing */
1525	udelay(10);
1526	at91_vbus_update(udc, gpiod_get_value(udc->board.vbus_pin));
1527
1528	return IRQ_HANDLED;
1529}
1530
1531static void at91_vbus_timer_work(struct work_struct *work)
1532{
1533	struct at91_udc *udc = container_of(work, struct at91_udc,
1534					    vbus_timer_work);
1535
1536	at91_vbus_update(udc, gpiod_get_value_cansleep(udc->board.vbus_pin));
1537
1538	if (!timer_pending(&udc->vbus_timer))
1539		mod_timer(&udc->vbus_timer, jiffies + VBUS_POLL_TIMEOUT);
1540}
1541
1542static void at91_vbus_timer(struct timer_list *t)
1543{
1544	struct at91_udc *udc = timer_container_of(udc, t, vbus_timer);
1545
1546	/*
1547	 * If we are polling vbus it is likely that the gpio is on an
1548	 * bus such as i2c or spi which may sleep, so schedule some work
1549	 * to read the vbus gpio
1550	 */
1551	schedule_work(&udc->vbus_timer_work);
1552}
1553
1554static int at91_start(struct usb_gadget *gadget,
1555		struct usb_gadget_driver *driver)
1556{
1557	struct at91_udc	*udc;
1558
1559	udc = container_of(gadget, struct at91_udc, gadget);
1560	udc->driver = driver;
1561	udc->gadget.dev.of_node = udc->pdev->dev.of_node;
1562	udc->enabled = 1;
1563	udc->gadget.is_selfpowered = 1;
1564
1565	return 0;
1566}
1567
1568static int at91_stop(struct usb_gadget *gadget)
1569{
1570	struct at91_udc *udc;
1571	unsigned long	flags;
1572
1573	udc = container_of(gadget, struct at91_udc, gadget);
1574	spin_lock_irqsave(&udc->lock, flags);
1575	udc->enabled = 0;
1576	at91_udp_write(udc, AT91_UDP_IDR, ~0);
1577	spin_unlock_irqrestore(&udc->lock, flags);
1578
1579	udc->driver = NULL;
1580
1581	return 0;
1582}
1583
1584/*-------------------------------------------------------------------------*/
1585
1586static void at91udc_shutdown(struct platform_device *dev)
1587{
1588	struct at91_udc *udc = platform_get_drvdata(dev);
1589	unsigned long	flags;
1590
1591	/* force disconnect on reboot */
1592	spin_lock_irqsave(&udc->lock, flags);
1593	pullup(platform_get_drvdata(dev), 0);
1594	spin_unlock_irqrestore(&udc->lock, flags);
1595}
1596
1597static int at91rm9200_udc_init(struct at91_udc *udc)
1598{
1599	struct at91_ep *ep;
1600	int i;
1601
1602	for (i = 0; i < NUM_ENDPOINTS; i++) {
1603		ep = &udc->ep[i];
1604
1605		switch (i) {
1606		case 0:
1607		case 3:
1608			ep->maxpacket = 8;
1609			break;
1610		case 1 ... 2:
1611			ep->maxpacket = 64;
1612			break;
1613		case 4 ... 5:
1614			ep->maxpacket = 256;
1615			break;
1616		}
1617	}
1618
1619	if (!udc->board.pullup_pin) {
1620		DBG("no D+ pullup?\n");
1621		return -ENODEV;
1622	}
1623
1624	gpiod_direction_output(udc->board.pullup_pin,
1625			       gpiod_is_active_low(udc->board.pullup_pin));
1626
1627	return 0;
1628}
1629
1630static void at91rm9200_udc_pullup(struct at91_udc *udc, int is_on)
1631{
1632	gpiod_set_value(udc->board.pullup_pin, is_on);
1633}
1634
1635static const struct at91_udc_caps at91rm9200_udc_caps = {
1636	.init = at91rm9200_udc_init,
1637	.pullup = at91rm9200_udc_pullup,
1638};
1639
1640static int at91sam9260_udc_init(struct at91_udc *udc)
1641{
1642	struct at91_ep *ep;
1643	int i;
1644
1645	for (i = 0; i < NUM_ENDPOINTS; i++) {
1646		ep = &udc->ep[i];
1647
1648		switch (i) {
1649		case 0 ... 3:
1650			ep->maxpacket = 64;
1651			break;
1652		case 4 ... 5:
1653			ep->maxpacket = 512;
1654			break;
1655		}
1656	}
1657
1658	return 0;
1659}
1660
1661static void at91sam9260_udc_pullup(struct at91_udc *udc, int is_on)
1662{
1663	u32 txvc = at91_udp_read(udc, AT91_UDP_TXVC);
1664
1665	if (is_on)
1666		txvc |= AT91_UDP_TXVC_PUON;
1667	else
1668		txvc &= ~AT91_UDP_TXVC_PUON;
1669
1670	at91_udp_write(udc, AT91_UDP_TXVC, txvc);
1671}
1672
1673static const struct at91_udc_caps at91sam9260_udc_caps = {
1674	.init = at91sam9260_udc_init,
1675	.pullup = at91sam9260_udc_pullup,
1676};
1677
1678static int at91sam9261_udc_init(struct at91_udc *udc)
1679{
1680	struct at91_ep *ep;
1681	int i;
1682
1683	for (i = 0; i < NUM_ENDPOINTS; i++) {
1684		ep = &udc->ep[i];
1685
1686		switch (i) {
1687		case 0:
1688			ep->maxpacket = 8;
1689			break;
1690		case 1 ... 3:
1691			ep->maxpacket = 64;
1692			break;
1693		case 4 ... 5:
1694			ep->maxpacket = 256;
1695			break;
1696		}
1697	}
1698
1699	udc->matrix = syscon_regmap_lookup_by_phandle(udc->pdev->dev.of_node,
1700						      "atmel,matrix");
1701	return PTR_ERR_OR_ZERO(udc->matrix);
1702}
1703
1704static void at91sam9261_udc_pullup(struct at91_udc *udc, int is_on)
1705{
1706	u32 usbpucr = 0;
1707
1708	if (is_on)
1709		usbpucr = AT91_MATRIX_USBPUCR_PUON;
1710
1711	regmap_update_bits(udc->matrix, AT91SAM9261_MATRIX_USBPUCR,
1712			   AT91_MATRIX_USBPUCR_PUON, usbpucr);
1713}
1714
1715static const struct at91_udc_caps at91sam9261_udc_caps = {
1716	.init = at91sam9261_udc_init,
1717	.pullup = at91sam9261_udc_pullup,
1718};
1719
1720static int at91sam9263_udc_init(struct at91_udc *udc)
1721{
1722	struct at91_ep *ep;
1723	int i;
1724
1725	for (i = 0; i < NUM_ENDPOINTS; i++) {
1726		ep = &udc->ep[i];
1727
1728		switch (i) {
1729		case 0:
1730		case 1:
1731		case 2:
1732		case 3:
1733			ep->maxpacket = 64;
1734			break;
1735		case 4:
1736		case 5:
1737			ep->maxpacket = 256;
1738			break;
1739		}
1740	}
1741
1742	return 0;
1743}
1744
1745static const struct at91_udc_caps at91sam9263_udc_caps = {
1746	.init = at91sam9263_udc_init,
1747	.pullup = at91sam9260_udc_pullup,
1748};
1749
1750static const struct of_device_id at91_udc_dt_ids[] = {
1751	{
1752		.compatible = "atmel,at91rm9200-udc",
1753		.data = &at91rm9200_udc_caps,
1754	},
1755	{
1756		.compatible = "atmel,at91sam9260-udc",
1757		.data = &at91sam9260_udc_caps,
1758	},
1759	{
1760		.compatible = "atmel,at91sam9261-udc",
1761		.data = &at91sam9261_udc_caps,
1762	},
1763	{
1764		.compatible = "atmel,at91sam9263-udc",
1765		.data = &at91sam9263_udc_caps,
1766	},
1767	{ /* sentinel */ }
1768};
1769MODULE_DEVICE_TABLE(of, at91_udc_dt_ids);
1770
1771static void at91udc_of_init(struct at91_udc *udc, struct device_node *np)
1772{
1773	struct at91_udc_data *board = &udc->board;
1774	const struct of_device_id *match;
1775	u32 val;
1776
1777	if (of_property_read_u32(np, "atmel,vbus-polled", &val) == 0)
1778		board->vbus_polled = 1;
1779
1780	board->vbus_pin = fwnode_gpiod_get_index(of_fwnode_handle(np),
1781						 "atmel,vbus", 0, GPIOD_IN,
1782						 "udc_vbus");
1783	if (IS_ERR(board->vbus_pin))
1784		board->vbus_pin = NULL;
1785
1786	board->pullup_pin = fwnode_gpiod_get_index(of_fwnode_handle(np),
1787						   "atmel,pullup", 0,
1788						   GPIOD_ASIS, "udc_pullup");
1789	if (IS_ERR(board->pullup_pin))
1790		board->pullup_pin = NULL;
1791
1792	match = of_match_node(at91_udc_dt_ids, np);
1793	if (match)
1794		udc->caps = match->data;
1795}
1796
1797static int at91udc_probe(struct platform_device *pdev)
1798{
1799	struct device	*dev = &pdev->dev;
1800	struct at91_udc	*udc;
1801	int		retval;
1802	struct at91_ep	*ep;
1803	int		i;
1804
1805	udc = devm_kzalloc(dev, sizeof(*udc), GFP_KERNEL);
1806	if (!udc)
1807		return -ENOMEM;
1808
1809	/* init software state */
1810	udc->gadget.dev.parent = dev;
1811	at91udc_of_init(udc, pdev->dev.of_node);
1812	udc->pdev = pdev;
1813	udc->enabled = 0;
1814	spin_lock_init(&udc->lock);
1815
1816	udc->gadget.ops = &at91_udc_ops;
1817	udc->gadget.ep0 = &udc->ep[0].ep;
1818	udc->gadget.name = driver_name;
1819	udc->gadget.dev.init_name = "gadget";
1820
1821	for (i = 0; i < NUM_ENDPOINTS; i++) {
1822		ep = &udc->ep[i];
1823		ep->ep.name = ep_info[i].name;
1824		ep->ep.caps = ep_info[i].caps;
1825		ep->ep.ops = &at91_ep_ops;
1826		ep->udc = udc;
1827		ep->int_mask = BIT(i);
1828		if (i != 0 && i != 3)
1829			ep->is_pingpong = 1;
1830	}
1831
1832	udc->udp_baseaddr = devm_platform_ioremap_resource(pdev, 0);
1833	if (IS_ERR(udc->udp_baseaddr))
1834		return PTR_ERR(udc->udp_baseaddr);
1835
1836	if (udc->caps && udc->caps->init) {
1837		retval = udc->caps->init(udc);
1838		if (retval)
1839			return retval;
1840	}
1841
1842	udc_reinit(udc);
1843
1844	/* get interface and function clocks */
1845	udc->iclk = devm_clk_get(dev, "pclk");
1846	if (IS_ERR(udc->iclk))
1847		return PTR_ERR(udc->iclk);
1848
1849	udc->fclk = devm_clk_get(dev, "hclk");
1850	if (IS_ERR(udc->fclk))
1851		return PTR_ERR(udc->fclk);
1852
1853	/* don't do anything until we have both gadget driver and VBUS */
1854	clk_set_rate(udc->fclk, 48000000);
1855	retval = clk_prepare(udc->fclk);
1856	if (retval)
1857		return retval;
1858
1859	retval = clk_prepare_enable(udc->iclk);
1860	if (retval)
1861		goto err_unprepare_fclk;
1862
1863	at91_udp_write(udc, AT91_UDP_TXVC, AT91_UDP_TXVC_TXVDIS);
1864	at91_udp_write(udc, AT91_UDP_IDR, 0xffffffff);
1865	/* Clear all pending interrupts - UDP may be used by bootloader. */
1866	at91_udp_write(udc, AT91_UDP_ICR, 0xffffffff);
1867	clk_disable(udc->iclk);
1868
1869	/* request UDC and maybe VBUS irqs */
1870	udc->udp_irq = retval = platform_get_irq(pdev, 0);
1871	if (retval < 0)
1872		goto err_unprepare_iclk;
1873	retval = devm_request_irq(dev, udc->udp_irq, at91_udc_irq, 0,
1874				  driver_name, udc);
1875	if (retval) {
1876		DBG("request irq %d failed\n", udc->udp_irq);
1877		goto err_unprepare_iclk;
1878	}
1879
1880	if (udc->board.vbus_pin) {
1881		gpiod_direction_input(udc->board.vbus_pin);
1882
1883		/*
1884		 * Get the initial state of VBUS - we cannot expect
1885		 * a pending interrupt.
1886		 */
1887		udc->vbus = gpiod_get_value_cansleep(udc->board.vbus_pin);
1888
1889		if (udc->board.vbus_polled) {
1890			INIT_WORK(&udc->vbus_timer_work, at91_vbus_timer_work);
1891			timer_setup(&udc->vbus_timer, at91_vbus_timer, 0);
1892			mod_timer(&udc->vbus_timer,
1893				  jiffies + VBUS_POLL_TIMEOUT);
1894		} else {
1895			retval = devm_request_irq(dev,
1896					gpiod_to_irq(udc->board.vbus_pin),
1897					at91_vbus_irq, 0, driver_name, udc);
1898			if (retval) {
1899				DBG("request vbus irq %d failed\n",
1900				    desc_to_gpio(udc->board.vbus_pin));
1901				goto err_unprepare_iclk;
1902			}
1903		}
1904	} else {
1905		DBG("no VBUS detection, assuming always-on\n");
1906		udc->vbus = 1;
1907	}
1908	retval = usb_add_gadget_udc(dev, &udc->gadget);
1909	if (retval)
1910		goto err_unprepare_iclk;
1911	dev_set_drvdata(dev, udc);
1912	device_init_wakeup(dev, 1);
1913	create_debug_file(udc);
1914
1915	INFO("%s version %s\n", driver_name, DRIVER_VERSION);
1916	return 0;
1917
1918err_unprepare_iclk:
1919	clk_unprepare(udc->iclk);
1920err_unprepare_fclk:
1921	clk_unprepare(udc->fclk);
1922
1923	DBG("%s probe failed, %d\n", driver_name, retval);
1924
1925	return retval;
1926}
1927
1928static void at91udc_remove(struct platform_device *pdev)
1929{
1930	struct at91_udc *udc = platform_get_drvdata(pdev);
1931	unsigned long	flags;
1932
1933	DBG("remove\n");
1934
1935	usb_del_gadget_udc(&udc->gadget);
1936	if (udc->driver) {
1937		dev_err(&pdev->dev,
1938			"Driver still in use but removing anyhow\n");
1939		return;
1940	}
1941
1942	spin_lock_irqsave(&udc->lock, flags);
1943	pullup(udc, 0);
1944	spin_unlock_irqrestore(&udc->lock, flags);
1945
1946	device_init_wakeup(&pdev->dev, 0);
1947	remove_debug_file(udc);
1948	clk_unprepare(udc->fclk);
1949	clk_unprepare(udc->iclk);
1950}
1951
1952#ifdef CONFIG_PM
1953static int at91udc_suspend(struct platform_device *pdev, pm_message_t mesg)
1954{
1955	struct at91_udc *udc = platform_get_drvdata(pdev);
1956	int		wake = udc->driver && device_may_wakeup(&pdev->dev);
1957	unsigned long	flags;
1958
1959	/* Unless we can act normally to the host (letting it wake us up
1960	 * whenever it has work for us) force disconnect.  Wakeup requires
1961	 * PLLB for USB events (signaling for reset, wakeup, or incoming
1962	 * tokens) and VBUS irqs (on systems which support them).
1963	 */
1964	if ((!udc->suspended && udc->addr)
1965			|| !wake
1966			|| at91_suspend_entering_slow_clock()) {
1967		spin_lock_irqsave(&udc->lock, flags);
1968		pullup(udc, 0);
1969		wake = 0;
1970		spin_unlock_irqrestore(&udc->lock, flags);
1971	} else
1972		enable_irq_wake(udc->udp_irq);
1973
1974	udc->active_suspend = wake;
1975	if (udc->board.vbus_pin && !udc->board.vbus_polled && wake)
1976		enable_irq_wake(gpiod_to_irq(udc->board.vbus_pin));
1977	return 0;
1978}
1979
1980static int at91udc_resume(struct platform_device *pdev)
1981{
1982	struct at91_udc *udc = platform_get_drvdata(pdev);
1983	unsigned long	flags;
1984
1985	if (udc->board.vbus_pin && !udc->board.vbus_polled &&
1986	    udc->active_suspend)
1987		disable_irq_wake(gpiod_to_irq(udc->board.vbus_pin));
1988
1989	/* maybe reconnect to host; if so, clocks on */
1990	if (udc->active_suspend)
1991		disable_irq_wake(udc->udp_irq);
1992	else {
1993		spin_lock_irqsave(&udc->lock, flags);
1994		pullup(udc, 1);
1995		spin_unlock_irqrestore(&udc->lock, flags);
1996	}
1997	return 0;
1998}
1999#else
2000#define	at91udc_suspend	NULL
2001#define	at91udc_resume	NULL
2002#endif
2003
2004static struct platform_driver at91_udc_driver = {
2005	.probe		= at91udc_probe,
2006	.remove		= at91udc_remove,
2007	.shutdown	= at91udc_shutdown,
2008	.suspend	= at91udc_suspend,
2009	.resume		= at91udc_resume,
2010	.driver		= {
2011		.name	= driver_name,
2012		.of_match_table	= at91_udc_dt_ids,
2013	},
2014};
2015
2016module_platform_driver(at91_udc_driver);
2017
2018MODULE_DESCRIPTION("AT91 udc driver");
2019MODULE_AUTHOR("Thomas Rathbone, David Brownell");
2020MODULE_LICENSE("GPL");
2021MODULE_ALIAS("platform:at91_udc");