bdc_ep.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * bdc_ep.c - BRCM BDC USB3.0 device controller endpoint related functions
   4 *
   5 * Copyright (C) 2014 Broadcom Corporation
   6 *
   7 * Author: Ashwini Pahuja
   8 *
   9 * Based on drivers under drivers/usb/
  10 */
  11#include <linux/module.h>
  12#include <linux/pci.h>
  13#include <linux/dma-mapping.h>
  14#include <linux/kernel.h>
  15#include <linux/delay.h>
  16#include <linux/dmapool.h>
  17#include <linux/ioport.h>
  18#include <linux/sched.h>
  19#include <linux/slab.h>
  20#include <linux/errno.h>
  21#include <linux/init.h>
  22#include <linux/timer.h>
  23#include <linux/list.h>
  24#include <linux/interrupt.h>
  25#include <linux/moduleparam.h>
  26#include <linux/device.h>
  27#include <linux/usb/ch9.h>
  28#include <linux/usb/gadget.h>
  29#include <linux/usb/otg.h>
  30#include <linux/pm.h>
  31#include <linux/io.h>
  32#include <linux/irq.h>
  33#include <linux/unaligned.h>
  34#include <linux/platform_device.h>
  35#include <linux/usb/composite.h>
  36
  37#include "bdc.h"
  38#include "bdc_ep.h"
  39#include "bdc_cmd.h"
  40#include "bdc_dbg.h"
  41
  42static const char * const ep0_state_string[] =  {
  43	"WAIT_FOR_SETUP",
  44	"WAIT_FOR_DATA_START",
  45	"WAIT_FOR_DATA_XMIT",
  46	"WAIT_FOR_STATUS_START",
  47	"WAIT_FOR_STATUS_XMIT",
  48	"STATUS_PENDING"
  49};
  50
  51/* Free the bdl during ep disable */
  52static void ep_bd_list_free(struct bdc_ep *ep, u32 num_tabs)
  53{
  54	struct bd_list *bd_list = &ep->bd_list;
  55	struct bdc *bdc = ep->bdc;
  56	struct bd_table *bd_table;
  57	int index;
  58
  59	dev_dbg(bdc->dev, "%s ep:%s num_tabs:%d\n",
  60				 __func__, ep->name, num_tabs);
  61
  62	if (!bd_list->bd_table_array) {
  63		dev_dbg(bdc->dev, "%s already freed\n", ep->name);
  64		return;
  65	}
  66	for (index = 0; index < num_tabs; index++) {
  67		/*
  68		 * check if the bd_table struct is allocated ?
  69		 * if yes, then check if bd memory has been allocated, then
  70		 * free the dma_pool and also the bd_table struct memory
  71		 */
  72		bd_table = bd_list->bd_table_array[index];
  73		dev_dbg(bdc->dev, "bd_table:%p index:%d\n", bd_table, index);
  74		if (!bd_table) {
  75			dev_dbg(bdc->dev, "bd_table not allocated\n");
  76			continue;
  77		}
  78		if (!bd_table->start_bd) {
  79			dev_dbg(bdc->dev, "bd dma pool not allocated\n");
  80			continue;
  81		}
  82
  83		dev_dbg(bdc->dev,
  84				"Free dma pool start_bd:%p dma:%llx\n",
  85				bd_table->start_bd,
  86				(unsigned long long)bd_table->dma);
  87
  88		dma_pool_free(bdc->bd_table_pool,
  89				bd_table->start_bd,
  90				bd_table->dma);
  91		/* Free the bd_table structure */
  92		kfree(bd_table);
  93	}
  94	/* Free the bd table array */
  95	kfree(ep->bd_list.bd_table_array);
  96}
  97
  98/*
  99 * chain the tables, by insteting a chain bd at the end of prev_table, pointing
 100 * to next_table
 101 */
 102static inline void chain_table(struct bd_table *prev_table,
 103					struct bd_table *next_table,
 104					u32 bd_p_tab)
 105{
 106	/* Chain the prev table to next table */
 107	prev_table->start_bd[bd_p_tab-1].offset[0] =
 108				cpu_to_le32(lower_32_bits(next_table->dma));
 109
 110	prev_table->start_bd[bd_p_tab-1].offset[1] =
 111				cpu_to_le32(upper_32_bits(next_table->dma));
 112
 113	prev_table->start_bd[bd_p_tab-1].offset[2] =
 114				0x0;
 115
 116	prev_table->start_bd[bd_p_tab-1].offset[3] =
 117				cpu_to_le32(MARK_CHAIN_BD);
 118}
 119
 120/* Allocate the bdl for ep, during config ep */
 121static int ep_bd_list_alloc(struct bdc_ep *ep)
 122{
 123	struct bd_table *prev_table = NULL;
 124	int index, num_tabs, bd_p_tab;
 125	struct bdc *bdc = ep->bdc;
 126	struct bd_table *bd_table;
 127	dma_addr_t dma;
 128
 129	if (usb_endpoint_xfer_isoc(ep->desc))
 130		num_tabs = NUM_TABLES_ISOCH;
 131	else
 132		num_tabs = NUM_TABLES;
 133
 134	bd_p_tab = NUM_BDS_PER_TABLE;
 135	/* if there is only 1 table in bd list then loop chain to self */
 136	dev_dbg(bdc->dev,
 137		"%s ep:%p num_tabs:%d\n",
 138		__func__, ep, num_tabs);
 139
 140	/* Allocate memory for table array */
 141	ep->bd_list.bd_table_array = kcalloc(num_tabs,
 142					     sizeof(struct bd_table *),
 143					     GFP_ATOMIC);
 144	if (!ep->bd_list.bd_table_array)
 145		return -ENOMEM;
 146
 147	/* Allocate memory for each table */
 148	for (index = 0; index < num_tabs; index++) {
 149		/* Allocate memory for bd_table structure */
 150		bd_table = kzalloc(sizeof(*bd_table), GFP_ATOMIC);
 151		if (!bd_table)
 152			goto fail;
 153
 154		bd_table->start_bd = dma_pool_zalloc(bdc->bd_table_pool,
 155							GFP_ATOMIC,
 156							&dma);
 157		if (!bd_table->start_bd) {
 158			kfree(bd_table);
 159			goto fail;
 160		}
 161
 162		bd_table->dma = dma;
 163
 164		dev_dbg(bdc->dev,
 165			"index:%d start_bd:%p dma=%08llx prev_table:%p\n",
 166			index, bd_table->start_bd,
 167			(unsigned long long)bd_table->dma, prev_table);
 168
 169		ep->bd_list.bd_table_array[index] = bd_table;
 170		if (prev_table)
 171			chain_table(prev_table, bd_table, bd_p_tab);
 172
 173		prev_table = bd_table;
 174	}
 175	chain_table(prev_table, ep->bd_list.bd_table_array[0], bd_p_tab);
 176	/* Memory allocation is successful, now init the internal fields */
 177	ep->bd_list.num_tabs = num_tabs;
 178	ep->bd_list.max_bdi  = (num_tabs * bd_p_tab) - 1;
 179	ep->bd_list.num_tabs = num_tabs;
 180	ep->bd_list.num_bds_table = bd_p_tab;
 181	ep->bd_list.eqp_bdi = 0;
 182	ep->bd_list.hwd_bdi = 0;
 183
 184	return 0;
 185fail:
 186	/* Free the bd_table_array, bd_table struct, bd's */
 187	ep_bd_list_free(ep, num_tabs);
 188
 189	return -ENOMEM;
 190}
 191
 192/* returns how many bd's are need for this transfer */
 193static inline int bd_needed_req(struct bdc_req *req)
 194{
 195	int bd_needed = 0;
 196	int remaining;
 197
 198	/* 1 bd needed for 0 byte transfer */
 199	if (req->usb_req.length == 0)
 200		return 1;
 201
 202	/* remaining bytes after tranfering all max BD size BD's */
 203	remaining = req->usb_req.length % BD_MAX_BUFF_SIZE;
 204	if (remaining)
 205		bd_needed++;
 206
 207	/* How many maximum BUFF size BD's ? */
 208	remaining = req->usb_req.length / BD_MAX_BUFF_SIZE;
 209	bd_needed += remaining;
 210
 211	return bd_needed;
 212}
 213
 214/* returns the bd index(bdi) corresponding to bd dma address */
 215static int bd_add_to_bdi(struct bdc_ep *ep, dma_addr_t bd_dma_addr)
 216{
 217	struct bd_list *bd_list = &ep->bd_list;
 218	dma_addr_t dma_first_bd, dma_last_bd;
 219	struct bdc *bdc = ep->bdc;
 220	struct bd_table *bd_table;
 221	bool found = false;
 222	int tbi, bdi;
 223
 224	dma_first_bd = dma_last_bd = 0;
 225	dev_dbg(bdc->dev, "%s  %llx\n",
 226			__func__, (unsigned long long)bd_dma_addr);
 227	/*
 228	 * Find in which table this bd_dma_addr belongs?, go through the table
 229	 * array and compare addresses of first and last address of bd of each
 230	 * table
 231	 */
 232	for (tbi = 0; tbi < bd_list->num_tabs; tbi++) {
 233		bd_table = bd_list->bd_table_array[tbi];
 234		dma_first_bd = bd_table->dma;
 235		dma_last_bd = bd_table->dma +
 236					(sizeof(struct bdc_bd) *
 237					(bd_list->num_bds_table - 1));
 238		dev_dbg(bdc->dev, "dma_first_bd:%llx dma_last_bd:%llx\n",
 239					(unsigned long long)dma_first_bd,
 240					(unsigned long long)dma_last_bd);
 241		if (bd_dma_addr >= dma_first_bd && bd_dma_addr <= dma_last_bd) {
 242			found = true;
 243			break;
 244		}
 245	}
 246	if (unlikely(!found)) {
 247		dev_err(bdc->dev, "%s FATAL err, bd not found\n", __func__);
 248		return -EINVAL;
 249	}
 250	/* Now we know the table, find the bdi */
 251	bdi = (bd_dma_addr - dma_first_bd) / sizeof(struct bdc_bd);
 252
 253	/* return the global bdi, to compare with ep eqp_bdi */
 254	return (bdi + (tbi * bd_list->num_bds_table));
 255}
 256
 257/* returns the table index(tbi) of the given bdi */
 258static int bdi_to_tbi(struct bdc_ep *ep, int bdi)
 259{
 260	int tbi;
 261
 262	tbi = bdi / ep->bd_list.num_bds_table;
 263	dev_vdbg(ep->bdc->dev,
 264		"bdi:%d num_bds_table:%d tbi:%d\n",
 265		bdi, ep->bd_list.num_bds_table, tbi);
 266
 267	return tbi;
 268}
 269
 270/* Find the bdi last bd in the transfer */
 271static inline int find_end_bdi(struct bdc_ep *ep, int next_hwd_bdi)
 272{
 273	int end_bdi;
 274
 275	end_bdi = next_hwd_bdi - 1;
 276	if (end_bdi < 0)
 277		end_bdi = ep->bd_list.max_bdi - 1;
 278	else if ((end_bdi % (ep->bd_list.num_bds_table-1)) == 0)
 279		end_bdi--;
 280
 281	return end_bdi;
 282}
 283
 284/*
 285 * How many transfer bd's are available on this ep bdl, chain bds are not
 286 * counted in available bds
 287 */
 288static int bd_available_ep(struct bdc_ep *ep)
 289{
 290	struct bd_list *bd_list = &ep->bd_list;
 291	int available1, available2;
 292	struct bdc *bdc = ep->bdc;
 293	int chain_bd1, chain_bd2;
 294	int available_bd = 0;
 295
 296	available1 = available2 = chain_bd1 = chain_bd2 = 0;
 297	/* if empty then we have all bd's available - number of chain bd's */
 298	if (bd_list->eqp_bdi == bd_list->hwd_bdi)
 299		return bd_list->max_bdi - bd_list->num_tabs;
 300
 301	/*
 302	 * Depending upon where eqp and dqp pointers are, caculate number
 303	 * of avaialble bd's
 304	 */
 305	if (bd_list->hwd_bdi < bd_list->eqp_bdi) {
 306		/* available bd's are from eqp..max_bds + 0..dqp - chain_bds */
 307		available1 = bd_list->max_bdi - bd_list->eqp_bdi;
 308		available2 = bd_list->hwd_bdi;
 309		chain_bd1 = available1 / bd_list->num_bds_table;
 310		chain_bd2 = available2 / bd_list->num_bds_table;
 311		dev_vdbg(bdc->dev, "chain_bd1:%d chain_bd2:%d\n",
 312						chain_bd1, chain_bd2);
 313		available_bd = available1 + available2 - chain_bd1 - chain_bd2;
 314	} else {
 315		/* available bd's are from eqp..dqp - number of chain bd's */
 316		available1 = bd_list->hwd_bdi -  bd_list->eqp_bdi;
 317		/* if gap between eqp and dqp is less than NUM_BDS_PER_TABLE */
 318		if ((bd_list->hwd_bdi - bd_list->eqp_bdi)
 319					<= bd_list->num_bds_table) {
 320			/* If there any chain bd in between */
 321			if (!(bdi_to_tbi(ep, bd_list->hwd_bdi)
 322					== bdi_to_tbi(ep, bd_list->eqp_bdi))) {
 323				available_bd = available1 - 1;
 324			}
 325		} else {
 326			chain_bd1 = available1 / bd_list->num_bds_table;
 327			available_bd = available1 - chain_bd1;
 328		}
 329	}
 330	/*
 331	 * we need to keep one extra bd to check if ring is full or empty so
 332	 * reduce by 1
 333	 */
 334	available_bd--;
 335	dev_vdbg(bdc->dev, "available_bd:%d\n", available_bd);
 336
 337	return available_bd;
 338}
 339
 340/* Notify the hardware after queueing the bd to bdl */
 341void bdc_notify_xfr(struct bdc *bdc, u32 epnum)
 342{
 343	struct bdc_ep *ep = bdc->bdc_ep_array[epnum];
 344
 345	dev_vdbg(bdc->dev, "%s epnum:%d\n", __func__, epnum);
 346	/*
 347	 * We don't have anyway to check if ep state is running,
 348	 * except the software flags.
 349	 */
 350	if (unlikely(ep->flags & BDC_EP_STOP))
 351		ep->flags &= ~BDC_EP_STOP;
 352
 353	bdc_writel(bdc->regs, BDC_XSFNTF, epnum);
 354}
 355
 356/* returns the bd corresponding to bdi */
 357static struct bdc_bd *bdi_to_bd(struct bdc_ep *ep, int bdi)
 358{
 359	int tbi = bdi_to_tbi(ep, bdi);
 360	int local_bdi = 0;
 361
 362	local_bdi = bdi - (tbi * ep->bd_list.num_bds_table);
 363	dev_vdbg(ep->bdc->dev,
 364		"%s bdi:%d local_bdi:%d\n",
 365		 __func__, bdi, local_bdi);
 366
 367	return (ep->bd_list.bd_table_array[tbi]->start_bd + local_bdi);
 368}
 369
 370/* Advance the enqueue pointer */
 371static void ep_bdlist_eqp_adv(struct bdc_ep *ep)
 372{
 373	ep->bd_list.eqp_bdi++;
 374	/* if it's chain bd, then move to next */
 375	if (((ep->bd_list.eqp_bdi + 1) % ep->bd_list.num_bds_table) == 0)
 376		ep->bd_list.eqp_bdi++;
 377
 378	/* if the eqp is pointing to last + 1 then move back to 0 */
 379	if (ep->bd_list.eqp_bdi == (ep->bd_list.max_bdi + 1))
 380		ep->bd_list.eqp_bdi = 0;
 381}
 382
 383/* Setup the first bd for ep0 transfer */
 384static int setup_first_bd_ep0(struct bdc *bdc, struct bdc_req *req, u32 *dword3)
 385{
 386	u16 wValue;
 387	u32 req_len;
 388
 389	req->ep->dir = 0;
 390	req_len = req->usb_req.length;
 391	switch (bdc->ep0_state) {
 392	case WAIT_FOR_DATA_START:
 393		*dword3 |= BD_TYPE_DS;
 394		if (bdc->setup_pkt.bRequestType & USB_DIR_IN)
 395			*dword3 |= BD_DIR_IN;
 396
 397		/* check if zlp will be needed */
 398		wValue = le16_to_cpu(bdc->setup_pkt.wValue);
 399		if ((wValue > req_len) &&
 400				(req_len % bdc->gadget.ep0->maxpacket == 0)) {
 401			dev_dbg(bdc->dev, "ZLP needed wVal:%d len:%d MaxP:%d\n",
 402					wValue, req_len,
 403					bdc->gadget.ep0->maxpacket);
 404			bdc->zlp_needed = true;
 405		}
 406		break;
 407
 408	case WAIT_FOR_STATUS_START:
 409		*dword3 |= BD_TYPE_SS;
 410		if (!le16_to_cpu(bdc->setup_pkt.wLength) ||
 411				!(bdc->setup_pkt.bRequestType & USB_DIR_IN))
 412			*dword3 |= BD_DIR_IN;
 413		break;
 414	default:
 415		dev_err(bdc->dev,
 416			"Unknown ep0 state for queueing bd ep0_state:%s\n",
 417			ep0_state_string[bdc->ep0_state]);
 418		return -EINVAL;
 419	}
 420
 421	return 0;
 422}
 423
 424/* Setup the bd dma descriptor for a given request */
 425static int setup_bd_list_xfr(struct bdc *bdc, struct bdc_req *req, int num_bds)
 426{
 427	dma_addr_t buf_add = req->usb_req.dma;
 428	u32 maxp, tfs, dword2, dword3;
 429	struct bd_transfer *bd_xfr;
 430	struct bd_list *bd_list;
 431	struct bdc_ep *ep;
 432	struct bdc_bd *bd;
 433	int ret, bdnum;
 434	u32 req_len;
 435
 436	ep = req->ep;
 437	bd_list = &ep->bd_list;
 438	bd_xfr = &req->bd_xfr;
 439	bd_xfr->req = req;
 440	bd_xfr->start_bdi = bd_list->eqp_bdi;
 441	bd = bdi_to_bd(ep, bd_list->eqp_bdi);
 442	req_len = req->usb_req.length;
 443	maxp = usb_endpoint_maxp(ep->desc);
 444	tfs = roundup(req->usb_req.length, maxp);
 445	tfs = tfs/maxp;
 446	dev_vdbg(bdc->dev, "%s ep:%s num_bds:%d tfs:%d r_len:%d bd:%p\n",
 447				__func__, ep->name, num_bds, tfs, req_len, bd);
 448
 449	for (bdnum = 0; bdnum < num_bds; bdnum++) {
 450		dword2 = dword3 = 0;
 451		/* First bd */
 452		if (!bdnum) {
 453			dword3 |= BD_SOT|BD_SBF|(tfs<<BD_TFS_SHIFT);
 454			dword2 |= BD_LTF;
 455			/* format of first bd for ep0 is different than other */
 456			if (ep->ep_num == 1) {
 457				ret = setup_first_bd_ep0(bdc, req, &dword3);
 458				if (ret)
 459					return ret;
 460			}
 461		}
 462		if (!req->ep->dir)
 463			dword3 |= BD_ISP;
 464
 465		if (req_len > BD_MAX_BUFF_SIZE) {
 466			dword2 |= BD_MAX_BUFF_SIZE;
 467			req_len -= BD_MAX_BUFF_SIZE;
 468		} else {
 469			/* this should be the last bd */
 470			dword2 |= req_len;
 471			dword3 |= BD_IOC;
 472			dword3 |= BD_EOT;
 473		}
 474		/* Currently only 1 INT target is supported */
 475		dword2 |= BD_INTR_TARGET(0);
 476		bd = bdi_to_bd(ep, ep->bd_list.eqp_bdi);
 477		if (unlikely(!bd)) {
 478			dev_err(bdc->dev, "Err bd pointing to wrong addr\n");
 479			return -EINVAL;
 480		}
 481		/* write bd */
 482		bd->offset[0] = cpu_to_le32(lower_32_bits(buf_add));
 483		bd->offset[1] = cpu_to_le32(upper_32_bits(buf_add));
 484		bd->offset[2] = cpu_to_le32(dword2);
 485		bd->offset[3] = cpu_to_le32(dword3);
 486		/* advance eqp pointer */
 487		ep_bdlist_eqp_adv(ep);
 488		/* advance the buff pointer */
 489		buf_add += BD_MAX_BUFF_SIZE;
 490		dev_vdbg(bdc->dev, "buf_add:%08llx req_len:%d bd:%p eqp:%d\n",
 491				(unsigned long long)buf_add, req_len, bd,
 492							ep->bd_list.eqp_bdi);
 493		bd = bdi_to_bd(ep, ep->bd_list.eqp_bdi);
 494		bd->offset[3] = cpu_to_le32(BD_SBF);
 495	}
 496	/* clear the STOP BD fetch bit from the first bd of this xfr */
 497	bd = bdi_to_bd(ep, bd_xfr->start_bdi);
 498	bd->offset[3] &= cpu_to_le32(~BD_SBF);
 499	/* the new eqp will be next hw dqp */
 500	bd_xfr->num_bds  = num_bds;
 501	bd_xfr->next_hwd_bdi = ep->bd_list.eqp_bdi;
 502	/* everything is written correctly before notifying the HW */
 503	wmb();
 504
 505	return 0;
 506}
 507
 508/* Queue the xfr */
 509static int bdc_queue_xfr(struct bdc *bdc, struct bdc_req *req)
 510{
 511	int num_bds, bd_available;
 512	struct bdc_ep *ep;
 513	int ret;
 514
 515	ep = req->ep;
 516	dev_dbg(bdc->dev, "%s req:%p\n", __func__, req);
 517	dev_dbg(bdc->dev, "eqp_bdi:%d hwd_bdi:%d\n",
 518			ep->bd_list.eqp_bdi, ep->bd_list.hwd_bdi);
 519
 520	num_bds =  bd_needed_req(req);
 521	bd_available = bd_available_ep(ep);
 522
 523	/* how many bd's are avaialble on ep */
 524	if (num_bds > bd_available)
 525		return -ENOMEM;
 526
 527	ret = setup_bd_list_xfr(bdc, req, num_bds);
 528	if (ret)
 529		return ret;
 530	list_add_tail(&req->queue, &ep->queue);
 531	bdc_dbg_bd_list(bdc, ep);
 532	bdc_notify_xfr(bdc, ep->ep_num);
 533
 534	return 0;
 535}
 536
 537/* callback to gadget layer when xfr completes */
 538static void bdc_req_complete(struct bdc_ep *ep, struct bdc_req *req,
 539						int status)
 540{
 541	struct bdc *bdc = ep->bdc;
 542
 543	if (req == NULL)
 544		return;
 545
 546	dev_dbg(bdc->dev, "%s ep:%s status:%d\n", __func__, ep->name, status);
 547	list_del(&req->queue);
 548	req->usb_req.status = status;
 549	usb_gadget_unmap_request(&bdc->gadget, &req->usb_req, ep->dir);
 550	if (req->usb_req.complete) {
 551		spin_unlock(&bdc->lock);
 552		usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
 553		spin_lock(&bdc->lock);
 554	}
 555}
 556
 557/* Disable the endpoint */
 558int bdc_ep_disable(struct bdc_ep *ep)
 559{
 560	struct bdc_req *req;
 561	struct bdc *bdc;
 562	int ret;
 563
 564	ret = 0;
 565	bdc = ep->bdc;
 566	dev_dbg(bdc->dev, "%s() ep->ep_num=%d\n", __func__, ep->ep_num);
 567	/* Stop the endpoint */
 568	ret = bdc_stop_ep(bdc, ep->ep_num);
 569
 570	/*
 571	 * Intentionally don't check the ret value of stop, it can fail in
 572	 * disconnect scenarios, continue with dconfig
 573	 */
 574	/* de-queue any pending requests */
 575	while (!list_empty(&ep->queue)) {
 576		req = list_entry(ep->queue.next, struct bdc_req,
 577				queue);
 578		bdc_req_complete(ep, req, -ESHUTDOWN);
 579	}
 580	/* deconfigure the endpoint */
 581	ret = bdc_dconfig_ep(bdc, ep);
 582	if (ret)
 583		dev_warn(bdc->dev,
 584			"dconfig fail but continue with memory free");
 585
 586	ep->flags = 0;
 587	/* ep0 memory is not freed, but reused on next connect sr */
 588	if (ep->ep_num == 1)
 589		return 0;
 590
 591	/* Free the bdl memory */
 592	ep_bd_list_free(ep, ep->bd_list.num_tabs);
 593	ep->desc = NULL;
 594	ep->comp_desc = NULL;
 595	ep->usb_ep.desc = NULL;
 596	ep->ep_type = 0;
 597
 598	return ret;
 599}
 600
 601/* Enable the ep */
 602int bdc_ep_enable(struct bdc_ep *ep)
 603{
 604	struct bdc *bdc;
 605	int ret = 0;
 606
 607	bdc = ep->bdc;
 608	dev_dbg(bdc->dev, "%s NUM_TABLES:%d %d\n",
 609					__func__, NUM_TABLES, NUM_TABLES_ISOCH);
 610
 611	ret = ep_bd_list_alloc(ep);
 612	if (ret) {
 613		dev_err(bdc->dev, "ep bd list allocation failed:%d\n", ret);
 614		return -ENOMEM;
 615	}
 616	bdc_dbg_bd_list(bdc, ep);
 617	/* only for ep0: config ep is called for ep0 from connect event */
 618	if (ep->ep_num == 1)
 619		return ret;
 620
 621	/* Issue a configure endpoint command */
 622	ret = bdc_config_ep(bdc, ep);
 623	if (ret)
 624		return ret;
 625
 626	ep->usb_ep.maxpacket = usb_endpoint_maxp(ep->desc);
 627	ep->usb_ep.desc = ep->desc;
 628	ep->usb_ep.comp_desc = ep->comp_desc;
 629	ep->ep_type = usb_endpoint_type(ep->desc);
 630	ep->flags |= BDC_EP_ENABLED;
 631
 632	return 0;
 633}
 634
 635/* EP0 related code */
 636
 637/* Queue a status stage BD */
 638static int ep0_queue_status_stage(struct bdc *bdc)
 639{
 640	struct bdc_req *status_req;
 641	struct bdc_ep *ep;
 642
 643	status_req = &bdc->status_req;
 644	ep = bdc->bdc_ep_array[1];
 645	status_req->ep = ep;
 646	status_req->usb_req.length = 0;
 647	status_req->usb_req.status = -EINPROGRESS;
 648	status_req->usb_req.actual = 0;
 649	status_req->usb_req.complete = NULL;
 650	bdc_queue_xfr(bdc, status_req);
 651
 652	return 0;
 653}
 654
 655/* Queue xfr on ep0 */
 656static int ep0_queue(struct bdc_ep *ep, struct bdc_req *req)
 657{
 658	struct bdc *bdc;
 659	int ret;
 660
 661	bdc = ep->bdc;
 662	dev_dbg(bdc->dev, "%s()\n", __func__);
 663	req->usb_req.actual = 0;
 664	req->usb_req.status = -EINPROGRESS;
 665	req->epnum = ep->ep_num;
 666
 667	if (bdc->delayed_status) {
 668		bdc->delayed_status = false;
 669		/* if status stage was delayed? */
 670		if (bdc->ep0_state == WAIT_FOR_STATUS_START) {
 671			/* Queue a status stage BD */
 672			ep0_queue_status_stage(bdc);
 673			bdc->ep0_state = WAIT_FOR_STATUS_XMIT;
 674			return 0;
 675		}
 676	} else {
 677		/*
 678		 * if delayed status is false and 0 length transfer is requested
 679		 * i.e. for status stage of some setup request, then just
 680		 * return from here the status stage is queued independently
 681		 */
 682		if (req->usb_req.length == 0)
 683			return 0;
 684
 685	}
 686	ret = usb_gadget_map_request(&bdc->gadget, &req->usb_req, ep->dir);
 687	if (ret) {
 688		dev_err(bdc->dev, "dma mapping failed %s\n", ep->name);
 689		return ret;
 690	}
 691
 692	return bdc_queue_xfr(bdc, req);
 693}
 694
 695/* Queue data stage */
 696static int ep0_queue_data_stage(struct bdc *bdc)
 697{
 698	struct bdc_ep *ep;
 699
 700	dev_dbg(bdc->dev, "%s\n", __func__);
 701	ep = bdc->bdc_ep_array[1];
 702	bdc->ep0_req.ep = ep;
 703	bdc->ep0_req.usb_req.complete = NULL;
 704
 705	return ep0_queue(ep, &bdc->ep0_req);
 706}
 707
 708/* Queue req on ep */
 709static int ep_queue(struct bdc_ep *ep, struct bdc_req *req)
 710{
 711	struct bdc *bdc;
 712	int ret = 0;
 713
 714	if (!req || !ep->usb_ep.desc)
 715		return -EINVAL;
 716
 717	bdc = ep->bdc;
 718
 719	req->usb_req.actual = 0;
 720	req->usb_req.status = -EINPROGRESS;
 721	req->epnum = ep->ep_num;
 722
 723	ret = usb_gadget_map_request(&bdc->gadget, &req->usb_req, ep->dir);
 724	if (ret) {
 725		dev_err(bdc->dev, "dma mapping failed\n");
 726		return ret;
 727	}
 728
 729	return bdc_queue_xfr(bdc, req);
 730}
 731
 732/* Dequeue a request from ep */
 733static int ep_dequeue(struct bdc_ep *ep, struct bdc_req *req)
 734{
 735	int start_bdi, end_bdi, tbi, eqp_bdi, curr_hw_dqpi;
 736	bool start_pending, end_pending;
 737	bool first_remove = false;
 738	struct bdc_req *first_req;
 739	struct bdc_bd *bd_start;
 740	struct bd_table *table;
 741	dma_addr_t next_bd_dma;
 742	u64   deq_ptr_64 = 0;
 743	struct bdc  *bdc;
 744	u32    tmp_32;
 745	int ret;
 746
 747	bdc = ep->bdc;
 748	start_pending = end_pending = false;
 749	eqp_bdi = ep->bd_list.eqp_bdi - 1;
 750
 751	if (eqp_bdi < 0)
 752		eqp_bdi = ep->bd_list.max_bdi;
 753
 754	start_bdi = req->bd_xfr.start_bdi;
 755	end_bdi = find_end_bdi(ep, req->bd_xfr.next_hwd_bdi);
 756
 757	dev_dbg(bdc->dev, "%s ep:%s start:%d end:%d\n",
 758					__func__, ep->name, start_bdi, end_bdi);
 759	dev_dbg(bdc->dev, "%s ep=%p ep->desc=%p\n", __func__,
 760						ep, (void *)ep->usb_ep.desc);
 761	/* if still connected, stop the ep to see where the HW is ? */
 762	if (!(bdc_readl(bdc->regs, BDC_USPC) & BDC_PST_MASK)) {
 763		ret = bdc_stop_ep(bdc, ep->ep_num);
 764		/* if there is an issue, then no need to go further */
 765		if (ret)
 766			return 0;
 767	} else
 768		return 0;
 769
 770	/*
 771	 * After endpoint is stopped, there can be 3 cases, the request
 772	 * is processed, pending or in the middle of processing
 773	 */
 774
 775	/* The current hw dequeue pointer */
 776	tmp_32 = bdc_readl(bdc->regs, BDC_EPSTS0);
 777	deq_ptr_64 = tmp_32;
 778	tmp_32 = bdc_readl(bdc->regs, BDC_EPSTS1);
 779	deq_ptr_64 |= ((u64)tmp_32 << 32);
 780
 781	/* we have the dma addr of next bd that will be fetched by hardware */
 782	curr_hw_dqpi = bd_add_to_bdi(ep, deq_ptr_64);
 783	if (curr_hw_dqpi < 0)
 784		return curr_hw_dqpi;
 785
 786	/*
 787	 * curr_hw_dqpi points to actual dqp of HW and HW owns bd's from
 788	 * curr_hw_dqbdi..eqp_bdi.
 789	 */
 790
 791	/* Check if start_bdi and end_bdi are in range of HW owned BD's */
 792	if (curr_hw_dqpi > eqp_bdi) {
 793		/* there is a wrap from last to 0 */
 794		if (start_bdi >= curr_hw_dqpi || start_bdi <= eqp_bdi) {
 795			start_pending = true;
 796			end_pending = true;
 797		} else if (end_bdi >= curr_hw_dqpi || end_bdi <= eqp_bdi) {
 798			end_pending = true;
 799		}
 800	} else {
 801		if (start_bdi >= curr_hw_dqpi) {
 802			start_pending = true;
 803			end_pending = true;
 804		} else if (end_bdi >= curr_hw_dqpi) {
 805			end_pending = true;
 806		}
 807	}
 808	dev_dbg(bdc->dev,
 809		"start_pending:%d end_pending:%d speed:%d\n",
 810		start_pending, end_pending, bdc->gadget.speed);
 811
 812	/* If both start till end are processes, we cannot deq req */
 813	if (!start_pending && !end_pending)
 814		return -EINVAL;
 815
 816	/*
 817	 * if ep_dequeue is called after disconnect then just return
 818	 * success from here
 819	 */
 820	if (bdc->gadget.speed == USB_SPEED_UNKNOWN)
 821		return 0;
 822	tbi = bdi_to_tbi(ep, req->bd_xfr.next_hwd_bdi);
 823	table = ep->bd_list.bd_table_array[tbi];
 824	next_bd_dma =  table->dma +
 825			sizeof(struct bdc_bd)*(req->bd_xfr.next_hwd_bdi -
 826					tbi * ep->bd_list.num_bds_table);
 827
 828	first_req = list_first_entry(&ep->queue, struct bdc_req,
 829			queue);
 830
 831	if (req == first_req)
 832		first_remove = true;
 833
 834	/*
 835	 * Due to HW limitation we need to bypadd chain bd's and issue ep_bla,
 836	 * incase if start is pending this is the first request in the list
 837	 * then issue ep_bla instead of marking as chain bd
 838	 */
 839	if (start_pending && !first_remove) {
 840		/*
 841		 * Mark the start bd as Chain bd, and point the chain
 842		 * bd to next_bd_dma
 843		 */
 844		bd_start = bdi_to_bd(ep, start_bdi);
 845		bd_start->offset[0] = cpu_to_le32(lower_32_bits(next_bd_dma));
 846		bd_start->offset[1] = cpu_to_le32(upper_32_bits(next_bd_dma));
 847		bd_start->offset[2] = 0x0;
 848		bd_start->offset[3] = cpu_to_le32(MARK_CHAIN_BD);
 849		bdc_dbg_bd_list(bdc, ep);
 850	} else if (end_pending) {
 851		/*
 852		 * The transfer is stopped in the middle, move the
 853		 * HW deq pointer to next_bd_dma
 854		 */
 855		ret = bdc_ep_bla(bdc, ep, next_bd_dma);
 856		if (ret) {
 857			dev_err(bdc->dev, "error in ep_bla:%d\n", ret);
 858			return ret;
 859		}
 860	}
 861
 862	return 0;
 863}
 864
 865/* Halt/Clear the ep based on value */
 866static int ep_set_halt(struct bdc_ep *ep, u32 value)
 867{
 868	struct bdc *bdc;
 869	int ret;
 870
 871	bdc = ep->bdc;
 872	dev_dbg(bdc->dev, "%s ep:%s value=%d\n", __func__, ep->name, value);
 873
 874	if (value) {
 875		dev_dbg(bdc->dev, "Halt\n");
 876		if (ep->ep_num == 1)
 877			bdc->ep0_state = WAIT_FOR_SETUP;
 878
 879		ret = bdc_ep_set_stall(bdc, ep->ep_num);
 880		if (ret)
 881			dev_err(bdc->dev, "failed to set STALL on %s\n",
 882				ep->name);
 883		else
 884			ep->flags |= BDC_EP_STALL;
 885	} else {
 886		/* Clear */
 887		dev_dbg(bdc->dev, "Before Clear\n");
 888		ret = bdc_ep_clear_stall(bdc, ep->ep_num);
 889		if (ret)
 890			dev_err(bdc->dev, "failed to clear STALL on %s\n",
 891				ep->name);
 892		else
 893			ep->flags &= ~BDC_EP_STALL;
 894		dev_dbg(bdc->dev, "After  Clear\n");
 895	}
 896
 897	return ret;
 898}
 899
 900/* Free all the ep */
 901void bdc_free_ep(struct bdc *bdc)
 902{
 903	struct bdc_ep *ep;
 904	u8	epnum;
 905
 906	dev_dbg(bdc->dev, "%s\n", __func__);
 907	for (epnum = 1; epnum < bdc->num_eps; epnum++) {
 908		ep = bdc->bdc_ep_array[epnum];
 909		if (!ep)
 910			continue;
 911
 912		if (ep->flags & BDC_EP_ENABLED)
 913			ep_bd_list_free(ep, ep->bd_list.num_tabs);
 914
 915		/* ep0 is not in this gadget list */
 916		if (epnum != 1)
 917			list_del(&ep->usb_ep.ep_list);
 918
 919		kfree(ep);
 920	}
 921}
 922
 923/* USB2 spec, section 7.1.20 */
 924static int bdc_set_test_mode(struct bdc *bdc)
 925{
 926	u32 usb2_pm;
 927
 928	usb2_pm = bdc_readl(bdc->regs, BDC_USPPM2);
 929	usb2_pm &= ~BDC_PTC_MASK;
 930	dev_dbg(bdc->dev, "%s\n", __func__);
 931	switch (bdc->test_mode) {
 932	case USB_TEST_J:
 933	case USB_TEST_K:
 934	case USB_TEST_SE0_NAK:
 935	case USB_TEST_PACKET:
 936	case USB_TEST_FORCE_ENABLE:
 937		usb2_pm |= bdc->test_mode << 28;
 938		break;
 939	default:
 940		return -EINVAL;
 941	}
 942	dev_dbg(bdc->dev, "usb2_pm=%08x", usb2_pm);
 943	bdc_writel(bdc->regs, BDC_USPPM2, usb2_pm);
 944
 945	return 0;
 946}
 947
 948/*
 949 * Helper function to handle Transfer status report with status as either
 950 * success or short
 951 */
 952static void handle_xsr_succ_status(struct bdc *bdc, struct bdc_ep *ep,
 953							struct bdc_sr *sreport)
 954{
 955	int short_bdi, start_bdi, end_bdi, max_len_bds, chain_bds;
 956	struct bd_list *bd_list = &ep->bd_list;
 957	int actual_length, length_short;
 958	struct bd_transfer *bd_xfr;
 959	struct bdc_bd *short_bd;
 960	struct bdc_req *req;
 961	u64   deq_ptr_64 = 0;
 962	int status = 0;
 963	int sr_status;
 964	u32    tmp_32;
 965
 966	dev_dbg(bdc->dev, "%s  ep:%p\n", __func__, ep);
 967	bdc_dbg_srr(bdc, 0);
 968	/* do not process thie sr if ignore flag is set */
 969	if (ep->ignore_next_sr) {
 970		ep->ignore_next_sr = false;
 971		return;
 972	}
 973
 974	if (unlikely(list_empty(&ep->queue))) {
 975		dev_warn(bdc->dev, "xfr srr with no BD's queued\n");
 976		return;
 977	}
 978	req = list_entry(ep->queue.next, struct bdc_req,
 979			queue);
 980
 981	bd_xfr = &req->bd_xfr;
 982	sr_status = XSF_STS(le32_to_cpu(sreport->offset[3]));
 983
 984	/*
 985	 * sr_status is short and this transfer has more than 1 bd then it needs
 986	 * special handling,  this is only applicable for bulk and ctrl
 987	 */
 988	if (sr_status == XSF_SHORT &&  bd_xfr->num_bds > 1) {
 989		/*
 990		 * This is multi bd xfr, lets see which bd
 991		 * caused short transfer and how many bytes have been
 992		 * transferred so far.
 993		 */
 994		tmp_32 = le32_to_cpu(sreport->offset[0]);
 995		deq_ptr_64 = tmp_32;
 996		tmp_32 = le32_to_cpu(sreport->offset[1]);
 997		deq_ptr_64 |= ((u64)tmp_32 << 32);
 998		short_bdi = bd_add_to_bdi(ep, deq_ptr_64);
 999		if (unlikely(short_bdi < 0))
1000			dev_warn(bdc->dev, "bd doesn't exist?\n");
1001
1002		start_bdi =  bd_xfr->start_bdi;
1003		/*
1004		 * We know the start_bdi and short_bdi, how many xfr
1005		 * bds in between
1006		 */
1007		if (start_bdi <= short_bdi) {
1008			max_len_bds = short_bdi - start_bdi;
1009			if (max_len_bds <= bd_list->num_bds_table) {
1010				if (!(bdi_to_tbi(ep, start_bdi) ==
1011						bdi_to_tbi(ep, short_bdi)))
1012					max_len_bds--;
1013			} else {
1014				chain_bds = max_len_bds/bd_list->num_bds_table;
1015				max_len_bds -= chain_bds;
1016			}
1017		} else {
1018			/* there is a wrap in the ring within a xfr */
1019			chain_bds = (bd_list->max_bdi - start_bdi)/
1020							bd_list->num_bds_table;
1021			chain_bds += short_bdi/bd_list->num_bds_table;
1022			max_len_bds = bd_list->max_bdi - start_bdi;
1023			max_len_bds += short_bdi;
1024			max_len_bds -= chain_bds;
1025		}
1026		/* max_len_bds is the number of full length bds */
1027		end_bdi = find_end_bdi(ep, bd_xfr->next_hwd_bdi);
1028		if (!(end_bdi == short_bdi))
1029			ep->ignore_next_sr = true;
1030
1031		actual_length = max_len_bds * BD_MAX_BUFF_SIZE;
1032		short_bd = bdi_to_bd(ep, short_bdi);
1033		/* length queued */
1034		length_short = le32_to_cpu(short_bd->offset[2]) & 0x1FFFFF;
1035		/* actual length trensfered */
1036		length_short -= SR_BD_LEN(le32_to_cpu(sreport->offset[2]));
1037		actual_length += length_short;
1038		req->usb_req.actual = actual_length;
1039	} else {
1040		req->usb_req.actual = req->usb_req.length -
1041			SR_BD_LEN(le32_to_cpu(sreport->offset[2]));
1042		dev_dbg(bdc->dev,
1043			"len=%d actual=%d bd_xfr->next_hwd_bdi:%d\n",
1044			req->usb_req.length, req->usb_req.actual,
1045			bd_xfr->next_hwd_bdi);
1046	}
1047
1048	/* Update the dequeue pointer */
1049	ep->bd_list.hwd_bdi = bd_xfr->next_hwd_bdi;
1050	if (req->usb_req.actual < req->usb_req.length) {
1051		dev_dbg(bdc->dev, "short xfr on %d\n", ep->ep_num);
1052		if (req->usb_req.short_not_ok)
1053			status = -EREMOTEIO;
1054	}
1055	bdc_req_complete(ep, bd_xfr->req, status);
1056}
1057
1058/* EP0 setup related packet handlers */
1059
1060/*
1061 * Setup packet received, just store the packet and process on next DS or SS
1062 * started SR
1063 */
1064void bdc_xsf_ep0_setup_recv(struct bdc *bdc, struct bdc_sr *sreport)
1065{
1066	struct usb_ctrlrequest *setup_pkt;
1067	u32 len;
1068
1069	dev_dbg(bdc->dev,
1070		"%s ep0_state:%s\n",
1071		__func__, ep0_state_string[bdc->ep0_state]);
1072	/* Store received setup packet */
1073	setup_pkt = &bdc->setup_pkt;
1074	memcpy(setup_pkt, &sreport->offset[0], sizeof(*setup_pkt));
1075	len = le16_to_cpu(setup_pkt->wLength);
1076	if (!len)
1077		bdc->ep0_state = WAIT_FOR_STATUS_START;
1078	else
1079		bdc->ep0_state = WAIT_FOR_DATA_START;
1080
1081
1082	dev_dbg(bdc->dev,
1083		"%s exit ep0_state:%s\n",
1084		__func__, ep0_state_string[bdc->ep0_state]);
1085}
1086
1087/* Stall ep0 */
1088static void ep0_stall(struct bdc *bdc)
1089{
1090	struct bdc_ep	*ep = bdc->bdc_ep_array[1];
1091	struct bdc_req *req;
1092
1093	dev_dbg(bdc->dev, "%s\n", __func__);
1094	bdc->delayed_status = false;
1095	ep_set_halt(ep, 1);
1096
1097	/* de-queue any pendig requests */
1098	while (!list_empty(&ep->queue)) {
1099		req = list_entry(ep->queue.next, struct bdc_req,
1100				queue);
1101		bdc_req_complete(ep, req, -ESHUTDOWN);
1102	}
1103}
1104
1105/* SET_ADD handlers */
1106static int ep0_set_address(struct bdc *bdc, struct usb_ctrlrequest *ctrl)
1107{
1108	enum usb_device_state state = bdc->gadget.state;
1109	int ret = 0;
1110	u32 addr;
1111
1112	addr = le16_to_cpu(ctrl->wValue);
1113	dev_dbg(bdc->dev,
1114		"%s addr:%d dev state:%d\n",
1115		__func__, addr, state);
1116
1117	if (addr > 127)
1118		return -EINVAL;
1119
1120	switch (state) {
1121	case USB_STATE_DEFAULT:
1122	case USB_STATE_ADDRESS:
1123		/* Issue Address device command */
1124		ret = bdc_address_device(bdc, addr);
1125		if (ret)
1126			return ret;
1127
1128		if (addr)
1129			usb_gadget_set_state(&bdc->gadget, USB_STATE_ADDRESS);
1130		else
1131			usb_gadget_set_state(&bdc->gadget, USB_STATE_DEFAULT);
1132
1133		bdc->dev_addr = addr;
1134		break;
1135	default:
1136		dev_warn(bdc->dev,
1137			"SET Address in wrong device state %d\n",
1138			state);
1139		ret = -EINVAL;
1140	}
1141
1142	return ret;
1143}
1144
1145/* Handler for SET/CLEAR FEATURE requests for device */
1146static int ep0_handle_feature_dev(struct bdc *bdc, u16 wValue,
1147							u16 wIndex, bool set)
1148{
1149	enum usb_device_state state = bdc->gadget.state;
1150	u32	usppms = 0;
1151
1152	dev_dbg(bdc->dev, "%s set:%d dev state:%d\n",
1153					__func__, set, state);
1154	switch (wValue) {
1155	case USB_DEVICE_REMOTE_WAKEUP:
1156		dev_dbg(bdc->dev, "USB_DEVICE_REMOTE_WAKEUP\n");
1157		if (set)
1158			bdc->devstatus |= REMOTE_WAKE_ENABLE;
1159		else
1160			bdc->devstatus &= ~REMOTE_WAKE_ENABLE;
1161		break;
1162
1163	case USB_DEVICE_TEST_MODE:
1164		dev_dbg(bdc->dev, "USB_DEVICE_TEST_MODE\n");
1165		if ((wIndex & 0xFF) ||
1166				(bdc->gadget.speed != USB_SPEED_HIGH) || !set)
1167			return -EINVAL;
1168
1169		bdc->test_mode = wIndex >> 8;
1170		break;
1171
1172	case USB_DEVICE_U1_ENABLE:
1173		dev_dbg(bdc->dev, "USB_DEVICE_U1_ENABLE\n");
1174
1175		if (bdc->gadget.speed != USB_SPEED_SUPER ||
1176						state != USB_STATE_CONFIGURED)
1177			return -EINVAL;
1178
1179		usppms =  bdc_readl(bdc->regs, BDC_USPPMS);
1180		if (set) {
1181			/* clear previous u1t */
1182			usppms &= ~BDC_U1T(BDC_U1T_MASK);
1183			usppms |= BDC_U1T(U1_TIMEOUT);
1184			usppms |= BDC_U1E | BDC_PORT_W1S;
1185			bdc->devstatus |= (1 << USB_DEV_STAT_U1_ENABLED);
1186		} else {
1187			usppms &= ~BDC_U1E;
1188			usppms |= BDC_PORT_W1S;
1189			bdc->devstatus &= ~(1 << USB_DEV_STAT_U1_ENABLED);
1190		}
1191		bdc_writel(bdc->regs, BDC_USPPMS, usppms);
1192		break;
1193
1194	case USB_DEVICE_U2_ENABLE:
1195		dev_dbg(bdc->dev, "USB_DEVICE_U2_ENABLE\n");
1196
1197		if (bdc->gadget.speed != USB_SPEED_SUPER ||
1198						state != USB_STATE_CONFIGURED)
1199			return -EINVAL;
1200
1201		usppms = bdc_readl(bdc->regs, BDC_USPPMS);
1202		if (set) {
1203			usppms |= BDC_U2E;
1204			usppms |= BDC_U2A;
1205			bdc->devstatus |= (1 << USB_DEV_STAT_U2_ENABLED);
1206		} else {
1207			usppms &= ~BDC_U2E;
1208			usppms &= ~BDC_U2A;
1209			bdc->devstatus &= ~(1 << USB_DEV_STAT_U2_ENABLED);
1210		}
1211		bdc_writel(bdc->regs, BDC_USPPMS, usppms);
1212		break;
1213
1214	case USB_DEVICE_LTM_ENABLE:
1215		dev_dbg(bdc->dev, "USB_DEVICE_LTM_ENABLE?\n");
1216		if (bdc->gadget.speed != USB_SPEED_SUPER ||
1217						state != USB_STATE_CONFIGURED)
1218			return -EINVAL;
1219		break;
1220	default:
1221		dev_err(bdc->dev, "Unknown wValue:%d\n", wValue);
1222		return -EOPNOTSUPP;
1223	} /* USB_RECIP_DEVICE end */
1224
1225	return 0;
1226}
1227
1228/* SET/CLEAR FEATURE handler */
1229static int ep0_handle_feature(struct bdc *bdc,
1230			      struct usb_ctrlrequest *setup_pkt, bool set)
1231{
1232	enum usb_device_state state = bdc->gadget.state;
1233	struct bdc_ep *ep;
1234	u16 wValue;
1235	u16 wIndex;
1236	int epnum;
1237
1238	wValue = le16_to_cpu(setup_pkt->wValue);
1239	wIndex = le16_to_cpu(setup_pkt->wIndex);
1240
1241	dev_dbg(bdc->dev,
1242		"%s wValue=%d wIndex=%d	devstate=%08x speed=%d set=%d",
1243		__func__, wValue, wIndex, state,
1244		bdc->gadget.speed, set);
1245
1246	switch (setup_pkt->bRequestType & USB_RECIP_MASK) {
1247	case USB_RECIP_DEVICE:
1248		return ep0_handle_feature_dev(bdc, wValue, wIndex, set);
1249	case USB_RECIP_INTERFACE:
1250		dev_dbg(bdc->dev, "USB_RECIP_INTERFACE\n");
1251		/* USB3 spec, sec 9.4.9 */
1252		if (wValue != USB_INTRF_FUNC_SUSPEND)
1253			return -EINVAL;
1254		/* USB3 spec, Table 9-8 */
1255		if (set) {
1256			if (wIndex & USB_INTRF_FUNC_SUSPEND_RW) {
1257				dev_dbg(bdc->dev, "SET REMOTE_WAKEUP\n");
1258				bdc->devstatus |= REMOTE_WAKE_ENABLE;
1259			} else {
1260				dev_dbg(bdc->dev, "CLEAR REMOTE_WAKEUP\n");
1261				bdc->devstatus &= ~REMOTE_WAKE_ENABLE;
1262			}
1263		}
1264		break;
1265
1266	case USB_RECIP_ENDPOINT:
1267		dev_dbg(bdc->dev, "USB_RECIP_ENDPOINT\n");
1268		if (wValue != USB_ENDPOINT_HALT)
1269			return -EINVAL;
1270
1271		epnum = wIndex & USB_ENDPOINT_NUMBER_MASK;
1272		if (epnum) {
1273			if ((wIndex & USB_ENDPOINT_DIR_MASK) == USB_DIR_IN)
1274				epnum = epnum * 2 + 1;
1275			else
1276				epnum *= 2;
1277		} else {
1278			epnum = 1; /*EP0*/
1279		}
1280		/*
1281		 * If CLEAR_FEATURE on ep0 then don't do anything as the stall
1282		 * condition on ep0 has already been cleared when SETUP packet
1283		 * was received.
1284		 */
1285		if (epnum == 1 && !set) {
1286			dev_dbg(bdc->dev, "ep0 stall already cleared\n");
1287			return 0;
1288		}
1289		dev_dbg(bdc->dev, "epnum=%d\n", epnum);
1290		ep = bdc->bdc_ep_array[epnum];
1291		if (!ep)
1292			return -EINVAL;
1293
1294		return ep_set_halt(ep, set);
1295	default:
1296		dev_err(bdc->dev, "Unknown recipient\n");
1297		return -EINVAL;
1298	}
1299
1300	return 0;
1301}
1302
1303/* GET_STATUS request handler */
1304static int ep0_handle_status(struct bdc *bdc,
1305			     struct usb_ctrlrequest *setup_pkt)
1306{
1307	enum usb_device_state state = bdc->gadget.state;
1308	struct bdc_ep *ep;
1309	u16 usb_status = 0;
1310	u32 epnum;
1311	u16 wIndex;
1312
1313	/* USB2.0 spec sec 9.4.5 */
1314	if (state == USB_STATE_DEFAULT)
1315		return -EINVAL;
1316	wIndex = le16_to_cpu(setup_pkt->wIndex);
1317	dev_dbg(bdc->dev, "%s\n", __func__);
1318	usb_status = bdc->devstatus;
1319	switch (setup_pkt->bRequestType & USB_RECIP_MASK) {
1320	case USB_RECIP_DEVICE:
1321		dev_dbg(bdc->dev,
1322			"USB_RECIP_DEVICE devstatus:%08x\n",
1323			bdc->devstatus);
1324		/* USB3 spec, sec 9.4.5 */
1325		if (bdc->gadget.speed == USB_SPEED_SUPER)
1326			usb_status &= ~REMOTE_WAKE_ENABLE;
1327		break;
1328
1329	case USB_RECIP_INTERFACE:
1330		dev_dbg(bdc->dev, "USB_RECIP_INTERFACE\n");
1331		if (bdc->gadget.speed == USB_SPEED_SUPER) {
1332			/*
1333			 * This should come from func for Func remote wkup
1334			 * usb_status |=1;
1335			 */
1336			if (bdc->devstatus & REMOTE_WAKE_ENABLE)
1337				usb_status |= REMOTE_WAKE_ENABLE;
1338		} else {
1339			usb_status = 0;
1340		}
1341
1342		break;
1343
1344	case USB_RECIP_ENDPOINT:
1345		dev_dbg(bdc->dev, "USB_RECIP_ENDPOINT\n");
1346		epnum = wIndex & USB_ENDPOINT_NUMBER_MASK;
1347		if (epnum) {
1348			if ((wIndex & USB_ENDPOINT_DIR_MASK) == USB_DIR_IN)
1349				epnum = epnum*2 + 1;
1350			else
1351				epnum *= 2;
1352		} else {
1353			epnum = 1; /* EP0 */
1354		}
1355
1356		ep = bdc->bdc_ep_array[epnum];
1357		if (!ep) {
1358			dev_err(bdc->dev, "ISSUE, GET_STATUS for invalid EP ?");
1359			return -EINVAL;
1360		}
1361		if (ep->flags & BDC_EP_STALL)
1362			usb_status |= 1 << USB_ENDPOINT_HALT;
1363
1364		break;
1365	default:
1366		dev_err(bdc->dev, "Unknown recipient for get_status\n");
1367		return -EINVAL;
1368	}
1369	/* prepare a data stage for GET_STATUS */
1370	dev_dbg(bdc->dev, "usb_status=%08x\n", usb_status);
1371	*(__le16 *)bdc->ep0_response_buff = cpu_to_le16(usb_status);
1372	bdc->ep0_req.usb_req.length = 2;
1373	bdc->ep0_req.usb_req.buf = &bdc->ep0_response_buff;
1374	ep0_queue_data_stage(bdc);
1375
1376	return 0;
1377}
1378
1379static void ep0_set_sel_cmpl(struct usb_ep *_ep, struct usb_request *_req)
1380{
1381	/* ep0_set_sel_cmpl */
1382}
1383
1384/* Queue data stage to handle 6 byte SET_SEL request */
1385static int ep0_set_sel(struct bdc *bdc,
1386			     struct usb_ctrlrequest *setup_pkt)
1387{
1388	struct bdc_ep	*ep;
1389	u16	wLength;
1390
1391	dev_dbg(bdc->dev, "%s\n", __func__);
1392	wLength = le16_to_cpu(setup_pkt->wLength);
1393	if (unlikely(wLength != 6)) {
1394		dev_err(bdc->dev, "%s Wrong wLength:%d\n", __func__, wLength);
1395		return -EINVAL;
1396	}
1397	ep = bdc->bdc_ep_array[1];
1398	bdc->ep0_req.ep = ep;
1399	bdc->ep0_req.usb_req.length = 6;
1400	bdc->ep0_req.usb_req.buf = bdc->ep0_response_buff;
1401	bdc->ep0_req.usb_req.complete = ep0_set_sel_cmpl;
1402	ep0_queue_data_stage(bdc);
1403
1404	return 0;
1405}
1406
1407/*
1408 * Queue a 0 byte bd only if wLength is more than the length and length is
1409 * a multiple of MaxPacket then queue 0 byte BD
1410 */
1411static int ep0_queue_zlp(struct bdc *bdc)
1412{
1413	int ret;
1414
1415	dev_dbg(bdc->dev, "%s\n", __func__);
1416	bdc->ep0_req.ep = bdc->bdc_ep_array[1];
1417	bdc->ep0_req.usb_req.length = 0;
1418	bdc->ep0_req.usb_req.complete = NULL;
1419	bdc->ep0_state = WAIT_FOR_DATA_START;
1420	ret = bdc_queue_xfr(bdc, &bdc->ep0_req);
1421	if (ret) {
1422		dev_err(bdc->dev, "err queueing zlp :%d\n", ret);
1423		return ret;
1424	}
1425	bdc->ep0_state = WAIT_FOR_DATA_XMIT;
1426
1427	return 0;
1428}
1429
1430/* Control request handler */
1431static int handle_control_request(struct bdc *bdc)
1432{
1433	enum usb_device_state state = bdc->gadget.state;
1434	struct usb_ctrlrequest *setup_pkt;
1435	int delegate_setup = 0;
1436	int ret = 0;
1437	int config = 0;
1438
1439	setup_pkt = &bdc->setup_pkt;
1440	dev_dbg(bdc->dev, "%s\n", __func__);
1441	if ((setup_pkt->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1442		switch (setup_pkt->bRequest) {
1443		case USB_REQ_SET_ADDRESS:
1444			dev_dbg(bdc->dev, "USB_REQ_SET_ADDRESS\n");
1445			ret = ep0_set_address(bdc, setup_pkt);
1446			bdc->devstatus &= DEVSTATUS_CLEAR;
1447			break;
1448
1449		case USB_REQ_SET_CONFIGURATION:
1450			dev_dbg(bdc->dev, "USB_REQ_SET_CONFIGURATION\n");
1451			if (state == USB_STATE_ADDRESS) {
1452				usb_gadget_set_state(&bdc->gadget,
1453							USB_STATE_CONFIGURED);
1454			} else if (state == USB_STATE_CONFIGURED) {
1455				/*
1456				 * USB2 spec sec 9.4.7, if wValue is 0 then dev
1457				 * is moved to addressed state
1458				 */
1459				config = le16_to_cpu(setup_pkt->wValue);
1460				if (!config)
1461					usb_gadget_set_state(
1462							&bdc->gadget,
1463							USB_STATE_ADDRESS);
1464			}
1465			delegate_setup = 1;
1466			break;
1467
1468		case USB_REQ_SET_FEATURE:
1469			dev_dbg(bdc->dev, "USB_REQ_SET_FEATURE\n");
1470			ret = ep0_handle_feature(bdc, setup_pkt, 1);
1471			break;
1472
1473		case USB_REQ_CLEAR_FEATURE:
1474			dev_dbg(bdc->dev, "USB_REQ_CLEAR_FEATURE\n");
1475			ret = ep0_handle_feature(bdc, setup_pkt, 0);
1476			break;
1477
1478		case USB_REQ_GET_STATUS:
1479			dev_dbg(bdc->dev, "USB_REQ_GET_STATUS\n");
1480			ret = ep0_handle_status(bdc, setup_pkt);
1481			break;
1482
1483		case USB_REQ_SET_SEL:
1484			dev_dbg(bdc->dev, "USB_REQ_SET_SEL\n");
1485			ret = ep0_set_sel(bdc, setup_pkt);
1486			break;
1487
1488		case USB_REQ_SET_ISOCH_DELAY:
1489			dev_warn(bdc->dev,
1490			"USB_REQ_SET_ISOCH_DELAY not handled\n");
1491			ret = 0;
1492			break;
1493		default:
1494			delegate_setup = 1;
1495		}
1496	} else {
1497		delegate_setup = 1;
1498	}
1499
1500	if (delegate_setup) {
1501		spin_unlock(&bdc->lock);
1502		ret = bdc->gadget_driver->setup(&bdc->gadget, setup_pkt);
1503		spin_lock(&bdc->lock);
1504	}
1505
1506	return ret;
1507}
1508
1509/* EP0: Data stage started */
1510void bdc_xsf_ep0_data_start(struct bdc *bdc, struct bdc_sr *sreport)
1511{
1512	struct bdc_ep *ep;
1513	int ret = 0;
1514
1515	dev_dbg(bdc->dev, "%s\n", __func__);
1516	ep = bdc->bdc_ep_array[1];
1517	/* If ep0 was stalled, the clear it first */
1518	if (ep->flags & BDC_EP_STALL) {
1519		ret = ep_set_halt(ep, 0);
1520		if (ret)
1521			goto err;
1522	}
1523	if (bdc->ep0_state != WAIT_FOR_DATA_START)
1524		dev_warn(bdc->dev,
1525			"Data stage not expected ep0_state:%s\n",
1526			ep0_state_string[bdc->ep0_state]);
1527
1528	ret = handle_control_request(bdc);
1529	if (ret == USB_GADGET_DELAYED_STATUS) {
1530		/*
1531		 * The ep0 state will remain WAIT_FOR_DATA_START till
1532		 * we received ep_queue on ep0
1533		 */
1534		bdc->delayed_status = true;
1535		return;
1536	}
1537	if (!ret) {
1538		bdc->ep0_state = WAIT_FOR_DATA_XMIT;
1539		dev_dbg(bdc->dev,
1540			"ep0_state:%s", ep0_state_string[bdc->ep0_state]);
1541		return;
1542	}
1543err:
1544	ep0_stall(bdc);
1545}
1546
1547/* EP0: status stage started */
1548void bdc_xsf_ep0_status_start(struct bdc *bdc, struct bdc_sr *sreport)
1549{
1550	struct usb_ctrlrequest *setup_pkt;
1551	struct bdc_ep *ep;
1552	int ret = 0;
1553
1554	dev_dbg(bdc->dev,
1555		"%s ep0_state:%s",
1556		__func__, ep0_state_string[bdc->ep0_state]);
1557	ep = bdc->bdc_ep_array[1];
1558
1559	/* check if ZLP was queued? */
1560	if (bdc->zlp_needed)
1561		bdc->zlp_needed = false;
1562
1563	if (ep->flags & BDC_EP_STALL) {
1564		ret = ep_set_halt(ep, 0);
1565		if (ret)
1566			goto err;
1567	}
1568
1569	if ((bdc->ep0_state != WAIT_FOR_STATUS_START) &&
1570				(bdc->ep0_state != WAIT_FOR_DATA_XMIT))
1571		dev_err(bdc->dev,
1572			"Status stage recv but ep0_state:%s\n",
1573			ep0_state_string[bdc->ep0_state]);
1574
1575	/* check if data stage is in progress ? */
1576	if (bdc->ep0_state == WAIT_FOR_DATA_XMIT) {
1577		bdc->ep0_state = STATUS_PENDING;
1578		/* Status stage will be queued upon Data stage transmit event */
1579		dev_dbg(bdc->dev,
1580			"status started but data  not transmitted yet\n");
1581		return;
1582	}
1583	setup_pkt = &bdc->setup_pkt;
1584
1585	/*
1586	 * 2 stage setup then only process the setup, for 3 stage setup the date
1587	 * stage is already handled
1588	 */
1589	if (!le16_to_cpu(setup_pkt->wLength)) {
1590		ret = handle_control_request(bdc);
1591		if (ret == USB_GADGET_DELAYED_STATUS) {
1592			bdc->delayed_status = true;
1593			/* ep0_state will remain WAIT_FOR_STATUS_START */
1594			return;
1595		}
1596	}
1597	if (!ret) {
1598		/* Queue a status stage BD */
1599		ep0_queue_status_stage(bdc);
1600		bdc->ep0_state = WAIT_FOR_STATUS_XMIT;
1601		dev_dbg(bdc->dev,
1602			"ep0_state:%s", ep0_state_string[bdc->ep0_state]);
1603		return;
1604	}
1605err:
1606	ep0_stall(bdc);
1607}
1608
1609/* Helper function to update ep0 upon SR with xsf_succ or xsf_short */
1610static void ep0_xsf_complete(struct bdc *bdc, struct bdc_sr *sreport)
1611{
1612	dev_dbg(bdc->dev, "%s\n", __func__);
1613	switch (bdc->ep0_state) {
1614	case WAIT_FOR_DATA_XMIT:
1615		bdc->ep0_state = WAIT_FOR_STATUS_START;
1616		break;
1617	case WAIT_FOR_STATUS_XMIT:
1618		bdc->ep0_state = WAIT_FOR_SETUP;
1619		if (bdc->test_mode) {
1620			int ret;
1621
1622			dev_dbg(bdc->dev, "test_mode:%d\n", bdc->test_mode);
1623			ret = bdc_set_test_mode(bdc);
1624			if (ret < 0) {
1625				dev_err(bdc->dev, "Err in setting Test mode\n");
1626				return;
1627			}
1628			bdc->test_mode = 0;
1629		}
1630		break;
1631	case STATUS_PENDING:
1632		bdc_xsf_ep0_status_start(bdc, sreport);
1633		break;
1634
1635	default:
1636		dev_err(bdc->dev,
1637			"Unknown ep0_state:%s\n",
1638			ep0_state_string[bdc->ep0_state]);
1639
1640	}
1641}
1642
1643/* xfr completion status report handler */
1644void bdc_sr_xsf(struct bdc *bdc, struct bdc_sr *sreport)
1645{
1646	struct bdc_ep *ep;
1647	u32 sr_status;
1648	u8 ep_num;
1649
1650	ep_num = (le32_to_cpu(sreport->offset[3])>>4) & 0x1f;
1651	ep = bdc->bdc_ep_array[ep_num];
1652	if (!ep || !(ep->flags & BDC_EP_ENABLED)) {
1653		dev_err(bdc->dev, "xsf for ep not enabled\n");
1654		return;
1655	}
1656	/*
1657	 * check if this transfer is after link went from U3->U0 due
1658	 * to remote wakeup
1659	 */
1660	if (bdc->devstatus & FUNC_WAKE_ISSUED) {
1661		bdc->devstatus &= ~(FUNC_WAKE_ISSUED);
1662		dev_dbg(bdc->dev, "%s clearing FUNC_WAKE_ISSUED flag\n",
1663								__func__);
1664	}
1665	sr_status = XSF_STS(le32_to_cpu(sreport->offset[3]));
1666	dev_dbg_ratelimited(bdc->dev, "%s sr_status=%d ep:%s\n",
1667					__func__, sr_status, ep->name);
1668
1669	switch (sr_status) {
1670	case XSF_SUCC:
1671	case XSF_SHORT:
1672		handle_xsr_succ_status(bdc, ep, sreport);
1673		if (ep_num == 1)
1674			ep0_xsf_complete(bdc, sreport);
1675		break;
1676
1677	case XSF_SETUP_RECV:
1678	case XSF_DATA_START:
1679	case XSF_STATUS_START:
1680		if (ep_num != 1) {
1681			dev_err(bdc->dev,
1682				"ep0 related packets on non ep0 endpoint");
1683			return;
1684		}
1685		bdc->sr_xsf_ep0[sr_status - XSF_SETUP_RECV](bdc, sreport);
1686		break;
1687
1688	case XSF_BABB:
1689		if (ep_num == 1) {
1690			dev_dbg(bdc->dev, "Babble on ep0 zlp_need:%d\n",
1691							bdc->zlp_needed);
1692			/*
1693			 * If the last completed transfer had wLength >Data Len,
1694			 * and Len is multiple of MaxPacket,then queue ZLP
1695			 */
1696			if (bdc->zlp_needed) {
1697				/* queue 0 length bd */
1698				ep0_queue_zlp(bdc);
1699				return;
1700			}
1701		}
1702		dev_warn(bdc->dev, "Babble on ep not handled\n");
1703		break;
1704	default:
1705		dev_warn(bdc->dev, "sr status not handled:%x\n", sr_status);
1706		break;
1707	}
1708}
1709
1710static int bdc_gadget_ep_queue(struct usb_ep *_ep,
1711				struct usb_request *_req, gfp_t gfp_flags)
1712{
1713	struct bdc_req *req;
1714	unsigned long flags;
1715	struct bdc_ep *ep;
1716	struct bdc *bdc;
1717	int ret;
1718
1719	if (!_ep || !_ep->desc)
1720		return -ESHUTDOWN;
1721
1722	if (!_req || !_req->complete || !_req->buf)
1723		return -EINVAL;
1724
1725	ep = to_bdc_ep(_ep);
1726	req = to_bdc_req(_req);
1727	bdc = ep->bdc;
1728	dev_dbg(bdc->dev, "%s ep:%p req:%p\n", __func__, ep, req);
1729	dev_dbg(bdc->dev, "queuing request %p to %s length %d zero:%d\n",
1730				_req, ep->name, _req->length, _req->zero);
1731
1732	if (!ep->usb_ep.desc) {
1733		dev_warn(bdc->dev,
1734			"trying to queue req %p to disabled %s\n",
1735			_req, ep->name);
1736		return -ESHUTDOWN;
1737	}
1738
1739	if (_req->length > MAX_XFR_LEN) {
1740		dev_warn(bdc->dev,
1741			"req length > supported MAX:%d requested:%d\n",
1742			MAX_XFR_LEN, _req->length);
1743		return -EOPNOTSUPP;
1744	}
1745	spin_lock_irqsave(&bdc->lock, flags);
1746	if (ep == bdc->bdc_ep_array[1])
1747		ret = ep0_queue(ep, req);
1748	else
1749		ret = ep_queue(ep, req);
1750
1751	spin_unlock_irqrestore(&bdc->lock, flags);
1752
1753	return ret;
1754}
1755
1756static int bdc_gadget_ep_dequeue(struct usb_ep *_ep,
1757				  struct usb_request *_req)
1758{
1759	struct bdc_req *req;
1760	struct bdc_req *iter;
1761	unsigned long flags;
1762	struct bdc_ep *ep;
1763	struct bdc *bdc;
1764	int ret;
1765
1766	if (!_ep || !_req)
1767		return -EINVAL;
1768
1769	ep = to_bdc_ep(_ep);
1770	req = to_bdc_req(_req);
1771	bdc = ep->bdc;
1772	dev_dbg(bdc->dev, "%s ep:%s req:%p\n", __func__, ep->name, req);
1773	bdc_dbg_bd_list(bdc, ep);
1774	spin_lock_irqsave(&bdc->lock, flags);
1775
1776	req = NULL;
1777	/* make sure it's still queued on this endpoint */
1778	list_for_each_entry(iter, &ep->queue, queue) {
1779		if (&iter->usb_req != _req)
1780			continue;
1781		req = iter;
1782		break;
1783	}
1784	if (!req) {
1785		spin_unlock_irqrestore(&bdc->lock, flags);
1786		dev_err(bdc->dev, "usb_req !=req n");
1787		return -EINVAL;
1788	}
1789	ret = ep_dequeue(ep, req);
1790	if (ret) {
1791		ret = -EOPNOTSUPP;
1792		goto err;
1793	}
1794	bdc_req_complete(ep, req, -ECONNRESET);
1795
1796err:
1797	bdc_dbg_bd_list(bdc, ep);
1798	spin_unlock_irqrestore(&bdc->lock, flags);
1799
1800	return ret;
1801}
1802
1803static int bdc_gadget_ep_set_halt(struct usb_ep *_ep, int value)
1804{
1805	unsigned long flags;
1806	struct bdc_ep *ep;
1807	struct bdc *bdc;
1808	int ret;
1809
1810	ep = to_bdc_ep(_ep);
1811	bdc = ep->bdc;
1812	dev_dbg(bdc->dev, "%s ep:%s value=%d\n", __func__, ep->name, value);
1813	spin_lock_irqsave(&bdc->lock, flags);
1814	if (usb_endpoint_xfer_isoc(ep->usb_ep.desc))
1815		ret = -EINVAL;
1816	else if (!list_empty(&ep->queue))
1817		ret = -EAGAIN;
1818	else
1819		ret = ep_set_halt(ep, value);
1820
1821	spin_unlock_irqrestore(&bdc->lock, flags);
1822
1823	return ret;
1824}
1825
1826static struct usb_request *bdc_gadget_alloc_request(struct usb_ep *_ep,
1827						     gfp_t gfp_flags)
1828{
1829	struct bdc_req *req;
1830	struct bdc_ep *ep;
1831
1832	req = kzalloc(sizeof(*req), gfp_flags);
1833	if (!req)
1834		return NULL;
1835
1836	ep = to_bdc_ep(_ep);
1837	req->ep = ep;
1838	req->epnum = ep->ep_num;
1839	req->usb_req.dma = DMA_ADDR_INVALID;
1840	dev_dbg(ep->bdc->dev, "%s ep:%s req:%p\n", __func__, ep->name, req);
1841
1842	return &req->usb_req;
1843}
1844
1845static void bdc_gadget_free_request(struct usb_ep *_ep,
1846				     struct usb_request *_req)
1847{
1848	struct bdc_req *req;
1849
1850	req = to_bdc_req(_req);
1851	kfree(req);
1852}
1853
1854/* endpoint operations */
1855
1856/* configure endpoint and also allocate resources */
1857static int bdc_gadget_ep_enable(struct usb_ep *_ep,
1858				 const struct usb_endpoint_descriptor *desc)
1859{
1860	unsigned long flags;
1861	struct bdc_ep *ep;
1862	struct bdc *bdc;
1863	int ret;
1864
1865	if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1866		pr_debug("%s invalid parameters\n", __func__);
1867		return -EINVAL;
1868	}
1869
1870	if (!desc->wMaxPacketSize) {
1871		pr_debug("%s missing wMaxPacketSize\n", __func__);
1872		return -EINVAL;
1873	}
1874
1875	ep = to_bdc_ep(_ep);
1876	bdc = ep->bdc;
1877
1878	/* Sanity check, upper layer will not send enable for ep0 */
1879	if (ep == bdc->bdc_ep_array[1])
1880		return -EINVAL;
1881
1882	if (!bdc->gadget_driver
1883	    || bdc->gadget.speed == USB_SPEED_UNKNOWN) {
1884		return -ESHUTDOWN;
1885	}
1886
1887	dev_dbg(bdc->dev, "%s Enabling %s\n", __func__, ep->name);
1888	spin_lock_irqsave(&bdc->lock, flags);
1889	ep->desc = desc;
1890	ep->comp_desc = _ep->comp_desc;
1891	ret = bdc_ep_enable(ep);
1892	spin_unlock_irqrestore(&bdc->lock, flags);
1893
1894	return ret;
1895}
1896
1897static int bdc_gadget_ep_disable(struct usb_ep *_ep)
1898{
1899	unsigned long flags;
1900	struct bdc_ep *ep;
1901	struct bdc *bdc;
1902	int ret;
1903
1904	if (!_ep) {
1905		pr_debug("bdc: invalid parameters\n");
1906		return -EINVAL;
1907	}
1908	ep = to_bdc_ep(_ep);
1909	bdc = ep->bdc;
1910
1911	/* Upper layer will not call this for ep0, but do a sanity check */
1912	if (ep == bdc->bdc_ep_array[1]) {
1913		dev_warn(bdc->dev, "%s called for ep0\n", __func__);
1914		return -EINVAL;
1915	}
1916	dev_dbg(bdc->dev,
1917		"%s() ep:%s ep->flags:%08x\n",
1918		__func__, ep->name, ep->flags);
1919
1920	if (!(ep->flags & BDC_EP_ENABLED)) {
1921		if (bdc->gadget.speed != USB_SPEED_UNKNOWN)
1922			dev_warn(bdc->dev, "%s is already disabled\n",
1923				 ep->name);
1924		return 0;
1925	}
1926	spin_lock_irqsave(&bdc->lock, flags);
1927	ret = bdc_ep_disable(ep);
1928	spin_unlock_irqrestore(&bdc->lock, flags);
1929
1930	return ret;
1931}
1932
1933static const struct usb_ep_ops bdc_gadget_ep_ops = {
1934	.enable = bdc_gadget_ep_enable,
1935	.disable = bdc_gadget_ep_disable,
1936	.alloc_request = bdc_gadget_alloc_request,
1937	.free_request = bdc_gadget_free_request,
1938	.queue = bdc_gadget_ep_queue,
1939	.dequeue = bdc_gadget_ep_dequeue,
1940	.set_halt = bdc_gadget_ep_set_halt
1941};
1942
1943/* dir = 1 is IN */
1944static int init_ep(struct bdc *bdc, u32 epnum, u32 dir)
1945{
1946	struct bdc_ep *ep;
1947
1948	dev_dbg(bdc->dev, "%s epnum=%d dir=%d\n", __func__, epnum, dir);
1949	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1950	if (!ep)
1951		return -ENOMEM;
1952
1953	ep->bdc = bdc;
1954	ep->dir = dir;
1955
1956	if (dir)
1957		ep->usb_ep.caps.dir_in = true;
1958	else
1959		ep->usb_ep.caps.dir_out = true;
1960
1961	/* ep->ep_num is the index inside bdc_ep */
1962	if (epnum == 1) {
1963		ep->ep_num = 1;
1964		bdc->bdc_ep_array[ep->ep_num] = ep;
1965		snprintf(ep->name, sizeof(ep->name), "ep%d", epnum - 1);
1966		usb_ep_set_maxpacket_limit(&ep->usb_ep, EP0_MAX_PKT_SIZE);
1967		ep->usb_ep.caps.type_control = true;
1968		ep->comp_desc = NULL;
1969		bdc->gadget.ep0 = &ep->usb_ep;
1970	} else {
1971		if (dir)
1972			ep->ep_num = epnum * 2 - 1;
1973		else
1974			ep->ep_num = epnum * 2 - 2;
1975
1976		bdc->bdc_ep_array[ep->ep_num] = ep;
1977		snprintf(ep->name, sizeof(ep->name), "ep%d%s", epnum - 1,
1978			 dir & 1 ? "in" : "out");
1979
1980		usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
1981		ep->usb_ep.caps.type_iso = true;
1982		ep->usb_ep.caps.type_bulk = true;
1983		ep->usb_ep.caps.type_int = true;
1984		ep->usb_ep.max_streams = 0;
1985		list_add_tail(&ep->usb_ep.ep_list, &bdc->gadget.ep_list);
1986	}
1987	ep->usb_ep.ops = &bdc_gadget_ep_ops;
1988	ep->usb_ep.name = ep->name;
1989	ep->flags = 0;
1990	ep->ignore_next_sr = false;
1991	dev_dbg(bdc->dev, "ep=%p ep->usb_ep.name=%s epnum=%d ep->epnum=%d\n",
1992				ep, ep->usb_ep.name, epnum, ep->ep_num);
1993
1994	INIT_LIST_HEAD(&ep->queue);
1995
1996	return 0;
1997}
1998
1999/* Init all ep */
2000int bdc_init_ep(struct bdc *bdc)
2001{
2002	u8 epnum;
2003	int ret;
2004
2005	dev_dbg(bdc->dev, "%s()\n", __func__);
2006	INIT_LIST_HEAD(&bdc->gadget.ep_list);
2007	/* init ep0 */
2008	ret = init_ep(bdc, 1, 0);
2009	if (ret) {
2010		dev_err(bdc->dev, "init ep ep0 fail %d\n", ret);
2011		return ret;
2012	}
2013
2014	for (epnum = 2; epnum <= bdc->num_eps / 2; epnum++) {
2015		/* OUT */
2016		ret = init_ep(bdc, epnum, 0);
2017		if (ret) {
2018			dev_err(bdc->dev,
2019				"init ep failed for:%d error: %d\n",
2020				epnum, ret);
2021			return ret;
2022		}
2023
2024		/* IN */
2025		ret = init_ep(bdc, epnum, 1);
2026		if (ret) {
2027			dev_err(bdc->dev,
2028				"init ep failed for:%d error: %d\n",
2029				epnum, ret);
2030			return ret;
2031		}
2032	}
2033
2034	return 0;
2035}