FILE
gadget.c
File preview with syntax highlighting.
linux-6.19.14/drivers/usb/dwc3/gadget.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link 4 * 5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com 6 * 7 * Authors: Felipe Balbi <balbi@ti.com>, 8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de> 9 */ 10 11#include <linux/kernel.h> 12#include <linux/delay.h> 13#include <linux/slab.h> 14#include <linux/spinlock.h> 15#include <linux/platform_device.h> 16#include <linux/pm_runtime.h> 17#include <linux/interrupt.h> 18#include <linux/io.h> 19#include <linux/list.h> 20#include <linux/dma-mapping.h> 21 22#include <linux/usb/ch9.h> 23#include <linux/usb/gadget.h> 24 25#include "debug.h" 26#include "core.h" 27#include "gadget.h" 28#include "io.h" 29 30#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \ 31 & ~((d)->interval - 1)) 32 33/** 34 * dwc3_gadget_set_test_mode - enables usb2 test modes 35 * @dwc: pointer to our context structure 36 * @mode: the mode to set (J, K SE0 NAK, Force Enable) 37 * 38 * Caller should take care of locking. This function will return 0 on 39 * success or -EINVAL if wrong Test Selector is passed. 40 */ 41int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) 42{ 43 u32 reg; 44 45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 46 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 47 48 switch (mode) { 49 case USB_TEST_J: 50 case USB_TEST_K: 51 case USB_TEST_SE0_NAK: 52 case USB_TEST_PACKET: 53 case USB_TEST_FORCE_ENABLE: 54 reg |= mode << 1; 55 break; 56 default: 57 return -EINVAL; 58 } 59 60 dwc3_gadget_dctl_write_safe(dwc, reg); 61 62 return 0; 63} 64 65/** 66 * dwc3_gadget_get_link_state - gets current state of usb link 67 * @dwc: pointer to our context structure 68 * 69 * Caller should take care of locking. This function will 70 * return the link state on success (>= 0) or -ETIMEDOUT. 71 */ 72int dwc3_gadget_get_link_state(struct dwc3 *dwc) 73{ 74 u32 reg; 75 76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 77 78 return DWC3_DSTS_USBLNKST(reg); 79} 80 81/** 82 * dwc3_gadget_set_link_state - sets usb link to a particular state 83 * @dwc: pointer to our context structure 84 * @state: the state to put link into 85 * 86 * Caller should take care of locking. This function will 87 * return 0 on success or -ETIMEDOUT. 88 */ 89int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state) 90{ 91 int retries = 10000; 92 u32 reg; 93 94 /* 95 * Wait until device controller is ready. Only applies to 1.94a and 96 * later RTL. 97 */ 98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) { 99 while (--retries) { 100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 101 if (reg & DWC3_DSTS_DCNRD) 102 udelay(5); 103 else 104 break; 105 } 106 107 if (retries <= 0) 108 return -ETIMEDOUT; 109 } 110 111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 113 114 /* set no action before sending new link state change */ 115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 116 117 /* set requested state */ 118 reg |= DWC3_DCTL_ULSTCHNGREQ(state); 119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 120 121 /* 122 * The following code is racy when called from dwc3_gadget_wakeup, 123 * and is not needed, at least on newer versions 124 */ 125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 126 return 0; 127 128 /* wait for a change in DSTS */ 129 retries = 10000; 130 while (--retries) { 131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 132 133 if (DWC3_DSTS_USBLNKST(reg) == state) 134 return 0; 135 136 udelay(5); 137 } 138 139 return -ETIMEDOUT; 140} 141 142static void dwc3_ep0_reset_state(struct dwc3 *dwc) 143{ 144 unsigned int dir; 145 146 if (dwc->ep0state != EP0_SETUP_PHASE) { 147 dir = !!dwc->ep0_expect_in; 148 if (dwc->ep0state == EP0_DATA_PHASE) 149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); 150 else 151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); 152 153 dwc->eps[0]->trb_enqueue = 0; 154 dwc->eps[1]->trb_enqueue = 0; 155 156 dwc3_ep0_stall_and_restart(dwc); 157 } 158} 159 160/** 161 * dwc3_ep_inc_trb - increment a trb index. 162 * @index: Pointer to the TRB index to increment. 163 * 164 * The index should never point to the link TRB. After incrementing, 165 * if it is point to the link TRB, wrap around to the beginning. The 166 * link TRB is always at the last TRB entry. 167 */ 168static void dwc3_ep_inc_trb(u8 *index) 169{ 170 (*index)++; 171 if (*index == (DWC3_TRB_NUM - 1)) 172 *index = 0; 173} 174 175/** 176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer 177 * @dep: The endpoint whose enqueue pointer we're incrementing 178 */ 179static void dwc3_ep_inc_enq(struct dwc3_ep *dep) 180{ 181 dwc3_ep_inc_trb(&dep->trb_enqueue); 182} 183 184/** 185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer 186 * @dep: The endpoint whose enqueue pointer we're incrementing 187 */ 188static void dwc3_ep_inc_deq(struct dwc3_ep *dep) 189{ 190 dwc3_ep_inc_trb(&dep->trb_dequeue); 191} 192 193static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, 194 struct dwc3_request *req, int status) 195{ 196 struct dwc3 *dwc = dep->dwc; 197 198 list_del(&req->list); 199 req->remaining = 0; 200 req->num_trbs = 0; 201 202 if (req->request.status == -EINPROGRESS) 203 req->request.status = status; 204 205 if (req->trb) 206 usb_gadget_unmap_request_by_dev(dwc->sysdev, 207 &req->request, req->direction); 208 209 req->trb = NULL; 210 trace_dwc3_gadget_giveback(req); 211 212 if (dep->number > 1) 213 pm_runtime_put(dwc->dev); 214} 215 216/** 217 * dwc3_gadget_giveback - call struct usb_request's ->complete callback 218 * @dep: The endpoint to whom the request belongs to 219 * @req: The request we're giving back 220 * @status: completion code for the request 221 * 222 * Must be called with controller's lock held and interrupts disabled. This 223 * function will unmap @req and call its ->complete() callback to notify upper 224 * layers that it has completed. 225 */ 226void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, 227 int status) 228{ 229 struct dwc3 *dwc = dep->dwc; 230 231 /* 232 * The request might have been processed and completed while the 233 * spinlock was released. Skip processing if already completed. 234 */ 235 if (req->status == DWC3_REQUEST_STATUS_COMPLETED) 236 return; 237 238 dwc3_gadget_del_and_unmap_request(dep, req, status); 239 req->status = DWC3_REQUEST_STATUS_COMPLETED; 240 241 spin_unlock(&dwc->lock); 242 usb_gadget_giveback_request(&dep->endpoint, &req->request); 243 spin_lock(&dwc->lock); 244} 245 246/** 247 * dwc3_send_gadget_generic_command - issue a generic command for the controller 248 * @dwc: pointer to the controller context 249 * @cmd: the command to be issued 250 * @param: command parameter 251 * 252 * Caller should take care of locking. Issue @cmd with a given @param to @dwc 253 * and wait for its completion. 254 */ 255int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd, 256 u32 param) 257{ 258 u32 timeout = 500; 259 int status = 0; 260 int ret = 0; 261 u32 reg; 262 263 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); 264 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); 265 266 do { 267 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); 268 if (!(reg & DWC3_DGCMD_CMDACT)) { 269 status = DWC3_DGCMD_STATUS(reg); 270 if (status) 271 ret = -EINVAL; 272 break; 273 } 274 } while (--timeout); 275 276 if (!timeout) { 277 ret = -ETIMEDOUT; 278 status = -ETIMEDOUT; 279 } 280 281 trace_dwc3_gadget_generic_cmd(cmd, param, status); 282 283 return ret; 284} 285 286/** 287 * dwc3_send_gadget_ep_cmd - issue an endpoint command 288 * @dep: the endpoint to which the command is going to be issued 289 * @cmd: the command to be issued 290 * @params: parameters to the command 291 * 292 * Caller should handle locking. This function will issue @cmd with given 293 * @params to @dep and wait for its completion. 294 * 295 * According to the programming guide, if the link state is in L1/L2/U3, 296 * then sending the Start Transfer command may not complete. The 297 * programming guide suggested to bring the link state back to ON/U0 by 298 * performing remote wakeup prior to sending the command. However, don't 299 * initiate remote wakeup when the user/function does not send wakeup 300 * request via wakeup ops. Send the command when it's allowed. 301 * 302 * Notes: 303 * For L1 link state, issuing a command requires the clearing of 304 * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete 305 * the given command (usually within 50us). This should happen within the 306 * command timeout set by driver. No additional step is needed. 307 * 308 * For L2 or U3 link state, the gadget is in USB suspend. Care should be 309 * taken when sending Start Transfer command to ensure that it's done after 310 * USB resume. 311 */ 312int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, 313 struct dwc3_gadget_ep_cmd_params *params) 314{ 315 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 316 struct dwc3 *dwc = dep->dwc; 317 u32 timeout = 5000; 318 u32 saved_config = 0; 319 u32 reg; 320 321 int cmd_status = 0; 322 int ret = -EINVAL; 323 324 /* 325 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or 326 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an 327 * endpoint command. 328 * 329 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY 330 * settings. Restore them after the command is completed. 331 * 332 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2 333 */ 334 if (dwc->gadget->speed <= USB_SPEED_HIGH || 335 DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) { 336 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 337 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) { 338 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY; 339 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 340 } 341 342 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) { 343 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM; 344 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM; 345 } 346 347 if (saved_config) 348 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 349 } 350 351 /* 352 * For some commands such as Update Transfer command, DEPCMDPARn 353 * registers are reserved. Since the driver often sends Update Transfer 354 * command, don't write to DEPCMDPARn to avoid register write delays and 355 * improve performance. 356 */ 357 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) { 358 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); 359 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); 360 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); 361 } 362 363 /* 364 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're 365 * not relying on XferNotReady, we can make use of a special "No 366 * Response Update Transfer" command where we should clear both CmdAct 367 * and CmdIOC bits. 368 * 369 * With this, we don't need to wait for command completion and can 370 * straight away issue further commands to the endpoint. 371 * 372 * NOTICE: We're making an assumption that control endpoints will never 373 * make use of Update Transfer command. This is a safe assumption 374 * because we can never have more than one request at a time with 375 * Control Endpoints. If anybody changes that assumption, this chunk 376 * needs to be updated accordingly. 377 */ 378 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER && 379 !usb_endpoint_xfer_isoc(desc)) 380 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT); 381 else 382 cmd |= DWC3_DEPCMD_CMDACT; 383 384 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); 385 386 if (!(cmd & DWC3_DEPCMD_CMDACT) || 387 (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER && 388 !(cmd & DWC3_DEPCMD_CMDIOC))) { 389 ret = 0; 390 goto skip_status; 391 } 392 393 do { 394 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); 395 if (!(reg & DWC3_DEPCMD_CMDACT)) { 396 cmd_status = DWC3_DEPCMD_STATUS(reg); 397 398 switch (cmd_status) { 399 case 0: 400 ret = 0; 401 break; 402 case DEPEVT_TRANSFER_NO_RESOURCE: 403 dev_WARN(dwc->dev, "No resource for %s\n", 404 dep->name); 405 ret = -EINVAL; 406 break; 407 case DEPEVT_TRANSFER_BUS_EXPIRY: 408 /* 409 * SW issues START TRANSFER command to 410 * isochronous ep with future frame interval. If 411 * future interval time has already passed when 412 * core receives the command, it will respond 413 * with an error status of 'Bus Expiry'. 414 * 415 * Instead of always returning -EINVAL, let's 416 * give a hint to the gadget driver that this is 417 * the case by returning -EAGAIN. 418 */ 419 ret = -EAGAIN; 420 break; 421 default: 422 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); 423 } 424 425 break; 426 } 427 } while (--timeout); 428 429 if (timeout == 0) { 430 ret = -ETIMEDOUT; 431 cmd_status = -ETIMEDOUT; 432 } 433 434skip_status: 435 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); 436 437 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 438 if (ret == 0) 439 dep->flags |= DWC3_EP_TRANSFER_STARTED; 440 441 if (ret != -ETIMEDOUT) 442 dwc3_gadget_ep_get_transfer_index(dep); 443 } 444 445 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER && 446 !(cmd & DWC3_DEPCMD_CMDIOC)) 447 mdelay(1); 448 449 if (saved_config) { 450 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 451 reg |= saved_config; 452 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 453 } 454 455 return ret; 456} 457 458static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) 459{ 460 struct dwc3 *dwc = dep->dwc; 461 struct dwc3_gadget_ep_cmd_params params; 462 u32 cmd = DWC3_DEPCMD_CLEARSTALL; 463 464 /* 465 * As of core revision 2.60a the recommended programming model 466 * is to set the ClearPendIN bit when issuing a Clear Stall EP 467 * command for IN endpoints. This is to prevent an issue where 468 * some (non-compliant) hosts may not send ACK TPs for pending 469 * IN transfers due to a mishandled error condition. Synopsys 470 * STAR 9000614252. 471 */ 472 if (dep->direction && 473 !DWC3_VER_IS_PRIOR(DWC3, 260A) && 474 (dwc->gadget->speed >= USB_SPEED_SUPER)) 475 cmd |= DWC3_DEPCMD_CLEARPENDIN; 476 477 memset(¶ms, 0, sizeof(params)); 478 479 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 480} 481 482static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, 483 struct dwc3_trb *trb) 484{ 485 u32 offset = (char *) trb - (char *) dep->trb_pool; 486 487 return dep->trb_pool_dma + offset; 488} 489 490static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) 491{ 492 struct dwc3 *dwc = dep->dwc; 493 494 if (dep->trb_pool) 495 return 0; 496 497 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, 498 sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 499 &dep->trb_pool_dma, GFP_KERNEL); 500 if (!dep->trb_pool) { 501 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", 502 dep->name); 503 return -ENOMEM; 504 } 505 506 return 0; 507} 508 509static void dwc3_free_trb_pool(struct dwc3_ep *dep) 510{ 511 struct dwc3 *dwc = dep->dwc; 512 513 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 514 dep->trb_pool, dep->trb_pool_dma); 515 516 dep->trb_pool = NULL; 517 dep->trb_pool_dma = 0; 518} 519 520static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) 521{ 522 struct dwc3_gadget_ep_cmd_params params; 523 int ret; 524 525 if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED) 526 return 0; 527 528 memset(¶ms, 0x00, sizeof(params)); 529 530 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); 531 532 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, 533 ¶ms); 534 if (ret) 535 return ret; 536 537 dep->flags |= DWC3_EP_RESOURCE_ALLOCATED; 538 return 0; 539} 540 541/** 542 * dwc3_gadget_start_config - reset endpoint resources 543 * @dwc: pointer to the DWC3 context 544 * @resource_index: DEPSTARTCFG.XferRscIdx value (must be 0 or 2) 545 * 546 * Set resource_index=0 to reset all endpoints' resources allocation. Do this as 547 * part of the power-on/soft-reset initialization. 548 * 549 * Set resource_index=2 to reset only non-control endpoints' resources. Do this 550 * on receiving the SET_CONFIGURATION request or hibernation resume. 551 */ 552int dwc3_gadget_start_config(struct dwc3 *dwc, unsigned int resource_index) 553{ 554 struct dwc3_gadget_ep_cmd_params params; 555 struct dwc3_ep *dep; 556 u32 cmd; 557 int i; 558 int ret; 559 560 if (resource_index != 0 && resource_index != 2) 561 return -EINVAL; 562 563 memset(¶ms, 0x00, sizeof(params)); 564 cmd = DWC3_DEPCMD_DEPSTARTCFG; 565 cmd |= DWC3_DEPCMD_PARAM(resource_index); 566 567 ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, ¶ms); 568 if (ret) 569 return ret; 570 571 /* Reset resource allocation flags */ 572 for (i = resource_index; i < dwc->num_eps; i++) { 573 dep = dwc->eps[i]; 574 if (!dep) 575 continue; 576 577 dep->flags &= ~DWC3_EP_RESOURCE_ALLOCATED; 578 } 579 580 return 0; 581} 582 583static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) 584{ 585 const struct usb_ss_ep_comp_descriptor *comp_desc; 586 const struct usb_endpoint_descriptor *desc; 587 struct dwc3_gadget_ep_cmd_params params; 588 struct dwc3 *dwc = dep->dwc; 589 590 comp_desc = dep->endpoint.comp_desc; 591 desc = dep->endpoint.desc; 592 593 memset(¶ms, 0x00, sizeof(params)); 594 595 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc)) 596 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc)); 597 598 /* Burst size is only needed in SuperSpeed mode */ 599 if (dwc->gadget->speed >= USB_SPEED_SUPER) { 600 u32 burst = dep->endpoint.maxburst; 601 602 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); 603 } 604 605 params.param0 |= action; 606 if (action == DWC3_DEPCFG_ACTION_RESTORE) 607 params.param2 |= dep->saved_state; 608 609 if (usb_endpoint_xfer_control(desc)) 610 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN; 611 612 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) 613 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN; 614 615 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 616 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE 617 | DWC3_DEPCFG_XFER_COMPLETE_EN 618 | DWC3_DEPCFG_STREAM_EVENT_EN; 619 dep->stream_capable = true; 620 } 621 622 if (!usb_endpoint_xfer_control(desc)) 623 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN; 624 625 /* 626 * We are doing 1:1 mapping for endpoints, meaning 627 * Physical Endpoints 2 maps to Logical Endpoint 2 and 628 * so on. We consider the direction bit as part of the physical 629 * endpoint number. So USB endpoint 0x81 is 0x03. 630 */ 631 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); 632 633 /* 634 * We must use the lower 16 TX FIFOs even though 635 * HW might have more 636 */ 637 if (dep->direction) 638 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); 639 640 if (desc->bInterval) { 641 u8 bInterval_m1; 642 643 /* 644 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13. 645 * 646 * NOTE: The programming guide incorrectly stated bInterval_m1 647 * must be set to 0 when operating in fullspeed. Internally the 648 * controller does not have this limitation. See DWC_usb3x 649 * programming guide section 3.2.2.1. 650 */ 651 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); 652 653 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT && 654 dwc->gadget->speed == USB_SPEED_FULL) 655 dep->interval = desc->bInterval; 656 else 657 dep->interval = 1 << (desc->bInterval - 1); 658 659 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1); 660 } 661 662 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); 663} 664 665/** 666 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value 667 * @dwc: pointer to the DWC3 context 668 * @mult: multiplier to be used when calculating the fifo_size 669 * 670 * Calculates the size value based on the equation below: 671 * 672 * DWC3 revision 280A and prior: 673 * fifo_size = mult * (max_packet / mdwidth) + 1; 674 * 675 * DWC3 revision 290A and onwards: 676 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 677 * 678 * The max packet size is set to 1024, as the txfifo requirements mainly apply 679 * to super speed USB use cases. However, it is safe to overestimate the fifo 680 * allocations for other scenarios, i.e. high speed USB. 681 */ 682static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult) 683{ 684 int max_packet = 1024; 685 int fifo_size; 686 int mdwidth; 687 688 mdwidth = dwc3_mdwidth(dwc); 689 690 /* MDWIDTH is represented in bits, we need it in bytes */ 691 mdwidth >>= 3; 692 693 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 694 fifo_size = mult * (max_packet / mdwidth) + 1; 695 else 696 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1; 697 return fifo_size; 698} 699 700/** 701 * dwc3_gadget_calc_ram_depth - calculates the ram depth for txfifo 702 * @dwc: pointer to the DWC3 context 703 */ 704static int dwc3_gadget_calc_ram_depth(struct dwc3 *dwc) 705{ 706 int ram_depth; 707 int fifo_0_start; 708 bool is_single_port_ram; 709 710 /* Check supporting RAM type by HW */ 711 is_single_port_ram = DWC3_SPRAM_TYPE(dwc->hwparams.hwparams1); 712 713 /* 714 * If a single port RAM is utilized, then allocate TxFIFOs from 715 * RAM0. otherwise, allocate them from RAM1. 716 */ 717 ram_depth = is_single_port_ram ? DWC3_RAM0_DEPTH(dwc->hwparams.hwparams6) : 718 DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 719 720 /* 721 * In a single port RAM configuration, the available RAM is shared 722 * between the RX and TX FIFOs. This means that the txfifo can begin 723 * at a non-zero address. 724 */ 725 if (is_single_port_ram) { 726 u32 reg; 727 728 /* Check if TXFIFOs start at non-zero addr */ 729 reg = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 730 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(reg); 731 732 ram_depth -= (fifo_0_start >> 16); 733 } 734 735 return ram_depth; 736} 737 738/** 739 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation 740 * @dwc: pointer to the DWC3 context 741 * 742 * Iterates through all the endpoint registers and clears the previous txfifo 743 * allocations. 744 */ 745void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc) 746{ 747 struct dwc3_ep *dep; 748 int fifo_depth; 749 int size; 750 int num; 751 752 if (!dwc->do_fifo_resize) 753 return; 754 755 /* Read ep0IN related TXFIFO size */ 756 dep = dwc->eps[1]; 757 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 758 if (DWC3_IP_IS(DWC3)) 759 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size); 760 else 761 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size); 762 763 dwc->last_fifo_depth = fifo_depth; 764 /* Clear existing TXFIFO for all IN eps except ep0 */ 765 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); num += 2) { 766 dep = dwc->eps[num]; 767 if (!dep) 768 continue; 769 770 /* Don't change TXFRAMNUM on usb31 version */ 771 size = DWC3_IP_IS(DWC3) ? 0 : 772 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & 773 DWC31_GTXFIFOSIZ_TXFRAMNUM; 774 775 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); 776 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; 777 } 778 dwc->num_ep_resized = 0; 779} 780 781/* 782 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case 783 * @dwc: pointer to our context structure 784 * 785 * This function will a best effort FIFO allocation in order 786 * to improve FIFO usage and throughput, while still allowing 787 * us to enable as many endpoints as possible. 788 * 789 * Keep in mind that this operation will be highly dependent 790 * on the configured size for RAM1 - which contains TxFifo -, 791 * the amount of endpoints enabled on coreConsultant tool, and 792 * the width of the Master Bus. 793 * 794 * In general, FIFO depths are represented with the following equation: 795 * 796 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 797 * 798 * In conjunction with dwc3_gadget_check_config(), this resizing logic will 799 * ensure that all endpoints will have enough internal memory for one max 800 * packet per endpoint. 801 */ 802static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) 803{ 804 struct dwc3 *dwc = dep->dwc; 805 int fifo_0_start; 806 int ram_depth; 807 int fifo_size; 808 int min_depth; 809 int num_in_ep; 810 int remaining; 811 int num_fifos = 1; 812 int fifo; 813 int tmp; 814 815 if (!dwc->do_fifo_resize) 816 return 0; 817 818 /* resize IN endpoints except ep0 */ 819 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) 820 return 0; 821 822 /* bail if already resized */ 823 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) 824 return 0; 825 826 ram_depth = dwc3_gadget_calc_ram_depth(dwc); 827 828 switch (dwc->gadget->speed) { 829 case USB_SPEED_SUPER_PLUS: 830 case USB_SPEED_SUPER: 831 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) || 832 usb_endpoint_xfer_isoc(dep->endpoint.desc)) 833 num_fifos = min_t(unsigned int, 834 dep->endpoint.maxburst, 835 dwc->tx_fifo_resize_max_num); 836 break; 837 case USB_SPEED_HIGH: 838 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 839 num_fifos = min_t(unsigned int, 840 usb_endpoint_maxp_mult(dep->endpoint.desc) + 1, 841 dwc->tx_fifo_resize_max_num); 842 break; 843 } 844 fallthrough; 845 case USB_SPEED_FULL: 846 if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) 847 num_fifos = 2; 848 break; 849 default: 850 break; 851 } 852 853 /* FIFO size for a single buffer */ 854 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1); 855 856 /* Calculate the number of remaining EPs w/o any FIFO */ 857 num_in_ep = dwc->max_cfg_eps; 858 num_in_ep -= dwc->num_ep_resized; 859 860 /* Reserve at least one FIFO for the number of IN EPs */ 861 min_depth = num_in_ep * (fifo + 1); 862 remaining = ram_depth - min_depth - dwc->last_fifo_depth; 863 remaining = max_t(int, 0, remaining); 864 /* 865 * We've already reserved 1 FIFO per EP, so check what we can fit in 866 * addition to it. If there is not enough remaining space, allocate 867 * all the remaining space to the EP. 868 */ 869 fifo_size = (num_fifos - 1) * fifo; 870 if (remaining < fifo_size) 871 fifo_size = remaining; 872 873 fifo_size += fifo; 874 /* Last increment according to the TX FIFO size equation */ 875 fifo_size++; 876 877 /* Check if TXFIFOs start at non-zero addr */ 878 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 879 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp); 880 881 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); 882 if (DWC3_IP_IS(DWC3)) 883 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 884 else 885 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 886 887 /* Check fifo size allocation doesn't exceed available RAM size. */ 888 if (dwc->last_fifo_depth >= ram_depth) { 889 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", 890 dwc->last_fifo_depth, ram_depth, 891 dep->endpoint.name, fifo_size); 892 if (DWC3_IP_IS(DWC3)) 893 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 894 else 895 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 896 897 dwc->last_fifo_depth -= fifo_size; 898 return -ENOMEM; 899 } 900 901 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); 902 dep->flags |= DWC3_EP_TXFIFO_RESIZED; 903 dwc->num_ep_resized++; 904 905 return 0; 906} 907 908/** 909 * __dwc3_gadget_ep_enable - initializes a hw endpoint 910 * @dep: endpoint to be initialized 911 * @action: one of INIT, MODIFY or RESTORE 912 * 913 * Caller should take care of locking. Execute all necessary commands to 914 * initialize a HW endpoint so it can be used by a gadget driver. 915 */ 916static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) 917{ 918 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 919 struct dwc3 *dwc = dep->dwc; 920 921 u32 reg; 922 int ret; 923 924 if (!(dep->flags & DWC3_EP_ENABLED)) { 925 ret = dwc3_gadget_resize_tx_fifos(dep); 926 if (ret) 927 return ret; 928 } 929 930 ret = dwc3_gadget_set_ep_config(dep, action); 931 if (ret) 932 return ret; 933 934 ret = dwc3_gadget_set_xfer_resource(dep); 935 if (ret) 936 return ret; 937 938 if (!(dep->flags & DWC3_EP_ENABLED)) { 939 struct dwc3_trb *trb_st_hw; 940 struct dwc3_trb *trb_link; 941 942 dep->type = usb_endpoint_type(desc); 943 dep->flags |= DWC3_EP_ENABLED; 944 945 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 946 reg |= DWC3_DALEPENA_EP(dep->number); 947 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 948 949 dep->trb_dequeue = 0; 950 dep->trb_enqueue = 0; 951 952 if (usb_endpoint_xfer_control(desc)) 953 goto out; 954 955 /* Initialize the TRB ring */ 956 memset(dep->trb_pool, 0, 957 sizeof(struct dwc3_trb) * DWC3_TRB_NUM); 958 959 /* Link TRB. The HWO bit is never reset */ 960 trb_st_hw = &dep->trb_pool[0]; 961 962 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; 963 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 964 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 965 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; 966 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; 967 } 968 969 /* 970 * Issue StartTransfer here with no-op TRB so we can always rely on No 971 * Response Update Transfer command. 972 */ 973 if (usb_endpoint_xfer_bulk(desc) || 974 usb_endpoint_xfer_int(desc)) { 975 struct dwc3_gadget_ep_cmd_params params; 976 struct dwc3_trb *trb; 977 dma_addr_t trb_dma; 978 u32 cmd; 979 980 memset(¶ms, 0, sizeof(params)); 981 trb = &dep->trb_pool[0]; 982 trb_dma = dwc3_trb_dma_offset(dep, trb); 983 984 params.param0 = upper_32_bits(trb_dma); 985 params.param1 = lower_32_bits(trb_dma); 986 987 cmd = DWC3_DEPCMD_STARTTRANSFER; 988 989 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 990 if (ret < 0) 991 return ret; 992 993 if (dep->stream_capable) { 994 /* 995 * For streams, at start, there maybe a race where the 996 * host primes the endpoint before the function driver 997 * queues a request to initiate a stream. In that case, 998 * the controller will not see the prime to generate the 999 * ERDY and start stream. To workaround this, issue a 1000 * no-op TRB as normal, but end it immediately. As a 1001 * result, when the function driver queues the request, 1002 * the next START_TRANSFER command will cause the 1003 * controller to generate an ERDY to initiate the 1004 * stream. 1005 */ 1006 dwc3_stop_active_transfer(dep, true, true); 1007 1008 /* 1009 * All stream eps will reinitiate stream on NoStream 1010 * rejection. 1011 * 1012 * However, if the controller is capable of 1013 * TXF_FLUSH_BYPASS, then IN direction endpoints will 1014 * automatically restart the stream without the driver 1015 * initiation. 1016 */ 1017 if (!dep->direction || 1018 !(dwc->hwparams.hwparams9 & 1019 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS)) 1020 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; 1021 } 1022 } 1023 1024out: 1025 trace_dwc3_gadget_ep_enable(dep); 1026 1027 return 0; 1028} 1029 1030void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) 1031{ 1032 struct dwc3_request *req; 1033 1034 dwc3_stop_active_transfer(dep, true, false); 1035 1036 /* If endxfer is delayed, avoid unmapping requests */ 1037 if (dep->flags & DWC3_EP_DELAY_STOP) 1038 return; 1039 1040 /* - giveback all requests to gadget driver */ 1041 while (!list_empty(&dep->started_list)) { 1042 req = next_request(&dep->started_list); 1043 1044 dwc3_gadget_giveback(dep, req, status); 1045 } 1046 1047 while (!list_empty(&dep->pending_list)) { 1048 req = next_request(&dep->pending_list); 1049 1050 dwc3_gadget_giveback(dep, req, status); 1051 } 1052 1053 while (!list_empty(&dep->cancelled_list)) { 1054 req = next_request(&dep->cancelled_list); 1055 1056 dwc3_gadget_giveback(dep, req, status); 1057 } 1058} 1059 1060/** 1061 * __dwc3_gadget_ep_disable - disables a hw endpoint 1062 * @dep: the endpoint to disable 1063 * 1064 * This function undoes what __dwc3_gadget_ep_enable did and also removes 1065 * requests which are currently being processed by the hardware and those which 1066 * are not yet scheduled. 1067 * 1068 * Caller should take care of locking. 1069 */ 1070static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) 1071{ 1072 struct dwc3 *dwc = dep->dwc; 1073 u32 reg; 1074 u32 mask; 1075 1076 trace_dwc3_gadget_ep_disable(dep); 1077 1078 /* make sure HW endpoint isn't stalled */ 1079 if (dep->flags & DWC3_EP_STALL) 1080 __dwc3_gadget_ep_set_halt(dep, 0, false); 1081 1082 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 1083 reg &= ~DWC3_DALEPENA_EP(dep->number); 1084 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 1085 1086 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); 1087 1088 dep->stream_capable = false; 1089 dep->type = 0; 1090 mask = DWC3_EP_TXFIFO_RESIZED | DWC3_EP_RESOURCE_ALLOCATED; 1091 /* 1092 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is 1093 * set. Do not clear DEP flags, so that the end transfer command will 1094 * be reattempted during the next SETUP stage. 1095 */ 1096 if (dep->flags & DWC3_EP_DELAY_STOP) 1097 mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED); 1098 dep->flags &= mask; 1099 1100 /* Clear out the ep descriptors for non-ep0 */ 1101 if (dep->number > 1) { 1102 dep->endpoint.comp_desc = NULL; 1103 dep->endpoint.desc = NULL; 1104 } 1105 1106 return 0; 1107} 1108 1109/* -------------------------------------------------------------------------- */ 1110 1111static int dwc3_gadget_ep0_enable(struct usb_ep *ep, 1112 const struct usb_endpoint_descriptor *desc) 1113{ 1114 return -EINVAL; 1115} 1116 1117static int dwc3_gadget_ep0_disable(struct usb_ep *ep) 1118{ 1119 return -EINVAL; 1120} 1121 1122/* -------------------------------------------------------------------------- */ 1123 1124static int dwc3_gadget_ep_enable(struct usb_ep *ep, 1125 const struct usb_endpoint_descriptor *desc) 1126{ 1127 struct dwc3_ep *dep; 1128 struct dwc3 *dwc; 1129 unsigned long flags; 1130 int ret; 1131 1132 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 1133 pr_debug("dwc3: invalid parameters\n"); 1134 return -EINVAL; 1135 } 1136 1137 if (!desc->wMaxPacketSize) { 1138 pr_debug("dwc3: missing wMaxPacketSize\n"); 1139 return -EINVAL; 1140 } 1141 1142 dep = to_dwc3_ep(ep); 1143 dwc = dep->dwc; 1144 1145 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, 1146 "%s is already enabled\n", 1147 dep->name)) 1148 return 0; 1149 1150 spin_lock_irqsave(&dwc->lock, flags); 1151 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 1152 spin_unlock_irqrestore(&dwc->lock, flags); 1153 1154 return ret; 1155} 1156 1157static int dwc3_gadget_ep_disable(struct usb_ep *ep) 1158{ 1159 struct dwc3_ep *dep; 1160 struct dwc3 *dwc; 1161 unsigned long flags; 1162 int ret; 1163 1164 if (!ep) { 1165 pr_debug("dwc3: invalid parameters\n"); 1166 return -EINVAL; 1167 } 1168 1169 dep = to_dwc3_ep(ep); 1170 dwc = dep->dwc; 1171 1172 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), 1173 "%s is already disabled\n", 1174 dep->name)) 1175 return 0; 1176 1177 spin_lock_irqsave(&dwc->lock, flags); 1178 ret = __dwc3_gadget_ep_disable(dep); 1179 spin_unlock_irqrestore(&dwc->lock, flags); 1180 1181 return ret; 1182} 1183 1184static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, 1185 gfp_t gfp_flags) 1186{ 1187 struct dwc3_request *req; 1188 struct dwc3_ep *dep = to_dwc3_ep(ep); 1189 1190 req = kzalloc(sizeof(*req), gfp_flags); 1191 if (!req) 1192 return NULL; 1193 1194 req->direction = dep->direction; 1195 req->epnum = dep->number; 1196 req->dep = dep; 1197 req->status = DWC3_REQUEST_STATUS_UNKNOWN; 1198 1199 trace_dwc3_alloc_request(req); 1200 1201 return &req->request; 1202} 1203 1204static void dwc3_gadget_ep_free_request(struct usb_ep *ep, 1205 struct usb_request *request) 1206{ 1207 struct dwc3_request *req = to_dwc3_request(request); 1208 1209 trace_dwc3_free_request(req); 1210 kfree(req); 1211} 1212 1213/** 1214 * dwc3_ep_prev_trb - returns the previous TRB in the ring 1215 * @dep: The endpoint with the TRB ring 1216 * @index: The index of the current TRB in the ring 1217 * 1218 * Returns the TRB prior to the one pointed to by the index. If the 1219 * index is 0, we will wrap backwards, skip the link TRB, and return 1220 * the one just before that. 1221 */ 1222static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) 1223{ 1224 u8 tmp = index; 1225 1226 if (!tmp) 1227 tmp = DWC3_TRB_NUM - 1; 1228 1229 return &dep->trb_pool[tmp - 1]; 1230} 1231 1232static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) 1233{ 1234 u8 trbs_left; 1235 1236 /* 1237 * If the enqueue & dequeue are equal then the TRB ring is either full 1238 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs 1239 * pending to be processed by the driver. 1240 */ 1241 if (dep->trb_enqueue == dep->trb_dequeue) { 1242 struct dwc3_request *req; 1243 1244 /* 1245 * If there is any request remained in the started_list with 1246 * active TRBs at this point, then there is no TRB available. 1247 */ 1248 req = next_request(&dep->started_list); 1249 if (req && req->num_trbs) 1250 return 0; 1251 1252 return DWC3_TRB_NUM - 1; 1253 } 1254 1255 trbs_left = dep->trb_dequeue - dep->trb_enqueue; 1256 trbs_left &= (DWC3_TRB_NUM - 1); 1257 1258 if (dep->trb_dequeue < dep->trb_enqueue) 1259 trbs_left--; 1260 1261 return trbs_left; 1262} 1263 1264/** 1265 * dwc3_prepare_one_trb - setup one TRB from one request 1266 * @dep: endpoint for which this request is prepared 1267 * @req: dwc3_request pointer 1268 * @trb_length: buffer size of the TRB 1269 * @chain: should this TRB be chained to the next? 1270 * @node: only for isochronous endpoints. First TRB needs different type. 1271 * @use_bounce_buffer: set to use bounce buffer 1272 * @must_interrupt: set to interrupt on TRB completion 1273 */ 1274static void dwc3_prepare_one_trb(struct dwc3_ep *dep, 1275 struct dwc3_request *req, unsigned int trb_length, 1276 unsigned int chain, unsigned int node, bool use_bounce_buffer, 1277 bool must_interrupt) 1278{ 1279 struct dwc3_trb *trb; 1280 dma_addr_t dma; 1281 unsigned int stream_id = req->request.stream_id; 1282 unsigned int short_not_ok = req->request.short_not_ok; 1283 unsigned int no_interrupt = req->request.no_interrupt; 1284 unsigned int is_last = req->request.is_last; 1285 struct dwc3 *dwc = dep->dwc; 1286 struct usb_gadget *gadget = dwc->gadget; 1287 enum usb_device_speed speed = gadget->speed; 1288 1289 if (use_bounce_buffer) 1290 dma = dep->dwc->bounce_addr; 1291 else if (req->request.num_sgs > 0) 1292 dma = sg_dma_address(req->start_sg); 1293 else 1294 dma = req->request.dma; 1295 1296 trb = &dep->trb_pool[dep->trb_enqueue]; 1297 1298 if (!req->trb) { 1299 dwc3_gadget_move_started_request(req); 1300 req->trb = trb; 1301 req->trb_dma = dwc3_trb_dma_offset(dep, trb); 1302 } 1303 1304 req->num_trbs++; 1305 1306 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); 1307 trb->bpl = lower_32_bits(dma); 1308 trb->bph = upper_32_bits(dma); 1309 1310 switch (usb_endpoint_type(dep->endpoint.desc)) { 1311 case USB_ENDPOINT_XFER_CONTROL: 1312 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; 1313 break; 1314 1315 case USB_ENDPOINT_XFER_ISOC: 1316 if (!node) { 1317 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; 1318 1319 /* 1320 * USB Specification 2.0 Section 5.9.2 states that: "If 1321 * there is only a single transaction in the microframe, 1322 * only a DATA0 data packet PID is used. If there are 1323 * two transactions per microframe, DATA1 is used for 1324 * the first transaction data packet and DATA0 is used 1325 * for the second transaction data packet. If there are 1326 * three transactions per microframe, DATA2 is used for 1327 * the first transaction data packet, DATA1 is used for 1328 * the second, and DATA0 is used for the third." 1329 * 1330 * IOW, we should satisfy the following cases: 1331 * 1332 * 1) length <= maxpacket 1333 * - DATA0 1334 * 1335 * 2) maxpacket < length <= (2 * maxpacket) 1336 * - DATA1, DATA0 1337 * 1338 * 3) (2 * maxpacket) < length <= (3 * maxpacket) 1339 * - DATA2, DATA1, DATA0 1340 */ 1341 if (speed == USB_SPEED_HIGH) { 1342 struct usb_ep *ep = &dep->endpoint; 1343 unsigned int mult = 2; 1344 unsigned int maxp = usb_endpoint_maxp(ep->desc); 1345 1346 if (req->request.length <= (2 * maxp)) 1347 mult--; 1348 1349 if (req->request.length <= maxp) 1350 mult--; 1351 1352 trb->size |= DWC3_TRB_SIZE_PCM1(mult); 1353 } 1354 } else { 1355 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; 1356 } 1357 1358 if (!no_interrupt && !chain) 1359 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1360 break; 1361 1362 case USB_ENDPOINT_XFER_BULK: 1363 case USB_ENDPOINT_XFER_INT: 1364 trb->ctrl = DWC3_TRBCTL_NORMAL; 1365 break; 1366 default: 1367 /* 1368 * This is only possible with faulty memory because we 1369 * checked it already :) 1370 */ 1371 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", 1372 usb_endpoint_type(dep->endpoint.desc)); 1373 } 1374 1375 /* 1376 * Enable Continue on Short Packet 1377 * when endpoint is not a stream capable 1378 */ 1379 if (usb_endpoint_dir_out(dep->endpoint.desc)) { 1380 if (!dep->stream_capable) 1381 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1382 1383 if (short_not_ok) 1384 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1385 } 1386 1387 /* All TRBs setup for MST must set CSP=1 when LST=0 */ 1388 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) 1389 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1390 1391 if ((!no_interrupt && !chain) || must_interrupt) 1392 trb->ctrl |= DWC3_TRB_CTRL_IOC; 1393 1394 if (chain) 1395 trb->ctrl |= DWC3_TRB_CTRL_CHN; 1396 else if (dep->stream_capable && is_last && 1397 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1398 trb->ctrl |= DWC3_TRB_CTRL_LST; 1399 1400 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) 1401 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); 1402 1403 /* 1404 * As per data book 4.2.3.2TRB Control Bit Rules section 1405 * 1406 * The controller autonomously checks the HWO field of a TRB to determine if the 1407 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB 1408 * is valid before setting the HWO field to '1'. In most systems, this means that 1409 * software must update the fourth DWORD of a TRB last. 1410 * 1411 * However there is a possibility of CPU re-ordering here which can cause 1412 * controller to observe the HWO bit set prematurely. 1413 * Add a write memory barrier to prevent CPU re-ordering. 1414 */ 1415 wmb(); 1416 trb->ctrl |= DWC3_TRB_CTRL_HWO; 1417 1418 dwc3_ep_inc_enq(dep); 1419 1420 trace_dwc3_prepare_trb(dep, trb); 1421} 1422 1423static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req) 1424{ 1425 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1426 unsigned int rem = req->request.length % maxp; 1427 1428 if ((req->request.length && req->request.zero && !rem && 1429 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || 1430 (!req->direction && rem)) 1431 return true; 1432 1433 return false; 1434} 1435 1436/** 1437 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry 1438 * @dep: The endpoint that the request belongs to 1439 * @req: The request to prepare 1440 * @entry_length: The last SG entry size 1441 * @node: Indicates whether this is not the first entry (for isoc only) 1442 * 1443 * Return the number of TRBs prepared. 1444 */ 1445static int dwc3_prepare_last_sg(struct dwc3_ep *dep, 1446 struct dwc3_request *req, unsigned int entry_length, 1447 unsigned int node) 1448{ 1449 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1450 unsigned int rem = req->request.length % maxp; 1451 unsigned int num_trbs = 1; 1452 bool needs_extra_trb; 1453 1454 if (dwc3_needs_extra_trb(dep, req)) 1455 num_trbs++; 1456 1457 if (dwc3_calc_trbs_left(dep) < num_trbs) 1458 return 0; 1459 1460 needs_extra_trb = num_trbs > 1; 1461 1462 /* Prepare a normal TRB */ 1463 if (req->direction || req->request.length) 1464 dwc3_prepare_one_trb(dep, req, entry_length, 1465 needs_extra_trb, node, false, false); 1466 1467 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */ 1468 if ((!req->direction && !req->request.length) || needs_extra_trb) 1469 dwc3_prepare_one_trb(dep, req, 1470 req->direction ? 0 : maxp - rem, 1471 false, 1, true, false); 1472 1473 return num_trbs; 1474} 1475 1476static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep, 1477 struct dwc3_request *req) 1478{ 1479 struct scatterlist *sg = req->start_sg; 1480 struct scatterlist *s; 1481 int i; 1482 unsigned int length = req->request.length; 1483 unsigned int remaining = req->num_pending_sgs; 1484 unsigned int num_queued_sgs = req->request.num_mapped_sgs - remaining; 1485 unsigned int num_trbs = req->num_trbs; 1486 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req); 1487 1488 /* 1489 * If we resume preparing the request, then get the remaining length of 1490 * the request and resume where we left off. 1491 */ 1492 for_each_sg(req->request.sg, s, num_queued_sgs, i) 1493 length -= sg_dma_len(s); 1494 1495 for_each_sg(sg, s, remaining, i) { 1496 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep); 1497 unsigned int trb_length; 1498 bool must_interrupt = false; 1499 bool last_sg = false; 1500 1501 trb_length = min_t(unsigned int, length, sg_dma_len(s)); 1502 1503 length -= trb_length; 1504 1505 /* 1506 * IOMMU driver is coalescing the list of sgs which shares a 1507 * page boundary into one and giving it to USB driver. With 1508 * this the number of sgs mapped is not equal to the number of 1509 * sgs passed. So mark the chain bit to false if it isthe last 1510 * mapped sg. 1511 */ 1512 if ((i == remaining - 1) || !length) 1513 last_sg = true; 1514 1515 if (!num_trbs_left) 1516 break; 1517 1518 if (last_sg) { 1519 if (!dwc3_prepare_last_sg(dep, req, trb_length, i)) 1520 break; 1521 } else { 1522 /* 1523 * Look ahead to check if we have enough TRBs for the 1524 * next SG entry. If not, set interrupt on this TRB to 1525 * resume preparing the next SG entry when more TRBs are 1526 * free. 1527 */ 1528 if (num_trbs_left == 1 || (needs_extra_trb && 1529 num_trbs_left <= 2 && 1530 sg_dma_len(sg_next(s)) >= length)) { 1531 struct dwc3_request *r; 1532 1533 /* Check if previous requests already set IOC */ 1534 list_for_each_entry(r, &dep->started_list, list) { 1535 if (r != req && !r->request.no_interrupt) 1536 break; 1537 1538 if (r == req) 1539 must_interrupt = true; 1540 } 1541 } 1542 1543 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false, 1544 must_interrupt); 1545 } 1546 1547 /* 1548 * There can be a situation where all sgs in sglist are not 1549 * queued because of insufficient trb number. To handle this 1550 * case, update start_sg to next sg to be queued, so that 1551 * we have free trbs we can continue queuing from where we 1552 * previously stopped 1553 */ 1554 if (!last_sg) 1555 req->start_sg = sg_next(s); 1556 1557 req->num_pending_sgs--; 1558 1559 /* 1560 * The number of pending SG entries may not correspond to the 1561 * number of mapped SG entries. If all the data are queued, then 1562 * don't include unused SG entries. 1563 */ 1564 if (length == 0) { 1565 req->num_pending_sgs = 0; 1566 break; 1567 } 1568 1569 if (must_interrupt) 1570 break; 1571 } 1572 1573 return req->num_trbs - num_trbs; 1574} 1575 1576static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep, 1577 struct dwc3_request *req) 1578{ 1579 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); 1580} 1581 1582/* 1583 * dwc3_prepare_trbs - setup TRBs from requests 1584 * @dep: endpoint for which requests are being prepared 1585 * 1586 * The function goes through the requests list and sets up TRBs for the 1587 * transfers. The function returns once there are no more TRBs available or 1588 * it runs out of requests. 1589 * 1590 * Returns the number of TRBs prepared or negative errno. 1591 */ 1592static int dwc3_prepare_trbs(struct dwc3_ep *dep) 1593{ 1594 struct dwc3_request *req, *n; 1595 int ret = 0; 1596 1597 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM); 1598 1599 /* 1600 * We can get in a situation where there's a request in the started list 1601 * but there weren't enough TRBs to fully kick it in the first time 1602 * around, so it has been waiting for more TRBs to be freed up. 1603 * 1604 * In that case, we should check if we have a request with pending_sgs 1605 * in the started list and prepare TRBs for that request first, 1606 * otherwise we will prepare TRBs completely out of order and that will 1607 * break things. 1608 */ 1609 list_for_each_entry(req, &dep->started_list, list) { 1610 if (req->num_pending_sgs > 0) { 1611 ret = dwc3_prepare_trbs_sg(dep, req); 1612 if (!ret || req->num_pending_sgs) 1613 return ret; 1614 } 1615 1616 if (!dwc3_calc_trbs_left(dep)) 1617 return ret; 1618 1619 /* 1620 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1621 * burst capability may try to read and use TRBs beyond the 1622 * active transfer instead of stopping. 1623 */ 1624 if (dep->stream_capable && req->request.is_last && 1625 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1626 return ret; 1627 } 1628 1629 list_for_each_entry_safe(req, n, &dep->pending_list, list) { 1630 struct dwc3 *dwc = dep->dwc; 1631 1632 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, 1633 dep->direction); 1634 if (ret) 1635 return ret; 1636 1637 req->start_sg = req->request.sg; 1638 req->num_pending_sgs = req->request.num_mapped_sgs; 1639 1640 if (req->num_pending_sgs > 0) { 1641 ret = dwc3_prepare_trbs_sg(dep, req); 1642 if (req->num_pending_sgs) 1643 return ret; 1644 } else { 1645 ret = dwc3_prepare_trbs_linear(dep, req); 1646 } 1647 1648 if (!ret || !dwc3_calc_trbs_left(dep)) 1649 return ret; 1650 1651 /* 1652 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1653 * burst capability may try to read and use TRBs beyond the 1654 * active transfer instead of stopping. 1655 */ 1656 if (dep->stream_capable && req->request.is_last && 1657 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1658 return ret; 1659 } 1660 1661 return ret; 1662} 1663 1664static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep); 1665 1666static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) 1667{ 1668 struct dwc3_gadget_ep_cmd_params params; 1669 struct dwc3_request *req; 1670 int starting; 1671 int ret; 1672 u32 cmd; 1673 1674 /* 1675 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0). 1676 * This happens when we need to stop and restart a transfer such as in 1677 * the case of reinitiating a stream or retrying an isoc transfer. 1678 */ 1679 ret = dwc3_prepare_trbs(dep); 1680 if (ret < 0) 1681 return ret; 1682 1683 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); 1684 1685 /* 1686 * If there's no new TRB prepared and we don't need to restart a 1687 * transfer, there's no need to update the transfer. 1688 */ 1689 if (!ret && !starting) 1690 return ret; 1691 1692 req = next_request(&dep->started_list); 1693 if (!req) { 1694 dep->flags |= DWC3_EP_PENDING_REQUEST; 1695 return 0; 1696 } 1697 1698 memset(¶ms, 0, sizeof(params)); 1699 1700 if (starting) { 1701 params.param0 = upper_32_bits(req->trb_dma); 1702 params.param1 = lower_32_bits(req->trb_dma); 1703 cmd = DWC3_DEPCMD_STARTTRANSFER; 1704 1705 if (dep->stream_capable) 1706 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); 1707 1708 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 1709 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); 1710 } else { 1711 cmd = DWC3_DEPCMD_UPDATETRANSFER | 1712 DWC3_DEPCMD_PARAM(dep->resource_index); 1713 } 1714 1715 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1716 if (ret < 0) { 1717 struct dwc3_request *tmp; 1718 1719 if (ret == -EAGAIN) 1720 return ret; 1721 1722 dwc3_stop_active_transfer(dep, true, true); 1723 1724 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 1725 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED); 1726 1727 /* If ep isn't started, then there's no end transfer pending */ 1728 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 1729 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 1730 1731 return ret; 1732 } 1733 1734 if (dep->stream_capable && req->request.is_last && 1735 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1736 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; 1737 1738 return 0; 1739} 1740 1741static int __dwc3_gadget_get_frame(struct dwc3 *dwc) 1742{ 1743 u32 reg; 1744 1745 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1746 return DWC3_DSTS_SOFFN(reg); 1747} 1748 1749/** 1750 * __dwc3_stop_active_transfer - stop the current active transfer 1751 * @dep: isoc endpoint 1752 * @force: set forcerm bit in the command 1753 * @interrupt: command complete interrupt after End Transfer command 1754 * 1755 * When setting force, the ForceRM bit will be set. In that case 1756 * the controller won't update the TRB progress on command 1757 * completion. It also won't clear the HWO bit in the TRB. 1758 * The command will also not complete immediately in that case. 1759 */ 1760static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) 1761{ 1762 struct dwc3_gadget_ep_cmd_params params; 1763 u32 cmd; 1764 int ret; 1765 1766 cmd = DWC3_DEPCMD_ENDTRANSFER; 1767 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; 1768 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0; 1769 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); 1770 memset(¶ms, 0, sizeof(params)); 1771 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1772 /* 1773 * If the End Transfer command was timed out while the device is 1774 * not in SETUP phase, it's possible that an incoming Setup packet 1775 * may prevent the command's completion. Let's retry when the 1776 * ep0state returns to EP0_SETUP_PHASE. 1777 */ 1778 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { 1779 dep->flags |= DWC3_EP_DELAY_STOP; 1780 return 0; 1781 } 1782 1783 if (ret) 1784 dev_err_ratelimited(dep->dwc->dev, 1785 "end transfer failed: %d\n", ret); 1786 1787 dep->resource_index = 0; 1788 1789 if (!interrupt) 1790 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 1791 else if (!ret) 1792 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; 1793 1794 dep->flags &= ~DWC3_EP_DELAY_STOP; 1795 return ret; 1796} 1797 1798/** 1799 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number 1800 * @dep: isoc endpoint 1801 * 1802 * This function tests for the correct combination of BIT[15:14] from the 16-bit 1803 * microframe number reported by the XferNotReady event for the future frame 1804 * number to start the isoc transfer. 1805 * 1806 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed 1807 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the 1808 * XferNotReady event are invalid. The driver uses this number to schedule the 1809 * isochronous transfer and passes it to the START TRANSFER command. Because 1810 * this number is invalid, the command may fail. If BIT[15:14] matches the 1811 * internal 16-bit microframe, the START TRANSFER command will pass and the 1812 * transfer will start at the scheduled time, if it is off by 1, the command 1813 * will still pass, but the transfer will start 2 seconds in the future. For all 1814 * other conditions, the START TRANSFER command will fail with bus-expiry. 1815 * 1816 * In order to workaround this issue, we can test for the correct combination of 1817 * BIT[15:14] by sending START TRANSFER commands with different values of 1818 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart 1819 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status. 1820 * As the result, within the 4 possible combinations for BIT[15:14], there will 1821 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful 1822 * command status will result in a 2-second delay start. The smaller BIT[15:14] 1823 * value is the correct combination. 1824 * 1825 * Since there are only 4 outcomes and the results are ordered, we can simply 1826 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to 1827 * deduce the smaller successful combination. 1828 * 1829 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01 1830 * of BIT[15:14]. The correct combination is as follow: 1831 * 1832 * if test0 fails and test1 passes, BIT[15:14] is 'b01 1833 * if test0 fails and test1 fails, BIT[15:14] is 'b10 1834 * if test0 passes and test1 fails, BIT[15:14] is 'b11 1835 * if test0 passes and test1 passes, BIT[15:14] is 'b00 1836 * 1837 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN 1838 * endpoints. 1839 */ 1840static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) 1841{ 1842 int cmd_status = 0; 1843 bool test0; 1844 bool test1; 1845 1846 while (dep->combo_num < 2) { 1847 struct dwc3_gadget_ep_cmd_params params; 1848 u32 test_frame_number; 1849 u32 cmd; 1850 1851 /* 1852 * Check if we can start isoc transfer on the next interval or 1853 * 4 uframes in the future with BIT[15:14] as dep->combo_num 1854 */ 1855 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; 1856 test_frame_number |= dep->combo_num << 14; 1857 test_frame_number += max_t(u32, 4, dep->interval); 1858 1859 params.param0 = upper_32_bits(dep->dwc->bounce_addr); 1860 params.param1 = lower_32_bits(dep->dwc->bounce_addr); 1861 1862 cmd = DWC3_DEPCMD_STARTTRANSFER; 1863 cmd |= DWC3_DEPCMD_PARAM(test_frame_number); 1864 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1865 1866 /* Redo if some other failure beside bus-expiry is received */ 1867 if (cmd_status && cmd_status != -EAGAIN) { 1868 dep->start_cmd_status = 0; 1869 dep->combo_num = 0; 1870 return 0; 1871 } 1872 1873 /* Store the first test status */ 1874 if (dep->combo_num == 0) 1875 dep->start_cmd_status = cmd_status; 1876 1877 dep->combo_num++; 1878 1879 /* 1880 * End the transfer if the START_TRANSFER command is successful 1881 * to wait for the next XferNotReady to test the command again 1882 */ 1883 if (cmd_status == 0) { 1884 dwc3_stop_active_transfer(dep, true, true); 1885 return 0; 1886 } 1887 } 1888 1889 /* test0 and test1 are both completed at this point */ 1890 test0 = (dep->start_cmd_status == 0); 1891 test1 = (cmd_status == 0); 1892 1893 if (!test0 && test1) 1894 dep->combo_num = 1; 1895 else if (!test0 && !test1) 1896 dep->combo_num = 2; 1897 else if (test0 && !test1) 1898 dep->combo_num = 3; 1899 else if (test0 && test1) 1900 dep->combo_num = 0; 1901 1902 dep->frame_number &= DWC3_FRNUMBER_MASK; 1903 dep->frame_number |= dep->combo_num << 14; 1904 dep->frame_number += max_t(u32, 4, dep->interval); 1905 1906 /* Reinitialize test variables */ 1907 dep->start_cmd_status = 0; 1908 dep->combo_num = 0; 1909 1910 return __dwc3_gadget_kick_transfer(dep); 1911} 1912 1913static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) 1914{ 1915 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 1916 struct dwc3 *dwc = dep->dwc; 1917 int ret; 1918 int i; 1919 1920 if (list_empty(&dep->pending_list) && 1921 list_empty(&dep->started_list)) { 1922 dep->flags |= DWC3_EP_PENDING_REQUEST; 1923 return -EAGAIN; 1924 } 1925 1926 if (!dwc->dis_start_transfer_quirk && 1927 (DWC3_VER_IS_PRIOR(DWC31, 170A) || 1928 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) { 1929 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) 1930 return dwc3_gadget_start_isoc_quirk(dep); 1931 } 1932 1933 if (desc->bInterval <= 14 && 1934 dwc->gadget->speed >= USB_SPEED_HIGH) { 1935 u32 frame = __dwc3_gadget_get_frame(dwc); 1936 bool rollover = frame < 1937 (dep->frame_number & DWC3_FRNUMBER_MASK); 1938 1939 /* 1940 * frame_number is set from XferNotReady and may be already 1941 * out of date. DSTS only provides the lower 14 bit of the 1942 * current frame number. So add the upper two bits of 1943 * frame_number and handle a possible rollover. 1944 * This will provide the correct frame_number unless more than 1945 * rollover has happened since XferNotReady. 1946 */ 1947 1948 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | 1949 frame; 1950 if (rollover) 1951 dep->frame_number += BIT(14); 1952 } 1953 1954 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) { 1955 int future_interval = i + 1; 1956 1957 /* Give the controller at least 500us to schedule transfers */ 1958 if (desc->bInterval < 3) 1959 future_interval += 3 - desc->bInterval; 1960 1961 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); 1962 1963 ret = __dwc3_gadget_kick_transfer(dep); 1964 if (ret != -EAGAIN) 1965 break; 1966 } 1967 1968 /* 1969 * After a number of unsuccessful start attempts due to bus-expiry 1970 * status, issue END_TRANSFER command and retry on the next XferNotReady 1971 * event. 1972 */ 1973 if (ret == -EAGAIN) 1974 ret = __dwc3_stop_active_transfer(dep, false, true); 1975 1976 return ret; 1977} 1978 1979static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) 1980{ 1981 struct dwc3 *dwc = dep->dwc; 1982 1983 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { 1984 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n", 1985 dep->name); 1986 return -ESHUTDOWN; 1987 } 1988 1989 if (WARN(req->dep != dep, "request %p belongs to '%s'\n", 1990 &req->request, req->dep->name)) 1991 return -EINVAL; 1992 1993 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, 1994 "%s: request %p already in flight\n", 1995 dep->name, &req->request)) 1996 return -EINVAL; 1997 1998 pm_runtime_get(dwc->dev); 1999 2000 req->request.actual = 0; 2001 req->request.status = -EINPROGRESS; 2002 2003 trace_dwc3_ep_queue(req); 2004 2005 list_add_tail(&req->list, &dep->pending_list); 2006 req->status = DWC3_REQUEST_STATUS_QUEUED; 2007 2008 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) 2009 return 0; 2010 2011 /* 2012 * Start the transfer only after the END_TRANSFER is completed 2013 * and endpoint STALL is cleared. 2014 */ 2015 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || 2016 (dep->flags & DWC3_EP_WEDGE) || 2017 (dep->flags & DWC3_EP_DELAY_STOP) || 2018 (dep->flags & DWC3_EP_STALL)) { 2019 dep->flags |= DWC3_EP_DELAY_START; 2020 return 0; 2021 } 2022 2023 /* 2024 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must 2025 * wait for a XferNotReady event so we will know what's the current 2026 * (micro-)frame number. 2027 * 2028 * Without this trick, we are very, very likely gonna get Bus Expiry 2029 * errors which will force us issue EndTransfer command. 2030 */ 2031 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2032 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { 2033 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) 2034 return __dwc3_gadget_start_isoc(dep); 2035 2036 return 0; 2037 } 2038 } 2039 2040 __dwc3_gadget_kick_transfer(dep); 2041 2042 return 0; 2043} 2044 2045static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 2046 gfp_t gfp_flags) 2047{ 2048 struct dwc3_request *req = to_dwc3_request(request); 2049 struct dwc3_ep *dep = to_dwc3_ep(ep); 2050 struct dwc3 *dwc = dep->dwc; 2051 2052 unsigned long flags; 2053 2054 int ret; 2055 2056 spin_lock_irqsave(&dwc->lock, flags); 2057 ret = __dwc3_gadget_ep_queue(dep, req); 2058 spin_unlock_irqrestore(&dwc->lock, flags); 2059 2060 return ret; 2061} 2062 2063static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req) 2064{ 2065 int i; 2066 2067 /* If req->trb is not set, then the request has not started */ 2068 if (!req->trb) 2069 return; 2070 2071 /* 2072 * If request was already started, this means we had to 2073 * stop the transfer. With that we also need to ignore 2074 * all TRBs used by the request, however TRBs can only 2075 * be modified after completion of END_TRANSFER 2076 * command. So what we do here is that we wait for 2077 * END_TRANSFER completion and only after that, we jump 2078 * over TRBs by clearing HWO and incrementing dequeue 2079 * pointer. 2080 */ 2081 for (i = 0; i < req->num_trbs; i++) { 2082 struct dwc3_trb *trb; 2083 2084 trb = &dep->trb_pool[dep->trb_dequeue]; 2085 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 2086 dwc3_ep_inc_deq(dep); 2087 } 2088 2089 req->num_trbs = 0; 2090} 2091 2092static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) 2093{ 2094 struct dwc3_request *req; 2095 struct dwc3 *dwc = dep->dwc; 2096 2097 while (!list_empty(&dep->cancelled_list)) { 2098 req = next_request(&dep->cancelled_list); 2099 dwc3_gadget_ep_skip_trbs(dep, req); 2100 switch (req->status) { 2101 case DWC3_REQUEST_STATUS_DISCONNECTED: 2102 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 2103 break; 2104 case DWC3_REQUEST_STATUS_DEQUEUED: 2105 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2106 break; 2107 case DWC3_REQUEST_STATUS_STALLED: 2108 dwc3_gadget_giveback(dep, req, -EPIPE); 2109 break; 2110 default: 2111 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); 2112 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2113 break; 2114 } 2115 /* 2116 * The endpoint is disabled, let the dwc3_remove_requests() 2117 * handle the cleanup. 2118 */ 2119 if (!dep->endpoint.desc) 2120 break; 2121 } 2122} 2123 2124static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, 2125 struct usb_request *request) 2126{ 2127 struct dwc3_request *req = to_dwc3_request(request); 2128 struct dwc3_request *r = NULL; 2129 2130 struct dwc3_ep *dep = to_dwc3_ep(ep); 2131 struct dwc3 *dwc = dep->dwc; 2132 2133 unsigned long flags; 2134 int ret = 0; 2135 2136 trace_dwc3_ep_dequeue(req); 2137 2138 spin_lock_irqsave(&dwc->lock, flags); 2139 2140 list_for_each_entry(r, &dep->cancelled_list, list) { 2141 if (r == req) 2142 goto out; 2143 } 2144 2145 list_for_each_entry(r, &dep->pending_list, list) { 2146 if (r == req) { 2147 /* 2148 * Explicitly check for EP0/1 as dequeue for those 2149 * EPs need to be handled differently. Control EP 2150 * only deals with one USB req, and giveback will 2151 * occur during dwc3_ep0_stall_and_restart(). EP0 2152 * requests are never added to started_list. 2153 */ 2154 if (dep->number > 1) 2155 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2156 else 2157 dwc3_ep0_reset_state(dwc); 2158 goto out; 2159 } 2160 } 2161 2162 list_for_each_entry(r, &dep->started_list, list) { 2163 if (r == req) { 2164 struct dwc3_request *t; 2165 2166 /* wait until it is processed */ 2167 dwc3_stop_active_transfer(dep, true, true); 2168 2169 /* 2170 * Remove any started request if the transfer is 2171 * cancelled. 2172 */ 2173 list_for_each_entry_safe(r, t, &dep->started_list, list) 2174 dwc3_gadget_move_cancelled_request(r, 2175 DWC3_REQUEST_STATUS_DEQUEUED); 2176 2177 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 2178 2179 goto out; 2180 } 2181 } 2182 2183 dev_err(dwc->dev, "request %p was not queued to %s\n", 2184 request, ep->name); 2185 ret = -EINVAL; 2186out: 2187 spin_unlock_irqrestore(&dwc->lock, flags); 2188 2189 return ret; 2190} 2191 2192int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) 2193{ 2194 struct dwc3_gadget_ep_cmd_params params; 2195 struct dwc3 *dwc = dep->dwc; 2196 struct dwc3_request *req; 2197 struct dwc3_request *tmp; 2198 int ret; 2199 2200 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2201 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); 2202 return -EINVAL; 2203 } 2204 2205 memset(¶ms, 0x00, sizeof(params)); 2206 2207 if (value) { 2208 struct dwc3_trb *trb; 2209 2210 unsigned int transfer_in_flight; 2211 unsigned int started; 2212 2213 if (dep->number > 1) 2214 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); 2215 else 2216 trb = &dwc->ep0_trb[dep->trb_enqueue]; 2217 2218 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; 2219 started = !list_empty(&dep->started_list); 2220 2221 if (!protocol && ((dep->direction && transfer_in_flight) || 2222 (!dep->direction && started))) { 2223 return -EAGAIN; 2224 } 2225 2226 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, 2227 ¶ms); 2228 if (ret) 2229 dev_err(dwc->dev, "failed to set STALL on %s\n", 2230 dep->name); 2231 else 2232 dep->flags |= DWC3_EP_STALL; 2233 } else { 2234 /* 2235 * Don't issue CLEAR_STALL command to control endpoints. The 2236 * controller automatically clears the STALL when it receives 2237 * the SETUP token. 2238 */ 2239 if (dep->number <= 1) { 2240 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2241 return 0; 2242 } 2243 2244 dwc3_stop_active_transfer(dep, true, true); 2245 2246 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 2247 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED); 2248 2249 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || 2250 (dep->flags & DWC3_EP_DELAY_STOP)) { 2251 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; 2252 if (protocol) 2253 dwc->clear_stall_protocol = dep->number; 2254 2255 return 0; 2256 } 2257 2258 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 2259 2260 ret = dwc3_send_clear_stall_ep_cmd(dep); 2261 if (ret) { 2262 dev_err(dwc->dev, "failed to clear STALL on %s\n", 2263 dep->name); 2264 return ret; 2265 } 2266 2267 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2268 2269 if ((dep->flags & DWC3_EP_DELAY_START) && 2270 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 2271 __dwc3_gadget_kick_transfer(dep); 2272 2273 dep->flags &= ~DWC3_EP_DELAY_START; 2274 } 2275 2276 return ret; 2277} 2278 2279static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) 2280{ 2281 struct dwc3_ep *dep = to_dwc3_ep(ep); 2282 struct dwc3 *dwc = dep->dwc; 2283 2284 unsigned long flags; 2285 2286 int ret; 2287 2288 spin_lock_irqsave(&dwc->lock, flags); 2289 ret = __dwc3_gadget_ep_set_halt(dep, value, false); 2290 spin_unlock_irqrestore(&dwc->lock, flags); 2291 2292 return ret; 2293} 2294 2295static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) 2296{ 2297 struct dwc3_ep *dep = to_dwc3_ep(ep); 2298 struct dwc3 *dwc = dep->dwc; 2299 unsigned long flags; 2300 int ret; 2301 2302 spin_lock_irqsave(&dwc->lock, flags); 2303 dep->flags |= DWC3_EP_WEDGE; 2304 2305 if (dep->number == 0 || dep->number == 1) 2306 ret = __dwc3_gadget_ep0_set_halt(ep, 1); 2307 else 2308 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); 2309 spin_unlock_irqrestore(&dwc->lock, flags); 2310 2311 return ret; 2312} 2313 2314/* -------------------------------------------------------------------------- */ 2315 2316static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = { 2317 .bLength = USB_DT_ENDPOINT_SIZE, 2318 .bDescriptorType = USB_DT_ENDPOINT, 2319 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 2320}; 2321 2322static const struct usb_ep_ops dwc3_gadget_ep0_ops = { 2323 .enable = dwc3_gadget_ep0_enable, 2324 .disable = dwc3_gadget_ep0_disable, 2325 .alloc_request = dwc3_gadget_ep_alloc_request, 2326 .free_request = dwc3_gadget_ep_free_request, 2327 .queue = dwc3_gadget_ep0_queue, 2328 .dequeue = dwc3_gadget_ep_dequeue, 2329 .set_halt = dwc3_gadget_ep0_set_halt, 2330 .set_wedge = dwc3_gadget_ep_set_wedge, 2331}; 2332 2333static const struct usb_ep_ops dwc3_gadget_ep_ops = { 2334 .enable = dwc3_gadget_ep_enable, 2335 .disable = dwc3_gadget_ep_disable, 2336 .alloc_request = dwc3_gadget_ep_alloc_request, 2337 .free_request = dwc3_gadget_ep_free_request, 2338 .queue = dwc3_gadget_ep_queue, 2339 .dequeue = dwc3_gadget_ep_dequeue, 2340 .set_halt = dwc3_gadget_ep_set_halt, 2341 .set_wedge = dwc3_gadget_ep_set_wedge, 2342}; 2343 2344/* -------------------------------------------------------------------------- */ 2345 2346static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set) 2347{ 2348 u32 reg; 2349 2350 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2351 return; 2352 2353 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN); 2354 if (set) 2355 reg |= DWC3_DEVTEN_ULSTCNGEN; 2356 else 2357 reg &= ~DWC3_DEVTEN_ULSTCNGEN; 2358 2359 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2360} 2361 2362static int dwc3_gadget_get_frame(struct usb_gadget *g) 2363{ 2364 struct dwc3 *dwc = gadget_to_dwc(g); 2365 2366 return __dwc3_gadget_get_frame(dwc); 2367} 2368 2369static int __dwc3_gadget_wakeup(struct dwc3 *dwc) 2370{ 2371 int ret; 2372 u32 reg; 2373 2374 u8 link_state; 2375 2376 /* 2377 * According to the Databook Remote wakeup request should 2378 * be issued only when the device is in early suspend state. 2379 * 2380 * We can check that via USB Link State bits in DSTS register. 2381 */ 2382 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2383 2384 link_state = DWC3_DSTS_USBLNKST(reg); 2385 2386 switch (link_state) { 2387 case DWC3_LINK_STATE_RESET: 2388 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ 2389 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ 2390 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */ 2391 case DWC3_LINK_STATE_U1: 2392 case DWC3_LINK_STATE_RESUME: 2393 break; 2394 default: 2395 return -EINVAL; 2396 } 2397 2398 dwc3_gadget_enable_linksts_evts(dwc, true); 2399 2400 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV); 2401 if (ret < 0) { 2402 dev_err(dwc->dev, "failed to put link in Recovery\n"); 2403 dwc3_gadget_enable_linksts_evts(dwc, false); 2404 return ret; 2405 } 2406 2407 /* Recent versions do this automatically */ 2408 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) { 2409 /* write zeroes to Link Change Request */ 2410 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2411 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 2412 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2413 } 2414 2415 /* 2416 * Since link status change events are enabled we will receive 2417 * an U0 event when wakeup is successful. 2418 */ 2419 return 0; 2420} 2421 2422static int dwc3_gadget_wakeup(struct usb_gadget *g) 2423{ 2424 struct dwc3 *dwc = gadget_to_dwc(g); 2425 unsigned long flags; 2426 int ret; 2427 2428 if (!dwc->wakeup_configured) { 2429 dev_err(dwc->dev, "remote wakeup not configured\n"); 2430 return -EINVAL; 2431 } 2432 2433 spin_lock_irqsave(&dwc->lock, flags); 2434 if (!dwc->gadget->wakeup_armed) { 2435 dev_err(dwc->dev, "not armed for remote wakeup\n"); 2436 spin_unlock_irqrestore(&dwc->lock, flags); 2437 return -EINVAL; 2438 } 2439 ret = __dwc3_gadget_wakeup(dwc); 2440 2441 spin_unlock_irqrestore(&dwc->lock, flags); 2442 2443 return ret; 2444} 2445 2446static void dwc3_resume_gadget(struct dwc3 *dwc); 2447 2448static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id) 2449{ 2450 struct dwc3 *dwc = gadget_to_dwc(g); 2451 unsigned long flags; 2452 int ret; 2453 int link_state; 2454 2455 if (!dwc->wakeup_configured) { 2456 dev_err(dwc->dev, "remote wakeup not configured\n"); 2457 return -EINVAL; 2458 } 2459 2460 spin_lock_irqsave(&dwc->lock, flags); 2461 /* 2462 * If the link is in U3, signal for remote wakeup and wait for the 2463 * link to transition to U0 before sending device notification. 2464 */ 2465 link_state = dwc3_gadget_get_link_state(dwc); 2466 if (link_state == DWC3_LINK_STATE_U3) { 2467 dwc->wakeup_pending_funcs |= BIT(intf_id); 2468 ret = __dwc3_gadget_wakeup(dwc); 2469 spin_unlock_irqrestore(&dwc->lock, flags); 2470 return ret; 2471 } 2472 2473 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION, 2474 DWC3_DGCMDPAR_DN_FUNC_WAKE | 2475 DWC3_DGCMDPAR_INTF_SEL(intf_id)); 2476 if (ret) 2477 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret); 2478 2479 spin_unlock_irqrestore(&dwc->lock, flags); 2480 2481 return ret; 2482} 2483 2484static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set) 2485{ 2486 struct dwc3 *dwc = gadget_to_dwc(g); 2487 unsigned long flags; 2488 2489 spin_lock_irqsave(&dwc->lock, flags); 2490 dwc->wakeup_configured = !!set; 2491 spin_unlock_irqrestore(&dwc->lock, flags); 2492 2493 return 0; 2494} 2495 2496static int dwc3_gadget_set_selfpowered(struct usb_gadget *g, 2497 int is_selfpowered) 2498{ 2499 struct dwc3 *dwc = gadget_to_dwc(g); 2500 unsigned long flags; 2501 2502 spin_lock_irqsave(&dwc->lock, flags); 2503 g->is_selfpowered = !!is_selfpowered; 2504 spin_unlock_irqrestore(&dwc->lock, flags); 2505 2506 return 0; 2507} 2508 2509static void dwc3_stop_active_transfers(struct dwc3 *dwc) 2510{ 2511 u32 epnum; 2512 2513 for (epnum = 2; epnum < dwc->num_eps; epnum++) { 2514 struct dwc3_ep *dep; 2515 2516 dep = dwc->eps[epnum]; 2517 if (!dep) 2518 continue; 2519 2520 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); 2521 } 2522} 2523 2524static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc) 2525{ 2526 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; 2527 u32 reg; 2528 2529 if (ssp_rate == USB_SSP_GEN_UNKNOWN) 2530 ssp_rate = dwc->max_ssp_rate; 2531 2532 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2533 reg &= ~DWC3_DCFG_SPEED_MASK; 2534 reg &= ~DWC3_DCFG_NUMLANES(~0); 2535 2536 if (ssp_rate == USB_SSP_GEN_1x2) 2537 reg |= DWC3_DCFG_SUPERSPEED; 2538 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) 2539 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2540 2541 if (ssp_rate != USB_SSP_GEN_2x1 && 2542 dwc->max_ssp_rate != USB_SSP_GEN_2x1) 2543 reg |= DWC3_DCFG_NUMLANES(1); 2544 2545 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2546} 2547 2548static void __dwc3_gadget_set_speed(struct dwc3 *dwc) 2549{ 2550 enum usb_device_speed speed; 2551 u32 reg; 2552 2553 speed = dwc->gadget_max_speed; 2554 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) 2555 speed = dwc->maximum_speed; 2556 2557 if (speed == USB_SPEED_SUPER_PLUS && 2558 DWC3_IP_IS(DWC32)) { 2559 __dwc3_gadget_set_ssp_rate(dwc); 2560 return; 2561 } 2562 2563 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2564 reg &= ~(DWC3_DCFG_SPEED_MASK); 2565 2566 /* 2567 * WORKAROUND: DWC3 revision < 2.20a have an issue 2568 * which would cause metastability state on Run/Stop 2569 * bit if we try to force the IP to USB2-only mode. 2570 * 2571 * Because of that, we cannot configure the IP to any 2572 * speed other than the SuperSpeed 2573 * 2574 * Refers to: 2575 * 2576 * STAR#9000525659: Clock Domain Crossing on DCTL in 2577 * USB 2.0 Mode 2578 */ 2579 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 2580 !dwc->dis_metastability_quirk) { 2581 reg |= DWC3_DCFG_SUPERSPEED; 2582 } else { 2583 switch (speed) { 2584 case USB_SPEED_FULL: 2585 reg |= DWC3_DCFG_FULLSPEED; 2586 break; 2587 case USB_SPEED_HIGH: 2588 reg |= DWC3_DCFG_HIGHSPEED; 2589 break; 2590 case USB_SPEED_SUPER: 2591 reg |= DWC3_DCFG_SUPERSPEED; 2592 break; 2593 case USB_SPEED_SUPER_PLUS: 2594 if (DWC3_IP_IS(DWC3)) 2595 reg |= DWC3_DCFG_SUPERSPEED; 2596 else 2597 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2598 break; 2599 default: 2600 dev_err(dwc->dev, "invalid speed (%d)\n", speed); 2601 2602 if (DWC3_IP_IS(DWC3)) 2603 reg |= DWC3_DCFG_SUPERSPEED; 2604 else 2605 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2606 } 2607 } 2608 2609 if (DWC3_IP_IS(DWC32) && 2610 speed > USB_SPEED_UNKNOWN && 2611 speed < USB_SPEED_SUPER_PLUS) 2612 reg &= ~DWC3_DCFG_NUMLANES(~0); 2613 2614 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2615} 2616 2617static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on) 2618{ 2619 u32 reg; 2620 u32 timeout = 2000; 2621 u32 saved_config = 0; 2622 2623 if (pm_runtime_suspended(dwc->dev)) 2624 return 0; 2625 2626 /* 2627 * When operating in USB 2.0 speeds (HS/FS), ensure that 2628 * GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY are cleared before starting 2629 * or stopping the controller. This resolves timeout issues that occur 2630 * during frequent role switches between host and device modes. 2631 * 2632 * Save and clear these settings, then restore them after completing the 2633 * controller start or stop sequence. 2634 * 2635 * This solution was discovered through experimentation as it is not 2636 * mentioned in the dwc3 programming guide. It has been tested on an 2637 * Exynos platforms. 2638 */ 2639 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 2640 if (reg & DWC3_GUSB2PHYCFG_SUSPHY) { 2641 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY; 2642 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 2643 } 2644 2645 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) { 2646 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM; 2647 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM; 2648 } 2649 2650 if (saved_config) 2651 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 2652 2653 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2654 if (is_on) { 2655 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) { 2656 reg &= ~DWC3_DCTL_TRGTULST_MASK; 2657 reg |= DWC3_DCTL_TRGTULST_RX_DET; 2658 } 2659 2660 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 2661 reg &= ~DWC3_DCTL_KEEP_CONNECT; 2662 reg |= DWC3_DCTL_RUN_STOP; 2663 2664 __dwc3_gadget_set_speed(dwc); 2665 dwc->pullups_connected = true; 2666 } else { 2667 reg &= ~DWC3_DCTL_RUN_STOP; 2668 2669 dwc->pullups_connected = false; 2670 } 2671 2672 dwc3_pre_run_stop(dwc, is_on); 2673 dwc3_gadget_dctl_write_safe(dwc, reg); 2674 2675 do { 2676 usleep_range(1000, 2000); 2677 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2678 reg &= DWC3_DSTS_DEVCTRLHLT; 2679 } while (--timeout && !(!is_on ^ !reg)); 2680 2681 if (saved_config) { 2682 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 2683 reg |= saved_config; 2684 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 2685 } 2686 2687 if (!timeout) 2688 return -ETIMEDOUT; 2689 2690 return 0; 2691} 2692 2693static void dwc3_gadget_disable_irq(struct dwc3 *dwc); 2694static void __dwc3_gadget_stop(struct dwc3 *dwc); 2695static int __dwc3_gadget_start(struct dwc3 *dwc); 2696 2697static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc) 2698{ 2699 unsigned long flags; 2700 int ret; 2701 2702 spin_lock_irqsave(&dwc->lock, flags); 2703 if (!dwc->pullups_connected) { 2704 spin_unlock_irqrestore(&dwc->lock, flags); 2705 return 0; 2706 } 2707 2708 dwc->connected = false; 2709 2710 /* 2711 * Attempt to end pending SETUP status phase, and not wait for the 2712 * function to do so. 2713 */ 2714 if (dwc->delayed_status) 2715 dwc3_ep0_send_delayed_status(dwc); 2716 2717 /* 2718 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a 2719 * Section 4.1.8 Table 4-7, it states that for a device-initiated 2720 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER 2721 * command for any active transfers" before clearing the RunStop 2722 * bit. 2723 */ 2724 dwc3_stop_active_transfers(dwc); 2725 spin_unlock_irqrestore(&dwc->lock, flags); 2726 2727 /* 2728 * Per databook, when we want to stop the gadget, if a control transfer 2729 * is still in process, complete it and get the core into setup phase. 2730 * In case the host is unresponsive to a SETUP transaction, forcefully 2731 * stall the transfer, and move back to the SETUP phase, so that any 2732 * pending endxfers can be executed. 2733 */ 2734 if (dwc->ep0state != EP0_SETUP_PHASE) { 2735 reinit_completion(&dwc->ep0_in_setup); 2736 2737 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, 2738 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); 2739 if (ret == 0) { 2740 dev_warn(dwc->dev, "wait for SETUP phase timed out\n"); 2741 spin_lock_irqsave(&dwc->lock, flags); 2742 dwc3_ep0_reset_state(dwc); 2743 spin_unlock_irqrestore(&dwc->lock, flags); 2744 } 2745 } 2746 2747 /* 2748 * Note: if the GEVNTCOUNT indicates events in the event buffer, the 2749 * driver needs to acknowledge them before the controller can halt. 2750 * Simply let the interrupt handler acknowledges and handle the 2751 * remaining event generated by the controller while polling for 2752 * DSTS.DEVCTLHLT. 2753 */ 2754 ret = dwc3_gadget_run_stop(dwc, false); 2755 2756 /* 2757 * Stop the gadget after controller is halted, so that if needed, the 2758 * events to update EP0 state can still occur while the run/stop 2759 * routine polls for the halted state. DEVTEN is cleared as part of 2760 * gadget stop. 2761 */ 2762 spin_lock_irqsave(&dwc->lock, flags); 2763 __dwc3_gadget_stop(dwc); 2764 spin_unlock_irqrestore(&dwc->lock, flags); 2765 2766 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); 2767 2768 return ret; 2769} 2770 2771static int dwc3_gadget_soft_connect(struct dwc3 *dwc) 2772{ 2773 int ret; 2774 2775 /* 2776 * In the Synopsys DWC_usb31 1.90a programming guide section 2777 * 4.1.9, it specifies that for a reconnect after a 2778 * device-initiated disconnect requires a core soft reset 2779 * (DCTL.CSftRst) before enabling the run/stop bit. 2780 */ 2781 ret = dwc3_core_soft_reset(dwc); 2782 if (ret) 2783 return ret; 2784 2785 dwc3_event_buffers_setup(dwc); 2786 __dwc3_gadget_start(dwc); 2787 return dwc3_gadget_run_stop(dwc, true); 2788} 2789 2790static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) 2791{ 2792 struct dwc3 *dwc = gadget_to_dwc(g); 2793 int ret; 2794 2795 is_on = !!is_on; 2796 2797 dwc->softconnect = is_on; 2798 2799 /* 2800 * Avoid issuing a runtime resume if the device is already in the 2801 * suspended state during gadget disconnect. DWC3 gadget was already 2802 * halted/stopped during runtime suspend. 2803 */ 2804 if (!is_on) { 2805 pm_runtime_barrier(dwc->dev); 2806 if (pm_runtime_suspended(dwc->dev)) 2807 return 0; 2808 } 2809 2810 /* 2811 * Check the return value for successful resume, or error. For a 2812 * successful resume, the DWC3 runtime PM resume routine will handle 2813 * the run stop sequence, so avoid duplicate operations here. 2814 */ 2815 ret = pm_runtime_get_sync(dwc->dev); 2816 if (!ret || ret < 0) { 2817 pm_runtime_put(dwc->dev); 2818 if (ret < 0) 2819 pm_runtime_set_suspended(dwc->dev); 2820 return ret; 2821 } 2822 2823 if (dwc->pullups_connected == is_on) { 2824 pm_runtime_put(dwc->dev); 2825 return 0; 2826 } 2827 2828 synchronize_irq(dwc->irq_gadget); 2829 2830 if (!is_on) 2831 ret = dwc3_gadget_soft_disconnect(dwc); 2832 else 2833 ret = dwc3_gadget_soft_connect(dwc); 2834 2835 pm_runtime_put(dwc->dev); 2836 2837 return ret; 2838} 2839 2840static void dwc3_gadget_enable_irq(struct dwc3 *dwc) 2841{ 2842 u32 reg; 2843 2844 /* Enable all but Start and End of Frame IRQs */ 2845 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN | 2846 DWC3_DEVTEN_CMDCMPLTEN | 2847 DWC3_DEVTEN_ERRTICERREN | 2848 DWC3_DEVTEN_WKUPEVTEN | 2849 DWC3_DEVTEN_CONNECTDONEEN | 2850 DWC3_DEVTEN_USBRSTEN | 2851 DWC3_DEVTEN_DISCONNEVTEN); 2852 2853 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2854 reg |= DWC3_DEVTEN_ULSTCNGEN; 2855 2856 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ 2857 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 2858 reg |= DWC3_DEVTEN_U3L2L1SUSPEN; 2859 2860 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2861} 2862 2863static void dwc3_gadget_disable_irq(struct dwc3 *dwc) 2864{ 2865 /* mask all interrupts */ 2866 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); 2867} 2868 2869static irqreturn_t dwc3_interrupt(int irq, void *_dwc); 2870static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc); 2871 2872/** 2873 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG 2874 * @dwc: pointer to our context structure 2875 * 2876 * The following looks like complex but it's actually very simple. In order to 2877 * calculate the number of packets we can burst at once on OUT transfers, we're 2878 * gonna use RxFIFO size. 2879 * 2880 * To calculate RxFIFO size we need two numbers: 2881 * MDWIDTH = size, in bits, of the internal memory bus 2882 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits) 2883 * 2884 * Given these two numbers, the formula is simple: 2885 * 2886 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16; 2887 * 2888 * 24 bytes is for 3x SETUP packets 2889 * 16 bytes is a clock domain crossing tolerance 2890 * 2891 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024; 2892 */ 2893static void dwc3_gadget_setup_nump(struct dwc3 *dwc) 2894{ 2895 u32 ram2_depth; 2896 u32 mdwidth; 2897 u32 nump; 2898 u32 reg; 2899 2900 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); 2901 mdwidth = dwc3_mdwidth(dwc); 2902 2903 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; 2904 nump = min_t(u32, nump, 16); 2905 2906 /* update NumP */ 2907 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2908 reg &= ~DWC3_DCFG_NUMP_MASK; 2909 reg |= nump << DWC3_DCFG_NUMP_SHIFT; 2910 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2911} 2912 2913static int __dwc3_gadget_start(struct dwc3 *dwc) 2914{ 2915 struct dwc3_ep *dep; 2916 int ret = 0; 2917 u32 reg; 2918 2919 /* 2920 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if 2921 * the core supports IMOD, disable it. 2922 */ 2923 if (dwc->imod_interval) { 2924 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 2925 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 2926 } else if (dwc3_has_imod(dwc)) { 2927 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); 2928 } 2929 2930 /* 2931 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP 2932 * field instead of letting dwc3 itself calculate that automatically. 2933 * 2934 * This way, we maximize the chances that we'll be able to get several 2935 * bursts of data without going through any sort of endpoint throttling. 2936 */ 2937 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); 2938 if (DWC3_IP_IS(DWC3)) 2939 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL; 2940 else 2941 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL; 2942 2943 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); 2944 2945 dwc3_gadget_setup_nump(dwc); 2946 2947 /* 2948 * Currently the controller handles single stream only. So, Ignore 2949 * Packet Pending bit for stream selection and don't search for another 2950 * stream if the host sends Data Packet with PP=0 (for OUT direction) or 2951 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves 2952 * the stream performance. 2953 */ 2954 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2955 reg |= DWC3_DCFG_IGNSTRMPP; 2956 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2957 2958 /* Enable MST by default if the device is capable of MST */ 2959 if (DWC3_MST_CAPABLE(&dwc->hwparams)) { 2960 reg = dwc3_readl(dwc->regs, DWC3_DCFG1); 2961 reg &= ~DWC3_DCFG1_DIS_MST_ENH; 2962 dwc3_writel(dwc->regs, DWC3_DCFG1, reg); 2963 } 2964 2965 /* Start with SuperSpeed Default */ 2966 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2967 2968 ret = dwc3_gadget_start_config(dwc, 0); 2969 if (ret) { 2970 dev_err(dwc->dev, "failed to config endpoints\n"); 2971 return ret; 2972 } 2973 2974 dep = dwc->eps[0]; 2975 dep->flags = 0; 2976 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2977 if (ret) { 2978 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2979 goto err0; 2980 } 2981 2982 dep = dwc->eps[1]; 2983 dep->flags = 0; 2984 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2985 if (ret) { 2986 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2987 goto err1; 2988 } 2989 2990 /* begin to receive SETUP packets */ 2991 dwc->ep0state = EP0_SETUP_PHASE; 2992 dwc->ep0_bounced = false; 2993 dwc->link_state = DWC3_LINK_STATE_SS_DIS; 2994 dwc->delayed_status = false; 2995 dwc3_ep0_out_start(dwc); 2996 2997 dwc3_gadget_enable_irq(dwc); 2998 dwc3_enable_susphy(dwc, true); 2999 3000 return 0; 3001 3002err1: 3003 __dwc3_gadget_ep_disable(dwc->eps[0]); 3004 3005err0: 3006 return ret; 3007} 3008 3009static int dwc3_gadget_start(struct usb_gadget *g, 3010 struct usb_gadget_driver *driver) 3011{ 3012 struct dwc3 *dwc = gadget_to_dwc(g); 3013 unsigned long flags; 3014 int ret; 3015 int irq; 3016 3017 irq = dwc->irq_gadget; 3018 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt, 3019 IRQF_SHARED, "dwc3", dwc->ev_buf); 3020 if (ret) { 3021 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", 3022 irq, ret); 3023 return ret; 3024 } 3025 3026 spin_lock_irqsave(&dwc->lock, flags); 3027 dwc->gadget_driver = driver; 3028 spin_unlock_irqrestore(&dwc->lock, flags); 3029 3030 if (dwc->sys_wakeup) 3031 device_wakeup_enable(dwc->sysdev); 3032 3033 return 0; 3034} 3035 3036static void __dwc3_gadget_stop(struct dwc3 *dwc) 3037{ 3038 dwc3_gadget_disable_irq(dwc); 3039 __dwc3_gadget_ep_disable(dwc->eps[0]); 3040 __dwc3_gadget_ep_disable(dwc->eps[1]); 3041} 3042 3043static int dwc3_gadget_stop(struct usb_gadget *g) 3044{ 3045 struct dwc3 *dwc = gadget_to_dwc(g); 3046 unsigned long flags; 3047 3048 if (dwc->sys_wakeup) 3049 device_wakeup_disable(dwc->sysdev); 3050 3051 spin_lock_irqsave(&dwc->lock, flags); 3052 dwc->gadget_driver = NULL; 3053 dwc->max_cfg_eps = 0; 3054 spin_unlock_irqrestore(&dwc->lock, flags); 3055 3056 free_irq(dwc->irq_gadget, dwc->ev_buf); 3057 3058 return 0; 3059} 3060 3061static void dwc3_gadget_config_params(struct usb_gadget *g, 3062 struct usb_dcd_config_params *params) 3063{ 3064 struct dwc3 *dwc = gadget_to_dwc(g); 3065 3066 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; 3067 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; 3068 3069 /* Recommended BESL */ 3070 if (!dwc->dis_enblslpm_quirk) { 3071 /* 3072 * If the recommended BESL baseline is 0 or if the BESL deep is 3073 * less than 2, Microsoft's Windows 10 host usb stack will issue 3074 * a usb reset immediately after it receives the extended BOS 3075 * descriptor and the enumeration will fail. To maintain 3076 * compatibility with the Windows' usb stack, let's set the 3077 * recommended BESL baseline to 1 and clamp the BESL deep to be 3078 * within 2 to 15. 3079 */ 3080 params->besl_baseline = 1; 3081 if (dwc->is_utmi_l1_suspend) 3082 params->besl_deep = 3083 clamp_t(u8, dwc->hird_threshold, 2, 15); 3084 } 3085 3086 /* U1 Device exit Latency */ 3087 if (dwc->dis_u1_entry_quirk) 3088 params->bU1devExitLat = 0; 3089 else 3090 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; 3091 3092 /* U2 Device exit Latency */ 3093 if (dwc->dis_u2_entry_quirk) 3094 params->bU2DevExitLat = 0; 3095 else 3096 params->bU2DevExitLat = 3097 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT); 3098} 3099 3100static void dwc3_gadget_set_speed(struct usb_gadget *g, 3101 enum usb_device_speed speed) 3102{ 3103 struct dwc3 *dwc = gadget_to_dwc(g); 3104 unsigned long flags; 3105 3106 spin_lock_irqsave(&dwc->lock, flags); 3107 dwc->gadget_max_speed = speed; 3108 spin_unlock_irqrestore(&dwc->lock, flags); 3109} 3110 3111static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, 3112 enum usb_ssp_rate rate) 3113{ 3114 struct dwc3 *dwc = gadget_to_dwc(g); 3115 unsigned long flags; 3116 3117 spin_lock_irqsave(&dwc->lock, flags); 3118 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; 3119 dwc->gadget_ssp_rate = rate; 3120 spin_unlock_irqrestore(&dwc->lock, flags); 3121} 3122 3123static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) 3124{ 3125 struct dwc3 *dwc = gadget_to_dwc(g); 3126 3127 if (dwc->usb2_phy) 3128 return usb_phy_set_power(dwc->usb2_phy, mA); 3129 3130 if (!dwc->usb_psy) 3131 return -EOPNOTSUPP; 3132 3133 dwc->current_limit = mA; 3134 schedule_work(&dwc->vbus_draw_work); 3135 3136 return 0; 3137} 3138 3139/** 3140 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration 3141 * @g: pointer to the USB gadget 3142 * 3143 * Used to record the maximum number of endpoints being used in a USB composite 3144 * device. (across all configurations) This is to be used in the calculation 3145 * of the TXFIFO sizes when resizing internal memory for individual endpoints. 3146 * It will help ensured that the resizing logic reserves enough space for at 3147 * least one max packet. 3148 */ 3149static int dwc3_gadget_check_config(struct usb_gadget *g) 3150{ 3151 struct dwc3 *dwc = gadget_to_dwc(g); 3152 struct usb_ep *ep; 3153 int fifo_size = 0; 3154 int ram_depth; 3155 int ep_num = 0; 3156 3157 if (!dwc->do_fifo_resize) 3158 return 0; 3159 3160 list_for_each_entry(ep, &g->ep_list, ep_list) { 3161 /* Only interested in the IN endpoints */ 3162 if (ep->claimed && (ep->address & USB_DIR_IN)) 3163 ep_num++; 3164 } 3165 3166 if (ep_num <= dwc->max_cfg_eps) 3167 return 0; 3168 3169 /* Update the max number of eps in the composition */ 3170 dwc->max_cfg_eps = ep_num; 3171 3172 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); 3173 /* Based on the equation, increment by one for every ep */ 3174 fifo_size += dwc->max_cfg_eps; 3175 3176 /* Check if we can fit a single fifo per endpoint */ 3177 ram_depth = dwc3_gadget_calc_ram_depth(dwc); 3178 if (fifo_size > ram_depth) 3179 return -ENOMEM; 3180 3181 return 0; 3182} 3183 3184static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable) 3185{ 3186 struct dwc3 *dwc = gadget_to_dwc(g); 3187 unsigned long flags; 3188 3189 spin_lock_irqsave(&dwc->lock, flags); 3190 dwc->async_callbacks = enable; 3191 spin_unlock_irqrestore(&dwc->lock, flags); 3192} 3193 3194static const struct usb_gadget_ops dwc3_gadget_ops = { 3195 .get_frame = dwc3_gadget_get_frame, 3196 .wakeup = dwc3_gadget_wakeup, 3197 .func_wakeup = dwc3_gadget_func_wakeup, 3198 .set_remote_wakeup = dwc3_gadget_set_remote_wakeup, 3199 .set_selfpowered = dwc3_gadget_set_selfpowered, 3200 .pullup = dwc3_gadget_pullup, 3201 .udc_start = dwc3_gadget_start, 3202 .udc_stop = dwc3_gadget_stop, 3203 .udc_set_speed = dwc3_gadget_set_speed, 3204 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate, 3205 .get_config_params = dwc3_gadget_config_params, 3206 .vbus_draw = dwc3_gadget_vbus_draw, 3207 .check_config = dwc3_gadget_check_config, 3208 .udc_async_callbacks = dwc3_gadget_async_callbacks, 3209}; 3210 3211/* -------------------------------------------------------------------------- */ 3212 3213static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) 3214{ 3215 struct dwc3 *dwc = dep->dwc; 3216 3217 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); 3218 dep->endpoint.maxburst = 1; 3219 dep->endpoint.ops = &dwc3_gadget_ep0_ops; 3220 if (!dep->direction) 3221 dwc->gadget->ep0 = &dep->endpoint; 3222 3223 dep->endpoint.caps.type_control = true; 3224 3225 return 0; 3226} 3227 3228static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) 3229{ 3230 struct dwc3 *dwc = dep->dwc; 3231 u32 mdwidth; 3232 int size; 3233 int maxpacket; 3234 3235 mdwidth = dwc3_mdwidth(dwc); 3236 3237 /* MDWIDTH is represented in bits, we need it in bytes */ 3238 mdwidth /= 8; 3239 3240 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); 3241 if (DWC3_IP_IS(DWC3)) 3242 size = DWC3_GTXFIFOSIZ_TXFDEP(size); 3243 else 3244 size = DWC31_GTXFIFOSIZ_TXFDEP(size); 3245 3246 /* 3247 * maxpacket size is determined as part of the following, after assuming 3248 * a mult value of one maxpacket: 3249 * DWC3 revision 280A and prior: 3250 * fifo_size = mult * (max_packet / mdwidth) + 1; 3251 * maxpacket = mdwidth * (fifo_size - 1); 3252 * 3253 * DWC3 revision 290A and onwards: 3254 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 3255 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; 3256 */ 3257 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 3258 maxpacket = mdwidth * (size - 1); 3259 else 3260 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; 3261 3262 /* Functionally, space for one max packet is sufficient */ 3263 size = min_t(int, maxpacket, 1024); 3264 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3265 3266 dep->endpoint.max_streams = 16; 3267 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3268 list_add_tail(&dep->endpoint.ep_list, 3269 &dwc->gadget->ep_list); 3270 dep->endpoint.caps.type_iso = true; 3271 dep->endpoint.caps.type_bulk = true; 3272 dep->endpoint.caps.type_int = true; 3273 3274 return dwc3_alloc_trb_pool(dep); 3275} 3276 3277static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) 3278{ 3279 struct dwc3 *dwc = dep->dwc; 3280 u32 mdwidth; 3281 int size; 3282 3283 mdwidth = dwc3_mdwidth(dwc); 3284 3285 /* MDWIDTH is represented in bits, convert to bytes */ 3286 mdwidth /= 8; 3287 3288 /* All OUT endpoints share a single RxFIFO space */ 3289 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); 3290 if (DWC3_IP_IS(DWC3)) 3291 size = DWC3_GRXFIFOSIZ_RXFDEP(size); 3292 else 3293 size = DWC31_GRXFIFOSIZ_RXFDEP(size); 3294 3295 /* FIFO depth is in MDWDITH bytes */ 3296 size *= mdwidth; 3297 3298 /* 3299 * To meet performance requirement, a minimum recommended RxFIFO size 3300 * is defined as follow: 3301 * RxFIFO size >= (3 x MaxPacketSize) + 3302 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin) 3303 * 3304 * Then calculate the max packet limit as below. 3305 */ 3306 size -= (3 * 8) + 16; 3307 if (size < 0) 3308 size = 0; 3309 else 3310 size /= 3; 3311 3312 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3313 dep->endpoint.max_streams = 16; 3314 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3315 list_add_tail(&dep->endpoint.ep_list, 3316 &dwc->gadget->ep_list); 3317 dep->endpoint.caps.type_iso = true; 3318 dep->endpoint.caps.type_bulk = true; 3319 dep->endpoint.caps.type_int = true; 3320 3321 return dwc3_alloc_trb_pool(dep); 3322} 3323 3324#define nostream_work_to_dep(w) (container_of(to_delayed_work(w), struct dwc3_ep, nostream_work)) 3325static void dwc3_nostream_work(struct work_struct *work) 3326{ 3327 struct dwc3_ep *dep = nostream_work_to_dep(work); 3328 struct dwc3 *dwc = dep->dwc; 3329 unsigned long flags; 3330 3331 spin_lock_irqsave(&dwc->lock, flags); 3332 if (dep->flags & DWC3_EP_STREAM_PRIMED) 3333 goto out; 3334 3335 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || 3336 (!DWC3_MST_CAPABLE(&dwc->hwparams) && 3337 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) 3338 goto out; 3339 /* 3340 * If the host rejects a stream due to no active stream, by the 3341 * USB and xHCI spec, the endpoint will be put back to idle 3342 * state. When the host is ready (buffer added/updated), it will 3343 * prime the endpoint to inform the usb device controller. This 3344 * triggers the device controller to issue ERDY to restart the 3345 * stream. However, some hosts don't follow this and keep the 3346 * endpoint in the idle state. No prime will come despite host 3347 * streams are updated, and the device controller will not be 3348 * triggered to generate ERDY to move the next stream data. To 3349 * workaround this and maintain compatibility with various 3350 * hosts, force to reinitiate the stream until the host is ready 3351 * instead of waiting for the host to prime the endpoint. 3352 */ 3353 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) { 3354 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME; 3355 3356 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); 3357 } else { 3358 dep->flags |= DWC3_EP_DELAY_START; 3359 dwc3_stop_active_transfer(dep, true, true); 3360 spin_unlock_irqrestore(&dwc->lock, flags); 3361 return; 3362 } 3363out: 3364 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; 3365 spin_unlock_irqrestore(&dwc->lock, flags); 3366} 3367 3368static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum) 3369{ 3370 struct dwc3_ep *dep; 3371 bool direction = epnum & 1; 3372 int ret; 3373 u8 num = epnum >> 1; 3374 3375 dep = kzalloc(sizeof(*dep), GFP_KERNEL); 3376 if (!dep) 3377 return -ENOMEM; 3378 3379 dep->dwc = dwc; 3380 dep->number = epnum; 3381 dep->direction = direction; 3382 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); 3383 dwc->eps[epnum] = dep; 3384 dep->combo_num = 0; 3385 dep->start_cmd_status = 0; 3386 3387 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, 3388 direction ? "in" : "out"); 3389 3390 dep->endpoint.name = dep->name; 3391 3392 if (!(dep->number > 1)) { 3393 dep->endpoint.desc = &dwc3_gadget_ep0_desc; 3394 dep->endpoint.comp_desc = NULL; 3395 } 3396 3397 if (num == 0) 3398 ret = dwc3_gadget_init_control_endpoint(dep); 3399 else if (direction) 3400 ret = dwc3_gadget_init_in_endpoint(dep); 3401 else 3402 ret = dwc3_gadget_init_out_endpoint(dep); 3403 3404 if (ret) 3405 return ret; 3406 3407 dep->endpoint.caps.dir_in = direction; 3408 dep->endpoint.caps.dir_out = !direction; 3409 3410 INIT_LIST_HEAD(&dep->pending_list); 3411 INIT_LIST_HEAD(&dep->started_list); 3412 INIT_LIST_HEAD(&dep->cancelled_list); 3413 INIT_DELAYED_WORK(&dep->nostream_work, dwc3_nostream_work); 3414 3415 dwc3_debugfs_create_endpoint_dir(dep); 3416 3417 return 0; 3418} 3419 3420static int dwc3_gadget_get_reserved_endpoints(struct dwc3 *dwc, const char *propname, 3421 u8 *eps, u8 num) 3422{ 3423 u8 count; 3424 int ret; 3425 3426 if (!device_property_present(dwc->dev, propname)) 3427 return 0; 3428 3429 ret = device_property_count_u8(dwc->dev, propname); 3430 if (ret < 0) 3431 return ret; 3432 count = ret; 3433 3434 ret = device_property_read_u8_array(dwc->dev, propname, eps, min(num, count)); 3435 if (ret) 3436 return ret; 3437 3438 return count; 3439} 3440 3441static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) 3442{ 3443 const char *propname = "snps,reserved-endpoints"; 3444 u8 epnum; 3445 u8 reserved_eps[DWC3_ENDPOINTS_NUM]; 3446 u8 count; 3447 u8 num; 3448 int ret; 3449 3450 INIT_LIST_HEAD(&dwc->gadget->ep_list); 3451 3452 ret = dwc3_gadget_get_reserved_endpoints(dwc, propname, 3453 reserved_eps, ARRAY_SIZE(reserved_eps)); 3454 if (ret < 0) { 3455 dev_err(dwc->dev, "failed to read %s\n", propname); 3456 return ret; 3457 } 3458 count = ret; 3459 3460 for (epnum = 0; epnum < total; epnum++) { 3461 for (num = 0; num < count; num++) { 3462 if (epnum == reserved_eps[num]) 3463 break; 3464 } 3465 if (num < count) 3466 continue; 3467 3468 ret = dwc3_gadget_init_endpoint(dwc, epnum); 3469 if (ret) 3470 return ret; 3471 } 3472 3473 return 0; 3474} 3475 3476static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) 3477{ 3478 struct dwc3_ep *dep; 3479 u8 epnum; 3480 3481 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3482 dep = dwc->eps[epnum]; 3483 if (!dep) 3484 continue; 3485 /* 3486 * Physical endpoints 0 and 1 are special; they form the 3487 * bi-directional USB endpoint 0. 3488 * 3489 * For those two physical endpoints, we don't allocate a TRB 3490 * pool nor do we add them the endpoints list. Due to that, we 3491 * shouldn't do these two operations otherwise we would end up 3492 * with all sorts of bugs when removing dwc3.ko. 3493 */ 3494 if (epnum != 0 && epnum != 1) { 3495 dwc3_free_trb_pool(dep); 3496 list_del(&dep->endpoint.ep_list); 3497 } 3498 3499 dwc3_debugfs_remove_endpoint_dir(dep); 3500 kfree(dep); 3501 } 3502} 3503 3504/* -------------------------------------------------------------------------- */ 3505 3506static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, 3507 struct dwc3_request *req, struct dwc3_trb *trb, 3508 const struct dwc3_event_depevt *event, int status) 3509{ 3510 unsigned int count; 3511 3512 dwc3_ep_inc_deq(dep); 3513 3514 trace_dwc3_complete_trb(dep, trb); 3515 req->num_trbs--; 3516 3517 /* 3518 * If we're in the middle of series of chained TRBs and we 3519 * receive a short transfer along the way, DWC3 will skip 3520 * through all TRBs including the last TRB in the chain (the 3521 * where CHN bit is zero. DWC3 will also avoid clearing HWO 3522 * bit and SW has to do it manually. 3523 * 3524 * We're going to do that here to avoid problems of HW trying 3525 * to use bogus TRBs for transfers. 3526 */ 3527 if (trb->ctrl & DWC3_TRB_CTRL_HWO) 3528 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3529 3530 /* 3531 * For isochronous transfers, the first TRB in a service interval must 3532 * have the Isoc-First type. Track and report its interval frame number. 3533 */ 3534 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3535 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { 3536 unsigned int frame_number; 3537 3538 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); 3539 frame_number &= ~(dep->interval - 1); 3540 req->request.frame_number = frame_number; 3541 } 3542 3543 /* 3544 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If 3545 * this TRB points to the bounce buffer address, it's a MPS alignment 3546 * TRB. Don't add it to req->remaining calculation. 3547 */ 3548 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && 3549 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { 3550 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3551 return 1; 3552 } 3553 3554 count = trb->size & DWC3_TRB_SIZE_MASK; 3555 req->remaining += count; 3556 3557 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) 3558 return 1; 3559 3560 if (event->status & DEPEVT_STATUS_SHORT && 3561 !(trb->ctrl & DWC3_TRB_CTRL_CHN)) 3562 return 1; 3563 3564 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && 3565 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) 3566 return 1; 3567 3568 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || 3569 (trb->ctrl & DWC3_TRB_CTRL_LST)) 3570 return 1; 3571 3572 return 0; 3573} 3574 3575static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, 3576 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3577 int status) 3578{ 3579 struct dwc3_trb *trb; 3580 unsigned int num_completed_trbs = req->num_trbs; 3581 unsigned int i; 3582 int ret = 0; 3583 3584 for (i = 0; i < num_completed_trbs; i++) { 3585 trb = &dep->trb_pool[dep->trb_dequeue]; 3586 3587 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, 3588 trb, event, status); 3589 if (ret) 3590 break; 3591 } 3592 3593 return ret; 3594} 3595 3596static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req) 3597{ 3598 return req->num_pending_sgs == 0 && req->num_trbs == 0; 3599} 3600 3601static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, 3602 const struct dwc3_event_depevt *event, 3603 struct dwc3_request *req, int status) 3604{ 3605 int request_status; 3606 int ret; 3607 3608 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, status); 3609 3610 req->request.actual = req->request.length - req->remaining; 3611 3612 if (!dwc3_gadget_ep_request_completed(req)) 3613 goto out; 3614 3615 /* 3616 * The event status only reflects the status of the TRB with IOC set. 3617 * For the requests that don't set interrupt on completion, the driver 3618 * needs to check and return the status of the completed TRBs associated 3619 * with the request. Use the status of the last TRB of the request. 3620 */ 3621 if (req->request.no_interrupt) { 3622 struct dwc3_trb *trb; 3623 3624 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); 3625 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { 3626 case DWC3_TRBSTS_MISSED_ISOC: 3627 /* Isoc endpoint only */ 3628 request_status = -EXDEV; 3629 break; 3630 case DWC3_TRB_STS_XFER_IN_PROG: 3631 /* Applicable when End Transfer with ForceRM=0 */ 3632 case DWC3_TRBSTS_SETUP_PENDING: 3633 /* Control endpoint only */ 3634 case DWC3_TRBSTS_OK: 3635 default: 3636 request_status = 0; 3637 break; 3638 } 3639 } else { 3640 request_status = status; 3641 } 3642 3643 dwc3_gadget_giveback(dep, req, request_status); 3644 3645out: 3646 return ret; 3647} 3648 3649static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, 3650 const struct dwc3_event_depevt *event, int status) 3651{ 3652 struct dwc3_request *req; 3653 3654 while (!list_empty(&dep->started_list)) { 3655 int ret; 3656 3657 req = next_request(&dep->started_list); 3658 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, 3659 req, status); 3660 if (ret) 3661 break; 3662 /* 3663 * The endpoint is disabled, let the dwc3_remove_requests() 3664 * handle the cleanup. 3665 */ 3666 if (!dep->endpoint.desc) 3667 break; 3668 } 3669} 3670 3671static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep) 3672{ 3673 struct dwc3_request *req; 3674 struct dwc3 *dwc = dep->dwc; 3675 3676 if (!dep->endpoint.desc || !dwc->pullups_connected || 3677 !dwc->connected) 3678 return false; 3679 3680 if (!list_empty(&dep->pending_list)) 3681 return true; 3682 3683 /* 3684 * We only need to check the first entry of the started list. We can 3685 * assume the completed requests are removed from the started list. 3686 */ 3687 req = next_request(&dep->started_list); 3688 if (!req) 3689 return false; 3690 3691 return !dwc3_gadget_ep_request_completed(req); 3692} 3693 3694static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, 3695 const struct dwc3_event_depevt *event) 3696{ 3697 dep->frame_number = event->parameters; 3698} 3699 3700static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, 3701 const struct dwc3_event_depevt *event, int status) 3702{ 3703 struct dwc3 *dwc = dep->dwc; 3704 bool no_started_trb = true; 3705 3706 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); 3707 3708 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3709 goto out; 3710 3711 if (!dep->endpoint.desc) 3712 return no_started_trb; 3713 3714 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3715 list_empty(&dep->started_list) && 3716 (list_empty(&dep->pending_list) || status == -EXDEV)) 3717 dwc3_stop_active_transfer(dep, true, true); 3718 else if (dwc3_gadget_ep_should_continue(dep)) 3719 if (__dwc3_gadget_kick_transfer(dep) == 0) 3720 no_started_trb = false; 3721 3722out: 3723 /* 3724 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. 3725 * See dwc3_gadget_linksts_change_interrupt() for 1st half. 3726 */ 3727 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 3728 u32 reg; 3729 int i; 3730 3731 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 3732 dep = dwc->eps[i]; 3733 if (!dep) 3734 continue; 3735 3736 if (!(dep->flags & DWC3_EP_ENABLED)) 3737 continue; 3738 3739 if (!list_empty(&dep->started_list)) 3740 return no_started_trb; 3741 } 3742 3743 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3744 reg |= dwc->u1u2; 3745 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 3746 3747 dwc->u1u2 = 0; 3748 } 3749 3750 return no_started_trb; 3751} 3752 3753static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, 3754 const struct dwc3_event_depevt *event) 3755{ 3756 int status = 0; 3757 3758 if (!dep->endpoint.desc) 3759 return; 3760 3761 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3762 dwc3_gadget_endpoint_frame_from_event(dep, event); 3763 3764 if (event->status & DEPEVT_STATUS_BUSERR) 3765 status = -ECONNRESET; 3766 3767 if (event->status & DEPEVT_STATUS_MISSED_ISOC) 3768 status = -EXDEV; 3769 3770 dwc3_gadget_endpoint_trbs_complete(dep, event, status); 3771} 3772 3773static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, 3774 const struct dwc3_event_depevt *event) 3775{ 3776 int status = 0; 3777 3778 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3779 3780 if (event->status & DEPEVT_STATUS_BUSERR) 3781 status = -ECONNRESET; 3782 3783 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status)) 3784 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 3785} 3786 3787static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, 3788 const struct dwc3_event_depevt *event) 3789{ 3790 /* 3791 * During a device-initiated disconnect, a late xferNotReady event can 3792 * be generated after the End Transfer command resets the event filter, 3793 * but before the controller is halted. Ignore it to prevent a new 3794 * transfer from starting. 3795 */ 3796 if (!dep->dwc->connected) 3797 return; 3798 3799 dwc3_gadget_endpoint_frame_from_event(dep, event); 3800 3801 /* 3802 * The XferNotReady event is generated only once before the endpoint 3803 * starts. It will be generated again when END_TRANSFER command is 3804 * issued. For some controller versions, the XferNotReady event may be 3805 * generated while the END_TRANSFER command is still in process. Ignore 3806 * it and wait for the next XferNotReady event after the command is 3807 * completed. 3808 */ 3809 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3810 return; 3811 3812 (void) __dwc3_gadget_start_isoc(dep); 3813} 3814 3815static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep, 3816 const struct dwc3_event_depevt *event) 3817{ 3818 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); 3819 3820 if (cmd != DWC3_DEPCMD_ENDTRANSFER) 3821 return; 3822 3823 /* 3824 * The END_TRANSFER command will cause the controller to generate a 3825 * NoStream Event, and it's not due to the host DP NoStream rejection. 3826 * Ignore the next NoStream event. 3827 */ 3828 if (dep->stream_capable) 3829 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; 3830 3831 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; 3832 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3833 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 3834 3835 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { 3836 struct dwc3 *dwc = dep->dwc; 3837 3838 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; 3839 if (dwc3_send_clear_stall_ep_cmd(dep)) { 3840 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; 3841 3842 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); 3843 if (dwc->delayed_status) 3844 __dwc3_gadget_ep0_set_halt(ep0, 1); 3845 return; 3846 } 3847 3848 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 3849 if (dwc->clear_stall_protocol == dep->number) 3850 dwc3_ep0_send_delayed_status(dwc); 3851 } 3852 3853 if ((dep->flags & DWC3_EP_DELAY_START) && 3854 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3855 __dwc3_gadget_kick_transfer(dep); 3856 3857 dep->flags &= ~DWC3_EP_DELAY_START; 3858} 3859 3860static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, 3861 const struct dwc3_event_depevt *event) 3862{ 3863 if (event->status == DEPEVT_STREAMEVT_FOUND) { 3864 cancel_delayed_work(&dep->nostream_work); 3865 dep->flags |= DWC3_EP_STREAM_PRIMED; 3866 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; 3867 return; 3868 } 3869 3870 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */ 3871 switch (event->parameters) { 3872 case DEPEVT_STREAM_PRIME: 3873 cancel_delayed_work(&dep->nostream_work); 3874 dep->flags |= DWC3_EP_STREAM_PRIMED; 3875 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; 3876 break; 3877 case DEPEVT_STREAM_NOSTREAM: 3878 dep->flags &= ~DWC3_EP_STREAM_PRIMED; 3879 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) 3880 queue_delayed_work(system_percpu_wq, &dep->nostream_work, 3881 msecs_to_jiffies(100)); 3882 break; 3883 } 3884} 3885 3886static void dwc3_endpoint_interrupt(struct dwc3 *dwc, 3887 const struct dwc3_event_depevt *event) 3888{ 3889 struct dwc3_ep *dep; 3890 u8 epnum = event->endpoint_number; 3891 3892 dep = dwc->eps[epnum]; 3893 if (!dep) { 3894 dev_warn(dwc->dev, "spurious event, endpoint %u is not allocated\n", epnum); 3895 return; 3896 } 3897 3898 if (!(dep->flags & DWC3_EP_ENABLED)) { 3899 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) 3900 return; 3901 3902 /* Handle only EPCMDCMPLT when EP disabled */ 3903 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) && 3904 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE)) 3905 return; 3906 } 3907 3908 if (epnum == 0 || epnum == 1) { 3909 dwc3_ep0_interrupt(dwc, event); 3910 return; 3911 } 3912 3913 switch (event->endpoint_event) { 3914 case DWC3_DEPEVT_XFERINPROGRESS: 3915 dwc3_gadget_endpoint_transfer_in_progress(dep, event); 3916 break; 3917 case DWC3_DEPEVT_XFERNOTREADY: 3918 dwc3_gadget_endpoint_transfer_not_ready(dep, event); 3919 break; 3920 case DWC3_DEPEVT_EPCMDCMPLT: 3921 dwc3_gadget_endpoint_command_complete(dep, event); 3922 break; 3923 case DWC3_DEPEVT_XFERCOMPLETE: 3924 dwc3_gadget_endpoint_transfer_complete(dep, event); 3925 break; 3926 case DWC3_DEPEVT_STREAMEVT: 3927 dwc3_gadget_endpoint_stream_event(dep, event); 3928 break; 3929 case DWC3_DEPEVT_RXTXFIFOEVT: 3930 break; 3931 default: 3932 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event); 3933 break; 3934 } 3935} 3936 3937static void dwc3_disconnect_gadget(struct dwc3 *dwc) 3938{ 3939 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { 3940 spin_unlock(&dwc->lock); 3941 dwc->gadget_driver->disconnect(dwc->gadget); 3942 spin_lock(&dwc->lock); 3943 } 3944} 3945 3946static void dwc3_suspend_gadget(struct dwc3 *dwc) 3947{ 3948 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { 3949 spin_unlock(&dwc->lock); 3950 dwc->gadget_driver->suspend(dwc->gadget); 3951 spin_lock(&dwc->lock); 3952 } 3953} 3954 3955static void dwc3_resume_gadget(struct dwc3 *dwc) 3956{ 3957 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 3958 spin_unlock(&dwc->lock); 3959 dwc->gadget_driver->resume(dwc->gadget); 3960 spin_lock(&dwc->lock); 3961 } 3962} 3963 3964static void dwc3_reset_gadget(struct dwc3 *dwc) 3965{ 3966 if (!dwc->gadget_driver) 3967 return; 3968 3969 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { 3970 spin_unlock(&dwc->lock); 3971 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); 3972 spin_lock(&dwc->lock); 3973 } 3974} 3975 3976void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, 3977 bool interrupt) 3978{ 3979 struct dwc3 *dwc = dep->dwc; 3980 3981 /* 3982 * Only issue End Transfer command to the control endpoint of a started 3983 * Data Phase. Typically we should only do so in error cases such as 3984 * invalid/unexpected direction as described in the control transfer 3985 * flow of the programming guide. 3986 */ 3987 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) 3988 return; 3989 3990 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP)) 3991 return; 3992 3993 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || 3994 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 3995 return; 3996 3997 /* 3998 * If a Setup packet is received but yet to DMA out, the controller will 3999 * not process the End Transfer command of any endpoint. Polling of its 4000 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a 4001 * timeout. Delay issuing the End Transfer command until the Setup TRB is 4002 * prepared. 4003 */ 4004 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { 4005 dep->flags |= DWC3_EP_DELAY_STOP; 4006 return; 4007 } 4008 4009 /* 4010 * NOTICE: We are violating what the Databook says about the 4011 * EndTransfer command. Ideally we would _always_ wait for the 4012 * EndTransfer Command Completion IRQ, but that's causing too 4013 * much trouble synchronizing between us and gadget driver. 4014 * 4015 * We have discussed this with the IP Provider and it was 4016 * suggested to giveback all requests here. 4017 * 4018 * Note also that a similar handling was tested by Synopsys 4019 * (thanks a lot Paul) and nothing bad has come out of it. 4020 * In short, what we're doing is issuing EndTransfer with 4021 * CMDIOC bit set and delay kicking transfer until the 4022 * EndTransfer command had completed. 4023 * 4024 * As of IP version 3.10a of the DWC_usb3 IP, the controller 4025 * supports a mode to work around the above limitation. The 4026 * software can poll the CMDACT bit in the DEPCMD register 4027 * after issuing a EndTransfer command. This mode is enabled 4028 * by writing GUCTL2[14]. This polling is already done in the 4029 * dwc3_send_gadget_ep_cmd() function so if the mode is 4030 * enabled, the EndTransfer command will have completed upon 4031 * returning from this function. 4032 * 4033 * This mode is NOT available on the DWC_usb31 IP. In this 4034 * case, if the IOC bit is not set, then delay by 1ms 4035 * after issuing the EndTransfer command. This allows for the 4036 * controller to handle the command completely before DWC3 4037 * remove requests attempts to unmap USB request buffers. 4038 */ 4039 4040 __dwc3_stop_active_transfer(dep, force, interrupt); 4041} 4042 4043static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) 4044{ 4045 u32 epnum; 4046 4047 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 4048 struct dwc3_ep *dep; 4049 int ret; 4050 4051 dep = dwc->eps[epnum]; 4052 if (!dep) 4053 continue; 4054 4055 if (!(dep->flags & DWC3_EP_STALL)) 4056 continue; 4057 4058 dep->flags &= ~DWC3_EP_STALL; 4059 4060 ret = dwc3_send_clear_stall_ep_cmd(dep); 4061 if (ret) 4062 dev_err_ratelimited(dwc->dev, 4063 "failed to clear STALL on %s\n", dep->name); 4064 } 4065} 4066 4067static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) 4068{ 4069 int reg; 4070 4071 dwc->suspended = false; 4072 4073 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET); 4074 4075 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4076 reg &= ~DWC3_DCTL_INITU1ENA; 4077 reg &= ~DWC3_DCTL_INITU2ENA; 4078 dwc3_gadget_dctl_write_safe(dwc, reg); 4079 4080 dwc->connected = false; 4081 4082 dwc3_disconnect_gadget(dwc); 4083 4084 dwc->gadget->speed = USB_SPEED_UNKNOWN; 4085 dwc->setup_packet_pending = false; 4086 dwc->gadget->wakeup_armed = false; 4087 dwc3_gadget_enable_linksts_evts(dwc, false); 4088 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); 4089 4090 dwc3_ep0_reset_state(dwc); 4091 4092 /* 4093 * Request PM idle to address condition where usage count is 4094 * already decremented to zero, but waiting for the disconnect 4095 * interrupt to set dwc->connected to FALSE. 4096 */ 4097 pm_request_idle(dwc->dev); 4098} 4099 4100static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) 4101{ 4102 u32 reg; 4103 4104 dwc->suspended = false; 4105 4106 /* 4107 * Ideally, dwc3_reset_gadget() would trigger the function 4108 * drivers to stop any active transfers through ep disable. 4109 * However, for functions which defer ep disable, such as mass 4110 * storage, we will need to rely on the call to stop active 4111 * transfers here, and avoid allowing of request queuing. 4112 */ 4113 dwc->connected = false; 4114 4115 /* 4116 * WORKAROUND: DWC3 revisions <1.88a have an issue which 4117 * would cause a missing Disconnect Event if there's a 4118 * pending Setup Packet in the FIFO. 4119 * 4120 * There's no suggested workaround on the official Bug 4121 * report, which states that "unless the driver/application 4122 * is doing any special handling of a disconnect event, 4123 * there is no functional issue". 4124 * 4125 * Unfortunately, it turns out that we _do_ some special 4126 * handling of a disconnect event, namely complete all 4127 * pending transfers, notify gadget driver of the 4128 * disconnection, and so on. 4129 * 4130 * Our suggested workaround is to follow the Disconnect 4131 * Event steps here, instead, based on a setup_packet_pending 4132 * flag. Such flag gets set whenever we have a SETUP_PENDING 4133 * status for EP0 TRBs and gets cleared on XferComplete for the 4134 * same endpoint. 4135 * 4136 * Refers to: 4137 * 4138 * STAR#9000466709: RTL: Device : Disconnect event not 4139 * generated if setup packet pending in FIFO 4140 */ 4141 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) { 4142 if (dwc->setup_packet_pending) 4143 dwc3_gadget_disconnect_interrupt(dwc); 4144 } 4145 4146 dwc3_reset_gadget(dwc); 4147 4148 /* 4149 * From SNPS databook section 8.1.2, the EP0 should be in setup 4150 * phase. So ensure that EP0 is in setup phase by issuing a stall 4151 * and restart if EP0 is not in setup phase. 4152 */ 4153 dwc3_ep0_reset_state(dwc); 4154 4155 /* 4156 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a 4157 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW 4158 * needs to ensure that it sends "a DEPENDXFER command for any active 4159 * transfers." 4160 */ 4161 dwc3_stop_active_transfers(dwc); 4162 dwc->connected = true; 4163 4164 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4165 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 4166 dwc3_gadget_dctl_write_safe(dwc, reg); 4167 dwc->test_mode = false; 4168 dwc->gadget->wakeup_armed = false; 4169 dwc3_gadget_enable_linksts_evts(dwc, false); 4170 dwc3_clear_stall_all_ep(dwc); 4171 4172 /* Reset device address to zero */ 4173 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4174 reg &= ~(DWC3_DCFG_DEVADDR_MASK); 4175 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4176} 4177 4178static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) 4179{ 4180 struct dwc3_ep *dep; 4181 int ret; 4182 u32 reg; 4183 u8 lanes = 1; 4184 u8 speed; 4185 4186 if (!dwc->softconnect) 4187 return; 4188 4189 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 4190 speed = reg & DWC3_DSTS_CONNECTSPD; 4191 dwc->speed = speed; 4192 4193 if (DWC3_IP_IS(DWC32)) 4194 lanes = DWC3_DSTS_CONNLANES(reg) + 1; 4195 4196 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4197 4198 /* 4199 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed 4200 * each time on Connect Done. 4201 * 4202 * Currently we always use the reset value. If any platform 4203 * wants to set this to a different value, we need to add a 4204 * setting and update GCTL.RAMCLKSEL here. 4205 */ 4206 4207 switch (speed) { 4208 case DWC3_DSTS_SUPERSPEED_PLUS: 4209 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 4210 dwc->gadget->ep0->maxpacket = 512; 4211 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 4212 4213 if (lanes > 1) 4214 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; 4215 else 4216 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; 4217 break; 4218 case DWC3_DSTS_SUPERSPEED: 4219 /* 4220 * WORKAROUND: DWC3 revisions <1.90a have an issue which 4221 * would cause a missing USB3 Reset event. 4222 * 4223 * In such situations, we should force a USB3 Reset 4224 * event by calling our dwc3_gadget_reset_interrupt() 4225 * routine. 4226 * 4227 * Refers to: 4228 * 4229 * STAR#9000483510: RTL: SS : USB3 reset event may 4230 * not be generated always when the link enters poll 4231 */ 4232 if (DWC3_VER_IS_PRIOR(DWC3, 190A)) 4233 dwc3_gadget_reset_interrupt(dwc); 4234 4235 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 4236 dwc->gadget->ep0->maxpacket = 512; 4237 dwc->gadget->speed = USB_SPEED_SUPER; 4238 4239 if (lanes > 1) { 4240 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 4241 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; 4242 } 4243 break; 4244 case DWC3_DSTS_HIGHSPEED: 4245 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 4246 dwc->gadget->ep0->maxpacket = 64; 4247 dwc->gadget->speed = USB_SPEED_HIGH; 4248 break; 4249 case DWC3_DSTS_FULLSPEED: 4250 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 4251 dwc->gadget->ep0->maxpacket = 64; 4252 dwc->gadget->speed = USB_SPEED_FULL; 4253 break; 4254 } 4255 4256 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; 4257 4258 /* Enable USB2 LPM Capability */ 4259 4260 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) && 4261 !dwc->usb2_gadget_lpm_disable && 4262 (speed != DWC3_DSTS_SUPERSPEED) && 4263 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) { 4264 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4265 reg |= DWC3_DCFG_LPM_CAP; 4266 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4267 4268 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4269 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN); 4270 4271 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | 4272 (dwc->is_utmi_l1_suspend << 4)); 4273 4274 /* 4275 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and 4276 * DCFG.LPMCap is set, core responses with an ACK and the 4277 * BESL value in the LPM token is less than or equal to LPM 4278 * NYET threshold. 4279 */ 4280 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, 4281 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); 4282 4283 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) { 4284 reg &= ~DWC3_DCTL_NYET_THRES_MASK; 4285 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); 4286 } 4287 4288 dwc3_gadget_dctl_write_safe(dwc, reg); 4289 } else { 4290 if (dwc->usb2_gadget_lpm_disable) { 4291 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4292 reg &= ~DWC3_DCFG_LPM_CAP; 4293 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4294 } 4295 4296 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4297 reg &= ~DWC3_DCTL_HIRD_THRES_MASK; 4298 dwc3_gadget_dctl_write_safe(dwc, reg); 4299 } 4300 4301 dep = dwc->eps[0]; 4302 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 4303 if (ret) { 4304 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 4305 return; 4306 } 4307 4308 dep = dwc->eps[1]; 4309 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 4310 if (ret) { 4311 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 4312 return; 4313 } 4314 4315 /* 4316 * Configure PHY via GUSB3PIPECTLn if required. 4317 * 4318 * Update GTXFIFOSIZn 4319 * 4320 * In both cases reset values should be sufficient. 4321 */ 4322} 4323 4324static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo) 4325{ 4326 dwc->suspended = false; 4327 4328 /* 4329 * TODO take core out of low power mode when that's 4330 * implemented. 4331 */ 4332 4333 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 4334 spin_unlock(&dwc->lock); 4335 dwc->gadget_driver->resume(dwc->gadget); 4336 spin_lock(&dwc->lock); 4337 } 4338 4339 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK; 4340} 4341 4342static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc, 4343 unsigned int evtinfo) 4344{ 4345 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4346 unsigned int pwropt; 4347 int ret; 4348 int intf_id; 4349 4350 /* 4351 * WORKAROUND: DWC3 < 2.50a have an issue when configured without 4352 * Hibernation mode enabled which would show up when device detects 4353 * host-initiated U3 exit. 4354 * 4355 * In that case, device will generate a Link State Change Interrupt 4356 * from U3 to RESUME which is only necessary if Hibernation is 4357 * configured in. 4358 * 4359 * There are no functional changes due to such spurious event and we 4360 * just need to ignore it. 4361 * 4362 * Refers to: 4363 * 4364 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation 4365 * operational mode 4366 */ 4367 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); 4368 if (DWC3_VER_IS_PRIOR(DWC3, 250A) && 4369 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) { 4370 if ((dwc->link_state == DWC3_LINK_STATE_U3) && 4371 (next == DWC3_LINK_STATE_RESUME)) { 4372 return; 4373 } 4374 } 4375 4376 /* 4377 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending 4378 * on the link partner, the USB session might do multiple entry/exit 4379 * of low power states before a transfer takes place. 4380 * 4381 * Due to this problem, we might experience lower throughput. The 4382 * suggested workaround is to disable DCTL[12:9] bits if we're 4383 * transitioning from U1/U2 to U0 and enable those bits again 4384 * after a transfer completes and there are no pending transfers 4385 * on any of the enabled endpoints. 4386 * 4387 * This is the first half of that workaround. 4388 * 4389 * Refers to: 4390 * 4391 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us 4392 * core send LGO_Ux entering U0 4393 */ 4394 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 4395 if (next == DWC3_LINK_STATE_U0) { 4396 u32 u1u2; 4397 u32 reg; 4398 4399 switch (dwc->link_state) { 4400 case DWC3_LINK_STATE_U1: 4401 case DWC3_LINK_STATE_U2: 4402 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4403 u1u2 = reg & (DWC3_DCTL_INITU2ENA 4404 | DWC3_DCTL_ACCEPTU2ENA 4405 | DWC3_DCTL_INITU1ENA 4406 | DWC3_DCTL_ACCEPTU1ENA); 4407 4408 if (!dwc->u1u2) 4409 dwc->u1u2 = reg & u1u2; 4410 4411 reg &= ~u1u2; 4412 4413 dwc3_gadget_dctl_write_safe(dwc, reg); 4414 break; 4415 default: 4416 /* do nothing */ 4417 break; 4418 } 4419 } 4420 } 4421 4422 switch (next) { 4423 case DWC3_LINK_STATE_U0: 4424 if (dwc->gadget->wakeup_armed || dwc->wakeup_pending_funcs) { 4425 dwc3_gadget_enable_linksts_evts(dwc, false); 4426 dwc3_resume_gadget(dwc); 4427 dwc->suspended = false; 4428 } 4429 break; 4430 case DWC3_LINK_STATE_U1: 4431 if (dwc->speed == USB_SPEED_SUPER) 4432 dwc3_suspend_gadget(dwc); 4433 break; 4434 case DWC3_LINK_STATE_U2: 4435 case DWC3_LINK_STATE_U3: 4436 dwc3_suspend_gadget(dwc); 4437 break; 4438 case DWC3_LINK_STATE_RESUME: 4439 dwc3_resume_gadget(dwc); 4440 break; 4441 default: 4442 /* do nothing */ 4443 break; 4444 } 4445 4446 dwc->link_state = next; 4447 4448 /* Proceed with func wakeup if any interfaces that has requested */ 4449 while (dwc->wakeup_pending_funcs && (next == DWC3_LINK_STATE_U0)) { 4450 intf_id = ffs(dwc->wakeup_pending_funcs) - 1; 4451 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION, 4452 DWC3_DGCMDPAR_DN_FUNC_WAKE | 4453 DWC3_DGCMDPAR_INTF_SEL(intf_id)); 4454 if (ret) 4455 dev_err(dwc->dev, "Failed to send DN wake for intf %d\n", intf_id); 4456 4457 dwc->wakeup_pending_funcs &= ~BIT(intf_id); 4458 } 4459} 4460 4461static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc, 4462 unsigned int evtinfo) 4463{ 4464 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4465 4466 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) { 4467 dwc->suspended = true; 4468 dwc3_suspend_gadget(dwc); 4469 } 4470 4471 dwc->link_state = next; 4472} 4473 4474static void dwc3_gadget_interrupt(struct dwc3 *dwc, 4475 const struct dwc3_event_devt *event) 4476{ 4477 switch (event->type) { 4478 case DWC3_DEVICE_EVENT_DISCONNECT: 4479 dwc3_gadget_disconnect_interrupt(dwc); 4480 break; 4481 case DWC3_DEVICE_EVENT_RESET: 4482 dwc3_gadget_reset_interrupt(dwc); 4483 break; 4484 case DWC3_DEVICE_EVENT_CONNECT_DONE: 4485 dwc3_gadget_conndone_interrupt(dwc); 4486 break; 4487 case DWC3_DEVICE_EVENT_WAKEUP: 4488 dwc3_gadget_wakeup_interrupt(dwc, event->event_info); 4489 break; 4490 case DWC3_DEVICE_EVENT_HIBER_REQ: 4491 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n"); 4492 break; 4493 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE: 4494 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); 4495 break; 4496 case DWC3_DEVICE_EVENT_SUSPEND: 4497 /* It changed to be suspend event for version 2.30a and above */ 4498 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 4499 dwc3_gadget_suspend_interrupt(dwc, event->event_info); 4500 break; 4501 case DWC3_DEVICE_EVENT_SOF: 4502 case DWC3_DEVICE_EVENT_ERRATIC_ERROR: 4503 case DWC3_DEVICE_EVENT_CMD_CMPL: 4504 case DWC3_DEVICE_EVENT_OVERFLOW: 4505 break; 4506 default: 4507 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); 4508 } 4509} 4510 4511static void dwc3_process_event_entry(struct dwc3 *dwc, 4512 const union dwc3_event *event) 4513{ 4514 trace_dwc3_event(event->raw, dwc); 4515 4516 if (!event->type.is_devspec) 4517 dwc3_endpoint_interrupt(dwc, &event->depevt); 4518 else if (event->type.type == DWC3_EVENT_TYPE_DEV) 4519 dwc3_gadget_interrupt(dwc, &event->devt); 4520 else 4521 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); 4522} 4523 4524static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) 4525{ 4526 struct dwc3 *dwc = evt->dwc; 4527 irqreturn_t ret = IRQ_NONE; 4528 int left; 4529 4530 left = evt->count; 4531 4532 if (!(evt->flags & DWC3_EVENT_PENDING)) 4533 return IRQ_NONE; 4534 4535 while (left > 0) { 4536 union dwc3_event event; 4537 4538 event.raw = *(u32 *) (evt->cache + evt->lpos); 4539 4540 dwc3_process_event_entry(dwc, &event); 4541 4542 /* 4543 * FIXME we wrap around correctly to the next entry as 4544 * almost all entries are 4 bytes in size. There is one 4545 * entry which has 12 bytes which is a regular entry 4546 * followed by 8 bytes data. ATM I don't know how 4547 * things are organized if we get next to the a 4548 * boundary so I worry about that once we try to handle 4549 * that. 4550 */ 4551 evt->lpos = (evt->lpos + 4) % evt->length; 4552 left -= 4; 4553 } 4554 4555 evt->count = 0; 4556 ret = IRQ_HANDLED; 4557 4558 /* Unmask interrupt */ 4559 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4560 DWC3_GEVNTSIZ_SIZE(evt->length)); 4561 4562 evt->flags &= ~DWC3_EVENT_PENDING; 4563 /* 4564 * Add an explicit write memory barrier to make sure that the update of 4565 * clearing DWC3_EVENT_PENDING is observed in dwc3_check_event_buf() 4566 */ 4567 wmb(); 4568 4569 if (dwc->imod_interval) { 4570 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 4571 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 4572 } 4573 4574 return ret; 4575} 4576 4577static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) 4578{ 4579 struct dwc3_event_buffer *evt = _evt; 4580 struct dwc3 *dwc = evt->dwc; 4581 unsigned long flags; 4582 irqreturn_t ret = IRQ_NONE; 4583 4584 local_bh_disable(); 4585 spin_lock_irqsave(&dwc->lock, flags); 4586 ret = dwc3_process_event_buf(evt); 4587 spin_unlock_irqrestore(&dwc->lock, flags); 4588 local_bh_enable(); 4589 4590 return ret; 4591} 4592 4593static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) 4594{ 4595 struct dwc3 *dwc = evt->dwc; 4596 u32 amount; 4597 u32 count; 4598 4599 if (pm_runtime_suspended(dwc->dev)) { 4600 dwc->pending_events = true; 4601 /* 4602 * Trigger runtime resume. The get() function will be balanced 4603 * after processing the pending events in dwc3_process_pending 4604 * events(). 4605 */ 4606 pm_runtime_get(dwc->dev); 4607 disable_irq_nosync(dwc->irq_gadget); 4608 return IRQ_HANDLED; 4609 } 4610 4611 /* 4612 * With PCIe legacy interrupt, test shows that top-half irq handler can 4613 * be called again after HW interrupt deassertion. Check if bottom-half 4614 * irq event handler completes before caching new event to prevent 4615 * losing events. 4616 */ 4617 if (evt->flags & DWC3_EVENT_PENDING) 4618 return IRQ_HANDLED; 4619 4620 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); 4621 count &= DWC3_GEVNTCOUNT_MASK; 4622 if (!count) 4623 return IRQ_NONE; 4624 4625 if (count > evt->length) { 4626 dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n", 4627 count, evt->length); 4628 return IRQ_NONE; 4629 } 4630 4631 evt->count = count; 4632 evt->flags |= DWC3_EVENT_PENDING; 4633 4634 /* Mask interrupt */ 4635 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4636 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); 4637 4638 amount = min(count, evt->length - evt->lpos); 4639 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); 4640 4641 if (amount < count) 4642 memcpy(evt->cache, evt->buf, count - amount); 4643 4644 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); 4645 4646 return IRQ_WAKE_THREAD; 4647} 4648 4649static irqreturn_t dwc3_interrupt(int irq, void *_evt) 4650{ 4651 struct dwc3_event_buffer *evt = _evt; 4652 4653 return dwc3_check_event_buf(evt); 4654} 4655 4656static int dwc3_gadget_get_irq(struct dwc3 *dwc) 4657{ 4658 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); 4659 int irq; 4660 4661 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral"); 4662 if (irq > 0) 4663 goto out; 4664 4665 if (irq == -EPROBE_DEFER) 4666 goto out; 4667 4668 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3"); 4669 if (irq > 0) 4670 goto out; 4671 4672 if (irq == -EPROBE_DEFER) 4673 goto out; 4674 4675 irq = platform_get_irq(dwc3_pdev, 0); 4676 4677out: 4678 return irq; 4679} 4680 4681static void dwc_gadget_release(struct device *dev) 4682{ 4683 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev); 4684 4685 kfree(gadget); 4686} 4687 4688/** 4689 * dwc3_gadget_init - initializes gadget related registers 4690 * @dwc: pointer to our controller context structure 4691 * 4692 * Returns 0 on success otherwise negative errno. 4693 */ 4694int dwc3_gadget_init(struct dwc3 *dwc) 4695{ 4696 int ret; 4697 int irq; 4698 struct device *dev; 4699 4700 irq = dwc3_gadget_get_irq(dwc); 4701 if (irq < 0) { 4702 ret = irq; 4703 goto err0; 4704 } 4705 4706 dwc->irq_gadget = irq; 4707 4708 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, 4709 sizeof(*dwc->ep0_trb) * 2, 4710 &dwc->ep0_trb_addr, GFP_KERNEL); 4711 if (!dwc->ep0_trb) { 4712 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); 4713 ret = -ENOMEM; 4714 goto err0; 4715 } 4716 4717 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); 4718 if (!dwc->setup_buf) { 4719 ret = -ENOMEM; 4720 goto err1; 4721 } 4722 4723 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, 4724 &dwc->bounce_addr, GFP_KERNEL); 4725 if (!dwc->bounce) { 4726 ret = -ENOMEM; 4727 goto err2; 4728 } 4729 4730 init_completion(&dwc->ep0_in_setup); 4731 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); 4732 if (!dwc->gadget) { 4733 ret = -ENOMEM; 4734 goto err3; 4735 } 4736 4737 4738 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); 4739 dev = &dwc->gadget->dev; 4740 dev->platform_data = dwc; 4741 dwc->gadget->ops = &dwc3_gadget_ops; 4742 dwc->gadget->speed = USB_SPEED_UNKNOWN; 4743 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4744 dwc->gadget->sg_supported = true; 4745 dwc->gadget->name = "dwc3-gadget"; 4746 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; 4747 dwc->gadget->wakeup_capable = true; 4748 4749 /* 4750 * FIXME We might be setting max_speed to <SUPER, however versions 4751 * <2.20a of dwc3 have an issue with metastability (documented 4752 * elsewhere in this driver) which tells us we can't set max speed to 4753 * anything lower than SUPER. 4754 * 4755 * Because gadget.max_speed is only used by composite.c and function 4756 * drivers (i.e. it won't go into dwc3's registers) we are allowing this 4757 * to happen so we avoid sending SuperSpeed Capability descriptor 4758 * together with our BOS descriptor as that could confuse host into 4759 * thinking we can handle super speed. 4760 * 4761 * Note that, in fact, we won't even support GetBOS requests when speed 4762 * is less than super speed because we don't have means, yet, to tell 4763 * composite.c that we are USB 2.0 + LPM ECN. 4764 */ 4765 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 4766 !dwc->dis_metastability_quirk) 4767 dev_info(dwc->dev, "changing max_speed on rev %08x\n", 4768 dwc->revision); 4769 4770 dwc->gadget->max_speed = dwc->maximum_speed; 4771 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; 4772 4773 /* 4774 * REVISIT: Here we should clear all pending IRQs to be 4775 * sure we're starting from a well known location. 4776 */ 4777 4778 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); 4779 if (ret) 4780 goto err4; 4781 4782 ret = usb_add_gadget(dwc->gadget); 4783 if (ret) { 4784 dev_err(dwc->dev, "failed to add gadget\n"); 4785 goto err5; 4786 } 4787 4788 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) 4789 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); 4790 else 4791 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); 4792 4793 /* No system wakeup if no gadget driver bound */ 4794 if (dwc->sys_wakeup) 4795 device_wakeup_disable(dwc->sysdev); 4796 4797 return 0; 4798 4799err5: 4800 dwc3_gadget_free_endpoints(dwc); 4801err4: 4802 usb_put_gadget(dwc->gadget); 4803 dwc->gadget = NULL; 4804err3: 4805 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4806 dwc->bounce_addr); 4807 4808err2: 4809 kfree(dwc->setup_buf); 4810 4811err1: 4812 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4813 dwc->ep0_trb, dwc->ep0_trb_addr); 4814 4815err0: 4816 return ret; 4817} 4818EXPORT_SYMBOL_GPL(dwc3_gadget_init); 4819 4820/* -------------------------------------------------------------------------- */ 4821 4822void dwc3_gadget_exit(struct dwc3 *dwc) 4823{ 4824 if (!dwc->gadget) 4825 return; 4826 4827 dwc3_enable_susphy(dwc, true); 4828 usb_del_gadget(dwc->gadget); 4829 dwc3_gadget_free_endpoints(dwc); 4830 usb_put_gadget(dwc->gadget); 4831 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4832 dwc->bounce_addr); 4833 kfree(dwc->setup_buf); 4834 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4835 dwc->ep0_trb, dwc->ep0_trb_addr); 4836} 4837EXPORT_SYMBOL_GPL(dwc3_gadget_exit); 4838 4839int dwc3_gadget_suspend(struct dwc3 *dwc) 4840{ 4841 unsigned long flags; 4842 int ret; 4843 4844 ret = dwc3_gadget_soft_disconnect(dwc); 4845 /* 4846 * Attempt to reset the controller's state. Likely no 4847 * communication can be established until the host 4848 * performs a port reset. 4849 */ 4850 if (ret && dwc->softconnect) { 4851 dwc3_gadget_soft_connect(dwc); 4852 return -EAGAIN; 4853 } 4854 4855 spin_lock_irqsave(&dwc->lock, flags); 4856 if (dwc->gadget_driver) 4857 dwc3_disconnect_gadget(dwc); 4858 spin_unlock_irqrestore(&dwc->lock, flags); 4859 4860 return 0; 4861} 4862 4863int dwc3_gadget_resume(struct dwc3 *dwc) 4864{ 4865 if (!dwc->gadget_driver || !dwc->softconnect) 4866 return 0; 4867 4868 return dwc3_gadget_soft_connect(dwc); 4869}
~/
Read-only file preview.