cdnsp-gadget.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence CDNSP DRD Driver.
   4 *
   5 * Copyright (C) 2020 Cadence.
   6 *
   7 * Author: Pawel Laszczak <pawell@cadence.com>
   8 *
   9 */
  10
  11#include <linux/moduleparam.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/module.h>
  14#include <linux/iopoll.h>
  15#include <linux/delay.h>
  16#include <linux/log2.h>
  17#include <linux/slab.h>
  18#include <linux/string_choices.h>
  19#include <linux/pci.h>
  20#include <linux/irq.h>
  21#include <linux/dmi.h>
  22
  23#include "core.h"
  24#include "gadget-export.h"
  25#include "drd.h"
  26#include "cdnsp-gadget.h"
  27#include "cdnsp-trace.h"
  28
  29unsigned int cdnsp_port_speed(unsigned int port_status)
  30{
  31	/*Detect gadget speed based on PORTSC register*/
  32	if (DEV_SUPERSPEEDPLUS(port_status) ||
  33	    DEV_SSP_GEN1x2(port_status) || DEV_SSP_GEN2x2(port_status))
  34		return USB_SPEED_SUPER_PLUS;
  35	else if (DEV_SUPERSPEED(port_status))
  36		return USB_SPEED_SUPER;
  37	else if (DEV_HIGHSPEED(port_status))
  38		return USB_SPEED_HIGH;
  39	else if (DEV_FULLSPEED(port_status))
  40		return USB_SPEED_FULL;
  41
  42	/* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
  43	return USB_SPEED_UNKNOWN;
  44}
  45
  46/*
  47 * Given a port state, this function returns a value that would result in the
  48 * port being in the same state, if the value was written to the port status
  49 * control register.
  50 * Save Read Only (RO) bits and save read/write bits where
  51 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
  52 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
  53 */
  54u32 cdnsp_port_state_to_neutral(u32 state)
  55{
  56	/* Save read-only status and port state. */
  57	return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
  58}
  59
  60/**
  61 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
  62 *                           with capability ID id.
  63 * @base: PCI MMIO registers base address.
  64 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
  65 *         beginning of list)
  66 * @id: Extended capability ID to search for.
  67 *
  68 * Returns the offset of the next matching extended capability structure.
  69 * Some capabilities can occur several times,
  70 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
  71 */
  72int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
  73{
  74	u32 offset = start;
  75	u32 next;
  76	u32 val;
  77
  78	if (!start || start == HCC_PARAMS_OFFSET) {
  79		val = readl(base + HCC_PARAMS_OFFSET);
  80		if (val == ~0)
  81			return 0;
  82
  83		offset = HCC_EXT_CAPS(val) << 2;
  84		if (!offset)
  85			return 0;
  86	}
  87
  88	do {
  89		val = readl(base + offset);
  90		if (val == ~0)
  91			return 0;
  92
  93		if (EXT_CAPS_ID(val) == id && offset != start)
  94			return offset;
  95
  96		next = EXT_CAPS_NEXT(val);
  97		offset += next << 2;
  98	} while (next);
  99
 100	return 0;
 101}
 102
 103void cdnsp_set_link_state(struct cdnsp_device *pdev,
 104			  __le32 __iomem *port_regs,
 105			  u32 link_state)
 106{
 107	int port_num = 0xFF;
 108	u32 temp;
 109
 110	temp = readl(port_regs);
 111	temp = cdnsp_port_state_to_neutral(temp);
 112	temp |= PORT_WKCONN_E | PORT_WKDISC_E;
 113	writel(temp, port_regs);
 114
 115	temp &= ~PORT_PLS_MASK;
 116	temp |= PORT_LINK_STROBE | link_state;
 117
 118	if (pdev->active_port)
 119		port_num = pdev->active_port->port_num;
 120
 121	trace_cdnsp_handle_port_status(port_num, readl(port_regs));
 122	writel(temp, port_regs);
 123	trace_cdnsp_link_state_changed(port_num, readl(port_regs));
 124}
 125
 126static void cdnsp_disable_port(struct cdnsp_device *pdev,
 127			       __le32 __iomem *port_regs)
 128{
 129	u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
 130
 131	writel(temp | PORT_PED, port_regs);
 132}
 133
 134static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
 135					__le32 __iomem *port_regs)
 136{
 137	u32 portsc = readl(port_regs);
 138
 139	writel(cdnsp_port_state_to_neutral(portsc) |
 140	       (portsc & PORT_CHANGE_BITS), port_regs);
 141}
 142
 143static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev)
 144{
 145	struct cdns *cdns = dev_get_drvdata(pdev->dev);
 146	__le32 __iomem *reg;
 147	void __iomem *base;
 148	u32 offset = 0;
 149	u32 val;
 150
 151	if (!cdns->override_apb_timeout)
 152		return;
 153
 154	base = &pdev->cap_regs->hc_capbase;
 155	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 156	reg = base + offset + REG_CHICKEN_BITS_3_OFFSET;
 157
 158	val  = le32_to_cpu(readl(reg));
 159	val = CHICKEN_APB_TIMEOUT_SET(val, cdns->override_apb_timeout);
 160	writel(cpu_to_le32(val), reg);
 161}
 162
 163static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 164{
 165	__le32 __iomem *reg;
 166	void __iomem *base;
 167	u32 offset = 0;
 168
 169	base = &pdev->cap_regs->hc_capbase;
 170	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 171	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 172
 173	bit = readl(reg) | bit;
 174	writel(bit, reg);
 175}
 176
 177static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 178{
 179	__le32 __iomem *reg;
 180	void __iomem *base;
 181	u32 offset = 0;
 182
 183	base = &pdev->cap_regs->hc_capbase;
 184	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 185	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 186
 187	bit = readl(reg) & ~bit;
 188	writel(bit, reg);
 189}
 190
 191/*
 192 * Disable interrupts and begin the controller halting process.
 193 */
 194static void cdnsp_quiesce(struct cdnsp_device *pdev)
 195{
 196	u32 halted;
 197	u32 mask;
 198	u32 cmd;
 199
 200	mask = ~(u32)(CDNSP_IRQS);
 201
 202	halted = readl(&pdev->op_regs->status) & STS_HALT;
 203	if (!halted)
 204		mask &= ~(CMD_R_S | CMD_DEVEN);
 205
 206	cmd = readl(&pdev->op_regs->command);
 207	cmd &= mask;
 208	writel(cmd, &pdev->op_regs->command);
 209}
 210
 211/*
 212 * Force controller into halt state.
 213 *
 214 * Disable any IRQs and clear the run/stop bit.
 215 * Controller will complete any current and actively pipelined transactions, and
 216 * should halt within 16 ms of the run/stop bit being cleared.
 217 * Read controller Halted bit in the status register to see when the
 218 * controller is finished.
 219 */
 220int cdnsp_halt(struct cdnsp_device *pdev)
 221{
 222	int ret;
 223	u32 val;
 224
 225	cdnsp_quiesce(pdev);
 226
 227	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
 228					val & STS_HALT, 1,
 229					CDNSP_MAX_HALT_USEC);
 230	if (ret) {
 231		dev_err(pdev->dev, "ERROR: Device halt failed\n");
 232		return ret;
 233	}
 234
 235	pdev->cdnsp_state |= CDNSP_STATE_HALTED;
 236
 237	return 0;
 238}
 239
 240/*
 241 * device controller died, register read returns 0xffffffff, or command never
 242 * ends.
 243 */
 244void cdnsp_died(struct cdnsp_device *pdev)
 245{
 246	dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
 247	pdev->cdnsp_state |= CDNSP_STATE_DYING;
 248	cdnsp_halt(pdev);
 249}
 250
 251/*
 252 * Set the run bit and wait for the device to be running.
 253 */
 254static int cdnsp_start(struct cdnsp_device *pdev)
 255{
 256	u32 temp;
 257	int ret;
 258
 259	temp = readl(&pdev->op_regs->command);
 260	temp |= (CMD_R_S | CMD_DEVEN);
 261	writel(temp, &pdev->op_regs->command);
 262
 263	pdev->cdnsp_state = 0;
 264
 265	/*
 266	 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
 267	 * running.
 268	 */
 269	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 270					!(temp & STS_HALT), 1,
 271					CDNSP_MAX_HALT_USEC);
 272	if (ret) {
 273		pdev->cdnsp_state = CDNSP_STATE_DYING;
 274		dev_err(pdev->dev, "ERROR: Controller run failed\n");
 275	}
 276
 277	return ret;
 278}
 279
 280/*
 281 * Reset a halted controller.
 282 *
 283 * This resets pipelines, timers, counters, state machines, etc.
 284 * Transactions will be terminated immediately, and operational registers
 285 * will be set to their defaults.
 286 */
 287int cdnsp_reset(struct cdnsp_device *pdev)
 288{
 289	u32 command;
 290	u32 temp;
 291	int ret;
 292
 293	temp = readl(&pdev->op_regs->status);
 294
 295	if (temp == ~(u32)0) {
 296		dev_err(pdev->dev, "Device not accessible, reset failed.\n");
 297		return -ENODEV;
 298	}
 299
 300	if ((temp & STS_HALT) == 0) {
 301		dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
 302		return -EINVAL;
 303	}
 304
 305	command = readl(&pdev->op_regs->command);
 306	command |= CMD_RESET;
 307	writel(command, &pdev->op_regs->command);
 308
 309	ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
 310					!(temp & CMD_RESET), 1,
 311					10 * 1000);
 312	if (ret) {
 313		dev_err(pdev->dev, "ERROR: Controller reset failed\n");
 314		return ret;
 315	}
 316
 317	/*
 318	 * CDNSP cannot write any doorbells or operational registers other
 319	 * than status until the "Controller Not Ready" flag is cleared.
 320	 */
 321	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 322					!(temp & STS_CNR), 1,
 323					10 * 1000);
 324
 325	if (ret) {
 326		dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
 327		return ret;
 328	}
 329
 330	dev_dbg(pdev->dev, "Controller ready to work");
 331
 332	return ret;
 333}
 334
 335/*
 336 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
 337 * descriptor.Use the return value to right shift 1 for the bitmask.
 338 *
 339 * Index = (epnum * 2) + direction - 1,
 340 * where direction = 0 for OUT, 1 for IN.
 341 * For control endpoints, the IN index is used (OUT index is unused), so
 342 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
 343 */
 344static unsigned int
 345	cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
 346{
 347	unsigned int index = (unsigned int)usb_endpoint_num(desc);
 348
 349	if (usb_endpoint_xfer_control(desc))
 350		return index * 2;
 351
 352	return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
 353}
 354
 355/*
 356 * Find the flag for this endpoint (for use in the control context). Use the
 357 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
 358 * bit 1, etc.
 359 */
 360static unsigned int
 361	cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
 362{
 363	return 1 << (cdnsp_get_endpoint_index(desc) + 1);
 364}
 365
 366int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 367{
 368	struct cdnsp_device *pdev = pep->pdev;
 369	struct usb_request *request;
 370	int ret;
 371
 372	if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
 373		trace_cdnsp_request_enqueue_busy(preq);
 374		return -EBUSY;
 375	}
 376
 377	request = &preq->request;
 378	request->actual = 0;
 379	request->status = -EINPROGRESS;
 380	preq->direction = pep->direction;
 381	preq->epnum = pep->number;
 382	preq->td.drbl = 0;
 383
 384	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
 385	if (ret) {
 386		trace_cdnsp_request_enqueue_error(preq);
 387		return ret;
 388	}
 389
 390	list_add_tail(&preq->list, &pep->pending_list);
 391
 392	trace_cdnsp_request_enqueue(preq);
 393
 394	switch (usb_endpoint_type(pep->endpoint.desc)) {
 395	case USB_ENDPOINT_XFER_CONTROL:
 396		ret = cdnsp_queue_ctrl_tx(pdev, preq);
 397		break;
 398	case USB_ENDPOINT_XFER_BULK:
 399	case USB_ENDPOINT_XFER_INT:
 400		ret = cdnsp_queue_bulk_tx(pdev, preq);
 401		break;
 402	case USB_ENDPOINT_XFER_ISOC:
 403		ret = cdnsp_queue_isoc_tx(pdev, preq);
 404	}
 405
 406	if (ret)
 407		goto unmap;
 408
 409	return 0;
 410
 411unmap:
 412	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
 413					pep->direction);
 414	list_del(&preq->list);
 415	trace_cdnsp_request_enqueue_error(preq);
 416
 417	return ret;
 418}
 419
 420/*
 421 * Remove the request's TD from the endpoint ring. This may cause the
 422 * controller to stop USB transfers, potentially stopping in the middle of a
 423 * TRB buffer. The controller should pick up where it left off in the TD,
 424 * unless a Set Transfer Ring Dequeue Pointer is issued.
 425 *
 426 * The TRBs that make up the buffers for the canceled request will be "removed"
 427 * from the ring. Since the ring is a contiguous structure, they can't be
 428 * physically removed. Instead, there are two options:
 429 *
 430 *  1) If the controller is in the middle of processing the request to be
 431 *     canceled, we simply move the ring's dequeue pointer past those TRBs
 432 *     using the Set Transfer Ring Dequeue Pointer command. This will be
 433 *     the common case, when drivers timeout on the last submitted request
 434 *     and attempt to cancel.
 435 *
 436 *  2) If the controller is in the middle of a different TD, we turn the TRBs
 437 *     into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
 438 *     The controller will need to invalidate the any TRBs it has cached after
 439 *     the stop endpoint command.
 440 *
 441 *  3) The TD may have completed by the time the Stop Endpoint Command
 442 *     completes, so software needs to handle that case too.
 443 *
 444 */
 445int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 446{
 447	struct cdnsp_device *pdev = pep->pdev;
 448	int ret_stop = 0;
 449	int ret_rem;
 450
 451	trace_cdnsp_request_dequeue(preq);
 452
 453	if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
 454		ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
 455
 456	ret_rem = cdnsp_remove_request(pdev, preq, pep);
 457
 458	return ret_rem ? ret_rem : ret_stop;
 459}
 460
 461static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
 462{
 463	struct cdnsp_input_control_ctx *ctrl_ctx;
 464	struct cdnsp_slot_ctx *slot_ctx;
 465	struct cdnsp_ep_ctx *ep_ctx;
 466	int i;
 467
 468	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 469
 470	/*
 471	 * When a device's add flag and drop flag are zero, any subsequent
 472	 * configure endpoint command will leave that endpoint's state
 473	 * untouched. Make sure we don't leave any old state in the input
 474	 * endpoint contexts.
 475	 */
 476	ctrl_ctx->drop_flags = 0;
 477	ctrl_ctx->add_flags = 0;
 478	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 479	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 480
 481	/* Endpoint 0 is always valid */
 482	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
 483	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
 484		ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
 485		ep_ctx->ep_info = 0;
 486		ep_ctx->ep_info2 = 0;
 487		ep_ctx->deq = 0;
 488		ep_ctx->tx_info = 0;
 489	}
 490}
 491
 492/* Issue a configure endpoint command and wait for it to finish. */
 493static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
 494{
 495	int ret;
 496
 497	cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
 498	cdnsp_ring_cmd_db(pdev);
 499	ret = cdnsp_wait_for_cmd_compl(pdev);
 500	if (ret) {
 501		dev_err(pdev->dev,
 502			"ERR: unexpected command completion code 0x%x.\n", ret);
 503		return -EINVAL;
 504	}
 505
 506	return ret;
 507}
 508
 509static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
 510				       struct cdnsp_ep *pep)
 511{
 512	struct cdnsp_segment *segment;
 513	union cdnsp_trb *event;
 514	u32 cycle_state;
 515	u32  data;
 516
 517	event = pdev->event_ring->dequeue;
 518	segment = pdev->event_ring->deq_seg;
 519	cycle_state = pdev->event_ring->cycle_state;
 520
 521	while (1) {
 522		data = le32_to_cpu(event->trans_event.flags);
 523
 524		/* Check the owner of the TRB. */
 525		if ((data & TRB_CYCLE) != cycle_state)
 526			break;
 527
 528		if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
 529		    TRB_TO_EP_ID(data) == (pep->idx + 1)) {
 530			data |= TRB_EVENT_INVALIDATE;
 531			event->trans_event.flags = cpu_to_le32(data);
 532		}
 533
 534		if (cdnsp_last_trb_on_seg(segment, event)) {
 535			cycle_state ^= 1;
 536			segment = pdev->event_ring->deq_seg->next;
 537			event = segment->trbs;
 538		} else {
 539			event++;
 540		}
 541	}
 542}
 543
 544int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
 545{
 546	struct cdnsp_segment *event_deq_seg;
 547	union cdnsp_trb *cmd_trb;
 548	dma_addr_t cmd_deq_dma;
 549	union cdnsp_trb *event;
 550	u32 cycle_state;
 551	u32 retry = 10;
 552	int ret, val;
 553	u64 cmd_dma;
 554	u32  flags;
 555
 556	cmd_trb = pdev->cmd.command_trb;
 557	pdev->cmd.status = 0;
 558
 559	trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
 560
 561	ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
 562					!CMD_RING_BUSY(val), 1,
 563					CDNSP_CMD_TIMEOUT);
 564	if (ret) {
 565		dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
 566		trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
 567		pdev->cdnsp_state = CDNSP_STATE_DYING;
 568		return -ETIMEDOUT;
 569	}
 570
 571	event = pdev->event_ring->dequeue;
 572	event_deq_seg = pdev->event_ring->deq_seg;
 573	cycle_state = pdev->event_ring->cycle_state;
 574
 575	cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
 576	if (!cmd_deq_dma)
 577		return -EINVAL;
 578
 579	while (1) {
 580		flags = le32_to_cpu(event->event_cmd.flags);
 581
 582		/* Check the owner of the TRB. */
 583		if ((flags & TRB_CYCLE) != cycle_state) {
 584			/*
 585			 * Give some extra time to get chance controller
 586			 * to finish command before returning error code.
 587			 * Checking CMD_RING_BUSY is not sufficient because
 588			 * this bit is cleared to '0' when the Command
 589			 * Descriptor has been executed by controller
 590			 * and not when command completion event has
 591			 * be added to event ring.
 592			 */
 593			if (retry--) {
 594				udelay(20);
 595				continue;
 596			}
 597
 598			return -EINVAL;
 599		}
 600
 601		cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
 602
 603		/*
 604		 * Check whether the completion event is for last queued
 605		 * command.
 606		 */
 607		if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
 608		    cmd_dma != (u64)cmd_deq_dma) {
 609			if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
 610				event++;
 611				continue;
 612			}
 613
 614			if (cdnsp_last_trb_on_ring(pdev->event_ring,
 615						   event_deq_seg, event))
 616				cycle_state ^= 1;
 617
 618			event_deq_seg = event_deq_seg->next;
 619			event = event_deq_seg->trbs;
 620			continue;
 621		}
 622
 623		trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
 624
 625		pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
 626		if (pdev->cmd.status == COMP_SUCCESS)
 627			return 0;
 628
 629		return -pdev->cmd.status;
 630	}
 631}
 632
 633int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
 634			struct cdnsp_ep *pep,
 635			int value)
 636{
 637	int ret;
 638
 639	trace_cdnsp_ep_halt(value ? "Set" : "Clear");
 640
 641	ret = cdnsp_cmd_stop_ep(pdev, pep);
 642	if (ret)
 643		return ret;
 644
 645	if (value) {
 646		if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
 647			cdnsp_queue_halt_endpoint(pdev, pep->idx);
 648			cdnsp_ring_cmd_db(pdev);
 649			ret = cdnsp_wait_for_cmd_compl(pdev);
 650		}
 651
 652		pep->ep_state |= EP_HALTED;
 653	} else {
 654		cdnsp_queue_reset_ep(pdev, pep->idx);
 655		cdnsp_ring_cmd_db(pdev);
 656		ret = cdnsp_wait_for_cmd_compl(pdev);
 657		trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
 658
 659		if (ret)
 660			return ret;
 661
 662		pep->ep_state &= ~EP_HALTED;
 663
 664		if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
 665			cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 666
 667		pep->ep_state &= ~EP_WEDGE;
 668	}
 669
 670	return 0;
 671}
 672
 673static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
 674					  struct cdnsp_ep *pep)
 675{
 676	struct cdnsp_input_control_ctx *ctrl_ctx;
 677	struct cdnsp_slot_ctx *slot_ctx;
 678	int ret = 0;
 679	u32 ep_sts;
 680	int i;
 681
 682	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 683
 684	/* Don't issue the command if there's no endpoints to update. */
 685	if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
 686		return 0;
 687
 688	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
 689	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
 690	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
 691
 692	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
 693	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 694	for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
 695		__le32 le32 = cpu_to_le32(BIT(i));
 696
 697		if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
 698		    (ctrl_ctx->add_flags & le32) || i == 1) {
 699			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 700			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
 701			break;
 702		}
 703	}
 704
 705	ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
 706
 707	if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
 708	     ep_sts == EP_STATE_DISABLED) ||
 709	    (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
 710		ret = cdnsp_configure_endpoint(pdev);
 711
 712	trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
 713	trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
 714
 715	cdnsp_zero_in_ctx(pdev);
 716
 717	return ret;
 718}
 719
 720/*
 721 * This submits a Reset Device Command, which will set the device state to 0,
 722 * set the device address to 0, and disable all the endpoints except the default
 723 * control endpoint. The USB core should come back and call
 724 * cdnsp_setup_device(), and then re-set up the configuration.
 725 */
 726int cdnsp_reset_device(struct cdnsp_device *pdev)
 727{
 728	struct cdnsp_slot_ctx *slot_ctx;
 729	int slot_state;
 730	int ret, i;
 731
 732	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 733	slot_ctx->dev_info = 0;
 734	pdev->device_address = 0;
 735
 736	/* If device is not setup, there is no point in resetting it. */
 737	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 738	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 739	trace_cdnsp_reset_device(slot_ctx);
 740
 741	if (slot_state <= SLOT_STATE_DEFAULT &&
 742	    pdev->eps[0].ep_state & EP_HALTED) {
 743		cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
 744	}
 745
 746	/*
 747	 * During Reset Device command controller shall transition the
 748	 * endpoint ep0 to the Running State.
 749	 */
 750	pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
 751	pdev->eps[0].ep_state |= EP_ENABLED;
 752
 753	if (slot_state <= SLOT_STATE_DEFAULT)
 754		return 0;
 755
 756	cdnsp_queue_reset_device(pdev);
 757	cdnsp_ring_cmd_db(pdev);
 758	ret = cdnsp_wait_for_cmd_compl(pdev);
 759
 760	/*
 761	 * After Reset Device command all not default endpoints
 762	 * are in Disabled state.
 763	 */
 764	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
 765		pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
 766
 767	trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
 768
 769	if (ret)
 770		dev_err(pdev->dev, "Reset device failed with error code %d",
 771			ret);
 772
 773	return ret;
 774}
 775
 776/*
 777 * Sets the MaxPStreams field and the Linear Stream Array field.
 778 * Sets the dequeue pointer to the stream context array.
 779 */
 780static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
 781					     struct cdnsp_ep_ctx *ep_ctx,
 782					     struct cdnsp_stream_info *stream_info)
 783{
 784	u32 max_primary_streams;
 785
 786	/* MaxPStreams is the number of stream context array entries, not the
 787	 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
 788	 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
 789	 */
 790	max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
 791	ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
 792	ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
 793				       | EP_HAS_LSA);
 794	ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
 795}
 796
 797/*
 798 * The drivers use this function to prepare a bulk endpoints to use streams.
 799 *
 800 * Don't allow the call to succeed if endpoint only supports one stream
 801 * (which means it doesn't support streams at all).
 802 */
 803int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
 804{
 805	unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
 806	unsigned int num_stream_ctxs;
 807	int ret;
 808
 809	if (num_streams ==  0)
 810		return 0;
 811
 812	if (num_streams > STREAM_NUM_STREAMS)
 813		return -EINVAL;
 814
 815	/*
 816	 * Add two to the number of streams requested to account for
 817	 * stream 0 that is reserved for controller usage and one additional
 818	 * for TASK SET FULL response.
 819	 */
 820	num_streams += 2;
 821
 822	/* The stream context array size must be a power of two */
 823	num_stream_ctxs = roundup_pow_of_two(num_streams);
 824
 825	trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
 826
 827	ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
 828	if (ret)
 829		return ret;
 830
 831	cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
 832
 833	pep->ep_state |= EP_HAS_STREAMS;
 834	pep->stream_info.td_count = 0;
 835	pep->stream_info.first_prime_det = 0;
 836
 837	/* Subtract 1 for stream 0, which drivers can't use. */
 838	return num_streams - 1;
 839}
 840
 841int cdnsp_disable_slot(struct cdnsp_device *pdev)
 842{
 843	int ret;
 844
 845	cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
 846	cdnsp_ring_cmd_db(pdev);
 847	ret = cdnsp_wait_for_cmd_compl(pdev);
 848
 849	pdev->slot_id = 0;
 850	pdev->active_port = NULL;
 851
 852	trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 853
 854	memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
 855	memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
 856
 857	return ret;
 858}
 859
 860int cdnsp_enable_slot(struct cdnsp_device *pdev)
 861{
 862	struct cdnsp_slot_ctx *slot_ctx;
 863	int slot_state;
 864	int ret;
 865
 866	/* If device is not setup, there is no point in resetting it */
 867	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 868	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 869
 870	if (slot_state != SLOT_STATE_DISABLED)
 871		return 0;
 872
 873	cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
 874	cdnsp_ring_cmd_db(pdev);
 875	ret = cdnsp_wait_for_cmd_compl(pdev);
 876	if (ret)
 877		goto show_trace;
 878
 879	pdev->slot_id = 1;
 880
 881show_trace:
 882	trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 883
 884	return ret;
 885}
 886
 887/*
 888 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
 889 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
 890 */
 891int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
 892{
 893	struct cdnsp_input_control_ctx *ctrl_ctx;
 894	struct cdnsp_slot_ctx *slot_ctx;
 895	int dev_state = 0;
 896	int ret;
 897
 898	if (!pdev->slot_id) {
 899		trace_cdnsp_slot_id("incorrect");
 900		return -EINVAL;
 901	}
 902
 903	if (!pdev->active_port->port_num)
 904		return -EINVAL;
 905
 906	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 907	dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 908
 909	if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
 910		trace_cdnsp_slot_already_in_default(slot_ctx);
 911		return 0;
 912	}
 913
 914	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 915	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 916
 917	if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
 918		ret = cdnsp_setup_addressable_priv_dev(pdev);
 919		if (ret)
 920			return ret;
 921	}
 922
 923	cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
 924
 925	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
 926	ctrl_ctx->drop_flags = 0;
 927
 928	trace_cdnsp_setup_device_slot(slot_ctx);
 929
 930	cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
 931	cdnsp_ring_cmd_db(pdev);
 932	ret = cdnsp_wait_for_cmd_compl(pdev);
 933
 934	trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
 935
 936	/* Zero the input context control for later use. */
 937	ctrl_ctx->add_flags = 0;
 938	ctrl_ctx->drop_flags = 0;
 939
 940	return ret;
 941}
 942
 943void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
 944				 struct usb_request *req,
 945				 int enable)
 946{
 947	if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
 948		return;
 949
 950	trace_cdnsp_lpm(enable);
 951
 952	if (enable)
 953		writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
 954		       &pdev->active_port->regs->portpmsc);
 955	else
 956		writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
 957}
 958
 959static int cdnsp_get_frame(struct cdnsp_device *pdev)
 960{
 961	return readl(&pdev->run_regs->microframe_index) >> 3;
 962}
 963
 964static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
 965				  const struct usb_endpoint_descriptor *desc)
 966{
 967	struct cdnsp_input_control_ctx *ctrl_ctx;
 968	struct cdnsp_device *pdev;
 969	struct cdnsp_ep *pep;
 970	unsigned long flags;
 971	u32 added_ctxs;
 972	int ret;
 973
 974	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
 975	    !desc->wMaxPacketSize)
 976		return -EINVAL;
 977
 978	pep = to_cdnsp_ep(ep);
 979	pdev = pep->pdev;
 980	pep->ep_state &= ~EP_UNCONFIGURED;
 981
 982	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
 983			  "%s is already enabled\n", pep->name))
 984		return 0;
 985
 986	spin_lock_irqsave(&pdev->lock, flags);
 987
 988	added_ctxs = cdnsp_get_endpoint_flag(desc);
 989	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
 990		dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
 991		ret = -EINVAL;
 992		goto unlock;
 993	}
 994
 995	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
 996
 997	if (pdev->gadget.speed == USB_SPEED_FULL) {
 998		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
 999			pep->interval = desc->bInterval << 3;
1000		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1001			pep->interval = BIT(desc->bInterval - 1) << 3;
1002	}
1003
1004	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
1005		if (pep->interval > BIT(12)) {
1006			dev_err(pdev->dev, "bInterval %d not supported\n",
1007				desc->bInterval);
1008			ret = -EINVAL;
1009			goto unlock;
1010		}
1011		cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1012	}
1013
1014	ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
1015	if (ret)
1016		goto unlock;
1017
1018	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1019	ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
1020	ctrl_ctx->drop_flags = 0;
1021
1022	ret = cdnsp_update_eps_configuration(pdev, pep);
1023	if (ret) {
1024		cdnsp_free_endpoint_rings(pdev, pep);
1025		goto unlock;
1026	}
1027
1028	pep->ep_state |= EP_ENABLED;
1029	pep->ep_state &= ~EP_STOPPED;
1030
1031unlock:
1032	trace_cdnsp_ep_enable_end(pep, 0);
1033	spin_unlock_irqrestore(&pdev->lock, flags);
1034
1035	return ret;
1036}
1037
1038static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1039{
1040	struct cdnsp_input_control_ctx *ctrl_ctx;
1041	struct cdnsp_request *preq;
1042	struct cdnsp_device *pdev;
1043	struct cdnsp_ep *pep;
1044	unsigned long flags;
1045	u32 drop_flag;
1046	int ret = 0;
1047
1048	if (!ep)
1049		return -EINVAL;
1050
1051	pep = to_cdnsp_ep(ep);
1052	pdev = pep->pdev;
1053
1054	spin_lock_irqsave(&pdev->lock, flags);
1055
1056	if (!(pep->ep_state & EP_ENABLED)) {
1057		dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1058		ret = -EINVAL;
1059		goto finish;
1060	}
1061
1062	pep->ep_state |= EP_DIS_IN_RROGRESS;
1063
1064	/* Endpoint was unconfigured by Reset Device command. */
1065	if (!(pep->ep_state & EP_UNCONFIGURED))
1066		cdnsp_cmd_stop_ep(pdev, pep);
1067
1068	/* Remove all queued USB requests. */
1069	while (!list_empty(&pep->pending_list)) {
1070		preq = next_request(&pep->pending_list);
1071		cdnsp_ep_dequeue(pep, preq);
1072	}
1073
1074	cdnsp_invalidate_ep_events(pdev, pep);
1075
1076	pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1077	drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1078	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1079	ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1080	ctrl_ctx->add_flags = 0;
1081
1082	cdnsp_endpoint_zero(pdev, pep);
1083
1084	if (!(pep->ep_state & EP_UNCONFIGURED))
1085		ret = cdnsp_update_eps_configuration(pdev, pep);
1086
1087	cdnsp_free_endpoint_rings(pdev, pep);
1088
1089	pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1090	pep->ep_state |= EP_STOPPED;
1091
1092finish:
1093	trace_cdnsp_ep_disable_end(pep, 0);
1094	spin_unlock_irqrestore(&pdev->lock, flags);
1095
1096	return ret;
1097}
1098
1099static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1100							 gfp_t gfp_flags)
1101{
1102	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1103	struct cdnsp_request *preq;
1104
1105	preq = kzalloc(sizeof(*preq), gfp_flags);
1106	if (!preq)
1107		return NULL;
1108
1109	preq->epnum = pep->number;
1110	preq->pep = pep;
1111
1112	trace_cdnsp_alloc_request(preq);
1113
1114	return &preq->request;
1115}
1116
1117static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1118					 struct usb_request *request)
1119{
1120	struct cdnsp_request *preq = to_cdnsp_request(request);
1121
1122	trace_cdnsp_free_request(preq);
1123	kfree(preq);
1124}
1125
1126static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1127				 struct usb_request *request,
1128				 gfp_t gfp_flags)
1129{
1130	struct cdnsp_request *preq;
1131	struct cdnsp_device *pdev;
1132	struct cdnsp_ep *pep;
1133	unsigned long flags;
1134	int ret;
1135
1136	if (!request || !ep)
1137		return -EINVAL;
1138
1139	pep = to_cdnsp_ep(ep);
1140	pdev = pep->pdev;
1141
1142	if (!(pep->ep_state & EP_ENABLED)) {
1143		dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1144			pep->name);
1145		return -EINVAL;
1146	}
1147
1148	preq = to_cdnsp_request(request);
1149	spin_lock_irqsave(&pdev->lock, flags);
1150	ret = cdnsp_ep_enqueue(pep, preq);
1151	spin_unlock_irqrestore(&pdev->lock, flags);
1152
1153	return ret;
1154}
1155
1156static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1157				   struct usb_request *request)
1158{
1159	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1160	struct cdnsp_device *pdev = pep->pdev;
1161	unsigned long flags;
1162	int ret;
1163
1164	if (request->status != -EINPROGRESS)
1165		return 0;
1166
1167	if (!pep->endpoint.desc) {
1168		dev_err(pdev->dev,
1169			"%s: can't dequeue to disabled endpoint\n",
1170			pep->name);
1171		return -ESHUTDOWN;
1172	}
1173
1174	/* Requests has been dequeued during disabling endpoint. */
1175	if (!(pep->ep_state & EP_ENABLED))
1176		return 0;
1177
1178	spin_lock_irqsave(&pdev->lock, flags);
1179	ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1180	spin_unlock_irqrestore(&pdev->lock, flags);
1181
1182	return ret;
1183}
1184
1185static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1186{
1187	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1188	struct cdnsp_device *pdev = pep->pdev;
1189	struct cdnsp_request *preq;
1190	unsigned long flags;
1191	int ret;
1192
1193	spin_lock_irqsave(&pdev->lock, flags);
1194
1195	preq = next_request(&pep->pending_list);
1196	if (value) {
1197		if (preq) {
1198			trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1199			ret = -EAGAIN;
1200			goto done;
1201		}
1202	}
1203
1204	ret = cdnsp_halt_endpoint(pdev, pep, value);
1205
1206done:
1207	spin_unlock_irqrestore(&pdev->lock, flags);
1208	return ret;
1209}
1210
1211static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1212{
1213	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1214	struct cdnsp_device *pdev = pep->pdev;
1215	unsigned long flags;
1216	int ret;
1217
1218	spin_lock_irqsave(&pdev->lock, flags);
1219	pep->ep_state |= EP_WEDGE;
1220	ret = cdnsp_halt_endpoint(pdev, pep, 1);
1221	spin_unlock_irqrestore(&pdev->lock, flags);
1222
1223	return ret;
1224}
1225
1226static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1227	.enable		= cdnsp_gadget_ep_enable,
1228	.disable	= cdnsp_gadget_ep_disable,
1229	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1230	.free_request	= cdnsp_gadget_ep_free_request,
1231	.queue		= cdnsp_gadget_ep_queue,
1232	.dequeue	= cdnsp_gadget_ep_dequeue,
1233	.set_halt	= cdnsp_gadget_ep_set_halt,
1234	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1235};
1236
1237static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1238	.enable		= cdnsp_gadget_ep_enable,
1239	.disable	= cdnsp_gadget_ep_disable,
1240	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1241	.free_request	= cdnsp_gadget_ep_free_request,
1242	.queue		= cdnsp_gadget_ep_queue,
1243	.dequeue	= cdnsp_gadget_ep_dequeue,
1244	.set_halt	= cdnsp_gadget_ep_set_halt,
1245	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1246};
1247
1248void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1249			   struct cdnsp_request *preq,
1250			   int status)
1251{
1252	struct cdnsp_device *pdev = pep->pdev;
1253
1254	list_del(&preq->list);
1255
1256	if (preq->request.status == -EINPROGRESS)
1257		preq->request.status = status;
1258
1259	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1260					preq->direction);
1261
1262	trace_cdnsp_request_giveback(preq);
1263
1264	if (preq != &pdev->ep0_preq) {
1265		spin_unlock(&pdev->lock);
1266		usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1267		spin_lock(&pdev->lock);
1268	}
1269}
1270
1271static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1272	.bLength =		USB_DT_ENDPOINT_SIZE,
1273	.bDescriptorType =	USB_DT_ENDPOINT,
1274	.bmAttributes =		USB_ENDPOINT_XFER_CONTROL,
1275};
1276
1277static int cdnsp_run(struct cdnsp_device *pdev,
1278		     enum usb_device_speed speed)
1279{
1280	u32 fs_speed = 0;
1281	u32 temp;
1282	int ret;
1283
1284	temp = readl(&pdev->ir_set->irq_control);
1285	temp &= ~IMOD_INTERVAL_MASK;
1286	temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1287	writel(temp, &pdev->ir_set->irq_control);
1288
1289	temp = readl(&pdev->port3x_regs->mode_addr);
1290
1291	switch (speed) {
1292	case USB_SPEED_SUPER_PLUS:
1293		temp |= CFG_3XPORT_SSP_SUPPORT;
1294		break;
1295	case USB_SPEED_SUPER:
1296		temp &= ~CFG_3XPORT_SSP_SUPPORT;
1297		break;
1298	case USB_SPEED_HIGH:
1299		break;
1300	case USB_SPEED_FULL:
1301		fs_speed = PORT_REG6_FORCE_FS;
1302		break;
1303	default:
1304		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1305			speed);
1306		fallthrough;
1307	case USB_SPEED_UNKNOWN:
1308		/* Default to superspeed. */
1309		speed = USB_SPEED_SUPER;
1310		break;
1311	}
1312
1313	if (speed >= USB_SPEED_SUPER) {
1314		writel(temp, &pdev->port3x_regs->mode_addr);
1315		cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1316				     XDEV_RXDETECT);
1317	} else {
1318		cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1319	}
1320
1321	cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1322			     XDEV_RXDETECT);
1323
1324	cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1325
1326	writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1327
1328	ret = cdnsp_start(pdev);
1329	if (ret) {
1330		ret = -ENODEV;
1331		goto err;
1332	}
1333
1334	temp = readl(&pdev->op_regs->command);
1335	temp |= (CMD_INTE);
1336	writel(temp, &pdev->op_regs->command);
1337
1338	temp = readl(&pdev->ir_set->irq_pending);
1339	writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1340
1341	trace_cdnsp_init("Controller ready to work");
1342	return 0;
1343err:
1344	cdnsp_halt(pdev);
1345	return ret;
1346}
1347
1348static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1349				  struct usb_gadget_driver *driver)
1350{
1351	enum usb_device_speed max_speed = driver->max_speed;
1352	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1353	unsigned long flags;
1354	int ret;
1355
1356	spin_lock_irqsave(&pdev->lock, flags);
1357	pdev->gadget_driver = driver;
1358
1359	/* limit speed if necessary */
1360	max_speed = min(driver->max_speed, g->max_speed);
1361	ret = cdnsp_run(pdev, max_speed);
1362
1363	spin_unlock_irqrestore(&pdev->lock, flags);
1364
1365	return ret;
1366}
1367
1368/*
1369 * Update Event Ring Dequeue Pointer:
1370 * - When all events have finished
1371 * - To avoid "Event Ring Full Error" condition
1372 */
1373void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1374			       union cdnsp_trb *event_ring_deq,
1375			       u8 clear_ehb)
1376{
1377	u64 temp_64;
1378	dma_addr_t deq;
1379
1380	temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1381
1382	/* If necessary, update the HW's version of the event ring deq ptr. */
1383	if (event_ring_deq != pdev->event_ring->dequeue) {
1384		deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1385					    pdev->event_ring->dequeue);
1386		temp_64 &= ERST_PTR_MASK;
1387		temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1388	}
1389
1390	/* Clear the event handler busy flag (RW1C). */
1391	if (clear_ehb)
1392		temp_64 |= ERST_EHB;
1393	else
1394		temp_64 &= ~ERST_EHB;
1395
1396	cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1397}
1398
1399static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1400{
1401	struct cdnsp_segment *seg;
1402	u64 val_64;
1403	int i;
1404
1405	cdnsp_initialize_ring_info(pdev->cmd_ring);
1406
1407	seg = pdev->cmd_ring->first_seg;
1408	for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1409		memset(seg->trbs, 0,
1410		       sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1411		seg = seg->next;
1412	}
1413
1414	/* Set the address in the Command Ring Control register. */
1415	val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1416	val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1417		 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1418		 pdev->cmd_ring->cycle_state;
1419	cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1420}
1421
1422static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1423{
1424	struct cdnsp_segment *event_deq_seg;
1425	union cdnsp_trb *event_ring_deq;
1426	union cdnsp_trb *event;
1427	u32 cycle_bit;
1428
1429	event_ring_deq = pdev->event_ring->dequeue;
1430	event_deq_seg = pdev->event_ring->deq_seg;
1431	event = pdev->event_ring->dequeue;
1432
1433	/* Update ring dequeue pointer. */
1434	while (1) {
1435		cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1436
1437		/* Does the controller or driver own the TRB? */
1438		if (cycle_bit != pdev->event_ring->cycle_state)
1439			break;
1440
1441		cdnsp_inc_deq(pdev, pdev->event_ring);
1442
1443		if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1444			event++;
1445			continue;
1446		}
1447
1448		if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1449					   event))
1450			cycle_bit ^= 1;
1451
1452		event_deq_seg = event_deq_seg->next;
1453		event = event_deq_seg->trbs;
1454	}
1455
1456	cdnsp_update_erst_dequeue(pdev,  event_ring_deq, 1);
1457}
1458
1459static void cdnsp_stop(struct cdnsp_device *pdev)
1460{
1461	u32 temp;
1462
1463	/* Remove internally queued request for ep0. */
1464	if (!list_empty(&pdev->eps[0].pending_list)) {
1465		struct cdnsp_request *req;
1466
1467		req = next_request(&pdev->eps[0].pending_list);
1468		if (req == &pdev->ep0_preq)
1469			cdnsp_ep_dequeue(&pdev->eps[0], req);
1470	}
1471
1472	cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1473	cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1474	cdnsp_disable_slot(pdev);
1475	cdnsp_halt(pdev);
1476
1477	temp = readl(&pdev->op_regs->status);
1478	writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1479	temp = readl(&pdev->ir_set->irq_pending);
1480	writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1481
1482	cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1483	cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1484
1485	/* Clear interrupt line */
1486	temp = readl(&pdev->ir_set->irq_pending);
1487	temp |= IMAN_IP;
1488	writel(temp, &pdev->ir_set->irq_pending);
1489
1490	cdnsp_consume_all_events(pdev);
1491	cdnsp_clear_cmd_ring(pdev);
1492
1493	trace_cdnsp_exit("Controller stopped.");
1494}
1495
1496/*
1497 * Stop controller.
1498 * This function is called by the gadget core when the driver is removed.
1499 * Disable slot, disable IRQs, and quiesce the controller.
1500 */
1501static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1502{
1503	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1504	unsigned long flags;
1505
1506	spin_lock_irqsave(&pdev->lock, flags);
1507	cdnsp_stop(pdev);
1508	pdev->gadget_driver = NULL;
1509	spin_unlock_irqrestore(&pdev->lock, flags);
1510
1511	return 0;
1512}
1513
1514static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1515{
1516	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1517
1518	return cdnsp_get_frame(pdev);
1519}
1520
1521static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1522{
1523	struct cdnsp_port_regs __iomem *port_regs;
1524	u32 portpm, portsc;
1525
1526	port_regs = pdev->active_port->regs;
1527	portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1528
1529	/* Remote wakeup feature is not enabled by host. */
1530	if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1531		portpm = readl(&port_regs->portpmsc);
1532
1533		if (!(portpm & PORT_RWE))
1534			return;
1535	}
1536
1537	if (portsc == XDEV_U3 && !pdev->may_wakeup)
1538		return;
1539
1540	cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1541
1542	pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1543}
1544
1545static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1546{
1547	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1548	unsigned long flags;
1549
1550	spin_lock_irqsave(&pdev->lock, flags);
1551	__cdnsp_gadget_wakeup(pdev);
1552	spin_unlock_irqrestore(&pdev->lock, flags);
1553
1554	return 0;
1555}
1556
1557static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1558					int is_selfpowered)
1559{
1560	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1561	unsigned long flags;
1562
1563	spin_lock_irqsave(&pdev->lock, flags);
1564	g->is_selfpowered = !!is_selfpowered;
1565	spin_unlock_irqrestore(&pdev->lock, flags);
1566
1567	return 0;
1568}
1569
1570static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1571{
1572	struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1573	struct cdns *cdns = dev_get_drvdata(pdev->dev);
1574	unsigned long flags;
1575
1576	trace_cdnsp_pullup(is_on);
1577
1578	/*
1579	 * Disable events handling while controller is being
1580	 * enabled/disabled.
1581	 */
1582	disable_irq(cdns->dev_irq);
1583	spin_lock_irqsave(&pdev->lock, flags);
1584
1585	if (!is_on) {
1586		cdnsp_reset_device(pdev);
1587		cdns_clear_vbus(cdns);
1588	} else {
1589		cdns_set_vbus(cdns);
1590	}
1591
1592	spin_unlock_irqrestore(&pdev->lock, flags);
1593	enable_irq(cdns->dev_irq);
1594
1595	return 0;
1596}
1597
1598static const struct usb_gadget_ops cdnsp_gadget_ops = {
1599	.get_frame		= cdnsp_gadget_get_frame,
1600	.wakeup			= cdnsp_gadget_wakeup,
1601	.set_selfpowered	= cdnsp_gadget_set_selfpowered,
1602	.pullup			= cdnsp_gadget_pullup,
1603	.udc_start		= cdnsp_gadget_udc_start,
1604	.udc_stop		= cdnsp_gadget_udc_stop,
1605};
1606
1607static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1608				   struct cdnsp_ep *pep)
1609{
1610	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1611	int endpoints;
1612
1613	reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1614
1615	if (!pep->direction) {
1616		pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1617		pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1618		pep->buffering = (pep->buffering + 1) / 2;
1619		pep->buffering_period = (pep->buffering_period + 1) / 2;
1620		return;
1621	}
1622
1623	endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1624
1625	/* Set to XBUF_TX_TAG_MASK_0 register. */
1626	reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1627	/* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1628	reg += pep->number * sizeof(u32) * 2;
1629
1630	pep->buffering = (readl(reg) + 1) / 2;
1631	pep->buffering_period = pep->buffering;
1632}
1633
1634static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1635{
1636	int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1637	struct cdnsp_ep *pep;
1638	int i;
1639
1640	INIT_LIST_HEAD(&pdev->gadget.ep_list);
1641
1642	if (max_streams < STREAM_LOG_STREAMS) {
1643		dev_err(pdev->dev, "Stream size %d not supported\n",
1644			max_streams);
1645		return -EINVAL;
1646	}
1647
1648	max_streams = STREAM_LOG_STREAMS;
1649
1650	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1651		bool direction = !(i & 1); /* Start from OUT endpoint. */
1652		u8 epnum = ((i + 1) >> 1);
1653
1654		if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1655			continue;
1656
1657		pep = &pdev->eps[i];
1658		pep->pdev = pdev;
1659		pep->number = epnum;
1660		pep->direction = direction; /* 0 for OUT, 1 for IN. */
1661
1662		/*
1663		 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1664		 * pdev->eps[0]
1665		 */
1666		if (epnum == 0) {
1667			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1668				 epnum, "BiDir");
1669
1670			pep->idx = 0;
1671			usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1672			pep->endpoint.maxburst = 1;
1673			pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1674			pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1675			pep->endpoint.comp_desc = NULL;
1676			pep->endpoint.caps.type_control = true;
1677			pep->endpoint.caps.dir_in = true;
1678			pep->endpoint.caps.dir_out = true;
1679
1680			pdev->ep0_preq.epnum = pep->number;
1681			pdev->ep0_preq.pep = pep;
1682			pdev->gadget.ep0 = &pep->endpoint;
1683		} else {
1684			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1685				 epnum, (pep->direction) ? "in" : "out");
1686
1687			pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;
1688			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1689
1690			pep->endpoint.max_streams = max_streams;
1691			pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1692			list_add_tail(&pep->endpoint.ep_list,
1693				      &pdev->gadget.ep_list);
1694
1695			pep->endpoint.caps.type_iso = true;
1696			pep->endpoint.caps.type_bulk = true;
1697			pep->endpoint.caps.type_int = true;
1698
1699			pep->endpoint.caps.dir_in = direction;
1700			pep->endpoint.caps.dir_out = !direction;
1701		}
1702
1703		pep->endpoint.name = pep->name;
1704		pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1705		pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1706		cdnsp_get_ep_buffering(pdev, pep);
1707
1708		dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1709			"CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1710			"SupDir IN: %s, OUT: %s\n",
1711			pep->name, 1024,
1712			str_yes_no(pep->endpoint.caps.type_control),
1713			str_yes_no(pep->endpoint.caps.type_int),
1714			str_yes_no(pep->endpoint.caps.type_bulk),
1715			str_yes_no(pep->endpoint.caps.type_iso),
1716			str_yes_no(pep->endpoint.caps.dir_in),
1717			str_yes_no(pep->endpoint.caps.dir_out));
1718
1719		INIT_LIST_HEAD(&pep->pending_list);
1720	}
1721
1722	return 0;
1723}
1724
1725static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1726{
1727	struct cdnsp_ep *pep;
1728	int i;
1729
1730	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1731		pep = &pdev->eps[i];
1732		if (pep->number != 0 && pep->out_ctx)
1733			list_del(&pep->endpoint.ep_list);
1734	}
1735}
1736
1737void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1738{
1739	pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1740
1741	if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1742		spin_unlock(&pdev->lock);
1743		pdev->gadget_driver->disconnect(&pdev->gadget);
1744		spin_lock(&pdev->lock);
1745	}
1746
1747	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1748	usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1749
1750	pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1751}
1752
1753void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1754{
1755	if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1756		spin_unlock(&pdev->lock);
1757		pdev->gadget_driver->suspend(&pdev->gadget);
1758		spin_lock(&pdev->lock);
1759	}
1760}
1761
1762void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1763{
1764	if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1765		spin_unlock(&pdev->lock);
1766		pdev->gadget_driver->resume(&pdev->gadget);
1767		spin_lock(&pdev->lock);
1768	}
1769}
1770
1771void cdnsp_irq_reset(struct cdnsp_device *pdev)
1772{
1773	struct cdnsp_port_regs __iomem *port_regs;
1774
1775	cdnsp_reset_device(pdev);
1776
1777	port_regs = pdev->active_port->regs;
1778	pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1779
1780	spin_unlock(&pdev->lock);
1781	usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1782	spin_lock(&pdev->lock);
1783
1784	switch (pdev->gadget.speed) {
1785	case USB_SPEED_SUPER_PLUS:
1786	case USB_SPEED_SUPER:
1787		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1788		pdev->gadget.ep0->maxpacket = 512;
1789		break;
1790	case USB_SPEED_HIGH:
1791	case USB_SPEED_FULL:
1792		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1793		pdev->gadget.ep0->maxpacket = 64;
1794		break;
1795	default:
1796		/* Low speed is not supported. */
1797		dev_err(pdev->dev, "Unknown device speed\n");
1798		break;
1799	}
1800
1801	cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1802	cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1803	usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1804}
1805
1806static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1807{
1808	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1809
1810	reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1811	pdev->rev_cap  = reg;
1812
1813	pdev->rtl_revision = readl(&pdev->rev_cap->rtl_revision);
1814
1815	dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1816		 readl(&pdev->rev_cap->ctrl_revision),
1817		 readl(&pdev->rev_cap->rtl_revision),
1818		 readl(&pdev->rev_cap->ep_supported),
1819		 readl(&pdev->rev_cap->rx_buff_size),
1820		 readl(&pdev->rev_cap->tx_buff_size));
1821}
1822
1823static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1824{
1825	int ret;
1826	u32 reg;
1827
1828	pdev->cap_regs = pdev->regs;
1829	pdev->op_regs = pdev->regs +
1830		HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1831	pdev->run_regs = pdev->regs +
1832		(readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1833
1834	/* Cache read-only capability registers */
1835	pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1836	pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1837	pdev->hci_version = HC_VERSION(pdev->hcc_params);
1838	pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1839
1840	/*
1841	 * Override the APB timeout value to give the controller more time for
1842	 * enabling UTMI clock and synchronizing APB and UTMI clock domains.
1843	 * This fix is platform specific and is required to fixes issue with
1844	 * reading incorrect value from PORTSC register after resuming
1845	 * from L1 state.
1846	 */
1847	cdnsp_set_apb_timeout_value(pdev);
1848
1849	cdnsp_get_rev_cap(pdev);
1850
1851	/* Make sure the Device Controller is halted. */
1852	ret = cdnsp_halt(pdev);
1853	if (ret)
1854		return ret;
1855
1856	/* Reset the internal controller memory state and registers. */
1857	ret = cdnsp_reset(pdev);
1858	if (ret)
1859		return ret;
1860
1861	/*
1862	 * Set dma_mask and coherent_dma_mask to 64-bits,
1863	 * if controller supports 64-bit addressing.
1864	 */
1865	if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1866	    !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1867		dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1868		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1869	} else {
1870		/*
1871		 * This is to avoid error in cases where a 32-bit USB
1872		 * controller is used on a 64-bit capable system.
1873		 */
1874		ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1875		if (ret)
1876			return ret;
1877
1878		dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1879		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1880	}
1881
1882	spin_lock_init(&pdev->lock);
1883
1884	ret = cdnsp_mem_init(pdev);
1885	if (ret)
1886		return ret;
1887
1888	/*
1889	 * Software workaround for U1: after transition
1890	 * to U1 the controller starts gating clock, and in some cases,
1891	 * it causes that controller stack.
1892	 */
1893	reg = readl(&pdev->port3x_regs->mode_2);
1894	reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1895	writel(reg, &pdev->port3x_regs->mode_2);
1896
1897	return 0;
1898}
1899
1900static int __cdnsp_gadget_init(struct cdns *cdns)
1901{
1902	struct cdnsp_device *pdev;
1903	u32 max_speed;
1904	int ret = -ENOMEM;
1905
1906	cdns_drd_gadget_on(cdns);
1907
1908	pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1909	if (!pdev)
1910		return -ENOMEM;
1911
1912	pm_runtime_get_sync(cdns->dev);
1913
1914	cdns->gadget_dev = pdev;
1915	pdev->dev = cdns->dev;
1916	pdev->regs = cdns->dev_regs;
1917	max_speed = usb_get_maximum_speed(cdns->dev);
1918
1919	switch (max_speed) {
1920	case USB_SPEED_FULL:
1921	case USB_SPEED_HIGH:
1922	case USB_SPEED_SUPER:
1923	case USB_SPEED_SUPER_PLUS:
1924		break;
1925	default:
1926		dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1927		fallthrough;
1928	case USB_SPEED_UNKNOWN:
1929		/* Default to SSP */
1930		max_speed = USB_SPEED_SUPER_PLUS;
1931		break;
1932	}
1933
1934	pdev->gadget.ops = &cdnsp_gadget_ops;
1935	pdev->gadget.name = "cdnsp-gadget";
1936	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1937	pdev->gadget.sg_supported = 1;
1938	pdev->gadget.max_speed = max_speed;
1939	pdev->gadget.lpm_capable = 1;
1940
1941	pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1942	if (!pdev->setup_buf)
1943		goto free_pdev;
1944
1945	/*
1946	 * Controller supports not aligned buffer but it should improve
1947	 * performance.
1948	 */
1949	pdev->gadget.quirk_ep_out_aligned_size = true;
1950
1951	ret = cdnsp_gen_setup(pdev);
1952	if (ret) {
1953		dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1954		goto free_setup;
1955	}
1956
1957	ret = cdnsp_gadget_init_endpoints(pdev);
1958	if (ret) {
1959		dev_err(pdev->dev, "failed to initialize endpoints\n");
1960		goto halt_pdev;
1961	}
1962
1963	ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1964	if (ret) {
1965		dev_err(pdev->dev, "failed to register udc\n");
1966		goto free_endpoints;
1967	}
1968
1969	ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1970					cdnsp_irq_handler,
1971					cdnsp_thread_irq_handler, IRQF_SHARED,
1972					dev_name(pdev->dev), pdev);
1973	if (ret)
1974		goto del_gadget;
1975
1976	return 0;
1977
1978del_gadget:
1979	usb_del_gadget(&pdev->gadget);
1980	cdnsp_gadget_free_endpoints(pdev);
1981	usb_put_gadget(&pdev->gadget);
1982	goto halt_pdev;
1983free_endpoints:
1984	cdnsp_gadget_free_endpoints(pdev);
1985halt_pdev:
1986	cdnsp_halt(pdev);
1987	cdnsp_reset(pdev);
1988	cdnsp_mem_cleanup(pdev);
1989free_setup:
1990	kfree(pdev->setup_buf);
1991free_pdev:
1992	kfree(pdev);
1993
1994	return ret;
1995}
1996
1997static void cdnsp_gadget_exit(struct cdns *cdns)
1998{
1999	struct cdnsp_device *pdev = cdns->gadget_dev;
2000
2001	devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
2002	pm_runtime_put_autosuspend(cdns->dev);
2003	usb_del_gadget(&pdev->gadget);
2004	cdnsp_gadget_free_endpoints(pdev);
2005	usb_put_gadget(&pdev->gadget);
2006	cdnsp_mem_cleanup(pdev);
2007	kfree(pdev);
2008	cdns->gadget_dev = NULL;
2009	cdns_drd_gadget_off(cdns);
2010}
2011
2012static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
2013{
2014	struct cdnsp_device *pdev = cdns->gadget_dev;
2015	unsigned long flags;
2016
2017	if (pdev->link_state == XDEV_U3)
2018		return 0;
2019
2020	spin_lock_irqsave(&pdev->lock, flags);
2021	cdnsp_disconnect_gadget(pdev);
2022	cdnsp_stop(pdev);
2023	spin_unlock_irqrestore(&pdev->lock, flags);
2024
2025	return 0;
2026}
2027
2028static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power)
2029{
2030	struct cdnsp_device *pdev = cdns->gadget_dev;
2031	enum usb_device_speed max_speed;
2032	unsigned long flags;
2033	int ret;
2034
2035	if (!pdev->gadget_driver)
2036		return 0;
2037
2038	spin_lock_irqsave(&pdev->lock, flags);
2039	max_speed = pdev->gadget_driver->max_speed;
2040
2041	/* Limit speed if necessary. */
2042	max_speed = min(max_speed, pdev->gadget.max_speed);
2043
2044	ret = cdnsp_run(pdev, max_speed);
2045
2046	if (pdev->link_state == XDEV_U3)
2047		__cdnsp_gadget_wakeup(pdev);
2048
2049	spin_unlock_irqrestore(&pdev->lock, flags);
2050
2051	return ret;
2052}
2053
2054/**
2055 * cdnsp_gadget_init - initialize device structure
2056 * @cdns: cdnsp instance
2057 *
2058 * This function initializes the gadget.
2059 */
2060int cdnsp_gadget_init(struct cdns *cdns)
2061{
2062	struct cdns_role_driver *rdrv;
2063
2064	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2065	if (!rdrv)
2066		return -ENOMEM;
2067
2068	rdrv->start	= __cdnsp_gadget_init;
2069	rdrv->stop	= cdnsp_gadget_exit;
2070	rdrv->suspend	= cdnsp_gadget_suspend;
2071	rdrv->resume	= cdnsp_gadget_resume;
2072	rdrv->state	= CDNS_ROLE_STATE_INACTIVE;
2073	rdrv->name	= "gadget";
2074	cdns->roles[USB_ROLE_DEVICE] = rdrv;
2075
2076	return 0;
2077}