pcie-dw-rockchip.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * PCIe host controller driver for Rockchip SoCs.
  4 *
  5 * Copyright (C) 2021 Rockchip Electronics Co., Ltd.
  6 *		http://www.rock-chips.com
  7 *
  8 * Author: Simon Xue <xxm@rock-chips.com>
  9 */
 10
 11#include <linux/bitfield.h>
 12#include <linux/clk.h>
 13#include <linux/gpio/consumer.h>
 14#include <linux/hw_bitfield.h>
 15#include <linux/irqchip/chained_irq.h>
 16#include <linux/irqdomain.h>
 17#include <linux/mfd/syscon.h>
 18#include <linux/module.h>
 19#include <linux/of.h>
 20#include <linux/of_irq.h>
 21#include <linux/phy/phy.h>
 22#include <linux/platform_device.h>
 23#include <linux/regmap.h>
 24#include <linux/reset.h>
 25
 26#include "../../pci.h"
 27#include "pcie-designware.h"
 28
 29/*
 30 * The upper 16 bits of PCIE_CLIENT_CONFIG are a write
 31 * mask for the lower 16 bits.
 32 */
 33
 34#define to_rockchip_pcie(x) dev_get_drvdata((x)->dev)
 35
 36/* General Control Register */
 37#define PCIE_CLIENT_GENERAL_CON		0x0
 38#define  PCIE_CLIENT_MODE_MASK		GENMASK(7, 4)
 39#define  PCIE_CLIENT_MODE_EP		0x0UL
 40#define  PCIE_CLIENT_MODE_RC		0x4UL
 41#define  PCIE_CLIENT_SET_MODE(x)	FIELD_PREP_WM16(PCIE_CLIENT_MODE_MASK, (x))
 42#define  PCIE_CLIENT_LD_RQ_RST_GRT	FIELD_PREP_WM16(BIT(3), 1)
 43#define  PCIE_CLIENT_ENABLE_LTSSM	FIELD_PREP_WM16(BIT(2), 1)
 44#define  PCIE_CLIENT_DISABLE_LTSSM	FIELD_PREP_WM16(BIT(2), 0)
 45
 46/* Interrupt Status Register Related to Legacy Interrupt */
 47#define PCIE_CLIENT_INTR_STATUS_LEGACY	0x8
 48
 49/* Interrupt Status Register Related to Miscellaneous Operation */
 50#define PCIE_CLIENT_INTR_STATUS_MISC	0x10
 51#define  PCIE_RDLH_LINK_UP_CHGED	BIT(1)
 52#define  PCIE_LINK_REQ_RST_NOT_INT	BIT(2)
 53
 54/* Interrupt Mask Register Related to Legacy Interrupt */
 55#define PCIE_CLIENT_INTR_MASK_LEGACY	0x1c
 56#define  PCIE_INTR_MASK			GENMASK(7, 0)
 57#define  PCIE_INTR_CLAMP(_x)		((BIT((_x)) & PCIE_INTR_MASK))
 58#define  PCIE_INTR_LEGACY_MASK(x)	(PCIE_INTR_CLAMP((x)) | \
 59					 (PCIE_INTR_CLAMP((x)) << 16))
 60#define  PCIE_INTR_LEGACY_UNMASK(x)	(PCIE_INTR_CLAMP((x)) << 16)
 61
 62/* Interrupt Mask Register Related to Miscellaneous Operation */
 63#define PCIE_CLIENT_INTR_MASK_MISC	0x24
 64
 65/* Power Management Control Register */
 66#define PCIE_CLIENT_POWER_CON		0x2c
 67#define  PCIE_CLKREQ_READY		FIELD_PREP_WM16(BIT(0), 1)
 68#define  PCIE_CLKREQ_NOT_READY		FIELD_PREP_WM16(BIT(0), 0)
 69#define  PCIE_CLKREQ_PULL_DOWN		FIELD_PREP_WM16(GENMASK(13, 12), 1)
 70
 71/* RASDES TBA information */
 72#define PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN	0x154
 73#define  PCIE_CLIENT_CDM_RASDES_TBA_L1_1	BIT(4)
 74#define  PCIE_CLIENT_CDM_RASDES_TBA_L1_2	BIT(5)
 75
 76/* Hot Reset Control Register */
 77#define PCIE_CLIENT_HOT_RESET_CTRL	0x180
 78#define  PCIE_LTSSM_APP_DLY2_EN		BIT(1)
 79#define  PCIE_LTSSM_APP_DLY2_DONE	BIT(3)
 80#define  PCIE_LTSSM_ENABLE_ENHANCE	BIT(4)
 81
 82/* LTSSM Status Register */
 83#define PCIE_CLIENT_LTSSM_STATUS	0x300
 84#define  PCIE_LINKUP			0x3
 85#define  PCIE_LINKUP_MASK		GENMASK(17, 16)
 86#define  PCIE_LTSSM_STATUS_MASK		GENMASK(5, 0)
 87
 88#define PCIE_TYPE0_HDR_DBI2_OFFSET      0x100000
 89
 90struct rockchip_pcie {
 91	struct dw_pcie pci;
 92	void __iomem *apb_base;
 93	struct phy *phy;
 94	struct clk_bulk_data *clks;
 95	unsigned int clk_cnt;
 96	struct reset_control *rst;
 97	struct gpio_desc *rst_gpio;
 98	struct irq_domain *irq_domain;
 99	const struct rockchip_pcie_of_data *data;
100	bool supports_clkreq;
101};
102
103struct rockchip_pcie_of_data {
104	enum dw_pcie_device_mode mode;
105	const struct pci_epc_features *epc_features;
106};
107
108static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg)
109{
110	return readl_relaxed(rockchip->apb_base + reg);
111}
112
113static void rockchip_pcie_writel_apb(struct rockchip_pcie *rockchip, u32 val,
114				     u32 reg)
115{
116	writel_relaxed(val, rockchip->apb_base + reg);
117}
118
119static void rockchip_pcie_intx_handler(struct irq_desc *desc)
120{
121	struct irq_chip *chip = irq_desc_get_chip(desc);
122	struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
123	unsigned long reg, hwirq;
124
125	chained_irq_enter(chip, desc);
126
127	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_LEGACY);
128
129	for_each_set_bit(hwirq, &reg, 4)
130		generic_handle_domain_irq(rockchip->irq_domain, hwirq);
131
132	chained_irq_exit(chip, desc);
133}
134
135static void rockchip_intx_mask(struct irq_data *data)
136{
137	rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data),
138				 PCIE_INTR_LEGACY_MASK(data->hwirq),
139				 PCIE_CLIENT_INTR_MASK_LEGACY);
140};
141
142static void rockchip_intx_unmask(struct irq_data *data)
143{
144	rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data),
145				 PCIE_INTR_LEGACY_UNMASK(data->hwirq),
146				 PCIE_CLIENT_INTR_MASK_LEGACY);
147};
148
149static struct irq_chip rockchip_intx_irq_chip = {
150	.name			= "INTx",
151	.irq_mask		= rockchip_intx_mask,
152	.irq_unmask		= rockchip_intx_unmask,
153	.flags			= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
154};
155
156static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
157				  irq_hw_number_t hwirq)
158{
159	irq_set_chip_and_handler(irq, &rockchip_intx_irq_chip, handle_level_irq);
160	irq_set_chip_data(irq, domain->host_data);
161
162	return 0;
163}
164
165static const struct irq_domain_ops intx_domain_ops = {
166	.map = rockchip_pcie_intx_map,
167};
168
169static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
170{
171	struct device *dev = rockchip->pci.dev;
172	struct device_node *intc;
173
174	intc = of_get_child_by_name(dev->of_node, "legacy-interrupt-controller");
175	if (!intc) {
176		dev_err(dev, "missing child interrupt-controller node\n");
177		return -EINVAL;
178	}
179
180	rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX,
181							&intx_domain_ops, rockchip);
182	of_node_put(intc);
183	if (!rockchip->irq_domain) {
184		dev_err(dev, "failed to get a INTx IRQ domain\n");
185		return -EINVAL;
186	}
187
188	return 0;
189}
190
191static u32 rockchip_pcie_get_ltssm_reg(struct rockchip_pcie *rockchip)
192{
193	return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS);
194}
195
196static enum dw_pcie_ltssm rockchip_pcie_get_ltssm(struct dw_pcie *pci)
197{
198	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
199	u32 val = rockchip_pcie_readl_apb(rockchip,
200			PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN);
201
202	if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_1)
203		return DW_PCIE_LTSSM_L1_1;
204
205	if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_2)
206		return DW_PCIE_LTSSM_L1_2;
207
208	return rockchip_pcie_get_ltssm_reg(rockchip) & PCIE_LTSSM_STATUS_MASK;
209}
210
211static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip)
212{
213	rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM,
214				 PCIE_CLIENT_GENERAL_CON);
215}
216
217static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip)
218{
219	rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_DISABLE_LTSSM,
220				 PCIE_CLIENT_GENERAL_CON);
221}
222
223static bool rockchip_pcie_link_up(struct dw_pcie *pci)
224{
225	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
226	u32 val = rockchip_pcie_get_ltssm_reg(rockchip);
227
228	return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP;
229}
230
231/*
232 * See e.g. section '11.6.6.4 L1 Substate' in the RK3588 TRM V1.0 for the steps
233 * needed to support L1 substates. Currently, just enable L1 substates for RC
234 * mode if CLKREQ# is properly connected and supports-clkreq is present in DT.
235 * For EP mode, there are more things should be done to actually save power in
236 * L1 substates, so disable L1 substates until there is proper support.
237 */
238static void rockchip_pcie_configure_l1ss(struct dw_pcie *pci)
239{
240	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
241
242	/* Enable L1 substates if CLKREQ# is properly connected */
243	if (rockchip->supports_clkreq) {
244		rockchip_pcie_writel_apb(rockchip, PCIE_CLKREQ_READY,
245					 PCIE_CLIENT_POWER_CON);
246		pci->l1ss_support = true;
247		return;
248	}
249
250	/*
251	 * Otherwise, assert CLKREQ# unconditionally.  Since
252	 * pci->l1ss_support is not set, the DWC core will prevent L1
253	 * Substates support from being advertised.
254	 */
255	rockchip_pcie_writel_apb(rockchip,
256				 PCIE_CLKREQ_PULL_DOWN | PCIE_CLKREQ_NOT_READY,
257				 PCIE_CLIENT_POWER_CON);
258}
259
260static void rockchip_pcie_enable_l0s(struct dw_pcie *pci)
261{
262	u32 cap, lnkcap;
263
264	/* Enable L0S capability for all SoCs */
265	cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
266	if (cap) {
267		lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP);
268		lnkcap |= PCI_EXP_LNKCAP_ASPM_L0S;
269		dw_pcie_dbi_ro_wr_en(pci);
270		dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap);
271		dw_pcie_dbi_ro_wr_dis(pci);
272	}
273}
274
275static int rockchip_pcie_start_link(struct dw_pcie *pci)
276{
277	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
278
279	/* Reset device */
280	gpiod_set_value_cansleep(rockchip->rst_gpio, 0);
281
282	rockchip_pcie_enable_ltssm(rockchip);
283
284	/*
285	 * PCIe requires the refclk to be stable for 100µs prior to releasing
286	 * PERST. See table 2-4 in section 2.6.2 AC Specifications of the PCI
287	 * Express Card Electromechanical Specification, 1.1. However, we don't
288	 * know if the refclk is coming from RC's PHY or external OSC. If it's
289	 * from RC, so enabling LTSSM is the just right place to release #PERST.
290	 * We need more extra time as before, rather than setting just
291	 * 100us as we don't know how long should the device need to reset.
292	 */
293	msleep(PCIE_T_PVPERL_MS);
294	gpiod_set_value_cansleep(rockchip->rst_gpio, 1);
295
296	return 0;
297}
298
299static void rockchip_pcie_stop_link(struct dw_pcie *pci)
300{
301	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
302
303	rockchip_pcie_disable_ltssm(rockchip);
304}
305
306static int rockchip_pcie_host_init(struct dw_pcie_rp *pp)
307{
308	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
309	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
310	struct device *dev = rockchip->pci.dev;
311	int irq, ret;
312
313	irq = of_irq_get_byname(dev->of_node, "legacy");
314	if (irq < 0)
315		return irq;
316
317	pci->dbi_base2 = pci->dbi_base + PCIE_TYPE0_HDR_DBI2_OFFSET;
318
319	ret = rockchip_pcie_init_irq_domain(rockchip);
320	if (ret < 0)
321		dev_err(dev, "failed to init irq domain\n");
322
323	irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler,
324					 rockchip);
325
326	rockchip_pcie_configure_l1ss(pci);
327	rockchip_pcie_enable_l0s(pci);
328
329	/* Disable Root Ports BAR0 and BAR1 as they report bogus size */
330	dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, 0x0);
331	dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_1, 0x0);
332
333	return 0;
334}
335
336static const struct dw_pcie_host_ops rockchip_pcie_host_ops = {
337	.init = rockchip_pcie_host_init,
338};
339
340/*
341 * ATS does not work on RK3588 when running in EP mode.
342 *
343 * After the host has enabled ATS on the EP side, it will send an IOTLB
344 * invalidation request to the EP side. However, the RK3588 will never send
345 * a completion back and eventually the host will print an IOTLB_INV_TIMEOUT
346 * error, and the EP will not be operational. If we hide the ATS capability,
347 * things work as expected.
348 */
349static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep)
350{
351	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
352	struct device *dev = pci->dev;
353
354	/* Only hide the ATS capability for RK3588 running in EP mode. */
355	if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep"))
356		return;
357
358	dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS);
359}
360
361static void rockchip_pcie_ep_init(struct dw_pcie_ep *ep)
362{
363	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
364	enum pci_barno bar;
365
366	rockchip_pcie_enable_l0s(pci);
367	rockchip_pcie_ep_hide_broken_ats_cap_rk3588(ep);
368
369	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
370		dw_pcie_ep_reset_bar(pci, bar);
371};
372
373static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
374				   unsigned int type, u16 interrupt_num)
375{
376	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
377
378	switch (type) {
379	case PCI_IRQ_INTX:
380		return dw_pcie_ep_raise_intx_irq(ep, func_no);
381	case PCI_IRQ_MSI:
382		return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
383	case PCI_IRQ_MSIX:
384		return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
385	default:
386		dev_err(pci->dev, "UNKNOWN IRQ type\n");
387	}
388
389	return 0;
390}
391
392static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = {
393	DWC_EPC_COMMON_FEATURES,
394	.linkup_notifier = true,
395	.msi_capable = true,
396	.msix_capable = true,
397	.align = SZ_64K,
398	.bar[BAR_0] = { .type = BAR_RESIZABLE, },
399	.bar[BAR_1] = { .type = BAR_RESIZABLE, },
400	.bar[BAR_2] = { .type = BAR_RESIZABLE, },
401	.bar[BAR_3] = { .type = BAR_RESIZABLE, },
402	.bar[BAR_4] = { .type = BAR_RESIZABLE, },
403	.bar[BAR_5] = { .type = BAR_RESIZABLE, },
404};
405
406/*
407 * BAR4 on rk3588 exposes the ATU Port Logic Structure to the host regardless of
408 * iATU settings for BAR4. This means that BAR4 cannot be used by an EPF driver,
409 * so mark it as RESERVED. (rockchip_pcie_ep_init() will disable all BARs by
410 * default.) If the host could write to BAR4, the iATU settings (for all other
411 * BARs) would be overwritten, resulting in (all other BARs) no longer working.
412 */
413static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = {
414	DWC_EPC_COMMON_FEATURES,
415	.linkup_notifier = true,
416	.msi_capable = true,
417	.msix_capable = true,
418	.align = SZ_64K,
419	.bar[BAR_0] = { .type = BAR_RESIZABLE, },
420	.bar[BAR_1] = { .type = BAR_RESIZABLE, },
421	.bar[BAR_2] = { .type = BAR_RESIZABLE, },
422	.bar[BAR_3] = { .type = BAR_RESIZABLE, },
423	.bar[BAR_4] = { .type = BAR_RESERVED, },
424	.bar[BAR_5] = { .type = BAR_RESIZABLE, },
425};
426
427static const struct pci_epc_features *
428rockchip_pcie_get_features(struct dw_pcie_ep *ep)
429{
430	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
431	struct rockchip_pcie *rockchip = to_rockchip_pcie(pci);
432
433	return rockchip->data->epc_features;
434}
435
436static const struct dw_pcie_ep_ops rockchip_pcie_ep_ops = {
437	.init = rockchip_pcie_ep_init,
438	.raise_irq = rockchip_pcie_raise_irq,
439	.get_features = rockchip_pcie_get_features,
440};
441
442static int rockchip_pcie_clk_init(struct rockchip_pcie *rockchip)
443{
444	struct device *dev = rockchip->pci.dev;
445	int ret;
446
447	ret = devm_clk_bulk_get_all(dev, &rockchip->clks);
448	if (ret < 0)
449		return dev_err_probe(dev, ret, "failed to get clocks\n");
450
451	rockchip->clk_cnt = ret;
452
453	ret = clk_bulk_prepare_enable(rockchip->clk_cnt, rockchip->clks);
454	if (ret)
455		return dev_err_probe(dev, ret, "failed to enable clocks\n");
456
457	return 0;
458}
459
460static int rockchip_pcie_resource_get(struct platform_device *pdev,
461				      struct rockchip_pcie *rockchip)
462{
463	rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb");
464	if (IS_ERR(rockchip->apb_base))
465		return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->apb_base),
466				     "failed to map apb registers\n");
467
468	rockchip->rst_gpio = devm_gpiod_get_optional(&pdev->dev, "reset",
469						     GPIOD_OUT_LOW);
470	if (IS_ERR(rockchip->rst_gpio))
471		return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst_gpio),
472				     "failed to get reset gpio\n");
473
474	rockchip->rst = devm_reset_control_array_get_exclusive(&pdev->dev);
475	if (IS_ERR(rockchip->rst))
476		return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst),
477				     "failed to get reset lines\n");
478
479	rockchip->supports_clkreq = of_property_read_bool(pdev->dev.of_node,
480							  "supports-clkreq");
481
482	return 0;
483}
484
485static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip)
486{
487	struct device *dev = rockchip->pci.dev;
488	int ret;
489
490	rockchip->phy = devm_phy_get(dev, "pcie-phy");
491	if (IS_ERR(rockchip->phy))
492		return dev_err_probe(dev, PTR_ERR(rockchip->phy),
493				     "missing PHY\n");
494
495	ret = phy_init(rockchip->phy);
496	if (ret < 0)
497		return ret;
498
499	ret = phy_power_on(rockchip->phy);
500	if (ret)
501		phy_exit(rockchip->phy);
502
503	return ret;
504}
505
506static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip)
507{
508	phy_power_off(rockchip->phy);
509	phy_exit(rockchip->phy);
510}
511
512static const struct dw_pcie_ops dw_pcie_ops = {
513	.link_up = rockchip_pcie_link_up,
514	.start_link = rockchip_pcie_start_link,
515	.stop_link = rockchip_pcie_stop_link,
516	.get_ltssm = rockchip_pcie_get_ltssm,
517};
518
519static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg)
520{
521	struct rockchip_pcie *rockchip = arg;
522	struct dw_pcie *pci = &rockchip->pci;
523	struct device *dev = pci->dev;
524	u32 reg, val;
525
526	reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC);
527	rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC);
528
529	dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg);
530	dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip));
531
532	if (reg & PCIE_LINK_REQ_RST_NOT_INT) {
533		dev_dbg(dev, "hot reset or link-down reset\n");
534		dw_pcie_ep_linkdown(&pci->ep);
535		/* Stop delaying link training. */
536		val = FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_DONE, 1);
537		rockchip_pcie_writel_apb(rockchip, val,
538					 PCIE_CLIENT_HOT_RESET_CTRL);
539	}
540
541	if (reg & PCIE_RDLH_LINK_UP_CHGED) {
542		if (rockchip_pcie_link_up(pci)) {
543			dev_dbg(dev, "link up\n");
544			dw_pcie_ep_linkup(&pci->ep);
545		}
546	}
547
548	return IRQ_HANDLED;
549}
550
551static int rockchip_pcie_configure_rc(struct rockchip_pcie *rockchip)
552{
553	struct dw_pcie_rp *pp;
554	u32 val;
555
556	if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST))
557		return -ENODEV;
558
559	/* LTSSM enable control mode */
560	val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1);
561	rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
562
563	rockchip_pcie_writel_apb(rockchip,
564				 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_RC),
565				 PCIE_CLIENT_GENERAL_CON);
566
567	pp = &rockchip->pci.pp;
568	pp->ops = &rockchip_pcie_host_ops;
569
570	return dw_pcie_host_init(pp);
571}
572
573static int rockchip_pcie_configure_ep(struct platform_device *pdev,
574				      struct rockchip_pcie *rockchip)
575{
576	struct device *dev = &pdev->dev;
577	int irq, ret;
578	u32 val;
579
580	if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_EP))
581		return -ENODEV;
582
583	irq = platform_get_irq_byname(pdev, "sys");
584	if (irq < 0)
585		return irq;
586
587	ret = devm_request_threaded_irq(dev, irq, NULL,
588					rockchip_pcie_ep_sys_irq_thread,
589					IRQF_ONESHOT, "pcie-sys-ep", rockchip);
590	if (ret) {
591		dev_err(dev, "failed to request PCIe sys IRQ\n");
592		return ret;
593	}
594
595	/*
596	 * LTSSM enable control mode, and automatically delay link training on
597	 * hot reset/link-down reset.
598	 */
599	val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1) |
600	      FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_EN, 1);
601	rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL);
602
603	rockchip_pcie_writel_apb(rockchip,
604				 PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_EP),
605				 PCIE_CLIENT_GENERAL_CON);
606
607	rockchip->pci.ep.ops = &rockchip_pcie_ep_ops;
608	rockchip->pci.ep.page_size = SZ_64K;
609
610	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
611
612	ret = dw_pcie_ep_init(&rockchip->pci.ep);
613	if (ret) {
614		dev_err(dev, "failed to initialize endpoint\n");
615		return ret;
616	}
617
618	ret = dw_pcie_ep_init_registers(&rockchip->pci.ep);
619	if (ret) {
620		dev_err(dev, "failed to initialize DWC endpoint registers\n");
621		dw_pcie_ep_deinit(&rockchip->pci.ep);
622		return ret;
623	}
624
625	pci_epc_init_notify(rockchip->pci.ep.epc);
626
627	/* unmask DLL up/down indicator and hot reset/link-down reset */
628	val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0) |
629	      FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0);
630	rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC);
631
632	return ret;
633}
634
635static int rockchip_pcie_probe(struct platform_device *pdev)
636{
637	struct device *dev = &pdev->dev;
638	struct rockchip_pcie *rockchip;
639	const struct rockchip_pcie_of_data *data;
640	int ret;
641
642	data = of_device_get_match_data(dev);
643	if (!data)
644		return -EINVAL;
645
646	rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL);
647	if (!rockchip)
648		return -ENOMEM;
649
650	platform_set_drvdata(pdev, rockchip);
651
652	rockchip->pci.dev = dev;
653	rockchip->pci.ops = &dw_pcie_ops;
654	rockchip->data = data;
655
656	/* Default N_FTS value (210) is broken, override it to 255 */
657	rockchip->pci.n_fts[0] = 255; /* Gen1 */
658	rockchip->pci.n_fts[1] = 255; /* Gen2+ */
659
660	ret = rockchip_pcie_resource_get(pdev, rockchip);
661	if (ret)
662		return ret;
663
664	ret = reset_control_assert(rockchip->rst);
665	if (ret)
666		return ret;
667
668	/* DON'T MOVE ME: must be enable before PHY init */
669	ret = devm_regulator_get_enable_optional(dev, "vpcie3v3");
670	if (ret < 0 && ret != -ENODEV)
671		return dev_err_probe(dev, ret,
672				     "failed to enable vpcie3v3 regulator\n");
673
674	ret = rockchip_pcie_phy_init(rockchip);
675	if (ret)
676		return dev_err_probe(dev, ret,
677				     "failed to initialize the phy\n");
678
679	ret = reset_control_deassert(rockchip->rst);
680	if (ret)
681		goto deinit_phy;
682
683	ret = rockchip_pcie_clk_init(rockchip);
684	if (ret)
685		goto deinit_phy;
686
687	switch (data->mode) {
688	case DW_PCIE_RC_TYPE:
689		ret = rockchip_pcie_configure_rc(rockchip);
690		if (ret)
691			goto deinit_clk;
692		break;
693	case DW_PCIE_EP_TYPE:
694		ret = rockchip_pcie_configure_ep(pdev, rockchip);
695		if (ret)
696			goto deinit_clk;
697		break;
698	default:
699		dev_err(dev, "INVALID device type %d\n", data->mode);
700		ret = -EINVAL;
701		goto deinit_clk;
702	}
703
704	return 0;
705
706deinit_clk:
707	clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks);
708deinit_phy:
709	rockchip_pcie_phy_deinit(rockchip);
710
711	return ret;
712}
713
714static const struct rockchip_pcie_of_data rockchip_pcie_rc_of_data_rk3568 = {
715	.mode = DW_PCIE_RC_TYPE,
716};
717
718static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3568 = {
719	.mode = DW_PCIE_EP_TYPE,
720	.epc_features = &rockchip_pcie_epc_features_rk3568,
721};
722
723static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3588 = {
724	.mode = DW_PCIE_EP_TYPE,
725	.epc_features = &rockchip_pcie_epc_features_rk3588,
726};
727
728static const struct of_device_id rockchip_pcie_of_match[] = {
729	{
730		.compatible = "rockchip,rk3568-pcie",
731		.data = &rockchip_pcie_rc_of_data_rk3568,
732	},
733	{
734		.compatible = "rockchip,rk3568-pcie-ep",
735		.data = &rockchip_pcie_ep_of_data_rk3568,
736	},
737	{
738		.compatible = "rockchip,rk3588-pcie-ep",
739		.data = &rockchip_pcie_ep_of_data_rk3588,
740	},
741	{},
742};
743
744static struct platform_driver rockchip_pcie_driver = {
745	.driver = {
746		.name	= "rockchip-dw-pcie",
747		.of_match_table = rockchip_pcie_of_match,
748		.suppress_bind_attrs = true,
749	},
750	.probe = rockchip_pcie_probe,
751};
752builtin_platform_driver(rockchip_pcie_driver);