File Index
Files and directories
Sign in
FILE

pci-imx6.c

File preview with syntax highlighting.

linux-6.19.14/drivers/pci/controller/dwc/pci-imx6.c C 
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * PCIe host controller driver for Freescale i.MX6 SoCs
   4 *
   5 * Copyright (C) 2013 Kosagi
   6 *		https://www.kosagi.com
   7 *
   8 * Author: Sean Cross <xobs@kosagi.com>
   9 */
  10
  11#include <linux/bitfield.h>
  12#include <linux/clk.h>
  13#include <linux/delay.h>
  14#include <linux/gpio/consumer.h>
  15#include <linux/kernel.h>
  16#include <linux/mfd/syscon.h>
  17#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
  18#include <linux/mfd/syscon/imx7-iomuxc-gpr.h>
  19#include <linux/module.h>
  20#include <linux/of.h>
  21#include <linux/of_address.h>
  22#include <linux/pci.h>
  23#include <linux/platform_device.h>
  24#include <linux/regmap.h>
  25#include <linux/regulator/consumer.h>
  26#include <linux/resource.h>
  27#include <linux/signal.h>
  28#include <linux/types.h>
  29#include <linux/interrupt.h>
  30#include <linux/reset.h>
  31#include <linux/phy/pcie.h>
  32#include <linux/phy/phy.h>
  33#include <linux/pm_domain.h>
  34#include <linux/pm_runtime.h>
  35
  36#include "../../pci.h"
  37#include "pcie-designware.h"
  38
  39#define IMX8MQ_GPR_PCIE_REF_USE_PAD		BIT(9)
  40#define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN	BIT(10)
  41#define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE	BIT(11)
  42#define IMX8MQ_GPR_PCIE_VREG_BYPASS		BIT(12)
  43#define IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE	GENMASK(11, 8)
  44
  45#define IMX95_PCIE_PHY_GEN_CTRL			0x0
  46#define IMX95_PCIE_REF_USE_PAD			BIT(17)
  47
  48#define IMX95_PCIE_PHY_MPLLA_CTRL		0x10
  49#define IMX95_PCIE_PHY_MPLL_STATE		BIT(30)
  50
  51#define IMX95_PCIE_SS_RW_REG_0			0xf0
  52#define IMX95_PCIE_REF_CLKEN			BIT(23)
  53#define IMX95_PCIE_PHY_CR_PARA_SEL		BIT(9)
  54#define IMX95_PCIE_SS_RW_REG_1			0xf4
  55#define IMX95_PCIE_CLKREQ_OVERRIDE_EN		BIT(8)
  56#define IMX95_PCIE_CLKREQ_OVERRIDE_VAL		BIT(9)
  57#define IMX95_PCIE_SYS_AUX_PWR_DET		BIT(31)
  58
  59#define IMX95_PE0_GEN_CTRL_1			0x1050
  60#define IMX95_PCIE_DEVICE_TYPE			GENMASK(3, 0)
  61
  62#define IMX95_PE0_GEN_CTRL_3			0x1058
  63#define IMX95_PCIE_LTSSM_EN			BIT(0)
  64
  65#define IMX95_PE0_LUT_ACSCTRL			0x1008
  66#define IMX95_PEO_LUT_RWA			BIT(16)
  67#define IMX95_PE0_LUT_ENLOC			GENMASK(4, 0)
  68
  69#define IMX95_PE0_LUT_DATA1			0x100c
  70#define IMX95_PE0_LUT_VLD			BIT(31)
  71#define IMX95_PE0_LUT_DAC_ID			GENMASK(10, 8)
  72#define IMX95_PE0_LUT_STREAM_ID			GENMASK(5, 0)
  73
  74#define IMX95_PE0_LUT_DATA2			0x1010
  75#define IMX95_PE0_LUT_REQID			GENMASK(31, 16)
  76#define IMX95_PE0_LUT_MASK			GENMASK(15, 0)
  77
  78#define IMX95_SID_MASK				GENMASK(5, 0)
  79#define IMX95_MAX_LUT				32
  80
  81#define IMX95_PCIE_RST_CTRL			0x3010
  82#define IMX95_PCIE_COLD_RST			BIT(0)
  83
  84#define to_imx_pcie(x)	dev_get_drvdata((x)->dev)
  85
  86enum imx_pcie_variants {
  87	IMX6Q,
  88	IMX6SX,
  89	IMX6QP,
  90	IMX7D,
  91	IMX8MQ,
  92	IMX8MM,
  93	IMX8MP,
  94	IMX8Q,
  95	IMX95,
  96	IMX8MQ_EP,
  97	IMX8MM_EP,
  98	IMX8MP_EP,
  99	IMX8Q_EP,
 100	IMX95_EP,
 101};
 102
 103#define IMX_PCIE_FLAG_IMX_PHY			BIT(0)
 104#define IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND	BIT(1)
 105#define IMX_PCIE_FLAG_SUPPORTS_SUSPEND		BIT(2)
 106#define IMX_PCIE_FLAG_HAS_PHYDRV		BIT(3)
 107#define IMX_PCIE_FLAG_HAS_APP_RESET		BIT(4)
 108#define IMX_PCIE_FLAG_HAS_PHY_RESET		BIT(5)
 109#define IMX_PCIE_FLAG_HAS_SERDES		BIT(6)
 110#define IMX_PCIE_FLAG_SUPPORT_64BIT		BIT(7)
 111#define IMX_PCIE_FLAG_CPU_ADDR_FIXUP		BIT(8)
 112/*
 113 * Because of ERR005723 (PCIe does not support L2 power down) we need to
 114 * workaround suspend resume on some devices which are affected by this errata.
 115 */
 116#define IMX_PCIE_FLAG_BROKEN_SUSPEND		BIT(9)
 117#define IMX_PCIE_FLAG_HAS_LUT			BIT(10)
 118#define IMX_PCIE_FLAG_8GT_ECN_ERR051586		BIT(11)
 119#define IMX_PCIE_FLAG_SKIP_L23_READY		BIT(12)
 120
 121#define imx_check_flag(pci, val)	(pci->drvdata->flags & val)
 122
 123#define IMX_PCIE_MAX_INSTANCES	2
 124
 125struct imx_pcie;
 126
 127struct imx_pcie_drvdata {
 128	enum imx_pcie_variants variant;
 129	enum dw_pcie_device_mode mode;
 130	u32 flags;
 131	int dbi_length;
 132	const char *gpr;
 133	const u32 ltssm_off;
 134	const u32 ltssm_mask;
 135	const u32 mode_off[IMX_PCIE_MAX_INSTANCES];
 136	const u32 mode_mask[IMX_PCIE_MAX_INSTANCES];
 137	const struct pci_epc_features *epc_features;
 138	int (*init_phy)(struct imx_pcie *pcie);
 139	int (*enable_ref_clk)(struct imx_pcie *pcie, bool enable);
 140	int (*core_reset)(struct imx_pcie *pcie, bool assert);
 141	int (*wait_pll_lock)(struct imx_pcie *pcie);
 142	const struct dw_pcie_host_ops *ops;
 143};
 144
 145struct imx_lut_data {
 146	u32 data1;
 147	u32 data2;
 148};
 149
 150struct imx_pcie {
 151	struct dw_pcie		*pci;
 152	struct gpio_desc	*reset_gpiod;
 153	struct clk_bulk_data	*clks;
 154	int			num_clks;
 155	struct regmap		*iomuxc_gpr;
 156	u16			msi_ctrl;
 157	u32			controller_id;
 158	struct reset_control	*pciephy_reset;
 159	struct reset_control	*apps_reset;
 160	u32			tx_deemph_gen1;
 161	u32			tx_deemph_gen2_3p5db;
 162	u32			tx_deemph_gen2_6db;
 163	u32			tx_swing_full;
 164	u32			tx_swing_low;
 165	struct regulator	*vpcie;
 166	struct regulator	*vph;
 167	void __iomem		*phy_base;
 168
 169	/* LUT data for pcie */
 170	struct imx_lut_data	luts[IMX95_MAX_LUT];
 171	/* power domain for pcie */
 172	struct device		*pd_pcie;
 173	/* power domain for pcie phy */
 174	struct device		*pd_pcie_phy;
 175	struct phy		*phy;
 176	const struct imx_pcie_drvdata *drvdata;
 177
 178	/* Ensure that only one device's LUT is configured at any given time */
 179	struct mutex		lock;
 180};
 181
 182/* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */
 183#define PHY_PLL_LOCK_WAIT_USLEEP_MAX	200
 184#define PHY_PLL_LOCK_WAIT_TIMEOUT	(2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX)
 185
 186/* PCIe Port Logic registers (memory-mapped) */
 187#define PL_OFFSET 0x700
 188
 189#define PCIE_PHY_CTRL (PL_OFFSET + 0x114)
 190#define PCIE_PHY_CTRL_DATA(x)		FIELD_PREP(GENMASK(15, 0), (x))
 191#define PCIE_PHY_CTRL_CAP_ADR		BIT(16)
 192#define PCIE_PHY_CTRL_CAP_DAT		BIT(17)
 193#define PCIE_PHY_CTRL_WR		BIT(18)
 194#define PCIE_PHY_CTRL_RD		BIT(19)
 195
 196#define PCIE_PHY_STAT (PL_OFFSET + 0x110)
 197#define PCIE_PHY_STAT_ACK		BIT(16)
 198
 199/* PHY registers (not memory-mapped) */
 200#define PCIE_PHY_ATEOVRD			0x10
 201#define  PCIE_PHY_ATEOVRD_EN			BIT(2)
 202#define  PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT	0
 203#define  PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK	0x1
 204
 205#define PCIE_PHY_MPLL_OVRD_IN_LO		0x11
 206#define  PCIE_PHY_MPLL_MULTIPLIER_SHIFT		2
 207#define  PCIE_PHY_MPLL_MULTIPLIER_MASK		0x7f
 208#define  PCIE_PHY_MPLL_MULTIPLIER_OVRD		BIT(9)
 209
 210#define PCIE_PHY_RX_ASIC_OUT 0x100D
 211#define PCIE_PHY_RX_ASIC_OUT_VALID	(1 << 0)
 212
 213/* iMX7 PCIe PHY registers */
 214#define PCIE_PHY_CMN_REG4		0x14
 215/* These are probably the bits that *aren't* DCC_FB_EN */
 216#define PCIE_PHY_CMN_REG4_DCC_FB_EN	0x29
 217
 218#define PCIE_PHY_CMN_REG15	        0x54
 219#define PCIE_PHY_CMN_REG15_DLY_4	BIT(2)
 220#define PCIE_PHY_CMN_REG15_PLL_PD	BIT(5)
 221#define PCIE_PHY_CMN_REG15_OVRD_PLL_PD	BIT(7)
 222
 223#define PCIE_PHY_CMN_REG24		0x90
 224#define PCIE_PHY_CMN_REG24_RX_EQ	BIT(6)
 225#define PCIE_PHY_CMN_REG24_RX_EQ_SEL	BIT(3)
 226
 227#define PCIE_PHY_CMN_REG26		0x98
 228#define PCIE_PHY_CMN_REG26_ATT_MODE	0xBC
 229
 230#define PHY_RX_OVRD_IN_LO 0x1005
 231#define PHY_RX_OVRD_IN_LO_RX_DATA_EN		BIT(5)
 232#define PHY_RX_OVRD_IN_LO_RX_PLL_EN		BIT(3)
 233
 234static unsigned int imx_pcie_grp_offset(const struct imx_pcie *imx_pcie)
 235{
 236	WARN_ON(imx_pcie->drvdata->variant != IMX8MQ &&
 237		imx_pcie->drvdata->variant != IMX8MQ_EP &&
 238		imx_pcie->drvdata->variant != IMX8MM &&
 239		imx_pcie->drvdata->variant != IMX8MM_EP &&
 240		imx_pcie->drvdata->variant != IMX8MP &&
 241		imx_pcie->drvdata->variant != IMX8MP_EP);
 242	return imx_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;
 243}
 244
 245static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie)
 246{
 247	/*
 248	 * ERR051624: The Controller Without Vaux Cannot Exit L23 Ready
 249	 * Through Beacon or PERST# De-assertion
 250	 *
 251	 * When the auxiliary power is not available, the controller
 252	 * cannot exit from L23 Ready with beacon or PERST# de-assertion
 253	 * when main power is not removed.
 254	 *
 255	 * Workaround: Set SS_RW_REG_1[SYS_AUX_PWR_DET] to 1.
 256	 */
 257	regmap_set_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1,
 258			IMX95_PCIE_SYS_AUX_PWR_DET);
 259
 260	regmap_update_bits(imx_pcie->iomuxc_gpr,
 261			IMX95_PCIE_SS_RW_REG_0,
 262			IMX95_PCIE_PHY_CR_PARA_SEL,
 263			IMX95_PCIE_PHY_CR_PARA_SEL);
 264
 265	regmap_update_bits(imx_pcie->iomuxc_gpr,
 266			   IMX95_PCIE_PHY_GEN_CTRL,
 267			   IMX95_PCIE_REF_USE_PAD, 0);
 268	regmap_update_bits(imx_pcie->iomuxc_gpr,
 269			   IMX95_PCIE_SS_RW_REG_0,
 270			   IMX95_PCIE_REF_CLKEN,
 271			   IMX95_PCIE_REF_CLKEN);
 272
 273	return 0;
 274}
 275
 276static void imx_pcie_configure_type(struct imx_pcie *imx_pcie)
 277{
 278	const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata;
 279	unsigned int mask, val, mode, id;
 280
 281	if (drvdata->mode == DW_PCIE_EP_TYPE)
 282		mode = PCI_EXP_TYPE_ENDPOINT;
 283	else
 284		mode = PCI_EXP_TYPE_ROOT_PORT;
 285
 286	id = imx_pcie->controller_id;
 287
 288	/* If mode_mask is 0, generic PHY driver is used to set the mode */
 289	if (!drvdata->mode_mask[0])
 290		return;
 291
 292	/* If mode_mask[id] is 0, each controller has its individual GPR */
 293	if (!drvdata->mode_mask[id])
 294		id = 0;
 295
 296	mask = drvdata->mode_mask[id];
 297	val = mode << (ffs(mask) - 1);
 298
 299	regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->mode_off[id], mask, val);
 300}
 301
 302static int pcie_phy_poll_ack(struct imx_pcie *imx_pcie, bool exp_val)
 303{
 304	struct dw_pcie *pci = imx_pcie->pci;
 305	bool val;
 306	u32 max_iterations = 10;
 307	u32 wait_counter = 0;
 308
 309	do {
 310		val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT) &
 311			PCIE_PHY_STAT_ACK;
 312		wait_counter++;
 313
 314		if (val == exp_val)
 315			return 0;
 316
 317		udelay(1);
 318	} while (wait_counter < max_iterations);
 319
 320	return -ETIMEDOUT;
 321}
 322
 323static int pcie_phy_wait_ack(struct imx_pcie *imx_pcie, int addr)
 324{
 325	struct dw_pcie *pci = imx_pcie->pci;
 326	u32 val;
 327	int ret;
 328
 329	val = PCIE_PHY_CTRL_DATA(addr);
 330	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
 331
 332	val |= PCIE_PHY_CTRL_CAP_ADR;
 333	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
 334
 335	ret = pcie_phy_poll_ack(imx_pcie, true);
 336	if (ret)
 337		return ret;
 338
 339	val = PCIE_PHY_CTRL_DATA(addr);
 340	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val);
 341
 342	return pcie_phy_poll_ack(imx_pcie, false);
 343}
 344
 345/* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */
 346static int pcie_phy_read(struct imx_pcie *imx_pcie, int addr, u16 *data)
 347{
 348	struct dw_pcie *pci = imx_pcie->pci;
 349	u32 phy_ctl;
 350	int ret;
 351
 352	ret = pcie_phy_wait_ack(imx_pcie, addr);
 353	if (ret)
 354		return ret;
 355
 356	/* assert Read signal */
 357	phy_ctl = PCIE_PHY_CTRL_RD;
 358	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl);
 359
 360	ret = pcie_phy_poll_ack(imx_pcie, true);
 361	if (ret)
 362		return ret;
 363
 364	*data = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT);
 365
 366	/* deassert Read signal */
 367	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00);
 368
 369	return pcie_phy_poll_ack(imx_pcie, false);
 370}
 371
 372static int pcie_phy_write(struct imx_pcie *imx_pcie, int addr, u16 data)
 373{
 374	struct dw_pcie *pci = imx_pcie->pci;
 375	u32 var;
 376	int ret;
 377
 378	/* write addr */
 379	/* cap addr */
 380	ret = pcie_phy_wait_ack(imx_pcie, addr);
 381	if (ret)
 382		return ret;
 383
 384	var = PCIE_PHY_CTRL_DATA(data);
 385	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 386
 387	/* capture data */
 388	var |= PCIE_PHY_CTRL_CAP_DAT;
 389	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 390
 391	ret = pcie_phy_poll_ack(imx_pcie, true);
 392	if (ret)
 393		return ret;
 394
 395	/* deassert cap data */
 396	var = PCIE_PHY_CTRL_DATA(data);
 397	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 398
 399	/* wait for ack de-assertion */
 400	ret = pcie_phy_poll_ack(imx_pcie, false);
 401	if (ret)
 402		return ret;
 403
 404	/* assert wr signal */
 405	var = PCIE_PHY_CTRL_WR;
 406	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 407
 408	/* wait for ack */
 409	ret = pcie_phy_poll_ack(imx_pcie, true);
 410	if (ret)
 411		return ret;
 412
 413	/* deassert wr signal */
 414	var = PCIE_PHY_CTRL_DATA(data);
 415	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var);
 416
 417	/* wait for ack de-assertion */
 418	ret = pcie_phy_poll_ack(imx_pcie, false);
 419	if (ret)
 420		return ret;
 421
 422	dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x0);
 423
 424	return 0;
 425}
 426
 427static int imx8mq_pcie_init_phy(struct imx_pcie *imx_pcie)
 428{
 429	/* TODO: This code assumes external oscillator is being used */
 430	regmap_update_bits(imx_pcie->iomuxc_gpr,
 431			   imx_pcie_grp_offset(imx_pcie),
 432			   IMX8MQ_GPR_PCIE_REF_USE_PAD,
 433			   IMX8MQ_GPR_PCIE_REF_USE_PAD);
 434	/*
 435	 * Per the datasheet, the PCIE_VPH is suggested to be 1.8V.  If the
 436	 * PCIE_VPH is supplied by 3.3V, the VREG_BYPASS should be cleared
 437	 * to zero.
 438	 */
 439	if (imx_pcie->vph && regulator_get_voltage(imx_pcie->vph) > 3000000)
 440		regmap_update_bits(imx_pcie->iomuxc_gpr,
 441				   imx_pcie_grp_offset(imx_pcie),
 442				   IMX8MQ_GPR_PCIE_VREG_BYPASS,
 443				   0);
 444
 445	return 0;
 446}
 447
 448static int imx_pcie_init_phy(struct imx_pcie *imx_pcie)
 449{
 450	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 451				   IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);
 452
 453	/* configure constant input signal to the pcie ctrl and phy */
 454	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 455			   IMX6Q_GPR12_LOS_LEVEL, 9 << 4);
 456
 457	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
 458			   IMX6Q_GPR8_TX_DEEMPH_GEN1,
 459			   imx_pcie->tx_deemph_gen1 << 0);
 460	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
 461			   IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,
 462			   imx_pcie->tx_deemph_gen2_3p5db << 6);
 463	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
 464			   IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,
 465			   imx_pcie->tx_deemph_gen2_6db << 12);
 466	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
 467			   IMX6Q_GPR8_TX_SWING_FULL,
 468			   imx_pcie->tx_swing_full << 18);
 469	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8,
 470			   IMX6Q_GPR8_TX_SWING_LOW,
 471			   imx_pcie->tx_swing_low << 25);
 472	return 0;
 473}
 474
 475static int imx6sx_pcie_init_phy(struct imx_pcie *imx_pcie)
 476{
 477	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 478			   IMX6SX_GPR12_PCIE_RX_EQ_MASK, IMX6SX_GPR12_PCIE_RX_EQ_2);
 479
 480	return imx_pcie_init_phy(imx_pcie);
 481}
 482
 483static void imx7d_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie)
 484{
 485	u32 val;
 486	struct device *dev = imx_pcie->pci->dev;
 487
 488	if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr,
 489				     IOMUXC_GPR22, val,
 490				     val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED,
 491				     PHY_PLL_LOCK_WAIT_USLEEP_MAX,
 492				     PHY_PLL_LOCK_WAIT_TIMEOUT))
 493		dev_err(dev, "PCIe PLL lock timeout\n");
 494}
 495
 496static int imx95_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie)
 497{
 498	u32 val;
 499	struct device *dev = imx_pcie->pci->dev;
 500
 501	if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr,
 502				     IMX95_PCIE_PHY_MPLLA_CTRL, val,
 503				     val & IMX95_PCIE_PHY_MPLL_STATE,
 504				     PHY_PLL_LOCK_WAIT_USLEEP_MAX,
 505				     PHY_PLL_LOCK_WAIT_TIMEOUT)) {
 506		dev_err(dev, "PCIe PLL lock timeout\n");
 507		return -ETIMEDOUT;
 508	}
 509
 510	return 0;
 511}
 512
 513static int imx_setup_phy_mpll(struct imx_pcie *imx_pcie)
 514{
 515	unsigned long phy_rate = 0;
 516	int mult, div;
 517	u16 val;
 518	int i;
 519	struct clk_bulk_data *clks = imx_pcie->clks;
 520
 521	if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY))
 522		return 0;
 523
 524	for (i = 0; i < imx_pcie->num_clks; i++)
 525		if (strncmp(clks[i].id, "pcie_phy", 8) == 0)
 526			phy_rate = clk_get_rate(clks[i].clk);
 527
 528	switch (phy_rate) {
 529	case 125000000:
 530		/*
 531		 * The default settings of the MPLL are for a 125MHz input
 532		 * clock, so no need to reconfigure anything in that case.
 533		 */
 534		return 0;
 535	case 100000000:
 536		mult = 25;
 537		div = 0;
 538		break;
 539	case 200000000:
 540		mult = 25;
 541		div = 1;
 542		break;
 543	default:
 544		dev_err(imx_pcie->pci->dev,
 545			"Unsupported PHY reference clock rate %lu\n", phy_rate);
 546		return -EINVAL;
 547	}
 548
 549	pcie_phy_read(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val);
 550	val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK <<
 551		 PCIE_PHY_MPLL_MULTIPLIER_SHIFT);
 552	val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT;
 553	val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD;
 554	pcie_phy_write(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val);
 555
 556	pcie_phy_read(imx_pcie, PCIE_PHY_ATEOVRD, &val);
 557	val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK <<
 558		 PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT);
 559	val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT;
 560	val |= PCIE_PHY_ATEOVRD_EN;
 561	pcie_phy_write(imx_pcie, PCIE_PHY_ATEOVRD, val);
 562
 563	return 0;
 564}
 565
 566static void imx_pcie_reset_phy(struct imx_pcie *imx_pcie)
 567{
 568	u16 tmp;
 569
 570	if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY))
 571		return;
 572
 573	pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp);
 574	tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN |
 575		PHY_RX_OVRD_IN_LO_RX_PLL_EN);
 576	pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp);
 577
 578	usleep_range(2000, 3000);
 579
 580	pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp);
 581	tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN |
 582		  PHY_RX_OVRD_IN_LO_RX_PLL_EN);
 583	pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp);
 584}
 585
 586#ifdef CONFIG_ARM
 587/*  Added for PCI abort handling */
 588static int imx6q_pcie_abort_handler(unsigned long addr,
 589		unsigned int fsr, struct pt_regs *regs)
 590{
 591	unsigned long pc = instruction_pointer(regs);
 592	unsigned long instr = *(unsigned long *)pc;
 593	int reg = (instr >> 12) & 15;
 594
 595	/*
 596	 * If the instruction being executed was a read,
 597	 * make it look like it read all-ones.
 598	 */
 599	if ((instr & 0x0c100000) == 0x04100000) {
 600		unsigned long val;
 601
 602		if (instr & 0x00400000)
 603			val = 255;
 604		else
 605			val = -1;
 606
 607		regs->uregs[reg] = val;
 608		regs->ARM_pc += 4;
 609		return 0;
 610	}
 611
 612	if ((instr & 0x0e100090) == 0x00100090) {
 613		regs->uregs[reg] = -1;
 614		regs->ARM_pc += 4;
 615		return 0;
 616	}
 617
 618	return 1;
 619}
 620#endif
 621
 622static int imx_pcie_attach_pd(struct device *dev)
 623{
 624	struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
 625	struct device_link *link;
 626
 627	/* Do nothing when in a single power domain */
 628	if (dev->pm_domain)
 629		return 0;
 630
 631	imx_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie");
 632	if (IS_ERR(imx_pcie->pd_pcie))
 633		return PTR_ERR(imx_pcie->pd_pcie);
 634	/* Do nothing when power domain missing */
 635	if (!imx_pcie->pd_pcie)
 636		return 0;
 637	link = device_link_add(dev, imx_pcie->pd_pcie,
 638			DL_FLAG_STATELESS |
 639			DL_FLAG_PM_RUNTIME |
 640			DL_FLAG_RPM_ACTIVE);
 641	if (!link) {
 642		dev_err(dev, "Failed to add device_link to pcie pd\n");
 643		return -EINVAL;
 644	}
 645
 646	imx_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy");
 647	if (IS_ERR(imx_pcie->pd_pcie_phy))
 648		return PTR_ERR(imx_pcie->pd_pcie_phy);
 649
 650	link = device_link_add(dev, imx_pcie->pd_pcie_phy,
 651			DL_FLAG_STATELESS |
 652			DL_FLAG_PM_RUNTIME |
 653			DL_FLAG_RPM_ACTIVE);
 654	if (!link) {
 655		dev_err(dev, "Failed to add device_link to pcie_phy pd\n");
 656		return -EINVAL;
 657	}
 658
 659	return 0;
 660}
 661
 662static int imx6sx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
 663{
 664	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 665			   IMX6SX_GPR12_PCIE_TEST_POWERDOWN,
 666			   enable ? 0 : IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
 667	return 0;
 668}
 669
 670static int imx6q_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
 671{
 672	if (enable) {
 673		/* power up core phy and enable ref clock */
 674		regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD);
 675		/*
 676		 * The async reset input need ref clock to sync internally,
 677		 * when the ref clock comes after reset, internal synced
 678		 * reset time is too short, cannot meet the requirement.
 679		 * Add a ~10us delay here.
 680		 */
 681		usleep_range(10, 100);
 682		regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN);
 683	} else {
 684		regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN);
 685		regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD);
 686	}
 687
 688	return 0;
 689}
 690
 691static int imx8mm_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
 692{
 693	int offset = imx_pcie_grp_offset(imx_pcie);
 694
 695	regmap_update_bits(imx_pcie->iomuxc_gpr, offset,
 696			   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE,
 697			   enable ? 0 : IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE);
 698	regmap_update_bits(imx_pcie->iomuxc_gpr, offset,
 699			   IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN,
 700			   enable ? IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN : 0);
 701	return 0;
 702}
 703
 704static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
 705{
 706	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 707			   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL,
 708			   enable ? 0 : IMX7D_GPR12_PCIE_PHY_REFCLK_SEL);
 709	return 0;
 710}
 711
 712static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable)
 713{
 714	regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1,
 715			   IMX95_PCIE_CLKREQ_OVERRIDE_EN,
 716			   enable ? IMX95_PCIE_CLKREQ_OVERRIDE_EN : 0);
 717	regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1,
 718			   IMX95_PCIE_CLKREQ_OVERRIDE_VAL,
 719			   enable ? IMX95_PCIE_CLKREQ_OVERRIDE_VAL : 0);
 720}
 721
 722static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
 723{
 724	imx95_pcie_clkreq_override(imx_pcie, enable);
 725	return 0;
 726}
 727
 728static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie)
 729{
 730	struct dw_pcie *pci = imx_pcie->pci;
 731	struct device *dev = pci->dev;
 732	int ret;
 733
 734	ret = clk_bulk_prepare_enable(imx_pcie->num_clks, imx_pcie->clks);
 735	if (ret)
 736		return ret;
 737
 738	if (imx_pcie->drvdata->enable_ref_clk) {
 739		ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true);
 740		if (ret) {
 741			dev_err(dev, "Failed to enable PCIe REFCLK\n");
 742			goto err_ref_clk;
 743		}
 744	}
 745
 746	/* allow the clocks to stabilize */
 747	usleep_range(200, 500);
 748	return 0;
 749
 750err_ref_clk:
 751	clk_bulk_disable_unprepare(imx_pcie->num_clks, imx_pcie->clks);
 752
 753	return ret;
 754}
 755
 756static void imx_pcie_clk_disable(struct imx_pcie *imx_pcie)
 757{
 758	if (imx_pcie->drvdata->enable_ref_clk)
 759		imx_pcie->drvdata->enable_ref_clk(imx_pcie, false);
 760	clk_bulk_disable_unprepare(imx_pcie->num_clks, imx_pcie->clks);
 761}
 762
 763static int imx6sx_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
 764{
 765	if (assert)
 766		regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 767				IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
 768
 769	/* Force PCIe PHY reset */
 770	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET,
 771			   assert ? IMX6SX_GPR5_PCIE_BTNRST_RESET : 0);
 772	return 0;
 773}
 774
 775static int imx6qp_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
 776{
 777	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_SW_RST,
 778			   assert ? IMX6Q_GPR1_PCIE_SW_RST : 0);
 779	if (!assert)
 780		usleep_range(200, 500);
 781
 782	return 0;
 783}
 784
 785static int imx6q_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
 786{
 787	if (!assert)
 788		return 0;
 789
 790	regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_TEST_PD);
 791	regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_REF_CLK_EN);
 792
 793	return 0;
 794}
 795
 796static int imx7d_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
 797{
 798	struct dw_pcie *pci = imx_pcie->pci;
 799	struct device *dev = pci->dev;
 800
 801	if (assert)
 802		return 0;
 803
 804	/*
 805	 * Workaround for ERR010728 (IMX7DS_2N09P, Rev. 1.1, 4/2023):
 806	 *
 807	 * PCIe: PLL may fail to lock under corner conditions.
 808	 *
 809	 * Initial VCO oscillation may fail under corner conditions such as
 810	 * cold temperature which will cause the PCIe PLL fail to lock in the
 811	 * initialization phase.
 812	 *
 813	 * The Duty-cycle Corrector calibration must be disabled.
 814	 *
 815	 * 1. De-assert the G_RST signal by clearing
 816	 *    SRC_PCIEPHY_RCR[PCIEPHY_G_RST].
 817	 * 2. De-assert DCC_FB_EN by writing data “0x29” to the register
 818	 *    address 0x306d0014 (PCIE_PHY_CMN_REG4).
 819	 * 3. Assert RX_EQS, RX_EQ_SEL by writing data “0x48” to the register
 820	 *    address 0x306d0090 (PCIE_PHY_CMN_REG24).
 821	 * 4. Assert ATT_MODE by writing data “0xbc” to the register
 822	 *    address 0x306d0098 (PCIE_PHY_CMN_REG26).
 823	 * 5. De-assert the CMN_RST signal by clearing register bit
 824	 *    SRC_PCIEPHY_RCR[PCIEPHY_BTN]
 825	 */
 826
 827	if (likely(imx_pcie->phy_base)) {
 828		/* De-assert DCC_FB_EN */
 829		writel(PCIE_PHY_CMN_REG4_DCC_FB_EN, imx_pcie->phy_base + PCIE_PHY_CMN_REG4);
 830		/* Assert RX_EQS and RX_EQS_SEL */
 831		writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL | PCIE_PHY_CMN_REG24_RX_EQ,
 832		       imx_pcie->phy_base + PCIE_PHY_CMN_REG24);
 833		/* Assert ATT_MODE */
 834		writel(PCIE_PHY_CMN_REG26_ATT_MODE, imx_pcie->phy_base + PCIE_PHY_CMN_REG26);
 835	} else {
 836		dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n");
 837	}
 838	imx7d_pcie_wait_for_phy_pll_lock(imx_pcie);
 839	return 0;
 840}
 841
 842static int imx95_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
 843{
 844	u32 val;
 845
 846	if (assert) {
 847		/*
 848		 * From i.MX95 PCIe PHY perspective, the COLD reset toggle
 849		 * should be complete after power-up by the following sequence.
 850		 *                 > 10us(at power-up)
 851		 *                 > 10ns(warm reset)
 852		 *               |<------------>|
 853		 *                ______________
 854		 * phy_reset ____/              \________________
 855		 *                                   ____________
 856		 * ref_clk_en_______________________/
 857		 * Toggle COLD reset aligned with this sequence for i.MX95 PCIe.
 858		 */
 859		regmap_set_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
 860				IMX95_PCIE_COLD_RST);
 861		/*
 862		 * Make sure the write to IMX95_PCIE_RST_CTRL is flushed to the
 863		 * hardware by doing a read. Otherwise, there is no guarantee
 864		 * that the write has reached the hardware before udelay().
 865		 */
 866		regmap_read_bypassed(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
 867				     &val);
 868		udelay(15);
 869		regmap_clear_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
 870				  IMX95_PCIE_COLD_RST);
 871		regmap_read_bypassed(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL,
 872				     &val);
 873		udelay(10);
 874	}
 875
 876	return 0;
 877}
 878
 879static void imx_pcie_assert_core_reset(struct imx_pcie *imx_pcie)
 880{
 881	reset_control_assert(imx_pcie->pciephy_reset);
 882
 883	if (imx_pcie->drvdata->core_reset)
 884		imx_pcie->drvdata->core_reset(imx_pcie, true);
 885
 886	/* Some boards don't have PCIe reset GPIO. */
 887	gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 1);
 888}
 889
 890static int imx_pcie_deassert_core_reset(struct imx_pcie *imx_pcie)
 891{
 892	reset_control_deassert(imx_pcie->pciephy_reset);
 893
 894	if (imx_pcie->drvdata->core_reset)
 895		imx_pcie->drvdata->core_reset(imx_pcie, false);
 896
 897	/* Some boards don't have PCIe reset GPIO. */
 898	if (imx_pcie->reset_gpiod) {
 899		msleep(100);
 900		gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 0);
 901		/* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */
 902		msleep(100);
 903	}
 904
 905	return 0;
 906}
 907
 908static int imx_pcie_wait_for_speed_change(struct imx_pcie *imx_pcie)
 909{
 910	struct dw_pcie *pci = imx_pcie->pci;
 911	struct device *dev = pci->dev;
 912	u32 tmp;
 913	unsigned int retries;
 914
 915	for (retries = 0; retries < 200; retries++) {
 916		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
 917		/* Test if the speed change finished. */
 918		if (!(tmp & PORT_LOGIC_SPEED_CHANGE))
 919			return 0;
 920		usleep_range(100, 1000);
 921	}
 922
 923	dev_err(dev, "Speed change timeout\n");
 924	return -ETIMEDOUT;
 925}
 926
 927static void imx_pcie_ltssm_enable(struct device *dev)
 928{
 929	struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
 930	const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata;
 931	u8 offset = dw_pcie_find_capability(imx_pcie->pci, PCI_CAP_ID_EXP);
 932	u32 tmp;
 933
 934	tmp = dw_pcie_readl_dbi(imx_pcie->pci, offset + PCI_EXP_LNKCAP);
 935	phy_set_speed(imx_pcie->phy, FIELD_GET(PCI_EXP_LNKCAP_SLS, tmp));
 936	if (drvdata->ltssm_mask)
 937		regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask,
 938				   drvdata->ltssm_mask);
 939
 940	reset_control_deassert(imx_pcie->apps_reset);
 941}
 942
 943static void imx_pcie_ltssm_disable(struct device *dev)
 944{
 945	struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
 946	const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata;
 947
 948	phy_set_speed(imx_pcie->phy, 0);
 949	if (drvdata->ltssm_mask)
 950		regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off,
 951				   drvdata->ltssm_mask, 0);
 952
 953	reset_control_assert(imx_pcie->apps_reset);
 954}
 955
 956static int imx_pcie_start_link(struct dw_pcie *pci)
 957{
 958	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
 959	struct device *dev = pci->dev;
 960	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 961	u32 tmp;
 962	int ret;
 963
 964	if (!(imx_pcie->drvdata->flags &
 965	    IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND)) {
 966		imx_pcie_ltssm_enable(dev);
 967		return 0;
 968	}
 969
 970	/*
 971	 * Force Gen1 operation when starting the link.  In case the link is
 972	 * started in Gen2 mode, there is a possibility the devices on the
 973	 * bus will not be detected at all.  This happens with PCIe switches.
 974	 */
 975	dw_pcie_dbi_ro_wr_en(pci);
 976	tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
 977	tmp &= ~PCI_EXP_LNKCAP_SLS;
 978	tmp |= PCI_EXP_LNKCAP_SLS_2_5GB;
 979	dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
 980	dw_pcie_dbi_ro_wr_dis(pci);
 981
 982	/* Start LTSSM. */
 983	imx_pcie_ltssm_enable(dev);
 984
 985	if (pci->max_link_speed > 1) {
 986		ret = dw_pcie_wait_for_link(pci);
 987		if (ret)
 988			goto err_reset_phy;
 989
 990		/* Allow faster modes after the link is up */
 991		dw_pcie_dbi_ro_wr_en(pci);
 992		tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
 993		tmp &= ~PCI_EXP_LNKCAP_SLS;
 994		tmp |= pci->max_link_speed;
 995		dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);
 996
 997		/*
 998		 * Start Directed Speed Change so the best possible
 999		 * speed both link partners support can be negotiated.
1000		 */
1001		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
1002		tmp |= PORT_LOGIC_SPEED_CHANGE;
1003		dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);
1004		dw_pcie_dbi_ro_wr_dis(pci);
1005
1006		ret = imx_pcie_wait_for_speed_change(imx_pcie);
1007		if (ret) {
1008			dev_err(dev, "Failed to bring link up!\n");
1009			goto err_reset_phy;
1010		}
1011	} else {
1012		dev_info(dev, "Link: Only Gen1 is enabled\n");
1013	}
1014
1015	return 0;
1016
1017err_reset_phy:
1018	dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n",
1019		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0),
1020		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1));
1021	imx_pcie_reset_phy(imx_pcie);
1022	return 0;
1023}
1024
1025static void imx_pcie_stop_link(struct dw_pcie *pci)
1026{
1027	struct device *dev = pci->dev;
1028
1029	/* Turn off PCIe LTSSM */
1030	imx_pcie_ltssm_disable(dev);
1031}
1032
1033static int imx_pcie_add_lut(struct imx_pcie *imx_pcie, u16 rid, u8 sid)
1034{
1035	struct dw_pcie *pci = imx_pcie->pci;
1036	struct device *dev = pci->dev;
1037	u32 data1, data2;
1038	int free = -1;
1039	int i;
1040
1041	if (sid >= 64) {
1042		dev_err(dev, "Invalid SID for index %d\n", sid);
1043		return -EINVAL;
1044	}
1045
1046	guard(mutex)(&imx_pcie->lock);
1047
1048	/*
1049	 * Iterate through all LUT entries to check for duplicate RID and
1050	 * identify the first available entry. Configure this available entry
1051	 * immediately after verification to avoid rescanning it.
1052	 */
1053	for (i = 0; i < IMX95_MAX_LUT; i++) {
1054		regmap_write(imx_pcie->iomuxc_gpr,
1055			     IMX95_PE0_LUT_ACSCTRL, IMX95_PEO_LUT_RWA | i);
1056		regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, &data1);
1057
1058		if (!(data1 & IMX95_PE0_LUT_VLD)) {
1059			if (free < 0)
1060				free = i;
1061			continue;
1062		}
1063
1064		regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2);
1065
1066		/* Do not add duplicate RID */
1067		if (rid == FIELD_GET(IMX95_PE0_LUT_REQID, data2)) {
1068			dev_warn(dev, "Existing LUT entry available for RID (%d)", rid);
1069			return 0;
1070		}
1071	}
1072
1073	if (free < 0) {
1074		dev_err(dev, "LUT entry is not available\n");
1075		return -ENOSPC;
1076	}
1077
1078	data1 = FIELD_PREP(IMX95_PE0_LUT_DAC_ID, 0);
1079	data1 |= FIELD_PREP(IMX95_PE0_LUT_STREAM_ID, sid);
1080	data1 |= IMX95_PE0_LUT_VLD;
1081	regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, data1);
1082
1083	if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE)
1084		data2 = 0x7; /* In the EP mode, only 'Device ID' is required */
1085	else
1086		data2 = IMX95_PE0_LUT_MASK; /* Match all bits of RID */
1087	data2 |= FIELD_PREP(IMX95_PE0_LUT_REQID, rid);
1088	regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, data2);
1089
1090	regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, free);
1091
1092	return 0;
1093}
1094
1095static void imx_pcie_remove_lut(struct imx_pcie *imx_pcie, u16 rid)
1096{
1097	u32 data2;
1098	int i;
1099
1100	guard(mutex)(&imx_pcie->lock);
1101
1102	for (i = 0; i < IMX95_MAX_LUT; i++) {
1103		regmap_write(imx_pcie->iomuxc_gpr,
1104			     IMX95_PE0_LUT_ACSCTRL, IMX95_PEO_LUT_RWA | i);
1105		regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2);
1106		if (FIELD_GET(IMX95_PE0_LUT_REQID, data2) == rid) {
1107			regmap_write(imx_pcie->iomuxc_gpr,
1108				     IMX95_PE0_LUT_DATA1, 0);
1109			regmap_write(imx_pcie->iomuxc_gpr,
1110				     IMX95_PE0_LUT_DATA2, 0);
1111			regmap_write(imx_pcie->iomuxc_gpr,
1112				     IMX95_PE0_LUT_ACSCTRL, i);
1113
1114			break;
1115		}
1116	}
1117}
1118
1119static int imx_pcie_add_lut_by_rid(struct imx_pcie *imx_pcie, u32 rid)
1120{
1121	struct device *dev = imx_pcie->pci->dev;
1122	struct device_node *target;
1123	u32 sid_i, sid_m;
1124	int err_i, err_m;
1125	u32 sid = 0;
1126
1127	target = NULL;
1128	err_i = of_map_id(dev->of_node, rid, "iommu-map", "iommu-map-mask",
1129			  &target, &sid_i);
1130	if (target) {
1131		of_node_put(target);
1132	} else {
1133		/*
1134		 * "target == NULL && err_i == 0" means RID out of map range.
1135		 * Use 1:1 map RID to streamID. Hardware can't support this
1136		 * because the streamID is only 6 bits
1137		 */
1138		err_i = -EINVAL;
1139	}
1140
1141	target = NULL;
1142	err_m = of_map_id(dev->of_node, rid, "msi-map", "msi-map-mask",
1143			  &target, &sid_m);
1144
1145	/*
1146	 *   err_m      target
1147	 *	0	NULL		RID out of range. Use 1:1 map RID to
1148	 *				streamID, Current hardware can't
1149	 *				support it, so return -EINVAL.
1150	 *      != 0    NULL		msi-map does not exist, use built-in MSI
1151	 *	0	!= NULL		Get correct streamID from RID
1152	 *	!= 0	!= NULL		Invalid combination
1153	 */
1154	if (!err_m && !target)
1155		return -EINVAL;
1156	else if (target)
1157		of_node_put(target); /* Find streamID map entry for RID in msi-map */
1158
1159	/*
1160	 * msi-map        iommu-map
1161	 *   N                N            DWC MSI Ctrl
1162	 *   Y                Y            ITS + SMMU, require the same SID
1163	 *   Y                N            ITS
1164	 *   N                Y            DWC MSI Ctrl + SMMU
1165	 */
1166	if (err_i && err_m)
1167		return 0;
1168
1169	if (!err_i && !err_m) {
1170		/*
1171		 *	    Glue Layer
1172		 *          <==========>
1173		 * ┌─────┐                  ┌──────────┐
1174		 * │ LUT │ 6-bit streamID   │          │
1175		 * │     │─────────────────►│  MSI     │
1176		 * └─────┘   2-bit ctrl ID  │          │
1177		 *             ┌───────────►│          │
1178		 *  (i.MX95)   │            │          │
1179		 *  00 PCIe0   │            │          │
1180		 *  01 ENETC   │            │          │
1181		 *  10 PCIe1   │            │          │
1182		 *             │            └──────────┘
1183		 * The MSI glue layer auto adds 2 bits controller ID ahead of
1184		 * streamID, so mask these 2 bits to get streamID. The
1185		 * IOMMU glue layer doesn't do that.
1186		 */
1187		if (sid_i != (sid_m & IMX95_SID_MASK)) {
1188			dev_err(dev, "iommu-map and msi-map entries mismatch!\n");
1189			return -EINVAL;
1190		}
1191	}
1192
1193	if (!err_i)
1194		sid = sid_i;
1195	else if (!err_m)
1196		sid = sid_m & IMX95_SID_MASK;
1197
1198	return imx_pcie_add_lut(imx_pcie, rid, sid);
1199}
1200
1201static int imx_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev)
1202{
1203	struct imx_pcie *imx_pcie = to_imx_pcie(to_dw_pcie_from_pp(bridge->sysdata));
1204
1205	return imx_pcie_add_lut_by_rid(imx_pcie, pci_dev_id(pdev));
1206}
1207
1208static void imx_pcie_disable_device(struct pci_host_bridge *bridge,
1209				    struct pci_dev *pdev)
1210{
1211	struct imx_pcie *imx_pcie;
1212
1213	imx_pcie = to_imx_pcie(to_dw_pcie_from_pp(bridge->sysdata));
1214	imx_pcie_remove_lut(imx_pcie, pci_dev_id(pdev));
1215}
1216
1217static int imx_pcie_host_init(struct dw_pcie_rp *pp)
1218{
1219	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1220	struct device *dev = pci->dev;
1221	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1222	int ret;
1223
1224	if (imx_pcie->vpcie) {
1225		ret = regulator_enable(imx_pcie->vpcie);
1226		if (ret) {
1227			dev_err(dev, "failed to enable vpcie regulator: %d\n",
1228				ret);
1229			return ret;
1230		}
1231	}
1232
1233	if (pp->bridge && imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) {
1234		pp->bridge->enable_device = imx_pcie_enable_device;
1235		pp->bridge->disable_device = imx_pcie_disable_device;
1236	}
1237
1238	imx_pcie_assert_core_reset(imx_pcie);
1239
1240	if (imx_pcie->drvdata->init_phy)
1241		imx_pcie->drvdata->init_phy(imx_pcie);
1242
1243	imx_pcie_configure_type(imx_pcie);
1244
1245	ret = imx_pcie_clk_enable(imx_pcie);
1246	if (ret) {
1247		dev_err(dev, "unable to enable pcie clocks: %d\n", ret);
1248		goto err_reg_disable;
1249	}
1250
1251	if (imx_pcie->phy) {
1252		ret = phy_init(imx_pcie->phy);
1253		if (ret) {
1254			dev_err(dev, "pcie PHY power up failed\n");
1255			goto err_clk_disable;
1256		}
1257
1258		ret = phy_set_mode_ext(imx_pcie->phy, PHY_MODE_PCIE,
1259				       imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE ?
1260						PHY_MODE_PCIE_EP : PHY_MODE_PCIE_RC);
1261		if (ret) {
1262			dev_err(dev, "unable to set PCIe PHY mode\n");
1263			goto err_phy_exit;
1264		}
1265
1266		ret = phy_power_on(imx_pcie->phy);
1267		if (ret) {
1268			dev_err(dev, "waiting for PHY ready timeout!\n");
1269			goto err_phy_exit;
1270		}
1271	}
1272
1273	/* Make sure that PCIe LTSSM is cleared */
1274	imx_pcie_ltssm_disable(dev);
1275
1276	ret = imx_pcie_deassert_core_reset(imx_pcie);
1277	if (ret < 0) {
1278		dev_err(dev, "pcie deassert core reset failed: %d\n", ret);
1279		goto err_phy_off;
1280	}
1281
1282	if (imx_pcie->drvdata->wait_pll_lock) {
1283		ret = imx_pcie->drvdata->wait_pll_lock(imx_pcie);
1284		if (ret < 0)
1285			goto err_phy_off;
1286	}
1287
1288	imx_setup_phy_mpll(imx_pcie);
1289
1290	return 0;
1291
1292err_phy_off:
1293	phy_power_off(imx_pcie->phy);
1294err_phy_exit:
1295	phy_exit(imx_pcie->phy);
1296err_clk_disable:
1297	imx_pcie_clk_disable(imx_pcie);
1298err_reg_disable:
1299	if (imx_pcie->vpcie)
1300		regulator_disable(imx_pcie->vpcie);
1301	return ret;
1302}
1303
1304static void imx_pcie_host_exit(struct dw_pcie_rp *pp)
1305{
1306	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1307	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1308
1309	if (imx_pcie->phy) {
1310		if (phy_power_off(imx_pcie->phy))
1311			dev_err(pci->dev, "unable to power off PHY\n");
1312		phy_exit(imx_pcie->phy);
1313	}
1314	imx_pcie_clk_disable(imx_pcie);
1315
1316	if (imx_pcie->vpcie)
1317		regulator_disable(imx_pcie->vpcie);
1318}
1319
1320static void imx_pcie_host_post_init(struct dw_pcie_rp *pp)
1321{
1322	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1323	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1324	u32 val;
1325
1326	if (imx_pcie->drvdata->flags & IMX_PCIE_FLAG_8GT_ECN_ERR051586) {
1327		/*
1328		 * ERR051586: Compliance with 8GT/s Receiver Impedance ECN
1329		 *
1330		 * The default value of GEN3_RELATED_OFF[GEN3_ZRXDC_NONCOMPL]
1331		 * is 1 which makes receiver non-compliant with the ZRX-DC
1332		 * parameter for 2.5 GT/s when operating at 8 GT/s or higher.
1333		 * It causes unnecessary timeout in L1.
1334		 *
1335		 * Workaround: Program GEN3_RELATED_OFF[GEN3_ZRXDC_NONCOMPL]
1336		 * to 0.
1337		 */
1338		dw_pcie_dbi_ro_wr_en(pci);
1339		val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1340		val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1341		dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1342		dw_pcie_dbi_ro_wr_dis(pci);
1343	}
1344}
1345
1346/*
1347 * In old DWC implementations, PCIE_ATU_INHIBIT_PAYLOAD in iATU Ctrl2
1348 * register is reserved, so the generic DWC implementation of sending the
1349 * PME_Turn_Off message using a dummy MMIO write cannot be used.
1350 */
1351static void imx_pcie_pme_turn_off(struct dw_pcie_rp *pp)
1352{
1353	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1354	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1355
1356	regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF);
1357	regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF);
1358
1359	usleep_range(PCIE_PME_TO_L2_TIMEOUT_US/10, PCIE_PME_TO_L2_TIMEOUT_US);
1360}
1361
1362static const struct dw_pcie_host_ops imx_pcie_host_ops = {
1363	.init = imx_pcie_host_init,
1364	.deinit = imx_pcie_host_exit,
1365	.pme_turn_off = imx_pcie_pme_turn_off,
1366};
1367
1368static const struct dw_pcie_host_ops imx_pcie_host_dw_pme_ops = {
1369	.init = imx_pcie_host_init,
1370	.deinit = imx_pcie_host_exit,
1371	.post_init = imx_pcie_host_post_init,
1372};
1373
1374static const struct dw_pcie_ops dw_pcie_ops = {
1375	.start_link = imx_pcie_start_link,
1376	.stop_link = imx_pcie_stop_link,
1377};
1378
1379static void imx_pcie_ep_init(struct dw_pcie_ep *ep)
1380{
1381	enum pci_barno bar;
1382	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1383
1384	for (bar = BAR_0; bar <= BAR_5; bar++)
1385		dw_pcie_ep_reset_bar(pci, bar);
1386}
1387
1388static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1389				  unsigned int type, u16 interrupt_num)
1390{
1391	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1392
1393	switch (type) {
1394	case PCI_IRQ_INTX:
1395		return dw_pcie_ep_raise_intx_irq(ep, func_no);
1396	case PCI_IRQ_MSI:
1397		return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
1398	case PCI_IRQ_MSIX:
1399		return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
1400	default:
1401		dev_err(pci->dev, "UNKNOWN IRQ type\n");
1402		return -EINVAL;
1403	}
1404
1405	return 0;
1406}
1407
1408static const struct pci_epc_features imx8m_pcie_epc_features = {
1409	DWC_EPC_COMMON_FEATURES,
1410	.msi_capable = true,
1411	.bar[BAR_1] = { .type = BAR_RESERVED, },
1412	.bar[BAR_3] = { .type = BAR_RESERVED, },
1413	.bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = SZ_256, },
1414	.bar[BAR_5] = { .type = BAR_RESERVED, },
1415	.align = SZ_64K,
1416};
1417
1418static const struct pci_epc_features imx8q_pcie_epc_features = {
1419	DWC_EPC_COMMON_FEATURES,
1420	.msi_capable = true,
1421	.bar[BAR_1] = { .type = BAR_RESERVED, },
1422	.bar[BAR_3] = { .type = BAR_RESERVED, },
1423	.bar[BAR_5] = { .type = BAR_RESERVED, },
1424	.align = SZ_64K,
1425};
1426
1427/*
1428 *     	| Default  | Default | Default | BAR Sizing
1429 * BAR#	| Enable?  | Type    | Size    | Scheme
1430 * =======================================================
1431 * BAR0	| Enable   | 64-bit  |  1 MB   | Programmable Size
1432 * BAR1	| Disable  | 32-bit  | 64 KB   | Fixed Size
1433 *       (BAR1 should be disabled if BAR0 is 64-bit)
1434 * BAR2	| Enable   | 32-bit  |  1 MB   | Programmable Size
1435 * BAR3	| Enable   | 32-bit  | 64 KB   | Programmable Size
1436 * BAR4	| Enable   | 32-bit  |  1 MB   | Programmable Size
1437 * BAR5	| Enable   | 32-bit  | 64 KB   | Programmable Size
1438 */
1439static const struct pci_epc_features imx95_pcie_epc_features = {
1440	DWC_EPC_COMMON_FEATURES,
1441	.msi_capable = true,
1442	.bar[BAR_1] = { .type = BAR_FIXED, .fixed_size = SZ_64K, },
1443	.align = SZ_4K,
1444};
1445
1446static const struct pci_epc_features*
1447imx_pcie_ep_get_features(struct dw_pcie_ep *ep)
1448{
1449	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1450	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
1451
1452	return imx_pcie->drvdata->epc_features;
1453}
1454
1455static const struct dw_pcie_ep_ops pcie_ep_ops = {
1456	.init = imx_pcie_ep_init,
1457	.raise_irq = imx_pcie_ep_raise_irq,
1458	.get_features = imx_pcie_ep_get_features,
1459};
1460
1461static int imx_add_pcie_ep(struct imx_pcie *imx_pcie,
1462			   struct platform_device *pdev)
1463{
1464	int ret;
1465	struct dw_pcie_ep *ep;
1466	struct dw_pcie *pci = imx_pcie->pci;
1467	struct dw_pcie_rp *pp = &pci->pp;
1468	struct device *dev = pci->dev;
1469
1470	imx_pcie_host_init(pp);
1471	ep = &pci->ep;
1472	ep->ops = &pcie_ep_ops;
1473
1474	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SUPPORT_64BIT))
1475		dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1476
1477	ep->page_size = imx_pcie->drvdata->epc_features->align;
1478
1479	ret = dw_pcie_ep_init(ep);
1480	if (ret) {
1481		dev_err(dev, "failed to initialize endpoint\n");
1482		return ret;
1483	}
1484	imx_pcie_host_post_init(pp);
1485
1486	ret = dw_pcie_ep_init_registers(ep);
1487	if (ret) {
1488		dev_err(dev, "Failed to initialize DWC endpoint registers\n");
1489		dw_pcie_ep_deinit(ep);
1490		return ret;
1491	}
1492
1493	pci_epc_init_notify(ep->epc);
1494
1495	return 0;
1496}
1497
1498static void imx_pcie_msi_save_restore(struct imx_pcie *imx_pcie, bool save)
1499{
1500	u8 offset;
1501	u16 val;
1502	struct dw_pcie *pci = imx_pcie->pci;
1503
1504	if (pci_msi_enabled()) {
1505		offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
1506		if (save) {
1507			val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS);
1508			imx_pcie->msi_ctrl = val;
1509		} else {
1510			dw_pcie_dbi_ro_wr_en(pci);
1511			val = imx_pcie->msi_ctrl;
1512			dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val);
1513			dw_pcie_dbi_ro_wr_dis(pci);
1514		}
1515	}
1516}
1517
1518static void imx_pcie_lut_save(struct imx_pcie *imx_pcie)
1519{
1520	u32 data1, data2;
1521	int i;
1522
1523	for (i = 0; i < IMX95_MAX_LUT; i++) {
1524		regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL,
1525			     IMX95_PEO_LUT_RWA | i);
1526		regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, &data1);
1527		regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2);
1528		if (data1 & IMX95_PE0_LUT_VLD) {
1529			imx_pcie->luts[i].data1 = data1;
1530			imx_pcie->luts[i].data2 = data2;
1531		} else {
1532			imx_pcie->luts[i].data1 = 0;
1533			imx_pcie->luts[i].data2 = 0;
1534		}
1535	}
1536}
1537
1538static void imx_pcie_lut_restore(struct imx_pcie *imx_pcie)
1539{
1540	int i;
1541
1542	for (i = 0; i < IMX95_MAX_LUT; i++) {
1543		if ((imx_pcie->luts[i].data1 & IMX95_PE0_LUT_VLD) == 0)
1544			continue;
1545
1546		regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1,
1547			     imx_pcie->luts[i].data1);
1548		regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2,
1549			     imx_pcie->luts[i].data2);
1550		regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, i);
1551	}
1552}
1553
1554static int imx_pcie_suspend_noirq(struct device *dev)
1555{
1556	struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
1557
1558	if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND))
1559		return 0;
1560
1561	imx_pcie_msi_save_restore(imx_pcie, true);
1562	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT))
1563		imx_pcie_lut_save(imx_pcie);
1564	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) {
1565		/*
1566		 * The minimum for a workaround would be to set PERST# and to
1567		 * set the PCIE_TEST_PD flag. However, we can also disable the
1568		 * clock which saves some power.
1569		 */
1570		imx_pcie_assert_core_reset(imx_pcie);
1571		imx_pcie->drvdata->enable_ref_clk(imx_pcie, false);
1572	} else {
1573		return dw_pcie_suspend_noirq(imx_pcie->pci);
1574	}
1575
1576	return 0;
1577}
1578
1579static int imx_pcie_resume_noirq(struct device *dev)
1580{
1581	int ret;
1582	struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
1583
1584	if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND))
1585		return 0;
1586
1587	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) {
1588		ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true);
1589		if (ret)
1590			return ret;
1591		ret = imx_pcie_deassert_core_reset(imx_pcie);
1592		if (ret)
1593			return ret;
1594
1595		/*
1596		 * Using PCIE_TEST_PD seems to disable MSI and powers down the
1597		 * root complex. This is why we have to setup the rc again and
1598		 * why we have to restore the MSI register.
1599		 */
1600		ret = dw_pcie_setup_rc(&imx_pcie->pci->pp);
1601		if (ret)
1602			return ret;
1603	} else {
1604		ret = dw_pcie_resume_noirq(imx_pcie->pci);
1605		if (ret)
1606			return ret;
1607	}
1608	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT))
1609		imx_pcie_lut_restore(imx_pcie);
1610	imx_pcie_msi_save_restore(imx_pcie, false);
1611
1612	return 0;
1613}
1614
1615static const struct dev_pm_ops imx_pcie_pm_ops = {
1616	NOIRQ_SYSTEM_SLEEP_PM_OPS(imx_pcie_suspend_noirq,
1617				  imx_pcie_resume_noirq)
1618};
1619
1620static int imx_pcie_probe(struct platform_device *pdev)
1621{
1622	struct device *dev = &pdev->dev;
1623	struct dw_pcie *pci;
1624	struct imx_pcie *imx_pcie;
1625	struct device_node *np;
1626	struct device_node *node = dev->of_node;
1627	int ret, domain;
1628	u16 val;
1629
1630	imx_pcie = devm_kzalloc(dev, sizeof(*imx_pcie), GFP_KERNEL);
1631	if (!imx_pcie)
1632		return -ENOMEM;
1633
1634	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
1635	if (!pci)
1636		return -ENOMEM;
1637
1638	pci->dev = dev;
1639	pci->ops = &dw_pcie_ops;
1640
1641	imx_pcie->pci = pci;
1642	imx_pcie->drvdata = of_device_get_match_data(dev);
1643
1644	mutex_init(&imx_pcie->lock);
1645
1646	if (imx_pcie->drvdata->ops)
1647		pci->pp.ops = imx_pcie->drvdata->ops;
1648	else
1649		pci->pp.ops = &imx_pcie_host_dw_pme_ops;
1650
1651	/* Find the PHY if one is defined, only imx7d uses it */
1652	np = of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0);
1653	if (np) {
1654		struct resource res;
1655
1656		ret = of_address_to_resource(np, 0, &res);
1657		if (ret) {
1658			dev_err(dev, "Unable to map PCIe PHY\n");
1659			return ret;
1660		}
1661		imx_pcie->phy_base = devm_ioremap_resource(dev, &res);
1662		if (IS_ERR(imx_pcie->phy_base))
1663			return PTR_ERR(imx_pcie->phy_base);
1664	}
1665
1666	/* Fetch GPIOs */
1667	imx_pcie->reset_gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
1668	if (IS_ERR(imx_pcie->reset_gpiod))
1669		return dev_err_probe(dev, PTR_ERR(imx_pcie->reset_gpiod),
1670				     "unable to get reset gpio\n");
1671	gpiod_set_consumer_name(imx_pcie->reset_gpiod, "PCIe reset");
1672
1673	/* Fetch clocks */
1674	imx_pcie->num_clks = devm_clk_bulk_get_all(dev, &imx_pcie->clks);
1675	if (imx_pcie->num_clks < 0)
1676		return dev_err_probe(dev, imx_pcie->num_clks,
1677				     "failed to get clocks\n");
1678
1679	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHYDRV)) {
1680		imx_pcie->phy = devm_phy_get(dev, "pcie-phy");
1681		if (IS_ERR(imx_pcie->phy))
1682			return dev_err_probe(dev, PTR_ERR(imx_pcie->phy),
1683					     "failed to get pcie phy\n");
1684	}
1685
1686	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_APP_RESET)) {
1687		imx_pcie->apps_reset = devm_reset_control_get_exclusive(dev, "apps");
1688		if (IS_ERR(imx_pcie->apps_reset))
1689			return dev_err_probe(dev, PTR_ERR(imx_pcie->apps_reset),
1690					     "failed to get pcie apps reset control\n");
1691	}
1692
1693	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHY_RESET)) {
1694		imx_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev, "pciephy");
1695		if (IS_ERR(imx_pcie->pciephy_reset))
1696			return dev_err_probe(dev, PTR_ERR(imx_pcie->pciephy_reset),
1697					     "Failed to get PCIEPHY reset control\n");
1698	}
1699
1700	switch (imx_pcie->drvdata->variant) {
1701	case IMX8MQ:
1702	case IMX8MQ_EP:
1703		domain = of_get_pci_domain_nr(node);
1704		if (domain < 0 || domain > 1)
1705			return dev_err_probe(dev, -ENODEV, "no \"linux,pci-domain\" property in devicetree\n");
1706
1707		imx_pcie->controller_id = domain;
1708		break;
1709	default:
1710		break;
1711	}
1712
1713	if (imx_pcie->drvdata->gpr) {
1714	/* Grab GPR config register range */
1715		imx_pcie->iomuxc_gpr =
1716			 syscon_regmap_lookup_by_compatible(imx_pcie->drvdata->gpr);
1717		if (IS_ERR(imx_pcie->iomuxc_gpr))
1718			return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr),
1719					     "unable to find iomuxc registers\n");
1720	}
1721
1722	if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_SERDES)) {
1723		void __iomem *off = devm_platform_ioremap_resource_byname(pdev, "app");
1724
1725		if (IS_ERR(off))
1726			return dev_err_probe(dev, PTR_ERR(off),
1727					     "unable to find serdes registers\n");
1728
1729		static const struct regmap_config regmap_config = {
1730			.reg_bits = 32,
1731			.val_bits = 32,
1732			.reg_stride = 4,
1733		};
1734
1735		imx_pcie->iomuxc_gpr = devm_regmap_init_mmio(dev, off, &regmap_config);
1736		if (IS_ERR(imx_pcie->iomuxc_gpr))
1737			return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr),
1738					     "unable to find iomuxc registers\n");
1739	}
1740
1741	/* Grab PCIe PHY Tx Settings */
1742	if (of_property_read_u32(node, "fsl,tx-deemph-gen1",
1743				 &imx_pcie->tx_deemph_gen1))
1744		imx_pcie->tx_deemph_gen1 = 0;
1745
1746	if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db",
1747				 &imx_pcie->tx_deemph_gen2_3p5db))
1748		imx_pcie->tx_deemph_gen2_3p5db = 0;
1749
1750	if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db",
1751				 &imx_pcie->tx_deemph_gen2_6db))
1752		imx_pcie->tx_deemph_gen2_6db = 20;
1753
1754	if (of_property_read_u32(node, "fsl,tx-swing-full",
1755				 &imx_pcie->tx_swing_full))
1756		imx_pcie->tx_swing_full = 127;
1757
1758	if (of_property_read_u32(node, "fsl,tx-swing-low",
1759				 &imx_pcie->tx_swing_low))
1760		imx_pcie->tx_swing_low = 127;
1761
1762	/* Limit link speed */
1763	pci->max_link_speed = 1;
1764	of_property_read_u32(node, "fsl,max-link-speed", &pci->max_link_speed);
1765
1766	ret = devm_regulator_get_enable_optional(&pdev->dev, "vpcie3v3aux");
1767	if (ret < 0 && ret != -ENODEV)
1768		return dev_err_probe(dev, ret, "failed to enable Vaux supply\n");
1769
1770	imx_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie");
1771	if (IS_ERR(imx_pcie->vpcie)) {
1772		if (PTR_ERR(imx_pcie->vpcie) != -ENODEV)
1773			return PTR_ERR(imx_pcie->vpcie);
1774		imx_pcie->vpcie = NULL;
1775	}
1776
1777	imx_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph");
1778	if (IS_ERR(imx_pcie->vph)) {
1779		if (PTR_ERR(imx_pcie->vph) != -ENODEV)
1780			return PTR_ERR(imx_pcie->vph);
1781		imx_pcie->vph = NULL;
1782	}
1783
1784	platform_set_drvdata(pdev, imx_pcie);
1785
1786	ret = imx_pcie_attach_pd(dev);
1787	if (ret)
1788		return ret;
1789
1790	pci->use_parent_dt_ranges = true;
1791	if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE) {
1792		ret = imx_add_pcie_ep(imx_pcie, pdev);
1793		if (ret < 0)
1794			return ret;
1795
1796		/*
1797		 * FIXME: Only single Device (EPF) is supported due to the
1798		 * Endpoint framework limitation.
1799		 */
1800		imx_pcie_add_lut_by_rid(imx_pcie, 0);
1801	} else {
1802		if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY))
1803			pci->pp.skip_l23_ready = true;
1804		pci->pp.use_atu_msg = true;
1805		ret = dw_pcie_host_init(&pci->pp);
1806		if (ret < 0)
1807			return ret;
1808
1809		if (pci_msi_enabled()) {
1810			u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
1811
1812			val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS);
1813			val |= PCI_MSI_FLAGS_ENABLE;
1814			dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val);
1815		}
1816	}
1817
1818	return 0;
1819}
1820
1821static void imx_pcie_shutdown(struct platform_device *pdev)
1822{
1823	struct imx_pcie *imx_pcie = platform_get_drvdata(pdev);
1824
1825	/* bring down link, so bootloader gets clean state in case of reboot */
1826	imx_pcie_assert_core_reset(imx_pcie);
1827}
1828
1829static const struct imx_pcie_drvdata drvdata[] = {
1830	[IMX6Q] = {
1831		.variant = IMX6Q,
1832		.flags = IMX_PCIE_FLAG_IMX_PHY |
1833			 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND |
1834			 IMX_PCIE_FLAG_BROKEN_SUSPEND |
1835			 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
1836		.dbi_length = 0x200,
1837		.gpr = "fsl,imx6q-iomuxc-gpr",
1838		.ltssm_off = IOMUXC_GPR12,
1839		.ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2,
1840		.mode_off[0] = IOMUXC_GPR12,
1841		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1842		.init_phy = imx_pcie_init_phy,
1843		.enable_ref_clk = imx6q_pcie_enable_ref_clk,
1844		.core_reset = imx6q_pcie_core_reset,
1845	},
1846	[IMX6SX] = {
1847		.variant = IMX6SX,
1848		.flags = IMX_PCIE_FLAG_IMX_PHY |
1849			 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND |
1850			 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
1851		.gpr = "fsl,imx6q-iomuxc-gpr",
1852		.ltssm_off = IOMUXC_GPR12,
1853		.ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2,
1854		.mode_off[0] = IOMUXC_GPR12,
1855		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1856		.init_phy = imx6sx_pcie_init_phy,
1857		.enable_ref_clk = imx6sx_pcie_enable_ref_clk,
1858		.core_reset = imx6sx_pcie_core_reset,
1859		.ops = &imx_pcie_host_ops,
1860	},
1861	[IMX6QP] = {
1862		.variant = IMX6QP,
1863		.flags = IMX_PCIE_FLAG_IMX_PHY |
1864			 IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND |
1865			 IMX_PCIE_FLAG_SKIP_L23_READY |
1866			 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
1867		.dbi_length = 0x200,
1868		.gpr = "fsl,imx6q-iomuxc-gpr",
1869		.ltssm_off = IOMUXC_GPR12,
1870		.ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2,
1871		.mode_off[0] = IOMUXC_GPR12,
1872		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1873		.init_phy = imx_pcie_init_phy,
1874		.enable_ref_clk = imx6q_pcie_enable_ref_clk,
1875		.core_reset = imx6qp_pcie_core_reset,
1876		.ops = &imx_pcie_host_ops,
1877	},
1878	[IMX7D] = {
1879		.variant = IMX7D,
1880		.flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
1881			 IMX_PCIE_FLAG_HAS_APP_RESET |
1882			 IMX_PCIE_FLAG_SKIP_L23_READY |
1883			 IMX_PCIE_FLAG_HAS_PHY_RESET,
1884		.gpr = "fsl,imx7d-iomuxc-gpr",
1885		.mode_off[0] = IOMUXC_GPR12,
1886		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1887		.enable_ref_clk = imx7d_pcie_enable_ref_clk,
1888		.core_reset = imx7d_pcie_core_reset,
1889	},
1890	[IMX8MQ] = {
1891		.variant = IMX8MQ,
1892		.flags = IMX_PCIE_FLAG_HAS_APP_RESET |
1893			 IMX_PCIE_FLAG_HAS_PHY_RESET |
1894			 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
1895		.gpr = "fsl,imx8mq-iomuxc-gpr",
1896		.mode_off[0] = IOMUXC_GPR12,
1897		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1898		.mode_off[1] = IOMUXC_GPR12,
1899		.mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,
1900		.init_phy = imx8mq_pcie_init_phy,
1901		.enable_ref_clk = imx8mm_pcie_enable_ref_clk,
1902	},
1903	[IMX8MM] = {
1904		.variant = IMX8MM,
1905		.flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
1906			 IMX_PCIE_FLAG_HAS_PHYDRV |
1907			 IMX_PCIE_FLAG_HAS_APP_RESET,
1908		.gpr = "fsl,imx8mm-iomuxc-gpr",
1909		.mode_off[0] = IOMUXC_GPR12,
1910		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1911		.enable_ref_clk = imx8mm_pcie_enable_ref_clk,
1912	},
1913	[IMX8MP] = {
1914		.variant = IMX8MP,
1915		.flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND |
1916			 IMX_PCIE_FLAG_HAS_PHYDRV |
1917			 IMX_PCIE_FLAG_HAS_APP_RESET,
1918		.gpr = "fsl,imx8mp-iomuxc-gpr",
1919		.mode_off[0] = IOMUXC_GPR12,
1920		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1921		.enable_ref_clk = imx8mm_pcie_enable_ref_clk,
1922	},
1923	[IMX8Q] = {
1924		.variant = IMX8Q,
1925		.flags = IMX_PCIE_FLAG_HAS_PHYDRV |
1926			 IMX_PCIE_FLAG_CPU_ADDR_FIXUP |
1927			 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
1928	},
1929	[IMX95] = {
1930		.variant = IMX95,
1931		.flags = IMX_PCIE_FLAG_HAS_SERDES |
1932			 IMX_PCIE_FLAG_HAS_LUT |
1933			 IMX_PCIE_FLAG_8GT_ECN_ERR051586 |
1934			 IMX_PCIE_FLAG_SUPPORTS_SUSPEND,
1935		.ltssm_off = IMX95_PE0_GEN_CTRL_3,
1936		.ltssm_mask = IMX95_PCIE_LTSSM_EN,
1937		.mode_off[0]  = IMX95_PE0_GEN_CTRL_1,
1938		.mode_mask[0] = IMX95_PCIE_DEVICE_TYPE,
1939		.core_reset = imx95_pcie_core_reset,
1940		.init_phy = imx95_pcie_init_phy,
1941		.wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock,
1942		.enable_ref_clk = imx95_pcie_enable_ref_clk,
1943	},
1944	[IMX8MQ_EP] = {
1945		.variant = IMX8MQ_EP,
1946		.flags = IMX_PCIE_FLAG_HAS_APP_RESET |
1947			 IMX_PCIE_FLAG_HAS_PHY_RESET,
1948		.mode = DW_PCIE_EP_TYPE,
1949		.gpr = "fsl,imx8mq-iomuxc-gpr",
1950		.mode_off[0] = IOMUXC_GPR12,
1951		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1952		.mode_off[1] = IOMUXC_GPR12,
1953		.mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,
1954		.epc_features = &imx8q_pcie_epc_features,
1955		.init_phy = imx8mq_pcie_init_phy,
1956		.enable_ref_clk = imx8mm_pcie_enable_ref_clk,
1957	},
1958	[IMX8MM_EP] = {
1959		.variant = IMX8MM_EP,
1960		.flags = IMX_PCIE_FLAG_HAS_APP_RESET |
1961			 IMX_PCIE_FLAG_HAS_PHYDRV,
1962		.mode = DW_PCIE_EP_TYPE,
1963		.gpr = "fsl,imx8mm-iomuxc-gpr",
1964		.mode_off[0] = IOMUXC_GPR12,
1965		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1966		.epc_features = &imx8m_pcie_epc_features,
1967		.enable_ref_clk = imx8mm_pcie_enable_ref_clk,
1968	},
1969	[IMX8MP_EP] = {
1970		.variant = IMX8MP_EP,
1971		.flags = IMX_PCIE_FLAG_HAS_APP_RESET |
1972			 IMX_PCIE_FLAG_HAS_PHYDRV,
1973		.mode = DW_PCIE_EP_TYPE,
1974		.gpr = "fsl,imx8mp-iomuxc-gpr",
1975		.mode_off[0] = IOMUXC_GPR12,
1976		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
1977		.epc_features = &imx8m_pcie_epc_features,
1978		.enable_ref_clk = imx8mm_pcie_enable_ref_clk,
1979	},
1980	[IMX8Q_EP] = {
1981		.variant = IMX8Q_EP,
1982		.flags = IMX_PCIE_FLAG_HAS_PHYDRV,
1983		.mode = DW_PCIE_EP_TYPE,
1984		.epc_features = &imx8q_pcie_epc_features,
1985	},
1986	[IMX95_EP] = {
1987		.variant = IMX95_EP,
1988		.flags = IMX_PCIE_FLAG_HAS_SERDES |
1989			 IMX_PCIE_FLAG_8GT_ECN_ERR051586 |
1990			 IMX_PCIE_FLAG_SUPPORT_64BIT,
1991		.ltssm_off = IMX95_PE0_GEN_CTRL_3,
1992		.ltssm_mask = IMX95_PCIE_LTSSM_EN,
1993		.mode_off[0]  = IMX95_PE0_GEN_CTRL_1,
1994		.mode_mask[0] = IMX95_PCIE_DEVICE_TYPE,
1995		.init_phy = imx95_pcie_init_phy,
1996		.core_reset = imx95_pcie_core_reset,
1997		.wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock,
1998		.epc_features = &imx95_pcie_epc_features,
1999		.enable_ref_clk = imx95_pcie_enable_ref_clk,
2000		.mode = DW_PCIE_EP_TYPE,
2001	},
2002};
2003
2004static const struct of_device_id imx_pcie_of_match[] = {
2005	{ .compatible = "fsl,imx6q-pcie",  .data = &drvdata[IMX6Q],  },
2006	{ .compatible = "fsl,imx6sx-pcie", .data = &drvdata[IMX6SX], },
2007	{ .compatible = "fsl,imx6qp-pcie", .data = &drvdata[IMX6QP], },
2008	{ .compatible = "fsl,imx7d-pcie",  .data = &drvdata[IMX7D],  },
2009	{ .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], },
2010	{ .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], },
2011	{ .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], },
2012	{ .compatible = "fsl,imx8q-pcie", .data = &drvdata[IMX8Q], },
2013	{ .compatible = "fsl,imx95-pcie", .data = &drvdata[IMX95], },
2014	{ .compatible = "fsl,imx8mq-pcie-ep", .data = &drvdata[IMX8MQ_EP], },
2015	{ .compatible = "fsl,imx8mm-pcie-ep", .data = &drvdata[IMX8MM_EP], },
2016	{ .compatible = "fsl,imx8mp-pcie-ep", .data = &drvdata[IMX8MP_EP], },
2017	{ .compatible = "fsl,imx8q-pcie-ep", .data = &drvdata[IMX8Q_EP], },
2018	{ .compatible = "fsl,imx95-pcie-ep", .data = &drvdata[IMX95_EP], },
2019	{},
2020};
2021
2022static struct platform_driver imx_pcie_driver = {
2023	.driver = {
2024		.name	= "imx6q-pcie",
2025		.of_match_table = imx_pcie_of_match,
2026		.suppress_bind_attrs = true,
2027		.pm = &imx_pcie_pm_ops,
2028		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
2029	},
2030	.probe    = imx_pcie_probe,
2031	.shutdown = imx_pcie_shutdown,
2032};
2033
2034static void imx_pcie_quirk(struct pci_dev *dev)
2035{
2036	struct pci_bus *bus = dev->bus;
2037	struct dw_pcie_rp *pp = bus->sysdata;
2038
2039	/* Bus parent is the PCI bridge, its parent is this platform driver */
2040	if (!bus->dev.parent || !bus->dev.parent->parent)
2041		return;
2042
2043	/* Make sure we only quirk devices associated with this driver */
2044	if (bus->dev.parent->parent->driver != &imx_pcie_driver.driver)
2045		return;
2046
2047	if (pci_is_root_bus(bus)) {
2048		struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
2049		struct imx_pcie *imx_pcie = to_imx_pcie(pci);
2050
2051		/*
2052		 * Limit config length to avoid the kernel reading beyond
2053		 * the register set and causing an abort on i.MX 6Quad
2054		 */
2055		if (imx_pcie->drvdata->dbi_length) {
2056			dev->cfg_size = imx_pcie->drvdata->dbi_length;
2057			dev_info(&dev->dev, "Limiting cfg_size to %d\n",
2058					dev->cfg_size);
2059		}
2060	}
2061}
2062DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd,
2063			PCI_CLASS_BRIDGE_PCI, 8, imx_pcie_quirk);
2064
2065static int __init imx_pcie_init(void)
2066{
2067#ifdef CONFIG_ARM
2068	struct device_node *np;
2069
2070	np = of_find_matching_node(NULL, imx_pcie_of_match);
2071	if (!np)
2072		return -ENODEV;
2073	of_node_put(np);
2074
2075	/*
2076	 * Since probe() can be deferred we need to make sure that
2077	 * hook_fault_code is not called after __init memory is freed
2078	 * by kernel and since imx6q_pcie_abort_handler() is a no-op,
2079	 * we can install the handler here without risking it
2080	 * accessing some uninitialized driver state.
2081	 */
2082	hook_fault_code(8, imx6q_pcie_abort_handler, SIGBUS, 0,
2083			"external abort on non-linefetch");
2084#endif
2085
2086	return platform_driver_register(&imx_pcie_driver);
2087}
2088device_initcall(imx_pcie_init);
~/ Read-only file preview.