pcie-mediatek-gen3.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * MediaTek PCIe host controller driver.
   4 *
   5 * Copyright (c) 2020 MediaTek Inc.
   6 * Author: Jianjun Wang <jianjun.wang@mediatek.com>
   7 */
   8
   9#include <linux/bitfield.h>
  10#include <linux/clk.h>
  11#include <linux/clk-provider.h>
  12#include <linux/delay.h>
  13#include <linux/iopoll.h>
  14#include <linux/irq.h>
  15#include <linux/irqchip/irq-msi-lib.h>
  16#include <linux/irqchip/chained_irq.h>
  17#include <linux/irqdomain.h>
  18#include <linux/kernel.h>
  19#include <linux/mfd/syscon.h>
  20#include <linux/module.h>
  21#include <linux/msi.h>
  22#include <linux/of_device.h>
  23#include <linux/of_pci.h>
  24#include <linux/pci.h>
  25#include <linux/phy/phy.h>
  26#include <linux/platform_device.h>
  27#include <linux/pm_domain.h>
  28#include <linux/pm_runtime.h>
  29#include <linux/regmap.h>
  30#include <linux/reset.h>
  31
  32#include "../pci.h"
  33
  34#define PCIE_BASE_CFG_REG		0x14
  35#define PCIE_BASE_CFG_SPEED		GENMASK(15, 8)
  36
  37#define PCIE_SETTING_REG		0x80
  38#define PCIE_SETTING_LINK_WIDTH		GENMASK(11, 8)
  39#define PCIE_SETTING_GEN_SUPPORT	GENMASK(14, 12)
  40#define PCIE_PCI_IDS_1			0x9c
  41#define PCI_CLASS(class)		(class << 8)
  42#define PCIE_RC_MODE			BIT(0)
  43
  44#define PCIE_EQ_PRESET_01_REG		0x100
  45#define PCIE_VAL_LN0_DOWNSTREAM		GENMASK(6, 0)
  46#define PCIE_VAL_LN0_UPSTREAM		GENMASK(14, 8)
  47#define PCIE_VAL_LN1_DOWNSTREAM		GENMASK(22, 16)
  48#define PCIE_VAL_LN1_UPSTREAM		GENMASK(30, 24)
  49
  50#define PCIE_CFGNUM_REG			0x140
  51#define PCIE_CFG_DEVFN(devfn)		((devfn) & GENMASK(7, 0))
  52#define PCIE_CFG_BUS(bus)		(((bus) << 8) & GENMASK(15, 8))
  53#define PCIE_CFG_BYTE_EN(bytes)		(((bytes) << 16) & GENMASK(19, 16))
  54#define PCIE_CFG_FORCE_BYTE_EN		BIT(20)
  55#define PCIE_CFG_OFFSET_ADDR		0x1000
  56#define PCIE_CFG_HEADER(bus, devfn) \
  57	(PCIE_CFG_BUS(bus) | PCIE_CFG_DEVFN(devfn))
  58
  59#define PCIE_RST_CTRL_REG		0x148
  60#define PCIE_MAC_RSTB			BIT(0)
  61#define PCIE_PHY_RSTB			BIT(1)
  62#define PCIE_BRG_RSTB			BIT(2)
  63#define PCIE_PE_RSTB			BIT(3)
  64
  65#define PCIE_LTSSM_STATUS_REG		0x150
  66#define PCIE_LTSSM_STATE_MASK		GENMASK(28, 24)
  67#define PCIE_LTSSM_STATE(val)		((val & PCIE_LTSSM_STATE_MASK) >> 24)
  68#define PCIE_LTSSM_STATE_L2_IDLE	0x14
  69
  70#define PCIE_LINK_STATUS_REG		0x154
  71#define PCIE_PORT_LINKUP		BIT(8)
  72
  73#define PCIE_MSI_SET_NUM		8
  74#define PCIE_MSI_IRQS_PER_SET		32
  75#define PCIE_MSI_IRQS_NUM \
  76	(PCIE_MSI_IRQS_PER_SET * PCIE_MSI_SET_NUM)
  77
  78#define PCIE_INT_ENABLE_REG		0x180
  79#define PCIE_MSI_ENABLE			GENMASK(PCIE_MSI_SET_NUM + 8 - 1, 8)
  80#define PCIE_MSI_SHIFT			8
  81#define PCIE_INTX_SHIFT			24
  82#define PCIE_INTX_ENABLE \
  83	GENMASK(PCIE_INTX_SHIFT + PCI_NUM_INTX - 1, PCIE_INTX_SHIFT)
  84
  85#define PCIE_INT_STATUS_REG		0x184
  86#define PCIE_MSI_SET_ENABLE_REG		0x190
  87#define PCIE_MSI_SET_ENABLE		GENMASK(PCIE_MSI_SET_NUM - 1, 0)
  88
  89#define PCIE_PIPE4_PIE8_REG		0x338
  90#define PCIE_K_FINETUNE_MAX		GENMASK(5, 0)
  91#define PCIE_K_FINETUNE_ERR		GENMASK(7, 6)
  92#define PCIE_K_PRESET_TO_USE		GENMASK(18, 8)
  93#define PCIE_K_PHYPARAM_QUERY		BIT(19)
  94#define PCIE_K_QUERY_TIMEOUT		BIT(20)
  95#define PCIE_K_PRESET_TO_USE_16G	GENMASK(31, 21)
  96
  97#define PCIE_MSI_SET_BASE_REG		0xc00
  98#define PCIE_MSI_SET_OFFSET		0x10
  99#define PCIE_MSI_SET_STATUS_OFFSET	0x04
 100#define PCIE_MSI_SET_ENABLE_OFFSET	0x08
 101
 102#define PCIE_MSI_SET_ADDR_HI_BASE	0xc80
 103#define PCIE_MSI_SET_ADDR_HI_OFFSET	0x04
 104
 105#define PCIE_RESOURCE_CTRL_REG		0xd2c
 106#define PCIE_RSRC_SYS_CLK_RDY_TIME_MASK	GENMASK(7, 0)
 107
 108#define PCIE_ICMD_PM_REG		0x198
 109#define PCIE_TURN_OFF_LINK		BIT(4)
 110
 111#define PCIE_MISC_CTRL_REG		0x348
 112#define PCIE_DISABLE_DVFSRC_VLT_REQ	BIT(1)
 113
 114#define PCIE_TRANS_TABLE_BASE_REG	0x800
 115#define PCIE_ATR_SRC_ADDR_MSB_OFFSET	0x4
 116#define PCIE_ATR_TRSL_ADDR_LSB_OFFSET	0x8
 117#define PCIE_ATR_TRSL_ADDR_MSB_OFFSET	0xc
 118#define PCIE_ATR_TRSL_PARAM_OFFSET	0x10
 119#define PCIE_ATR_TLB_SET_OFFSET		0x20
 120
 121#define PCIE_MAX_TRANS_TABLES		8
 122#define PCIE_ATR_EN			BIT(0)
 123#define PCIE_ATR_SIZE(size) \
 124	(((((size) - 1) << 1) & GENMASK(6, 1)) | PCIE_ATR_EN)
 125#define PCIE_ATR_ID(id)			((id) & GENMASK(3, 0))
 126#define PCIE_ATR_TYPE_MEM		PCIE_ATR_ID(0)
 127#define PCIE_ATR_TYPE_IO		PCIE_ATR_ID(1)
 128#define PCIE_ATR_TLP_TYPE(type)		(((type) << 16) & GENMASK(18, 16))
 129#define PCIE_ATR_TLP_TYPE_MEM		PCIE_ATR_TLP_TYPE(0)
 130#define PCIE_ATR_TLP_TYPE_IO		PCIE_ATR_TLP_TYPE(2)
 131
 132#define MAX_NUM_PHY_RESETS		3
 133
 134#define PCIE_MTK_RESET_TIME_US		10
 135
 136/* Time in ms needed to complete PCIe reset on EN7581 SoC */
 137#define PCIE_EN7581_RESET_TIME_MS	100
 138
 139struct mtk_gen3_pcie;
 140
 141#define PCIE_CONF_LINK2_CTL_STS		(PCIE_CFG_OFFSET_ADDR + 0xb0)
 142#define PCIE_CONF_LINK2_LCR2_LINK_SPEED	GENMASK(3, 0)
 143
 144enum mtk_gen3_pcie_flags {
 145	SKIP_PCIE_RSTB	= BIT(0), /* Skip PERST# assertion during device
 146				   * probing or suspend/resume phase to
 147				   * avoid hw bugs/issues.
 148				   */
 149};
 150
 151/**
 152 * struct mtk_gen3_pcie_pdata - differentiate between host generations
 153 * @power_up: pcie power_up callback
 154 * @phy_resets: phy reset lines SoC data.
 155 * @sys_clk_rdy_time_us: System clock ready time override (microseconds)
 156 * @flags: pcie device flags.
 157 */
 158struct mtk_gen3_pcie_pdata {
 159	int (*power_up)(struct mtk_gen3_pcie *pcie);
 160	struct {
 161		const char *id[MAX_NUM_PHY_RESETS];
 162		int num_resets;
 163	} phy_resets;
 164	u8 sys_clk_rdy_time_us;
 165	u32 flags;
 166};
 167
 168/**
 169 * struct mtk_msi_set - MSI information for each set
 170 * @base: IO mapped register base
 171 * @msg_addr: MSI message address
 172 * @saved_irq_state: IRQ enable state saved at suspend time
 173 */
 174struct mtk_msi_set {
 175	void __iomem *base;
 176	phys_addr_t msg_addr;
 177	u32 saved_irq_state;
 178};
 179
 180/**
 181 * struct mtk_gen3_pcie - PCIe port information
 182 * @dev: pointer to PCIe device
 183 * @base: IO mapped register base
 184 * @reg_base: physical register base
 185 * @mac_reset: MAC reset control
 186 * @phy_resets: PHY reset controllers
 187 * @phy: PHY controller block
 188 * @clks: PCIe clocks
 189 * @num_clks: PCIe clocks count for this port
 190 * @max_link_speed: Maximum link speed (PCIe Gen) for this port
 191 * @num_lanes: Number of PCIe lanes for this port
 192 * @irq: PCIe controller interrupt number
 193 * @saved_irq_state: IRQ enable state saved at suspend time
 194 * @irq_lock: lock protecting IRQ register access
 195 * @intx_domain: legacy INTx IRQ domain
 196 * @msi_bottom_domain: MSI IRQ bottom domain
 197 * @msi_sets: MSI sets information
 198 * @lock: lock protecting IRQ bit map
 199 * @msi_irq_in_use: bit map for assigned MSI IRQ
 200 * @soc: pointer to SoC-dependent operations
 201 */
 202struct mtk_gen3_pcie {
 203	struct device *dev;
 204	void __iomem *base;
 205	phys_addr_t reg_base;
 206	struct reset_control *mac_reset;
 207	struct reset_control_bulk_data phy_resets[MAX_NUM_PHY_RESETS];
 208	struct phy *phy;
 209	struct clk_bulk_data *clks;
 210	int num_clks;
 211	u8 max_link_speed;
 212	u8 num_lanes;
 213
 214	int irq;
 215	u32 saved_irq_state;
 216	raw_spinlock_t irq_lock;
 217	struct irq_domain *intx_domain;
 218	struct irq_domain *msi_bottom_domain;
 219	struct mtk_msi_set msi_sets[PCIE_MSI_SET_NUM];
 220	struct mutex lock;
 221	DECLARE_BITMAP(msi_irq_in_use, PCIE_MSI_IRQS_NUM);
 222
 223	const struct mtk_gen3_pcie_pdata *soc;
 224};
 225
 226/* LTSSM state in PCIE_LTSSM_STATUS_REG bit[28:24] */
 227static const char *const ltssm_str[] = {
 228	"detect.quiet",			/* 0x00 */
 229	"detect.active",		/* 0x01 */
 230	"polling.active",		/* 0x02 */
 231	"polling.compliance",		/* 0x03 */
 232	"polling.configuration",	/* 0x04 */
 233	"config.linkwidthstart",	/* 0x05 */
 234	"config.linkwidthaccept",	/* 0x06 */
 235	"config.lanenumwait",		/* 0x07 */
 236	"config.lanenumaccept",		/* 0x08 */
 237	"config.complete",		/* 0x09 */
 238	"config.idle",			/* 0x0A */
 239	"recovery.receiverlock",	/* 0x0B */
 240	"recovery.equalization",	/* 0x0C */
 241	"recovery.speed",		/* 0x0D */
 242	"recovery.receiverconfig",	/* 0x0E */
 243	"recovery.idle",		/* 0x0F */
 244	"L0",				/* 0x10 */
 245	"L0s",				/* 0x11 */
 246	"L1.entry",			/* 0x12 */
 247	"L1.idle",			/* 0x13 */
 248	"L2.idle",			/* 0x14 */
 249	"L2.transmitwake",		/* 0x15 */
 250	"disable",			/* 0x16 */
 251	"loopback.entry",		/* 0x17 */
 252	"loopback.active",		/* 0x18 */
 253	"loopback.exit",		/* 0x19 */
 254	"hotreset",			/* 0x1A */
 255};
 256
 257/**
 258 * mtk_pcie_config_tlp_header() - Configure a configuration TLP header
 259 * @bus: PCI bus to query
 260 * @devfn: device/function number
 261 * @where: offset in config space
 262 * @size: data size in TLP header
 263 *
 264 * Set byte enable field and device information in configuration TLP header.
 265 */
 266static void mtk_pcie_config_tlp_header(struct pci_bus *bus, unsigned int devfn,
 267					int where, int size)
 268{
 269	struct mtk_gen3_pcie *pcie = bus->sysdata;
 270	int bytes;
 271	u32 val;
 272
 273	bytes = (GENMASK(size - 1, 0) & 0xf) << (where & 0x3);
 274
 275	val = PCIE_CFG_FORCE_BYTE_EN | PCIE_CFG_BYTE_EN(bytes) |
 276	      PCIE_CFG_HEADER(bus->number, devfn);
 277
 278	writel_relaxed(val, pcie->base + PCIE_CFGNUM_REG);
 279}
 280
 281static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus, unsigned int devfn,
 282				      int where)
 283{
 284	struct mtk_gen3_pcie *pcie = bus->sysdata;
 285
 286	return pcie->base + PCIE_CFG_OFFSET_ADDR + where;
 287}
 288
 289static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
 290				int where, int size, u32 *val)
 291{
 292	mtk_pcie_config_tlp_header(bus, devfn, where, size);
 293
 294	return pci_generic_config_read32(bus, devfn, where, size, val);
 295}
 296
 297static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
 298				 int where, int size, u32 val)
 299{
 300	mtk_pcie_config_tlp_header(bus, devfn, where, size);
 301
 302	if (size <= 2)
 303		val <<= (where & 0x3) * 8;
 304
 305	return pci_generic_config_write32(bus, devfn, where, 4, val);
 306}
 307
 308static struct pci_ops mtk_pcie_ops = {
 309	.map_bus = mtk_pcie_map_bus,
 310	.read  = mtk_pcie_config_read,
 311	.write = mtk_pcie_config_write,
 312};
 313
 314static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie,
 315				    resource_size_t cpu_addr,
 316				    resource_size_t pci_addr,
 317				    resource_size_t size,
 318				    unsigned long type, int *num)
 319{
 320	resource_size_t remaining = size;
 321	resource_size_t table_size;
 322	resource_size_t addr_align;
 323	const char *range_type;
 324	void __iomem *table;
 325	u32 val;
 326
 327	while (remaining && (*num < PCIE_MAX_TRANS_TABLES)) {
 328		/* Table size needs to be a power of 2 */
 329		table_size = BIT(fls(remaining) - 1);
 330
 331		if (cpu_addr > 0) {
 332			addr_align = BIT(ffs(cpu_addr) - 1);
 333			table_size = min(table_size, addr_align);
 334		}
 335
 336		/* Minimum size of translate table is 4KiB */
 337		if (table_size < 0x1000) {
 338			dev_err(pcie->dev, "illegal table size %#llx\n",
 339				(unsigned long long)table_size);
 340			return -EINVAL;
 341		}
 342
 343		table = pcie->base + PCIE_TRANS_TABLE_BASE_REG + *num * PCIE_ATR_TLB_SET_OFFSET;
 344		writel_relaxed(lower_32_bits(cpu_addr) | PCIE_ATR_SIZE(fls(table_size) - 1), table);
 345		writel_relaxed(upper_32_bits(cpu_addr), table + PCIE_ATR_SRC_ADDR_MSB_OFFSET);
 346		writel_relaxed(lower_32_bits(pci_addr), table + PCIE_ATR_TRSL_ADDR_LSB_OFFSET);
 347		writel_relaxed(upper_32_bits(pci_addr), table + PCIE_ATR_TRSL_ADDR_MSB_OFFSET);
 348
 349		if (type == IORESOURCE_IO) {
 350			val = PCIE_ATR_TYPE_IO | PCIE_ATR_TLP_TYPE_IO;
 351			range_type = "IO";
 352		} else {
 353			val = PCIE_ATR_TYPE_MEM | PCIE_ATR_TLP_TYPE_MEM;
 354			range_type = "MEM";
 355		}
 356
 357		writel_relaxed(val, table + PCIE_ATR_TRSL_PARAM_OFFSET);
 358
 359		dev_dbg(pcie->dev, "set %s trans window[%d]: cpu_addr = %#llx, pci_addr = %#llx, size = %#llx\n",
 360			range_type, *num, (unsigned long long)cpu_addr,
 361			(unsigned long long)pci_addr,
 362			(unsigned long long)table_size);
 363
 364		cpu_addr += table_size;
 365		pci_addr += table_size;
 366		remaining -= table_size;
 367		(*num)++;
 368	}
 369
 370	if (remaining)
 371		dev_warn(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n",
 372			 (unsigned long long)cpu_addr, PCIE_MAX_TRANS_TABLES);
 373
 374	return 0;
 375}
 376
 377static void mtk_pcie_enable_msi(struct mtk_gen3_pcie *pcie)
 378{
 379	int i;
 380	u32 val;
 381
 382	for (i = 0; i < PCIE_MSI_SET_NUM; i++) {
 383		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];
 384
 385		msi_set->base = pcie->base + PCIE_MSI_SET_BASE_REG +
 386				i * PCIE_MSI_SET_OFFSET;
 387		msi_set->msg_addr = pcie->reg_base + PCIE_MSI_SET_BASE_REG +
 388				    i * PCIE_MSI_SET_OFFSET;
 389
 390		/* Configure the MSI capture address */
 391		writel_relaxed(lower_32_bits(msi_set->msg_addr), msi_set->base);
 392		writel_relaxed(upper_32_bits(msi_set->msg_addr),
 393			       pcie->base + PCIE_MSI_SET_ADDR_HI_BASE +
 394			       i * PCIE_MSI_SET_ADDR_HI_OFFSET);
 395	}
 396
 397	val = readl_relaxed(pcie->base + PCIE_MSI_SET_ENABLE_REG);
 398	val |= PCIE_MSI_SET_ENABLE;
 399	writel_relaxed(val, pcie->base + PCIE_MSI_SET_ENABLE_REG);
 400
 401	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);
 402	val |= PCIE_MSI_ENABLE;
 403	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);
 404}
 405
 406static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 407{
 408	struct resource_entry *entry;
 409	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
 410	unsigned int table_index = 0;
 411	int err;
 412	u32 val;
 413
 414	/* Set as RC mode and set controller PCIe Gen speed restriction, if any */
 415	val = readl_relaxed(pcie->base + PCIE_SETTING_REG);
 416	val |= PCIE_RC_MODE;
 417	if (pcie->max_link_speed) {
 418		val &= ~PCIE_SETTING_GEN_SUPPORT;
 419
 420		/* Can enable link speed support only from Gen2 onwards */
 421		if (pcie->max_link_speed >= 2)
 422			val |= FIELD_PREP(PCIE_SETTING_GEN_SUPPORT,
 423					  GENMASK(pcie->max_link_speed - 2, 0));
 424	}
 425	if (pcie->num_lanes) {
 426		val &= ~PCIE_SETTING_LINK_WIDTH;
 427
 428		/* Zero means one lane, each bit activates x2/x4/x8/x16 */
 429		if (pcie->num_lanes > 1)
 430			val |= FIELD_PREP(PCIE_SETTING_LINK_WIDTH,
 431					  GENMASK(fls(pcie->num_lanes >> 2), 0));
 432	}
 433	writel_relaxed(val, pcie->base + PCIE_SETTING_REG);
 434
 435	/* Set Link Control 2 (LNKCTL2) speed restriction, if any */
 436	if (pcie->max_link_speed) {
 437		val = readl_relaxed(pcie->base + PCIE_CONF_LINK2_CTL_STS);
 438		val &= ~PCIE_CONF_LINK2_LCR2_LINK_SPEED;
 439		val |= FIELD_PREP(PCIE_CONF_LINK2_LCR2_LINK_SPEED, pcie->max_link_speed);
 440		writel_relaxed(val, pcie->base + PCIE_CONF_LINK2_CTL_STS);
 441	}
 442
 443	/* If parameter is present, adjust SYS_CLK_RDY_TIME to avoid glitching */
 444	if (pcie->soc->sys_clk_rdy_time_us) {
 445		val = readl_relaxed(pcie->base + PCIE_RESOURCE_CTRL_REG);
 446		FIELD_MODIFY(PCIE_RSRC_SYS_CLK_RDY_TIME_MASK, &val,
 447			     pcie->soc->sys_clk_rdy_time_us);
 448		writel_relaxed(val, pcie->base + PCIE_RESOURCE_CTRL_REG);
 449	}
 450
 451	/* Set class code */
 452	val = readl_relaxed(pcie->base + PCIE_PCI_IDS_1);
 453	val &= ~GENMASK(31, 8);
 454	val |= PCI_CLASS(PCI_CLASS_BRIDGE_PCI_NORMAL);
 455	writel_relaxed(val, pcie->base + PCIE_PCI_IDS_1);
 456
 457	/* Mask all INTx interrupts */
 458	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);
 459	val &= ~PCIE_INTX_ENABLE;
 460	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);
 461
 462	/* Disable DVFSRC voltage request */
 463	val = readl_relaxed(pcie->base + PCIE_MISC_CTRL_REG);
 464	val |= PCIE_DISABLE_DVFSRC_VLT_REQ;
 465	writel_relaxed(val, pcie->base + PCIE_MISC_CTRL_REG);
 466
 467	/*
 468	 * Airoha EN7581 has a hw bug asserting/releasing PCIE_PE_RSTB signal
 469	 * causing occasional PCIe link down. In order to overcome the issue,
 470	 * PCIE_RSTB signals are not asserted/released at this stage and the
 471	 * PCIe block is reset using en7523_reset_assert() and
 472	 * en7581_pci_enable().
 473	 */
 474	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
 475		/* Assert all reset signals */
 476		val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);
 477		val |= PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
 478		       PCIE_PE_RSTB;
 479		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
 480
 481		/*
 482		 * Described in PCIe CEM specification revision 6.0.
 483		 *
 484		 * The deassertion of PERST# should be delayed 100ms (TPVPERL)
 485		 * for the power and clock to become stable.
 486		 */
 487		msleep(PCIE_T_PVPERL_MS);
 488
 489		/* De-assert reset signals */
 490		val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
 491			 PCIE_PE_RSTB);
 492		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
 493	}
 494
 495	/* Check if the link is up or not */
 496	err = readl_poll_timeout(pcie->base + PCIE_LINK_STATUS_REG, val,
 497				 !!(val & PCIE_PORT_LINKUP), 20,
 498				 PCI_PM_D3COLD_WAIT * USEC_PER_MSEC);
 499	if (err) {
 500		const char *ltssm_state;
 501		int ltssm_index;
 502
 503		val = readl_relaxed(pcie->base + PCIE_LTSSM_STATUS_REG);
 504		ltssm_index = PCIE_LTSSM_STATE(val);
 505		ltssm_state = ltssm_index >= ARRAY_SIZE(ltssm_str) ?
 506			      "Unknown state" : ltssm_str[ltssm_index];
 507		dev_err(pcie->dev,
 508			"PCIe link down, current LTSSM state: %s (%#x)\n",
 509			ltssm_state, val);
 510		return err;
 511	}
 512
 513	mtk_pcie_enable_msi(pcie);
 514
 515	/* Set PCIe translation windows */
 516	resource_list_for_each_entry(entry, &host->windows) {
 517		struct resource *res = entry->res;
 518		unsigned long type = resource_type(res);
 519		resource_size_t cpu_addr;
 520		resource_size_t pci_addr;
 521		resource_size_t size;
 522
 523		if (type == IORESOURCE_IO)
 524			cpu_addr = pci_pio_to_address(res->start);
 525		else if (type == IORESOURCE_MEM)
 526			cpu_addr = res->start;
 527		else
 528			continue;
 529
 530		pci_addr = res->start - entry->offset;
 531		size = resource_size(res);
 532		err = mtk_pcie_set_trans_table(pcie, cpu_addr, pci_addr, size,
 533					       type, &table_index);
 534		if (err)
 535			return err;
 536	}
 537
 538	return 0;
 539}
 540
 541#define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
 542				MSI_FLAG_USE_DEF_CHIP_OPS	| \
 543				MSI_FLAG_NO_AFFINITY		| \
 544				MSI_FLAG_PCI_MSI_MASK_PARENT)
 545
 546#define MTK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
 547				 MSI_FLAG_PCI_MSIX		| \
 548				 MSI_FLAG_MULTI_PCI_MSI)
 549
 550static const struct msi_parent_ops mtk_msi_parent_ops = {
 551	.required_flags		= MTK_MSI_FLAGS_REQUIRED,
 552	.supported_flags	= MTK_MSI_FLAGS_SUPPORTED,
 553	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
 554	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
 555	.prefix			= "MTK3-",
 556	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 557};
 558
 559static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 560{
 561	struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data);
 562	struct mtk_gen3_pcie *pcie = data->domain->host_data;
 563	unsigned long hwirq;
 564
 565	hwirq =	data->hwirq % PCIE_MSI_IRQS_PER_SET;
 566
 567	msg->address_hi = upper_32_bits(msi_set->msg_addr);
 568	msg->address_lo = lower_32_bits(msi_set->msg_addr);
 569	msg->data = hwirq;
 570	dev_dbg(pcie->dev, "msi#%#lx address_hi %#x address_lo %#x data %d\n",
 571		hwirq, msg->address_hi, msg->address_lo, msg->data);
 572}
 573
 574static void mtk_msi_bottom_irq_ack(struct irq_data *data)
 575{
 576	struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data);
 577	unsigned long hwirq;
 578
 579	hwirq =	data->hwirq % PCIE_MSI_IRQS_PER_SET;
 580
 581	writel_relaxed(BIT(hwirq), msi_set->base + PCIE_MSI_SET_STATUS_OFFSET);
 582}
 583
 584static void mtk_msi_bottom_irq_mask(struct irq_data *data)
 585{
 586	struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data);
 587	struct mtk_gen3_pcie *pcie = data->domain->host_data;
 588	unsigned long hwirq, flags;
 589	u32 val;
 590
 591	hwirq =	data->hwirq % PCIE_MSI_IRQS_PER_SET;
 592
 593	raw_spin_lock_irqsave(&pcie->irq_lock, flags);
 594	val = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET);
 595	val &= ~BIT(hwirq);
 596	writel_relaxed(val, msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET);
 597	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
 598}
 599
 600static void mtk_msi_bottom_irq_unmask(struct irq_data *data)
 601{
 602	struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data);
 603	struct mtk_gen3_pcie *pcie = data->domain->host_data;
 604	unsigned long hwirq, flags;
 605	u32 val;
 606
 607	hwirq =	data->hwirq % PCIE_MSI_IRQS_PER_SET;
 608
 609	raw_spin_lock_irqsave(&pcie->irq_lock, flags);
 610	val = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET);
 611	val |= BIT(hwirq);
 612	writel_relaxed(val, msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET);
 613	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
 614}
 615
 616static struct irq_chip mtk_msi_bottom_irq_chip = {
 617	.irq_ack		= mtk_msi_bottom_irq_ack,
 618	.irq_mask		= mtk_msi_bottom_irq_mask,
 619	.irq_unmask		= mtk_msi_bottom_irq_unmask,
 620	.irq_compose_msi_msg	= mtk_compose_msi_msg,
 621	.name			= "MSI",
 622};
 623
 624static int mtk_msi_bottom_domain_alloc(struct irq_domain *domain,
 625				       unsigned int virq, unsigned int nr_irqs,
 626				       void *arg)
 627{
 628	struct mtk_gen3_pcie *pcie = domain->host_data;
 629	struct mtk_msi_set *msi_set;
 630	int i, hwirq, set_idx;
 631
 632	mutex_lock(&pcie->lock);
 633
 634	hwirq = bitmap_find_free_region(pcie->msi_irq_in_use, PCIE_MSI_IRQS_NUM,
 635					order_base_2(nr_irqs));
 636
 637	mutex_unlock(&pcie->lock);
 638
 639	if (hwirq < 0)
 640		return -ENOSPC;
 641
 642	set_idx = hwirq / PCIE_MSI_IRQS_PER_SET;
 643	msi_set = &pcie->msi_sets[set_idx];
 644
 645	for (i = 0; i < nr_irqs; i++)
 646		irq_domain_set_info(domain, virq + i, hwirq + i,
 647				    &mtk_msi_bottom_irq_chip, msi_set,
 648				    handle_edge_irq, NULL, NULL);
 649
 650	return 0;
 651}
 652
 653static void mtk_msi_bottom_domain_free(struct irq_domain *domain,
 654				       unsigned int virq, unsigned int nr_irqs)
 655{
 656	struct mtk_gen3_pcie *pcie = domain->host_data;
 657	struct irq_data *data = irq_domain_get_irq_data(domain, virq);
 658
 659	mutex_lock(&pcie->lock);
 660
 661	bitmap_release_region(pcie->msi_irq_in_use, data->hwirq,
 662			      order_base_2(nr_irqs));
 663
 664	mutex_unlock(&pcie->lock);
 665
 666	irq_domain_free_irqs_common(domain, virq, nr_irqs);
 667}
 668
 669static const struct irq_domain_ops mtk_msi_bottom_domain_ops = {
 670	.alloc = mtk_msi_bottom_domain_alloc,
 671	.free = mtk_msi_bottom_domain_free,
 672};
 673
 674static void mtk_intx_mask(struct irq_data *data)
 675{
 676	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);
 677	unsigned long flags;
 678	u32 val;
 679
 680	raw_spin_lock_irqsave(&pcie->irq_lock, flags);
 681	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);
 682	val &= ~BIT(data->hwirq + PCIE_INTX_SHIFT);
 683	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);
 684	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
 685}
 686
 687static void mtk_intx_unmask(struct irq_data *data)
 688{
 689	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);
 690	unsigned long flags;
 691	u32 val;
 692
 693	raw_spin_lock_irqsave(&pcie->irq_lock, flags);
 694	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);
 695	val |= BIT(data->hwirq + PCIE_INTX_SHIFT);
 696	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);
 697	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);
 698}
 699
 700/**
 701 * mtk_intx_eoi() - Clear INTx IRQ status at the end of interrupt
 702 * @data: pointer to chip specific data
 703 *
 704 * As an emulated level IRQ, its interrupt status will remain
 705 * until the corresponding de-assert message is received; hence that
 706 * the status can only be cleared when the interrupt has been serviced.
 707 */
 708static void mtk_intx_eoi(struct irq_data *data)
 709{
 710	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);
 711	unsigned long hwirq;
 712
 713	hwirq = data->hwirq + PCIE_INTX_SHIFT;
 714	writel_relaxed(BIT(hwirq), pcie->base + PCIE_INT_STATUS_REG);
 715}
 716
 717static struct irq_chip mtk_intx_irq_chip = {
 718	.irq_mask		= mtk_intx_mask,
 719	.irq_unmask		= mtk_intx_unmask,
 720	.irq_eoi		= mtk_intx_eoi,
 721	.name			= "INTx",
 722};
 723
 724static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 725			     irq_hw_number_t hwirq)
 726{
 727	irq_set_chip_data(irq, domain->host_data);
 728	irq_set_chip_and_handler_name(irq, &mtk_intx_irq_chip,
 729				      handle_fasteoi_irq, "INTx");
 730	return 0;
 731}
 732
 733static const struct irq_domain_ops intx_domain_ops = {
 734	.map = mtk_pcie_intx_map,
 735};
 736
 737static int mtk_pcie_init_irq_domains(struct mtk_gen3_pcie *pcie)
 738{
 739	struct device *dev = pcie->dev;
 740	struct device_node *intc_node, *node = dev->of_node;
 741	int ret;
 742
 743	raw_spin_lock_init(&pcie->irq_lock);
 744
 745	/* Setup INTx */
 746	intc_node = of_get_child_by_name(node, "interrupt-controller");
 747	if (!intc_node) {
 748		dev_err(dev, "missing interrupt-controller node\n");
 749		return -ENODEV;
 750	}
 751
 752	pcie->intx_domain = irq_domain_create_linear(of_fwnode_handle(intc_node), PCI_NUM_INTX,
 753						     &intx_domain_ops, pcie);
 754	if (!pcie->intx_domain) {
 755		dev_err(dev, "failed to create INTx IRQ domain\n");
 756		ret = -ENODEV;
 757		goto out_put_node;
 758	}
 759
 760	/* Setup MSI */
 761	mutex_init(&pcie->lock);
 762
 763	struct irq_domain_info info = {
 764		.fwnode		= dev_fwnode(dev),
 765		.ops		= &mtk_msi_bottom_domain_ops,
 766		.host_data	= pcie,
 767		.size		= PCIE_MSI_IRQS_NUM,
 768	};
 769
 770	pcie->msi_bottom_domain = msi_create_parent_irq_domain(&info, &mtk_msi_parent_ops);
 771	if (!pcie->msi_bottom_domain) {
 772		dev_err(dev, "failed to create MSI bottom domain\n");
 773		ret = -ENODEV;
 774		goto err_msi_bottom_domain;
 775	}
 776
 777	of_node_put(intc_node);
 778	return 0;
 779
 780err_msi_bottom_domain:
 781	irq_domain_remove(pcie->intx_domain);
 782out_put_node:
 783	of_node_put(intc_node);
 784	return ret;
 785}
 786
 787static void mtk_pcie_irq_teardown(struct mtk_gen3_pcie *pcie)
 788{
 789	irq_set_chained_handler_and_data(pcie->irq, NULL, NULL);
 790
 791	if (pcie->intx_domain)
 792		irq_domain_remove(pcie->intx_domain);
 793
 794	if (pcie->msi_bottom_domain)
 795		irq_domain_remove(pcie->msi_bottom_domain);
 796
 797	irq_dispose_mapping(pcie->irq);
 798}
 799
 800static void mtk_pcie_msi_handler(struct mtk_gen3_pcie *pcie, int set_idx)
 801{
 802	struct mtk_msi_set *msi_set = &pcie->msi_sets[set_idx];
 803	unsigned long msi_enable, msi_status;
 804	irq_hw_number_t bit, hwirq;
 805
 806	msi_enable = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET);
 807
 808	do {
 809		msi_status = readl_relaxed(msi_set->base +
 810					   PCIE_MSI_SET_STATUS_OFFSET);
 811		msi_status &= msi_enable;
 812		if (!msi_status)
 813			break;
 814
 815		for_each_set_bit(bit, &msi_status, PCIE_MSI_IRQS_PER_SET) {
 816			hwirq = bit + set_idx * PCIE_MSI_IRQS_PER_SET;
 817			generic_handle_domain_irq(pcie->msi_bottom_domain, hwirq);
 818		}
 819	} while (true);
 820}
 821
 822static void mtk_pcie_irq_handler(struct irq_desc *desc)
 823{
 824	struct mtk_gen3_pcie *pcie = irq_desc_get_handler_data(desc);
 825	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 826	unsigned long status;
 827	irq_hw_number_t irq_bit = PCIE_INTX_SHIFT;
 828
 829	chained_irq_enter(irqchip, desc);
 830
 831	status = readl_relaxed(pcie->base + PCIE_INT_STATUS_REG);
 832	for_each_set_bit_from(irq_bit, &status, PCI_NUM_INTX +
 833			      PCIE_INTX_SHIFT)
 834		generic_handle_domain_irq(pcie->intx_domain,
 835					  irq_bit - PCIE_INTX_SHIFT);
 836
 837	irq_bit = PCIE_MSI_SHIFT;
 838	for_each_set_bit_from(irq_bit, &status, PCIE_MSI_SET_NUM +
 839			      PCIE_MSI_SHIFT) {
 840		mtk_pcie_msi_handler(pcie, irq_bit - PCIE_MSI_SHIFT);
 841
 842		writel_relaxed(BIT(irq_bit), pcie->base + PCIE_INT_STATUS_REG);
 843	}
 844
 845	chained_irq_exit(irqchip, desc);
 846}
 847
 848static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie)
 849{
 850	struct device *dev = pcie->dev;
 851	struct platform_device *pdev = to_platform_device(dev);
 852	int err;
 853
 854	err = mtk_pcie_init_irq_domains(pcie);
 855	if (err)
 856		return err;
 857
 858	pcie->irq = platform_get_irq(pdev, 0);
 859	if (pcie->irq < 0)
 860		return pcie->irq;
 861
 862	irq_set_chained_handler_and_data(pcie->irq, mtk_pcie_irq_handler, pcie);
 863
 864	return 0;
 865}
 866
 867static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie)
 868{
 869	int i, ret, num_resets = pcie->soc->phy_resets.num_resets;
 870	struct device *dev = pcie->dev;
 871	struct platform_device *pdev = to_platform_device(dev);
 872	struct resource *regs;
 873	u32 num_lanes;
 874
 875	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac");
 876	if (!regs)
 877		return -EINVAL;
 878	pcie->base = devm_ioremap_resource(dev, regs);
 879	if (IS_ERR(pcie->base)) {
 880		dev_err(dev, "failed to map register base\n");
 881		return PTR_ERR(pcie->base);
 882	}
 883
 884	pcie->reg_base = regs->start;
 885
 886	for (i = 0; i < num_resets; i++)
 887		pcie->phy_resets[i].id = pcie->soc->phy_resets.id[i];
 888
 889	ret = devm_reset_control_bulk_get_optional_shared(dev, num_resets,
 890							  pcie->phy_resets);
 891	if (ret) {
 892		dev_err(dev, "failed to get PHY bulk reset\n");
 893		return ret;
 894	}
 895
 896	pcie->mac_reset = devm_reset_control_get_optional_exclusive(dev, "mac");
 897	if (IS_ERR(pcie->mac_reset)) {
 898		ret = PTR_ERR(pcie->mac_reset);
 899		if (ret != -EPROBE_DEFER)
 900			dev_err(dev, "failed to get MAC reset\n");
 901
 902		return ret;
 903	}
 904
 905	pcie->phy = devm_phy_optional_get(dev, "pcie-phy");
 906	if (IS_ERR(pcie->phy)) {
 907		ret = PTR_ERR(pcie->phy);
 908		if (ret != -EPROBE_DEFER)
 909			dev_err(dev, "failed to get PHY\n");
 910
 911		return ret;
 912	}
 913
 914	pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks);
 915	if (pcie->num_clks < 0) {
 916		dev_err(dev, "failed to get clocks\n");
 917		return pcie->num_clks;
 918	}
 919
 920	ret = of_property_read_u32(dev->of_node, "num-lanes", &num_lanes);
 921	if (ret == 0) {
 922		if (num_lanes == 0 || num_lanes > 16 ||
 923		    (num_lanes != 1 && num_lanes % 2))
 924			dev_warn(dev, "invalid num-lanes, using controller defaults\n");
 925		else
 926			pcie->num_lanes = num_lanes;
 927	}
 928
 929	return 0;
 930}
 931
 932static int mtk_pcie_en7581_power_up(struct mtk_gen3_pcie *pcie)
 933{
 934	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
 935	struct device *dev = pcie->dev;
 936	struct resource_entry *entry;
 937	struct regmap *pbus_regmap;
 938	u32 val, args[2], size;
 939	resource_size_t addr;
 940	int err;
 941
 942	/*
 943	 * The controller may have been left out of reset by the bootloader
 944	 * so make sure that we get a clean start by asserting resets here.
 945	 */
 946	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets,
 947				  pcie->phy_resets);
 948
 949	/* Wait for the time needed to complete the reset lines assert. */
 950	msleep(PCIE_EN7581_RESET_TIME_MS);
 951
 952	/*
 953	 * Configure PBus base address and base address mask to allow the
 954	 * hw to detect if a given address is accessible on PCIe controller.
 955	 */
 956	pbus_regmap = syscon_regmap_lookup_by_phandle_args(dev->of_node,
 957							   "mediatek,pbus-csr",
 958							   ARRAY_SIZE(args),
 959							   args);
 960	if (IS_ERR(pbus_regmap))
 961		return PTR_ERR(pbus_regmap);
 962
 963	entry = resource_list_first_type(&host->windows, IORESOURCE_MEM);
 964	if (!entry)
 965		return -ENODEV;
 966
 967	addr = entry->res->start - entry->offset;
 968	regmap_write(pbus_regmap, args[0], lower_32_bits(addr));
 969	size = lower_32_bits(resource_size(entry->res));
 970	regmap_write(pbus_regmap, args[1], GENMASK(31, __fls(size)));
 971
 972	/*
 973	 * Unlike the other MediaTek Gen3 controllers, the Airoha EN7581
 974	 * requires PHY initialization and power-on before PHY reset deassert.
 975	 */
 976	err = phy_init(pcie->phy);
 977	if (err) {
 978		dev_err(dev, "failed to initialize PHY\n");
 979		return err;
 980	}
 981
 982	err = phy_power_on(pcie->phy);
 983	if (err) {
 984		dev_err(dev, "failed to power on PHY\n");
 985		goto err_phy_on;
 986	}
 987
 988	err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets,
 989					  pcie->phy_resets);
 990	if (err) {
 991		dev_err(dev, "failed to deassert PHYs\n");
 992		goto err_phy_deassert;
 993	}
 994
 995	/*
 996	 * Wait for the time needed to complete the bulk de-assert above.
 997	 * This time is specific for EN7581 SoC.
 998	 */
 999	msleep(PCIE_EN7581_RESET_TIME_MS);
1000
1001	pm_runtime_enable(dev);
1002	pm_runtime_get_sync(dev);
1003
1004	val = FIELD_PREP(PCIE_VAL_LN0_DOWNSTREAM, 0x47) |
1005	      FIELD_PREP(PCIE_VAL_LN1_DOWNSTREAM, 0x47) |
1006	      FIELD_PREP(PCIE_VAL_LN0_UPSTREAM, 0x41) |
1007	      FIELD_PREP(PCIE_VAL_LN1_UPSTREAM, 0x41);
1008	writel_relaxed(val, pcie->base + PCIE_EQ_PRESET_01_REG);
1009
1010	val = PCIE_K_PHYPARAM_QUERY | PCIE_K_QUERY_TIMEOUT |
1011	      FIELD_PREP(PCIE_K_PRESET_TO_USE_16G, 0x80) |
1012	      FIELD_PREP(PCIE_K_PRESET_TO_USE, 0x2) |
1013	      FIELD_PREP(PCIE_K_FINETUNE_MAX, 0xf);
1014	writel_relaxed(val, pcie->base + PCIE_PIPE4_PIE8_REG);
1015
1016	err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks);
1017	if (err) {
1018		dev_err(dev, "failed to prepare clock\n");
1019		goto err_clk_prepare_enable;
1020	}
1021
1022	/*
1023	 * Airoha EN7581 performs PCIe reset via clk callbacks since it has a
1024	 * hw issue with PCIE_PE_RSTB signal. Add wait for the time needed to
1025	 * complete the PCIe reset.
1026	 */
1027	msleep(PCIE_T_PVPERL_MS);
1028
1029	return 0;
1030
1031err_clk_prepare_enable:
1032	pm_runtime_put_sync(dev);
1033	pm_runtime_disable(dev);
1034	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets,
1035				  pcie->phy_resets);
1036err_phy_deassert:
1037	phy_power_off(pcie->phy);
1038err_phy_on:
1039	phy_exit(pcie->phy);
1040
1041	return err;
1042}
1043
1044static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie)
1045{
1046	struct device *dev = pcie->dev;
1047	int err;
1048
1049	/*
1050	 * The controller may have been left out of reset by the bootloader
1051	 * so make sure that we get a clean start by asserting resets here.
1052	 */
1053	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets,
1054				  pcie->phy_resets);
1055	reset_control_assert(pcie->mac_reset);
1056	usleep_range(PCIE_MTK_RESET_TIME_US, 2 * PCIE_MTK_RESET_TIME_US);
1057
1058	/* PHY power on and enable pipe clock */
1059	err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets,
1060					  pcie->phy_resets);
1061	if (err) {
1062		dev_err(dev, "failed to deassert PHYs\n");
1063		return err;
1064	}
1065
1066	err = phy_init(pcie->phy);
1067	if (err) {
1068		dev_err(dev, "failed to initialize PHY\n");
1069		goto err_phy_init;
1070	}
1071
1072	err = phy_power_on(pcie->phy);
1073	if (err) {
1074		dev_err(dev, "failed to power on PHY\n");
1075		goto err_phy_on;
1076	}
1077
1078	/* MAC power on and enable transaction layer clocks */
1079	reset_control_deassert(pcie->mac_reset);
1080
1081	pm_runtime_enable(dev);
1082	pm_runtime_get_sync(dev);
1083
1084	err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks);
1085	if (err) {
1086		dev_err(dev, "failed to enable clocks\n");
1087		goto err_clk_init;
1088	}
1089
1090	return 0;
1091
1092err_clk_init:
1093	pm_runtime_put_sync(dev);
1094	pm_runtime_disable(dev);
1095	reset_control_assert(pcie->mac_reset);
1096	phy_power_off(pcie->phy);
1097err_phy_on:
1098	phy_exit(pcie->phy);
1099err_phy_init:
1100	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets,
1101				  pcie->phy_resets);
1102
1103	return err;
1104}
1105
1106static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie)
1107{
1108	clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks);
1109
1110	pm_runtime_put_sync(pcie->dev);
1111	pm_runtime_disable(pcie->dev);
1112	reset_control_assert(pcie->mac_reset);
1113
1114	phy_power_off(pcie->phy);
1115	phy_exit(pcie->phy);
1116	reset_control_bulk_assert(pcie->soc->phy_resets.num_resets,
1117				  pcie->phy_resets);
1118}
1119
1120static int mtk_pcie_get_controller_max_link_speed(struct mtk_gen3_pcie *pcie)
1121{
1122	u32 val;
1123	int ret;
1124
1125	val = readl_relaxed(pcie->base + PCIE_BASE_CFG_REG);
1126	val = FIELD_GET(PCIE_BASE_CFG_SPEED, val);
1127	ret = fls(val);
1128
1129	return ret > 0 ? ret : -EINVAL;
1130}
1131
1132static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie)
1133{
1134	int err, max_speed;
1135
1136	err = mtk_pcie_parse_port(pcie);
1137	if (err)
1138		return err;
1139
1140	/*
1141	 * Deassert the line in order to avoid unbalance in deassert_count
1142	 * counter since the bulk is shared.
1143	 */
1144	reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets,
1145				    pcie->phy_resets);
1146
1147	/* Don't touch the hardware registers before power up */
1148	err = pcie->soc->power_up(pcie);
1149	if (err)
1150		return err;
1151
1152	err = of_pci_get_max_link_speed(pcie->dev->of_node);
1153	if (err) {
1154		/* Get the maximum speed supported by the controller */
1155		max_speed = mtk_pcie_get_controller_max_link_speed(pcie);
1156
1157		/* Set max_link_speed only if the controller supports it */
1158		if (max_speed >= 0 && max_speed <= err) {
1159			pcie->max_link_speed = err;
1160			dev_info(pcie->dev,
1161				 "maximum controller link speed Gen%d, overriding to Gen%u",
1162				 max_speed, pcie->max_link_speed);
1163		}
1164	}
1165
1166	/* Try link up */
1167	err = mtk_pcie_startup_port(pcie);
1168	if (err)
1169		goto err_setup;
1170
1171	err = mtk_pcie_setup_irq(pcie);
1172	if (err)
1173		goto err_setup;
1174
1175	return 0;
1176
1177err_setup:
1178	mtk_pcie_power_down(pcie);
1179
1180	return err;
1181}
1182
1183static int mtk_pcie_probe(struct platform_device *pdev)
1184{
1185	struct device *dev = &pdev->dev;
1186	struct mtk_gen3_pcie *pcie;
1187	struct pci_host_bridge *host;
1188	int err;
1189
1190	host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
1191	if (!host)
1192		return -ENOMEM;
1193
1194	pcie = pci_host_bridge_priv(host);
1195
1196	pcie->dev = dev;
1197	pcie->soc = device_get_match_data(dev);
1198	platform_set_drvdata(pdev, pcie);
1199
1200	err = mtk_pcie_setup(pcie);
1201	if (err)
1202		return err;
1203
1204	host->ops = &mtk_pcie_ops;
1205	host->sysdata = pcie;
1206
1207	err = pci_host_probe(host);
1208	if (err) {
1209		mtk_pcie_irq_teardown(pcie);
1210		mtk_pcie_power_down(pcie);
1211		return err;
1212	}
1213
1214	return 0;
1215}
1216
1217static void mtk_pcie_remove(struct platform_device *pdev)
1218{
1219	struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev);
1220	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1221
1222	pci_lock_rescan_remove();
1223	pci_stop_root_bus(host->bus);
1224	pci_remove_root_bus(host->bus);
1225	pci_unlock_rescan_remove();
1226
1227	mtk_pcie_irq_teardown(pcie);
1228	mtk_pcie_power_down(pcie);
1229}
1230
1231static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie)
1232{
1233	int i;
1234
1235	raw_spin_lock(&pcie->irq_lock);
1236
1237	pcie->saved_irq_state = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);
1238
1239	for (i = 0; i < PCIE_MSI_SET_NUM; i++) {
1240		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];
1241
1242		msi_set->saved_irq_state = readl_relaxed(msi_set->base +
1243					   PCIE_MSI_SET_ENABLE_OFFSET);
1244	}
1245
1246	raw_spin_unlock(&pcie->irq_lock);
1247}
1248
1249static void mtk_pcie_irq_restore(struct mtk_gen3_pcie *pcie)
1250{
1251	int i;
1252
1253	raw_spin_lock(&pcie->irq_lock);
1254
1255	writel_relaxed(pcie->saved_irq_state, pcie->base + PCIE_INT_ENABLE_REG);
1256
1257	for (i = 0; i < PCIE_MSI_SET_NUM; i++) {
1258		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];
1259
1260		writel_relaxed(msi_set->saved_irq_state,
1261			       msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET);
1262	}
1263
1264	raw_spin_unlock(&pcie->irq_lock);
1265}
1266
1267static int mtk_pcie_turn_off_link(struct mtk_gen3_pcie *pcie)
1268{
1269	u32 val;
1270
1271	val = readl_relaxed(pcie->base + PCIE_ICMD_PM_REG);
1272	val |= PCIE_TURN_OFF_LINK;
1273	writel_relaxed(val, pcie->base + PCIE_ICMD_PM_REG);
1274
1275	/* Check the link is L2 */
1276	return readl_poll_timeout(pcie->base + PCIE_LTSSM_STATUS_REG, val,
1277				  (PCIE_LTSSM_STATE(val) ==
1278				   PCIE_LTSSM_STATE_L2_IDLE), 20,
1279				   50 * USEC_PER_MSEC);
1280}
1281
1282static int mtk_pcie_suspend_noirq(struct device *dev)
1283{
1284	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);
1285	int err;
1286	u32 val;
1287
1288	/* Trigger link to L2 state */
1289	err = mtk_pcie_turn_off_link(pcie);
1290	if (err) {
1291		dev_err(pcie->dev, "cannot enter L2 state\n");
1292		return err;
1293	}
1294
1295	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
1296		/* Assert the PERST# pin */
1297		val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);
1298		val |= PCIE_PE_RSTB;
1299		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
1300	}
1301
1302	dev_dbg(pcie->dev, "entered L2 states successfully");
1303
1304	mtk_pcie_irq_save(pcie);
1305	mtk_pcie_power_down(pcie);
1306
1307	return 0;
1308}
1309
1310static int mtk_pcie_resume_noirq(struct device *dev)
1311{
1312	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);
1313	int err;
1314
1315	err = pcie->soc->power_up(pcie);
1316	if (err)
1317		return err;
1318
1319	err = mtk_pcie_startup_port(pcie);
1320	if (err) {
1321		mtk_pcie_power_down(pcie);
1322		return err;
1323	}
1324
1325	mtk_pcie_irq_restore(pcie);
1326
1327	return 0;
1328}
1329
1330static const struct dev_pm_ops mtk_pcie_pm_ops = {
1331	NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
1332				  mtk_pcie_resume_noirq)
1333};
1334
1335static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8192 = {
1336	.power_up = mtk_pcie_power_up,
1337	.phy_resets = {
1338		.id[0] = "phy",
1339		.num_resets = 1,
1340	},
1341};
1342
1343static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8196 = {
1344	.power_up = mtk_pcie_power_up,
1345	.phy_resets = {
1346		.id[0] = "phy",
1347		.num_resets = 1,
1348	},
1349	.sys_clk_rdy_time_us = 10,
1350};
1351
1352static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_en7581 = {
1353	.power_up = mtk_pcie_en7581_power_up,
1354	.phy_resets = {
1355		.id[0] = "phy-lane0",
1356		.id[1] = "phy-lane1",
1357		.id[2] = "phy-lane2",
1358		.num_resets = 3,
1359	},
1360	.flags = SKIP_PCIE_RSTB,
1361};
1362
1363static const struct of_device_id mtk_pcie_of_match[] = {
1364	{ .compatible = "airoha,en7581-pcie", .data = &mtk_pcie_soc_en7581 },
1365	{ .compatible = "mediatek,mt8192-pcie", .data = &mtk_pcie_soc_mt8192 },
1366	{ .compatible = "mediatek,mt8196-pcie", .data = &mtk_pcie_soc_mt8196 },
1367	{},
1368};
1369MODULE_DEVICE_TABLE(of, mtk_pcie_of_match);
1370
1371static struct platform_driver mtk_pcie_driver = {
1372	.probe = mtk_pcie_probe,
1373	.remove = mtk_pcie_remove,
1374	.driver = {
1375		.name = "mtk-pcie-gen3",
1376		.of_match_table = mtk_pcie_of_match,
1377		.pm = &mtk_pcie_pm_ops,
1378		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
1379	},
1380};
1381
1382module_platform_driver(mtk_pcie_driver);
1383MODULE_DESCRIPTION("MediaTek Gen3 PCIe host controller driver");
1384MODULE_LICENSE("GPL v2");