pcie-mediatek.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * MediaTek PCIe host controller driver.
   4 *
   5 * Copyright (c) 2017 MediaTek Inc.
   6 * Author: Ryder Lee <ryder.lee@mediatek.com>
   7 *	   Honghui Zhang <honghui.zhang@mediatek.com>
   8 */
   9
  10#include <linux/clk.h>
  11#include <linux/delay.h>
  12#include <linux/iopoll.h>
  13#include <linux/irq.h>
  14#include <linux/irqchip/chained_irq.h>
  15#include <linux/irqchip/irq-msi-lib.h>
  16#include <linux/irqdomain.h>
  17#include <linux/kernel.h>
  18#include <linux/mfd/syscon.h>
  19#include <linux/msi.h>
  20#include <linux/module.h>
  21#include <linux/of_address.h>
  22#include <linux/of_pci.h>
  23#include <linux/of_platform.h>
  24#include <linux/pci.h>
  25#include <linux/phy/phy.h>
  26#include <linux/platform_device.h>
  27#include <linux/pm_runtime.h>
  28#include <linux/regmap.h>
  29#include <linux/reset.h>
  30
  31#include "../pci.h"
  32
  33/* PCIe shared registers */
  34#define PCIE_SYS_CFG		0x00
  35#define PCIE_INT_ENABLE		0x0c
  36#define PCIE_CFG_ADDR		0x20
  37#define PCIE_CFG_DATA		0x24
  38
  39/* PCIe per port registers */
  40#define PCIE_BAR0_SETUP		0x10
  41#define PCIE_CLASS		0x34
  42#define PCIE_LINK_STATUS	0x50
  43
  44#define PCIE_PORT_INT_EN(x)	BIT(20 + (x))
  45#define PCIE_PORT_PERST(x)	BIT(1 + (x))
  46#define PCIE_PORT_LINKUP	BIT(0)
  47#define PCIE_BAR_MAP_MAX	GENMASK(31, 16)
  48
  49#define PCIE_BAR_ENABLE		BIT(0)
  50#define PCIE_REVISION_ID	BIT(0)
  51#define PCIE_CLASS_CODE		(0x60400 << 8)
  52#define PCIE_CONF_REG(regn)	(((regn) & GENMASK(7, 2)) | \
  53				((((regn) >> 8) & GENMASK(3, 0)) << 24))
  54#define PCIE_CONF_FUN(fun)	(((fun) << 8) & GENMASK(10, 8))
  55#define PCIE_CONF_DEV(dev)	(((dev) << 11) & GENMASK(15, 11))
  56#define PCIE_CONF_BUS(bus)	(((bus) << 16) & GENMASK(23, 16))
  57#define PCIE_CONF_ADDR(regn, fun, dev, bus) \
  58	(PCIE_CONF_REG(regn) | PCIE_CONF_FUN(fun) | \
  59	 PCIE_CONF_DEV(dev) | PCIE_CONF_BUS(bus))
  60
  61/* MediaTek specific configuration registers */
  62#define PCIE_FTS_NUM		0x70c
  63#define PCIE_FTS_NUM_MASK	GENMASK(15, 8)
  64#define PCIE_FTS_NUM_L0(x)	((x) & 0xff << 8)
  65
  66#define PCIE_FC_CREDIT		0x73c
  67#define PCIE_FC_CREDIT_MASK	(GENMASK(31, 31) | GENMASK(28, 16))
  68#define PCIE_FC_CREDIT_VAL(x)	((x) << 16)
  69
  70/* PCIe V2 share registers */
  71#define PCIE_SYS_CFG_V2		0x0
  72#define PCIE_CSR_LTSSM_EN(x)	BIT(0 + (x) * 8)
  73#define PCIE_CSR_ASPM_L1_EN(x)	BIT(1 + (x) * 8)
  74
  75/* PCIe V2 per-port registers */
  76#define PCIE_MSI_VECTOR		0x0c0
  77
  78#define PCIE_CONF_VEND_ID	0x100
  79#define PCIE_CONF_DEVICE_ID	0x102
  80#define PCIE_CONF_CLASS_ID	0x106
  81
  82#define PCIE_INT_MASK		0x420
  83#define INTX_MASK		GENMASK(19, 16)
  84#define INTX_SHIFT		16
  85#define PCIE_INT_STATUS		0x424
  86#define MSI_STATUS		BIT(23)
  87#define PCIE_IMSI_STATUS	0x42c
  88#define PCIE_IMSI_ADDR		0x430
  89#define MSI_MASK		BIT(23)
  90#define MTK_MSI_IRQS_NUM	32
  91
  92#define PCIE_AHB_TRANS_BASE0_L	0x438
  93#define PCIE_AHB_TRANS_BASE0_H	0x43c
  94#define AHB2PCIE_SIZE(x)	((x) & GENMASK(4, 0))
  95#define PCIE_AXI_WINDOW0	0x448
  96#define WIN_ENABLE		BIT(7)
  97/*
  98 * Define PCIe to AHB window size as 2^33 to support max 8GB address space
  99 * translate, support least 4GB DRAM size access from EP DMA(physical DRAM
 100 * start from 0x40000000).
 101 */
 102#define PCIE2AHB_SIZE	0x21
 103
 104/* PCIe V2 configuration transaction header */
 105#define PCIE_CFG_HEADER0	0x460
 106#define PCIE_CFG_HEADER1	0x464
 107#define PCIE_CFG_HEADER2	0x468
 108#define PCIE_CFG_WDATA		0x470
 109#define PCIE_APP_TLP_REQ	0x488
 110#define PCIE_CFG_RDATA		0x48c
 111#define APP_CFG_REQ		BIT(0)
 112#define APP_CPL_STATUS		GENMASK(7, 5)
 113
 114#define CFG_WRRD_TYPE_0		4
 115#define CFG_WR_FMT		2
 116#define CFG_RD_FMT		0
 117
 118#define CFG_DW0_LENGTH(length)	((length) & GENMASK(9, 0))
 119#define CFG_DW0_TYPE(type)	(((type) << 24) & GENMASK(28, 24))
 120#define CFG_DW0_FMT(fmt)	(((fmt) << 29) & GENMASK(31, 29))
 121#define CFG_DW2_REGN(regn)	((regn) & GENMASK(11, 2))
 122#define CFG_DW2_FUN(fun)	(((fun) << 16) & GENMASK(18, 16))
 123#define CFG_DW2_DEV(dev)	(((dev) << 19) & GENMASK(23, 19))
 124#define CFG_DW2_BUS(bus)	(((bus) << 24) & GENMASK(31, 24))
 125#define CFG_HEADER_DW0(type, fmt) \
 126	(CFG_DW0_LENGTH(1) | CFG_DW0_TYPE(type) | CFG_DW0_FMT(fmt))
 127#define CFG_HEADER_DW1(where, size) \
 128	(GENMASK(((size) - 1), 0) << ((where) & 0x3))
 129#define CFG_HEADER_DW2(regn, fun, dev, bus) \
 130	(CFG_DW2_REGN(regn) | CFG_DW2_FUN(fun) | \
 131	CFG_DW2_DEV(dev) | CFG_DW2_BUS(bus))
 132
 133#define PCIE_RST_CTRL		0x510
 134#define PCIE_PHY_RSTB		BIT(0)
 135#define PCIE_PIPE_SRSTB		BIT(1)
 136#define PCIE_MAC_SRSTB		BIT(2)
 137#define PCIE_CRSTB		BIT(3)
 138#define PCIE_PERSTB		BIT(8)
 139#define PCIE_LINKDOWN_RST_EN	GENMASK(15, 13)
 140#define PCIE_LINK_STATUS_V2	0x804
 141#define PCIE_PORT_LINKUP_V2	BIT(10)
 142
 143struct mtk_pcie_port;
 144
 145/**
 146 * enum mtk_pcie_quirks - MTK PCIe quirks
 147 * @MTK_PCIE_FIX_CLASS_ID: host's class ID needed to be fixed
 148 * @MTK_PCIE_FIX_DEVICE_ID: host's device ID needed to be fixed
 149 * @MTK_PCIE_NO_MSI: Bridge has no MSI support, and relies on an external block
 150 * @MTK_PCIE_SKIP_RSTB: Skip calling RSTB bits on PCIe probe
 151 */
 152enum mtk_pcie_quirks {
 153	MTK_PCIE_FIX_CLASS_ID = BIT(0),
 154	MTK_PCIE_FIX_DEVICE_ID = BIT(1),
 155	MTK_PCIE_NO_MSI = BIT(2),
 156	MTK_PCIE_SKIP_RSTB = BIT(3),
 157};
 158
 159/**
 160 * struct mtk_pcie_soc - differentiate between host generations
 161 * @device_id: device ID which this host need to be fixed
 162 * @ops: pointer to configuration access functions
 163 * @startup: pointer to controller setting functions
 164 * @setup_irq: pointer to initialize IRQ functions
 165 * @quirks: PCIe device quirks.
 166 */
 167struct mtk_pcie_soc {
 168	unsigned int device_id;
 169	struct pci_ops *ops;
 170	int (*startup)(struct mtk_pcie_port *port);
 171	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
 172	enum mtk_pcie_quirks quirks;
 173};
 174
 175/**
 176 * struct mtk_pcie_port - PCIe port information
 177 * @base: IO mapped register base
 178 * @list: port list
 179 * @pcie: pointer to PCIe host info
 180 * @reset: pointer to port reset control
 181 * @sys_ck: pointer to transaction/data link layer clock
 182 * @ahb_ck: pointer to AHB slave interface operating clock for CSR access
 183 *          and RC initiated MMIO access
 184 * @axi_ck: pointer to application layer MMIO channel operating clock
 185 * @aux_ck: pointer to pe2_mac_bridge and pe2_mac_core operating clock
 186 *          when pcie_mac_ck/pcie_pipe_ck is turned off
 187 * @obff_ck: pointer to OBFF functional block operating clock
 188 * @pipe_ck: pointer to LTSSM and PHY/MAC layer operating clock
 189 * @phy: pointer to PHY control block
 190 * @slot: port slot
 191 * @irq: GIC irq
 192 * @irq_domain: legacy INTx IRQ domain
 193 * @inner_domain: inner IRQ domain
 194 * @lock: protect the msi_irq_in_use bitmap
 195 * @msi_irq_in_use: bit map for assigned MSI IRQ
 196 */
 197struct mtk_pcie_port {
 198	void __iomem *base;
 199	struct list_head list;
 200	struct mtk_pcie *pcie;
 201	struct reset_control *reset;
 202	struct clk *sys_ck;
 203	struct clk *ahb_ck;
 204	struct clk *axi_ck;
 205	struct clk *aux_ck;
 206	struct clk *obff_ck;
 207	struct clk *pipe_ck;
 208	struct phy *phy;
 209	u32 slot;
 210	int irq;
 211	struct irq_domain *irq_domain;
 212	struct irq_domain *inner_domain;
 213	struct mutex lock;
 214	DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM);
 215};
 216
 217/**
 218 * struct mtk_pcie - PCIe host information
 219 * @dev: pointer to PCIe device
 220 * @base: IO mapped register base
 221 * @cfg: IO mapped register map for PCIe config
 222 * @free_ck: free-run reference clock
 223 * @ports: pointer to PCIe port information
 224 * @soc: pointer to SoC-dependent operations
 225 */
 226struct mtk_pcie {
 227	struct device *dev;
 228	void __iomem *base;
 229	struct regmap *cfg;
 230	struct clk *free_ck;
 231
 232	struct list_head ports;
 233	const struct mtk_pcie_soc *soc;
 234};
 235
 236static void mtk_pcie_subsys_powerdown(struct mtk_pcie *pcie)
 237{
 238	struct device *dev = pcie->dev;
 239
 240	clk_disable_unprepare(pcie->free_ck);
 241
 242	pm_runtime_put_sync(dev);
 243	pm_runtime_disable(dev);
 244}
 245
 246static void mtk_pcie_port_free(struct mtk_pcie_port *port)
 247{
 248	struct mtk_pcie *pcie = port->pcie;
 249	struct device *dev = pcie->dev;
 250
 251	devm_iounmap(dev, port->base);
 252	list_del(&port->list);
 253	devm_kfree(dev, port);
 254}
 255
 256static void mtk_pcie_put_resources(struct mtk_pcie *pcie)
 257{
 258	struct mtk_pcie_port *port, *tmp;
 259
 260	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
 261		phy_power_off(port->phy);
 262		phy_exit(port->phy);
 263		clk_disable_unprepare(port->pipe_ck);
 264		clk_disable_unprepare(port->obff_ck);
 265		clk_disable_unprepare(port->axi_ck);
 266		clk_disable_unprepare(port->aux_ck);
 267		clk_disable_unprepare(port->ahb_ck);
 268		clk_disable_unprepare(port->sys_ck);
 269		mtk_pcie_port_free(port);
 270	}
 271
 272	mtk_pcie_subsys_powerdown(pcie);
 273}
 274
 275static int mtk_pcie_check_cfg_cpld(struct mtk_pcie_port *port)
 276{
 277	u32 val;
 278	int err;
 279
 280	err = readl_poll_timeout_atomic(port->base + PCIE_APP_TLP_REQ, val,
 281					!(val & APP_CFG_REQ), 10,
 282					100 * USEC_PER_MSEC);
 283	if (err)
 284		return PCIBIOS_SET_FAILED;
 285
 286	if (readl(port->base + PCIE_APP_TLP_REQ) & APP_CPL_STATUS)
 287		return PCIBIOS_SET_FAILED;
 288
 289	return PCIBIOS_SUCCESSFUL;
 290}
 291
 292static int mtk_pcie_hw_rd_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
 293			      int where, int size, u32 *val)
 294{
 295	u32 tmp;
 296
 297	/* Write PCIe configuration transaction header for Cfgrd */
 298	writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_RD_FMT),
 299	       port->base + PCIE_CFG_HEADER0);
 300	writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
 301	writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
 302	       port->base + PCIE_CFG_HEADER2);
 303
 304	/* Trigger h/w to transmit Cfgrd TLP */
 305	tmp = readl(port->base + PCIE_APP_TLP_REQ);
 306	tmp |= APP_CFG_REQ;
 307	writel(tmp, port->base + PCIE_APP_TLP_REQ);
 308
 309	/* Check completion status */
 310	if (mtk_pcie_check_cfg_cpld(port))
 311		return PCIBIOS_SET_FAILED;
 312
 313	/* Read cpld payload of Cfgrd */
 314	*val = readl(port->base + PCIE_CFG_RDATA);
 315
 316	if (size == 1)
 317		*val = (*val >> (8 * (where & 3))) & 0xff;
 318	else if (size == 2)
 319		*val = (*val >> (8 * (where & 3))) & 0xffff;
 320
 321	return PCIBIOS_SUCCESSFUL;
 322}
 323
 324static int mtk_pcie_hw_wr_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
 325			      int where, int size, u32 val)
 326{
 327	/* Write PCIe configuration transaction header for Cfgwr */
 328	writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_WR_FMT),
 329	       port->base + PCIE_CFG_HEADER0);
 330	writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
 331	writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
 332	       port->base + PCIE_CFG_HEADER2);
 333
 334	/* Write Cfgwr data */
 335	val = val << 8 * (where & 3);
 336	writel(val, port->base + PCIE_CFG_WDATA);
 337
 338	/* Trigger h/w to transmit Cfgwr TLP */
 339	val = readl(port->base + PCIE_APP_TLP_REQ);
 340	val |= APP_CFG_REQ;
 341	writel(val, port->base + PCIE_APP_TLP_REQ);
 342
 343	/* Check completion status */
 344	return mtk_pcie_check_cfg_cpld(port);
 345}
 346
 347static struct mtk_pcie_port *mtk_pcie_find_port(struct pci_bus *bus,
 348						unsigned int devfn)
 349{
 350	struct mtk_pcie *pcie = bus->sysdata;
 351	struct mtk_pcie_port *port;
 352	struct pci_dev *dev = NULL;
 353
 354	/*
 355	 * Walk the bus hierarchy to get the devfn value
 356	 * of the port in the root bus.
 357	 */
 358	while (bus && bus->number) {
 359		dev = bus->self;
 360		bus = dev->bus;
 361		devfn = dev->devfn;
 362	}
 363
 364	list_for_each_entry(port, &pcie->ports, list)
 365		if (port->slot == PCI_SLOT(devfn))
 366			return port;
 367
 368	return NULL;
 369}
 370
 371static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
 372				int where, int size, u32 *val)
 373{
 374	struct mtk_pcie_port *port;
 375	u32 bn = bus->number;
 376
 377	port = mtk_pcie_find_port(bus, devfn);
 378	if (!port)
 379		return PCIBIOS_DEVICE_NOT_FOUND;
 380
 381	return mtk_pcie_hw_rd_cfg(port, bn, devfn, where, size, val);
 382}
 383
 384static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
 385				 int where, int size, u32 val)
 386{
 387	struct mtk_pcie_port *port;
 388	u32 bn = bus->number;
 389
 390	port = mtk_pcie_find_port(bus, devfn);
 391	if (!port)
 392		return PCIBIOS_DEVICE_NOT_FOUND;
 393
 394	return mtk_pcie_hw_wr_cfg(port, bn, devfn, where, size, val);
 395}
 396
 397static struct pci_ops mtk_pcie_ops_v2 = {
 398	.read  = mtk_pcie_config_read,
 399	.write = mtk_pcie_config_write,
 400};
 401
 402static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 403{
 404	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
 405	phys_addr_t addr;
 406
 407	/* MT2712/MT7622 only support 32-bit MSI addresses */
 408	addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
 409	msg->address_hi = 0;
 410	msg->address_lo = lower_32_bits(addr);
 411
 412	msg->data = data->hwirq;
 413
 414	dev_dbg(port->pcie->dev, "msi#%d address_hi %#x address_lo %#x\n",
 415		(int)data->hwirq, msg->address_hi, msg->address_lo);
 416}
 417
 418static void mtk_msi_ack_irq(struct irq_data *data)
 419{
 420	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
 421	u32 hwirq = data->hwirq;
 422
 423	writel(1 << hwirq, port->base + PCIE_IMSI_STATUS);
 424}
 425
 426static struct irq_chip mtk_msi_bottom_irq_chip = {
 427	.name			= "MTK MSI",
 428	.irq_compose_msi_msg	= mtk_compose_msi_msg,
 429	.irq_ack		= mtk_msi_ack_irq,
 430};
 431
 432static int mtk_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 433				     unsigned int nr_irqs, void *args)
 434{
 435	struct mtk_pcie_port *port = domain->host_data;
 436	unsigned long bit;
 437
 438	WARN_ON(nr_irqs != 1);
 439	mutex_lock(&port->lock);
 440
 441	bit = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM);
 442	if (bit >= MTK_MSI_IRQS_NUM) {
 443		mutex_unlock(&port->lock);
 444		return -ENOSPC;
 445	}
 446
 447	__set_bit(bit, port->msi_irq_in_use);
 448
 449	mutex_unlock(&port->lock);
 450
 451	irq_domain_set_info(domain, virq, bit, &mtk_msi_bottom_irq_chip,
 452			    domain->host_data, handle_edge_irq,
 453			    NULL, NULL);
 454
 455	return 0;
 456}
 457
 458static void mtk_pcie_irq_domain_free(struct irq_domain *domain,
 459				     unsigned int virq, unsigned int nr_irqs)
 460{
 461	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 462	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(d);
 463
 464	mutex_lock(&port->lock);
 465
 466	if (!test_bit(d->hwirq, port->msi_irq_in_use))
 467		dev_err(port->pcie->dev, "trying to free unused MSI#%lu\n",
 468			d->hwirq);
 469	else
 470		__clear_bit(d->hwirq, port->msi_irq_in_use);
 471
 472	mutex_unlock(&port->lock);
 473
 474	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
 475}
 476
 477static const struct irq_domain_ops msi_domain_ops = {
 478	.alloc	= mtk_pcie_irq_domain_alloc,
 479	.free	= mtk_pcie_irq_domain_free,
 480};
 481
 482#define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
 483				MSI_FLAG_USE_DEF_CHIP_OPS	| \
 484				MSI_FLAG_NO_AFFINITY)
 485
 486#define MTK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
 487				 MSI_FLAG_PCI_MSIX)
 488
 489static const struct msi_parent_ops mtk_msi_parent_ops = {
 490	.required_flags		= MTK_MSI_FLAGS_REQUIRED,
 491	.supported_flags	= MTK_MSI_FLAGS_SUPPORTED,
 492	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
 493	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
 494	.prefix			= "MTK-",
 495	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 496};
 497
 498static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port)
 499{
 500	mutex_init(&port->lock);
 501
 502	struct irq_domain_info info = {
 503		.fwnode		= dev_fwnode(port->pcie->dev),
 504		.ops		= &msi_domain_ops,
 505		.host_data	= port,
 506		.size		= MTK_MSI_IRQS_NUM,
 507	};
 508
 509	port->inner_domain = msi_create_parent_irq_domain(&info, &mtk_msi_parent_ops);
 510	if (!port->inner_domain) {
 511		dev_err(port->pcie->dev, "failed to create IRQ domain\n");
 512		return -ENOMEM;
 513	}
 514
 515	return 0;
 516}
 517
 518static void mtk_pcie_enable_msi(struct mtk_pcie_port *port)
 519{
 520	u32 val;
 521	phys_addr_t msg_addr;
 522
 523	msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
 524	val = lower_32_bits(msg_addr);
 525	writel(val, port->base + PCIE_IMSI_ADDR);
 526
 527	val = readl(port->base + PCIE_INT_MASK);
 528	val &= ~MSI_MASK;
 529	writel(val, port->base + PCIE_INT_MASK);
 530}
 531
 532static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie)
 533{
 534	struct mtk_pcie_port *port, *tmp;
 535
 536	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
 537		irq_set_chained_handler_and_data(port->irq, NULL, NULL);
 538
 539		if (port->irq_domain)
 540			irq_domain_remove(port->irq_domain);
 541
 542		if (IS_ENABLED(CONFIG_PCI_MSI)) {
 543			if (port->inner_domain)
 544				irq_domain_remove(port->inner_domain);
 545		}
 546
 547		irq_dispose_mapping(port->irq);
 548	}
 549}
 550
 551static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 552			     irq_hw_number_t hwirq)
 553{
 554	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
 555	irq_set_chip_data(irq, domain->host_data);
 556
 557	return 0;
 558}
 559
 560static const struct irq_domain_ops intx_domain_ops = {
 561	.map = mtk_pcie_intx_map,
 562};
 563
 564static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
 565				    struct device_node *node)
 566{
 567	struct device *dev = port->pcie->dev;
 568	struct device_node *pcie_intc_node;
 569	int ret;
 570
 571	/* Setup INTx */
 572	pcie_intc_node = of_get_next_child(node, NULL);
 573	if (!pcie_intc_node) {
 574		dev_err(dev, "no PCIe Intc node found\n");
 575		return -ENODEV;
 576	}
 577
 578	port->irq_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX,
 579						    &intx_domain_ops, port);
 580	of_node_put(pcie_intc_node);
 581	if (!port->irq_domain) {
 582		dev_err(dev, "failed to get INTx IRQ domain\n");
 583		return -ENODEV;
 584	}
 585
 586	if (IS_ENABLED(CONFIG_PCI_MSI)) {
 587		ret = mtk_pcie_allocate_msi_domains(port);
 588		if (ret) {
 589			irq_domain_remove(port->irq_domain);
 590			return ret;
 591		}
 592	}
 593
 594	return 0;
 595}
 596
 597static void mtk_pcie_intr_handler(struct irq_desc *desc)
 598{
 599	struct mtk_pcie_port *port = irq_desc_get_handler_data(desc);
 600	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 601	unsigned long status;
 602	u32 bit = INTX_SHIFT;
 603
 604	chained_irq_enter(irqchip, desc);
 605
 606	status = readl(port->base + PCIE_INT_STATUS);
 607	if (status & INTX_MASK) {
 608		for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) {
 609			/* Clear the INTx */
 610			writel(1 << bit, port->base + PCIE_INT_STATUS);
 611			generic_handle_domain_irq(port->irq_domain,
 612						  bit - INTX_SHIFT);
 613		}
 614	}
 615
 616	if (IS_ENABLED(CONFIG_PCI_MSI)) {
 617		if (status & MSI_STATUS){
 618			unsigned long imsi_status;
 619
 620			/*
 621			 * The interrupt status can be cleared even if the
 622			 * MSI status remains pending. As such, given the
 623			 * edge-triggered interrupt type, its status should
 624			 * be cleared before being dispatched to the
 625			 * handler of the underlying device.
 626			 */
 627			writel(MSI_STATUS, port->base + PCIE_INT_STATUS);
 628			while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) {
 629				for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM)
 630					generic_handle_domain_irq(port->inner_domain, bit);
 631			}
 632		}
 633	}
 634
 635	chained_irq_exit(irqchip, desc);
 636}
 637
 638static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
 639			      struct device_node *node)
 640{
 641	struct mtk_pcie *pcie = port->pcie;
 642	struct device *dev = pcie->dev;
 643	struct platform_device *pdev = to_platform_device(dev);
 644	int err;
 645
 646	err = mtk_pcie_init_irq_domain(port, node);
 647	if (err) {
 648		dev_err(dev, "failed to init PCIe IRQ domain\n");
 649		return err;
 650	}
 651
 652	if (of_property_present(dev->of_node, "interrupt-names"))
 653		port->irq = platform_get_irq_byname(pdev, "pcie_irq");
 654	else
 655		port->irq = platform_get_irq(pdev, port->slot);
 656
 657	if (port->irq < 0)
 658		return port->irq;
 659
 660	irq_set_chained_handler_and_data(port->irq,
 661					 mtk_pcie_intr_handler, port);
 662
 663	return 0;
 664}
 665
 666static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
 667{
 668	struct mtk_pcie *pcie = port->pcie;
 669	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
 670	struct resource *mem = NULL;
 671	struct resource_entry *entry;
 672	const struct mtk_pcie_soc *soc = port->pcie->soc;
 673	u32 val;
 674	int err;
 675
 676	entry = resource_list_first_type(&host->windows, IORESOURCE_MEM);
 677	if (entry)
 678		mem = entry->res;
 679	if (!mem)
 680		return -EINVAL;
 681
 682	/* MT7622 platforms need to enable LTSSM and ASPM from PCIe subsys */
 683	if (pcie->base) {
 684		val = readl(pcie->base + PCIE_SYS_CFG_V2);
 685		val |= PCIE_CSR_LTSSM_EN(port->slot) |
 686		       PCIE_CSR_ASPM_L1_EN(port->slot);
 687		writel(val, pcie->base + PCIE_SYS_CFG_V2);
 688	} else if (pcie->cfg) {
 689		val = PCIE_CSR_LTSSM_EN(port->slot) |
 690		      PCIE_CSR_ASPM_L1_EN(port->slot);
 691		regmap_update_bits(pcie->cfg, PCIE_SYS_CFG_V2, val, val);
 692	}
 693
 694	if (!(soc->quirks & MTK_PCIE_SKIP_RSTB)) {
 695		/* Assert all reset signals */
 696		writel(0, port->base + PCIE_RST_CTRL);
 697
 698		/*
 699		 * Enable PCIe link down reset, if link status changed from
 700		 * link up to link down, this will reset MAC control registers
 701		 * and configuration space.
 702		 */
 703		writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL);
 704
 705		msleep(PCIE_T_PVPERL_MS);
 706
 707		/* De-assert PHY, PE, PIPE, MAC and configuration reset	*/
 708		val = readl(port->base + PCIE_RST_CTRL);
 709		val |= PCIE_PHY_RSTB | PCIE_PERSTB | PCIE_PIPE_SRSTB |
 710		       PCIE_MAC_SRSTB | PCIE_CRSTB;
 711		writel(val, port->base + PCIE_RST_CTRL);
 712	}
 713
 714	/* Set up vendor ID and class code */
 715	if (soc->quirks & MTK_PCIE_FIX_CLASS_ID) {
 716		val = PCI_VENDOR_ID_MEDIATEK;
 717		writew(val, port->base + PCIE_CONF_VEND_ID);
 718
 719		val = PCI_CLASS_BRIDGE_PCI;
 720		writew(val, port->base + PCIE_CONF_CLASS_ID);
 721	}
 722
 723	if (soc->quirks & MTK_PCIE_FIX_DEVICE_ID)
 724		writew(soc->device_id, port->base + PCIE_CONF_DEVICE_ID);
 725
 726	/* 100ms timeout value should be enough for Gen1/2 training */
 727	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val,
 728				 !!(val & PCIE_PORT_LINKUP_V2), 20,
 729				 100 * USEC_PER_MSEC);
 730	if (err)
 731		return -ETIMEDOUT;
 732
 733	/* Set INTx mask */
 734	val = readl(port->base + PCIE_INT_MASK);
 735	val &= ~INTX_MASK;
 736	writel(val, port->base + PCIE_INT_MASK);
 737
 738	if (IS_ENABLED(CONFIG_PCI_MSI))
 739		mtk_pcie_enable_msi(port);
 740
 741	/* Set AHB to PCIe translation windows */
 742	val = lower_32_bits(mem->start) |
 743	      AHB2PCIE_SIZE(fls(resource_size(mem)));
 744	writel(val, port->base + PCIE_AHB_TRANS_BASE0_L);
 745
 746	val = upper_32_bits(mem->start);
 747	writel(val, port->base + PCIE_AHB_TRANS_BASE0_H);
 748
 749	/* Set PCIe to AXI translation memory space.*/
 750	val = PCIE2AHB_SIZE | WIN_ENABLE;
 751	writel(val, port->base + PCIE_AXI_WINDOW0);
 752
 753	return 0;
 754}
 755
 756static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus,
 757				      unsigned int devfn, int where)
 758{
 759	struct mtk_pcie *pcie = bus->sysdata;
 760
 761	writel(PCIE_CONF_ADDR(where, PCI_FUNC(devfn), PCI_SLOT(devfn),
 762			      bus->number), pcie->base + PCIE_CFG_ADDR);
 763
 764	return pcie->base + PCIE_CFG_DATA + (where & 3);
 765}
 766
 767static struct pci_ops mtk_pcie_ops = {
 768	.map_bus = mtk_pcie_map_bus,
 769	.read  = pci_generic_config_read,
 770	.write = pci_generic_config_write,
 771};
 772
 773static int mtk_pcie_startup_port(struct mtk_pcie_port *port)
 774{
 775	struct mtk_pcie *pcie = port->pcie;
 776	u32 func = PCI_FUNC(port->slot);
 777	u32 slot = PCI_SLOT(port->slot << 3);
 778	u32 val;
 779	int err;
 780
 781	/* assert port PERST_N */
 782	val = readl(pcie->base + PCIE_SYS_CFG);
 783	val |= PCIE_PORT_PERST(port->slot);
 784	writel(val, pcie->base + PCIE_SYS_CFG);
 785
 786	/* de-assert port PERST_N */
 787	val = readl(pcie->base + PCIE_SYS_CFG);
 788	val &= ~PCIE_PORT_PERST(port->slot);
 789	writel(val, pcie->base + PCIE_SYS_CFG);
 790
 791	/* 100ms timeout value should be enough for Gen1/2 training */
 792	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS, val,
 793				 !!(val & PCIE_PORT_LINKUP), 20,
 794				 100 * USEC_PER_MSEC);
 795	if (err)
 796		return -ETIMEDOUT;
 797
 798	/* enable interrupt */
 799	val = readl(pcie->base + PCIE_INT_ENABLE);
 800	val |= PCIE_PORT_INT_EN(port->slot);
 801	writel(val, pcie->base + PCIE_INT_ENABLE);
 802
 803	/* map to all DDR region. We need to set it before cfg operation. */
 804	writel(PCIE_BAR_MAP_MAX | PCIE_BAR_ENABLE,
 805	       port->base + PCIE_BAR0_SETUP);
 806
 807	/* configure class code and revision ID */
 808	writel(PCIE_CLASS_CODE | PCIE_REVISION_ID, port->base + PCIE_CLASS);
 809
 810	/* configure FC credit */
 811	writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0),
 812	       pcie->base + PCIE_CFG_ADDR);
 813	val = readl(pcie->base + PCIE_CFG_DATA);
 814	val &= ~PCIE_FC_CREDIT_MASK;
 815	val |= PCIE_FC_CREDIT_VAL(0x806c);
 816	writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0),
 817	       pcie->base + PCIE_CFG_ADDR);
 818	writel(val, pcie->base + PCIE_CFG_DATA);
 819
 820	/* configure RC FTS number to 250 when it leaves L0s */
 821	writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
 822	       pcie->base + PCIE_CFG_ADDR);
 823	val = readl(pcie->base + PCIE_CFG_DATA);
 824	val &= ~PCIE_FTS_NUM_MASK;
 825	val |= PCIE_FTS_NUM_L0(0x50);
 826	writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
 827	       pcie->base + PCIE_CFG_ADDR);
 828	writel(val, pcie->base + PCIE_CFG_DATA);
 829
 830	return 0;
 831}
 832
 833static int mtk_pcie_startup_port_an7583(struct mtk_pcie_port *port)
 834{
 835	struct mtk_pcie *pcie = port->pcie;
 836	struct device *dev = pcie->dev;
 837	struct pci_host_bridge *host;
 838	struct resource_entry *entry;
 839	struct regmap *pbus_regmap;
 840	resource_size_t addr;
 841	u32 args[2], size;
 842
 843	/*
 844	 * Configure PBus base address and base address mask to allow
 845	 * the hw to detect if a given address is accessible on PCIe
 846	 * controller.
 847	 */
 848	pbus_regmap = syscon_regmap_lookup_by_phandle_args(dev->of_node,
 849							   "mediatek,pbus-csr",
 850							   ARRAY_SIZE(args),
 851							   args);
 852	if (IS_ERR(pbus_regmap))
 853		return PTR_ERR(pbus_regmap);
 854
 855	host = pci_host_bridge_from_priv(pcie);
 856	entry = resource_list_first_type(&host->windows, IORESOURCE_MEM);
 857	if (!entry)
 858		return -ENODEV;
 859
 860	addr = entry->res->start - entry->offset;
 861	regmap_write(pbus_regmap, args[0], lower_32_bits(addr));
 862	size = lower_32_bits(resource_size(entry->res));
 863	regmap_write(pbus_regmap, args[1], GENMASK(31, __fls(size)));
 864
 865	return mtk_pcie_startup_port_v2(port);
 866}
 867
 868static void mtk_pcie_enable_port(struct mtk_pcie_port *port)
 869{
 870	struct mtk_pcie *pcie = port->pcie;
 871	struct device *dev = pcie->dev;
 872	int err;
 873
 874	err = clk_prepare_enable(port->sys_ck);
 875	if (err) {
 876		dev_err(dev, "failed to enable sys_ck%d clock\n", port->slot);
 877		goto err_sys_clk;
 878	}
 879
 880	err = clk_prepare_enable(port->ahb_ck);
 881	if (err) {
 882		dev_err(dev, "failed to enable ahb_ck%d\n", port->slot);
 883		goto err_ahb_clk;
 884	}
 885
 886	err = clk_prepare_enable(port->aux_ck);
 887	if (err) {
 888		dev_err(dev, "failed to enable aux_ck%d\n", port->slot);
 889		goto err_aux_clk;
 890	}
 891
 892	err = clk_prepare_enable(port->axi_ck);
 893	if (err) {
 894		dev_err(dev, "failed to enable axi_ck%d\n", port->slot);
 895		goto err_axi_clk;
 896	}
 897
 898	err = clk_prepare_enable(port->obff_ck);
 899	if (err) {
 900		dev_err(dev, "failed to enable obff_ck%d\n", port->slot);
 901		goto err_obff_clk;
 902	}
 903
 904	err = clk_prepare_enable(port->pipe_ck);
 905	if (err) {
 906		dev_err(dev, "failed to enable pipe_ck%d\n", port->slot);
 907		goto err_pipe_clk;
 908	}
 909
 910	reset_control_assert(port->reset);
 911	reset_control_deassert(port->reset);
 912
 913	err = phy_init(port->phy);
 914	if (err) {
 915		dev_err(dev, "failed to initialize port%d phy\n", port->slot);
 916		goto err_phy_init;
 917	}
 918
 919	err = phy_power_on(port->phy);
 920	if (err) {
 921		dev_err(dev, "failed to power on port%d phy\n", port->slot);
 922		goto err_phy_on;
 923	}
 924
 925	if (!pcie->soc->startup(port))
 926		return;
 927
 928	dev_info(dev, "Port%d link down\n", port->slot);
 929
 930	phy_power_off(port->phy);
 931err_phy_on:
 932	phy_exit(port->phy);
 933err_phy_init:
 934	clk_disable_unprepare(port->pipe_ck);
 935err_pipe_clk:
 936	clk_disable_unprepare(port->obff_ck);
 937err_obff_clk:
 938	clk_disable_unprepare(port->axi_ck);
 939err_axi_clk:
 940	clk_disable_unprepare(port->aux_ck);
 941err_aux_clk:
 942	clk_disable_unprepare(port->ahb_ck);
 943err_ahb_clk:
 944	clk_disable_unprepare(port->sys_ck);
 945err_sys_clk:
 946	mtk_pcie_port_free(port);
 947}
 948
 949static int mtk_pcie_parse_port(struct mtk_pcie *pcie,
 950			       struct device_node *node,
 951			       int slot)
 952{
 953	struct mtk_pcie_port *port;
 954	struct device *dev = pcie->dev;
 955	struct platform_device *pdev = to_platform_device(dev);
 956	char name[10];
 957	int err;
 958
 959	port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
 960	if (!port)
 961		return -ENOMEM;
 962
 963	snprintf(name, sizeof(name), "port%d", slot);
 964	port->base = devm_platform_ioremap_resource_byname(pdev, name);
 965	if (IS_ERR(port->base)) {
 966		dev_err(dev, "failed to map port%d base\n", slot);
 967		return PTR_ERR(port->base);
 968	}
 969
 970	snprintf(name, sizeof(name), "sys_ck%d", slot);
 971	port->sys_ck = devm_clk_get(dev, name);
 972	if (IS_ERR(port->sys_ck)) {
 973		dev_err(dev, "failed to get sys_ck%d clock\n", slot);
 974		return PTR_ERR(port->sys_ck);
 975	}
 976
 977	/* sys_ck might be divided into the following parts in some chips */
 978	snprintf(name, sizeof(name), "ahb_ck%d", slot);
 979	port->ahb_ck = devm_clk_get_optional(dev, name);
 980	if (IS_ERR(port->ahb_ck))
 981		return PTR_ERR(port->ahb_ck);
 982
 983	snprintf(name, sizeof(name), "axi_ck%d", slot);
 984	port->axi_ck = devm_clk_get_optional(dev, name);
 985	if (IS_ERR(port->axi_ck))
 986		return PTR_ERR(port->axi_ck);
 987
 988	snprintf(name, sizeof(name), "aux_ck%d", slot);
 989	port->aux_ck = devm_clk_get_optional(dev, name);
 990	if (IS_ERR(port->aux_ck))
 991		return PTR_ERR(port->aux_ck);
 992
 993	snprintf(name, sizeof(name), "obff_ck%d", slot);
 994	port->obff_ck = devm_clk_get_optional(dev, name);
 995	if (IS_ERR(port->obff_ck))
 996		return PTR_ERR(port->obff_ck);
 997
 998	snprintf(name, sizeof(name), "pipe_ck%d", slot);
 999	port->pipe_ck = devm_clk_get_optional(dev, name);
1000	if (IS_ERR(port->pipe_ck))
1001		return PTR_ERR(port->pipe_ck);
1002
1003	snprintf(name, sizeof(name), "pcie-rst%d", slot);
1004	port->reset = devm_reset_control_get_optional_exclusive(dev, name);
1005	if (PTR_ERR(port->reset) == -EPROBE_DEFER)
1006		return PTR_ERR(port->reset);
1007
1008	/* some platforms may use default PHY setting */
1009	snprintf(name, sizeof(name), "pcie-phy%d", slot);
1010	port->phy = devm_phy_optional_get(dev, name);
1011	if (IS_ERR(port->phy))
1012		return PTR_ERR(port->phy);
1013
1014	port->slot = slot;
1015	port->pcie = pcie;
1016
1017	if (pcie->soc->setup_irq) {
1018		err = pcie->soc->setup_irq(port, node);
1019		if (err)
1020			return err;
1021	}
1022
1023	INIT_LIST_HEAD(&port->list);
1024	list_add_tail(&port->list, &pcie->ports);
1025
1026	return 0;
1027}
1028
1029static int mtk_pcie_subsys_powerup(struct mtk_pcie *pcie)
1030{
1031	struct device *dev = pcie->dev;
1032	struct platform_device *pdev = to_platform_device(dev);
1033	struct resource *regs;
1034	struct device_node *cfg_node;
1035	int err;
1036
1037	/* get shared registers, which are optional */
1038	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "subsys");
1039	if (regs) {
1040		pcie->base = devm_ioremap_resource(dev, regs);
1041		if (IS_ERR(pcie->base))
1042			return PTR_ERR(pcie->base);
1043	}
1044
1045	cfg_node = of_find_compatible_node(NULL, NULL,
1046					   "mediatek,generic-pciecfg");
1047	if (cfg_node) {
1048		pcie->cfg = syscon_node_to_regmap(cfg_node);
1049		of_node_put(cfg_node);
1050		if (IS_ERR(pcie->cfg))
1051			return PTR_ERR(pcie->cfg);
1052	}
1053
1054	pcie->free_ck = devm_clk_get(dev, "free_ck");
1055	if (IS_ERR(pcie->free_ck)) {
1056		if (PTR_ERR(pcie->free_ck) == -EPROBE_DEFER)
1057			return -EPROBE_DEFER;
1058
1059		pcie->free_ck = NULL;
1060	}
1061
1062	pm_runtime_enable(dev);
1063	pm_runtime_get_sync(dev);
1064
1065	/* enable top level clock */
1066	err = clk_prepare_enable(pcie->free_ck);
1067	if (err) {
1068		dev_err(dev, "failed to enable free_ck\n");
1069		goto err_free_ck;
1070	}
1071
1072	return 0;
1073
1074err_free_ck:
1075	pm_runtime_put_sync(dev);
1076	pm_runtime_disable(dev);
1077
1078	return err;
1079}
1080
1081static int mtk_pcie_setup(struct mtk_pcie *pcie)
1082{
1083	struct device *dev = pcie->dev;
1084	struct device_node *node = dev->of_node;
1085	struct mtk_pcie_port *port, *tmp;
1086	int err, slot;
1087
1088	slot = of_get_pci_domain_nr(dev->of_node);
1089	if (slot < 0) {
1090		for_each_available_child_of_node_scoped(node, child) {
1091			err = of_pci_get_devfn(child);
1092			if (err < 0)
1093				return dev_err_probe(dev, err, "failed to get devfn\n");
1094
1095			slot = PCI_SLOT(err);
1096
1097			err = mtk_pcie_parse_port(pcie, child, slot);
1098			if (err)
1099				return err;
1100		}
1101	} else {
1102		err = mtk_pcie_parse_port(pcie, node, slot);
1103		if (err)
1104			return err;
1105	}
1106
1107	err = mtk_pcie_subsys_powerup(pcie);
1108	if (err)
1109		return err;
1110
1111	/* enable each port, and then check link status */
1112	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
1113		mtk_pcie_enable_port(port);
1114
1115	/* power down PCIe subsys if slots are all empty (link down) */
1116	if (list_empty(&pcie->ports))
1117		mtk_pcie_subsys_powerdown(pcie);
1118
1119	return 0;
1120}
1121
1122static int mtk_pcie_probe(struct platform_device *pdev)
1123{
1124	struct device *dev = &pdev->dev;
1125	struct mtk_pcie *pcie;
1126	struct pci_host_bridge *host;
1127	int err;
1128
1129	host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
1130	if (!host)
1131		return -ENOMEM;
1132
1133	pcie = pci_host_bridge_priv(host);
1134
1135	pcie->dev = dev;
1136	pcie->soc = of_device_get_match_data(dev);
1137	platform_set_drvdata(pdev, pcie);
1138	INIT_LIST_HEAD(&pcie->ports);
1139
1140	err = mtk_pcie_setup(pcie);
1141	if (err)
1142		return err;
1143
1144	host->ops = pcie->soc->ops;
1145	host->sysdata = pcie;
1146	host->msi_domain = !!(pcie->soc->quirks & MTK_PCIE_NO_MSI);
1147
1148	err = pci_host_probe(host);
1149	if (err)
1150		goto put_resources;
1151
1152	return 0;
1153
1154put_resources:
1155	if (!list_empty(&pcie->ports))
1156		mtk_pcie_put_resources(pcie);
1157
1158	return err;
1159}
1160
1161
1162static void mtk_pcie_free_resources(struct mtk_pcie *pcie)
1163{
1164	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1165	struct list_head *windows = &host->windows;
1166
1167	pci_free_resource_list(windows);
1168}
1169
1170static void mtk_pcie_remove(struct platform_device *pdev)
1171{
1172	struct mtk_pcie *pcie = platform_get_drvdata(pdev);
1173	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
1174
1175	pci_stop_root_bus(host->bus);
1176	pci_remove_root_bus(host->bus);
1177	mtk_pcie_free_resources(pcie);
1178
1179	mtk_pcie_irq_teardown(pcie);
1180
1181	mtk_pcie_put_resources(pcie);
1182}
1183
1184static int mtk_pcie_suspend_noirq(struct device *dev)
1185{
1186	struct mtk_pcie *pcie = dev_get_drvdata(dev);
1187	struct mtk_pcie_port *port;
1188
1189	if (list_empty(&pcie->ports))
1190		return 0;
1191
1192	list_for_each_entry(port, &pcie->ports, list) {
1193		clk_disable_unprepare(port->pipe_ck);
1194		clk_disable_unprepare(port->obff_ck);
1195		clk_disable_unprepare(port->axi_ck);
1196		clk_disable_unprepare(port->aux_ck);
1197		clk_disable_unprepare(port->ahb_ck);
1198		clk_disable_unprepare(port->sys_ck);
1199		phy_power_off(port->phy);
1200		phy_exit(port->phy);
1201	}
1202
1203	clk_disable_unprepare(pcie->free_ck);
1204
1205	return 0;
1206}
1207
1208static int mtk_pcie_resume_noirq(struct device *dev)
1209{
1210	struct mtk_pcie *pcie = dev_get_drvdata(dev);
1211	struct mtk_pcie_port *port, *tmp;
1212
1213	if (list_empty(&pcie->ports))
1214		return 0;
1215
1216	clk_prepare_enable(pcie->free_ck);
1217
1218	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
1219		mtk_pcie_enable_port(port);
1220
1221	/* In case of EP was removed while system suspend. */
1222	if (list_empty(&pcie->ports))
1223		clk_disable_unprepare(pcie->free_ck);
1224
1225	return 0;
1226}
1227
1228static const struct dev_pm_ops mtk_pcie_pm_ops = {
1229	NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
1230				  mtk_pcie_resume_noirq)
1231};
1232
1233static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
1234	.ops = &mtk_pcie_ops,
1235	.startup = mtk_pcie_startup_port,
1236	.quirks = MTK_PCIE_NO_MSI,
1237};
1238
1239static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
1240	.ops = &mtk_pcie_ops_v2,
1241	.startup = mtk_pcie_startup_port_v2,
1242	.setup_irq = mtk_pcie_setup_irq,
1243};
1244
1245static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
1246	.ops = &mtk_pcie_ops_v2,
1247	.startup = mtk_pcie_startup_port_v2,
1248	.setup_irq = mtk_pcie_setup_irq,
1249	.quirks = MTK_PCIE_FIX_CLASS_ID,
1250};
1251
1252static const struct mtk_pcie_soc mtk_pcie_soc_an7583 = {
1253	.ops = &mtk_pcie_ops_v2,
1254	.startup = mtk_pcie_startup_port_an7583,
1255	.setup_irq = mtk_pcie_setup_irq,
1256	.quirks = MTK_PCIE_FIX_CLASS_ID | MTK_PCIE_SKIP_RSTB,
1257};
1258
1259static const struct mtk_pcie_soc mtk_pcie_soc_mt7629 = {
1260	.device_id = PCI_DEVICE_ID_MEDIATEK_7629,
1261	.ops = &mtk_pcie_ops_v2,
1262	.startup = mtk_pcie_startup_port_v2,
1263	.setup_irq = mtk_pcie_setup_irq,
1264	.quirks = MTK_PCIE_FIX_CLASS_ID | MTK_PCIE_FIX_DEVICE_ID,
1265};
1266
1267static const struct of_device_id mtk_pcie_ids[] = {
1268	{ .compatible = "airoha,an7583-pcie", .data = &mtk_pcie_soc_an7583 },
1269	{ .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 },
1270	{ .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 },
1271	{ .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_mt2712 },
1272	{ .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_mt7622 },
1273	{ .compatible = "mediatek,mt7629-pcie", .data = &mtk_pcie_soc_mt7629 },
1274	{},
1275};
1276MODULE_DEVICE_TABLE(of, mtk_pcie_ids);
1277
1278static struct platform_driver mtk_pcie_driver = {
1279	.probe = mtk_pcie_probe,
1280	.remove = mtk_pcie_remove,
1281	.driver = {
1282		.name = "mtk-pcie",
1283		.of_match_table = mtk_pcie_ids,
1284		.suppress_bind_attrs = true,
1285		.pm = &mtk_pcie_pm_ops,
1286	},
1287};
1288module_platform_driver(mtk_pcie_driver);
1289MODULE_DESCRIPTION("MediaTek PCIe host controller driver");
1290MODULE_LICENSE("GPL v2");