pcie-rockchip-host.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Rockchip AXI PCIe host controller driver
   4 *
   5 * Copyright (c) 2016 Rockchip, Inc.
   6 *
   7 * Author: Shawn Lin <shawn.lin@rock-chips.com>
   8 *         Wenrui Li <wenrui.li@rock-chips.com>
   9 *
  10 * Bits taken from Synopsys DesignWare Host controller driver and
  11 * ARM PCI Host generic driver.
  12 */
  13
  14#include <linux/bitfield.h>
  15#include <linux/bitrev.h>
  16#include <linux/gpio/consumer.h>
  17#include <linux/interrupt.h>
  18#include <linux/iopoll.h>
  19#include <linux/irq.h>
  20#include <linux/irqchip/chained_irq.h>
  21#include <linux/irqdomain.h>
  22#include <linux/module.h>
  23#include <linux/of.h>
  24#include <linux/of_pci.h>
  25#include <linux/phy/phy.h>
  26#include <linux/platform_device.h>
  27
  28#include "../pci.h"
  29#include "pcie-rockchip.h"
  30
  31static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
  32{
  33	u32 status;
  34
  35	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
  36	status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
  37	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
  38}
  39
  40static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
  41{
  42	u32 status;
  43
  44	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
  45	status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
  46	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
  47}
  48
  49static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
  50{
  51	u32 val;
  52
  53	/* Update Tx credit maximum update interval */
  54	val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
  55	val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
  56	val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000);	/* ns */
  57	rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
  58}
  59
  60static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
  61				      struct pci_bus *bus, int dev)
  62{
  63	/*
  64	 * Access only one slot on each root port.
  65	 * Do not read more than one device on the bus directly attached
  66	 * to RC's downstream side.
  67	 */
  68	if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent))
  69		return dev == 0;
  70
  71	return 1;
  72}
  73
  74static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
  75{
  76	u32 val;
  77	u8 map;
  78
  79	if (rockchip->legacy_phy)
  80		return GENMASK(MAX_LANE_NUM - 1, 0);
  81
  82	val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
  83	map = val & PCIE_CORE_LANE_MAP_MASK;
  84
  85	/* The link may be using a reverse-indexed mapping. */
  86	if (val & PCIE_CORE_LANE_MAP_REVERSE)
  87		map = bitrev8(map) >> 4;
  88
  89	return map;
  90}
  91
  92static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
  93				     int where, int size, u32 *val)
  94{
  95	void __iomem *addr;
  96
  97	addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
  98
  99	if (!IS_ALIGNED((uintptr_t)addr, size)) {
 100		*val = 0;
 101		return PCIBIOS_BAD_REGISTER_NUMBER;
 102	}
 103
 104	if (size == 4) {
 105		*val = readl(addr);
 106	} else if (size == 2) {
 107		*val = readw(addr);
 108	} else if (size == 1) {
 109		*val = readb(addr);
 110	} else {
 111		*val = 0;
 112		return PCIBIOS_BAD_REGISTER_NUMBER;
 113	}
 114	return PCIBIOS_SUCCESSFUL;
 115}
 116
 117static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
 118				     int where, int size, u32 val)
 119{
 120	u32 mask, tmp, offset;
 121	void __iomem *addr;
 122
 123	offset = where & ~0x3;
 124	addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
 125
 126	if (size == 4) {
 127		writel(val, addr);
 128		return PCIBIOS_SUCCESSFUL;
 129	}
 130
 131	mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 132
 133	/*
 134	 * N.B. This read/modify/write isn't safe in general because it can
 135	 * corrupt RW1C bits in adjacent registers.  But the hardware
 136	 * doesn't support smaller writes.
 137	 */
 138	tmp = readl(addr) & mask;
 139	tmp |= val << ((where & 0x3) * 8);
 140	writel(tmp, addr);
 141
 142	return PCIBIOS_SUCCESSFUL;
 143}
 144
 145static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
 146				       struct pci_bus *bus, u32 devfn,
 147				       int where, int size, u32 *val)
 148{
 149	void __iomem *addr;
 150
 151	addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);
 152
 153	if (!IS_ALIGNED((uintptr_t)addr, size)) {
 154		*val = 0;
 155		return PCIBIOS_BAD_REGISTER_NUMBER;
 156	}
 157
 158	if (pci_is_root_bus(bus->parent))
 159		rockchip_pcie_cfg_configuration_accesses(rockchip,
 160						AXI_WRAPPER_TYPE0_CFG);
 161	else
 162		rockchip_pcie_cfg_configuration_accesses(rockchip,
 163						AXI_WRAPPER_TYPE1_CFG);
 164
 165	if (size == 4) {
 166		*val = readl(addr);
 167	} else if (size == 2) {
 168		*val = readw(addr);
 169	} else if (size == 1) {
 170		*val = readb(addr);
 171	} else {
 172		*val = 0;
 173		return PCIBIOS_BAD_REGISTER_NUMBER;
 174	}
 175	return PCIBIOS_SUCCESSFUL;
 176}
 177
 178static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
 179				       struct pci_bus *bus, u32 devfn,
 180				       int where, int size, u32 val)
 181{
 182	void __iomem *addr;
 183
 184	addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);
 185
 186	if (!IS_ALIGNED((uintptr_t)addr, size))
 187		return PCIBIOS_BAD_REGISTER_NUMBER;
 188
 189	if (pci_is_root_bus(bus->parent))
 190		rockchip_pcie_cfg_configuration_accesses(rockchip,
 191						AXI_WRAPPER_TYPE0_CFG);
 192	else
 193		rockchip_pcie_cfg_configuration_accesses(rockchip,
 194						AXI_WRAPPER_TYPE1_CFG);
 195
 196	if (size == 4)
 197		writel(val, addr);
 198	else if (size == 2)
 199		writew(val, addr);
 200	else if (size == 1)
 201		writeb(val, addr);
 202	else
 203		return PCIBIOS_BAD_REGISTER_NUMBER;
 204
 205	return PCIBIOS_SUCCESSFUL;
 206}
 207
 208static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
 209				 int size, u32 *val)
 210{
 211	struct rockchip_pcie *rockchip = bus->sysdata;
 212
 213	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
 214		return PCIBIOS_DEVICE_NOT_FOUND;
 215
 216	if (pci_is_root_bus(bus))
 217		return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
 218
 219	return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size,
 220					   val);
 221}
 222
 223static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
 224				 int where, int size, u32 val)
 225{
 226	struct rockchip_pcie *rockchip = bus->sysdata;
 227
 228	if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
 229		return PCIBIOS_DEVICE_NOT_FOUND;
 230
 231	if (pci_is_root_bus(bus))
 232		return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
 233
 234	return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size,
 235					   val);
 236}
 237
 238static struct pci_ops rockchip_pcie_ops = {
 239	.read = rockchip_pcie_rd_conf,
 240	.write = rockchip_pcie_wr_conf,
 241};
 242
 243static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
 244{
 245	int curr;
 246	u32 status, scale, power;
 247
 248	if (IS_ERR(rockchip->vpcie3v3))
 249		return;
 250
 251	/*
 252	 * Set RC's captured slot power limit and scale if
 253	 * vpcie3v3 available. The default values are both zero
 254	 * which means the software should set these two according
 255	 * to the actual power supply.
 256	 */
 257	curr = regulator_get_current_limit(rockchip->vpcie3v3);
 258	if (curr <= 0)
 259		return;
 260
 261	scale = 3; /* 0.001x */
 262	curr = curr / 1000; /* convert to mA */
 263	power = (curr * 3300) / 1000; /* milliwatt */
 264	while (power > FIELD_MAX(PCI_EXP_DEVCAP_PWR_VAL)) {
 265		if (!scale) {
 266			dev_warn(rockchip->dev, "invalid power supply\n");
 267			return;
 268		}
 269		scale--;
 270		power = power / 10;
 271	}
 272
 273	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP);
 274	status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_VAL, power);
 275	status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_SCL, scale);
 276	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP);
 277}
 278
 279/**
 280 * rockchip_pcie_host_init_port - Initialize hardware
 281 * @rockchip: PCIe port information
 282 */
 283static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip)
 284{
 285	struct device *dev = rockchip->dev;
 286	int err, i = MAX_LANE_NUM;
 287	u32 status;
 288
 289	gpiod_set_value_cansleep(rockchip->perst_gpio, 0);
 290
 291	err = rockchip_pcie_init_port(rockchip);
 292	if (err)
 293		return err;
 294
 295	/* Fix the transmitted FTS count desired to exit from L0s. */
 296	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
 297	status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
 298		 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
 299	rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
 300
 301	rockchip_pcie_set_power_limit(rockchip);
 302
 303	/* Set RC's clock architecture as common clock */
 304	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
 305	status |= PCI_EXP_LNKSTA_SLC << 16;
 306	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
 307
 308	/* Set RC's RCB to 128 */
 309	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
 310	status |= PCI_EXP_LNKCTL_RCB;
 311	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
 312
 313	/* Enable Gen1 training */
 314	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
 315			    PCIE_CLIENT_CONFIG);
 316
 317	msleep(PCIE_T_PVPERL_MS);
 318	gpiod_set_value_cansleep(rockchip->perst_gpio, 1);
 319
 320	msleep(PCIE_RESET_CONFIG_WAIT_MS);
 321
 322	/* 500ms timeout value should be enough for Gen1/2 training */
 323	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
 324				 status, PCIE_LINK_UP(status), 20,
 325				 500 * USEC_PER_MSEC);
 326	if (err) {
 327		dev_err(dev, "PCIe link training gen1 timeout!\n");
 328		goto err_power_off_phy;
 329	}
 330
 331	if (rockchip->link_gen == 2) {
 332		/*
 333		 * Enable retrain for gen2. This should be configured only after
 334		 * gen1 finished.
 335		 */
 336		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2);
 337		status &= ~PCI_EXP_LNKCTL2_TLS;
 338		status |= PCI_EXP_LNKCTL2_TLS_5_0GT;
 339		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2);
 340		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
 341		status |= PCI_EXP_LNKCTL_RL;
 342		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL);
 343
 344		err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
 345					 status, PCIE_LINK_IS_GEN2(status), 20,
 346					 500 * USEC_PER_MSEC);
 347		if (err)
 348			dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
 349	}
 350
 351	/* Check the final link width from negotiated lane counter from MGMT */
 352	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
 353	status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
 354			  PCIE_CORE_PL_CONF_LANE_SHIFT);
 355	dev_dbg(dev, "current link width is x%d\n", status);
 356
 357	/* Power off unused lane(s) */
 358	rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
 359	for (i = 0; i < MAX_LANE_NUM; i++) {
 360		if (!(rockchip->lanes_map & BIT(i))) {
 361			dev_dbg(dev, "idling lane %d\n", i);
 362			phy_power_off(rockchip->phys[i]);
 363		}
 364	}
 365
 366	rockchip_pcie_write(rockchip, PCI_VENDOR_ID_ROCKCHIP,
 367			    PCIE_CORE_CONFIG_VENDOR);
 368	rockchip_pcie_write(rockchip,
 369			    PCI_CLASS_BRIDGE_PCI_NORMAL << 8,
 370			    PCIE_RC_CONFIG_RID_CCR);
 371
 372	/* Clear THP cap's next cap pointer to remove L1 substate cap */
 373	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
 374	status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
 375	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
 376
 377	/* Clear L0s from RC's link cap */
 378	if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
 379		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP);
 380		status &= ~PCI_EXP_LNKCAP_ASPM_L0S;
 381		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP);
 382	}
 383
 384	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL);
 385	status &= ~PCI_EXP_DEVCTL_PAYLOAD;
 386	status |= PCI_EXP_DEVCTL_PAYLOAD_256B;
 387	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL);
 388
 389	return 0;
 390err_power_off_phy:
 391	while (i--)
 392		phy_power_off(rockchip->phys[i]);
 393	i = MAX_LANE_NUM;
 394	while (i--)
 395		phy_exit(rockchip->phys[i]);
 396	return err;
 397}
 398
 399static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
 400{
 401	struct rockchip_pcie *rockchip = arg;
 402	struct device *dev = rockchip->dev;
 403	u32 reg;
 404	u32 sub_reg;
 405
 406	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
 407	if (reg & PCIE_CLIENT_INT_LOCAL) {
 408		dev_dbg(dev, "local interrupt received\n");
 409		sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
 410		if (sub_reg & PCIE_CORE_INT_PRFPE)
 411			dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
 412
 413		if (sub_reg & PCIE_CORE_INT_CRFPE)
 414			dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
 415
 416		if (sub_reg & PCIE_CORE_INT_RRPE)
 417			dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
 418
 419		if (sub_reg & PCIE_CORE_INT_PRFO)
 420			dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
 421
 422		if (sub_reg & PCIE_CORE_INT_CRFO)
 423			dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
 424
 425		if (sub_reg & PCIE_CORE_INT_RT)
 426			dev_dbg(dev, "replay timer timed out\n");
 427
 428		if (sub_reg & PCIE_CORE_INT_RTR)
 429			dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
 430
 431		if (sub_reg & PCIE_CORE_INT_PE)
 432			dev_dbg(dev, "phy error detected on receive side\n");
 433
 434		if (sub_reg & PCIE_CORE_INT_MTR)
 435			dev_dbg(dev, "malformed TLP received from the link\n");
 436
 437		if (sub_reg & PCIE_CORE_INT_UCR)
 438			dev_dbg(dev, "Unexpected Completion received from the link\n");
 439
 440		if (sub_reg & PCIE_CORE_INT_FCE)
 441			dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
 442
 443		if (sub_reg & PCIE_CORE_INT_CT)
 444			dev_dbg(dev, "a request timed out waiting for completion\n");
 445
 446		if (sub_reg & PCIE_CORE_INT_UTC)
 447			dev_dbg(dev, "unmapped TC error\n");
 448
 449		if (sub_reg & PCIE_CORE_INT_MMVC)
 450			dev_dbg(dev, "MSI mask register changes\n");
 451
 452		rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
 453	} else if (reg & PCIE_CLIENT_INT_PHY) {
 454		dev_dbg(dev, "phy link changes\n");
 455		rockchip_pcie_update_txcredit_mui(rockchip);
 456		rockchip_pcie_clr_bw_int(rockchip);
 457	}
 458
 459	rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
 460			    PCIE_CLIENT_INT_STATUS);
 461
 462	return IRQ_HANDLED;
 463}
 464
 465static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
 466{
 467	struct rockchip_pcie *rockchip = arg;
 468	struct device *dev = rockchip->dev;
 469	u32 reg;
 470
 471	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
 472	if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
 473		dev_dbg(dev, "legacy done interrupt received\n");
 474
 475	if (reg & PCIE_CLIENT_INT_MSG)
 476		dev_dbg(dev, "message done interrupt received\n");
 477
 478	if (reg & PCIE_CLIENT_INT_HOT_RST)
 479		dev_dbg(dev, "hot reset interrupt received\n");
 480
 481	if (reg & PCIE_CLIENT_INT_DPA)
 482		dev_dbg(dev, "dpa interrupt received\n");
 483
 484	if (reg & PCIE_CLIENT_INT_FATAL_ERR)
 485		dev_dbg(dev, "fatal error interrupt received\n");
 486
 487	if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
 488		dev_dbg(dev, "non fatal error interrupt received\n");
 489
 490	if (reg & PCIE_CLIENT_INT_CORR_ERR)
 491		dev_dbg(dev, "correctable error interrupt received\n");
 492
 493	if (reg & PCIE_CLIENT_INT_PHY)
 494		dev_dbg(dev, "phy interrupt received\n");
 495
 496	rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
 497			      PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
 498			      PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
 499			      PCIE_CLIENT_INT_NFATAL_ERR |
 500			      PCIE_CLIENT_INT_CORR_ERR |
 501			      PCIE_CLIENT_INT_PHY),
 502		   PCIE_CLIENT_INT_STATUS);
 503
 504	return IRQ_HANDLED;
 505}
 506
 507static void rockchip_pcie_intx_handler(struct irq_desc *desc)
 508{
 509	struct irq_chip *chip = irq_desc_get_chip(desc);
 510	struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
 511	struct device *dev = rockchip->dev;
 512	u32 reg;
 513	u32 hwirq;
 514	int ret;
 515
 516	chained_irq_enter(chip, desc);
 517
 518	reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
 519	reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
 520
 521	while (reg) {
 522		hwirq = ffs(reg) - 1;
 523		reg &= ~BIT(hwirq);
 524
 525		ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq);
 526		if (ret)
 527			dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
 528	}
 529
 530	chained_irq_exit(chip, desc);
 531}
 532
 533static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
 534{
 535	int irq, err;
 536	struct device *dev = rockchip->dev;
 537	struct platform_device *pdev = to_platform_device(dev);
 538
 539	irq = platform_get_irq_byname(pdev, "sys");
 540	if (irq < 0)
 541		return irq;
 542
 543	err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
 544			       IRQF_SHARED, "pcie-sys", rockchip);
 545	if (err) {
 546		dev_err(dev, "failed to request PCIe subsystem IRQ\n");
 547		return err;
 548	}
 549
 550	irq = platform_get_irq_byname(pdev, "legacy");
 551	if (irq < 0)
 552		return irq;
 553
 554	irq_set_chained_handler_and_data(irq,
 555					 rockchip_pcie_intx_handler,
 556					 rockchip);
 557
 558	irq = platform_get_irq_byname(pdev, "client");
 559	if (irq < 0)
 560		return irq;
 561
 562	err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
 563			       IRQF_SHARED, "pcie-client", rockchip);
 564	if (err) {
 565		dev_err(dev, "failed to request PCIe client IRQ\n");
 566		return err;
 567	}
 568
 569	return 0;
 570}
 571
 572/**
 573 * rockchip_pcie_parse_host_dt - Parse Device Tree
 574 * @rockchip: PCIe port information
 575 *
 576 * Return: '0' on success and error value on failure
 577 */
 578static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip)
 579{
 580	struct device *dev = rockchip->dev;
 581	int err;
 582
 583	err = rockchip_pcie_parse_dt(rockchip);
 584	if (err)
 585		return err;
 586
 587	rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
 588	if (IS_ERR(rockchip->vpcie12v)) {
 589		if (PTR_ERR(rockchip->vpcie12v) != -ENODEV)
 590			return PTR_ERR(rockchip->vpcie12v);
 591		dev_info(dev, "no vpcie12v regulator found\n");
 592	}
 593
 594	rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
 595	if (IS_ERR(rockchip->vpcie3v3)) {
 596		if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)
 597			return PTR_ERR(rockchip->vpcie3v3);
 598		dev_info(dev, "no vpcie3v3 regulator found\n");
 599	}
 600
 601	rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8");
 602	if (IS_ERR(rockchip->vpcie1v8))
 603		return PTR_ERR(rockchip->vpcie1v8);
 604
 605	rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9");
 606	if (IS_ERR(rockchip->vpcie0v9))
 607		return PTR_ERR(rockchip->vpcie0v9);
 608
 609	return 0;
 610}
 611
 612static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
 613{
 614	struct device *dev = rockchip->dev;
 615	int err;
 616
 617	if (!IS_ERR(rockchip->vpcie12v)) {
 618		err = regulator_enable(rockchip->vpcie12v);
 619		if (err) {
 620			dev_err(dev, "fail to enable vpcie12v regulator\n");
 621			goto err_out;
 622		}
 623	}
 624
 625	if (!IS_ERR(rockchip->vpcie3v3)) {
 626		err = regulator_enable(rockchip->vpcie3v3);
 627		if (err) {
 628			dev_err(dev, "fail to enable vpcie3v3 regulator\n");
 629			goto err_disable_12v;
 630		}
 631	}
 632
 633	err = regulator_enable(rockchip->vpcie1v8);
 634	if (err) {
 635		dev_err(dev, "fail to enable vpcie1v8 regulator\n");
 636		goto err_disable_3v3;
 637	}
 638
 639	err = regulator_enable(rockchip->vpcie0v9);
 640	if (err) {
 641		dev_err(dev, "fail to enable vpcie0v9 regulator\n");
 642		goto err_disable_1v8;
 643	}
 644
 645	return 0;
 646
 647err_disable_1v8:
 648	regulator_disable(rockchip->vpcie1v8);
 649err_disable_3v3:
 650	if (!IS_ERR(rockchip->vpcie3v3))
 651		regulator_disable(rockchip->vpcie3v3);
 652err_disable_12v:
 653	if (!IS_ERR(rockchip->vpcie12v))
 654		regulator_disable(rockchip->vpcie12v);
 655err_out:
 656	return err;
 657}
 658
 659static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
 660{
 661	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
 662			    (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
 663	rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
 664			    PCIE_CORE_INT_MASK);
 665
 666	rockchip_pcie_enable_bw_int(rockchip);
 667}
 668
 669static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 670				  irq_hw_number_t hwirq)
 671{
 672	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
 673	irq_set_chip_data(irq, domain->host_data);
 674
 675	return 0;
 676}
 677
 678static const struct irq_domain_ops intx_domain_ops = {
 679	.map = rockchip_pcie_intx_map,
 680};
 681
 682static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
 683{
 684	struct device *dev = rockchip->dev;
 685	struct device_node *intc = of_get_next_child(dev->of_node, NULL);
 686
 687	if (!intc) {
 688		dev_err(dev, "missing child interrupt-controller node\n");
 689		return -EINVAL;
 690	}
 691
 692	rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX,
 693							&intx_domain_ops, rockchip);
 694	of_node_put(intc);
 695	if (!rockchip->irq_domain) {
 696		dev_err(dev, "failed to get a INTx IRQ domain\n");
 697		return -EINVAL;
 698	}
 699
 700	return 0;
 701}
 702
 703static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
 704				     int region_no, int type, u8 num_pass_bits,
 705				     u32 lower_addr, u32 upper_addr)
 706{
 707	u32 ob_addr_0;
 708	u32 ob_addr_1;
 709	u32 ob_desc_0;
 710	u32 aw_offset;
 711
 712	if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
 713		return -EINVAL;
 714	if (num_pass_bits + 1 < 8)
 715		return -EINVAL;
 716	if (num_pass_bits > 63)
 717		return -EINVAL;
 718	if (region_no == 0) {
 719		if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
 720			return -EINVAL;
 721	}
 722	if (region_no != 0) {
 723		if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
 724			return -EINVAL;
 725	}
 726
 727	aw_offset = (region_no << OB_REG_SIZE_SHIFT);
 728
 729	ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
 730	ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
 731	ob_addr_1 = upper_addr;
 732	ob_desc_0 = (1 << 23 | type);
 733
 734	rockchip_pcie_write(rockchip, ob_addr_0,
 735			    PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
 736	rockchip_pcie_write(rockchip, ob_addr_1,
 737			    PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
 738	rockchip_pcie_write(rockchip, ob_desc_0,
 739			    PCIE_CORE_OB_REGION_DESC0 + aw_offset);
 740	rockchip_pcie_write(rockchip, 0,
 741			    PCIE_CORE_OB_REGION_DESC1 + aw_offset);
 742
 743	return 0;
 744}
 745
 746static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
 747				     int region_no, u8 num_pass_bits,
 748				     u32 lower_addr, u32 upper_addr)
 749{
 750	u32 ib_addr_0;
 751	u32 ib_addr_1;
 752	u32 aw_offset;
 753
 754	if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
 755		return -EINVAL;
 756	if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
 757		return -EINVAL;
 758	if (num_pass_bits > 63)
 759		return -EINVAL;
 760
 761	aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
 762
 763	ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
 764	ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
 765	ib_addr_1 = upper_addr;
 766
 767	rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
 768	rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
 769
 770	return 0;
 771}
 772
 773static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
 774{
 775	struct device *dev = rockchip->dev;
 776	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
 777	struct resource_entry *entry;
 778	u64 pci_addr, size;
 779	int offset;
 780	int err;
 781	int reg_no;
 782
 783	rockchip_pcie_cfg_configuration_accesses(rockchip,
 784						 AXI_WRAPPER_TYPE0_CFG);
 785	entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM);
 786	if (!entry)
 787		return -ENODEV;
 788
 789	size = resource_size(entry->res);
 790	pci_addr = entry->res->start - entry->offset;
 791	rockchip->msg_bus_addr = pci_addr;
 792
 793	for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
 794		err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
 795						AXI_WRAPPER_MEM_WRITE,
 796						20 - 1,
 797						pci_addr + (reg_no << 20),
 798						0);
 799		if (err) {
 800			dev_err(dev, "program RC mem outbound ATU failed\n");
 801			return err;
 802		}
 803	}
 804
 805	err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
 806	if (err) {
 807		dev_err(dev, "program RC mem inbound ATU failed\n");
 808		return err;
 809	}
 810
 811	entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
 812	if (!entry)
 813		return -ENODEV;
 814
 815	/* store the register number offset to program RC io outbound ATU */
 816	offset = size >> 20;
 817
 818	size = resource_size(entry->res);
 819	pci_addr = entry->res->start - entry->offset;
 820
 821	for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
 822		err = rockchip_pcie_prog_ob_atu(rockchip,
 823						reg_no + 1 + offset,
 824						AXI_WRAPPER_IO_WRITE,
 825						20 - 1,
 826						pci_addr + (reg_no << 20),
 827						0);
 828		if (err) {
 829			dev_err(dev, "program RC io outbound ATU failed\n");
 830			return err;
 831		}
 832	}
 833
 834	/* assign message regions */
 835	rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
 836				  AXI_WRAPPER_NOR_MSG,
 837				  20 - 1, 0, 0);
 838
 839	rockchip->msg_bus_addr += ((reg_no + offset) << 20);
 840	return err;
 841}
 842
 843static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
 844{
 845	u32 value;
 846	int err;
 847
 848	/* send PME_TURN_OFF message */
 849	writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
 850
 851	/* read LTSSM and wait for falling into L2 link state */
 852	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
 853				 value, PCIE_LINK_IS_L2(value), 20,
 854				 jiffies_to_usecs(5 * HZ));
 855	if (err) {
 856		dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
 857		return err;
 858	}
 859
 860	return 0;
 861}
 862
 863static int rockchip_pcie_suspend_noirq(struct device *dev)
 864{
 865	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
 866	int ret;
 867
 868	/* disable core and cli int since we don't need to ack PME_ACK */
 869	rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
 870			    PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
 871	rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
 872
 873	ret = rockchip_pcie_wait_l2(rockchip);
 874	if (ret) {
 875		rockchip_pcie_enable_interrupts(rockchip);
 876		return ret;
 877	}
 878
 879	rockchip_pcie_deinit_phys(rockchip);
 880
 881	rockchip_pcie_disable_clocks(rockchip);
 882
 883	regulator_disable(rockchip->vpcie0v9);
 884
 885	return ret;
 886}
 887
 888static int rockchip_pcie_resume_noirq(struct device *dev)
 889{
 890	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
 891	int err;
 892
 893	err = regulator_enable(rockchip->vpcie0v9);
 894	if (err) {
 895		dev_err(dev, "fail to enable vpcie0v9 regulator\n");
 896		return err;
 897	}
 898
 899	err = rockchip_pcie_enable_clocks(rockchip);
 900	if (err)
 901		goto err_disable_0v9;
 902
 903	err = rockchip_pcie_host_init_port(rockchip);
 904	if (err)
 905		goto err_pcie_resume;
 906
 907	err = rockchip_pcie_cfg_atu(rockchip);
 908	if (err)
 909		goto err_err_deinit_port;
 910
 911	/* Need this to enter L1 again */
 912	rockchip_pcie_update_txcredit_mui(rockchip);
 913	rockchip_pcie_enable_interrupts(rockchip);
 914
 915	return 0;
 916
 917err_err_deinit_port:
 918	rockchip_pcie_deinit_phys(rockchip);
 919err_pcie_resume:
 920	rockchip_pcie_disable_clocks(rockchip);
 921err_disable_0v9:
 922	regulator_disable(rockchip->vpcie0v9);
 923	return err;
 924}
 925
 926static int rockchip_pcie_probe(struct platform_device *pdev)
 927{
 928	struct rockchip_pcie *rockchip;
 929	struct device *dev = &pdev->dev;
 930	struct pci_host_bridge *bridge;
 931	int err;
 932
 933	if (!dev->of_node)
 934		return -ENODEV;
 935
 936	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
 937	if (!bridge)
 938		return -ENOMEM;
 939
 940	rockchip = pci_host_bridge_priv(bridge);
 941
 942	platform_set_drvdata(pdev, rockchip);
 943	rockchip->dev = dev;
 944	rockchip->is_rc = true;
 945
 946	err = rockchip_pcie_parse_host_dt(rockchip);
 947	if (err)
 948		return err;
 949
 950	err = rockchip_pcie_enable_clocks(rockchip);
 951	if (err)
 952		return err;
 953
 954	err = rockchip_pcie_set_vpcie(rockchip);
 955	if (err) {
 956		dev_err(dev, "failed to set vpcie regulator\n");
 957		goto err_set_vpcie;
 958	}
 959
 960	err = rockchip_pcie_host_init_port(rockchip);
 961	if (err)
 962		goto err_vpcie;
 963
 964	err = rockchip_pcie_init_irq_domain(rockchip);
 965	if (err < 0)
 966		goto err_deinit_port;
 967
 968	err = rockchip_pcie_cfg_atu(rockchip);
 969	if (err)
 970		goto err_remove_irq_domain;
 971
 972	rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
 973	if (!rockchip->msg_region) {
 974		err = -ENOMEM;
 975		goto err_remove_irq_domain;
 976	}
 977
 978	bridge->sysdata = rockchip;
 979	bridge->ops = &rockchip_pcie_ops;
 980
 981	err = rockchip_pcie_setup_irq(rockchip);
 982	if (err)
 983		goto err_remove_irq_domain;
 984
 985	rockchip_pcie_enable_interrupts(rockchip);
 986
 987	err = pci_host_probe(bridge);
 988	if (err < 0)
 989		goto err_remove_irq_domain;
 990
 991	return 0;
 992
 993err_remove_irq_domain:
 994	irq_domain_remove(rockchip->irq_domain);
 995err_deinit_port:
 996	rockchip_pcie_deinit_phys(rockchip);
 997err_vpcie:
 998	if (!IS_ERR(rockchip->vpcie12v))
 999		regulator_disable(rockchip->vpcie12v);
1000	if (!IS_ERR(rockchip->vpcie3v3))
1001		regulator_disable(rockchip->vpcie3v3);
1002	regulator_disable(rockchip->vpcie1v8);
1003	regulator_disable(rockchip->vpcie0v9);
1004err_set_vpcie:
1005	rockchip_pcie_disable_clocks(rockchip);
1006	return err;
1007}
1008
1009static void rockchip_pcie_remove(struct platform_device *pdev)
1010{
1011	struct device *dev = &pdev->dev;
1012	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
1013	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
1014
1015	pci_stop_root_bus(bridge->bus);
1016	pci_remove_root_bus(bridge->bus);
1017	irq_domain_remove(rockchip->irq_domain);
1018
1019	rockchip_pcie_deinit_phys(rockchip);
1020
1021	rockchip_pcie_disable_clocks(rockchip);
1022
1023	if (!IS_ERR(rockchip->vpcie12v))
1024		regulator_disable(rockchip->vpcie12v);
1025	if (!IS_ERR(rockchip->vpcie3v3))
1026		regulator_disable(rockchip->vpcie3v3);
1027	regulator_disable(rockchip->vpcie1v8);
1028	regulator_disable(rockchip->vpcie0v9);
1029}
1030
1031static const struct dev_pm_ops rockchip_pcie_pm_ops = {
1032	NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
1033				  rockchip_pcie_resume_noirq)
1034};
1035
1036static const struct of_device_id rockchip_pcie_of_match[] = {
1037	{ .compatible = "rockchip,rk3399-pcie", },
1038	{}
1039};
1040MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
1041
1042static struct platform_driver rockchip_pcie_driver = {
1043	.driver = {
1044		.name = "rockchip-pcie",
1045		.of_match_table = rockchip_pcie_of_match,
1046		.pm = &rockchip_pcie_pm_ops,
1047	},
1048	.probe = rockchip_pcie_probe,
1049	.remove = rockchip_pcie_remove,
1050};
1051module_platform_driver(rockchip_pcie_driver);
1052
1053MODULE_AUTHOR("Rockchip Inc");
1054MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
1055MODULE_LICENSE("GPL v2");