File Index
Files and directories
Sign in
FILE

pcie-brcmstb.c

File preview with syntax highlighting.

linux-6.19.14/drivers/pci/controller/pcie-brcmstb.c C 
   1// SPDX-License-Identifier: GPL-2.0+
   2/* Copyright (C) 2009 - 2019 Broadcom */
   3
   4#include <linux/bitfield.h>
   5#include <linux/bitops.h>
   6#include <linux/clk.h>
   7#include <linux/compiler.h>
   8#include <linux/delay.h>
   9#include <linux/init.h>
  10#include <linux/interrupt.h>
  11#include <linux/io.h>
  12#include <linux/iopoll.h>
  13#include <linux/ioport.h>
  14#include <linux/irqchip/chained_irq.h>
  15#include <linux/irqchip/irq-msi-lib.h>
  16#include <linux/irqdomain.h>
  17#include <linux/kdebug.h>
  18#include <linux/kernel.h>
  19#include <linux/list.h>
  20#include <linux/log2.h>
  21#include <linux/module.h>
  22#include <linux/msi.h>
  23#include <linux/notifier.h>
  24#include <linux/of_address.h>
  25#include <linux/of_irq.h>
  26#include <linux/of_pci.h>
  27#include <linux/of_platform.h>
  28#include <linux/panic_notifier.h>
  29#include <linux/pci.h>
  30#include <linux/pci-ecam.h>
  31#include <linux/printk.h>
  32#include <linux/regulator/consumer.h>
  33#include <linux/reset.h>
  34#include <linux/sizes.h>
  35#include <linux/slab.h>
  36#include <linux/spinlock.h>
  37#include <linux/string.h>
  38#include <linux/string_choices.h>
  39#include <linux/types.h>
  40
  41#include "../pci.h"
  42
  43/* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */
  44#define BRCM_PCIE_CAP_REGS				0x00ac
  45
  46/* Broadcom STB PCIe Register Offsets */
  47#define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1				0x0188
  48#define  PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK	0xc
  49#define  PCIE_RC_CFG_VENDOR_SPECIFIC_REG1_LITTLE_ENDIAN			0x0
  50
  51#define PCIE_RC_CFG_PRIV1_ID_VAL3			0x043c
  52#define  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK	0xffffff
  53
  54#define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY			0x04dc
  55#define  PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK	0x1f0
  56
  57#define PCIE_RC_CFG_PRIV1_ROOT_CAP			0x4f8
  58#define  PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK	0xf8
  59
  60#define PCIE_RC_DL_MDIO_ADDR				0x1100
  61#define PCIE_RC_DL_MDIO_WR_DATA				0x1104
  62#define PCIE_RC_DL_MDIO_RD_DATA				0x1108
  63
  64#define PCIE_RC_PL_REG_PHY_CTL_1			0x1804
  65#define  PCIE_RC_PL_REG_PHY_CTL_1_REG_P2_POWERDOWN_ENA_NOSYNC_MASK	0x8
  66
  67#define PCIE_RC_PL_PHY_CTL_15				0x184c
  68#define  PCIE_RC_PL_PHY_CTL_15_DIS_PLL_PD_MASK		0x400000
  69#define  PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK	0xff
  70
  71#define PCIE_MISC_MISC_CTRL				0x4008
  72#define  PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK	0x80
  73#define  PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK	0x400
  74#define  PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK		0x1000
  75#define  PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK	0x2000
  76#define  PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK	0x300000
  77
  78#define  PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK		0xf8000000
  79#define  PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK		0x07c00000
  80#define  PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK		0x0000001f
  81#define  SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK
  82
  83#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO		0x400c
  84#define PCIE_MEM_WIN0_LO(win)	\
  85		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8)
  86
  87#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI		0x4010
  88#define PCIE_MEM_WIN0_HI(win)	\
  89		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8)
  90
  91/*
  92 * NOTE: You may see the term "BAR" in a number of register names used by
  93 *   this driver.  The term is an artifact of when the HW core was an
  94 *   endpoint device (EP).  Now it is a root complex (RC) and anywhere a
  95 *   register has the term "BAR" it is related to an inbound window.
  96 */
  97
  98#define PCIE_BRCM_MAX_INBOUND_WINS			16
  99#define PCIE_MISC_RC_BAR1_CONFIG_LO			0x402c
 100#define  PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK		0x1f
 101
 102#define PCIE_MISC_RC_BAR4_CONFIG_LO			0x40d4
 103
 104
 105#define PCIE_MISC_MSI_BAR_CONFIG_LO			0x4044
 106#define PCIE_MISC_MSI_BAR_CONFIG_HI			0x4048
 107
 108#define PCIE_MISC_MSI_DATA_CONFIG			0x404c
 109#define  PCIE_MISC_MSI_DATA_CONFIG_VAL_32		0xffe06540
 110#define  PCIE_MISC_MSI_DATA_CONFIG_VAL_8		0xfff86540
 111
 112#define PCIE_MISC_PCIE_CTRL				0x4064
 113#define  PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK	0x1
 114#define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK		0x4
 115
 116#define PCIE_MISC_PCIE_STATUS				0x4068
 117#define  PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK		0x80
 118#define  PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK	0x20
 119#define  PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK	0x10
 120#define  PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK	0x40
 121
 122#define PCIE_MISC_REVISION				0x406c
 123#define  BRCM_PCIE_HW_REV_33				0x0303
 124#define  BRCM_PCIE_HW_REV_3_20				0x0320
 125
 126#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT		0x4070
 127#define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK	0xfff00000
 128#define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK	0xfff0
 129#define PCIE_MEM_WIN0_BASE_LIMIT(win)	\
 130		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4)
 131
 132#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI			0x4080
 133#define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK	0xff
 134#define PCIE_MEM_WIN0_BASE_HI(win)	\
 135		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8)
 136
 137#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI			0x4084
 138#define  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK	0xff
 139#define PCIE_MEM_WIN0_LIMIT_HI(win)	\
 140		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8)
 141
 142#define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK	0x2
 143#define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK		0x200000
 144#define  PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x08000000
 145#define  PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK		0x00800000
 146#define  PCIE_CLKREQ_MASK \
 147	  (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \
 148	   PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK)
 149
 150#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP			0x40ac
 151#define  PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK	BIT(0)
 152#define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP			0x410c
 153
 154#define PCIE_MSI_INTR2_BASE		0x4500
 155
 156/* Offsets from INTR2_CPU and MSI_INTR2 BASE offsets */
 157#define  MSI_INT_STATUS			0x0
 158#define  MSI_INT_CLR			0x8
 159#define  MSI_INT_MASK_SET		0x10
 160#define  MSI_INT_MASK_CLR		0x14
 161
 162/* Error report registers */
 163#define PCIE_OUTB_ERR_TREAT				0x6000
 164#define  PCIE_OUTB_ERR_TREAT_CONFIG		0x1
 165#define  PCIE_OUTB_ERR_TREAT_MEM			0x2
 166#define PCIE_OUTB_ERR_VALID				0x6004
 167#define PCIE_OUTB_ERR_CLEAR				0x6008
 168#define PCIE_OUTB_ERR_ACC_INFO				0x600c
 169#define  PCIE_OUTB_ERR_ACC_INFO_CFG_ERR			BIT(0)
 170#define  PCIE_OUTB_ERR_ACC_INFO_MEM_ERR			BIT(1)
 171#define  PCIE_OUTB_ERR_ACC_INFO_TYPE_64			BIT(2)
 172#define  PCIE_OUTB_ERR_ACC_INFO_DIR_WRITE		BIT(4)
 173#define  PCIE_OUTB_ERR_ACC_INFO_BYTE_LANES		0xff00
 174#define PCIE_OUTB_ERR_ACC_ADDR				0x6010
 175#define PCIE_OUTB_ERR_ACC_ADDR_BUS			0xff00000
 176#define PCIE_OUTB_ERR_ACC_ADDR_DEV			0xf8000
 177#define PCIE_OUTB_ERR_ACC_ADDR_FUNC			0x7000
 178#define PCIE_OUTB_ERR_ACC_ADDR_REG			0xfff
 179#define PCIE_OUTB_ERR_CFG_CAUSE				0x6014
 180#define  PCIE_OUTB_ERR_CFG_CAUSE_TIMEOUT		BIT(6)
 181#define  PCIE_OUTB_ERR_CFG_CAUSE_ABORT			BIT(5)
 182#define  PCIE_OUTB_ERR_CFG_CAUSE_UNSUPP_REQ		BIT(4)
 183#define  PCIE_OUTB_ERR_CFG_CAUSE_ACC_TIMEOUT		BIT(2)
 184#define  PCIE_OUTB_ERR_CFG_CAUSE_ACC_DISABLED		BIT(1)
 185#define  PCIE_OUTB_ERR_CFG_CAUSE_ACC_64BIT		BIT(0)
 186#define PCIE_OUTB_ERR_MEM_ADDR_LO			0x6018
 187#define PCIE_OUTB_ERR_MEM_ADDR_HI			0x601c
 188#define PCIE_OUTB_ERR_MEM_CAUSE				0x6020
 189#define  PCIE_OUTB_ERR_MEM_CAUSE_TIMEOUT		BIT(6)
 190#define  PCIE_OUTB_ERR_MEM_CAUSE_ABORT			BIT(5)
 191#define  PCIE_OUTB_ERR_MEM_CAUSE_UNSUPP_REQ		BIT(4)
 192#define  PCIE_OUTB_ERR_MEM_CAUSE_ACC_DISABLED		BIT(1)
 193#define  PCIE_OUTB_ERR_MEM_CAUSE_BAD_ADDR		BIT(0)
 194
 195#define  PCIE_RGR1_SW_INIT_1_PERST_MASK			0x1
 196
 197#define RGR1_SW_INIT_1_INIT_GENERIC_MASK		0x2
 198#define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT		0x1
 199#define RGR1_SW_INIT_1_INIT_7278_MASK			0x1
 200#define RGR1_SW_INIT_1_INIT_7278_SHIFT			0x0
 201
 202/* PCIe parameters */
 203#define BRCM_NUM_PCIE_OUT_WINS		0x4
 204#define BRCM_INT_PCI_MSI_NR		32
 205#define BRCM_INT_PCI_MSI_LEGACY_NR	8
 206#define BRCM_INT_PCI_MSI_SHIFT		0
 207#define BRCM_INT_PCI_MSI_MASK		GENMASK(BRCM_INT_PCI_MSI_NR - 1, 0)
 208#define BRCM_INT_PCI_MSI_LEGACY_MASK	GENMASK(31, \
 209						32 - BRCM_INT_PCI_MSI_LEGACY_NR)
 210
 211/* MSI target addresses */
 212#define BRCM_MSI_TARGET_ADDR_LT_4GB	0x0fffffffcULL
 213#define BRCM_MSI_TARGET_ADDR_GT_4GB	0xffffffffcULL
 214
 215/* MDIO registers */
 216#define MDIO_PORT0			0x0
 217#define MDIO_DATA_MASK			0x7fffffff
 218#define MDIO_PORT_MASK			0xf0000
 219#define MDIO_PORT_EXT_MASK		0x200000
 220#define MDIO_REGAD_MASK			0xffff
 221#define MDIO_CMD_MASK			0x00100000
 222#define MDIO_CMD_READ			0x1
 223#define MDIO_CMD_WRITE			0x0
 224#define MDIO_DATA_DONE_MASK		0x80000000
 225#define MDIO_RD_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 1 : 0)
 226#define MDIO_WT_DONE(x)			(((x) & MDIO_DATA_DONE_MASK) ? 0 : 1)
 227#define SSC_REGS_ADDR			0x1100
 228#define SET_ADDR_OFFSET			0x1f
 229#define SSC_CNTL_OFFSET			0x2
 230#define SSC_CNTL_OVRD_EN_MASK		0x8000
 231#define SSC_CNTL_OVRD_VAL_MASK		0x4000
 232#define SSC_STATUS_OFFSET		0x1
 233#define SSC_STATUS_SSC_MASK		0x400
 234#define SSC_STATUS_PLL_LOCK_MASK	0x800
 235#define PCIE_BRCM_MAX_MEMC		3
 236
 237#define IDX_ADDR(pcie)			((pcie)->cfg->offsets[EXT_CFG_INDEX])
 238#define DATA_ADDR(pcie)			((pcie)->cfg->offsets[EXT_CFG_DATA])
 239#define PCIE_RGR1_SW_INIT_1(pcie)	((pcie)->cfg->offsets[RGR1_SW_INIT_1])
 240#define HARD_DEBUG(pcie)		((pcie)->cfg->offsets[PCIE_HARD_DEBUG])
 241#define INTR2_CPU_BASE(pcie)		((pcie)->cfg->offsets[PCIE_INTR2_CPU_BASE])
 242
 243/* Rescal registers */
 244#define PCIE_DVT_PMU_PCIE_PHY_CTRL				0xc700
 245#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS			0x3
 246#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK		0x4
 247#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT	0x2
 248#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK		0x2
 249#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT		0x1
 250#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK		0x1
 251#define  PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT		0x0
 252
 253/* Forward declarations */
 254struct brcm_pcie;
 255
 256enum {
 257	RGR1_SW_INIT_1,
 258	EXT_CFG_INDEX,
 259	EXT_CFG_DATA,
 260	PCIE_HARD_DEBUG,
 261	PCIE_INTR2_CPU_BASE,
 262};
 263
 264enum pcie_soc_base {
 265	GENERIC,
 266	BCM2711,
 267	BCM4908,
 268	BCM7278,
 269	BCM7425,
 270	BCM7435,
 271	BCM7712,
 272};
 273
 274struct inbound_win {
 275	u64 size;
 276	u64 pci_offset;
 277	u64 cpu_addr;
 278};
 279
 280/*
 281 * The RESCAL block is tied to PCIe controller #1, regardless of the number of
 282 * controllers, and turning off PCIe controller #1 prevents access to the RESCAL
 283 * register blocks, therefore no other controller can access this register
 284 * space, and depending upon the bus fabric we may get a timeout (UBUS/GISB),
 285 * or a hang (AXI).
 286 */
 287#define CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN		BIT(0)
 288
 289struct pcie_cfg_data {
 290	const int *offsets;
 291	const enum pcie_soc_base soc_base;
 292	const bool has_phy;
 293	const u32 quirks;
 294	u8 num_inbound_wins;
 295	int (*perst_set)(struct brcm_pcie *pcie, u32 val);
 296	int (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val);
 297	int (*post_setup)(struct brcm_pcie *pcie);
 298	bool has_err_report;
 299};
 300
 301struct subdev_regulators {
 302	unsigned int num_supplies;
 303	struct regulator_bulk_data supplies[];
 304};
 305
 306struct brcm_msi {
 307	struct device		*dev;
 308	void __iomem		*base;
 309	struct device_node	*np;
 310	struct irq_domain	*inner_domain;
 311	struct mutex		lock; /* guards the alloc/free operations */
 312	u64			target_addr;
 313	int			irq;
 314	DECLARE_BITMAP(used, BRCM_INT_PCI_MSI_NR);
 315	bool			legacy;
 316	/* Some chips have MSIs in bits [31..24] of a shared register. */
 317	int			legacy_shift;
 318	int			nr; /* No. of MSI available, depends on chip */
 319	/* This is the base pointer for interrupt status/set/clr regs */
 320	void __iomem		*intr_base;
 321};
 322
 323/* Internal PCIe Host Controller Information.*/
 324struct brcm_pcie {
 325	struct device		*dev;
 326	void __iomem		*base;
 327	struct clk		*clk;
 328	struct device_node	*np;
 329	bool			ssc;
 330	int			gen;
 331	u64			msi_target_addr;
 332	struct brcm_msi		*msi;
 333	struct reset_control	*rescal;
 334	struct reset_control	*perst_reset;
 335	struct reset_control	*bridge_reset;
 336	struct reset_control	*swinit_reset;
 337	int			num_memc;
 338	u64			memc_size[PCIE_BRCM_MAX_MEMC];
 339	u32			hw_rev;
 340	struct subdev_regulators *sr;
 341	bool			ep_wakeup_capable;
 342	const struct pcie_cfg_data	*cfg;
 343	bool			bridge_in_reset;
 344	struct notifier_block	die_notifier;
 345	struct notifier_block	panic_notifier;
 346	spinlock_t		bridge_lock;
 347};
 348
 349static inline bool is_bmips(const struct brcm_pcie *pcie)
 350{
 351	return pcie->cfg->soc_base == BCM7435 || pcie->cfg->soc_base == BCM7425;
 352}
 353
 354static int brcm_pcie_bridge_sw_init_set(struct brcm_pcie *pcie, u32 val)
 355{
 356	unsigned long flags;
 357	int ret;
 358
 359	if (pcie->cfg->has_err_report)
 360		spin_lock_irqsave(&pcie->bridge_lock, flags);
 361
 362	ret = pcie->cfg->bridge_sw_init_set(pcie, val);
 363	/* If we fail, assume the bridge is in reset (off) */
 364	pcie->bridge_in_reset = ret ? true : val;
 365
 366	if (pcie->cfg->has_err_report)
 367		spin_unlock_irqrestore(&pcie->bridge_lock, flags);
 368
 369	return ret;
 370}
 371
 372/*
 373 * This is to convert the size of the inbound "BAR" region to the
 374 * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE
 375 */
 376static int brcm_pcie_encode_ibar_size(u64 size)
 377{
 378	int log2_in = ilog2(size);
 379
 380	if (log2_in >= 12 && log2_in <= 15)
 381		/* Covers 4KB to 32KB (inclusive) */
 382		return (log2_in - 12) + 0x1c;
 383	else if (log2_in >= 16 && log2_in <= 36)
 384		/* Covers 64KB to 64GB, (inclusive) */
 385		return log2_in - 15;
 386	/* Something is awry so disable */
 387	return 0;
 388}
 389
 390static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd)
 391{
 392	u32 pkt = 0;
 393
 394	pkt |= FIELD_PREP(MDIO_PORT_EXT_MASK, port >> 4);
 395	pkt |= FIELD_PREP(MDIO_PORT_MASK, port);
 396	pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad);
 397	pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd);
 398
 399	return pkt;
 400}
 401
 402/* negative return value indicates error */
 403static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val)
 404{
 405	u32 data;
 406	int err;
 407
 408	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ),
 409		   base + PCIE_RC_DL_MDIO_ADDR);
 410	readl(base + PCIE_RC_DL_MDIO_ADDR);
 411	err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_RD_DATA, data,
 412					MDIO_RD_DONE(data), 10, 100);
 413	*val = FIELD_GET(MDIO_DATA_MASK, data);
 414
 415	return err;
 416}
 417
 418/* negative return value indicates error */
 419static int brcm_pcie_mdio_write(void __iomem *base, u8 port,
 420				u8 regad, u16 wrdata)
 421{
 422	u32 data;
 423	int err;
 424
 425	writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE),
 426		   base + PCIE_RC_DL_MDIO_ADDR);
 427	readl(base + PCIE_RC_DL_MDIO_ADDR);
 428	writel(MDIO_DATA_DONE_MASK | wrdata, base + PCIE_RC_DL_MDIO_WR_DATA);
 429
 430	err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_WR_DATA, data,
 431					MDIO_WT_DONE(data), 10, 100);
 432	return err;
 433}
 434
 435/*
 436 * Configures device for Spread Spectrum Clocking (SSC) mode; a negative
 437 * return value indicates error.
 438 */
 439static int brcm_pcie_set_ssc(struct brcm_pcie *pcie)
 440{
 441	int pll, ssc;
 442	int ret;
 443	u32 tmp;
 444
 445	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET,
 446				   SSC_REGS_ADDR);
 447	if (ret < 0)
 448		return ret;
 449
 450	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
 451				  SSC_CNTL_OFFSET, &tmp);
 452	if (ret < 0)
 453		return ret;
 454
 455	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_EN_MASK);
 456	u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK);
 457	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0,
 458				   SSC_CNTL_OFFSET, tmp);
 459	if (ret < 0)
 460		return ret;
 461
 462	usleep_range(1000, 2000);
 463	ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0,
 464				  SSC_STATUS_OFFSET, &tmp);
 465	if (ret < 0)
 466		return ret;
 467
 468	ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp);
 469	pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp);
 470
 471	return ssc && pll ? 0 : -EIO;
 472}
 473
 474/* Limits operation to a specific generation (1, 2, or 3) */
 475static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen)
 476{
 477	u16 lnkctl2 = readw(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
 478	u32 lnkcap = readl(pcie->base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
 479
 480	u32p_replace_bits(&lnkcap, gen, PCI_EXP_LNKCAP_SLS);
 481	writel(lnkcap, pcie->base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
 482
 483	u16p_replace_bits(&lnkctl2, gen, PCI_EXP_LNKCTL2_TLS);
 484	writew(lnkctl2, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2);
 485}
 486
 487static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie,
 488				       u8 win, u64 cpu_addr,
 489				       u64 pcie_addr, u64 size)
 490{
 491	u32 cpu_addr_mb_high, limit_addr_mb_high;
 492	phys_addr_t cpu_addr_mb, limit_addr_mb;
 493	int high_addr_shift;
 494	u32 tmp;
 495
 496	/* Set the base of the pcie_addr window */
 497	writel(lower_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_LO(win));
 498	writel(upper_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_HI(win));
 499
 500	/* Write the addr base & limit lower bits (in MBs) */
 501	cpu_addr_mb = cpu_addr / SZ_1M;
 502	limit_addr_mb = (cpu_addr + size - 1) / SZ_1M;
 503
 504	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
 505	u32p_replace_bits(&tmp, cpu_addr_mb,
 506			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
 507	u32p_replace_bits(&tmp, limit_addr_mb,
 508			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK);
 509	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win));
 510
 511	if (is_bmips(pcie))
 512		return;
 513
 514	/* Write the cpu & limit addr upper bits */
 515	high_addr_shift =
 516		HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK);
 517
 518	cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift;
 519	tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
 520	u32p_replace_bits(&tmp, cpu_addr_mb_high,
 521			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK);
 522	writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_HI(win));
 523
 524	limit_addr_mb_high = limit_addr_mb >> high_addr_shift;
 525	tmp = readl(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
 526	u32p_replace_bits(&tmp, limit_addr_mb_high,
 527			  PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK);
 528	writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
 529}
 530
 531#define BRCM_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
 532				 MSI_FLAG_USE_DEF_CHIP_OPS	| \
 533				 MSI_FLAG_NO_AFFINITY)
 534
 535#define BRCM_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
 536				  MSI_FLAG_MULTI_PCI_MSI)
 537
 538static const struct msi_parent_ops brcm_msi_parent_ops = {
 539	.required_flags		= BRCM_MSI_FLAGS_REQUIRED,
 540	.supported_flags	= BRCM_MSI_FLAGS_SUPPORTED,
 541	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
 542	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
 543	.prefix			= "BRCM-",
 544	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 545};
 546
 547static void brcm_pcie_msi_isr(struct irq_desc *desc)
 548{
 549	struct irq_chip *chip = irq_desc_get_chip(desc);
 550	unsigned long status;
 551	struct brcm_msi *msi;
 552	struct device *dev;
 553	u32 bit;
 554
 555	chained_irq_enter(chip, desc);
 556	msi = irq_desc_get_handler_data(desc);
 557	dev = msi->dev;
 558
 559	status = readl(msi->intr_base + MSI_INT_STATUS);
 560	status >>= msi->legacy_shift;
 561
 562	for_each_set_bit(bit, &status, msi->nr) {
 563		int ret;
 564		ret = generic_handle_domain_irq(msi->inner_domain, bit);
 565		if (ret)
 566			dev_dbg(dev, "unexpected MSI\n");
 567	}
 568
 569	chained_irq_exit(chip, desc);
 570}
 571
 572static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 573{
 574	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
 575
 576	msg->address_lo = lower_32_bits(msi->target_addr);
 577	msg->address_hi = upper_32_bits(msi->target_addr);
 578	msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq;
 579}
 580
 581static void brcm_msi_ack_irq(struct irq_data *data)
 582{
 583	struct brcm_msi *msi = irq_data_get_irq_chip_data(data);
 584	const int shift_amt = data->hwirq + msi->legacy_shift;
 585
 586	writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR);
 587}
 588
 589
 590static struct irq_chip brcm_msi_bottom_irq_chip = {
 591	.name			= "BRCM STB MSI",
 592	.irq_compose_msi_msg	= brcm_msi_compose_msi_msg,
 593	.irq_ack                = brcm_msi_ack_irq,
 594};
 595
 596static int brcm_msi_alloc(struct brcm_msi *msi, unsigned int nr_irqs)
 597{
 598	int hwirq;
 599
 600	mutex_lock(&msi->lock);
 601	hwirq = bitmap_find_free_region(msi->used, msi->nr,
 602					order_base_2(nr_irqs));
 603	mutex_unlock(&msi->lock);
 604
 605	return hwirq;
 606}
 607
 608static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq,
 609			  unsigned int nr_irqs)
 610{
 611	mutex_lock(&msi->lock);
 612	bitmap_release_region(msi->used, hwirq, order_base_2(nr_irqs));
 613	mutex_unlock(&msi->lock);
 614}
 615
 616static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 617				 unsigned int nr_irqs, void *args)
 618{
 619	struct brcm_msi *msi = domain->host_data;
 620	int hwirq, i;
 621
 622	hwirq = brcm_msi_alloc(msi, nr_irqs);
 623
 624	if (hwirq < 0)
 625		return hwirq;
 626
 627	for (i = 0; i < nr_irqs; i++)
 628		irq_domain_set_info(domain, virq + i, (irq_hw_number_t)hwirq + i,
 629				    &brcm_msi_bottom_irq_chip, domain->host_data,
 630				    handle_edge_irq, NULL, NULL);
 631	return 0;
 632}
 633
 634static void brcm_irq_domain_free(struct irq_domain *domain,
 635				 unsigned int virq, unsigned int nr_irqs)
 636{
 637	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 638	struct brcm_msi *msi = irq_data_get_irq_chip_data(d);
 639
 640	brcm_msi_free(msi, d->hwirq, nr_irqs);
 641}
 642
 643static const struct irq_domain_ops msi_domain_ops = {
 644	.alloc	= brcm_irq_domain_alloc,
 645	.free	= brcm_irq_domain_free,
 646};
 647
 648static int brcm_allocate_domains(struct brcm_msi *msi)
 649{
 650	struct device *dev = msi->dev;
 651
 652	struct irq_domain_info info = {
 653		.fwnode		= of_fwnode_handle(msi->np),
 654		.ops		= &msi_domain_ops,
 655		.host_data	= msi,
 656		.size		= msi->nr,
 657	};
 658
 659	msi->inner_domain = msi_create_parent_irq_domain(&info, &brcm_msi_parent_ops);
 660	if (!msi->inner_domain) {
 661		dev_err(dev, "failed to create MSI domain\n");
 662		return -ENOMEM;
 663	}
 664
 665	return 0;
 666}
 667
 668static void brcm_free_domains(struct brcm_msi *msi)
 669{
 670	irq_domain_remove(msi->inner_domain);
 671}
 672
 673static void brcm_msi_remove(struct brcm_pcie *pcie)
 674{
 675	struct brcm_msi *msi = pcie->msi;
 676
 677	if (!msi)
 678		return;
 679	irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
 680	brcm_free_domains(msi);
 681}
 682
 683static void brcm_msi_set_regs(struct brcm_msi *msi)
 684{
 685	u32 val = msi->legacy ? BRCM_INT_PCI_MSI_LEGACY_MASK :
 686				BRCM_INT_PCI_MSI_MASK;
 687
 688	writel(val, msi->intr_base + MSI_INT_MASK_CLR);
 689	writel(val, msi->intr_base + MSI_INT_CLR);
 690
 691	/*
 692	 * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI
 693	 * enable, which we set to 1.
 694	 */
 695	writel(lower_32_bits(msi->target_addr) | 0x1,
 696	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO);
 697	writel(upper_32_bits(msi->target_addr),
 698	       msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI);
 699
 700	val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32;
 701	writel(val, msi->base + PCIE_MISC_MSI_DATA_CONFIG);
 702}
 703
 704static int brcm_pcie_enable_msi(struct brcm_pcie *pcie)
 705{
 706	struct brcm_msi *msi;
 707	int irq, ret;
 708	struct device *dev = pcie->dev;
 709
 710	irq = irq_of_parse_and_map(dev->of_node, 1);
 711	if (irq <= 0) {
 712		dev_err(dev, "cannot map MSI interrupt\n");
 713		return -ENODEV;
 714	}
 715
 716	msi = devm_kzalloc(dev, sizeof(struct brcm_msi), GFP_KERNEL);
 717	if (!msi)
 718		return -ENOMEM;
 719
 720	mutex_init(&msi->lock);
 721	msi->dev = dev;
 722	msi->base = pcie->base;
 723	msi->np = pcie->np;
 724	msi->target_addr = pcie->msi_target_addr;
 725	msi->irq = irq;
 726	msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33;
 727
 728	/*
 729	 * Sanity check to make sure that the 'used' bitmap in struct brcm_msi
 730	 * is large enough.
 731	 */
 732	BUILD_BUG_ON(BRCM_INT_PCI_MSI_LEGACY_NR > BRCM_INT_PCI_MSI_NR);
 733
 734	if (msi->legacy) {
 735		msi->intr_base = msi->base + INTR2_CPU_BASE(pcie);
 736		msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR;
 737		msi->legacy_shift = 24;
 738	} else {
 739		msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE;
 740		msi->nr = BRCM_INT_PCI_MSI_NR;
 741		msi->legacy_shift = 0;
 742	}
 743
 744	ret = brcm_allocate_domains(msi);
 745	if (ret)
 746		return ret;
 747
 748	irq_set_chained_handler_and_data(msi->irq, brcm_pcie_msi_isr, msi);
 749
 750	brcm_msi_set_regs(msi);
 751	pcie->msi = msi;
 752
 753	return 0;
 754}
 755
 756/* The controller is capable of serving in both RC and EP roles */
 757static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie)
 758{
 759	void __iomem *base = pcie->base;
 760	u32 val = readl(base + PCIE_MISC_PCIE_STATUS);
 761
 762	return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val);
 763}
 764
 765static bool brcm_pcie_link_up(struct brcm_pcie *pcie)
 766{
 767	u32 val = readl(pcie->base + PCIE_MISC_PCIE_STATUS);
 768	u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val);
 769	u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val);
 770
 771	return dla && plu;
 772}
 773
 774static void __iomem *brcm_pcie_map_bus(struct pci_bus *bus,
 775				       unsigned int devfn, int where)
 776{
 777	struct brcm_pcie *pcie = bus->sysdata;
 778	void __iomem *base = pcie->base;
 779	int idx;
 780
 781	/* Accesses to the RC go right to the RC registers if !devfn */
 782	if (pci_is_root_bus(bus))
 783		return devfn ? NULL : base + PCIE_ECAM_REG(where);
 784
 785	/* An access to our HW w/o link-up will cause a CPU Abort */
 786	if (!brcm_pcie_link_up(pcie))
 787		return NULL;
 788
 789	/* For devices, write to the config space index register */
 790	idx = PCIE_ECAM_OFFSET(bus->number, devfn, 0);
 791	writel(idx, base + IDX_ADDR(pcie));
 792	return base + DATA_ADDR(pcie) + PCIE_ECAM_REG(where);
 793}
 794
 795static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus,
 796					   unsigned int devfn, int where)
 797{
 798	struct brcm_pcie *pcie = bus->sysdata;
 799	void __iomem *base = pcie->base;
 800	int idx;
 801
 802	/* Accesses to the RC go right to the RC registers if !devfn */
 803	if (pci_is_root_bus(bus))
 804		return devfn ? NULL : base + PCIE_ECAM_REG(where);
 805
 806	/* An access to our HW w/o link-up will cause a CPU Abort */
 807	if (!brcm_pcie_link_up(pcie))
 808		return NULL;
 809
 810	/* For devices, write to the config space index register */
 811	idx = PCIE_ECAM_OFFSET(bus->number, devfn, where);
 812	writel(idx, base + IDX_ADDR(pcie));
 813	return base + DATA_ADDR(pcie);
 814}
 815
 816static int brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val)
 817{
 818	u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK;
 819	u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT;
 820	int ret = 0;
 821
 822	if (pcie->bridge_reset) {
 823		if (val)
 824			ret = reset_control_assert(pcie->bridge_reset);
 825		else
 826			ret = reset_control_deassert(pcie->bridge_reset);
 827
 828		if (ret)
 829			dev_err(pcie->dev, "failed to %s 'bridge' reset, err=%d\n",
 830				val ? "assert" : "deassert", ret);
 831
 832		return ret;
 833	}
 834
 835	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
 836	tmp = (tmp & ~mask) | ((val << shift) & mask);
 837	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
 838
 839	return ret;
 840}
 841
 842static int brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val)
 843{
 844	u32 tmp, mask =  RGR1_SW_INIT_1_INIT_7278_MASK;
 845	u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT;
 846
 847	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
 848	tmp = (tmp & ~mask) | ((val << shift) & mask);
 849	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
 850
 851	return 0;
 852}
 853
 854static int brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val)
 855{
 856	int ret;
 857
 858	if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n"))
 859		return -EINVAL;
 860
 861	if (val)
 862		ret = reset_control_assert(pcie->perst_reset);
 863	else
 864		ret = reset_control_deassert(pcie->perst_reset);
 865
 866	if (ret)
 867		dev_err(pcie->dev, "failed to %s 'perst' reset, err=%d\n",
 868			val ? "assert" : "deassert", ret);
 869	return ret;
 870}
 871
 872static int brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val)
 873{
 874	u32 tmp;
 875
 876	/* Perst bit has moved and assert value is 0 */
 877	tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL);
 878	u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK);
 879	writel(tmp, pcie->base +  PCIE_MISC_PCIE_CTRL);
 880
 881	return 0;
 882}
 883
 884static int brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val)
 885{
 886	u32 tmp;
 887
 888	tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
 889	u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK);
 890	writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie));
 891
 892	return 0;
 893}
 894
 895static int brcm_pcie_post_setup_bcm2712(struct brcm_pcie *pcie)
 896{
 897	static const u16 data[] = { 0x50b9, 0xbda1, 0x0094, 0x97b4, 0x5030,
 898				    0x5030, 0x0007 };
 899	static const u8 regs[] = { 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1e };
 900	int ret, i;
 901	u32 tmp;
 902
 903	/* Allow a 54MHz (xosc) refclk source */
 904	ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, 0x1600);
 905	if (ret < 0)
 906		return ret;
 907
 908	for (i = 0; i < ARRAY_SIZE(regs); i++) {
 909		ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, regs[i], data[i]);
 910		if (ret < 0)
 911			return ret;
 912	}
 913
 914	usleep_range(100, 200);
 915
 916	/*
 917	 * Set L1SS sub-state timers to avoid lengthy state transitions,
 918	 * PM clock period is 18.52ns (1/54MHz, round down).
 919	 */
 920	tmp = readl(pcie->base + PCIE_RC_PL_PHY_CTL_15);
 921	tmp &= ~PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK;
 922	tmp |= 0x12;
 923	writel(tmp, pcie->base + PCIE_RC_PL_PHY_CTL_15);
 924
 925	return 0;
 926}
 927
 928static void add_inbound_win(struct inbound_win *b, u8 *count, u64 size,
 929			    u64 cpu_addr, u64 pci_offset)
 930{
 931	b->size = size;
 932	b->cpu_addr = cpu_addr;
 933	b->pci_offset = pci_offset;
 934	(*count)++;
 935}
 936
 937static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie,
 938				      struct inbound_win inbound_wins[])
 939{
 940	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
 941	u64 pci_offset, cpu_addr, size = 0, tot_size = 0;
 942	struct resource_entry *entry;
 943	struct device *dev = pcie->dev;
 944	u64 lowest_pcie_addr = ~(u64)0;
 945	int ret, i = 0;
 946	u8 n = 0;
 947
 948	/*
 949	 * The HW registers (and PCIe) use order-1 numbering for BARs.  As such,
 950	 * we have inbound_wins[0] unused and BAR1 starts at inbound_wins[1].
 951	 */
 952	struct inbound_win *b_begin = &inbound_wins[1];
 953	struct inbound_win *b = b_begin;
 954
 955	/*
 956	 * STB chips beside 7712 disable the first inbound window default.
 957	 * Rather being mapped to system memory it is mapped to the
 958	 * internal registers of the SoC.  This feature is deprecated, has
 959	 * security considerations, and is not implemented in our modern
 960	 * SoCs.
 961	 */
 962	if (pcie->cfg->soc_base != BCM7712)
 963		add_inbound_win(b++, &n, 0, 0, 0);
 964
 965	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
 966		u64 pcie_start = entry->res->start - entry->offset;
 967		u64 cpu_start = entry->res->start;
 968
 969		size = resource_size(entry->res);
 970		tot_size += size;
 971		if (pcie_start < lowest_pcie_addr)
 972			lowest_pcie_addr = pcie_start;
 973		/*
 974		 * 7712 and newer chips may have many BARs, with each
 975		 * offering a non-overlapping viewport to system memory.
 976		 * That being said, each BARs size must still be a power of
 977		 * two.
 978		 */
 979		if (pcie->cfg->soc_base == BCM7712)
 980			add_inbound_win(b++, &n, size, cpu_start, pcie_start);
 981
 982		if (n > pcie->cfg->num_inbound_wins)
 983			break;
 984	}
 985
 986	if (lowest_pcie_addr == ~(u64)0) {
 987		dev_err(dev, "DT node has no dma-ranges\n");
 988		return -EINVAL;
 989	}
 990
 991	/*
 992	 * 7712 and newer chips do not have an internal memory mapping system
 993	 * that enables multiple memory controllers.  As such, it can return
 994	 * now w/o doing special configuration.
 995	 */
 996	if (pcie->cfg->soc_base == BCM7712)
 997		return n;
 998
 999	ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1,
1000						  PCIE_BRCM_MAX_MEMC);
1001	if (ret <= 0) {
1002		/* Make an educated guess */
1003		pcie->num_memc = 1;
1004		pcie->memc_size[0] = 1ULL << fls64(tot_size - 1);
1005	} else {
1006		pcie->num_memc = ret;
1007	}
1008
1009	/* Each memc is viewed through a "port" that is a power of 2 */
1010	for (i = 0, size = 0; i < pcie->num_memc; i++)
1011		size += pcie->memc_size[i];
1012
1013	/* Our HW mandates that the window size must be a power of 2 */
1014	size = 1ULL << fls64(size - 1);
1015
1016	/*
1017	 * For STB chips, the BAR2 cpu_addr is hardwired to the start
1018	 * of system memory, so we set it to 0.
1019	 */
1020	cpu_addr = 0;
1021	pci_offset = lowest_pcie_addr;
1022
1023	/*
1024	 * We validate the inbound memory view even though we should trust
1025	 * whatever the device-tree provides. This is because of an HW issue on
1026	 * early Raspberry Pi 4's revisions (bcm2711). It turns out its
1027	 * firmware has to dynamically edit dma-ranges due to a bug on the
1028	 * PCIe controller integration, which prohibits any access above the
1029	 * lower 3GB of memory. Given this, we decided to keep the dma-ranges
1030	 * in check, avoiding hard to debug device-tree related issues in the
1031	 * future:
1032	 *
1033	 * The PCIe host controller by design must set the inbound viewport to
1034	 * be a contiguous arrangement of all of the system's memory.  In
1035	 * addition, its size must be a power of two.  To further complicate
1036	 * matters, the viewport must start on a pcie-address that is aligned
1037	 * on a multiple of its size.  If a portion of the viewport does not
1038	 * represent system memory -- e.g. 3GB of memory requires a 4GB
1039	 * viewport -- we can map the outbound memory in or after 3GB and even
1040	 * though the viewport will overlap the outbound memory the controller
1041	 * will know to send outbound memory downstream and everything else
1042	 * upstream.
1043	 *
1044	 * For example:
1045	 *
1046	 * - The best-case scenario, memory up to 3GB, is to place the inbound
1047	 *   region in the first 4GB of pcie-space, as some legacy devices can
1048	 *   only address 32bits. We would also like to put the MSI under 4GB
1049	 *   as well, since some devices require a 32bit MSI target address.
1050	 *
1051	 * - If the system memory is 4GB or larger we cannot start the inbound
1052	 *   region at location 0 (since we have to allow some space for
1053	 *   outbound memory @ 3GB). So instead it will  start at the 1x
1054	 *   multiple of its size
1055	 */
1056	if (!size || (pci_offset & (size - 1)) ||
1057	    (pci_offset < SZ_4G && pci_offset > SZ_2G)) {
1058		dev_err(dev, "Invalid inbound_win2_offset/size: size 0x%llx, off 0x%llx\n",
1059			size, pci_offset);
1060		return -EINVAL;
1061	}
1062
1063	/* Enable inbound window 2, the main inbound window for STB chips */
1064	add_inbound_win(b++, &n, size, cpu_addr, pci_offset);
1065
1066	/*
1067	 * Disable inbound window 3.  On some chips presents the same
1068	 * window as #2 but the data appears in a settable endianness.
1069	 */
1070	add_inbound_win(b++, &n, 0, 0, 0);
1071
1072	return n;
1073}
1074
1075static u32 brcm_bar_reg_offset(int bar)
1076{
1077	if (bar <= 3)
1078		return PCIE_MISC_RC_BAR1_CONFIG_LO + 8 * (bar - 1);
1079	else
1080		return PCIE_MISC_RC_BAR4_CONFIG_LO + 8 * (bar - 4);
1081}
1082
1083static u32 brcm_ubus_reg_offset(int bar)
1084{
1085	if (bar <= 3)
1086		return PCIE_MISC_UBUS_BAR1_CONFIG_REMAP + 8 * (bar - 1);
1087	else
1088		return PCIE_MISC_UBUS_BAR4_CONFIG_REMAP + 8 * (bar - 4);
1089}
1090
1091static void set_inbound_win_registers(struct brcm_pcie *pcie,
1092				      const struct inbound_win *inbound_wins,
1093				      u8 num_inbound_wins)
1094{
1095	void __iomem *base = pcie->base;
1096	int i;
1097
1098	for (i = 1; i <= num_inbound_wins; i++) {
1099		u64 pci_offset = inbound_wins[i].pci_offset;
1100		u64 cpu_addr = inbound_wins[i].cpu_addr;
1101		u64 size = inbound_wins[i].size;
1102		u32 reg_offset = brcm_bar_reg_offset(i);
1103		u32 tmp = lower_32_bits(pci_offset);
1104
1105		u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(size),
1106				  PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK);
1107
1108		/* Write low */
1109		writel_relaxed(tmp, base + reg_offset);
1110		/* Write high */
1111		writel_relaxed(upper_32_bits(pci_offset), base + reg_offset + 4);
1112
1113		/*
1114		 * Most STB chips:
1115		 *     Do nothing.
1116		 * 7712:
1117		 *     All of their BARs need to be set.
1118		 */
1119		if (pcie->cfg->soc_base == BCM7712) {
1120			/* BUS remap register settings */
1121			reg_offset = brcm_ubus_reg_offset(i);
1122			tmp = lower_32_bits(cpu_addr) & ~0xfff;
1123			tmp |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK;
1124			writel_relaxed(tmp, base + reg_offset);
1125			tmp = upper_32_bits(cpu_addr);
1126			writel_relaxed(tmp, base + reg_offset + 4);
1127		}
1128	}
1129}
1130
1131static int brcm_pcie_setup(struct brcm_pcie *pcie)
1132{
1133	struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS];
1134	void __iomem *base = pcie->base;
1135	struct pci_host_bridge *bridge;
1136	struct resource_entry *entry;
1137	u32 tmp, burst, num_lanes, num_lanes_cap;
1138	u8 num_out_wins = 0;
1139	int num_inbound_wins = 0;
1140	int memc, ret;
1141
1142	/* Reset the bridge */
1143	ret = brcm_pcie_bridge_sw_init_set(pcie, 1);
1144	if (ret)
1145		return ret;
1146
1147	/* Ensure that PERST# is asserted; some bootloaders may deassert it. */
1148	if (pcie->cfg->soc_base == BCM2711) {
1149		ret = pcie->cfg->perst_set(pcie, 1);
1150		if (ret) {
1151			pcie->cfg->bridge_sw_init_set(pcie, 0);
1152			return ret;
1153		}
1154	}
1155
1156	usleep_range(100, 200);
1157
1158	/* Take the bridge out of reset */
1159	ret = brcm_pcie_bridge_sw_init_set(pcie, 0);
1160	if (ret)
1161		return ret;
1162
1163	tmp = readl(base + HARD_DEBUG(pcie));
1164	if (is_bmips(pcie))
1165		tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
1166	else
1167		tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK;
1168	writel(tmp, base + HARD_DEBUG(pcie));
1169	/* Wait for SerDes to be stable */
1170	usleep_range(100, 200);
1171
1172	/*
1173	 * SCB_MAX_BURST_SIZE is a two bit field.  For GENERIC chips it
1174	 * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it
1175	 * is encoded as 0=Rsvd, 1=128, 2=256, 3=512.
1176	 */
1177	if (is_bmips(pcie))
1178		burst = 0x1; /* 256 bytes */
1179	else if (pcie->cfg->soc_base == BCM2711)
1180		burst = 0x0; /* 128 bytes */
1181	else if (pcie->cfg->soc_base == BCM7278)
1182		burst = 0x3; /* 512 bytes */
1183	else
1184		burst = 0x2; /* 512 bytes */
1185
1186	/*
1187	 * Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN,
1188	 * RCB_MPS_MODE, RCB_64B_MODE
1189	 */
1190	tmp = readl(base + PCIE_MISC_MISC_CTRL);
1191	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK);
1192	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK);
1193	u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK);
1194	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK);
1195	u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK);
1196	writel(tmp, base + PCIE_MISC_MISC_CTRL);
1197
1198	num_inbound_wins = brcm_pcie_get_inbound_wins(pcie, inbound_wins);
1199	if (num_inbound_wins < 0)
1200		return num_inbound_wins;
1201
1202	set_inbound_win_registers(pcie, inbound_wins, num_inbound_wins);
1203
1204	if (!brcm_pcie_rc_mode(pcie)) {
1205		dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n");
1206		return -EINVAL;
1207	}
1208
1209	tmp = readl(base + PCIE_MISC_MISC_CTRL);
1210	for (memc = 0; memc < pcie->num_memc; memc++) {
1211		u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15;
1212
1213		if (memc == 0)
1214			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(0));
1215		else if (memc == 1)
1216			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(1));
1217		else if (memc == 2)
1218			u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(2));
1219	}
1220	writel(tmp, base + PCIE_MISC_MISC_CTRL);
1221
1222	/*
1223	 * We ideally want the MSI target address to be located in the 32bit
1224	 * addressable memory area. Some devices might depend on it. This is
1225	 * possible either when the inbound window is located above the lower
1226	 * 4GB or when the inbound area is smaller than 4GB (taking into
1227	 * account the rounding-up we're forced to perform).
1228	 */
1229	if (inbound_wins[2].pci_offset >= SZ_4G ||
1230	    (inbound_wins[2].size + inbound_wins[2].pci_offset) < SZ_4G)
1231		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB;
1232	else
1233		pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB;
1234
1235
1236	/* Don't advertise L0s capability if 'aspm-no-l0s' */
1237	tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
1238	if (of_property_read_bool(pcie->np, "aspm-no-l0s"))
1239		tmp &= ~PCI_EXP_LNKCAP_ASPM_L0S;
1240	writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
1241
1242	/* 'tmp' still holds the contents of PRIV1_LINK_CAPABILITY */
1243	num_lanes_cap = u32_get_bits(tmp, PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK);
1244	num_lanes = 0;
1245
1246	/*
1247	 * Use hardware negotiated Max Link Width value by default.  If the
1248	 * "num-lanes" DT property is present, assume that the chip's default
1249	 * link width capability information is incorrect/undesired and use the
1250	 * specified value instead.
1251	 */
1252	if (!of_property_read_u32(pcie->np, "num-lanes", &num_lanes) &&
1253	    num_lanes && num_lanes <= 4 && num_lanes_cap != num_lanes) {
1254		u32p_replace_bits(&tmp, num_lanes,
1255			PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK);
1256		writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
1257		tmp = readl(base + PCIE_RC_PL_REG_PHY_CTL_1);
1258		u32p_replace_bits(&tmp, 1,
1259			PCIE_RC_PL_REG_PHY_CTL_1_REG_P2_POWERDOWN_ENA_NOSYNC_MASK);
1260		writel(tmp, base + PCIE_RC_PL_REG_PHY_CTL_1);
1261	}
1262
1263	/*
1264	 * For config space accesses on the RC, show the right class for
1265	 * a PCIe-PCIe bridge (the default setting is to be EP mode).
1266	 */
1267	tmp = readl(base + PCIE_RC_CFG_PRIV1_ID_VAL3);
1268	u32p_replace_bits(&tmp, 0x060400,
1269			  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK);
1270	writel(tmp, base + PCIE_RC_CFG_PRIV1_ID_VAL3);
1271
1272	bridge = pci_host_bridge_from_priv(pcie);
1273	resource_list_for_each_entry(entry, &bridge->windows) {
1274		struct resource *res = entry->res;
1275
1276		if (resource_type(res) != IORESOURCE_MEM)
1277			continue;
1278
1279		if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) {
1280			dev_err(pcie->dev, "too many outbound wins\n");
1281			return -EINVAL;
1282		}
1283
1284		if (is_bmips(pcie)) {
1285			u64 start = res->start;
1286			unsigned int j, nwins = resource_size(res) / SZ_128M;
1287
1288			/* bmips PCIe outbound windows have a 128MB max size */
1289			if (nwins > BRCM_NUM_PCIE_OUT_WINS)
1290				nwins = BRCM_NUM_PCIE_OUT_WINS;
1291			for (j = 0; j < nwins; j++, start += SZ_128M)
1292				brcm_pcie_set_outbound_win(pcie, j, start,
1293							   start - entry->offset,
1294							   SZ_128M);
1295			break;
1296		}
1297		brcm_pcie_set_outbound_win(pcie, num_out_wins, res->start,
1298					   res->start - entry->offset,
1299					   resource_size(res));
1300		num_out_wins++;
1301	}
1302
1303	/* PCIe->SCB endian mode for inbound window */
1304	tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1305	u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPECIFIC_REG1_LITTLE_ENDIAN,
1306		PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK);
1307	writel(tmp, base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1);
1308
1309	if (pcie->cfg->post_setup) {
1310		ret = pcie->cfg->post_setup(pcie);
1311		if (ret < 0)
1312			return ret;
1313	}
1314
1315	return 0;
1316}
1317
1318/*
1319 * This extends the timeout period for an access to an internal bus.  This
1320 * access timeout may occur during L1SS sleep periods, even without the
1321 * presence of a PCIe access.
1322 */
1323static void brcm_extend_rbus_timeout(struct brcm_pcie *pcie)
1324{
1325	/* TIMEOUT register is two registers before RGR1_SW_INIT_1 */
1326	const unsigned int REG_OFFSET = PCIE_RGR1_SW_INIT_1(pcie) - 8;
1327	u32 timeout_us = 4000000; /* 4 seconds, our setting for L1SS */
1328
1329	/* 7712 does not have this (RGR1) timer */
1330	if (pcie->cfg->soc_base == BCM7712)
1331		return;
1332
1333	/* Each unit in timeout register is 1/216,000,000 seconds */
1334	writel(216 * timeout_us, pcie->base + REG_OFFSET);
1335}
1336
1337static void brcm_config_clkreq(struct brcm_pcie *pcie)
1338{
1339	static const char err_msg[] = "invalid 'brcm,clkreq-mode' DT string\n";
1340	const char *mode = "default";
1341	u32 clkreq_cntl;
1342	int ret, tmp;
1343
1344	ret = of_property_read_string(pcie->np, "brcm,clkreq-mode", &mode);
1345	if (ret && ret != -EINVAL) {
1346		dev_err(pcie->dev, err_msg);
1347		mode = "safe";
1348	}
1349
1350	/* Start out assuming safe mode (both mode bits cleared) */
1351	clkreq_cntl = readl(pcie->base + HARD_DEBUG(pcie));
1352	clkreq_cntl &= ~PCIE_CLKREQ_MASK;
1353
1354	if (strcmp(mode, "no-l1ss") == 0) {
1355		/*
1356		 * "no-l1ss" -- Provides Clock Power Management, L0s, and
1357		 * L1, but cannot provide L1 substate (L1SS) power
1358		 * savings. If the downstream device connected to the RC is
1359		 * L1SS capable AND the OS enables L1SS, all PCIe traffic
1360		 * may abruptly halt, potentially hanging the system.
1361		 */
1362		clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK;
1363		/*
1364		 * We want to un-advertise L1 substates because if the OS
1365		 * tries to configure the controller into using L1 substate
1366		 * power savings it may fail or hang when the RC HW is in
1367		 * "no-l1ss" mode.
1368		 */
1369		tmp = readl(pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP);
1370		u32p_replace_bits(&tmp, 2, PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK);
1371		writel(tmp, pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP);
1372
1373	} else if (strcmp(mode, "default") == 0) {
1374		/*
1375		 * "default" -- Provides L0s, L1, and L1SS, but not
1376		 * compliant to provide Clock Power Management;
1377		 * specifically, may not be able to meet the Tclron max
1378		 * timing of 400ns as specified in "Dynamic Clock Control",
1379		 * section 3.2.5.2.2 of the PCIe spec.  This situation is
1380		 * atypical and should happen only with older devices.
1381		 */
1382		clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK;
1383		brcm_extend_rbus_timeout(pcie);
1384
1385	} else {
1386		/*
1387		 * "safe" -- No power savings; refclk is driven by RC
1388		 * unconditionally.
1389		 */
1390		if (strcmp(mode, "safe") != 0)
1391			dev_err(pcie->dev, err_msg);
1392		mode = "safe";
1393	}
1394	writel(clkreq_cntl, pcie->base + HARD_DEBUG(pcie));
1395
1396	dev_info(pcie->dev, "clkreq-mode set to %s\n", mode);
1397}
1398
1399static int brcm_pcie_start_link(struct brcm_pcie *pcie)
1400{
1401	struct device *dev = pcie->dev;
1402	void __iomem *base = pcie->base;
1403	u16 nlw, cls, lnksta;
1404	bool ssc_good = false;
1405	int ret, i;
1406
1407	/* Limit the generation if specified */
1408	if (pcie->gen)
1409		brcm_pcie_set_gen(pcie, pcie->gen);
1410
1411	/* Unassert the fundamental reset */
1412	ret = pcie->cfg->perst_set(pcie, 0);
1413	if (ret)
1414		return ret;
1415
1416	msleep(PCIE_RESET_CONFIG_WAIT_MS);
1417
1418	/*
1419	 * Give the RC/EP even more time to wake up, before trying to
1420	 * configure RC.  Intermittently check status for link-up, up to a
1421	 * total of 100ms.
1422	 */
1423	for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5)
1424		msleep(5);
1425
1426	if (!brcm_pcie_link_up(pcie)) {
1427		dev_err(dev, "link down\n");
1428		return -ENODEV;
1429	}
1430
1431	brcm_config_clkreq(pcie);
1432
1433	if (pcie->ssc) {
1434		ret = brcm_pcie_set_ssc(pcie);
1435		if (ret == 0)
1436			ssc_good = true;
1437		else
1438			dev_err(dev, "failed attempt to enter ssc mode\n");
1439	}
1440
1441	lnksta = readw(base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA);
1442	cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta);
1443	nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
1444	dev_info(dev, "link up, %s x%u %s\n",
1445		 pci_speed_string(pcie_link_speed[cls]), nlw,
1446		 ssc_good ? "(SSC)" : "(!SSC)");
1447
1448	return 0;
1449}
1450
1451static const char * const supplies[] = {
1452	"vpcie3v3",
1453	"vpcie3v3aux",
1454	"vpcie12v",
1455};
1456
1457static void *alloc_subdev_regulators(struct device *dev)
1458{
1459	const size_t size = sizeof(struct subdev_regulators) +
1460		sizeof(struct regulator_bulk_data) * ARRAY_SIZE(supplies);
1461	struct subdev_regulators *sr;
1462	int i;
1463
1464	sr = devm_kzalloc(dev, size, GFP_KERNEL);
1465	if (sr) {
1466		sr->num_supplies = ARRAY_SIZE(supplies);
1467		for (i = 0; i < ARRAY_SIZE(supplies); i++)
1468			sr->supplies[i].supply = supplies[i];
1469	}
1470
1471	return sr;
1472}
1473
1474static int brcm_pcie_add_bus(struct pci_bus *bus)
1475{
1476	struct brcm_pcie *pcie = bus->sysdata;
1477	struct device *dev = &bus->dev;
1478	struct subdev_regulators *sr;
1479	int ret;
1480
1481	if (!bus->parent || !pci_is_root_bus(bus->parent))
1482		return 0;
1483
1484	if (dev->of_node) {
1485		sr = alloc_subdev_regulators(dev);
1486		if (!sr) {
1487			dev_info(dev, "Can't allocate regulators for downstream device\n");
1488			goto no_regulators;
1489		}
1490
1491		pcie->sr = sr;
1492
1493		ret = regulator_bulk_get(dev, sr->num_supplies, sr->supplies);
1494		if (ret) {
1495			dev_info(dev, "Did not get regulators, err=%d\n", ret);
1496			pcie->sr = NULL;
1497			goto no_regulators;
1498		}
1499
1500		ret = regulator_bulk_enable(sr->num_supplies, sr->supplies);
1501		if (ret) {
1502			dev_err(dev, "Can't enable regulators for downstream device\n");
1503			regulator_bulk_free(sr->num_supplies, sr->supplies);
1504			pcie->sr = NULL;
1505		}
1506	}
1507
1508no_regulators:
1509	brcm_pcie_start_link(pcie);
1510	return 0;
1511}
1512
1513static void brcm_pcie_remove_bus(struct pci_bus *bus)
1514{
1515	struct brcm_pcie *pcie = bus->sysdata;
1516	struct subdev_regulators *sr = pcie->sr;
1517	struct device *dev = &bus->dev;
1518
1519	if (!sr || !bus->parent || !pci_is_root_bus(bus->parent))
1520		return;
1521
1522	if (regulator_bulk_disable(sr->num_supplies, sr->supplies))
1523		dev_err(dev, "Failed to disable regulators for downstream device\n");
1524	regulator_bulk_free(sr->num_supplies, sr->supplies);
1525	pcie->sr = NULL;
1526}
1527
1528/* L23 is a low-power PCIe link state */
1529static void brcm_pcie_enter_l23(struct brcm_pcie *pcie)
1530{
1531	void __iomem *base = pcie->base;
1532	int l23, i;
1533	u32 tmp;
1534
1535	/* Assert request for L23 */
1536	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1537	u32p_replace_bits(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1538	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1539
1540	/* Wait up to 36 msec for L23 */
1541	tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1542	l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp);
1543	for (i = 0; i < 15 && !l23; i++) {
1544		usleep_range(2000, 2400);
1545		tmp = readl(base + PCIE_MISC_PCIE_STATUS);
1546		l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK,
1547				tmp);
1548	}
1549
1550	if (!l23)
1551		dev_err(pcie->dev, "failed to enter low-power link state\n");
1552}
1553
1554static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start)
1555{
1556	static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1557		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT,
1558		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT,
1559		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,};
1560	static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = {
1561		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK,
1562		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK,
1563		PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,};
1564	const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1;
1565	const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1;
1566	u32 tmp, combined_mask = 0;
1567	u32 val;
1568	void __iomem *base = pcie->base;
1569	int i, ret;
1570
1571	for (i = beg; i != end; start ? i++ : i--) {
1572		val = start ? BIT_MASK(shifts[i]) : 0;
1573		tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1574		tmp = (tmp & ~masks[i]) | (val & masks[i]);
1575		writel(tmp, base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1576		usleep_range(50, 200);
1577		combined_mask |= masks[i];
1578	}
1579
1580	tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL);
1581	val = start ? combined_mask : 0;
1582
1583	ret = (tmp & combined_mask) == val ? 0 : -EIO;
1584	if (ret)
1585		dev_err(pcie->dev, "failed to %s phy\n", (start ? "start" : "stop"));
1586
1587	return ret;
1588}
1589
1590static inline int brcm_phy_start(struct brcm_pcie *pcie)
1591{
1592	return pcie->cfg->has_phy ? brcm_phy_cntl(pcie, 1) : 0;
1593}
1594
1595static inline int brcm_phy_stop(struct brcm_pcie *pcie)
1596{
1597	return pcie->cfg->has_phy ? brcm_phy_cntl(pcie, 0) : 0;
1598}
1599
1600static int brcm_pcie_turn_off(struct brcm_pcie *pcie)
1601{
1602	void __iomem *base = pcie->base;
1603	int tmp, ret;
1604
1605	if (brcm_pcie_link_up(pcie))
1606		brcm_pcie_enter_l23(pcie);
1607	/* Assert fundamental reset */
1608	ret = pcie->cfg->perst_set(pcie, 1);
1609	if (ret)
1610		return ret;
1611
1612	/* Deassert request for L23 in case it was asserted */
1613	tmp = readl(base + PCIE_MISC_PCIE_CTRL);
1614	u32p_replace_bits(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK);
1615	writel(tmp, base + PCIE_MISC_PCIE_CTRL);
1616
1617	/* Turn off SerDes */
1618	tmp = readl(base + HARD_DEBUG(pcie));
1619	u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1620	writel(tmp, base + HARD_DEBUG(pcie));
1621
1622	if (!(pcie->cfg->quirks & CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN))
1623		/* Shutdown PCIe bridge */
1624		ret = brcm_pcie_bridge_sw_init_set(pcie, 1);
1625
1626	return ret;
1627}
1628
1629static int pci_dev_may_wakeup(struct pci_dev *dev, void *data)
1630{
1631	bool *ret = data;
1632
1633	if (device_may_wakeup(&dev->dev)) {
1634		*ret = true;
1635		dev_info(&dev->dev, "Possible wake-up device; regulators will not be disabled\n");
1636	}
1637	return (int) *ret;
1638}
1639
1640static int brcm_pcie_suspend_noirq(struct device *dev)
1641{
1642	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1643	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1644	int ret, rret;
1645
1646	ret = brcm_pcie_turn_off(pcie);
1647	if (ret)
1648		return ret;
1649
1650	/*
1651	 * If brcm_phy_stop() returns an error, just dev_err(). If we
1652	 * return the error it will cause the suspend to fail and this is a
1653	 * forgivable offense that will probably be erased on resume.
1654	 */
1655	if (brcm_phy_stop(pcie))
1656		dev_err(dev, "Could not stop phy for suspend\n");
1657
1658	ret = reset_control_rearm(pcie->rescal);
1659	if (ret) {
1660		dev_err(dev, "Could not rearm rescal reset\n");
1661		return ret;
1662	}
1663
1664	if (pcie->sr) {
1665		/*
1666		 * Now turn off the regulators, but if at least one
1667		 * downstream device is enabled as a wake-up source, do not
1668		 * turn off regulators.
1669		 */
1670		pcie->ep_wakeup_capable = false;
1671		pci_walk_bus(bridge->bus, pci_dev_may_wakeup,
1672			     &pcie->ep_wakeup_capable);
1673		if (!pcie->ep_wakeup_capable) {
1674			ret = regulator_bulk_disable(pcie->sr->num_supplies,
1675						     pcie->sr->supplies);
1676			if (ret) {
1677				dev_err(dev, "Could not turn off regulators\n");
1678				rret = reset_control_reset(pcie->rescal);
1679				if (rret)
1680					dev_err(dev, "failed to reset 'rascal' controller ret=%d\n",
1681						rret);
1682				return ret;
1683			}
1684		}
1685	}
1686	clk_disable_unprepare(pcie->clk);
1687
1688	return 0;
1689}
1690
1691static int brcm_pcie_resume_noirq(struct device *dev)
1692{
1693	struct brcm_pcie *pcie = dev_get_drvdata(dev);
1694	void __iomem *base;
1695	u32 tmp;
1696	int ret, rret;
1697
1698	base = pcie->base;
1699	ret = clk_prepare_enable(pcie->clk);
1700	if (ret)
1701		return ret;
1702
1703	ret = reset_control_reset(pcie->rescal);
1704	if (ret)
1705		goto err_disable_clk;
1706
1707	ret = brcm_phy_start(pcie);
1708	if (ret)
1709		goto err_reset;
1710
1711	/* Take bridge out of reset so we can access the SERDES reg */
1712	ret = brcm_pcie_bridge_sw_init_set(pcie, 0);
1713	if (ret)
1714		goto err_reset;
1715
1716	/* SERDES_IDDQ = 0 */
1717	tmp = readl(base + HARD_DEBUG(pcie));
1718	u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK);
1719	writel(tmp, base + HARD_DEBUG(pcie));
1720
1721	/* wait for serdes to be stable */
1722	udelay(100);
1723
1724	ret = brcm_pcie_setup(pcie);
1725	if (ret)
1726		goto err_reset;
1727
1728	if (pcie->sr) {
1729		if (pcie->ep_wakeup_capable) {
1730			/*
1731			 * We are resuming from a suspend.  In the suspend we
1732			 * did not disable the power supplies, so there is
1733			 * no need to enable them (and falsely increase their
1734			 * usage count).
1735			 */
1736			pcie->ep_wakeup_capable = false;
1737		} else {
1738			ret = regulator_bulk_enable(pcie->sr->num_supplies,
1739						    pcie->sr->supplies);
1740			if (ret) {
1741				dev_err(dev, "Could not turn on regulators\n");
1742				goto err_reset;
1743			}
1744		}
1745	}
1746
1747	ret = brcm_pcie_start_link(pcie);
1748	if (ret)
1749		goto err_regulator;
1750
1751	if (pcie->msi)
1752		brcm_msi_set_regs(pcie->msi);
1753
1754	return 0;
1755
1756err_regulator:
1757	if (pcie->sr)
1758		regulator_bulk_disable(pcie->sr->num_supplies, pcie->sr->supplies);
1759err_reset:
1760	rret = reset_control_rearm(pcie->rescal);
1761	if (rret)
1762		dev_err(pcie->dev, "failed to rearm 'rescal' reset, err=%d\n", rret);
1763err_disable_clk:
1764	clk_disable_unprepare(pcie->clk);
1765	return ret;
1766}
1767
1768/* Dump out PCIe errors on die or panic */
1769static int brcm_pcie_dump_err(struct brcm_pcie *pcie,
1770			       const char *type)
1771{
1772	void __iomem *base = pcie->base;
1773	int i, is_cfg_err, is_mem_err, lanes;
1774	const char *width_str, *direction_str;
1775	u32 info, cfg_addr, cfg_cause, mem_cause, lo, hi;
1776	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1777	unsigned long flags;
1778	char lanes_str[9];
1779
1780	spin_lock_irqsave(&pcie->bridge_lock, flags);
1781	/* Don't access registers when the bridge is off */
1782	if (pcie->bridge_in_reset || readl(base + PCIE_OUTB_ERR_VALID) == 0) {
1783		spin_unlock_irqrestore(&pcie->bridge_lock, flags);
1784		return NOTIFY_DONE;
1785	}
1786
1787	/* Read all necessary registers so we can release the spinlock ASAP */
1788	info = readl(base + PCIE_OUTB_ERR_ACC_INFO);
1789	is_cfg_err = !!(info & PCIE_OUTB_ERR_ACC_INFO_CFG_ERR);
1790	is_mem_err = !!(info & PCIE_OUTB_ERR_ACC_INFO_MEM_ERR);
1791	if (is_cfg_err) {
1792		cfg_addr = readl(base + PCIE_OUTB_ERR_ACC_ADDR);
1793		cfg_cause = readl(base + PCIE_OUTB_ERR_CFG_CAUSE);
1794	}
1795	if (is_mem_err) {
1796		mem_cause = readl(base + PCIE_OUTB_ERR_MEM_CAUSE);
1797		lo = readl(base + PCIE_OUTB_ERR_MEM_ADDR_LO);
1798		hi = readl(base + PCIE_OUTB_ERR_MEM_ADDR_HI);
1799	}
1800	/* We've got all of the info, clear the error */
1801	writel(1, base + PCIE_OUTB_ERR_CLEAR);
1802	spin_unlock_irqrestore(&pcie->bridge_lock, flags);
1803
1804	dev_err(pcie->dev, "reporting PCIe info which may be related to %s error\n",
1805		type);
1806	width_str = (info & PCIE_OUTB_ERR_ACC_INFO_TYPE_64) ? "64bit" : "32bit";
1807	direction_str = str_read_write(!(info & PCIE_OUTB_ERR_ACC_INFO_DIR_WRITE));
1808	lanes = FIELD_GET(PCIE_OUTB_ERR_ACC_INFO_BYTE_LANES, info);
1809	for (i = 0, lanes_str[8] = 0; i < 8; i++)
1810		lanes_str[i] = (lanes & (1 << i)) ? '1' : '0';
1811
1812	if (is_cfg_err) {
1813		int bus = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_BUS, cfg_addr);
1814		int dev = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_DEV, cfg_addr);
1815		int func = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_FUNC, cfg_addr);
1816		int reg = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_REG, cfg_addr);
1817
1818		dev_err(pcie->dev, "Error: CFG Acc, %s, %s (%04x:%02x:%02x.%d) reg=0x%x, lanes=%s\n",
1819			width_str, direction_str, bridge->domain_nr, bus, dev,
1820			func, reg, lanes_str);
1821		dev_err(pcie->dev, " Type: TO=%d Abt=%d UnsupReq=%d AccTO=%d AccDsbld=%d Acc64bit=%d\n",
1822			!!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_TIMEOUT),
1823			!!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ABORT),
1824			!!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_UNSUPP_REQ),
1825			!!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ACC_TIMEOUT),
1826			!!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ACC_DISABLED),
1827			!!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ACC_64BIT));
1828	}
1829
1830	if (is_mem_err) {
1831		u64 addr = ((u64)hi << 32) | (u64)lo;
1832
1833		dev_err(pcie->dev, "Error: Mem Acc, %s, %s, @0x%llx, lanes=%s\n",
1834			width_str, direction_str, addr, lanes_str);
1835		dev_err(pcie->dev, " Type: TO=%d Abt=%d UnsupReq=%d AccDsble=%d BadAddr=%d\n",
1836			!!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_TIMEOUT),
1837			!!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_ABORT),
1838			!!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_UNSUPP_REQ),
1839			!!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_ACC_DISABLED),
1840			!!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_BAD_ADDR));
1841	}
1842
1843	return NOTIFY_DONE;
1844}
1845
1846static int brcm_pcie_die_notify_cb(struct notifier_block *self,
1847				   unsigned long v, void *p)
1848{
1849	struct brcm_pcie *pcie =
1850		container_of(self, struct brcm_pcie, die_notifier);
1851
1852	return brcm_pcie_dump_err(pcie, "Die");
1853}
1854
1855static int brcm_pcie_panic_notify_cb(struct notifier_block *self,
1856				     unsigned long v, void *p)
1857{
1858	struct brcm_pcie *pcie =
1859		container_of(self, struct brcm_pcie, panic_notifier);
1860
1861	return brcm_pcie_dump_err(pcie, "Panic");
1862}
1863
1864static void brcm_register_die_notifiers(struct brcm_pcie *pcie)
1865{
1866	pcie->panic_notifier.notifier_call = brcm_pcie_panic_notify_cb;
1867	atomic_notifier_chain_register(&panic_notifier_list,
1868				       &pcie->panic_notifier);
1869
1870	pcie->die_notifier.notifier_call = brcm_pcie_die_notify_cb;
1871	register_die_notifier(&pcie->die_notifier);
1872}
1873
1874static void brcm_unregister_die_notifiers(struct brcm_pcie *pcie)
1875{
1876	unregister_die_notifier(&pcie->die_notifier);
1877	atomic_notifier_chain_unregister(&panic_notifier_list,
1878					 &pcie->panic_notifier);
1879}
1880
1881static void __brcm_pcie_remove(struct brcm_pcie *pcie)
1882{
1883	brcm_msi_remove(pcie);
1884	brcm_pcie_turn_off(pcie);
1885	if (brcm_phy_stop(pcie))
1886		dev_err(pcie->dev, "Could not stop phy\n");
1887	if (reset_control_rearm(pcie->rescal))
1888		dev_err(pcie->dev, "Could not rearm rescal reset\n");
1889	clk_disable_unprepare(pcie->clk);
1890}
1891
1892static void brcm_pcie_remove(struct platform_device *pdev)
1893{
1894	struct brcm_pcie *pcie = platform_get_drvdata(pdev);
1895	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
1896
1897	pci_stop_root_bus(bridge->bus);
1898	pci_remove_root_bus(bridge->bus);
1899	if (pcie->cfg->has_err_report)
1900		brcm_unregister_die_notifiers(pcie);
1901
1902	__brcm_pcie_remove(pcie);
1903}
1904
1905static const int pcie_offsets[] = {
1906	[RGR1_SW_INIT_1]	= 0x9210,
1907	[EXT_CFG_INDEX]		= 0x9000,
1908	[EXT_CFG_DATA]		= 0x8000,
1909	[PCIE_HARD_DEBUG]	= 0x4204,
1910	[PCIE_INTR2_CPU_BASE]	= 0x4300,
1911};
1912
1913static const int pcie_offsets_bcm7278[] = {
1914	[RGR1_SW_INIT_1]	= 0xc010,
1915	[EXT_CFG_INDEX]		= 0x9000,
1916	[EXT_CFG_DATA]		= 0x8000,
1917	[PCIE_HARD_DEBUG]	= 0x4204,
1918	[PCIE_INTR2_CPU_BASE]	= 0x4300,
1919};
1920
1921static const int pcie_offsets_bcm7425[] = {
1922	[RGR1_SW_INIT_1]	= 0x8010,
1923	[EXT_CFG_INDEX]		= 0x8300,
1924	[EXT_CFG_DATA]		= 0x8304,
1925	[PCIE_HARD_DEBUG]	= 0x4204,
1926	[PCIE_INTR2_CPU_BASE]	= 0x4300,
1927};
1928
1929static const int pcie_offsets_bcm7712[] = {
1930	[RGR1_SW_INIT_1]	= 0x9210,
1931	[EXT_CFG_INDEX]		= 0x9000,
1932	[EXT_CFG_DATA]		= 0x8000,
1933	[PCIE_HARD_DEBUG]	= 0x4304,
1934	[PCIE_INTR2_CPU_BASE]	= 0x4400,
1935};
1936
1937static const struct pcie_cfg_data generic_cfg = {
1938	.offsets	= pcie_offsets,
1939	.soc_base	= GENERIC,
1940	.perst_set	= brcm_pcie_perst_set_generic,
1941	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1942	.num_inbound_wins = 3,
1943};
1944
1945static const struct pcie_cfg_data bcm2711_cfg = {
1946	.offsets	= pcie_offsets,
1947	.soc_base	= BCM2711,
1948	.perst_set	= brcm_pcie_perst_set_generic,
1949	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1950	.num_inbound_wins = 3,
1951};
1952
1953static const struct pcie_cfg_data bcm2712_cfg = {
1954	.offsets	= pcie_offsets_bcm7712,
1955	.soc_base	= BCM7712,
1956	.perst_set	= brcm_pcie_perst_set_7278,
1957	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1958	.post_setup	= brcm_pcie_post_setup_bcm2712,
1959	.quirks		= CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN,
1960	.num_inbound_wins = 10,
1961};
1962
1963static const struct pcie_cfg_data bcm4908_cfg = {
1964	.offsets	= pcie_offsets,
1965	.soc_base	= BCM4908,
1966	.perst_set	= brcm_pcie_perst_set_4908,
1967	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1968	.num_inbound_wins = 3,
1969};
1970
1971static const struct pcie_cfg_data bcm7278_cfg = {
1972	.offsets	= pcie_offsets_bcm7278,
1973	.soc_base	= BCM7278,
1974	.perst_set	= brcm_pcie_perst_set_7278,
1975	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278,
1976	.num_inbound_wins = 3,
1977};
1978
1979static const struct pcie_cfg_data bcm7425_cfg = {
1980	.offsets	= pcie_offsets_bcm7425,
1981	.soc_base	= BCM7425,
1982	.perst_set	= brcm_pcie_perst_set_generic,
1983	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1984	.num_inbound_wins = 3,
1985};
1986
1987static const struct pcie_cfg_data bcm7435_cfg = {
1988	.offsets	= pcie_offsets,
1989	.soc_base	= BCM7435,
1990	.perst_set	= brcm_pcie_perst_set_generic,
1991	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
1992	.num_inbound_wins = 3,
1993};
1994
1995static const struct pcie_cfg_data bcm7216_cfg = {
1996	.offsets	= pcie_offsets_bcm7278,
1997	.soc_base	= BCM7278,
1998	.perst_set	= brcm_pcie_perst_set_7278,
1999	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278,
2000	.has_phy	= true,
2001	.num_inbound_wins = 3,
2002	.has_err_report = true,
2003};
2004
2005static const struct pcie_cfg_data bcm7712_cfg = {
2006	.offsets	= pcie_offsets_bcm7712,
2007	.perst_set	= brcm_pcie_perst_set_7278,
2008	.bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic,
2009	.soc_base	= BCM7712,
2010	.num_inbound_wins = 10,
2011};
2012
2013static const struct of_device_id brcm_pcie_match[] = {
2014	{ .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg },
2015	{ .compatible = "brcm,bcm2712-pcie", .data = &bcm2712_cfg },
2016	{ .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg },
2017	{ .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg },
2018	{ .compatible = "brcm,bcm7216-pcie", .data = &bcm7216_cfg },
2019	{ .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg },
2020	{ .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg },
2021	{ .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg },
2022	{ .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg },
2023	{ .compatible = "brcm,bcm7712-pcie", .data = &bcm7712_cfg },
2024	{},
2025};
2026
2027static struct pci_ops brcm_pcie_ops = {
2028	.map_bus = brcm_pcie_map_bus,
2029	.read = pci_generic_config_read,
2030	.write = pci_generic_config_write,
2031	.add_bus = brcm_pcie_add_bus,
2032	.remove_bus = brcm_pcie_remove_bus,
2033};
2034
2035static struct pci_ops brcm7425_pcie_ops = {
2036	.map_bus = brcm7425_pcie_map_bus,
2037	.read = pci_generic_config_read32,
2038	.write = pci_generic_config_write32,
2039	.add_bus = brcm_pcie_add_bus,
2040	.remove_bus = brcm_pcie_remove_bus,
2041};
2042
2043static int brcm_pcie_probe(struct platform_device *pdev)
2044{
2045	struct device_node *np = pdev->dev.of_node;
2046	struct pci_host_bridge *bridge;
2047	const struct pcie_cfg_data *data;
2048	struct brcm_pcie *pcie;
2049	int ret;
2050
2051	bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie));
2052	if (!bridge)
2053		return -ENOMEM;
2054
2055	data = of_device_get_match_data(&pdev->dev);
2056	if (!data) {
2057		pr_err("failed to look up compatible string\n");
2058		return -EINVAL;
2059	}
2060
2061	pcie = pci_host_bridge_priv(bridge);
2062	pcie->dev = &pdev->dev;
2063	pcie->np = np;
2064	pcie->cfg = data;
2065
2066	pcie->base = devm_platform_ioremap_resource(pdev, 0);
2067	if (IS_ERR(pcie->base))
2068		return PTR_ERR(pcie->base);
2069
2070	pcie->clk = devm_clk_get_optional(&pdev->dev, "sw_pcie");
2071	if (IS_ERR(pcie->clk))
2072		return PTR_ERR(pcie->clk);
2073
2074	ret = of_pci_get_max_link_speed(np);
2075	pcie->gen = (ret < 0) ? 0 : ret;
2076
2077	pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc");
2078
2079	pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal");
2080	if (IS_ERR(pcie->rescal))
2081		return PTR_ERR(pcie->rescal);
2082
2083	pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst");
2084	if (IS_ERR(pcie->perst_reset))
2085		return PTR_ERR(pcie->perst_reset);
2086
2087	pcie->bridge_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "bridge");
2088	if (IS_ERR(pcie->bridge_reset))
2089		return PTR_ERR(pcie->bridge_reset);
2090
2091	pcie->swinit_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "swinit");
2092	if (IS_ERR(pcie->swinit_reset))
2093		return PTR_ERR(pcie->swinit_reset);
2094
2095	ret = clk_prepare_enable(pcie->clk);
2096	if (ret)
2097		return dev_err_probe(&pdev->dev, ret, "could not enable clock\n");
2098
2099	ret = brcm_pcie_bridge_sw_init_set(pcie, 0);
2100	if (ret)
2101		return dev_err_probe(&pdev->dev, ret,
2102				     "could not de-assert bridge reset\n");
2103
2104	if (pcie->swinit_reset) {
2105		ret = reset_control_assert(pcie->swinit_reset);
2106		if (ret) {
2107			clk_disable_unprepare(pcie->clk);
2108			return dev_err_probe(&pdev->dev, ret,
2109					     "could not assert reset 'swinit'\n");
2110		}
2111
2112		/* HW team recommends 1us for proper sync and propagation of reset */
2113		udelay(1);
2114
2115		ret = reset_control_deassert(pcie->swinit_reset);
2116		if (ret) {
2117			clk_disable_unprepare(pcie->clk);
2118			return dev_err_probe(&pdev->dev, ret,
2119					     "could not de-assert reset 'swinit'\n");
2120		}
2121	}
2122
2123	ret = reset_control_reset(pcie->rescal);
2124	if (ret) {
2125		clk_disable_unprepare(pcie->clk);
2126		return dev_err_probe(&pdev->dev, ret, "failed to deassert 'rescal'\n");
2127	}
2128
2129	ret = brcm_phy_start(pcie);
2130	if (ret) {
2131		reset_control_rearm(pcie->rescal);
2132		clk_disable_unprepare(pcie->clk);
2133		return ret;
2134	}
2135
2136	ret = brcm_pcie_setup(pcie);
2137	if (ret)
2138		goto fail;
2139
2140	pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION);
2141	if (pcie->cfg->soc_base == BCM4908 &&
2142	    pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) {
2143		dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n");
2144		ret = -ENODEV;
2145		goto fail;
2146	}
2147
2148	if (pci_msi_enabled()) {
2149		struct device_node *msi_np = of_parse_phandle(pcie->np, "msi-parent", 0);
2150
2151		if (msi_np == pcie->np)
2152			ret = brcm_pcie_enable_msi(pcie);
2153
2154		of_node_put(msi_np);
2155
2156		if (ret) {
2157			dev_err(pcie->dev, "probe of internal MSI failed");
2158			goto fail;
2159		}
2160	}
2161
2162	bridge->ops = pcie->cfg->soc_base == BCM7425 ?
2163				&brcm7425_pcie_ops : &brcm_pcie_ops;
2164	bridge->sysdata = pcie;
2165
2166	platform_set_drvdata(pdev, pcie);
2167
2168	ret = pci_host_probe(bridge);
2169	if (!ret && !brcm_pcie_link_up(pcie))
2170		ret = -ENODEV;
2171
2172	if (ret) {
2173		brcm_pcie_remove(pdev);
2174		return ret;
2175	}
2176
2177	if (pcie->cfg->has_err_report) {
2178		spin_lock_init(&pcie->bridge_lock);
2179		brcm_register_die_notifiers(pcie);
2180	}
2181
2182	return 0;
2183
2184fail:
2185	__brcm_pcie_remove(pcie);
2186
2187	return ret;
2188}
2189
2190MODULE_DEVICE_TABLE(of, brcm_pcie_match);
2191
2192static const struct dev_pm_ops brcm_pcie_pm_ops = {
2193	.suspend_noirq = brcm_pcie_suspend_noirq,
2194	.resume_noirq = brcm_pcie_resume_noirq,
2195};
2196
2197static struct platform_driver brcm_pcie_driver = {
2198	.probe = brcm_pcie_probe,
2199	.remove = brcm_pcie_remove,
2200	.driver = {
2201		.name = "brcm-pcie",
2202		.of_match_table = brcm_pcie_match,
2203		.pm = &brcm_pcie_pm_ops,
2204	},
2205};
2206module_platform_driver(brcm_pcie_driver);
2207
2208MODULE_LICENSE("GPL");
2209MODULE_DESCRIPTION("Broadcom STB PCIe RC driver");
2210MODULE_AUTHOR("Broadcom");
2211MODULE_SOFTDEP("pre: irq_bcm2712_mip");
~/ Read-only file preview.