mc.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2014-2025 NVIDIA CORPORATION.  All rights reserved.
   4 */
   5
   6#include <linux/clk.h>
   7#include <linux/delay.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/export.h>
  10#include <linux/interrupt.h>
  11#include <linux/kernel.h>
  12#include <linux/module.h>
  13#include <linux/of.h>
  14#include <linux/of_platform.h>
  15#include <linux/platform_device.h>
  16#include <linux/slab.h>
  17#include <linux/sort.h>
  18#include <linux/tegra-icc.h>
  19
  20#include <soc/tegra/fuse.h>
  21
  22#include "mc.h"
  23
  24static const struct of_device_id tegra_mc_of_match[] = {
  25#ifdef CONFIG_ARCH_TEGRA_2x_SOC
  26	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  27#endif
  28#ifdef CONFIG_ARCH_TEGRA_3x_SOC
  29	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  30#endif
  31#ifdef CONFIG_ARCH_TEGRA_114_SOC
  32	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  33#endif
  34#ifdef CONFIG_ARCH_TEGRA_124_SOC
  35	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  36#endif
  37#ifdef CONFIG_ARCH_TEGRA_132_SOC
  38	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  39#endif
  40#ifdef CONFIG_ARCH_TEGRA_210_SOC
  41	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  42#endif
  43#ifdef CONFIG_ARCH_TEGRA_186_SOC
  44	{ .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
  45#endif
  46#ifdef CONFIG_ARCH_TEGRA_194_SOC
  47	{ .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
  48#endif
  49#ifdef CONFIG_ARCH_TEGRA_234_SOC
  50	{ .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
  51#endif
  52#ifdef CONFIG_ARCH_TEGRA_264_SOC
  53	{ .compatible = "nvidia,tegra264-mc", .data = &tegra264_mc_soc },
  54#endif
  55	{ /* sentinel */ }
  56};
  57MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  58
  59static void tegra_mc_devm_action_put_device(void *data)
  60{
  61	struct tegra_mc *mc = data;
  62
  63	put_device(mc->dev);
  64}
  65
  66/**
  67 * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
  68 * @dev: device pointer for the consumer device
  69 *
  70 * This function will search for the Memory Controller node in a device-tree
  71 * and retrieve the Memory Controller handle.
  72 *
  73 * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
  74 */
  75struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
  76{
  77	struct platform_device *pdev;
  78	struct device_node *np;
  79	struct tegra_mc *mc;
  80	int err;
  81
  82	np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
  83	if (!np)
  84		return ERR_PTR(-ENOENT);
  85
  86	pdev = of_find_device_by_node(np);
  87	of_node_put(np);
  88	if (!pdev)
  89		return ERR_PTR(-ENODEV);
  90
  91	mc = platform_get_drvdata(pdev);
  92	if (!mc) {
  93		put_device(&pdev->dev);
  94		return ERR_PTR(-EPROBE_DEFER);
  95	}
  96
  97	err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
  98	if (err)
  99		return ERR_PTR(err);
 100
 101	return mc;
 102}
 103EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
 104
 105int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
 106{
 107	if (mc->soc->ops && mc->soc->ops->probe_device)
 108		return mc->soc->ops->probe_device(mc, dev);
 109
 110	return 0;
 111}
 112EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
 113
 114int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id,
 115                               phys_addr_t *base, u64 *size)
 116{
 117	u32 offset;
 118
 119	if (id < 1 || id >= mc->soc->num_carveouts)
 120		return -EINVAL;
 121
 122	if (id < 6)
 123		offset = 0xc0c + 0x50 * (id - 1);
 124	else
 125		offset = 0x2004 + 0x50 * (id - 6);
 126
 127	*base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0);
 128#ifdef CONFIG_PHYS_ADDR_T_64BIT
 129	*base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32;
 130#endif
 131
 132	if (size)
 133		*size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17;
 134
 135	return 0;
 136}
 137EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info);
 138
 139static int tegra_mc_block_dma_common(struct tegra_mc *mc,
 140				     const struct tegra_mc_reset *rst)
 141{
 142	unsigned long flags;
 143	u32 value;
 144
 145	spin_lock_irqsave(&mc->lock, flags);
 146
 147	value = mc_readl(mc, rst->control) | BIT(rst->bit);
 148	mc_writel(mc, value, rst->control);
 149
 150	spin_unlock_irqrestore(&mc->lock, flags);
 151
 152	return 0;
 153}
 154
 155static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
 156				       const struct tegra_mc_reset *rst)
 157{
 158	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
 159}
 160
 161static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
 162				       const struct tegra_mc_reset *rst)
 163{
 164	unsigned long flags;
 165	u32 value;
 166
 167	spin_lock_irqsave(&mc->lock, flags);
 168
 169	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 170	mc_writel(mc, value, rst->control);
 171
 172	spin_unlock_irqrestore(&mc->lock, flags);
 173
 174	return 0;
 175}
 176
 177static int tegra_mc_reset_status_common(struct tegra_mc *mc,
 178					const struct tegra_mc_reset *rst)
 179{
 180	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
 181}
 182
 183const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
 184	.block_dma = tegra_mc_block_dma_common,
 185	.dma_idling = tegra_mc_dma_idling_common,
 186	.unblock_dma = tegra_mc_unblock_dma_common,
 187	.reset_status = tegra_mc_reset_status_common,
 188};
 189
 190static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
 191{
 192	return container_of(rcdev, struct tegra_mc, reset);
 193}
 194
 195static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 196							unsigned long id)
 197{
 198	unsigned int i;
 199
 200	for (i = 0; i < mc->soc->num_resets; i++)
 201		if (mc->soc->resets[i].id == id)
 202			return &mc->soc->resets[i];
 203
 204	return NULL;
 205}
 206
 207static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 208				    unsigned long id)
 209{
 210	struct tegra_mc *mc = reset_to_mc(rcdev);
 211	const struct tegra_mc_reset_ops *rst_ops;
 212	const struct tegra_mc_reset *rst;
 213	int retries = 500;
 214	int err;
 215
 216	rst = tegra_mc_reset_find(mc, id);
 217	if (!rst)
 218		return -ENODEV;
 219
 220	rst_ops = mc->soc->reset_ops;
 221	if (!rst_ops)
 222		return -ENODEV;
 223
 224	/* DMA flushing will fail if reset is already asserted */
 225	if (rst_ops->reset_status) {
 226		/* check whether reset is asserted */
 227		if (rst_ops->reset_status(mc, rst))
 228			return 0;
 229	}
 230
 231	if (rst_ops->block_dma) {
 232		/* block clients DMA requests */
 233		err = rst_ops->block_dma(mc, rst);
 234		if (err) {
 235			dev_err(mc->dev, "failed to block %s DMA: %d\n",
 236				rst->name, err);
 237			return err;
 238		}
 239	}
 240
 241	if (rst_ops->dma_idling) {
 242		/* wait for completion of the outstanding DMA requests */
 243		while (!rst_ops->dma_idling(mc, rst)) {
 244			if (!retries--) {
 245				dev_err(mc->dev, "failed to flush %s DMA\n",
 246					rst->name);
 247				return -EBUSY;
 248			}
 249
 250			usleep_range(10, 100);
 251		}
 252	}
 253
 254	if (rst_ops->hotreset_assert) {
 255		/* clear clients DMA requests sitting before arbitration */
 256		err = rst_ops->hotreset_assert(mc, rst);
 257		if (err) {
 258			dev_err(mc->dev, "failed to hot reset %s: %d\n",
 259				rst->name, err);
 260			return err;
 261		}
 262	}
 263
 264	return 0;
 265}
 266
 267static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 268				      unsigned long id)
 269{
 270	struct tegra_mc *mc = reset_to_mc(rcdev);
 271	const struct tegra_mc_reset_ops *rst_ops;
 272	const struct tegra_mc_reset *rst;
 273	int err;
 274
 275	rst = tegra_mc_reset_find(mc, id);
 276	if (!rst)
 277		return -ENODEV;
 278
 279	rst_ops = mc->soc->reset_ops;
 280	if (!rst_ops)
 281		return -ENODEV;
 282
 283	if (rst_ops->hotreset_deassert) {
 284		/* take out client from hot reset */
 285		err = rst_ops->hotreset_deassert(mc, rst);
 286		if (err) {
 287			dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 288				rst->name, err);
 289			return err;
 290		}
 291	}
 292
 293	if (rst_ops->unblock_dma) {
 294		/* allow new DMA requests to proceed to arbitration */
 295		err = rst_ops->unblock_dma(mc, rst);
 296		if (err) {
 297			dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 298				rst->name, err);
 299			return err;
 300		}
 301	}
 302
 303	return 0;
 304}
 305
 306static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 307				    unsigned long id)
 308{
 309	struct tegra_mc *mc = reset_to_mc(rcdev);
 310	const struct tegra_mc_reset_ops *rst_ops;
 311	const struct tegra_mc_reset *rst;
 312
 313	rst = tegra_mc_reset_find(mc, id);
 314	if (!rst)
 315		return -ENODEV;
 316
 317	rst_ops = mc->soc->reset_ops;
 318	if (!rst_ops)
 319		return -ENODEV;
 320
 321	return rst_ops->reset_status(mc, rst);
 322}
 323
 324static const struct reset_control_ops tegra_mc_reset_ops = {
 325	.assert = tegra_mc_hotreset_assert,
 326	.deassert = tegra_mc_hotreset_deassert,
 327	.status = tegra_mc_hotreset_status,
 328};
 329
 330static int tegra_mc_reset_setup(struct tegra_mc *mc)
 331{
 332	int err;
 333
 334	mc->reset.ops = &tegra_mc_reset_ops;
 335	mc->reset.owner = THIS_MODULE;
 336	mc->reset.of_node = mc->dev->of_node;
 337	mc->reset.of_reset_n_cells = 1;
 338	mc->reset.nr_resets = mc->soc->num_resets;
 339
 340	err = reset_controller_register(&mc->reset);
 341	if (err < 0)
 342		return err;
 343
 344	return 0;
 345}
 346
 347int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 348{
 349	unsigned int i;
 350	struct tegra_mc_timing *timing = NULL;
 351
 352	for (i = 0; i < mc->num_timings; i++) {
 353		if (mc->timings[i].rate == rate) {
 354			timing = &mc->timings[i];
 355			break;
 356		}
 357	}
 358
 359	if (!timing) {
 360		dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 361			rate);
 362		return -EINVAL;
 363	}
 364
 365	for (i = 0; i < mc->soc->num_emem_regs; ++i)
 366		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 367
 368	return 0;
 369}
 370EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
 371
 372unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 373{
 374	u8 dram_count;
 375
 376	dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 377	dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 378	dram_count++;
 379
 380	return dram_count;
 381}
 382EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 383
 384#if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
 385    defined(CONFIG_ARCH_TEGRA_114_SOC) || \
 386    defined(CONFIG_ARCH_TEGRA_124_SOC) || \
 387    defined(CONFIG_ARCH_TEGRA_132_SOC) || \
 388    defined(CONFIG_ARCH_TEGRA_210_SOC)
 389static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 390{
 391	unsigned long long tick;
 392	unsigned int i;
 393	u32 value;
 394
 395	/* compute the number of MC clock cycles per tick */
 396	tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 397	do_div(tick, NSEC_PER_SEC);
 398
 399	value = mc_readl(mc, MC_EMEM_ARB_CFG);
 400	value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 401	value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 402	mc_writel(mc, value, MC_EMEM_ARB_CFG);
 403
 404	/* write latency allowance defaults */
 405	for (i = 0; i < mc->soc->num_clients; i++) {
 406		const struct tegra_mc_client *client = &mc->soc->clients[i];
 407		u32 value;
 408
 409		value = mc_readl(mc, client->regs.la.reg);
 410		value &= ~(client->regs.la.mask << client->regs.la.shift);
 411		value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
 412		mc_writel(mc, value, client->regs.la.reg);
 413	}
 414
 415	/* latch new values */
 416	mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 417
 418	return 0;
 419}
 420
 421static int load_one_timing(struct tegra_mc *mc,
 422			   struct tegra_mc_timing *timing,
 423			   struct device_node *node)
 424{
 425	int err;
 426	u32 tmp;
 427
 428	err = of_property_read_u32(node, "clock-frequency", &tmp);
 429	if (err) {
 430		dev_err(mc->dev,
 431			"timing %pOFn: failed to read rate\n", node);
 432		return err;
 433	}
 434
 435	timing->rate = tmp;
 436	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 437					 sizeof(u32), GFP_KERNEL);
 438	if (!timing->emem_data)
 439		return -ENOMEM;
 440
 441	err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 442					 timing->emem_data,
 443					 mc->soc->num_emem_regs);
 444	if (err) {
 445		dev_err(mc->dev,
 446			"timing %pOFn: failed to read EMEM configuration\n",
 447			node);
 448		return err;
 449	}
 450
 451	return 0;
 452}
 453
 454static int load_timings(struct tegra_mc *mc, struct device_node *node)
 455{
 456	struct tegra_mc_timing *timing;
 457	int child_count = of_get_child_count(node);
 458	int i = 0, err;
 459
 460	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 461				   GFP_KERNEL);
 462	if (!mc->timings)
 463		return -ENOMEM;
 464
 465	mc->num_timings = child_count;
 466
 467	for_each_child_of_node_scoped(node, child) {
 468		timing = &mc->timings[i++];
 469
 470		err = load_one_timing(mc, timing, child);
 471		if (err)
 472			return err;
 473	}
 474
 475	return 0;
 476}
 477
 478static int tegra_mc_setup_timings(struct tegra_mc *mc)
 479{
 480	u32 ram_code, node_ram_code;
 481	int err;
 482
 483	ram_code = tegra_read_ram_code();
 484
 485	mc->num_timings = 0;
 486
 487	for_each_child_of_node_scoped(mc->dev->of_node, node) {
 488		err = of_property_read_u32(node, "nvidia,ram-code",
 489					   &node_ram_code);
 490		if (err || (node_ram_code != ram_code))
 491			continue;
 492
 493		err = load_timings(mc, node);
 494		if (err)
 495			return err;
 496		break;
 497	}
 498
 499	if (mc->num_timings == 0)
 500		dev_warn(mc->dev,
 501			 "no memory timings for RAM code %u registered\n",
 502			 ram_code);
 503
 504	return 0;
 505}
 506
 507int tegra30_mc_probe(struct tegra_mc *mc)
 508{
 509	int err;
 510
 511	mc->clk = devm_clk_get_optional(mc->dev, "mc");
 512	if (IS_ERR(mc->clk)) {
 513		dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
 514		return PTR_ERR(mc->clk);
 515	}
 516
 517	/* ensure that debug features are disabled */
 518	mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 519
 520	err = tegra_mc_setup_latency_allowance(mc);
 521	if (err < 0) {
 522		dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
 523		return err;
 524	}
 525
 526	err = tegra_mc_setup_timings(mc);
 527	if (err < 0) {
 528		dev_err(mc->dev, "failed to setup timings: %d\n", err);
 529		return err;
 530	}
 531
 532	return 0;
 533}
 534
 535const struct tegra_mc_ops tegra30_mc_ops = {
 536	.probe = tegra30_mc_probe,
 537	.handle_irq = tegra30_mc_handle_irq,
 538};
 539#endif
 540
 541static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
 542					  unsigned int *mc_channel)
 543{
 544	if ((status & mc->soc->ch_intmask) == 0)
 545		return -EINVAL;
 546
 547	*mc_channel = __ffs((status & mc->soc->ch_intmask) >>
 548			    mc->soc->global_intstatus_channel_shift);
 549
 550	return 0;
 551}
 552
 553static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
 554					  unsigned int channel)
 555{
 556	return BIT(channel) << mc->soc->global_intstatus_channel_shift;
 557}
 558
 559irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 560{
 561	struct tegra_mc *mc = data;
 562	unsigned int bit, channel;
 563	unsigned long status;
 564
 565	if (mc->soc->num_channels) {
 566		u32 global_status;
 567		int err;
 568
 569		global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
 570		err = mc_global_intstatus_to_channel(mc, global_status, &channel);
 571		if (err < 0) {
 572			dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
 573					    global_status);
 574			return IRQ_NONE;
 575		}
 576
 577		/* mask all interrupts to avoid flooding */
 578		status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
 579	} else {
 580		status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 581	}
 582
 583	if (!status)
 584		return IRQ_NONE;
 585
 586	for_each_set_bit(bit, &status, 32) {
 587		const char *error = tegra_mc_status_names[bit] ?: "unknown";
 588		const char *client = "unknown", *desc;
 589		const char *direction, *secure;
 590		u32 status_reg, addr_reg;
 591		u32 intmask = BIT(bit);
 592		phys_addr_t addr = 0;
 593#ifdef CONFIG_PHYS_ADDR_T_64BIT
 594		u32 addr_hi_reg = 0;
 595#endif
 596		unsigned int i;
 597		char perm[7];
 598		u8 id, type;
 599		u32 value;
 600
 601		switch (intmask) {
 602		case MC_INT_DECERR_VPR:
 603			status_reg = MC_ERR_VPR_STATUS;
 604			addr_reg = MC_ERR_VPR_ADR;
 605			break;
 606
 607		case MC_INT_SECERR_SEC:
 608			status_reg = MC_ERR_SEC_STATUS;
 609			addr_reg = MC_ERR_SEC_ADR;
 610			break;
 611
 612		case MC_INT_DECERR_MTS:
 613			status_reg = MC_ERR_MTS_STATUS;
 614			addr_reg = MC_ERR_MTS_ADR;
 615			break;
 616
 617		case MC_INT_DECERR_GENERALIZED_CARVEOUT:
 618			status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
 619			addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
 620			break;
 621
 622		case MC_INT_DECERR_ROUTE_SANITY:
 623			status_reg = MC_ERR_ROUTE_SANITY_STATUS;
 624			addr_reg = MC_ERR_ROUTE_SANITY_ADR;
 625			break;
 626
 627		default:
 628			status_reg = MC_ERR_STATUS;
 629			addr_reg = MC_ERR_ADR;
 630
 631#ifdef CONFIG_PHYS_ADDR_T_64BIT
 632			if (mc->soc->has_addr_hi_reg)
 633				addr_hi_reg = MC_ERR_ADR_HI;
 634#endif
 635			break;
 636		}
 637
 638		if (mc->soc->num_channels)
 639			value = mc_ch_readl(mc, channel, status_reg);
 640		else
 641			value = mc_readl(mc, status_reg);
 642
 643#ifdef CONFIG_PHYS_ADDR_T_64BIT
 644		if (mc->soc->num_address_bits > 32) {
 645			if (addr_hi_reg) {
 646				if (mc->soc->num_channels)
 647					addr = mc_ch_readl(mc, channel, addr_hi_reg);
 648				else
 649					addr = mc_readl(mc, addr_hi_reg);
 650			} else {
 651				addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 652					MC_ERR_STATUS_ADR_HI_MASK);
 653			}
 654			addr <<= 32;
 655		}
 656#endif
 657
 658		if (value & MC_ERR_STATUS_RW)
 659			direction = "write";
 660		else
 661			direction = "read";
 662
 663		if (value & MC_ERR_STATUS_SECURITY)
 664			secure = "secure ";
 665		else
 666			secure = "";
 667
 668		id = value & mc->soc->client_id_mask;
 669
 670		for (i = 0; i < mc->soc->num_clients; i++) {
 671			if (mc->soc->clients[i].id == id) {
 672				client = mc->soc->clients[i].name;
 673				break;
 674			}
 675		}
 676
 677		type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 678		       MC_ERR_STATUS_TYPE_SHIFT;
 679		desc = tegra_mc_error_names[type];
 680
 681		switch (value & MC_ERR_STATUS_TYPE_MASK) {
 682		case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 683			perm[0] = ' ';
 684			perm[1] = '[';
 685
 686			if (value & MC_ERR_STATUS_READABLE)
 687				perm[2] = 'R';
 688			else
 689				perm[2] = '-';
 690
 691			if (value & MC_ERR_STATUS_WRITABLE)
 692				perm[3] = 'W';
 693			else
 694				perm[3] = '-';
 695
 696			if (value & MC_ERR_STATUS_NONSECURE)
 697				perm[4] = '-';
 698			else
 699				perm[4] = 'S';
 700
 701			perm[5] = ']';
 702			perm[6] = '\0';
 703			break;
 704
 705		default:
 706			perm[0] = '\0';
 707			break;
 708		}
 709
 710		if (mc->soc->num_channels)
 711			value = mc_ch_readl(mc, channel, addr_reg);
 712		else
 713			value = mc_readl(mc, addr_reg);
 714		addr |= value;
 715
 716		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 717				    client, secure, direction, &addr, error,
 718				    desc, perm);
 719	}
 720
 721	/* clear interrupts */
 722	if (mc->soc->num_channels) {
 723		mc_ch_writel(mc, channel, status, MC_INTSTATUS);
 724		mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
 725			     mc_channel_to_global_intstatus(mc, channel),
 726			     MC_GLOBAL_INTSTATUS);
 727	} else {
 728		mc_writel(mc, status, MC_INTSTATUS);
 729	}
 730
 731	return IRQ_HANDLED;
 732}
 733
 734const char *const tegra_mc_status_names[32] = {
 735	[ 1] = "External interrupt",
 736	[ 6] = "EMEM address decode error",
 737	[ 7] = "GART page fault",
 738	[ 8] = "Security violation",
 739	[ 9] = "EMEM arbitration error",
 740	[10] = "Page fault",
 741	[11] = "Invalid APB ASID update",
 742	[12] = "VPR violation",
 743	[13] = "Secure carveout violation",
 744	[16] = "MTS carveout violation",
 745	[17] = "Generalized carveout violation",
 746	[20] = "Route Sanity error",
 747};
 748
 749const char *const tegra_mc_error_names[8] = {
 750	[2] = "EMEM decode error",
 751	[3] = "TrustZone violation",
 752	[4] = "Carveout violation",
 753	[6] = "SMMU translation error",
 754};
 755
 756struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data)
 757{
 758	struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
 759	struct icc_node *node;
 760
 761	list_for_each_entry(node, &mc->provider.nodes, node_list) {
 762		if (node->id == spec->args[0])
 763			return node;
 764	}
 765
 766	/*
 767	 * If a client driver calls devm_of_icc_get() before the MC driver
 768	 * is probed, then return EPROBE_DEFER to the client driver.
 769	 */
 770	return ERR_PTR(-EPROBE_DEFER);
 771}
 772
 773static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak)
 774{
 775	*average = 0;
 776	*peak = 0;
 777
 778	return 0;
 779}
 780
 781static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst)
 782{
 783	return 0;
 784}
 785
 786const struct tegra_mc_icc_ops tegra_mc_icc_ops = {
 787	.xlate = tegra_mc_icc_xlate,
 788	.aggregate = icc_std_aggregate,
 789	.get_bw = tegra_mc_icc_get,
 790	.set = tegra_mc_icc_set,
 791};
 792
 793/*
 794 * Memory Controller (MC) has few Memory Clients that are issuing memory
 795 * bandwidth allocation requests to the MC interconnect provider. The MC
 796 * provider aggregates the requests and then sends the aggregated request
 797 * up to the External Memory Controller (EMC) interconnect provider which
 798 * re-configures hardware interface to External Memory (EMEM) in accordance
 799 * to the required bandwidth. Each MC interconnect node represents an
 800 * individual Memory Client.
 801 *
 802 * Memory interconnect topology:
 803 *
 804 *               +----+
 805 * +--------+    |    |
 806 * | TEXSRD +--->+    |
 807 * +--------+    |    |
 808 *               |    |    +-----+    +------+
 809 *    ...        | MC +--->+ EMC +--->+ EMEM |
 810 *               |    |    +-----+    +------+
 811 * +--------+    |    |
 812 * | DISP.. +--->+    |
 813 * +--------+    |    |
 814 *               +----+
 815 */
 816static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 817{
 818	struct icc_node *node;
 819	unsigned int i;
 820	int err;
 821
 822	/* older device-trees don't have interconnect properties */
 823	if (!device_property_present(mc->dev, "#interconnect-cells") ||
 824	    !mc->soc->icc_ops)
 825		return 0;
 826
 827	mc->provider.dev = mc->dev;
 828	mc->provider.data = &mc->provider;
 829	mc->provider.set = mc->soc->icc_ops->set;
 830	mc->provider.aggregate = mc->soc->icc_ops->aggregate;
 831	mc->provider.get_bw = mc->soc->icc_ops->get_bw;
 832	mc->provider.xlate = mc->soc->icc_ops->xlate;
 833	mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
 834
 835	icc_provider_init(&mc->provider);
 836
 837	/* create Memory Controller node */
 838	node = icc_node_create(TEGRA_ICC_MC);
 839	if (IS_ERR(node))
 840		return PTR_ERR(node);
 841
 842	node->name = "Memory Controller";
 843	icc_node_add(node, &mc->provider);
 844
 845	/* link Memory Controller to External Memory Controller */
 846	err = icc_link_create(node, TEGRA_ICC_EMC);
 847	if (err)
 848		goto remove_nodes;
 849
 850	for (i = 0; i < mc->soc->num_clients; i++) {
 851		/* create MC client node */
 852		node = icc_node_create(mc->soc->clients[i].id);
 853		if (IS_ERR(node)) {
 854			err = PTR_ERR(node);
 855			goto remove_nodes;
 856		}
 857
 858		node->name = mc->soc->clients[i].name;
 859		icc_node_add(node, &mc->provider);
 860
 861		/* link Memory Client to Memory Controller */
 862		err = icc_link_create(node, TEGRA_ICC_MC);
 863		if (err)
 864			goto remove_nodes;
 865
 866		node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]);
 867	}
 868
 869	err = icc_provider_register(&mc->provider);
 870	if (err)
 871		goto remove_nodes;
 872
 873	return 0;
 874
 875remove_nodes:
 876	icc_nodes_remove(&mc->provider);
 877
 878	return err;
 879}
 880
 881static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
 882{
 883	unsigned int i;
 884	u32 value;
 885
 886	value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE);
 887	if (value <= 0) {
 888		mc->num_channels = mc->soc->num_channels;
 889		return;
 890	}
 891
 892	for (i = 0; i < 32; i++) {
 893		if (value & BIT(i))
 894			mc->num_channels++;
 895	}
 896}
 897
 898static int tegra_mc_probe(struct platform_device *pdev)
 899{
 900	struct tegra_mc *mc;
 901	u64 mask;
 902	int err;
 903
 904	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 905	if (!mc)
 906		return -ENOMEM;
 907
 908	platform_set_drvdata(pdev, mc);
 909	spin_lock_init(&mc->lock);
 910	mc->soc = of_device_get_match_data(&pdev->dev);
 911	mc->dev = &pdev->dev;
 912
 913	mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 914
 915	err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 916	if (err < 0) {
 917		dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 918		return err;
 919	}
 920
 921	/* length of MC tick in nanoseconds */
 922	mc->tick = 30;
 923
 924	mc->regs = devm_platform_ioremap_resource(pdev, 0);
 925	if (IS_ERR(mc->regs))
 926		return PTR_ERR(mc->regs);
 927
 928	mc->debugfs.root = debugfs_create_dir("mc", NULL);
 929
 930	if (mc->soc->ops && mc->soc->ops->probe) {
 931		err = mc->soc->ops->probe(mc);
 932		if (err < 0)
 933			return err;
 934	}
 935
 936	tegra_mc_num_channel_enabled(mc);
 937
 938	if (mc->soc->ops && mc->soc->ops->handle_irq) {
 939		mc->irq = platform_get_irq(pdev, 0);
 940		if (mc->irq < 0)
 941			return mc->irq;
 942
 943		WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 944
 945		if (mc->soc->num_channels)
 946			mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
 947				     MC_INTMASK);
 948		else
 949			mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 950
 951		err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
 952				       dev_name(&pdev->dev), mc);
 953		if (err < 0) {
 954			dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 955				err);
 956			return err;
 957		}
 958	}
 959
 960	if (mc->soc->reset_ops) {
 961		err = tegra_mc_reset_setup(mc);
 962		if (err < 0)
 963			dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
 964	}
 965
 966	err = tegra_mc_interconnect_setup(mc);
 967	if (err < 0)
 968		dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
 969			err);
 970
 971	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 972		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 973		if (IS_ERR(mc->smmu)) {
 974			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 975				PTR_ERR(mc->smmu));
 976			mc->smmu = NULL;
 977		}
 978	}
 979
 980	return 0;
 981}
 982
 983static void tegra_mc_sync_state(struct device *dev)
 984{
 985	struct tegra_mc *mc = dev_get_drvdata(dev);
 986
 987	/* check whether ICC provider is registered */
 988	if (mc->provider.dev == dev)
 989		icc_sync_state(dev);
 990}
 991
 992static struct platform_driver tegra_mc_driver = {
 993	.driver = {
 994		.name = "tegra-mc",
 995		.of_match_table = tegra_mc_of_match,
 996		.suppress_bind_attrs = true,
 997		.sync_state = tegra_mc_sync_state,
 998	},
 999	.prevent_deferred_probe = true,
1000	.probe = tegra_mc_probe,
1001};
1002
1003static int tegra_mc_init(void)
1004{
1005	return platform_driver_register(&tegra_mc_driver);
1006}
1007arch_initcall(tegra_mc_init);
1008
1009MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
1010MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");