arm-smmu.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * IOMMU API for ARM architected SMMU implementations.
   4 *
   5 * Copyright (C) 2013 ARM Limited
   6 *
   7 * Author: Will Deacon <will.deacon@arm.com>
   8 *
   9 * This driver currently supports:
  10 *	- SMMUv1 and v2 implementations
  11 *	- Stream-matching and stream-indexing
  12 *	- v7/v8 long-descriptor format
  13 *	- Non-secure access to the SMMU
  14 *	- Context fault reporting
  15 *	- Extended Stream ID (16 bit)
  16 */
  17
  18#define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20#include <linux/acpi.h>
  21#include <linux/acpi_iort.h>
  22#include <linux/bitfield.h>
  23#include <linux/delay.h>
  24#include <linux/dma-mapping.h>
  25#include <linux/err.h>
  26#include <linux/interrupt.h>
  27#include <linux/io.h>
  28#include <linux/iopoll.h>
  29#include <linux/module.h>
  30#include <linux/of.h>
  31#include <linux/of_address.h>
  32#include <linux/pci.h>
  33#include <linux/platform_device.h>
  34#include <linux/pm_runtime.h>
  35#include <linux/ratelimit.h>
  36#include <linux/slab.h>
  37#include <linux/string_choices.h>
  38
  39#include <linux/fsl/mc.h>
  40
  41#include "arm-smmu.h"
  42#include "../../dma-iommu.h"
  43
  44/*
  45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  46 * global register space are still, in fact, using a hypervisor to mediate it
  47 * by trapping and emulating register accesses. Sadly, some deployed versions
  48 * of said trapping code have bugs wherein they go horribly wrong for stores
  49 * using r31 (i.e. XZR/WZR) as the source register.
  50 */
  51#define QCOM_DUMMY_VAL -1
  52
  53#define MSI_IOVA_BASE			0x8000000
  54#define MSI_IOVA_LENGTH			0x100000
  55
  56static int force_stage;
  57module_param(force_stage, int, S_IRUGO);
  58MODULE_PARM_DESC(force_stage,
  59	"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  60static bool disable_bypass =
  61	IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  62module_param(disable_bypass, bool, S_IRUGO);
  63MODULE_PARM_DESC(disable_bypass,
  64	"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  65
  66#define s2cr_init_val (struct arm_smmu_s2cr){				\
  67	.type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,	\
  68}
  69
  70static bool using_legacy_binding, using_generic_binding;
  71
  72static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
  73{
  74	if (pm_runtime_enabled(smmu->dev))
  75		return pm_runtime_resume_and_get(smmu->dev);
  76
  77	return 0;
  78}
  79
  80static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
  81{
  82	if (pm_runtime_enabled(smmu->dev)) {
  83		pm_runtime_mark_last_busy(smmu->dev);
  84		__pm_runtime_put_autosuspend(smmu->dev);
  85
  86	}
  87}
  88
  89static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
  90{
  91	/*
  92	 * Setup an autosuspend delay to avoid bouncing runpm state.
  93	 * Otherwise, if a driver for a suspended consumer device
  94	 * unmaps buffers, it will runpm resume/suspend for each one.
  95	 *
  96	 * For example, when used by a GPU device, when an application
  97	 * or game exits, it can trigger unmapping 100s or 1000s of
  98	 * buffers.  With a runpm cycle for each buffer, that adds up
  99	 * to 5-10sec worth of reprogramming the context bank, while
 100	 * the system appears to be locked up to the user.
 101	 */
 102	pm_runtime_set_autosuspend_delay(smmu->dev, 20);
 103	pm_runtime_use_autosuspend(smmu->dev);
 104}
 105
 106static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 107{
 108	return container_of(dom, struct arm_smmu_domain, domain);
 109}
 110
 111static struct platform_driver arm_smmu_driver;
 112static const struct iommu_ops arm_smmu_ops;
 113
 114#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
 115static struct device_node *dev_get_dev_node(struct device *dev)
 116{
 117	if (dev_is_pci(dev)) {
 118		struct pci_bus *bus = to_pci_dev(dev)->bus;
 119
 120		while (!pci_is_root_bus(bus))
 121			bus = bus->parent;
 122		return of_node_get(bus->bridge->parent->of_node);
 123	}
 124
 125	return of_node_get(dev->of_node);
 126}
 127
 128static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 129{
 130	*((__be32 *)data) = cpu_to_be32(alias);
 131	return 0; /* Continue walking */
 132}
 133
 134static int __find_legacy_master_phandle(struct device *dev, void *data)
 135{
 136	struct of_phandle_iterator *it = *(void **)data;
 137	struct device_node *np = it->node;
 138	int err;
 139
 140	of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 141			    "#stream-id-cells", -1)
 142		if (it->node == np) {
 143			*(void **)data = dev;
 144			return 1;
 145		}
 146	it->node = np;
 147	return err == -ENOENT ? 0 : err;
 148}
 149
 150static int arm_smmu_register_legacy_master(struct device *dev,
 151					   struct arm_smmu_device **smmu)
 152{
 153	struct device *smmu_dev;
 154	struct device_node *np;
 155	struct of_phandle_iterator it;
 156	void *data = &it;
 157	u32 *sids;
 158	__be32 pci_sid;
 159	int err;
 160
 161	np = dev_get_dev_node(dev);
 162	if (!np || !of_property_present(np, "#stream-id-cells")) {
 163		of_node_put(np);
 164		return -ENODEV;
 165	}
 166
 167	it.node = np;
 168	err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 169				     __find_legacy_master_phandle);
 170	smmu_dev = data;
 171	of_node_put(np);
 172	if (err == 0)
 173		return -ENODEV;
 174	if (err < 0)
 175		return err;
 176
 177	if (dev_is_pci(dev)) {
 178		/* "mmu-masters" assumes Stream ID == Requester ID */
 179		pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 180				       &pci_sid);
 181		it.cur = &pci_sid;
 182		it.cur_count = 1;
 183	}
 184
 185	err = iommu_fwspec_init(dev, NULL);
 186	if (err)
 187		return err;
 188
 189	sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 190	if (!sids)
 191		return -ENOMEM;
 192
 193	*smmu = dev_get_drvdata(smmu_dev);
 194	of_phandle_iterator_args(&it, sids, it.cur_count);
 195	err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 196	kfree(sids);
 197	return err;
 198}
 199#else
 200static int arm_smmu_register_legacy_master(struct device *dev,
 201					   struct arm_smmu_device **smmu)
 202{
 203	return -ENODEV;
 204}
 205#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
 206
 207static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 208{
 209	clear_bit(idx, map);
 210}
 211
 212/* Wait for any pending TLB invalidations to complete */
 213static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 214				int sync, int status)
 215{
 216	unsigned int spin_cnt, delay;
 217	u32 reg;
 218
 219	if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 220		return smmu->impl->tlb_sync(smmu, page, sync, status);
 221
 222	arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 223	for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 224		for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 225			reg = arm_smmu_readl(smmu, page, status);
 226			if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
 227				return;
 228			cpu_relax();
 229		}
 230		udelay(delay);
 231	}
 232	dev_err_ratelimited(smmu->dev,
 233			    "TLB sync timed out -- SMMU may be deadlocked\n");
 234}
 235
 236static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 237{
 238	unsigned long flags;
 239
 240	spin_lock_irqsave(&smmu->global_sync_lock, flags);
 241	__arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 242			    ARM_SMMU_GR0_sTLBGSTATUS);
 243	spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 244}
 245
 246static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 247{
 248	struct arm_smmu_device *smmu = smmu_domain->smmu;
 249	unsigned long flags;
 250
 251	spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 252	__arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 253			    ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 254	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 255}
 256
 257static void arm_smmu_tlb_inv_context_s1(void *cookie)
 258{
 259	struct arm_smmu_domain *smmu_domain = cookie;
 260	/*
 261	 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 262	 * current CPU are visible beforehand.
 263	 */
 264	wmb();
 265	arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 266			  ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 267	arm_smmu_tlb_sync_context(smmu_domain);
 268}
 269
 270static void arm_smmu_tlb_inv_context_s2(void *cookie)
 271{
 272	struct arm_smmu_domain *smmu_domain = cookie;
 273	struct arm_smmu_device *smmu = smmu_domain->smmu;
 274
 275	/* See above */
 276	wmb();
 277	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 278	arm_smmu_tlb_sync_global(smmu);
 279}
 280
 281static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 282				      size_t granule, void *cookie, int reg)
 283{
 284	struct arm_smmu_domain *smmu_domain = cookie;
 285	struct arm_smmu_device *smmu = smmu_domain->smmu;
 286	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 287	int idx = cfg->cbndx;
 288
 289	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 290		wmb();
 291
 292	if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 293		iova = (iova >> 12) << 12;
 294		iova |= cfg->asid;
 295		do {
 296			arm_smmu_cb_write(smmu, idx, reg, iova);
 297			iova += granule;
 298		} while (size -= granule);
 299	} else {
 300		iova >>= 12;
 301		iova |= (u64)cfg->asid << 48;
 302		do {
 303			arm_smmu_cb_writeq(smmu, idx, reg, iova);
 304			iova += granule >> 12;
 305		} while (size -= granule);
 306	}
 307}
 308
 309static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 310				      size_t granule, void *cookie, int reg)
 311{
 312	struct arm_smmu_domain *smmu_domain = cookie;
 313	struct arm_smmu_device *smmu = smmu_domain->smmu;
 314	int idx = smmu_domain->cfg.cbndx;
 315
 316	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 317		wmb();
 318
 319	iova >>= 12;
 320	do {
 321		if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 322			arm_smmu_cb_writeq(smmu, idx, reg, iova);
 323		else
 324			arm_smmu_cb_write(smmu, idx, reg, iova);
 325		iova += granule >> 12;
 326	} while (size -= granule);
 327}
 328
 329static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 330				     size_t granule, void *cookie)
 331{
 332	struct arm_smmu_domain *smmu_domain = cookie;
 333	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 334
 335	if (cfg->flush_walk_prefer_tlbiasid) {
 336		arm_smmu_tlb_inv_context_s1(cookie);
 337	} else {
 338		arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 339					  ARM_SMMU_CB_S1_TLBIVA);
 340		arm_smmu_tlb_sync_context(cookie);
 341	}
 342}
 343
 344static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 345				     unsigned long iova, size_t granule,
 346				     void *cookie)
 347{
 348	arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 349				  ARM_SMMU_CB_S1_TLBIVAL);
 350}
 351
 352static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 353				     size_t granule, void *cookie)
 354{
 355	arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 356				  ARM_SMMU_CB_S2_TLBIIPAS2);
 357	arm_smmu_tlb_sync_context(cookie);
 358}
 359
 360static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 361				     unsigned long iova, size_t granule,
 362				     void *cookie)
 363{
 364	arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 365				  ARM_SMMU_CB_S2_TLBIIPAS2L);
 366}
 367
 368static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
 369					size_t granule, void *cookie)
 370{
 371	arm_smmu_tlb_inv_context_s2(cookie);
 372}
 373/*
 374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 375 * almost negligible, but the benefit of getting the first one in as far ahead
 376 * of the sync as possible is significant, hence we don't just make this a
 377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 378 * think.
 379 */
 380static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 381					unsigned long iova, size_t granule,
 382					void *cookie)
 383{
 384	struct arm_smmu_domain *smmu_domain = cookie;
 385	struct arm_smmu_device *smmu = smmu_domain->smmu;
 386
 387	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 388		wmb();
 389
 390	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 391}
 392
 393static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 394	.tlb_flush_all	= arm_smmu_tlb_inv_context_s1,
 395	.tlb_flush_walk	= arm_smmu_tlb_inv_walk_s1,
 396	.tlb_add_page	= arm_smmu_tlb_add_page_s1,
 397};
 398
 399static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 400	.tlb_flush_all	= arm_smmu_tlb_inv_context_s2,
 401	.tlb_flush_walk	= arm_smmu_tlb_inv_walk_s2,
 402	.tlb_add_page	= arm_smmu_tlb_add_page_s2,
 403};
 404
 405static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 406	.tlb_flush_all	= arm_smmu_tlb_inv_context_s2,
 407	.tlb_flush_walk	= arm_smmu_tlb_inv_walk_s2_v1,
 408	.tlb_add_page	= arm_smmu_tlb_add_page_s2_v1,
 409};
 410
 411
 412void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
 413				      struct arm_smmu_context_fault_info *cfi)
 414{
 415	cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 416	cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 417	cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 418	cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 419}
 420
 421void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
 422				       const struct arm_smmu_context_fault_info *cfi)
 423{
 424	dev_err(smmu->dev,
 425		"Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 426		cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
 427
 428	dev_err(smmu->dev, "FSR    = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
 429		cfi->fsr,
 430		(cfi->fsr & ARM_SMMU_CB_FSR_MULTI)  ? "MULTI " : "",
 431		(cfi->fsr & ARM_SMMU_CB_FSR_SS)     ? "SS " : "",
 432		(u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
 433		(cfi->fsr & ARM_SMMU_CB_FSR_UUT)    ? " UUT" : "",
 434		(cfi->fsr & ARM_SMMU_CB_FSR_ASF)    ? " ASF" : "",
 435		(cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
 436		(cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
 437		(cfi->fsr & ARM_SMMU_CB_FSR_EF)     ? " EF" : "",
 438		(cfi->fsr & ARM_SMMU_CB_FSR_PF)     ? " PF" : "",
 439		(cfi->fsr & ARM_SMMU_CB_FSR_AFF)    ? " AFF" : "",
 440		(cfi->fsr & ARM_SMMU_CB_FSR_TF)     ? " TF" : "",
 441		cfi->cbfrsynra);
 442
 443	dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
 444		cfi->fsynr,
 445		(u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
 446		(cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
 447		(cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
 448		(cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
 449		(cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
 450		(cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
 451		(cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
 452		(u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
 453}
 454
 455static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 456{
 457	struct arm_smmu_context_fault_info cfi;
 458	struct arm_smmu_domain *smmu_domain = dev;
 459	struct arm_smmu_device *smmu = smmu_domain->smmu;
 460	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 461				      DEFAULT_RATELIMIT_BURST);
 462	int idx = smmu_domain->cfg.cbndx;
 463	int ret;
 464
 465	arm_smmu_read_context_fault_info(smmu, idx, &cfi);
 466
 467	if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
 468		return IRQ_NONE;
 469
 470	ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
 471		cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
 472
 473	if (ret == -ENOSYS && __ratelimit(&rs))
 474		arm_smmu_print_context_fault_info(smmu, idx, &cfi);
 475
 476	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
 477
 478	if (cfi.fsr & ARM_SMMU_CB_FSR_SS) {
 479		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
 480				  ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE);
 481	}
 482
 483	return IRQ_HANDLED;
 484}
 485
 486static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 487{
 488	u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 489	struct arm_smmu_device *smmu = dev;
 490	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 491				      DEFAULT_RATELIMIT_BURST);
 492
 493	gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 494	gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 495	gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 496	gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 497
 498	if (!gfsr)
 499		return IRQ_NONE;
 500
 501	if (__ratelimit(&rs)) {
 502		if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
 503		    (gfsr & ARM_SMMU_sGFSR_USF))
 504			dev_err(smmu->dev,
 505				"Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
 506				(u16)gfsynr1);
 507		else
 508			dev_err(smmu->dev,
 509				"Unexpected global fault, this could be serious\n");
 510		dev_err(smmu->dev,
 511			"\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 512			gfsr, gfsynr0, gfsynr1, gfsynr2);
 513	}
 514
 515	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 516	return IRQ_HANDLED;
 517}
 518
 519static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 520				       struct io_pgtable_cfg *pgtbl_cfg)
 521{
 522	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 523	struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 524	bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 525
 526	cb->cfg = cfg;
 527
 528	/* TCR */
 529	if (stage1) {
 530		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 531			cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 532		} else {
 533			cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
 534			cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
 535			if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 536				cb->tcr[1] |= ARM_SMMU_TCR2_AS;
 537			else
 538				cb->tcr[0] |= ARM_SMMU_TCR_EAE;
 539		}
 540	} else {
 541		cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
 542	}
 543
 544	/* TTBRs */
 545	if (stage1) {
 546		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 547			cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
 548			cb->ttbr[1] = 0;
 549		} else {
 550			cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 551						 cfg->asid);
 552			cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
 553						 cfg->asid);
 554
 555			if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
 556				cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 557			else
 558				cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 559		}
 560	} else {
 561		cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 562	}
 563
 564	/* MAIRs (stage-1 only) */
 565	if (stage1) {
 566		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 567			cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 568			cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 569		} else {
 570			cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
 571			cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
 572		}
 573	}
 574}
 575
 576void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 577{
 578	u32 reg;
 579	bool stage1;
 580	struct arm_smmu_cb *cb = &smmu->cbs[idx];
 581	struct arm_smmu_cfg *cfg = cb->cfg;
 582
 583	/* Unassigned context banks only need disabling */
 584	if (!cfg) {
 585		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 586		return;
 587	}
 588
 589	stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 590
 591	/* CBA2R */
 592	if (smmu->version > ARM_SMMU_V1) {
 593		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 594			reg = ARM_SMMU_CBA2R_VA64;
 595		else
 596			reg = 0;
 597		/* 16-bit VMIDs live in CBA2R */
 598		if (smmu->features & ARM_SMMU_FEAT_VMID16)
 599			reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
 600
 601		arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 602	}
 603
 604	/* CBAR */
 605	reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
 606	if (smmu->version < ARM_SMMU_V2)
 607		reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
 608
 609	/*
 610	 * Use the weakest shareability/memory types, so they are
 611	 * overridden by the ttbcr/pte.
 612	 */
 613	if (stage1) {
 614		reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
 615				  ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
 616		       FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
 617				  ARM_SMMU_CBAR_S1_MEMATTR_WB);
 618	} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 619		/* 8-bit VMIDs live in CBAR */
 620		reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
 621	}
 622	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 623
 624	/*
 625	 * TCR
 626	 * We must write this before the TTBRs, since it determines the
 627	 * access behaviour of some fields (in particular, ASID[15:8]).
 628	 */
 629	if (stage1 && smmu->version > ARM_SMMU_V1)
 630		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 631	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 632
 633	/* TTBRs */
 634	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 635		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 636		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 637		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 638	} else {
 639		arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 640		if (stage1)
 641			arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 642					   cb->ttbr[1]);
 643	}
 644
 645	/* MAIRs (stage-1 only) */
 646	if (stage1) {
 647		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 648		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 649	}
 650
 651	/* SCTLR */
 652	reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
 653	      ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
 654	if (stage1)
 655		reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
 656	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 657		reg |= ARM_SMMU_SCTLR_E;
 658
 659	if (smmu->impl && smmu->impl->write_sctlr)
 660		smmu->impl->write_sctlr(smmu, idx, reg);
 661	else
 662		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 663}
 664
 665static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
 666				       struct arm_smmu_device *smmu,
 667				       struct device *dev, unsigned int start)
 668{
 669	if (smmu->impl && smmu->impl->alloc_context_bank)
 670		return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
 671
 672	return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
 673}
 674
 675static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
 676					struct arm_smmu_device *smmu,
 677					struct device *dev)
 678{
 679	int irq, start, ret = 0;
 680	unsigned long ias, oas;
 681	struct io_pgtable_ops *pgtbl_ops;
 682	struct io_pgtable_cfg pgtbl_cfg;
 683	enum io_pgtable_fmt fmt;
 684	struct iommu_domain *domain = &smmu_domain->domain;
 685	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 686	irqreturn_t (*context_fault)(int irq, void *dev);
 687
 688	mutex_lock(&smmu_domain->init_mutex);
 689	if (smmu_domain->smmu)
 690		goto out_unlock;
 691
 692	/*
 693	 * Mapping the requested stage onto what we support is surprisingly
 694	 * complicated, mainly because the spec allows S1+S2 SMMUs without
 695	 * support for nested translation. That means we end up with the
 696	 * following table:
 697	 *
 698	 * Requested        Supported        Actual
 699	 *     S1               N              S1
 700	 *     S1             S1+S2            S1
 701	 *     S1               S2             S2
 702	 *     S1               S1             S1
 703	 *     N                N              N
 704	 *     N              S1+S2            S2
 705	 *     N                S2             S2
 706	 *     N                S1             S1
 707	 *
 708	 * Note that you can't actually request stage-2 mappings.
 709	 */
 710	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 711		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 712	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 713		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 714
 715	/*
 716	 * Choosing a suitable context format is even more fiddly. Until we
 717	 * grow some way for the caller to express a preference, and/or move
 718	 * the decision into the io-pgtable code where it arguably belongs,
 719	 * just aim for the closest thing to the rest of the system, and hope
 720	 * that the hardware isn't esoteric enough that we can't assume AArch64
 721	 * support to be a superset of AArch32 support...
 722	 */
 723	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 724		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 725	if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 726	    !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 727	    (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 728	    (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 729		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 730	if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 731	    (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 732			       ARM_SMMU_FEAT_FMT_AARCH64_16K |
 733			       ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 734		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 735
 736	if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 737		ret = -EINVAL;
 738		goto out_unlock;
 739	}
 740
 741	switch (smmu_domain->stage) {
 742	case ARM_SMMU_DOMAIN_S1:
 743		cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 744		start = smmu->num_s2_context_banks;
 745		ias = smmu->va_size;
 746		oas = smmu->ipa_size;
 747		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 748			fmt = ARM_64_LPAE_S1;
 749		} else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 750			fmt = ARM_32_LPAE_S1;
 751			ias = min(ias, 32UL);
 752			oas = min(oas, 40UL);
 753		} else {
 754			fmt = ARM_V7S;
 755			ias = min(ias, 32UL);
 756			oas = min(oas, 32UL);
 757		}
 758		smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 759		break;
 760	case ARM_SMMU_DOMAIN_NESTED:
 761		/*
 762		 * We will likely want to change this if/when KVM gets
 763		 * involved.
 764		 */
 765	case ARM_SMMU_DOMAIN_S2:
 766		cfg->cbar = CBAR_TYPE_S2_TRANS;
 767		start = 0;
 768		ias = smmu->ipa_size;
 769		oas = smmu->pa_size;
 770		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 771			fmt = ARM_64_LPAE_S2;
 772		} else {
 773			fmt = ARM_32_LPAE_S2;
 774			ias = min(ias, 40UL);
 775			oas = min(oas, 40UL);
 776		}
 777		if (smmu->version == ARM_SMMU_V2)
 778			smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 779		else
 780			smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 781		break;
 782	default:
 783		ret = -EINVAL;
 784		goto out_unlock;
 785	}
 786
 787	ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
 788	if (ret < 0) {
 789		goto out_unlock;
 790	}
 791
 792	smmu_domain->smmu = smmu;
 793
 794	cfg->cbndx = ret;
 795	if (smmu->version < ARM_SMMU_V2) {
 796		cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 797		cfg->irptndx %= smmu->num_context_irqs;
 798	} else {
 799		cfg->irptndx = cfg->cbndx;
 800	}
 801
 802	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 803		cfg->vmid = cfg->cbndx + 1;
 804	else
 805		cfg->asid = cfg->cbndx;
 806
 807	pgtbl_cfg = (struct io_pgtable_cfg) {
 808		.pgsize_bitmap	= smmu->pgsize_bitmap,
 809		.ias		= ias,
 810		.oas		= oas,
 811		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 812		.tlb		= smmu_domain->flush_ops,
 813		.iommu_dev	= smmu->dev,
 814	};
 815
 816	if (smmu->impl && smmu->impl->init_context) {
 817		ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
 818		if (ret)
 819			goto out_clear_smmu;
 820	}
 821
 822	if (smmu_domain->pgtbl_quirks)
 823		pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
 824
 825	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 826	if (!pgtbl_ops) {
 827		ret = -ENOMEM;
 828		goto out_clear_smmu;
 829	}
 830
 831	/* Update the domain's page sizes to reflect the page table format */
 832	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 833
 834	if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
 835		domain->geometry.aperture_start = ~0UL << ias;
 836		domain->geometry.aperture_end = ~0UL;
 837	} else {
 838		domain->geometry.aperture_end = (1UL << ias) - 1;
 839	}
 840
 841	domain->geometry.force_aperture = true;
 842
 843	/* Initialise the context bank with our page table cfg */
 844	arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 845	arm_smmu_write_context_bank(smmu, cfg->cbndx);
 846
 847	/*
 848	 * Request context fault interrupt. Do this last to avoid the
 849	 * handler seeing a half-initialised domain state.
 850	 */
 851	irq = smmu->irqs[cfg->irptndx];
 852
 853	if (smmu->impl && smmu->impl->context_fault)
 854		context_fault = smmu->impl->context_fault;
 855	else
 856		context_fault = arm_smmu_context_fault;
 857
 858	if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
 859		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
 860						context_fault,
 861						IRQF_ONESHOT | IRQF_SHARED,
 862						"arm-smmu-context-fault",
 863						smmu_domain);
 864	else
 865		ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
 866				       "arm-smmu-context-fault", smmu_domain);
 867
 868	if (ret < 0) {
 869		dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 870			cfg->irptndx, irq);
 871		cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
 872	}
 873
 874	mutex_unlock(&smmu_domain->init_mutex);
 875
 876	/* Publish page table ops for map/unmap */
 877	smmu_domain->pgtbl_ops = pgtbl_ops;
 878	return 0;
 879
 880out_clear_smmu:
 881	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 882	smmu_domain->smmu = NULL;
 883out_unlock:
 884	mutex_unlock(&smmu_domain->init_mutex);
 885	return ret;
 886}
 887
 888static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
 889{
 890	struct arm_smmu_device *smmu = smmu_domain->smmu;
 891	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 892	int ret, irq;
 893
 894	if (!smmu)
 895		return;
 896
 897	ret = arm_smmu_rpm_get(smmu);
 898	if (ret < 0)
 899		return;
 900
 901	/*
 902	 * Disable the context bank and free the page tables before freeing
 903	 * it.
 904	 */
 905	smmu->cbs[cfg->cbndx].cfg = NULL;
 906	arm_smmu_write_context_bank(smmu, cfg->cbndx);
 907
 908	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
 909		irq = smmu->irqs[cfg->irptndx];
 910		devm_free_irq(smmu->dev, irq, smmu_domain);
 911	}
 912
 913	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 914	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 915
 916	arm_smmu_rpm_put(smmu);
 917}
 918
 919static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
 920{
 921	struct arm_smmu_domain *smmu_domain;
 922	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
 923	struct arm_smmu_device *smmu = cfg->smmu;
 924
 925	/*
 926	 * Allocate the domain and initialise some of its data structures.
 927	 * We can't really do anything meaningful until we've added a
 928	 * master.
 929	 */
 930	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 931	if (!smmu_domain)
 932		return NULL;
 933
 934	mutex_init(&smmu_domain->init_mutex);
 935	spin_lock_init(&smmu_domain->cb_lock);
 936	smmu_domain->domain.pgsize_bitmap = smmu->pgsize_bitmap;
 937
 938	return &smmu_domain->domain;
 939}
 940
 941static void arm_smmu_domain_free(struct iommu_domain *domain)
 942{
 943	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 944
 945	/*
 946	 * Free the domain resources. We assume that all devices have
 947	 * already been detached.
 948	 */
 949	arm_smmu_destroy_domain_context(smmu_domain);
 950	kfree(smmu_domain);
 951}
 952
 953static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 954{
 955	struct arm_smmu_smr *smr = smmu->smrs + idx;
 956	u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
 957		  FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
 958
 959	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 960		reg |= ARM_SMMU_SMR_VALID;
 961	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 962}
 963
 964static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 965{
 966	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 967	u32 reg;
 968
 969	if (smmu->impl && smmu->impl->write_s2cr) {
 970		smmu->impl->write_s2cr(smmu, idx);
 971		return;
 972	}
 973
 974	reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
 975	      FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
 976	      FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
 977
 978	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 979	    smmu->smrs[idx].valid)
 980		reg |= ARM_SMMU_S2CR_EXIDVALID;
 981	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 982}
 983
 984static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 985{
 986	arm_smmu_write_s2cr(smmu, idx);
 987	if (smmu->smrs)
 988		arm_smmu_write_smr(smmu, idx);
 989}
 990
 991/*
 992 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 993 * should be called after sCR0 is written.
 994 */
 995static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 996{
 997	u32 smr;
 998	int i;
 999
1000	if (!smmu->smrs)
1001		return;
1002	/*
1003	 * If we've had to accommodate firmware memory regions, we may
1004	 * have live SMRs by now; tread carefully...
1005	 *
1006	 * Somewhat perversely, not having a free SMR for this test implies we
1007	 * can get away without it anyway, as we'll only be able to 'allocate'
1008	 * these SMRs for the ID/mask values we're already trusting to be OK.
1009	 */
1010	for (i = 0; i < smmu->num_mapping_groups; i++)
1011		if (!smmu->smrs[i].valid)
1012			goto smr_ok;
1013	return;
1014smr_ok:
1015	/*
1016	 * SMR.ID bits may not be preserved if the corresponding MASK
1017	 * bits are set, so check each one separately. We can reject
1018	 * masters later if they try to claim IDs outside these masks.
1019	 */
1020	smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1021	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1022	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1023	smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1024
1025	smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1026	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1027	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1028	smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1029}
1030
1031static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1032{
1033	struct arm_smmu_smr *smrs = smmu->smrs;
1034	int i, free_idx = -ENOSPC;
1035
1036	/* Stream indexing is blissfully easy */
1037	if (!smrs)
1038		return id;
1039
1040	/* Validating SMRs is... less so */
1041	for (i = 0; i < smmu->num_mapping_groups; ++i) {
1042		if (!smrs[i].valid) {
1043			/*
1044			 * Note the first free entry we come across, which
1045			 * we'll claim in the end if nothing else matches.
1046			 */
1047			if (free_idx < 0)
1048				free_idx = i;
1049			continue;
1050		}
1051		/*
1052		 * If the new entry is _entirely_ matched by an existing entry,
1053		 * then reuse that, with the guarantee that there also cannot
1054		 * be any subsequent conflicting entries. In normal use we'd
1055		 * expect simply identical entries for this case, but there's
1056		 * no harm in accommodating the generalisation.
1057		 */
1058		if ((mask & smrs[i].mask) == mask &&
1059		    !((id ^ smrs[i].id) & ~smrs[i].mask))
1060			return i;
1061		/*
1062		 * If the new entry has any other overlap with an existing one,
1063		 * though, then there always exists at least one stream ID
1064		 * which would cause a conflict, and we can't allow that risk.
1065		 */
1066		if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1067			return -EINVAL;
1068	}
1069
1070	return free_idx;
1071}
1072
1073static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1074{
1075	if (--smmu->s2crs[idx].count)
1076		return false;
1077
1078	smmu->s2crs[idx] = s2cr_init_val;
1079	if (smmu->smrs)
1080		smmu->smrs[idx].valid = false;
1081
1082	return true;
1083}
1084
1085static int arm_smmu_master_alloc_smes(struct device *dev)
1086{
1087	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1088	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1089	struct arm_smmu_device *smmu = cfg->smmu;
1090	struct arm_smmu_smr *smrs = smmu->smrs;
1091	int i, idx, ret;
1092
1093	mutex_lock(&smmu->stream_map_mutex);
1094	/* Figure out a viable stream map entry allocation */
1095	for_each_cfg_sme(cfg, fwspec, i, idx) {
1096		u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1097		u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1098
1099		if (idx != INVALID_SMENDX) {
1100			ret = -EEXIST;
1101			goto out_err;
1102		}
1103
1104		ret = arm_smmu_find_sme(smmu, sid, mask);
1105		if (ret < 0)
1106			goto out_err;
1107
1108		idx = ret;
1109		if (smrs && smmu->s2crs[idx].count == 0) {
1110			smrs[idx].id = sid;
1111			smrs[idx].mask = mask;
1112			smrs[idx].valid = true;
1113		}
1114		smmu->s2crs[idx].count++;
1115		cfg->smendx[i] = (s16)idx;
1116	}
1117
1118	/* It worked! Now, poke the actual hardware */
1119	for_each_cfg_sme(cfg, fwspec, i, idx)
1120		arm_smmu_write_sme(smmu, idx);
1121
1122	mutex_unlock(&smmu->stream_map_mutex);
1123	return 0;
1124
1125out_err:
1126	while (i--) {
1127		arm_smmu_free_sme(smmu, cfg->smendx[i]);
1128		cfg->smendx[i] = INVALID_SMENDX;
1129	}
1130	mutex_unlock(&smmu->stream_map_mutex);
1131	return ret;
1132}
1133
1134static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1135				      struct iommu_fwspec *fwspec)
1136{
1137	struct arm_smmu_device *smmu = cfg->smmu;
1138	int i, idx;
1139
1140	mutex_lock(&smmu->stream_map_mutex);
1141	for_each_cfg_sme(cfg, fwspec, i, idx) {
1142		if (arm_smmu_free_sme(smmu, idx))
1143			arm_smmu_write_sme(smmu, idx);
1144		cfg->smendx[i] = INVALID_SMENDX;
1145	}
1146	mutex_unlock(&smmu->stream_map_mutex);
1147}
1148
1149static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1150					  enum arm_smmu_s2cr_type type,
1151					  u8 cbndx, struct iommu_fwspec *fwspec)
1152{
1153	struct arm_smmu_device *smmu = cfg->smmu;
1154	struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1155	int i, idx;
1156
1157	for_each_cfg_sme(cfg, fwspec, i, idx) {
1158		if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1159			continue;
1160
1161		s2cr[idx].type = type;
1162		s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1163		s2cr[idx].cbndx = cbndx;
1164		arm_smmu_write_s2cr(smmu, idx);
1165	}
1166}
1167
1168static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev,
1169			       struct iommu_domain *old)
1170{
1171	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1172	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1173	struct arm_smmu_master_cfg *cfg;
1174	struct arm_smmu_device *smmu;
1175	int ret;
1176
1177	/*
1178	 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1179	 * domains between of_xlate() and probe_device() - we have no way to cope
1180	 * with that, so until ARM gets converted to rely on groups and default
1181	 * domains, just say no (but more politely than by dereferencing NULL).
1182	 * This should be at least a WARN_ON once that's sorted.
1183	 */
1184	cfg = dev_iommu_priv_get(dev);
1185	if (!cfg)
1186		return -ENODEV;
1187
1188	smmu = cfg->smmu;
1189
1190	ret = arm_smmu_rpm_get(smmu);
1191	if (ret < 0)
1192		return ret;
1193
1194	/* Ensure that the domain is finalised */
1195	ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1196	if (ret < 0)
1197		goto rpm_put;
1198
1199	/*
1200	 * Sanity check the domain. We don't support domains across
1201	 * different SMMUs.
1202	 */
1203	if (smmu_domain->smmu != smmu) {
1204		ret = -EINVAL;
1205		goto rpm_put;
1206	}
1207
1208	/* Looks ok, so add the device to the domain */
1209	arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1210				      smmu_domain->cfg.cbndx, fwspec);
1211rpm_put:
1212	arm_smmu_rpm_put(smmu);
1213	return ret;
1214}
1215
1216static int arm_smmu_attach_dev_type(struct device *dev,
1217				    enum arm_smmu_s2cr_type type)
1218{
1219	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1220	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1221	struct arm_smmu_device *smmu;
1222	int ret;
1223
1224	if (!cfg)
1225		return -ENODEV;
1226	smmu = cfg->smmu;
1227
1228	ret = arm_smmu_rpm_get(smmu);
1229	if (ret < 0)
1230		return ret;
1231
1232	arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1233	arm_smmu_rpm_put(smmu);
1234	return 0;
1235}
1236
1237static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1238					struct device *dev,
1239					struct iommu_domain *old)
1240{
1241	return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1242}
1243
1244static const struct iommu_domain_ops arm_smmu_identity_ops = {
1245	.attach_dev = arm_smmu_attach_dev_identity,
1246};
1247
1248static struct iommu_domain arm_smmu_identity_domain = {
1249	.type = IOMMU_DOMAIN_IDENTITY,
1250	.ops = &arm_smmu_identity_ops,
1251};
1252
1253static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1254				       struct device *dev,
1255				       struct iommu_domain *old)
1256{
1257	return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1258}
1259
1260static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1261	.attach_dev = arm_smmu_attach_dev_blocked,
1262};
1263
1264static struct iommu_domain arm_smmu_blocked_domain = {
1265	.type = IOMMU_DOMAIN_BLOCKED,
1266	.ops = &arm_smmu_blocked_ops,
1267};
1268
1269static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1270			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
1271			      int prot, gfp_t gfp, size_t *mapped)
1272{
1273	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1274	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1275	int ret;
1276
1277	if (!ops)
1278		return -ENODEV;
1279
1280	arm_smmu_rpm_get(smmu);
1281	ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1282	arm_smmu_rpm_put(smmu);
1283
1284	return ret;
1285}
1286
1287static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1288				   size_t pgsize, size_t pgcount,
1289				   struct iommu_iotlb_gather *iotlb_gather)
1290{
1291	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1292	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1293	size_t ret;
1294
1295	if (!ops)
1296		return 0;
1297
1298	arm_smmu_rpm_get(smmu);
1299	ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1300	arm_smmu_rpm_put(smmu);
1301
1302	return ret;
1303}
1304
1305static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1306{
1307	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1308	struct arm_smmu_device *smmu = smmu_domain->smmu;
1309
1310	if (smmu_domain->flush_ops) {
1311		arm_smmu_rpm_get(smmu);
1312		smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1313		arm_smmu_rpm_put(smmu);
1314	}
1315}
1316
1317static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1318				struct iommu_iotlb_gather *gather)
1319{
1320	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1321	struct arm_smmu_device *smmu = smmu_domain->smmu;
1322
1323	if (!smmu)
1324		return;
1325
1326	arm_smmu_rpm_get(smmu);
1327	if (smmu->version == ARM_SMMU_V2 ||
1328	    smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1329		arm_smmu_tlb_sync_context(smmu_domain);
1330	else
1331		arm_smmu_tlb_sync_global(smmu);
1332	arm_smmu_rpm_put(smmu);
1333}
1334
1335static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1336					      dma_addr_t iova)
1337{
1338	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1339	struct arm_smmu_device *smmu = smmu_domain->smmu;
1340	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1341	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1342	struct device *dev = smmu->dev;
1343	void __iomem *reg;
1344	u32 tmp;
1345	u64 phys;
1346	unsigned long va, flags;
1347	int ret, idx = cfg->cbndx;
1348	phys_addr_t addr = 0;
1349
1350	ret = arm_smmu_rpm_get(smmu);
1351	if (ret < 0)
1352		return 0;
1353
1354	spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1355	va = iova & ~0xfffUL;
1356	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1357		arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1358	else
1359		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1360
1361	reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1362	if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1363				      5, 50)) {
1364		spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1365		dev_err(dev,
1366			"iova to phys timed out on %pad. Falling back to software table walk.\n",
1367			&iova);
1368		arm_smmu_rpm_put(smmu);
1369		return ops->iova_to_phys(ops, iova);
1370	}
1371
1372	phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1373	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1374	if (phys & ARM_SMMU_CB_PAR_F) {
1375		dev_err(dev, "translation fault!\n");
1376		dev_err(dev, "PAR = 0x%llx\n", phys);
1377		goto out;
1378	}
1379
1380	addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1381out:
1382	arm_smmu_rpm_put(smmu);
1383
1384	return addr;
1385}
1386
1387static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1388					dma_addr_t iova)
1389{
1390	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1391	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1392
1393	if (!ops)
1394		return 0;
1395
1396	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1397			smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1398		return arm_smmu_iova_to_phys_hard(domain, iova);
1399
1400	return ops->iova_to_phys(ops, iova);
1401}
1402
1403static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1404{
1405	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1406
1407	switch (cap) {
1408	case IOMMU_CAP_CACHE_COHERENCY:
1409		/*
1410		 * It's overwhelmingly the case in practice that when the pagetable
1411		 * walk interface is connected to a coherent interconnect, all the
1412		 * translation interfaces are too. Furthermore if the device is
1413		 * natively coherent, then its translation interface must also be.
1414		 */
1415		return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1416			device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1417	case IOMMU_CAP_NOEXEC:
1418	case IOMMU_CAP_DEFERRED_FLUSH:
1419		return true;
1420	default:
1421		return false;
1422	}
1423}
1424
1425static
1426struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1427{
1428	struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
1429
1430	put_device(dev);
1431	return dev ? dev_get_drvdata(dev) : NULL;
1432}
1433
1434static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1435{
1436	struct arm_smmu_device *smmu = NULL;
1437	struct arm_smmu_master_cfg *cfg;
1438	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1439	int i, ret;
1440
1441	if (using_legacy_binding) {
1442		ret = arm_smmu_register_legacy_master(dev, &smmu);
1443
1444		/*
1445		 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1446		 * will allocate/initialise a new one. Thus we need to update fwspec for
1447		 * later use.
1448		 */
1449		fwspec = dev_iommu_fwspec_get(dev);
1450		if (ret)
1451			goto out_free;
1452	} else {
1453		smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1454	}
1455
1456	ret = -EINVAL;
1457	for (i = 0; i < fwspec->num_ids; i++) {
1458		u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1459		u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1460
1461		if (sid & ~smmu->streamid_mask) {
1462			dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1463				sid, smmu->streamid_mask);
1464			goto out_free;
1465		}
1466		if (mask & ~smmu->smr_mask_mask) {
1467			dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1468				mask, smmu->smr_mask_mask);
1469			goto out_free;
1470		}
1471	}
1472
1473	ret = -ENOMEM;
1474	cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1475		      GFP_KERNEL);
1476	if (!cfg)
1477		goto out_free;
1478
1479	cfg->smmu = smmu;
1480	dev_iommu_priv_set(dev, cfg);
1481	while (i--)
1482		cfg->smendx[i] = INVALID_SMENDX;
1483
1484	ret = arm_smmu_rpm_get(smmu);
1485	if (ret < 0)
1486		goto out_cfg_free;
1487
1488	ret = arm_smmu_master_alloc_smes(dev);
1489	arm_smmu_rpm_put(smmu);
1490
1491	if (ret)
1492		goto out_cfg_free;
1493
1494	device_link_add(dev, smmu->dev,
1495			DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1496
1497	return &smmu->iommu;
1498
1499out_cfg_free:
1500	kfree(cfg);
1501out_free:
1502	return ERR_PTR(ret);
1503}
1504
1505static void arm_smmu_release_device(struct device *dev)
1506{
1507	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1508	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1509	int ret;
1510
1511	ret = arm_smmu_rpm_get(cfg->smmu);
1512	if (ret < 0)
1513		return;
1514
1515	arm_smmu_master_free_smes(cfg, fwspec);
1516
1517	arm_smmu_rpm_put(cfg->smmu);
1518
1519	kfree(cfg);
1520}
1521
1522static void arm_smmu_probe_finalize(struct device *dev)
1523{
1524	struct arm_smmu_master_cfg *cfg;
1525	struct arm_smmu_device *smmu;
1526
1527	cfg = dev_iommu_priv_get(dev);
1528	smmu = cfg->smmu;
1529
1530	if (smmu->impl && smmu->impl->probe_finalize)
1531		smmu->impl->probe_finalize(smmu, dev);
1532}
1533
1534static struct iommu_group *arm_smmu_device_group(struct device *dev)
1535{
1536	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1537	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1538	struct arm_smmu_device *smmu = cfg->smmu;
1539	struct iommu_group *group = NULL;
1540	int i, idx;
1541
1542	mutex_lock(&smmu->stream_map_mutex);
1543	for_each_cfg_sme(cfg, fwspec, i, idx) {
1544		if (group && smmu->s2crs[idx].group &&
1545		    group != smmu->s2crs[idx].group) {
1546			mutex_unlock(&smmu->stream_map_mutex);
1547			return ERR_PTR(-EINVAL);
1548		}
1549
1550		group = smmu->s2crs[idx].group;
1551	}
1552
1553	if (group) {
1554		mutex_unlock(&smmu->stream_map_mutex);
1555		return iommu_group_ref_get(group);
1556	}
1557
1558	if (dev_is_pci(dev))
1559		group = pci_device_group(dev);
1560	else if (dev_is_fsl_mc(dev))
1561		group = fsl_mc_device_group(dev);
1562	else
1563		group = generic_device_group(dev);
1564
1565	/* Remember group for faster lookups */
1566	if (!IS_ERR(group))
1567		for_each_cfg_sme(cfg, fwspec, i, idx)
1568			smmu->s2crs[idx].group = group;
1569
1570	mutex_unlock(&smmu->stream_map_mutex);
1571	return group;
1572}
1573
1574static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1575		unsigned long quirks)
1576{
1577	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1578	int ret = 0;
1579
1580	mutex_lock(&smmu_domain->init_mutex);
1581	if (smmu_domain->smmu)
1582		ret = -EPERM;
1583	else
1584		smmu_domain->pgtbl_quirks = quirks;
1585	mutex_unlock(&smmu_domain->init_mutex);
1586
1587	return ret;
1588}
1589
1590static int arm_smmu_of_xlate(struct device *dev,
1591			     const struct of_phandle_args *args)
1592{
1593	u32 mask, fwid = 0;
1594
1595	if (args->args_count > 0)
1596		fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1597
1598	if (args->args_count > 1)
1599		fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1600	else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1601		fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1602
1603	return iommu_fwspec_add_ids(dev, &fwid, 1);
1604}
1605
1606static void arm_smmu_get_resv_regions(struct device *dev,
1607				      struct list_head *head)
1608{
1609	struct iommu_resv_region *region;
1610	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1611
1612	region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1613					 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1614	if (!region)
1615		return;
1616
1617	list_add_tail(&region->list, head);
1618
1619	iommu_dma_get_resv_regions(dev, head);
1620}
1621
1622static int arm_smmu_def_domain_type(struct device *dev)
1623{
1624	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1625	const struct arm_smmu_impl *impl = cfg->smmu->impl;
1626
1627	if (using_legacy_binding)
1628		return IOMMU_DOMAIN_IDENTITY;
1629
1630	if (impl && impl->def_domain_type)
1631		return impl->def_domain_type(dev);
1632
1633	return 0;
1634}
1635
1636static const struct iommu_ops arm_smmu_ops = {
1637	.identity_domain	= &arm_smmu_identity_domain,
1638	.blocked_domain		= &arm_smmu_blocked_domain,
1639	.capable		= arm_smmu_capable,
1640	.domain_alloc_paging	= arm_smmu_domain_alloc_paging,
1641	.probe_device		= arm_smmu_probe_device,
1642	.release_device		= arm_smmu_release_device,
1643	.probe_finalize		= arm_smmu_probe_finalize,
1644	.device_group		= arm_smmu_device_group,
1645	.of_xlate		= arm_smmu_of_xlate,
1646	.get_resv_regions	= arm_smmu_get_resv_regions,
1647	.def_domain_type	= arm_smmu_def_domain_type,
1648	.owner			= THIS_MODULE,
1649	.default_domain_ops = &(const struct iommu_domain_ops) {
1650		.attach_dev		= arm_smmu_attach_dev,
1651		.map_pages		= arm_smmu_map_pages,
1652		.unmap_pages		= arm_smmu_unmap_pages,
1653		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
1654		.iotlb_sync		= arm_smmu_iotlb_sync,
1655		.iova_to_phys		= arm_smmu_iova_to_phys,
1656		.set_pgtable_quirks	= arm_smmu_set_pgtable_quirks,
1657		.free			= arm_smmu_domain_free,
1658	}
1659};
1660
1661static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1662{
1663	int i;
1664	u32 reg;
1665
1666	/* clear global FSR */
1667	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1668	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1669
1670	/*
1671	 * Reset stream mapping groups: Initial values mark all SMRn as
1672	 * invalid and all S2CRn as bypass unless overridden.
1673	 */
1674	for (i = 0; i < smmu->num_mapping_groups; ++i)
1675		arm_smmu_write_sme(smmu, i);
1676
1677	/* Make sure all context banks are disabled and clear CB_FSR  */
1678	for (i = 0; i < smmu->num_context_banks; ++i) {
1679		arm_smmu_write_context_bank(smmu, i);
1680		arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1681	}
1682
1683	/* Invalidate the TLB, just in case */
1684	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1685	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1686
1687	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1688
1689	/* Enable fault reporting */
1690	reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1691		ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1692
1693	/* Disable TLB broadcasting. */
1694	reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1695
1696	/* Enable client access, handling unmatched streams as appropriate */
1697	reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1698	if (disable_bypass)
1699		reg |= ARM_SMMU_sCR0_USFCFG;
1700	else
1701		reg &= ~ARM_SMMU_sCR0_USFCFG;
1702
1703	/* Disable forced broadcasting */
1704	reg &= ~ARM_SMMU_sCR0_FB;
1705
1706	/* Don't upgrade barriers */
1707	reg &= ~(ARM_SMMU_sCR0_BSU);
1708
1709	if (smmu->features & ARM_SMMU_FEAT_VMID16)
1710		reg |= ARM_SMMU_sCR0_VMID16EN;
1711
1712	if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1713		reg |= ARM_SMMU_sCR0_EXIDENABLE;
1714
1715	if (smmu->impl && smmu->impl->reset)
1716		smmu->impl->reset(smmu);
1717
1718	/* Push the button */
1719	arm_smmu_tlb_sync_global(smmu);
1720	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1721}
1722
1723static int arm_smmu_id_size_to_bits(int size)
1724{
1725	switch (size) {
1726	case 0:
1727		return 32;
1728	case 1:
1729		return 36;
1730	case 2:
1731		return 40;
1732	case 3:
1733		return 42;
1734	case 4:
1735		return 44;
1736	case 5:
1737	default:
1738		return 48;
1739	}
1740}
1741
1742static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1743{
1744	unsigned int size;
1745	u32 id;
1746	bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1747	int i, ret;
1748
1749	dev_notice(smmu->dev, "probing hardware configuration...\n");
1750	dev_notice(smmu->dev, "SMMUv%d with:\n",
1751			smmu->version == ARM_SMMU_V2 ? 2 : 1);
1752
1753	/* ID0 */
1754	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1755
1756	/* Restrict available stages based on module parameter */
1757	if (force_stage == 1)
1758		id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1759	else if (force_stage == 2)
1760		id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1761
1762	if (id & ARM_SMMU_ID0_S1TS) {
1763		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1764		dev_notice(smmu->dev, "\tstage 1 translation\n");
1765	}
1766
1767	if (id & ARM_SMMU_ID0_S2TS) {
1768		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1769		dev_notice(smmu->dev, "\tstage 2 translation\n");
1770	}
1771
1772	if (id & ARM_SMMU_ID0_NTS) {
1773		smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1774		dev_notice(smmu->dev, "\tnested translation\n");
1775	}
1776
1777	if (!(smmu->features &
1778		(ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1779		dev_err(smmu->dev, "\tno translation support!\n");
1780		return -ENODEV;
1781	}
1782
1783	if ((id & ARM_SMMU_ID0_S1TS) &&
1784	    ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1785		smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1786		dev_notice(smmu->dev, "\taddress translation ops\n");
1787	}
1788
1789	/*
1790	 * In order for DMA API calls to work properly, we must defer to what
1791	 * the FW says about coherency, regardless of what the hardware claims.
1792	 * Fortunately, this also opens up a workaround for systems where the
1793	 * ID register value has ended up configured incorrectly.
1794	 */
1795	cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1796	if (cttw_fw || cttw_reg)
1797		dev_notice(smmu->dev, "\t%scoherent table walk\n",
1798			   cttw_fw ? "" : "non-");
1799	if (cttw_fw != cttw_reg)
1800		dev_notice(smmu->dev,
1801			   "\t(IDR0.CTTW overridden by FW configuration)\n");
1802
1803	/* Max. number of entries we have for stream matching/indexing */
1804	if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1805		smmu->features |= ARM_SMMU_FEAT_EXIDS;
1806		size = 1 << 16;
1807	} else {
1808		size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1809	}
1810	smmu->streamid_mask = size - 1;
1811	if (id & ARM_SMMU_ID0_SMS) {
1812		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1813		size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1814		if (size == 0) {
1815			dev_err(smmu->dev,
1816				"stream-matching supported, but no SMRs present!\n");
1817			return -ENODEV;
1818		}
1819
1820		/* Zero-initialised to mark as invalid */
1821		smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1822					  GFP_KERNEL);
1823		if (!smmu->smrs)
1824			return -ENOMEM;
1825
1826		dev_notice(smmu->dev,
1827			   "\tstream matching with %u register groups", size);
1828	}
1829	/* s2cr->type == 0 means translation, so initialise explicitly */
1830	smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1831					 GFP_KERNEL);
1832	if (!smmu->s2crs)
1833		return -ENOMEM;
1834	for (i = 0; i < size; i++)
1835		smmu->s2crs[i] = s2cr_init_val;
1836
1837	smmu->num_mapping_groups = size;
1838	mutex_init(&smmu->stream_map_mutex);
1839	spin_lock_init(&smmu->global_sync_lock);
1840
1841	if (smmu->version < ARM_SMMU_V2 ||
1842	    !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1843		smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1844		if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1845			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1846	}
1847
1848	/* ID1 */
1849	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1850	smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1851
1852	/* Check for size mismatch of SMMU address space from mapped region */
1853	size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1854	if (smmu->numpage != 2 * size << smmu->pgshift)
1855		dev_warn(smmu->dev,
1856			"SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1857			2 * size << smmu->pgshift, smmu->numpage);
1858	/* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1859	smmu->numpage = size;
1860
1861	smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1862	smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1863	if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1864		dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1865		return -ENODEV;
1866	}
1867	dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1868		   smmu->num_context_banks, smmu->num_s2_context_banks);
1869	smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1870				 sizeof(*smmu->cbs), GFP_KERNEL);
1871	if (!smmu->cbs)
1872		return -ENOMEM;
1873
1874	/* ID2 */
1875	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1876	size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1877	smmu->ipa_size = size;
1878
1879	/* The output mask is also applied for bypass */
1880	size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1881	smmu->pa_size = size;
1882
1883	if (id & ARM_SMMU_ID2_VMID16)
1884		smmu->features |= ARM_SMMU_FEAT_VMID16;
1885
1886	/*
1887	 * What the page table walker can address actually depends on which
1888	 * descriptor format is in use, but since a) we don't know that yet,
1889	 * and b) it can vary per context bank, this will have to do...
1890	 */
1891	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1892		dev_warn(smmu->dev,
1893			 "failed to set DMA mask for table walker\n");
1894
1895	if (smmu->version < ARM_SMMU_V2) {
1896		smmu->va_size = smmu->ipa_size;
1897		if (smmu->version == ARM_SMMU_V1_64K)
1898			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1899	} else {
1900		size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1901		smmu->va_size = arm_smmu_id_size_to_bits(size);
1902		if (id & ARM_SMMU_ID2_PTFS_4K)
1903			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1904		if (id & ARM_SMMU_ID2_PTFS_16K)
1905			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1906		if (id & ARM_SMMU_ID2_PTFS_64K)
1907			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1908	}
1909
1910	if (smmu->impl && smmu->impl->cfg_probe) {
1911		ret = smmu->impl->cfg_probe(smmu);
1912		if (ret)
1913			return ret;
1914	}
1915
1916	/* Now we've corralled the various formats, what'll it do? */
1917	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1918		smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1919	if (smmu->features &
1920	    (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1921		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1922	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1923		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1924	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1925		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1926
1927	dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1928		   smmu->pgsize_bitmap);
1929
1930
1931	if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1932		dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1933			   smmu->va_size, smmu->ipa_size);
1934
1935	if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1936		dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1937			   smmu->ipa_size, smmu->pa_size);
1938
1939	return 0;
1940}
1941
1942struct arm_smmu_match_data {
1943	enum arm_smmu_arch_version version;
1944	enum arm_smmu_implementation model;
1945};
1946
1947#define ARM_SMMU_MATCH_DATA(name, ver, imp)	\
1948static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1949
1950ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1951ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1952ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1953ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1954ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1955ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1956
1957static const struct of_device_id arm_smmu_of_match[] = {
1958	{ .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1959	{ .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1960	{ .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1961	{ .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1962	{ .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1963	{ .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1964	{ .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1965	{ .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1966	{ },
1967};
1968MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1969
1970#ifdef CONFIG_ACPI
1971static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1972{
1973	int ret = 0;
1974
1975	switch (model) {
1976	case ACPI_IORT_SMMU_V1:
1977	case ACPI_IORT_SMMU_CORELINK_MMU400:
1978		smmu->version = ARM_SMMU_V1;
1979		smmu->model = GENERIC_SMMU;
1980		break;
1981	case ACPI_IORT_SMMU_CORELINK_MMU401:
1982		smmu->version = ARM_SMMU_V1_64K;
1983		smmu->model = GENERIC_SMMU;
1984		break;
1985	case ACPI_IORT_SMMU_V2:
1986		smmu->version = ARM_SMMU_V2;
1987		smmu->model = GENERIC_SMMU;
1988		break;
1989	case ACPI_IORT_SMMU_CORELINK_MMU500:
1990		smmu->version = ARM_SMMU_V2;
1991		smmu->model = ARM_MMU500;
1992		break;
1993	case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1994		smmu->version = ARM_SMMU_V2;
1995		smmu->model = CAVIUM_SMMUV2;
1996		break;
1997	default:
1998		ret = -ENODEV;
1999	}
2000
2001	return ret;
2002}
2003
2004static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2005				      u32 *global_irqs, u32 *pmu_irqs)
2006{
2007	struct device *dev = smmu->dev;
2008	struct acpi_iort_node *node =
2009		*(struct acpi_iort_node **)dev_get_platdata(dev);
2010	struct acpi_iort_smmu *iort_smmu;
2011	int ret;
2012
2013	/* Retrieve SMMU1/2 specific data */
2014	iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2015
2016	ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2017	if (ret < 0)
2018		return ret;
2019
2020	/* Ignore the configuration access interrupt */
2021	*global_irqs = 1;
2022	*pmu_irqs = 0;
2023
2024	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2025		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2026
2027	return 0;
2028}
2029#else
2030static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2031					     u32 *global_irqs, u32 *pmu_irqs)
2032{
2033	return -ENODEV;
2034}
2035#endif
2036
2037static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2038				    u32 *global_irqs, u32 *pmu_irqs)
2039{
2040	const struct arm_smmu_match_data *data;
2041	struct device *dev = smmu->dev;
2042	bool legacy_binding;
2043
2044	if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2045		return dev_err_probe(dev, -ENODEV,
2046				     "missing #global-interrupts property\n");
2047	*pmu_irqs = 0;
2048
2049	data = of_device_get_match_data(dev);
2050	smmu->version = data->version;
2051	smmu->model = data->model;
2052
2053	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2054	if (legacy_binding && !using_generic_binding) {
2055		if (!using_legacy_binding) {
2056			pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2057				  IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2058		}
2059		using_legacy_binding = true;
2060	} else if (!legacy_binding && !using_legacy_binding) {
2061		using_generic_binding = true;
2062	} else {
2063		dev_err(dev, "not probing due to mismatched DT properties\n");
2064		return -ENODEV;
2065	}
2066
2067	if (of_dma_is_coherent(dev->of_node))
2068		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2069
2070	return 0;
2071}
2072
2073static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2074{
2075	struct list_head rmr_list;
2076	struct iommu_resv_region *e;
2077	int idx, cnt = 0;
2078	u32 reg;
2079
2080	INIT_LIST_HEAD(&rmr_list);
2081	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2082
2083	/*
2084	 * Rather than trying to look at existing mappings that
2085	 * are setup by the firmware and then invalidate the ones
2086	 * that do no have matching RMR entries, just disable the
2087	 * SMMU until it gets enabled again in the reset routine.
2088	 */
2089	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2090	reg |= ARM_SMMU_sCR0_CLIENTPD;
2091	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2092
2093	list_for_each_entry(e, &rmr_list, list) {
2094		struct iommu_iort_rmr_data *rmr;
2095		int i;
2096
2097		rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2098		for (i = 0; i < rmr->num_sids; i++) {
2099			idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2100			if (idx < 0)
2101				continue;
2102
2103			if (smmu->s2crs[idx].count == 0) {
2104				smmu->smrs[idx].id = rmr->sids[i];
2105				smmu->smrs[idx].mask = 0;
2106				smmu->smrs[idx].valid = true;
2107			}
2108			smmu->s2crs[idx].count++;
2109			smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2110			smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2111
2112			cnt++;
2113		}
2114	}
2115
2116	dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2117		   str_plural(cnt));
2118	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2119}
2120
2121static int arm_smmu_device_probe(struct platform_device *pdev)
2122{
2123	struct resource *res;
2124	struct arm_smmu_device *smmu;
2125	struct device *dev = &pdev->dev;
2126	int num_irqs, i, err;
2127	u32 global_irqs, pmu_irqs;
2128	irqreturn_t (*global_fault)(int irq, void *dev);
2129
2130	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2131	if (!smmu) {
2132		dev_err(dev, "failed to allocate arm_smmu_device\n");
2133		return -ENOMEM;
2134	}
2135	smmu->dev = dev;
2136
2137	if (dev->of_node)
2138		err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2139	else
2140		err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2141	if (err)
2142		return err;
2143
2144	smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2145	if (IS_ERR(smmu->base))
2146		return PTR_ERR(smmu->base);
2147	smmu->ioaddr = res->start;
2148
2149	/*
2150	 * The resource size should effectively match the value of SMMU_TOP;
2151	 * stash that temporarily until we know PAGESIZE to validate it with.
2152	 */
2153	smmu->numpage = resource_size(res);
2154
2155	smmu = arm_smmu_impl_init(smmu);
2156	if (IS_ERR(smmu))
2157		return PTR_ERR(smmu);
2158
2159	num_irqs = platform_irq_count(pdev);
2160
2161	smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2162	if (smmu->num_context_irqs <= 0)
2163		return dev_err_probe(dev, -ENODEV,
2164				"found %d interrupts but expected at least %d\n",
2165				num_irqs, global_irqs + pmu_irqs + 1);
2166
2167	smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2168				  sizeof(*smmu->irqs), GFP_KERNEL);
2169	if (!smmu->irqs)
2170		return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2171				     smmu->num_context_irqs);
2172
2173	for (i = 0; i < smmu->num_context_irqs; i++) {
2174		int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2175
2176		if (irq < 0)
2177			return irq;
2178		smmu->irqs[i] = irq;
2179	}
2180
2181	err = devm_clk_bulk_get_all(dev, &smmu->clks);
2182	if (err < 0) {
2183		dev_err(dev, "failed to get clocks %d\n", err);
2184		return err;
2185	}
2186	smmu->num_clks = err;
2187
2188	err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2189	if (err)
2190		return err;
2191
2192	err = arm_smmu_device_cfg_probe(smmu);
2193	if (err)
2194		return err;
2195
2196	if (smmu->version == ARM_SMMU_V2) {
2197		if (smmu->num_context_banks > smmu->num_context_irqs) {
2198			dev_err(dev,
2199			      "found only %d context irq(s) but %d required\n",
2200			      smmu->num_context_irqs, smmu->num_context_banks);
2201			return -ENODEV;
2202		}
2203
2204		/* Ignore superfluous interrupts */
2205		smmu->num_context_irqs = smmu->num_context_banks;
2206	}
2207
2208	if (smmu->impl && smmu->impl->global_fault)
2209		global_fault = smmu->impl->global_fault;
2210	else
2211		global_fault = arm_smmu_global_fault;
2212
2213	for (i = 0; i < global_irqs; i++) {
2214		int irq = platform_get_irq(pdev, i);
2215
2216		if (irq < 0)
2217			return irq;
2218
2219		err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2220				       "arm-smmu global fault", smmu);
2221		if (err)
2222			return dev_err_probe(dev, err,
2223					"failed to request global IRQ %d (%u)\n",
2224					i, irq);
2225	}
2226
2227	platform_set_drvdata(pdev, smmu);
2228
2229	/* Check for RMRs and install bypass SMRs if any */
2230	arm_smmu_rmr_install_bypass_smr(smmu);
2231
2232	arm_smmu_device_reset(smmu);
2233	arm_smmu_test_smr_masks(smmu);
2234
2235	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2236				     "smmu.%pa", &smmu->ioaddr);
2237	if (err)
2238		return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n");
2239
2240	err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2241				    using_legacy_binding ? NULL : dev);
2242	if (err) {
2243		iommu_device_sysfs_remove(&smmu->iommu);
2244		return dev_err_probe(dev, err, "Failed to register iommu\n");
2245	}
2246
2247	/*
2248	 * We want to avoid touching dev->power.lock in fastpaths unless
2249	 * it's really going to do something useful - pm_runtime_enabled()
2250	 * can serve as an ideal proxy for that decision. So, conditionally
2251	 * enable pm_runtime.
2252	 */
2253	if (dev->pm_domain) {
2254		pm_runtime_set_active(dev);
2255		pm_runtime_enable(dev);
2256		arm_smmu_rpm_use_autosuspend(smmu);
2257	}
2258
2259	return 0;
2260}
2261
2262static void arm_smmu_device_shutdown(struct platform_device *pdev)
2263{
2264	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2265
2266	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2267		dev_notice(&pdev->dev, "disabling translation\n");
2268
2269	arm_smmu_rpm_get(smmu);
2270	/* Turn the thing off */
2271	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2272	arm_smmu_rpm_put(smmu);
2273
2274	if (pm_runtime_enabled(smmu->dev))
2275		pm_runtime_force_suspend(smmu->dev);
2276	else
2277		clk_bulk_disable(smmu->num_clks, smmu->clks);
2278
2279	clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2280}
2281
2282static void arm_smmu_device_remove(struct platform_device *pdev)
2283{
2284	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2285
2286	iommu_device_unregister(&smmu->iommu);
2287	iommu_device_sysfs_remove(&smmu->iommu);
2288
2289	arm_smmu_device_shutdown(pdev);
2290}
2291
2292static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2293{
2294	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2295	int ret;
2296
2297	ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2298	if (ret)
2299		return ret;
2300
2301	arm_smmu_device_reset(smmu);
2302
2303	return 0;
2304}
2305
2306static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2307{
2308	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2309
2310	clk_bulk_disable(smmu->num_clks, smmu->clks);
2311
2312	return 0;
2313}
2314
2315static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2316{
2317	int ret;
2318	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2319
2320	ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2321	if (ret)
2322		return ret;
2323
2324	if (pm_runtime_suspended(dev))
2325		return 0;
2326
2327	ret = arm_smmu_runtime_resume(dev);
2328	if (ret)
2329		clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2330
2331	return ret;
2332}
2333
2334static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2335{
2336	int ret = 0;
2337	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2338
2339	if (pm_runtime_suspended(dev))
2340		goto clk_unprepare;
2341
2342	ret = arm_smmu_runtime_suspend(dev);
2343	if (ret)
2344		return ret;
2345
2346clk_unprepare:
2347	clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2348	return ret;
2349}
2350
2351static const struct dev_pm_ops arm_smmu_pm_ops = {
2352	SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2353	SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2354			   arm_smmu_runtime_resume, NULL)
2355};
2356
2357static struct platform_driver arm_smmu_driver = {
2358	.driver	= {
2359		.name			= "arm-smmu",
2360		.of_match_table		= arm_smmu_of_match,
2361		.pm			= &arm_smmu_pm_ops,
2362		.suppress_bind_attrs    = true,
2363	},
2364	.probe	= arm_smmu_device_probe,
2365	.remove = arm_smmu_device_remove,
2366	.shutdown = arm_smmu_device_shutdown,
2367};
2368
2369static int __init arm_smmu_init(void)
2370{
2371	int ret;
2372
2373	ret = platform_driver_register(&arm_smmu_driver);
2374	if (ret)
2375		return ret;
2376
2377	ret = arm_smmu_impl_module_init();
2378	if (ret)
2379		platform_driver_unregister(&arm_smmu_driver);
2380
2381	return ret;
2382}
2383module_init(arm_smmu_init);
2384
2385static void __exit arm_smmu_exit(void)
2386{
2387	arm_smmu_impl_module_exit();
2388	platform_driver_unregister(&arm_smmu_driver);
2389}
2390module_exit(arm_smmu_exit);
2391
2392MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2393MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2394MODULE_ALIAS("platform:arm-smmu");
2395MODULE_LICENSE("GPL v2");