arm-smmu-impl.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2// Miscellaneous Arm SMMU implementation and integration quirks
  3// Copyright (C) 2019 Arm Limited
  4
  5#define pr_fmt(fmt) "arm-smmu: " fmt
  6
  7#include <linux/bitfield.h>
  8#include <linux/of.h>
  9
 10#include "arm-smmu.h"
 11
 12
 13static int arm_smmu_gr0_ns(int offset)
 14{
 15	switch (offset) {
 16	case ARM_SMMU_GR0_sCR0:
 17	case ARM_SMMU_GR0_sACR:
 18	case ARM_SMMU_GR0_sGFSR:
 19	case ARM_SMMU_GR0_sGFSYNR0:
 20	case ARM_SMMU_GR0_sGFSYNR1:
 21	case ARM_SMMU_GR0_sGFSYNR2:
 22		return offset + 0x400;
 23	default:
 24		return offset;
 25	}
 26}
 27
 28static u32 arm_smmu_read_ns(struct arm_smmu_device *smmu, int page,
 29			    int offset)
 30{
 31	if (page == ARM_SMMU_GR0)
 32		offset = arm_smmu_gr0_ns(offset);
 33	return readl_relaxed(arm_smmu_page(smmu, page) + offset);
 34}
 35
 36static void arm_smmu_write_ns(struct arm_smmu_device *smmu, int page,
 37			      int offset, u32 val)
 38{
 39	if (page == ARM_SMMU_GR0)
 40		offset = arm_smmu_gr0_ns(offset);
 41	writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
 42}
 43
 44/* Since we don't care for sGFAR, we can do without 64-bit accessors */
 45static const struct arm_smmu_impl calxeda_impl = {
 46	.read_reg = arm_smmu_read_ns,
 47	.write_reg = arm_smmu_write_ns,
 48};
 49
 50
 51struct cavium_smmu {
 52	struct arm_smmu_device smmu;
 53	u32 id_base;
 54};
 55
 56static int cavium_cfg_probe(struct arm_smmu_device *smmu)
 57{
 58	static atomic_t context_count = ATOMIC_INIT(0);
 59	struct cavium_smmu *cs = container_of(smmu, struct cavium_smmu, smmu);
 60	/*
 61	 * Cavium CN88xx erratum #27704.
 62	 * Ensure ASID and VMID allocation is unique across all SMMUs in
 63	 * the system.
 64	 */
 65	cs->id_base = atomic_fetch_add(smmu->num_context_banks, &context_count);
 66	dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
 67
 68	return 0;
 69}
 70
 71static int cavium_init_context(struct arm_smmu_domain *smmu_domain,
 72		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
 73{
 74	struct cavium_smmu *cs = container_of(smmu_domain->smmu,
 75					      struct cavium_smmu, smmu);
 76
 77	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 78		smmu_domain->cfg.vmid += cs->id_base;
 79	else
 80		smmu_domain->cfg.asid += cs->id_base;
 81
 82	return 0;
 83}
 84
 85static const struct arm_smmu_impl cavium_impl = {
 86	.cfg_probe = cavium_cfg_probe,
 87	.init_context = cavium_init_context,
 88};
 89
 90static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smmu)
 91{
 92	struct cavium_smmu *cs;
 93
 94	cs = devm_krealloc(smmu->dev, smmu, sizeof(*cs), GFP_KERNEL);
 95	if (!cs)
 96		return ERR_PTR(-ENOMEM);
 97
 98	cs->smmu.impl = &cavium_impl;
 99
100	return &cs->smmu;
101}
102
103
104#define ARM_MMU500_ACTLR_CPRE		(1 << 1)
105
106#define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)
107#define ARM_MMU500_ACR_S2CRB_TLBEN	(1 << 10)
108#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)
109
110int arm_mmu500_reset(struct arm_smmu_device *smmu)
111{
112	u32 reg, major;
113	/*
114	 * On MMU-500 r2p0 onwards we need to clear ACR.CACHE_LOCK before
115	 * writes to the context bank ACTLRs will stick. And we just hope that
116	 * Secure has also cleared SACR.CACHE_LOCK for this to take effect...
117	 */
118	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
119	major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg);
120	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
121	if (major >= 2)
122		reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
123	/*
124	 * Allow unmatched Stream IDs to allocate bypass
125	 * TLB entries for reduced latency.
126	 */
127	reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
128	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
129
130#ifdef CONFIG_ARM_SMMU_MMU_500_CPRE_ERRATA
131	/*
132	 * Disable MMU-500's not-particularly-beneficial next-page
133	 * prefetcher for the sake of at least 5 known errata.
134	 */
135	for (int i = 0; i < smmu->num_context_banks; ++i) {
136		reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
137		reg &= ~ARM_MMU500_ACTLR_CPRE;
138		arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
139		reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
140		if (reg & ARM_MMU500_ACTLR_CPRE)
141			dev_warn_once(smmu->dev, "Failed to disable prefetcher for errata workarounds, check SACR.CACHE_LOCK\n");
142	}
143#endif
144
145	return 0;
146}
147
148static const struct arm_smmu_impl arm_mmu500_impl = {
149	.reset = arm_mmu500_reset,
150};
151
152static u64 mrvl_mmu500_readq(struct arm_smmu_device *smmu, int page, int off)
153{
154	/*
155	 * Marvell Armada-AP806 erratum #582743.
156	 * Split all the readq to double readl
157	 */
158	return hi_lo_readq_relaxed(arm_smmu_page(smmu, page) + off);
159}
160
161static void mrvl_mmu500_writeq(struct arm_smmu_device *smmu, int page, int off,
162			       u64 val)
163{
164	/*
165	 * Marvell Armada-AP806 erratum #582743.
166	 * Split all the writeq to double writel
167	 */
168	hi_lo_writeq_relaxed(val, arm_smmu_page(smmu, page) + off);
169}
170
171static int mrvl_mmu500_cfg_probe(struct arm_smmu_device *smmu)
172{
173
174	/*
175	 * Armada-AP806 erratum #582743.
176	 * Hide the SMMU_IDR2.PTFSv8 fields to sidestep the AArch64
177	 * formats altogether and allow using 32 bits access on the
178	 * interconnect.
179	 */
180	smmu->features &= ~(ARM_SMMU_FEAT_FMT_AARCH64_4K |
181			    ARM_SMMU_FEAT_FMT_AARCH64_16K |
182			    ARM_SMMU_FEAT_FMT_AARCH64_64K);
183
184	return 0;
185}
186
187static const struct arm_smmu_impl mrvl_mmu500_impl = {
188	.read_reg64 = mrvl_mmu500_readq,
189	.write_reg64 = mrvl_mmu500_writeq,
190	.cfg_probe = mrvl_mmu500_cfg_probe,
191	.reset = arm_mmu500_reset,
192};
193
194
195struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
196{
197	const struct device_node *np = smmu->dev->of_node;
198
199	/*
200	 * Set the impl for model-specific implementation quirks first,
201	 * such that platform integration quirks can pick it up and
202	 * inherit from it if necessary.
203	 */
204	switch (smmu->model) {
205	case ARM_MMU500:
206		smmu->impl = &arm_mmu500_impl;
207		break;
208	case CAVIUM_SMMUV2:
209		return cavium_smmu_impl_init(smmu);
210	default:
211		break;
212	}
213
214	/* This is implicitly MMU-400 */
215	if (of_property_read_bool(np, "calxeda,smmu-secure-config-access"))
216		smmu->impl = &calxeda_impl;
217
218	if (of_device_is_compatible(np, "nvidia,tegra234-smmu") ||
219	    of_device_is_compatible(np, "nvidia,tegra194-smmu") ||
220	    of_device_is_compatible(np, "nvidia,tegra186-smmu"))
221		return nvidia_smmu_impl_init(smmu);
222
223	if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM))
224		smmu = qcom_smmu_impl_init(smmu);
225
226	if (of_device_is_compatible(np, "marvell,ap806-smmu-500"))
227		smmu->impl = &mrvl_mmu500_impl;
228
229	return smmu;
230}
231
232int __init arm_smmu_impl_module_init(void)
233{
234	if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM))
235		return qcom_smmu_module_init();
236
237	return 0;
238}
239
240void __exit arm_smmu_impl_module_exit(void)
241{
242	if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM))
243		qcom_smmu_module_exit();
244}