arm-smmu-v3-sva.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Implementation of the IOMMU SVA API for the ARM SMMUv3
  4 */
  5
  6#include <linux/mm.h>
  7#include <linux/mmu_context.h>
  8#include <linux/mmu_notifier.h>
  9#include <linux/sched/mm.h>
 10#include <linux/slab.h>
 11#include <kunit/visibility.h>
 12
 13#include "arm-smmu-v3.h"
 14#include "../../io-pgtable-arm.h"
 15
 16static void __maybe_unused
 17arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain)
 18{
 19	struct arm_smmu_master_domain *master_domain;
 20	struct arm_smmu_cd target_cd;
 21	unsigned long flags;
 22
 23	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 24	list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) {
 25		struct arm_smmu_master *master = master_domain->master;
 26		struct arm_smmu_cd *cdptr;
 27
 28		cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid);
 29		if (WARN_ON(!cdptr))
 30			continue;
 31
 32		arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
 33		arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
 34					&target_cd);
 35	}
 36	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 37}
 38
 39static u64 page_size_to_cd(void)
 40{
 41	static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K ||
 42		      PAGE_SIZE == SZ_64K);
 43	if (PAGE_SIZE == SZ_64K)
 44		return ARM_LPAE_TCR_TG0_64K;
 45	if (PAGE_SIZE == SZ_16K)
 46		return ARM_LPAE_TCR_TG0_16K;
 47	return ARM_LPAE_TCR_TG0_4K;
 48}
 49
 50VISIBLE_IF_KUNIT
 51void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
 52			  struct arm_smmu_master *master, struct mm_struct *mm,
 53			  u16 asid)
 54{
 55	u64 par;
 56
 57	memset(target, 0, sizeof(*target));
 58
 59	par = cpuid_feature_extract_unsigned_field(
 60		read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1),
 61		ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 62
 63	target->data[0] = cpu_to_le64(
 64		CTXDESC_CD_0_TCR_EPD1 |
 65#ifdef __BIG_ENDIAN
 66		CTXDESC_CD_0_ENDI |
 67#endif
 68		CTXDESC_CD_0_V |
 69		FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) |
 70		CTXDESC_CD_0_AA64 |
 71		(master->stall_enabled ? CTXDESC_CD_0_S : 0) |
 72		CTXDESC_CD_0_R |
 73		CTXDESC_CD_0_A |
 74		CTXDESC_CD_0_ASET |
 75		FIELD_PREP(CTXDESC_CD_0_ASID, asid));
 76
 77	/*
 78	 * If no MM is passed then this creates a SVA entry that faults
 79	 * everything. arm_smmu_write_cd_entry() can hitlessly go between these
 80	 * two entries types since TTB0 is ignored by HW when EPD0 is set.
 81	 */
 82	if (mm) {
 83		target->data[0] |= cpu_to_le64(
 84			FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ,
 85				   64ULL - vabits_actual) |
 86			FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) |
 87			FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0,
 88				   ARM_LPAE_TCR_RGN_WBWA) |
 89			FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0,
 90				   ARM_LPAE_TCR_RGN_WBWA) |
 91			FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS));
 92
 93		target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) &
 94					      CTXDESC_CD_1_TTB0_MASK);
 95	} else {
 96		target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0);
 97
 98		/*
 99		 * Disable stall and immediately generate an abort if stall
100		 * disable is permitted. This speeds up cleanup for an unclean
101		 * exit if the device is still doing a lot of DMA.
102		 */
103		if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
104			target->data[0] &=
105				cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R));
106	}
107
108	/*
109	 * MAIR value is pretty much constant and global, so we can just get it
110	 * from the current CPU register
111	 */
112	target->data[3] = cpu_to_le64(read_sysreg(mair_el1));
113
114	/*
115	 * Note that we don't bother with S1PIE on the SMMU, we just rely on
116	 * our default encoding scheme matching direct permissions anyway.
117	 * SMMU has no notion of S1POE nor GCS, so make sure that is clear if
118	 * either is enabled for CPUs, just in case anyone imagines otherwise.
119	 */
120	if (system_supports_poe() || system_supports_gcs())
121		dev_warn_once(master->smmu->dev, "SVA devices ignore permission overlays and GCS\n");
122}
123EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd);
124
125/*
126 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this
127 * is used as a threshold to replace per-page TLBI commands to issue in the
128 * command queue with an address-space TLBI command, when SMMU w/o a range
129 * invalidation feature handles too many per-page TLBI commands, which will
130 * otherwise result in a soft lockup.
131 */
132#define CMDQ_MAX_TLBI_OPS		(1 << (PAGE_SHIFT - 3))
133
134static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
135						struct mm_struct *mm,
136						unsigned long start,
137						unsigned long end)
138{
139	struct arm_smmu_domain *smmu_domain =
140		container_of(mn, struct arm_smmu_domain, mmu_notifier);
141	size_t size;
142
143	/*
144	 * The mm_types defines vm_end as the first byte after the end address,
145	 * different from IOMMU subsystem using the last address of an address
146	 * range. So do a simple translation here by calculating size correctly.
147	 */
148	size = end - start;
149	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) {
150		if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE)
151			size = 0;
152	} else {
153		if (size == ULONG_MAX)
154			size = 0;
155	}
156
157	if (!size)
158		arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
159	else
160		arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid,
161					    PAGE_SIZE, false, smmu_domain);
162
163	arm_smmu_atc_inv_domain(smmu_domain, start, size);
164}
165
166static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
167{
168	struct arm_smmu_domain *smmu_domain =
169		container_of(mn, struct arm_smmu_domain, mmu_notifier);
170	struct arm_smmu_master_domain *master_domain;
171	unsigned long flags;
172
173	/*
174	 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
175	 * but disable translation.
176	 */
177	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
178	list_for_each_entry(master_domain, &smmu_domain->devices,
179			    devices_elm) {
180		struct arm_smmu_master *master = master_domain->master;
181		struct arm_smmu_cd target;
182		struct arm_smmu_cd *cdptr;
183
184		cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid);
185		if (WARN_ON(!cdptr))
186			continue;
187		arm_smmu_make_sva_cd(&target, master, NULL,
188				     smmu_domain->cd.asid);
189		arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
190					&target);
191	}
192	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
193
194	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
195	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
196}
197
198static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
199{
200	kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier));
201}
202
203static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
204	.arch_invalidate_secondary_tlbs	= arm_smmu_mm_arch_invalidate_secondary_tlbs,
205	.release			= arm_smmu_mm_release,
206	.free_notifier			= arm_smmu_mmu_notifier_free,
207};
208
209bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
210{
211	unsigned long reg, fld;
212	unsigned long oas;
213	unsigned long asid_bits;
214	u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
215
216	if (vabits_actual == 52) {
217		/* We don't support LPA2 */
218		if (PAGE_SIZE != SZ_64K)
219			return false;
220		feat_mask |= ARM_SMMU_FEAT_VAX;
221	}
222
223	if (system_supports_bbml2_noabort())
224		feat_mask |= ARM_SMMU_FEAT_BBML2;
225
226	if ((smmu->features & feat_mask) != feat_mask)
227		return false;
228
229	if (!(smmu->pgsize_bitmap & PAGE_SIZE))
230		return false;
231
232	/*
233	 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're
234	 * not even pretending to support AArch32 here. Abort if the MMU outputs
235	 * addresses larger than what we support.
236	 */
237	reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
238	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
239	oas = id_aa64mmfr0_parange_to_phys_shift(fld);
240	if (smmu->oas < oas)
241		return false;
242
243	/* We can support bigger ASIDs than the CPU, but not smaller */
244	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
245	asid_bits = fld ? 16 : 8;
246	if (smmu->asid_bits < asid_bits)
247		return false;
248
249	/*
250	 * See max_pinned_asids in arch/arm64/mm/context.c. The following is
251	 * generally the maximum number of bindable processes.
252	 */
253	if (arm64_kernel_unmapped_at_el0())
254		asid_bits--;
255	dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) -
256		num_possible_cpus() - 2);
257
258	return true;
259}
260
261void arm_smmu_sva_notifier_synchronize(void)
262{
263	/*
264	 * Some MMU notifiers may still be waiting to be freed, using
265	 * arm_smmu_mmu_notifier_free(). Wait for them.
266	 */
267	mmu_notifier_synchronize();
268}
269
270static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
271				      struct device *dev, ioasid_t id,
272				      struct iommu_domain *old)
273{
274	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
275	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
276	struct arm_smmu_cd target;
277	int ret;
278
279	if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
280		return -EOPNOTSUPP;
281
282	/* Prevent arm_smmu_mm_release from being called while we are attaching */
283	if (!mmget_not_zero(domain->mm))
284		return -EINVAL;
285
286	/*
287	 * This does not need the arm_smmu_asid_lock because SVA domains never
288	 * get reassigned
289	 */
290	arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid);
291	ret = arm_smmu_set_pasid(master, smmu_domain, id, &target, old);
292
293	mmput(domain->mm);
294	return ret;
295}
296
297static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
298{
299	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
300
301	/*
302	 * Ensure the ASID is empty in the iommu cache before allowing reuse.
303	 */
304	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
305
306	/*
307	 * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can
308	 * still be called/running at this point. We allow the ASID to be
309	 * reused, and if there is a race then it just suffers harmless
310	 * unnecessary invalidation.
311	 */
312	xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
313
314	/*
315	 * Actual free is defered to the SRCU callback
316	 * arm_smmu_mmu_notifier_free()
317	 */
318	mmu_notifier_put(&smmu_domain->mmu_notifier);
319}
320
321static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
322	.set_dev_pasid		= arm_smmu_sva_set_dev_pasid,
323	.free			= arm_smmu_sva_domain_free
324};
325
326struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
327					       struct mm_struct *mm)
328{
329	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
330	struct arm_smmu_device *smmu = master->smmu;
331	struct arm_smmu_domain *smmu_domain;
332	u32 asid;
333	int ret;
334
335	if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
336		return ERR_PTR(-EOPNOTSUPP);
337
338	smmu_domain = arm_smmu_domain_alloc();
339	if (IS_ERR(smmu_domain))
340		return ERR_CAST(smmu_domain);
341	smmu_domain->domain.type = IOMMU_DOMAIN_SVA;
342	smmu_domain->domain.ops = &arm_smmu_sva_domain_ops;
343
344	/*
345	 * Choose page_size as the leaf page size for invalidation when
346	 * ARM_SMMU_FEAT_RANGE_INV is present
347	 */
348	smmu_domain->domain.pgsize_bitmap = PAGE_SIZE;
349	smmu_domain->smmu = smmu;
350
351	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
352		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
353	if (ret)
354		goto err_free;
355
356	smmu_domain->cd.asid = asid;
357	smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops;
358	ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm);
359	if (ret)
360		goto err_asid;
361
362	return &smmu_domain->domain;
363
364err_asid:
365	xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
366err_free:
367	kfree(smmu_domain);
368	return ERR_PTR(ret);
369}