arm-smmu.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * IOMMU API for ARM architected SMMU implementations.
  4 *
  5 * Copyright (C) 2013 ARM Limited
  6 *
  7 * Author: Will Deacon <will.deacon@arm.com>
  8 */
  9
 10#ifndef _ARM_SMMU_H
 11#define _ARM_SMMU_H
 12
 13#include <linux/atomic.h>
 14#include <linux/bitfield.h>
 15#include <linux/bits.h>
 16#include <linux/clk.h>
 17#include <linux/device.h>
 18#include <linux/io-64-nonatomic-hi-lo.h>
 19#include <linux/io-pgtable.h>
 20#include <linux/iommu.h>
 21#include <linux/irqreturn.h>
 22#include <linux/mutex.h>
 23#include <linux/spinlock.h>
 24#include <linux/types.h>
 25
 26/* Configuration registers */
 27#define ARM_SMMU_GR0_sCR0		0x0
 28#define ARM_SMMU_sCR0_VMID16EN		BIT(31)
 29#define ARM_SMMU_sCR0_BSU		GENMASK(15, 14)
 30#define ARM_SMMU_sCR0_FB		BIT(13)
 31#define ARM_SMMU_sCR0_PTM		BIT(12)
 32#define ARM_SMMU_sCR0_VMIDPNE		BIT(11)
 33#define ARM_SMMU_sCR0_USFCFG		BIT(10)
 34#define ARM_SMMU_sCR0_GCFGFIE		BIT(5)
 35#define ARM_SMMU_sCR0_GCFGFRE		BIT(4)
 36#define ARM_SMMU_sCR0_EXIDENABLE	BIT(3)
 37#define ARM_SMMU_sCR0_GFIE		BIT(2)
 38#define ARM_SMMU_sCR0_GFRE		BIT(1)
 39#define ARM_SMMU_sCR0_CLIENTPD		BIT(0)
 40
 41/* Auxiliary Configuration register */
 42#define ARM_SMMU_GR0_sACR		0x10
 43
 44/* Identification registers */
 45#define ARM_SMMU_GR0_ID0		0x20
 46#define ARM_SMMU_ID0_S1TS		BIT(30)
 47#define ARM_SMMU_ID0_S2TS		BIT(29)
 48#define ARM_SMMU_ID0_NTS		BIT(28)
 49#define ARM_SMMU_ID0_SMS		BIT(27)
 50#define ARM_SMMU_ID0_ATOSNS		BIT(26)
 51#define ARM_SMMU_ID0_PTFS_NO_AARCH32	BIT(25)
 52#define ARM_SMMU_ID0_PTFS_NO_AARCH32S	BIT(24)
 53#define ARM_SMMU_ID0_NUMIRPT		GENMASK(23, 16)
 54#define ARM_SMMU_ID0_CTTW		BIT(14)
 55#define ARM_SMMU_ID0_NUMSIDB		GENMASK(12, 9)
 56#define ARM_SMMU_ID0_EXIDS		BIT(8)
 57#define ARM_SMMU_ID0_NUMSMRG		GENMASK(7, 0)
 58
 59#define ARM_SMMU_GR0_ID1		0x24
 60#define ARM_SMMU_ID1_PAGESIZE		BIT(31)
 61#define ARM_SMMU_ID1_NUMPAGENDXB	GENMASK(30, 28)
 62#define ARM_SMMU_ID1_NUMS2CB		GENMASK(23, 16)
 63#define ARM_SMMU_ID1_NUMCB		GENMASK(7, 0)
 64
 65#define ARM_SMMU_GR0_ID2		0x28
 66#define ARM_SMMU_ID2_VMID16		BIT(15)
 67#define ARM_SMMU_ID2_PTFS_64K		BIT(14)
 68#define ARM_SMMU_ID2_PTFS_16K		BIT(13)
 69#define ARM_SMMU_ID2_PTFS_4K		BIT(12)
 70#define ARM_SMMU_ID2_UBS		GENMASK(11, 8)
 71#define ARM_SMMU_ID2_OAS		GENMASK(7, 4)
 72#define ARM_SMMU_ID2_IAS		GENMASK(3, 0)
 73
 74#define ARM_SMMU_GR0_ID3		0x2c
 75#define ARM_SMMU_GR0_ID4		0x30
 76#define ARM_SMMU_GR0_ID5		0x34
 77#define ARM_SMMU_GR0_ID6		0x38
 78
 79#define ARM_SMMU_GR0_ID7		0x3c
 80#define ARM_SMMU_ID7_MAJOR		GENMASK(7, 4)
 81#define ARM_SMMU_ID7_MINOR		GENMASK(3, 0)
 82
 83#define ARM_SMMU_GR0_sGFSR		0x48
 84#define ARM_SMMU_sGFSR_USF		BIT(1)
 85
 86#define ARM_SMMU_GR0_sGFSYNR0		0x50
 87#define ARM_SMMU_GR0_sGFSYNR1		0x54
 88#define ARM_SMMU_GR0_sGFSYNR2		0x58
 89
 90/* Global TLB invalidation */
 91#define ARM_SMMU_GR0_TLBIVMID		0x64
 92#define ARM_SMMU_GR0_TLBIALLNSNH	0x68
 93#define ARM_SMMU_GR0_TLBIALLH		0x6c
 94#define ARM_SMMU_GR0_sTLBGSYNC		0x70
 95
 96#define ARM_SMMU_GR0_sTLBGSTATUS	0x74
 97#define ARM_SMMU_sTLBGSTATUS_GSACTIVE	BIT(0)
 98
 99/* Stream mapping registers */
100#define ARM_SMMU_GR0_SMR(n)		(0x800 + ((n) << 2))
101#define ARM_SMMU_SMR_VALID		BIT(31)
102#define ARM_SMMU_SMR_MASK		GENMASK(31, 16)
103#define ARM_SMMU_SMR_ID			GENMASK(15, 0)
104
105#define ARM_SMMU_GR0_S2CR(n)		(0xc00 + ((n) << 2))
106#define ARM_SMMU_S2CR_PRIVCFG		GENMASK(25, 24)
107enum arm_smmu_s2cr_privcfg {
108	S2CR_PRIVCFG_DEFAULT,
109	S2CR_PRIVCFG_DIPAN,
110	S2CR_PRIVCFG_UNPRIV,
111	S2CR_PRIVCFG_PRIV,
112};
113#define ARM_SMMU_S2CR_TYPE		GENMASK(17, 16)
114enum arm_smmu_s2cr_type {
115	S2CR_TYPE_TRANS,
116	S2CR_TYPE_BYPASS,
117	S2CR_TYPE_FAULT,
118};
119#define ARM_SMMU_S2CR_EXIDVALID		BIT(10)
120#define ARM_SMMU_S2CR_CBNDX		GENMASK(7, 0)
121
122/* Context bank attribute registers */
123#define ARM_SMMU_GR1_CBAR(n)		(0x0 + ((n) << 2))
124#define ARM_SMMU_CBAR_IRPTNDX		GENMASK(31, 24)
125#define ARM_SMMU_CBAR_TYPE		GENMASK(17, 16)
126enum arm_smmu_cbar_type {
127	CBAR_TYPE_S2_TRANS,
128	CBAR_TYPE_S1_TRANS_S2_BYPASS,
129	CBAR_TYPE_S1_TRANS_S2_FAULT,
130	CBAR_TYPE_S1_TRANS_S2_TRANS,
131};
132#define ARM_SMMU_CBAR_S1_MEMATTR	GENMASK(15, 12)
133#define ARM_SMMU_CBAR_S1_MEMATTR_WB	0xf
134#define ARM_SMMU_CBAR_S1_BPSHCFG	GENMASK(9, 8)
135#define ARM_SMMU_CBAR_S1_BPSHCFG_NSH	3
136#define ARM_SMMU_CBAR_VMID		GENMASK(7, 0)
137
138#define ARM_SMMU_GR1_CBFRSYNRA(n)	(0x400 + ((n) << 2))
139#define ARM_SMMU_CBFRSYNRA_SID		GENMASK(15, 0)
140
141#define ARM_SMMU_GR1_CBA2R(n)		(0x800 + ((n) << 2))
142#define ARM_SMMU_CBA2R_VMID16		GENMASK(31, 16)
143#define ARM_SMMU_CBA2R_VA64		BIT(0)
144
145#define ARM_SMMU_CB_SCTLR		0x0
146#define ARM_SMMU_SCTLR_S1_ASIDPNE	BIT(12)
147#define ARM_SMMU_SCTLR_CFCFG		BIT(7)
148#define ARM_SMMU_SCTLR_HUPCF		BIT(8)
149#define ARM_SMMU_SCTLR_CFIE		BIT(6)
150#define ARM_SMMU_SCTLR_CFRE		BIT(5)
151#define ARM_SMMU_SCTLR_E		BIT(4)
152#define ARM_SMMU_SCTLR_AFE		BIT(2)
153#define ARM_SMMU_SCTLR_TRE		BIT(1)
154#define ARM_SMMU_SCTLR_M		BIT(0)
155
156#define ARM_SMMU_CB_ACTLR		0x4
157#define ARM_SMMU_GFX_PRR_CFG_LADDR	0x6008
158#define ARM_SMMU_GFX_PRR_CFG_UADDR	0x600C
159
160#define ARM_SMMU_CB_RESUME		0x8
161#define ARM_SMMU_RESUME_TERMINATE	BIT(0)
162
163#define ARM_SMMU_CB_TCR2		0x10
164#define ARM_SMMU_TCR2_SEP		GENMASK(17, 15)
165#define ARM_SMMU_TCR2_SEP_UPSTREAM	0x7
166#define ARM_SMMU_TCR2_AS		BIT(4)
167#define ARM_SMMU_TCR2_PASIZE		GENMASK(3, 0)
168
169#define ARM_SMMU_CB_TTBR0		0x20
170#define ARM_SMMU_CB_TTBR1		0x28
171#define ARM_SMMU_TTBRn_ASID		GENMASK_ULL(63, 48)
172
173#define ARM_SMMU_CB_TCR			0x30
174#define ARM_SMMU_TCR_EAE		BIT(31)
175#define ARM_SMMU_TCR_EPD1		BIT(23)
176#define ARM_SMMU_TCR_A1			BIT(22)
177#define ARM_SMMU_TCR_TG0		GENMASK(15, 14)
178#define ARM_SMMU_TCR_SH0		GENMASK(13, 12)
179#define ARM_SMMU_TCR_ORGN0		GENMASK(11, 10)
180#define ARM_SMMU_TCR_IRGN0		GENMASK(9, 8)
181#define ARM_SMMU_TCR_EPD0		BIT(7)
182#define ARM_SMMU_TCR_T0SZ		GENMASK(5, 0)
183
184#define ARM_SMMU_VTCR_RES1		BIT(31)
185#define ARM_SMMU_VTCR_PS		GENMASK(18, 16)
186#define ARM_SMMU_VTCR_TG0		ARM_SMMU_TCR_TG0
187#define ARM_SMMU_VTCR_SH0		ARM_SMMU_TCR_SH0
188#define ARM_SMMU_VTCR_ORGN0		ARM_SMMU_TCR_ORGN0
189#define ARM_SMMU_VTCR_IRGN0		ARM_SMMU_TCR_IRGN0
190#define ARM_SMMU_VTCR_SL0		GENMASK(7, 6)
191#define ARM_SMMU_VTCR_T0SZ		ARM_SMMU_TCR_T0SZ
192
193#define ARM_SMMU_CB_CONTEXTIDR		0x34
194#define ARM_SMMU_CB_S1_MAIR0		0x38
195#define ARM_SMMU_CB_S1_MAIR1		0x3c
196
197#define ARM_SMMU_CB_PAR			0x50
198#define ARM_SMMU_CB_PAR_F		BIT(0)
199
200#define ARM_SMMU_CB_FSR			0x58
201#define ARM_SMMU_CB_FSR_MULTI		BIT(31)
202#define ARM_SMMU_CB_FSR_SS		BIT(30)
203#define ARM_SMMU_CB_FSR_FORMAT		GENMASK(10, 9)
204#define ARM_SMMU_CB_FSR_UUT		BIT(8)
205#define ARM_SMMU_CB_FSR_ASF		BIT(7)
206#define ARM_SMMU_CB_FSR_TLBLKF		BIT(6)
207#define ARM_SMMU_CB_FSR_TLBMCF		BIT(5)
208#define ARM_SMMU_CB_FSR_EF		BIT(4)
209#define ARM_SMMU_CB_FSR_PF		BIT(3)
210#define ARM_SMMU_CB_FSR_AFF		BIT(2)
211#define ARM_SMMU_CB_FSR_TF		BIT(1)
212
213#define ARM_SMMU_CB_FSR_IGN		(ARM_SMMU_CB_FSR_AFF |		\
214					 ARM_SMMU_CB_FSR_ASF |		\
215					 ARM_SMMU_CB_FSR_TLBMCF |	\
216					 ARM_SMMU_CB_FSR_TLBLKF)
217
218#define ARM_SMMU_CB_FSR_FAULT		(ARM_SMMU_CB_FSR_MULTI |	\
219					 ARM_SMMU_CB_FSR_SS |		\
220					 ARM_SMMU_CB_FSR_UUT |		\
221					 ARM_SMMU_CB_FSR_EF |		\
222					 ARM_SMMU_CB_FSR_PF |		\
223					 ARM_SMMU_CB_FSR_TF |		\
224					 ARM_SMMU_CB_FSR_IGN)
225
226#define ARM_SMMU_CB_FAR			0x60
227
228#define ARM_SMMU_CB_FSYNR0		0x68
229#define ARM_SMMU_CB_FSYNR0_PLVL		GENMASK(1, 0)
230#define ARM_SMMU_CB_FSYNR0_WNR		BIT(4)
231#define ARM_SMMU_CB_FSYNR0_PNU		BIT(5)
232#define ARM_SMMU_CB_FSYNR0_IND		BIT(6)
233#define ARM_SMMU_CB_FSYNR0_NSATTR	BIT(8)
234#define ARM_SMMU_CB_FSYNR0_PTWF		BIT(10)
235#define ARM_SMMU_CB_FSYNR0_AFR		BIT(11)
236#define ARM_SMMU_CB_FSYNR0_S1CBNDX	GENMASK(23, 16)
237
238#define ARM_SMMU_CB_FSYNR1		0x6c
239
240#define ARM_SMMU_CB_S1_TLBIVA		0x600
241#define ARM_SMMU_CB_S1_TLBIASID		0x610
242#define ARM_SMMU_CB_S1_TLBIVAL		0x620
243#define ARM_SMMU_CB_S2_TLBIIPAS2	0x630
244#define ARM_SMMU_CB_S2_TLBIIPAS2L	0x638
245#define ARM_SMMU_CB_TLBSYNC		0x7f0
246#define ARM_SMMU_CB_TLBSTATUS		0x7f4
247#define ARM_SMMU_CB_ATS1PR		0x800
248
249#define ARM_SMMU_CB_ATSR		0x8f0
250#define ARM_SMMU_CB_ATSR_ACTIVE		BIT(0)
251
252#define ARM_SMMU_RESUME_TERMINATE	BIT(0)
253
254/* Maximum number of context banks per SMMU */
255#define ARM_SMMU_MAX_CBS		128
256
257#define TLB_LOOP_TIMEOUT		1000000	/* 1s! */
258#define TLB_SPIN_COUNT			10
259
260/* Shared driver definitions */
261enum arm_smmu_arch_version {
262	ARM_SMMU_V1,
263	ARM_SMMU_V1_64K,
264	ARM_SMMU_V2,
265};
266
267enum arm_smmu_implementation {
268	GENERIC_SMMU,
269	ARM_MMU500,
270	CAVIUM_SMMUV2,
271	QCOM_SMMUV2,
272};
273
274struct arm_smmu_s2cr {
275	struct iommu_group		*group;
276	int				count;
277	enum arm_smmu_s2cr_type		type;
278	enum arm_smmu_s2cr_privcfg	privcfg;
279	u8				cbndx;
280};
281
282struct arm_smmu_smr {
283	u16				mask;
284	u16				id;
285	bool				valid;
286	bool				pinned;
287};
288
289struct arm_smmu_device {
290	struct device			*dev;
291
292	void __iomem			*base;
293	phys_addr_t			ioaddr;
294	unsigned int			numpage;
295	unsigned int			pgshift;
296
297#define ARM_SMMU_FEAT_COHERENT_WALK	(1 << 0)
298#define ARM_SMMU_FEAT_STREAM_MATCH	(1 << 1)
299#define ARM_SMMU_FEAT_TRANS_S1		(1 << 2)
300#define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)
301#define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)
302#define ARM_SMMU_FEAT_TRANS_OPS		(1 << 5)
303#define ARM_SMMU_FEAT_VMID16		(1 << 6)
304#define ARM_SMMU_FEAT_FMT_AARCH64_4K	(1 << 7)
305#define ARM_SMMU_FEAT_FMT_AARCH64_16K	(1 << 8)
306#define ARM_SMMU_FEAT_FMT_AARCH64_64K	(1 << 9)
307#define ARM_SMMU_FEAT_FMT_AARCH32_L	(1 << 10)
308#define ARM_SMMU_FEAT_FMT_AARCH32_S	(1 << 11)
309#define ARM_SMMU_FEAT_EXIDS		(1 << 12)
310	u32				features;
311
312	enum arm_smmu_arch_version	version;
313	enum arm_smmu_implementation	model;
314	const struct arm_smmu_impl	*impl;
315
316	u32				num_context_banks;
317	u32				num_s2_context_banks;
318	DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
319	struct arm_smmu_cb		*cbs;
320	atomic_t			irptndx;
321
322	u32				num_mapping_groups;
323	u16				streamid_mask;
324	u16				smr_mask_mask;
325	struct arm_smmu_smr		*smrs;
326	struct arm_smmu_s2cr		*s2crs;
327	struct mutex			stream_map_mutex;
328
329	unsigned long			va_size;
330	unsigned long			ipa_size;
331	unsigned long			pa_size;
332	unsigned long			pgsize_bitmap;
333
334	int				num_context_irqs;
335	int				num_clks;
336	unsigned int			*irqs;
337	struct clk_bulk_data		*clks;
338
339	spinlock_t			global_sync_lock;
340
341	/* IOMMU core code handle */
342	struct iommu_device		iommu;
343};
344
345enum arm_smmu_context_fmt {
346	ARM_SMMU_CTX_FMT_NONE,
347	ARM_SMMU_CTX_FMT_AARCH64,
348	ARM_SMMU_CTX_FMT_AARCH32_L,
349	ARM_SMMU_CTX_FMT_AARCH32_S,
350};
351
352struct arm_smmu_cfg {
353	u8				cbndx;
354	u8				irptndx;
355	union {
356		u16			asid;
357		u16			vmid;
358	};
359	enum arm_smmu_cbar_type		cbar;
360	enum arm_smmu_context_fmt	fmt;
361	bool				flush_walk_prefer_tlbiasid;
362};
363#define ARM_SMMU_INVALID_IRPTNDX	0xff
364
365struct arm_smmu_cb {
366	u64				ttbr[2];
367	u32				tcr[2];
368	u32				mair[2];
369	struct arm_smmu_cfg		*cfg;
370};
371
372enum arm_smmu_domain_stage {
373	ARM_SMMU_DOMAIN_S1 = 0,
374	ARM_SMMU_DOMAIN_S2,
375	ARM_SMMU_DOMAIN_NESTED,
376};
377
378struct arm_smmu_domain {
379	struct arm_smmu_device		*smmu;
380	struct io_pgtable_ops		*pgtbl_ops;
381	unsigned long			pgtbl_quirks;
382	const struct iommu_flush_ops	*flush_ops;
383	struct arm_smmu_cfg		cfg;
384	enum arm_smmu_domain_stage	stage;
385	struct mutex			init_mutex; /* Protects smmu pointer */
386	spinlock_t			cb_lock; /* Serialises ATS1* ops and TLB syncs */
387	struct iommu_domain		domain;
388};
389
390struct arm_smmu_master_cfg {
391	struct arm_smmu_device		*smmu;
392	s16				smendx[];
393};
394
395static inline u32 arm_smmu_lpae_tcr(const struct io_pgtable_cfg *cfg)
396{
397	u32 tcr = FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |
398		FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |
399		FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |
400		FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |
401		FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
402
403       /*
404	* When TTBR1 is selected shift the TCR fields by 16 bits and disable
405	* translation in TTBR0
406	*/
407	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
408		tcr = (tcr << 16) & ~ARM_SMMU_TCR_A1;
409		tcr |= ARM_SMMU_TCR_EPD0;
410	} else
411		tcr |= ARM_SMMU_TCR_EPD1;
412
413	return tcr;
414}
415
416static inline u32 arm_smmu_lpae_tcr2(const struct io_pgtable_cfg *cfg)
417{
418	return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |
419	       FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
420}
421
422static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg)
423{
424	return ARM_SMMU_VTCR_RES1 |
425	       FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |
426	       FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) |
427	       FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) |
428	       FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) |
429	       FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) |
430	       FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) |
431	       FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
432}
433
434/* Implementation details, yay! */
435struct arm_smmu_impl {
436	u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
437	void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
438			  u32 val);
439	u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
440	void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
441			    u64 val);
442	int (*cfg_probe)(struct arm_smmu_device *smmu);
443	int (*reset)(struct arm_smmu_device *smmu);
444	int (*init_context)(struct arm_smmu_domain *smmu_domain,
445			struct io_pgtable_cfg *cfg, struct device *dev);
446	void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
447			 int status);
448	int (*def_domain_type)(struct device *dev);
449	irqreturn_t (*global_fault)(int irq, void *dev);
450	irqreturn_t (*context_fault)(int irq, void *dev);
451	bool context_fault_needs_threaded_irq;
452	int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain,
453				  struct arm_smmu_device *smmu,
454				  struct device *dev, int start);
455	void (*write_s2cr)(struct arm_smmu_device *smmu, int idx);
456	void (*write_sctlr)(struct arm_smmu_device *smmu, int idx, u32 reg);
457	void (*probe_finalize)(struct arm_smmu_device *smmu, struct device *dev);
458};
459
460#define INVALID_SMENDX			-1
461#define cfg_smendx(cfg, fw, i) \
462	(i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
463#define for_each_cfg_sme(cfg, fw, i, idx) \
464	for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
465
466static inline int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
467{
468	int idx;
469
470	do {
471		idx = find_next_zero_bit(map, end, start);
472		if (idx == end)
473			return -ENOSPC;
474	} while (test_and_set_bit(idx, map));
475
476	return idx;
477}
478
479static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
480{
481	return smmu->base + (n << smmu->pgshift);
482}
483
484static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
485{
486	if (smmu->impl && unlikely(smmu->impl->read_reg))
487		return smmu->impl->read_reg(smmu, page, offset);
488	return readl_relaxed(arm_smmu_page(smmu, page) + offset);
489}
490
491static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
492				   int offset, u32 val)
493{
494	if (smmu->impl && unlikely(smmu->impl->write_reg))
495		smmu->impl->write_reg(smmu, page, offset, val);
496	else
497		writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
498}
499
500static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
501{
502	if (smmu->impl && unlikely(smmu->impl->read_reg64))
503		return smmu->impl->read_reg64(smmu, page, offset);
504	return readq_relaxed(arm_smmu_page(smmu, page) + offset);
505}
506
507static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
508				   int offset, u64 val)
509{
510	if (smmu->impl && unlikely(smmu->impl->write_reg64))
511		smmu->impl->write_reg64(smmu, page, offset, val);
512	else
513		writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
514}
515
516#define ARM_SMMU_GR0		0
517#define ARM_SMMU_GR1		1
518#define ARM_SMMU_CB(s, n)	((s)->numpage + (n))
519
520#define arm_smmu_gr0_read(s, o)		\
521	arm_smmu_readl((s), ARM_SMMU_GR0, (o))
522#define arm_smmu_gr0_write(s, o, v)	\
523	arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
524
525#define arm_smmu_gr1_read(s, o)		\
526	arm_smmu_readl((s), ARM_SMMU_GR1, (o))
527#define arm_smmu_gr1_write(s, o, v)	\
528	arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
529
530#define arm_smmu_cb_read(s, n, o)	\
531	arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
532#define arm_smmu_cb_write(s, n, o, v)	\
533	arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
534#define arm_smmu_cb_readq(s, n, o)	\
535	arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
536#define arm_smmu_cb_writeq(s, n, o, v)	\
537	arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
538
539struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
540struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);
541struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
542
543int __init arm_smmu_impl_module_init(void);
544void __exit arm_smmu_impl_module_exit(void);
545int __init qcom_smmu_module_init(void);
546void __exit qcom_smmu_module_exit(void);
547
548void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx);
549int arm_mmu500_reset(struct arm_smmu_device *smmu);
550
551struct arm_smmu_context_fault_info {
552	unsigned long iova;
553	u32 fsr;
554	u32 fsynr;
555	u32 cbfrsynra;
556};
557
558void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
559				      struct arm_smmu_context_fault_info *cfi);
560
561void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
562				       const struct arm_smmu_context_fault_info *cfi);
563
564#endif /* _ARM_SMMU_H */