arm-smmu-v3.h

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * IOMMU API for ARM architected SMMUv3 implementations.
   4 *
   5 * Copyright (C) 2015 ARM Limited
   6 */
   7
   8#ifndef _ARM_SMMU_V3_H
   9#define _ARM_SMMU_V3_H
  10
  11#include <linux/bitfield.h>
  12#include <linux/iommu.h>
  13#include <linux/iommufd.h>
  14#include <linux/kernel.h>
  15#include <linux/mmzone.h>
  16#include <linux/sizes.h>
  17
  18struct arm_smmu_device;
  19struct arm_vsmmu;
  20
  21/* MMIO registers */
  22#define ARM_SMMU_IDR0			0x0
  23#define IDR0_ST_LVL			GENMASK(28, 27)
  24#define IDR0_ST_LVL_2LVL		1
  25#define IDR0_STALL_MODEL		GENMASK(25, 24)
  26#define IDR0_STALL_MODEL_STALL		0
  27#define IDR0_STALL_MODEL_FORCE		2
  28#define IDR0_TTENDIAN			GENMASK(22, 21)
  29#define IDR0_TTENDIAN_MIXED		0
  30#define IDR0_TTENDIAN_LE		2
  31#define IDR0_TTENDIAN_BE		3
  32#define IDR0_CD2L			(1 << 19)
  33#define IDR0_VMID16			(1 << 18)
  34#define IDR0_PRI			(1 << 16)
  35#define IDR0_SEV			(1 << 14)
  36#define IDR0_MSI			(1 << 13)
  37#define IDR0_ASID16			(1 << 12)
  38#define IDR0_ATS			(1 << 10)
  39#define IDR0_HYP			(1 << 9)
  40#define IDR0_HTTU			GENMASK(7, 6)
  41#define IDR0_HTTU_ACCESS		1
  42#define IDR0_HTTU_ACCESS_DIRTY		2
  43#define IDR0_COHACC			(1 << 4)
  44#define IDR0_TTF			GENMASK(3, 2)
  45#define IDR0_TTF_AARCH64		2
  46#define IDR0_TTF_AARCH32_64		3
  47#define IDR0_S1P			(1 << 1)
  48#define IDR0_S2P			(1 << 0)
  49
  50#define ARM_SMMU_IDR1			0x4
  51#define IDR1_TABLES_PRESET		(1 << 30)
  52#define IDR1_QUEUES_PRESET		(1 << 29)
  53#define IDR1_REL			(1 << 28)
  54#define IDR1_ATTR_TYPES_OVR		(1 << 27)
  55#define IDR1_CMDQS			GENMASK(25, 21)
  56#define IDR1_EVTQS			GENMASK(20, 16)
  57#define IDR1_PRIQS			GENMASK(15, 11)
  58#define IDR1_SSIDSIZE			GENMASK(10, 6)
  59#define IDR1_SIDSIZE			GENMASK(5, 0)
  60
  61#define ARM_SMMU_IDR3			0xc
  62#define IDR3_FWB			(1 << 8)
  63#define IDR3_RIL			(1 << 10)
  64#define IDR3_BBM			GENMASK(12, 11)
  65
  66#define ARM_SMMU_IDR5			0x14
  67#define IDR5_STALL_MAX			GENMASK(31, 16)
  68#define IDR5_GRAN64K			(1 << 6)
  69#define IDR5_GRAN16K			(1 << 5)
  70#define IDR5_GRAN4K			(1 << 4)
  71#define IDR5_OAS			GENMASK(2, 0)
  72#define IDR5_OAS_32_BIT			0
  73#define IDR5_OAS_36_BIT			1
  74#define IDR5_OAS_40_BIT			2
  75#define IDR5_OAS_42_BIT			3
  76#define IDR5_OAS_44_BIT			4
  77#define IDR5_OAS_48_BIT			5
  78#define IDR5_OAS_52_BIT			6
  79#define IDR5_VAX			GENMASK(11, 10)
  80#define IDR5_VAX_52_BIT			1
  81
  82#define ARM_SMMU_IIDR			0x18
  83#define IIDR_PRODUCTID			GENMASK(31, 20)
  84#define IIDR_VARIANT			GENMASK(19, 16)
  85#define IIDR_REVISION			GENMASK(15, 12)
  86#define IIDR_IMPLEMENTER		GENMASK(11, 0)
  87
  88#define ARM_SMMU_AIDR			0x1C
  89
  90#define ARM_SMMU_CR0			0x20
  91#define CR0_ATSCHK			(1 << 4)
  92#define CR0_CMDQEN			(1 << 3)
  93#define CR0_EVTQEN			(1 << 2)
  94#define CR0_PRIQEN			(1 << 1)
  95#define CR0_SMMUEN			(1 << 0)
  96
  97#define ARM_SMMU_CR0ACK			0x24
  98
  99#define ARM_SMMU_CR1			0x28
 100#define CR1_TABLE_SH			GENMASK(11, 10)
 101#define CR1_TABLE_OC			GENMASK(9, 8)
 102#define CR1_TABLE_IC			GENMASK(7, 6)
 103#define CR1_QUEUE_SH			GENMASK(5, 4)
 104#define CR1_QUEUE_OC			GENMASK(3, 2)
 105#define CR1_QUEUE_IC			GENMASK(1, 0)
 106/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
 107#define CR1_CACHE_NC			0
 108#define CR1_CACHE_WB			1
 109#define CR1_CACHE_WT			2
 110
 111#define ARM_SMMU_CR2			0x2c
 112#define CR2_PTM				(1 << 2)
 113#define CR2_RECINVSID			(1 << 1)
 114#define CR2_E2H				(1 << 0)
 115
 116#define ARM_SMMU_GBPA			0x44
 117#define GBPA_UPDATE			(1 << 31)
 118#define GBPA_ABORT			(1 << 20)
 119
 120#define ARM_SMMU_IRQ_CTRL		0x50
 121#define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
 122#define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
 123#define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
 124
 125#define ARM_SMMU_IRQ_CTRLACK		0x54
 126
 127#define ARM_SMMU_GERROR			0x60
 128#define GERROR_SFM_ERR			(1 << 8)
 129#define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
 130#define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
 131#define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
 132#define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
 133#define GERROR_PRIQ_ABT_ERR		(1 << 3)
 134#define GERROR_EVTQ_ABT_ERR		(1 << 2)
 135#define GERROR_CMDQ_ERR			(1 << 0)
 136#define GERROR_ERR_MASK			0x1fd
 137
 138#define ARM_SMMU_GERRORN		0x64
 139
 140#define ARM_SMMU_GERROR_IRQ_CFG0	0x68
 141#define ARM_SMMU_GERROR_IRQ_CFG1	0x70
 142#define ARM_SMMU_GERROR_IRQ_CFG2	0x74
 143
 144#define ARM_SMMU_STRTAB_BASE		0x80
 145#define STRTAB_BASE_RA			(1UL << 62)
 146#define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
 147
 148#define ARM_SMMU_STRTAB_BASE_CFG	0x88
 149#define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
 150#define STRTAB_BASE_CFG_FMT_LINEAR	0
 151#define STRTAB_BASE_CFG_FMT_2LVL	1
 152#define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
 153#define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
 154
 155#define ARM_SMMU_CMDQ_BASE		0x90
 156#define ARM_SMMU_CMDQ_PROD		0x98
 157#define ARM_SMMU_CMDQ_CONS		0x9c
 158
 159#define ARM_SMMU_EVTQ_BASE		0xa0
 160#define ARM_SMMU_EVTQ_PROD		0xa8
 161#define ARM_SMMU_EVTQ_CONS		0xac
 162#define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
 163#define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
 164#define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
 165
 166#define ARM_SMMU_PRIQ_BASE		0xc0
 167#define ARM_SMMU_PRIQ_PROD		0xc8
 168#define ARM_SMMU_PRIQ_CONS		0xcc
 169#define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
 170#define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
 171#define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
 172
 173#define ARM_SMMU_REG_SZ			0xe00
 174
 175/* Common MSI config fields */
 176#define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
 177#define MSI_CFG2_SH			GENMASK(5, 4)
 178#define MSI_CFG2_MEMATTR		GENMASK(3, 0)
 179
 180/* Common memory attribute values */
 181#define ARM_SMMU_SH_NSH			0
 182#define ARM_SMMU_SH_OSH			2
 183#define ARM_SMMU_SH_ISH			3
 184#define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
 185#define ARM_SMMU_MEMATTR_OIWB		0xf
 186
 187#define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
 188#define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
 189#define Q_OVERFLOW_FLAG			(1U << 31)
 190#define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
 191#define Q_ENT(q, p)			((q)->base +			\
 192					 Q_IDX(&((q)->llq), p) *	\
 193					 (q)->ent_dwords)
 194
 195#define Q_BASE_RWA			(1UL << 62)
 196#define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
 197#define Q_BASE_LOG2SIZE			GENMASK(4, 0)
 198
 199/* Ensure DMA allocations are naturally aligned */
 200#ifdef CONFIG_CMA_ALIGNMENT
 201#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
 202#else
 203#define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_PAGE_ORDER)
 204#endif
 205
 206/*
 207 * Stream table.
 208 *
 209 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
 210 * 2lvl: 128k L1 entries,
 211 *       256 lazy entries per table (each table covers a PCI bus)
 212 */
 213#define STRTAB_SPLIT			8
 214
 215#define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
 216#define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
 217
 218#define STRTAB_STE_DWORDS		8
 219
 220struct arm_smmu_ste {
 221	__le64 data[STRTAB_STE_DWORDS];
 222};
 223
 224#define STRTAB_NUM_L2_STES		(1 << STRTAB_SPLIT)
 225struct arm_smmu_strtab_l2 {
 226	struct arm_smmu_ste stes[STRTAB_NUM_L2_STES];
 227};
 228
 229struct arm_smmu_strtab_l1 {
 230	__le64 l2ptr;
 231};
 232#define STRTAB_MAX_L1_ENTRIES		(1 << 17)
 233
 234static inline u32 arm_smmu_strtab_l1_idx(u32 sid)
 235{
 236	return sid / STRTAB_NUM_L2_STES;
 237}
 238
 239static inline u32 arm_smmu_strtab_l2_idx(u32 sid)
 240{
 241	return sid % STRTAB_NUM_L2_STES;
 242}
 243
 244#define STRTAB_STE_0_V			(1UL << 0)
 245#define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
 246#define STRTAB_STE_0_CFG_ABORT		0
 247#define STRTAB_STE_0_CFG_BYPASS		4
 248#define STRTAB_STE_0_CFG_S1_TRANS	5
 249#define STRTAB_STE_0_CFG_S2_TRANS	6
 250#define STRTAB_STE_0_CFG_NESTED		7
 251
 252#define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
 253#define STRTAB_STE_0_S1FMT_LINEAR	0
 254#define STRTAB_STE_0_S1FMT_64K_L2	2
 255#define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
 256#define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
 257
 258#define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
 259#define STRTAB_STE_1_S1DSS_TERMINATE	0x0
 260#define STRTAB_STE_1_S1DSS_BYPASS	0x1
 261#define STRTAB_STE_1_S1DSS_SSID0	0x2
 262
 263#define STRTAB_STE_1_S1C_CACHE_NC	0UL
 264#define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
 265#define STRTAB_STE_1_S1C_CACHE_WT	2UL
 266#define STRTAB_STE_1_S1C_CACHE_WB	3UL
 267#define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
 268#define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
 269#define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
 270
 271#define STRTAB_STE_1_MEV		(1UL << 19)
 272#define STRTAB_STE_1_S2FWB		(1UL << 25)
 273#define STRTAB_STE_1_S1STALLD		(1UL << 27)
 274
 275#define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
 276#define STRTAB_STE_1_EATS_ABT		0UL
 277#define STRTAB_STE_1_EATS_TRANS		1UL
 278#define STRTAB_STE_1_EATS_S1CHK		2UL
 279
 280#define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
 281#define STRTAB_STE_1_STRW_NSEL1		0UL
 282#define STRTAB_STE_1_STRW_EL2		2UL
 283
 284#define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
 285#define STRTAB_STE_1_SHCFG_INCOMING	1UL
 286
 287#define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
 288#define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
 289#define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
 290#define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
 291#define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
 292#define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
 293#define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
 294#define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
 295#define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
 296#define STRTAB_STE_2_S2AA64		(1UL << 51)
 297#define STRTAB_STE_2_S2ENDI		(1UL << 52)
 298#define STRTAB_STE_2_S2PTW		(1UL << 54)
 299#define STRTAB_STE_2_S2S		(1UL << 57)
 300#define STRTAB_STE_2_S2R		(1UL << 58)
 301
 302#define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
 303
 304/* These bits can be controlled by userspace for STRTAB_STE_0_CFG_NESTED */
 305#define STRTAB_STE_0_NESTING_ALLOWED                                         \
 306	cpu_to_le64(STRTAB_STE_0_V | STRTAB_STE_0_CFG | STRTAB_STE_0_S1FMT | \
 307		    STRTAB_STE_0_S1CTXPTR_MASK | STRTAB_STE_0_S1CDMAX)
 308#define STRTAB_STE_1_NESTING_ALLOWED                            \
 309	cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |   \
 310		    STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |   \
 311		    STRTAB_STE_1_S1STALLD | STRTAB_STE_1_EATS)
 312
 313/*
 314 * Context descriptors.
 315 *
 316 * Linear: when less than 1024 SSIDs are supported
 317 * 2lvl: at most 1024 L1 entries,
 318 *       1024 lazy entries per table.
 319 */
 320#define CTXDESC_L2_ENTRIES		1024
 321
 322#define CTXDESC_L1_DESC_V		(1UL << 0)
 323#define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
 324
 325#define CTXDESC_CD_DWORDS		8
 326
 327struct arm_smmu_cd {
 328	__le64 data[CTXDESC_CD_DWORDS];
 329};
 330
 331struct arm_smmu_cdtab_l2 {
 332	struct arm_smmu_cd cds[CTXDESC_L2_ENTRIES];
 333};
 334
 335struct arm_smmu_cdtab_l1 {
 336	__le64 l2ptr;
 337};
 338
 339static inline unsigned int arm_smmu_cdtab_l1_idx(unsigned int ssid)
 340{
 341	return ssid / CTXDESC_L2_ENTRIES;
 342}
 343
 344static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid)
 345{
 346	return ssid % CTXDESC_L2_ENTRIES;
 347}
 348
 349#define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
 350#define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
 351#define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
 352#define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
 353#define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
 354#define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
 355#define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
 356
 357#define CTXDESC_CD_0_ENDI		(1UL << 15)
 358#define CTXDESC_CD_0_V			(1UL << 31)
 359
 360#define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
 361#define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
 362
 363#define CTXDESC_CD_0_TCR_HA            (1UL << 43)
 364#define CTXDESC_CD_0_TCR_HD            (1UL << 42)
 365
 366#define CTXDESC_CD_0_AA64		(1UL << 41)
 367#define CTXDESC_CD_0_S			(1UL << 44)
 368#define CTXDESC_CD_0_R			(1UL << 45)
 369#define CTXDESC_CD_0_A			(1UL << 46)
 370#define CTXDESC_CD_0_ASET		(1UL << 47)
 371#define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
 372
 373#define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
 374
 375/*
 376 * When the SMMU only supports linear context descriptor tables, pick a
 377 * reasonable size limit (64kB).
 378 */
 379#define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / sizeof(struct arm_smmu_cd))
 380
 381/* Command queue */
 382#define CMDQ_ENT_SZ_SHIFT		4
 383#define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
 384#define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
 385
 386#define CMDQ_CONS_ERR			GENMASK(30, 24)
 387#define CMDQ_ERR_CERROR_NONE_IDX	0
 388#define CMDQ_ERR_CERROR_ILL_IDX		1
 389#define CMDQ_ERR_CERROR_ABT_IDX		2
 390#define CMDQ_ERR_CERROR_ATC_INV_IDX	3
 391
 392#define CMDQ_PROD_OWNED_FLAG		Q_OVERFLOW_FLAG
 393
 394/*
 395 * This is used to size the command queue and therefore must be at least
 396 * BITS_PER_LONG so that the valid_map works correctly (it relies on the
 397 * total number of queue entries being a multiple of BITS_PER_LONG).
 398 */
 399#define CMDQ_BATCH_ENTRIES		BITS_PER_LONG
 400
 401#define CMDQ_0_OP			GENMASK_ULL(7, 0)
 402#define CMDQ_0_SSV			(1UL << 11)
 403
 404#define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
 405#define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
 406#define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
 407
 408#define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
 409#define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
 410#define CMDQ_CFGI_1_LEAF		(1UL << 0)
 411#define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
 412
 413#define CMDQ_TLBI_0_NUM			GENMASK_ULL(16, 12)
 414#define CMDQ_TLBI_RANGE_NUM_MAX		31
 415#define CMDQ_TLBI_0_SCALE		GENMASK_ULL(24, 20)
 416#define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
 417#define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
 418#define CMDQ_TLBI_1_LEAF		(1UL << 0)
 419#define CMDQ_TLBI_1_TTL			GENMASK_ULL(9, 8)
 420#define CMDQ_TLBI_1_TG			GENMASK_ULL(11, 10)
 421#define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
 422#define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
 423
 424#define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
 425#define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
 426#define CMDQ_ATC_0_GLOBAL		(1UL << 9)
 427#define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
 428#define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
 429
 430#define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
 431#define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
 432#define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
 433#define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
 434
 435#define CMDQ_RESUME_0_RESP_TERM		0UL
 436#define CMDQ_RESUME_0_RESP_RETRY	1UL
 437#define CMDQ_RESUME_0_RESP_ABORT	2UL
 438#define CMDQ_RESUME_0_RESP		GENMASK_ULL(13, 12)
 439#define CMDQ_RESUME_0_SID		GENMASK_ULL(63, 32)
 440#define CMDQ_RESUME_1_STAG		GENMASK_ULL(15, 0)
 441
 442#define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
 443#define CMDQ_SYNC_0_CS_NONE		0
 444#define CMDQ_SYNC_0_CS_IRQ		1
 445#define CMDQ_SYNC_0_CS_SEV		2
 446#define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
 447#define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
 448#define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
 449#define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
 450
 451/* Event queue */
 452#define EVTQ_ENT_SZ_SHIFT		5
 453#define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
 454#define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
 455
 456#define EVTQ_0_ID			GENMASK_ULL(7, 0)
 457
 458#define EVT_ID_BAD_STREAMID_CONFIG	0x02
 459#define EVT_ID_STE_FETCH_FAULT		0x03
 460#define EVT_ID_BAD_STE_CONFIG		0x04
 461#define EVT_ID_STREAM_DISABLED_FAULT	0x06
 462#define EVT_ID_BAD_SUBSTREAMID_CONFIG	0x08
 463#define EVT_ID_CD_FETCH_FAULT		0x09
 464#define EVT_ID_BAD_CD_CONFIG		0x0a
 465#define EVT_ID_TRANSLATION_FAULT	0x10
 466#define EVT_ID_ADDR_SIZE_FAULT		0x11
 467#define EVT_ID_ACCESS_FAULT		0x12
 468#define EVT_ID_PERMISSION_FAULT		0x13
 469#define EVT_ID_VMS_FETCH_FAULT		0x25
 470
 471#define EVTQ_0_SSV			(1UL << 11)
 472#define EVTQ_0_SSID			GENMASK_ULL(31, 12)
 473#define EVTQ_0_SID			GENMASK_ULL(63, 32)
 474#define EVTQ_1_STAG			GENMASK_ULL(15, 0)
 475#define EVTQ_1_STALL			(1UL << 31)
 476#define EVTQ_1_PnU			(1UL << 33)
 477#define EVTQ_1_InD			(1UL << 34)
 478#define EVTQ_1_RnW			(1UL << 35)
 479#define EVTQ_1_S2			(1UL << 39)
 480#define EVTQ_1_CLASS			GENMASK_ULL(41, 40)
 481#define EVTQ_1_CLASS_TT			0x01
 482#define EVTQ_1_TT_READ			(1UL << 44)
 483#define EVTQ_2_ADDR			GENMASK_ULL(63, 0)
 484#define EVTQ_3_IPA			GENMASK_ULL(51, 12)
 485#define EVTQ_3_FETCH_ADDR		GENMASK_ULL(51, 3)
 486
 487/* PRI queue */
 488#define PRIQ_ENT_SZ_SHIFT		4
 489#define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
 490#define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
 491
 492#define PRIQ_0_SID			GENMASK_ULL(31, 0)
 493#define PRIQ_0_SSID			GENMASK_ULL(51, 32)
 494#define PRIQ_0_PERM_PRIV		(1UL << 58)
 495#define PRIQ_0_PERM_EXEC		(1UL << 59)
 496#define PRIQ_0_PERM_READ		(1UL << 60)
 497#define PRIQ_0_PERM_WRITE		(1UL << 61)
 498#define PRIQ_0_PRG_LAST			(1UL << 62)
 499#define PRIQ_0_SSID_V			(1UL << 63)
 500
 501#define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
 502#define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
 503
 504/* High-level queue structures */
 505#define ARM_SMMU_POLL_TIMEOUT_US	1000000 /* 1s! */
 506#define ARM_SMMU_POLL_SPIN_COUNT	10
 507
 508#define MSI_IOVA_BASE			0x8000000
 509#define MSI_IOVA_LENGTH			0x100000
 510
 511enum pri_resp {
 512	PRI_RESP_DENY = 0,
 513	PRI_RESP_FAIL = 1,
 514	PRI_RESP_SUCC = 2,
 515};
 516
 517struct arm_smmu_cmdq_ent {
 518	/* Common fields */
 519	u8				opcode;
 520	bool				substream_valid;
 521
 522	/* Command-specific fields */
 523	union {
 524		#define CMDQ_OP_PREFETCH_CFG	0x1
 525		struct {
 526			u32			sid;
 527		} prefetch;
 528
 529		#define CMDQ_OP_CFGI_STE	0x3
 530		#define CMDQ_OP_CFGI_ALL	0x4
 531		#define CMDQ_OP_CFGI_CD		0x5
 532		#define CMDQ_OP_CFGI_CD_ALL	0x6
 533		struct {
 534			u32			sid;
 535			u32			ssid;
 536			union {
 537				bool		leaf;
 538				u8		span;
 539			};
 540		} cfgi;
 541
 542		#define CMDQ_OP_TLBI_NH_ALL     0x10
 543		#define CMDQ_OP_TLBI_NH_ASID	0x11
 544		#define CMDQ_OP_TLBI_NH_VA	0x12
 545		#define CMDQ_OP_TLBI_NH_VAA	0x13
 546		#define CMDQ_OP_TLBI_EL2_ALL	0x20
 547		#define CMDQ_OP_TLBI_EL2_ASID	0x21
 548		#define CMDQ_OP_TLBI_EL2_VA	0x22
 549		#define CMDQ_OP_TLBI_S12_VMALL	0x28
 550		#define CMDQ_OP_TLBI_S2_IPA	0x2a
 551		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
 552		struct {
 553			u8			num;
 554			u8			scale;
 555			u16			asid;
 556			u16			vmid;
 557			bool			leaf;
 558			u8			ttl;
 559			u8			tg;
 560			u64			addr;
 561		} tlbi;
 562
 563		#define CMDQ_OP_ATC_INV		0x40
 564		#define ATC_INV_SIZE_ALL	52
 565		struct {
 566			u32			sid;
 567			u32			ssid;
 568			u64			addr;
 569			u8			size;
 570			bool			global;
 571		} atc;
 572
 573		#define CMDQ_OP_PRI_RESP	0x41
 574		struct {
 575			u32			sid;
 576			u32			ssid;
 577			u16			grpid;
 578			enum pri_resp		resp;
 579		} pri;
 580
 581		#define CMDQ_OP_RESUME		0x44
 582		struct {
 583			u32			sid;
 584			u16			stag;
 585			u8			resp;
 586		} resume;
 587
 588		#define CMDQ_OP_CMD_SYNC	0x46
 589		struct {
 590			u64			msiaddr;
 591		} sync;
 592	};
 593};
 594
 595struct arm_smmu_ll_queue {
 596	union {
 597		u64			val;
 598		struct {
 599			u32		prod;
 600			u32		cons;
 601		};
 602		struct {
 603			atomic_t	prod;
 604			atomic_t	cons;
 605		} atomic;
 606		u8			__pad[SMP_CACHE_BYTES];
 607	} ____cacheline_aligned_in_smp;
 608	u32				max_n_shift;
 609};
 610
 611struct arm_smmu_queue {
 612	struct arm_smmu_ll_queue	llq;
 613	int				irq; /* Wired interrupt */
 614
 615	__le64				*base;
 616	dma_addr_t			base_dma;
 617	u64				q_base;
 618
 619	size_t				ent_dwords;
 620
 621	u32 __iomem			*prod_reg;
 622	u32 __iomem			*cons_reg;
 623};
 624
 625struct arm_smmu_queue_poll {
 626	ktime_t				timeout;
 627	unsigned int			delay;
 628	unsigned int			spin_cnt;
 629	bool				wfe;
 630};
 631
 632struct arm_smmu_cmdq {
 633	struct arm_smmu_queue		q;
 634	atomic_long_t			*valid_map;
 635	atomic_t			owner_prod;
 636	atomic_t			lock;
 637	bool				(*supports_cmd)(struct arm_smmu_cmdq_ent *ent);
 638};
 639
 640static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq,
 641					      struct arm_smmu_cmdq_ent *ent)
 642{
 643	return cmdq->supports_cmd ? cmdq->supports_cmd(ent) : true;
 644}
 645
 646struct arm_smmu_cmdq_batch {
 647	u64				cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
 648	struct arm_smmu_cmdq		*cmdq;
 649	int				num;
 650};
 651
 652struct arm_smmu_evtq {
 653	struct arm_smmu_queue		q;
 654	struct iopf_queue		*iopf;
 655	u32				max_stalls;
 656};
 657
 658struct arm_smmu_priq {
 659	struct arm_smmu_queue		q;
 660};
 661
 662/* High-level stream table and context descriptor structures */
 663struct arm_smmu_ctx_desc {
 664	u16				asid;
 665};
 666
 667struct arm_smmu_ctx_desc_cfg {
 668	union {
 669		struct {
 670			struct arm_smmu_cd *table;
 671			unsigned int num_ents;
 672		} linear;
 673		struct {
 674			struct arm_smmu_cdtab_l1 *l1tab;
 675			struct arm_smmu_cdtab_l2 **l2ptrs;
 676			unsigned int num_l1_ents;
 677		} l2;
 678	};
 679	dma_addr_t			cdtab_dma;
 680	unsigned int			used_ssids;
 681	u8				in_ste;
 682	u8				s1fmt;
 683	/* log2 of the maximum number of CDs supported by this table */
 684	u8				s1cdmax;
 685};
 686
 687static inline bool
 688arm_smmu_cdtab_allocated(struct arm_smmu_ctx_desc_cfg *cfg)
 689{
 690	return cfg->linear.table || cfg->l2.l1tab;
 691}
 692
 693/* True if the cd table has SSIDS > 0 in use. */
 694static inline bool arm_smmu_ssids_in_use(struct arm_smmu_ctx_desc_cfg *cd_table)
 695{
 696	return cd_table->used_ssids;
 697}
 698
 699struct arm_smmu_s2_cfg {
 700	u16				vmid;
 701};
 702
 703struct arm_smmu_strtab_cfg {
 704	union {
 705		struct {
 706			struct arm_smmu_ste *table;
 707			dma_addr_t ste_dma;
 708			unsigned int num_ents;
 709		} linear;
 710		struct {
 711			struct arm_smmu_strtab_l1 *l1tab;
 712			struct arm_smmu_strtab_l2 **l2ptrs;
 713			dma_addr_t l1_dma;
 714			unsigned int num_l1_ents;
 715		} l2;
 716	};
 717};
 718
 719struct arm_smmu_impl_ops {
 720	int (*device_reset)(struct arm_smmu_device *smmu);
 721	void (*device_remove)(struct arm_smmu_device *smmu);
 722	int (*init_structures)(struct arm_smmu_device *smmu);
 723	struct arm_smmu_cmdq *(*get_secondary_cmdq)(
 724		struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent);
 725	/*
 726	 * An implementation should define its own type other than the default
 727	 * IOMMU_HW_INFO_TYPE_ARM_SMMUV3. And it must validate the input @type
 728	 * to return its own structure.
 729	 */
 730	void *(*hw_info)(struct arm_smmu_device *smmu, u32 *length,
 731			 enum iommu_hw_info_type *type);
 732	size_t (*get_viommu_size)(enum iommu_viommu_type viommu_type);
 733	int (*vsmmu_init)(struct arm_vsmmu *vsmmu,
 734			  const struct iommu_user_data *user_data);
 735};
 736
 737/* An SMMUv3 instance */
 738struct arm_smmu_device {
 739	struct device			*dev;
 740	struct device			*impl_dev;
 741	const struct arm_smmu_impl_ops	*impl_ops;
 742
 743	void __iomem			*base;
 744	void __iomem			*page1;
 745
 746#define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
 747#define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
 748#define ARM_SMMU_FEAT_TT_LE		(1 << 2)
 749#define ARM_SMMU_FEAT_TT_BE		(1 << 3)
 750#define ARM_SMMU_FEAT_PRI		(1 << 4)
 751#define ARM_SMMU_FEAT_ATS		(1 << 5)
 752#define ARM_SMMU_FEAT_SEV		(1 << 6)
 753#define ARM_SMMU_FEAT_MSI		(1 << 7)
 754#define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
 755#define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
 756#define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
 757#define ARM_SMMU_FEAT_STALLS		(1 << 11)
 758#define ARM_SMMU_FEAT_HYP		(1 << 12)
 759#define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
 760#define ARM_SMMU_FEAT_VAX		(1 << 14)
 761#define ARM_SMMU_FEAT_RANGE_INV		(1 << 15)
 762#define ARM_SMMU_FEAT_BTM		(1 << 16)
 763#define ARM_SMMU_FEAT_SVA		(1 << 17)
 764#define ARM_SMMU_FEAT_E2H		(1 << 18)
 765#define ARM_SMMU_FEAT_NESTING		(1 << 19)
 766#define ARM_SMMU_FEAT_ATTR_TYPES_OVR	(1 << 20)
 767#define ARM_SMMU_FEAT_HA		(1 << 21)
 768#define ARM_SMMU_FEAT_HD		(1 << 22)
 769#define ARM_SMMU_FEAT_S2FWB		(1 << 23)
 770#define ARM_SMMU_FEAT_BBML2		(1 << 24)
 771	u32				features;
 772
 773#define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
 774#define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
 775#define ARM_SMMU_OPT_MSIPOLL		(1 << 2)
 776#define ARM_SMMU_OPT_CMDQ_FORCE_SYNC	(1 << 3)
 777#define ARM_SMMU_OPT_TEGRA241_CMDQV	(1 << 4)
 778	u32				options;
 779
 780	struct arm_smmu_cmdq		cmdq;
 781	struct arm_smmu_evtq		evtq;
 782	struct arm_smmu_priq		priq;
 783
 784	int				gerr_irq;
 785	int				combined_irq;
 786
 787	unsigned long			ias; /* IPA */
 788	unsigned long			oas; /* PA */
 789	unsigned long			pgsize_bitmap;
 790
 791#define ARM_SMMU_MAX_ASIDS		(1 << 16)
 792	unsigned int			asid_bits;
 793
 794#define ARM_SMMU_MAX_VMIDS		(1 << 16)
 795	unsigned int			vmid_bits;
 796	struct ida			vmid_map;
 797
 798	unsigned int			ssid_bits;
 799	unsigned int			sid_bits;
 800
 801	struct arm_smmu_strtab_cfg	strtab_cfg;
 802
 803	/* IOMMU core code handle */
 804	struct iommu_device		iommu;
 805
 806	struct rb_root			streams;
 807	struct mutex			streams_mutex;
 808};
 809
 810struct arm_smmu_stream {
 811	u32				id;
 812	struct arm_smmu_master		*master;
 813	struct rb_node			node;
 814};
 815
 816struct arm_smmu_vmaster {
 817	struct arm_vsmmu		*vsmmu;
 818	unsigned long			vsid;
 819};
 820
 821struct arm_smmu_event {
 822	u8				stall : 1,
 823					ssv : 1,
 824					privileged : 1,
 825					instruction : 1,
 826					s2 : 1,
 827					read : 1,
 828					ttrnw : 1,
 829					class_tt : 1;
 830	u8				id;
 831	u8				class;
 832	u16				stag;
 833	u32				sid;
 834	u32				ssid;
 835	u64				iova;
 836	u64				ipa;
 837	u64				fetch_addr;
 838	struct device			*dev;
 839};
 840
 841/* SMMU private data for each master */
 842struct arm_smmu_master {
 843	struct arm_smmu_device		*smmu;
 844	struct device			*dev;
 845	struct arm_smmu_stream		*streams;
 846	struct arm_smmu_vmaster		*vmaster; /* use smmu->streams_mutex */
 847	/* Locked by the iommu core using the group mutex */
 848	struct arm_smmu_ctx_desc_cfg	cd_table;
 849	unsigned int			num_streams;
 850	bool				ats_enabled : 1;
 851	bool				ste_ats_enabled : 1;
 852	bool				stall_enabled;
 853	unsigned int			ssid_bits;
 854	unsigned int			iopf_refcount;
 855};
 856
 857/* SMMU private data for an IOMMU domain */
 858enum arm_smmu_domain_stage {
 859	ARM_SMMU_DOMAIN_S1 = 0,
 860	ARM_SMMU_DOMAIN_S2,
 861};
 862
 863struct arm_smmu_domain {
 864	struct arm_smmu_device		*smmu;
 865
 866	struct io_pgtable_ops		*pgtbl_ops;
 867	atomic_t			nr_ats_masters;
 868
 869	enum arm_smmu_domain_stage	stage;
 870	union {
 871		struct arm_smmu_ctx_desc	cd;
 872		struct arm_smmu_s2_cfg		s2_cfg;
 873	};
 874
 875	struct iommu_domain		domain;
 876
 877	/* List of struct arm_smmu_master_domain */
 878	struct list_head		devices;
 879	spinlock_t			devices_lock;
 880	bool				enforce_cache_coherency : 1;
 881	bool				nest_parent : 1;
 882
 883	struct mmu_notifier		mmu_notifier;
 884};
 885
 886struct arm_smmu_nested_domain {
 887	struct iommu_domain domain;
 888	struct arm_vsmmu *vsmmu;
 889	bool enable_ats : 1;
 890
 891	__le64 ste[2];
 892};
 893
 894/* The following are exposed for testing purposes. */
 895struct arm_smmu_entry_writer_ops;
 896struct arm_smmu_entry_writer {
 897	const struct arm_smmu_entry_writer_ops *ops;
 898	struct arm_smmu_master *master;
 899};
 900
 901struct arm_smmu_entry_writer_ops {
 902	void (*get_used)(const __le64 *entry, __le64 *used);
 903	void (*get_update_safe)(const __le64 *cur, const __le64 *target,
 904				__le64 *safe_bits);
 905	void (*sync)(struct arm_smmu_entry_writer *writer);
 906};
 907
 908void arm_smmu_make_abort_ste(struct arm_smmu_ste *target);
 909void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
 910				 struct arm_smmu_master *master,
 911				 struct arm_smmu_domain *smmu_domain,
 912				 bool ats_enabled);
 913
 914#if IS_ENABLED(CONFIG_KUNIT)
 915void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits);
 916void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
 917				  __le64 *safe_bits);
 918void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur,
 919			  const __le64 *target);
 920void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits);
 921void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
 922			      struct arm_smmu_ste *target);
 923void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
 924			       struct arm_smmu_master *master, bool ats_enabled,
 925			       unsigned int s1dss);
 926void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
 927			  struct arm_smmu_master *master, struct mm_struct *mm,
 928			  u16 asid);
 929#endif
 930
 931struct arm_smmu_master_domain {
 932	struct list_head devices_elm;
 933	struct arm_smmu_master *master;
 934	/*
 935	 * For nested domains the master_domain is threaded onto the S2 parent,
 936	 * this points to the IOMMU_DOMAIN_NESTED to disambiguate the masters.
 937	 */
 938	struct iommu_domain *domain;
 939	ioasid_t ssid;
 940	bool nested_ats_flush : 1;
 941	bool using_iopf : 1;
 942};
 943
 944static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 945{
 946	return container_of(dom, struct arm_smmu_domain, domain);
 947}
 948
 949static inline struct arm_smmu_nested_domain *
 950to_smmu_nested_domain(struct iommu_domain *dom)
 951{
 952	return container_of(dom, struct arm_smmu_nested_domain, domain);
 953}
 954
 955extern struct xarray arm_smmu_asid_xa;
 956extern struct mutex arm_smmu_asid_lock;
 957
 958struct arm_smmu_domain *arm_smmu_domain_alloc(void);
 959
 960void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid);
 961struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
 962					u32 ssid);
 963void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
 964			 struct arm_smmu_master *master,
 965			 struct arm_smmu_domain *smmu_domain);
 966void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
 967			     struct arm_smmu_cd *cdptr,
 968			     const struct arm_smmu_cd *target);
 969
 970int arm_smmu_set_pasid(struct arm_smmu_master *master,
 971		       struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
 972		       struct arm_smmu_cd *cd, struct iommu_domain *old);
 973
 974void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
 975void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
 976				 size_t granule, bool leaf,
 977				 struct arm_smmu_domain *smmu_domain);
 978int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
 979			    unsigned long iova, size_t size);
 980
 981void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
 982			      struct arm_smmu_cmdq *cmdq);
 983int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
 984			    struct arm_smmu_queue *q, void __iomem *page,
 985			    unsigned long prod_off, unsigned long cons_off,
 986			    size_t dwords, const char *name);
 987int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
 988		       struct arm_smmu_cmdq *cmdq);
 989
 990static inline bool arm_smmu_master_canwbs(struct arm_smmu_master *master)
 991{
 992	return dev_iommu_fwspec_get(master->dev)->flags &
 993	       IOMMU_FWSPEC_PCI_RC_CANWBS;
 994}
 995
 996struct arm_smmu_attach_state {
 997	/* Inputs */
 998	struct iommu_domain *old_domain;
 999	struct arm_smmu_master *master;
1000	bool cd_needs_ats;
1001	bool disable_ats;
1002	ioasid_t ssid;
1003	/* Resulting state */
1004	struct arm_smmu_vmaster *vmaster;
1005	bool ats_enabled;
1006};
1007
1008int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
1009			    struct iommu_domain *new_domain);
1010void arm_smmu_attach_commit(struct arm_smmu_attach_state *state);
1011void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
1012				  const struct arm_smmu_ste *target);
1013
1014int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
1015				struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
1016				bool sync);
1017
1018#ifdef CONFIG_ARM_SMMU_V3_SVA
1019bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
1020void arm_smmu_sva_notifier_synchronize(void);
1021struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
1022					       struct mm_struct *mm);
1023#else /* CONFIG_ARM_SMMU_V3_SVA */
1024static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
1025{
1026	return false;
1027}
1028
1029static inline void arm_smmu_sva_notifier_synchronize(void) {}
1030
1031#define arm_smmu_sva_domain_alloc NULL
1032
1033#endif /* CONFIG_ARM_SMMU_V3_SVA */
1034
1035#ifdef CONFIG_TEGRA241_CMDQV
1036struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu);
1037#else /* CONFIG_TEGRA241_CMDQV */
1038static inline struct arm_smmu_device *
1039tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
1040{
1041	return ERR_PTR(-ENODEV);
1042}
1043#endif /* CONFIG_TEGRA241_CMDQV */
1044
1045struct arm_vsmmu {
1046	struct iommufd_viommu core;
1047	struct arm_smmu_device *smmu;
1048	struct arm_smmu_domain *s2_parent;
1049	u16 vmid;
1050};
1051
1052#if IS_ENABLED(CONFIG_ARM_SMMU_V3_IOMMUFD)
1053void *arm_smmu_hw_info(struct device *dev, u32 *length,
1054		       enum iommu_hw_info_type *type);
1055size_t arm_smmu_get_viommu_size(struct device *dev,
1056				enum iommu_viommu_type viommu_type);
1057int arm_vsmmu_init(struct iommufd_viommu *viommu,
1058		   struct iommu_domain *parent_domain,
1059		   const struct iommu_user_data *user_data);
1060int arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,
1061				    struct arm_smmu_nested_domain *nested_domain);
1062void arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state);
1063void arm_smmu_master_clear_vmaster(struct arm_smmu_master *master);
1064int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster, u64 *evt);
1065struct iommu_domain *
1066arm_vsmmu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
1067			      const struct iommu_user_data *user_data);
1068int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
1069			       struct iommu_user_data_array *array);
1070#else
1071#define arm_smmu_get_viommu_size NULL
1072#define arm_smmu_hw_info NULL
1073#define arm_vsmmu_init NULL
1074#define arm_vsmmu_alloc_domain_nested NULL
1075#define arm_vsmmu_cache_invalidate NULL
1076
1077static inline int
1078arm_smmu_attach_prepare_vmaster(struct arm_smmu_attach_state *state,
1079				struct arm_smmu_nested_domain *nested_domain)
1080{
1081	return 0;
1082}
1083
1084static inline void
1085arm_smmu_attach_commit_vmaster(struct arm_smmu_attach_state *state)
1086{
1087}
1088
1089static inline void
1090arm_smmu_master_clear_vmaster(struct arm_smmu_master *master)
1091{
1092}
1093
1094static inline int arm_vmaster_report_event(struct arm_smmu_vmaster *vmaster,
1095					   u64 *evt)
1096{
1097	return -EOPNOTSUPP;
1098}
1099#endif /* CONFIG_ARM_SMMU_V3_IOMMUFD */
1100
1101#endif /* _ARM_SMMU_V3_H */