io-pgtable-arm.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * CPU-agnostic ARM page table allocator.
   4 *
   5 * Copyright (C) 2014 ARM Limited
   6 *
   7 * Author: Will Deacon <will.deacon@arm.com>
   8 */
   9
  10#define pr_fmt(fmt)	"arm-lpae io-pgtable: " fmt
  11
  12#include <linux/atomic.h>
  13#include <linux/bitops.h>
  14#include <linux/io-pgtable.h>
  15#include <linux/sizes.h>
  16#include <linux/slab.h>
  17#include <linux/types.h>
  18#include <linux/dma-mapping.h>
  19
  20#include <asm/barrier.h>
  21
  22#include "io-pgtable-arm.h"
  23#include "iommu-pages.h"
  24
  25#define ARM_LPAE_MAX_ADDR_BITS		52
  26#define ARM_LPAE_S2_MAX_CONCAT_PAGES	16
  27#define ARM_LPAE_MAX_LEVELS		4
  28
  29/* Struct accessors */
  30#define io_pgtable_to_data(x)						\
  31	container_of((x), struct arm_lpae_io_pgtable, iop)
  32
  33#define io_pgtable_ops_to_data(x)					\
  34	io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
  35
  36/*
  37 * Calculate the right shift amount to get to the portion describing level l
  38 * in a virtual address mapped by the pagetable in d.
  39 */
  40#define ARM_LPAE_LVL_SHIFT(l,d)						\
  41	(((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) +		\
  42	ilog2(sizeof(arm_lpae_iopte)))
  43
  44#define ARM_LPAE_GRANULE(d)						\
  45	(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
  46#define ARM_LPAE_PGD_SIZE(d)						\
  47	(sizeof(arm_lpae_iopte) << (d)->pgd_bits)
  48
  49#define ARM_LPAE_PTES_PER_TABLE(d)					\
  50	(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
  51
  52/*
  53 * Calculate the index at level l used to map virtual address a using the
  54 * pagetable in d.
  55 */
  56#define ARM_LPAE_PGD_IDX(l,d)						\
  57	((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
  58
  59#define ARM_LPAE_LVL_IDX(a,l,d)						\
  60	(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) &			\
  61	 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
  62
  63/* Calculate the block/page mapping size at level l for pagetable in d. */
  64#define ARM_LPAE_BLOCK_SIZE(l,d)	(1ULL << ARM_LPAE_LVL_SHIFT(l,d))
  65
  66/* Page table bits */
  67#define ARM_LPAE_PTE_TYPE_SHIFT		0
  68#define ARM_LPAE_PTE_TYPE_MASK		0x3
  69
  70#define ARM_LPAE_PTE_TYPE_BLOCK		1
  71#define ARM_LPAE_PTE_TYPE_TABLE		3
  72#define ARM_LPAE_PTE_TYPE_PAGE		3
  73
  74#define ARM_LPAE_PTE_ADDR_MASK		GENMASK_ULL(47,12)
  75
  76#define ARM_LPAE_PTE_NSTABLE		(((arm_lpae_iopte)1) << 63)
  77#define ARM_LPAE_PTE_XN			(((arm_lpae_iopte)3) << 53)
  78#define ARM_LPAE_PTE_DBM		(((arm_lpae_iopte)1) << 51)
  79#define ARM_LPAE_PTE_AF			(((arm_lpae_iopte)1) << 10)
  80#define ARM_LPAE_PTE_SH_NS		(((arm_lpae_iopte)0) << 8)
  81#define ARM_LPAE_PTE_SH_OS		(((arm_lpae_iopte)2) << 8)
  82#define ARM_LPAE_PTE_SH_IS		(((arm_lpae_iopte)3) << 8)
  83#define ARM_LPAE_PTE_NS			(((arm_lpae_iopte)1) << 5)
  84#define ARM_LPAE_PTE_VALID		(((arm_lpae_iopte)1) << 0)
  85
  86/* Software bit for solving coherency races */
  87#define ARM_LPAE_PTE_SW_SYNC		(((arm_lpae_iopte)1) << 55)
  88
  89/* Stage-1 PTE */
  90#define ARM_LPAE_PTE_AP_UNPRIV		(((arm_lpae_iopte)1) << 6)
  91#define ARM_LPAE_PTE_AP_RDONLY_BIT	7
  92#define ARM_LPAE_PTE_AP_RDONLY		(((arm_lpae_iopte)1) << \
  93					   ARM_LPAE_PTE_AP_RDONLY_BIT)
  94#define ARM_LPAE_PTE_AP_WR_CLEAN_MASK	(ARM_LPAE_PTE_AP_RDONLY | \
  95					 ARM_LPAE_PTE_DBM)
  96#define ARM_LPAE_PTE_ATTRINDX_SHIFT	2
  97#define ARM_LPAE_PTE_nG			(((arm_lpae_iopte)1) << 11)
  98
  99/* Stage-2 PTE */
 100#define ARM_LPAE_PTE_HAP_FAULT		(((arm_lpae_iopte)0) << 6)
 101#define ARM_LPAE_PTE_HAP_READ		(((arm_lpae_iopte)1) << 6)
 102#define ARM_LPAE_PTE_HAP_WRITE		(((arm_lpae_iopte)2) << 6)
 103/*
 104 * For !FWB these code to:
 105 *  1111 = Normal outer write back cachable / Inner Write Back Cachable
 106 *         Permit S1 to override
 107 *  0101 = Normal Non-cachable / Inner Non-cachable
 108 *  0001 = Device / Device-nGnRE
 109 * For S2FWB these code:
 110 *  0110 Force Normal Write Back
 111 *  0101 Normal* is forced Normal-NC, Device unchanged
 112 *  0001 Force Device-nGnRE
 113 */
 114#define ARM_LPAE_PTE_MEMATTR_FWB_WB	(((arm_lpae_iopte)0x6) << 2)
 115#define ARM_LPAE_PTE_MEMATTR_OIWB	(((arm_lpae_iopte)0xf) << 2)
 116#define ARM_LPAE_PTE_MEMATTR_NC		(((arm_lpae_iopte)0x5) << 2)
 117#define ARM_LPAE_PTE_MEMATTR_DEV	(((arm_lpae_iopte)0x1) << 2)
 118
 119/* Register bits */
 120#define ARM_LPAE_VTCR_SL0_MASK		0x3
 121
 122#define ARM_LPAE_TCR_T0SZ_SHIFT		0
 123
 124#define ARM_LPAE_VTCR_PS_SHIFT		16
 125#define ARM_LPAE_VTCR_PS_MASK		0x7
 126
 127#define ARM_LPAE_MAIR_ATTR_SHIFT(n)	((n) << 3)
 128#define ARM_LPAE_MAIR_ATTR_MASK		0xff
 129#define ARM_LPAE_MAIR_ATTR_DEVICE	0x04
 130#define ARM_LPAE_MAIR_ATTR_NC		0x44
 131#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA	0xf4
 132#define ARM_LPAE_MAIR_ATTR_WBRWA	0xff
 133#define ARM_LPAE_MAIR_ATTR_IDX_NC	0
 134#define ARM_LPAE_MAIR_ATTR_IDX_CACHE	1
 135#define ARM_LPAE_MAIR_ATTR_IDX_DEV	2
 136#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE	3
 137
 138#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
 139#define ARM_MALI_LPAE_TTBR_READ_INNER	BIT(2)
 140#define ARM_MALI_LPAE_TTBR_SHARE_OUTER	BIT(4)
 141
 142#define ARM_MALI_LPAE_MEMATTR_IMP_DEF	0x88ULL
 143#define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
 144
 145/* IOPTE accessors */
 146#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
 147
 148#define iopte_type(pte)					\
 149	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
 150
 151#define iopte_writeable_dirty(pte)				\
 152	(((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM)
 153
 154#define iopte_set_writeable_clean(ptep)				\
 155	set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep))
 156
 157struct arm_lpae_io_pgtable {
 158	struct io_pgtable	iop;
 159
 160	int			pgd_bits;
 161	int			start_level;
 162	int			bits_per_level;
 163
 164	void			*pgd;
 165};
 166
 167typedef u64 arm_lpae_iopte;
 168
 169static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
 170			      enum io_pgtable_fmt fmt)
 171{
 172	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
 173		return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;
 174
 175	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
 176}
 177
 178static inline bool iopte_table(arm_lpae_iopte pte, int lvl)
 179{
 180	if (lvl == (ARM_LPAE_MAX_LEVELS - 1))
 181		return false;
 182	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE;
 183}
 184
 185static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
 186				     struct arm_lpae_io_pgtable *data)
 187{
 188	arm_lpae_iopte pte = paddr;
 189
 190	/* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */
 191	return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK;
 192}
 193
 194static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
 195				  struct arm_lpae_io_pgtable *data)
 196{
 197	u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK;
 198
 199	if (ARM_LPAE_GRANULE(data) < SZ_64K)
 200		return paddr;
 201
 202	/* Rotate the packed high-order bits back to the top */
 203	return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
 204}
 205
 206/*
 207 * Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into
 208 * a concatenated PGD, into the maximum number of entries that can be
 209 * mapped in the same table page.
 210 */
 211static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data)
 212{
 213	int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
 214
 215	return ptes_per_table - (i & (ptes_per_table - 1));
 216}
 217
 218/*
 219 * Check if concatenated PGDs are mandatory according to Arm DDI0487 (K.a)
 220 * 1) R_DXBSH: For 16KB, and 48-bit input size, use level 1 instead of 0.
 221 * 2) R_SRKBC: After de-ciphering the table for PA size and valid initial lookup
 222 *   a) 40 bits PA size with 4K: use level 1 instead of level 0 (2 tables for ias = oas)
 223 *   b) 40 bits PA size with 16K: use level 2 instead of level 1 (16 tables for ias = oas)
 224 *   c) 42 bits PA size with 4K: use level 1 instead of level 0 (8 tables for ias = oas)
 225 *   d) 48 bits PA size with 16K: use level 1 instead of level 0 (2 tables for ias = oas)
 226 */
 227static inline bool arm_lpae_concat_mandatory(struct io_pgtable_cfg *cfg,
 228					     struct arm_lpae_io_pgtable *data)
 229{
 230	unsigned int ias = cfg->ias;
 231	unsigned int oas = cfg->oas;
 232
 233	/* Covers 1 and 2.d */
 234	if ((ARM_LPAE_GRANULE(data) == SZ_16K) && (data->start_level == 0))
 235		return (oas == 48) || (ias == 48);
 236
 237	/* Covers 2.a and 2.c */
 238	if ((ARM_LPAE_GRANULE(data) == SZ_4K) && (data->start_level == 0))
 239		return (oas == 40) || (oas == 42);
 240
 241	/* Case 2.b */
 242	return (ARM_LPAE_GRANULE(data) == SZ_16K) &&
 243	       (data->start_level == 1) && (oas == 40);
 244}
 245
 246static dma_addr_t __arm_lpae_dma_addr(void *pages)
 247{
 248	return (dma_addr_t)virt_to_phys(pages);
 249}
 250
 251static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
 252				    struct io_pgtable_cfg *cfg,
 253				    void *cookie)
 254{
 255	struct device *dev = cfg->iommu_dev;
 256	size_t alloc_size;
 257	dma_addr_t dma;
 258	void *pages;
 259
 260	/*
 261	 * For very small starting-level translation tables the HW requires a
 262	 * minimum alignment of at least 64 to cover all cases.
 263	 */
 264	alloc_size = max(size, 64);
 265	if (cfg->alloc)
 266		pages = cfg->alloc(cookie, alloc_size, gfp);
 267	else
 268		pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp,
 269						  alloc_size);
 270
 271	if (!pages)
 272		return NULL;
 273
 274	if (!cfg->coherent_walk) {
 275		dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
 276		if (dma_mapping_error(dev, dma))
 277			goto out_free;
 278		/*
 279		 * We depend on the IOMMU being able to work with any physical
 280		 * address directly, so if the DMA layer suggests otherwise by
 281		 * translating or truncating them, that bodes very badly...
 282		 */
 283		if (dma != virt_to_phys(pages))
 284			goto out_unmap;
 285	}
 286
 287	return pages;
 288
 289out_unmap:
 290	dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
 291	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
 292
 293out_free:
 294	if (cfg->free)
 295		cfg->free(cookie, pages, size);
 296	else
 297		iommu_free_pages(pages);
 298
 299	return NULL;
 300}
 301
 302static void __arm_lpae_free_pages(void *pages, size_t size,
 303				  struct io_pgtable_cfg *cfg,
 304				  void *cookie)
 305{
 306	if (!cfg->coherent_walk)
 307		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
 308				 size, DMA_TO_DEVICE);
 309
 310	if (cfg->free)
 311		cfg->free(cookie, pages, size);
 312	else
 313		iommu_free_pages(pages);
 314}
 315
 316static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
 317				struct io_pgtable_cfg *cfg)
 318{
 319	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),
 320				   sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
 321}
 322
 323static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries)
 324{
 325	for (int i = 0; i < num_entries; i++)
 326		ptep[i] = 0;
 327
 328	if (!cfg->coherent_walk && num_entries)
 329		__arm_lpae_sync_pte(ptep, num_entries, cfg);
 330}
 331
 332static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 333			       struct iommu_iotlb_gather *gather,
 334			       unsigned long iova, size_t size, size_t pgcount,
 335			       int lvl, arm_lpae_iopte *ptep);
 336
 337static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
 338				phys_addr_t paddr, arm_lpae_iopte prot,
 339				int lvl, int num_entries, arm_lpae_iopte *ptep)
 340{
 341	arm_lpae_iopte pte = prot;
 342	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 343	size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
 344	int i;
 345
 346	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
 347		pte |= ARM_LPAE_PTE_TYPE_PAGE;
 348	else
 349		pte |= ARM_LPAE_PTE_TYPE_BLOCK;
 350
 351	for (i = 0; i < num_entries; i++)
 352		ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data);
 353
 354	if (!cfg->coherent_walk)
 355		__arm_lpae_sync_pte(ptep, num_entries, cfg);
 356}
 357
 358static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
 359			     unsigned long iova, phys_addr_t paddr,
 360			     arm_lpae_iopte prot, int lvl, int num_entries,
 361			     arm_lpae_iopte *ptep)
 362{
 363	int i;
 364
 365	for (i = 0; i < num_entries; i++)
 366		if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {
 367			/* We require an unmap first */
 368			WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));
 369			return -EEXIST;
 370		} else if (iopte_type(ptep[i]) == ARM_LPAE_PTE_TYPE_TABLE) {
 371			/*
 372			 * We need to unmap and free the old table before
 373			 * overwriting it with a block entry.
 374			 */
 375			arm_lpae_iopte *tblp;
 376			size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
 377
 378			tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
 379			if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1,
 380					     lvl, tblp) != sz) {
 381				WARN_ON(1);
 382				return -EINVAL;
 383			}
 384		}
 385
 386	__arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);
 387	return 0;
 388}
 389
 390static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
 391					     arm_lpae_iopte *ptep,
 392					     arm_lpae_iopte curr,
 393					     struct arm_lpae_io_pgtable *data)
 394{
 395	arm_lpae_iopte old, new;
 396	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 397
 398	new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
 399	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
 400		new |= ARM_LPAE_PTE_NSTABLE;
 401
 402	/*
 403	 * Ensure the table itself is visible before its PTE can be.
 404	 * Whilst we could get away with cmpxchg64_release below, this
 405	 * doesn't have any ordering semantics when !CONFIG_SMP.
 406	 */
 407	dma_wmb();
 408
 409	old = cmpxchg64_relaxed(ptep, curr, new);
 410
 411	if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))
 412		return old;
 413
 414	/* Even if it's not ours, there's no point waiting; just kick it */
 415	__arm_lpae_sync_pte(ptep, 1, cfg);
 416	if (old == curr)
 417		WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);
 418
 419	return old;
 420}
 421
 422static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
 423			  phys_addr_t paddr, size_t size, size_t pgcount,
 424			  arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,
 425			  gfp_t gfp, size_t *mapped)
 426{
 427	arm_lpae_iopte *cptep, pte;
 428	size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
 429	size_t tblsz = ARM_LPAE_GRANULE(data);
 430	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 431	int ret = 0, num_entries, max_entries, map_idx_start;
 432
 433	/* Find our entry at the current level */
 434	map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
 435	ptep += map_idx_start;
 436
 437	/* If we can install a leaf entry at this level, then do so */
 438	if (size == block_size) {
 439		max_entries = arm_lpae_max_entries(map_idx_start, data);
 440		num_entries = min_t(int, pgcount, max_entries);
 441		ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
 442		if (!ret)
 443			*mapped += num_entries * size;
 444
 445		return ret;
 446	}
 447
 448	/* We can't allocate tables at the final level */
 449	if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
 450		return -EINVAL;
 451
 452	/* Grab a pointer to the next level */
 453	pte = READ_ONCE(*ptep);
 454	if (!pte) {
 455		cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie);
 456		if (!cptep)
 457			return -ENOMEM;
 458
 459		pte = arm_lpae_install_table(cptep, ptep, 0, data);
 460		if (pte)
 461			__arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie);
 462	} else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {
 463		__arm_lpae_sync_pte(ptep, 1, cfg);
 464	}
 465
 466	if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
 467		cptep = iopte_deref(pte, data);
 468	} else if (pte) {
 469		/* We require an unmap first */
 470		WARN_ON(!(cfg->quirks & IO_PGTABLE_QUIRK_NO_WARN));
 471		return -EEXIST;
 472	}
 473
 474	/* Rinse, repeat */
 475	return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,
 476			      cptep, gfp, mapped);
 477}
 478
 479static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
 480					   int prot)
 481{
 482	arm_lpae_iopte pte;
 483
 484	if (data->iop.fmt == ARM_64_LPAE_S1 ||
 485	    data->iop.fmt == ARM_32_LPAE_S1) {
 486		pte = ARM_LPAE_PTE_nG;
 487		if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
 488			pte |= ARM_LPAE_PTE_AP_RDONLY;
 489		else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD)
 490			pte |= ARM_LPAE_PTE_DBM;
 491		if (!(prot & IOMMU_PRIV))
 492			pte |= ARM_LPAE_PTE_AP_UNPRIV;
 493	} else {
 494		pte = ARM_LPAE_PTE_HAP_FAULT;
 495		if (prot & IOMMU_READ)
 496			pte |= ARM_LPAE_PTE_HAP_READ;
 497		if (prot & IOMMU_WRITE)
 498			pte |= ARM_LPAE_PTE_HAP_WRITE;
 499	}
 500
 501	/*
 502	 * Note that this logic is structured to accommodate Mali LPAE
 503	 * having stage-1-like attributes but stage-2-like permissions.
 504	 */
 505	if (data->iop.fmt == ARM_64_LPAE_S2 ||
 506	    data->iop.fmt == ARM_32_LPAE_S2) {
 507		if (prot & IOMMU_MMIO) {
 508			pte |= ARM_LPAE_PTE_MEMATTR_DEV;
 509		} else if (prot & IOMMU_CACHE) {
 510			if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
 511				pte |= ARM_LPAE_PTE_MEMATTR_FWB_WB;
 512			else
 513				pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
 514		} else {
 515			pte |= ARM_LPAE_PTE_MEMATTR_NC;
 516		}
 517	} else {
 518		if (prot & IOMMU_MMIO)
 519			pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
 520				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
 521		else if (prot & IOMMU_CACHE)
 522			pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
 523				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
 524	}
 525
 526	/*
 527	 * Also Mali has its own notions of shareability wherein its Inner
 528	 * domain covers the cores within the GPU, and its Outer domain is
 529	 * "outside the GPU" (i.e. either the Inner or System domain in CPU
 530	 * terms, depending on coherency).
 531	 */
 532	if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)
 533		pte |= ARM_LPAE_PTE_SH_IS;
 534	else
 535		pte |= ARM_LPAE_PTE_SH_OS;
 536
 537	if (prot & IOMMU_NOEXEC)
 538		pte |= ARM_LPAE_PTE_XN;
 539
 540	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
 541		pte |= ARM_LPAE_PTE_NS;
 542
 543	if (data->iop.fmt != ARM_MALI_LPAE)
 544		pte |= ARM_LPAE_PTE_AF;
 545
 546	return pte;
 547}
 548
 549static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
 550			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
 551			      int iommu_prot, gfp_t gfp, size_t *mapped)
 552{
 553	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
 554	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 555	arm_lpae_iopte *ptep = data->pgd;
 556	int ret, lvl = data->start_level;
 557	arm_lpae_iopte prot;
 558	long iaext = (s64)iova >> cfg->ias;
 559
 560	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))
 561		return -EINVAL;
 562
 563	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
 564		iaext = ~iaext;
 565	if (WARN_ON(iaext || paddr >> cfg->oas))
 566		return -ERANGE;
 567
 568	if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
 569		return -EINVAL;
 570
 571	prot = arm_lpae_prot_to_pte(data, iommu_prot);
 572	ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,
 573			     ptep, gfp, mapped);
 574	/*
 575	 * Synchronise all PTE updates for the new mapping before there's
 576	 * a chance for anything to kick off a table walk for the new iova.
 577	 */
 578	wmb();
 579
 580	return ret;
 581}
 582
 583static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
 584				    arm_lpae_iopte *ptep)
 585{
 586	arm_lpae_iopte *start, *end;
 587	unsigned long table_size;
 588
 589	if (lvl == data->start_level)
 590		table_size = ARM_LPAE_PGD_SIZE(data);
 591	else
 592		table_size = ARM_LPAE_GRANULE(data);
 593
 594	start = ptep;
 595
 596	/* Only leaf entries at the last level */
 597	if (lvl == ARM_LPAE_MAX_LEVELS - 1)
 598		end = ptep;
 599	else
 600		end = (void *)ptep + table_size;
 601
 602	while (ptep != end) {
 603		arm_lpae_iopte pte = *ptep++;
 604
 605		if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
 606			continue;
 607
 608		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
 609	}
 610
 611	__arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie);
 612}
 613
 614static void arm_lpae_free_pgtable(struct io_pgtable *iop)
 615{
 616	struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
 617
 618	__arm_lpae_free_pgtable(data, data->start_level, data->pgd);
 619	kfree(data);
 620}
 621
 622static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 623			       struct iommu_iotlb_gather *gather,
 624			       unsigned long iova, size_t size, size_t pgcount,
 625			       int lvl, arm_lpae_iopte *ptep)
 626{
 627	arm_lpae_iopte pte;
 628	struct io_pgtable *iop = &data->iop;
 629	int i = 0, num_entries, max_entries, unmap_idx_start;
 630
 631	/* Something went horribly wrong and we ran out of page table */
 632	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
 633		return 0;
 634
 635	unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
 636	ptep += unmap_idx_start;
 637	pte = READ_ONCE(*ptep);
 638	if (!pte) {
 639		WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));
 640		return 0;
 641	}
 642
 643	/* If the size matches this level, we're in the right place */
 644	if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
 645		max_entries = arm_lpae_max_entries(unmap_idx_start, data);
 646		num_entries = min_t(int, pgcount, max_entries);
 647
 648		/* Find and handle non-leaf entries */
 649		for (i = 0; i < num_entries; i++) {
 650			pte = READ_ONCE(ptep[i]);
 651			if (!pte) {
 652				WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));
 653				break;
 654			}
 655
 656			if (!iopte_leaf(pte, lvl, iop->fmt)) {
 657				__arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);
 658
 659				/* Also flush any partial walks */
 660				io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
 661							  ARM_LPAE_GRANULE(data));
 662				__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
 663			}
 664		}
 665
 666		/* Clear the remaining entries */
 667		__arm_lpae_clear_pte(ptep, &iop->cfg, i);
 668
 669		if (gather && !iommu_iotlb_gather_queued(gather))
 670			for (int j = 0; j < i; j++)
 671				io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);
 672
 673		return i * size;
 674	} else if (iopte_leaf(pte, lvl, iop->fmt)) {
 675		WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
 676		return 0;
 677	}
 678
 679	/* Keep on walkin' */
 680	ptep = iopte_deref(pte, data);
 681	return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);
 682}
 683
 684static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,
 685				   size_t pgsize, size_t pgcount,
 686				   struct iommu_iotlb_gather *gather)
 687{
 688	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
 689	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 690	arm_lpae_iopte *ptep = data->pgd;
 691	long iaext = (s64)iova >> cfg->ias;
 692
 693	if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))
 694		return 0;
 695
 696	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
 697		iaext = ~iaext;
 698	if (WARN_ON(iaext))
 699		return 0;
 700
 701	return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount,
 702				data->start_level, ptep);
 703}
 704
 705struct io_pgtable_walk_data {
 706	struct io_pgtable		*iop;
 707	void				*data;
 708	int (*visit)(struct io_pgtable_walk_data *walk_data, int lvl,
 709		     arm_lpae_iopte *ptep, size_t size);
 710	unsigned long			flags;
 711	u64				addr;
 712	const u64			end;
 713};
 714
 715static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
 716				 struct io_pgtable_walk_data *walk_data,
 717				 arm_lpae_iopte *ptep,
 718				 int lvl);
 719
 720struct iova_to_phys_data {
 721	arm_lpae_iopte pte;
 722	int lvl;
 723};
 724
 725static int visit_iova_to_phys(struct io_pgtable_walk_data *walk_data, int lvl,
 726			      arm_lpae_iopte *ptep, size_t size)
 727{
 728	struct iova_to_phys_data *data = walk_data->data;
 729	data->pte = *ptep;
 730	data->lvl = lvl;
 731	return 0;
 732}
 733
 734static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
 735					 unsigned long iova)
 736{
 737	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
 738	struct iova_to_phys_data d;
 739	struct io_pgtable_walk_data walk_data = {
 740		.data = &d,
 741		.visit = visit_iova_to_phys,
 742		.addr = iova,
 743		.end = iova + 1,
 744	};
 745	int ret;
 746
 747	ret = __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level);
 748	if (ret)
 749		return 0;
 750
 751	iova &= (ARM_LPAE_BLOCK_SIZE(d.lvl, data) - 1);
 752	return iopte_to_paddr(d.pte, data) | iova;
 753}
 754
 755static int visit_pgtable_walk(struct io_pgtable_walk_data *walk_data, int lvl,
 756			      arm_lpae_iopte *ptep, size_t size)
 757{
 758	struct arm_lpae_io_pgtable_walk_data *data = walk_data->data;
 759	data->ptes[lvl] = *ptep;
 760	return 0;
 761}
 762
 763static int arm_lpae_pgtable_walk(struct io_pgtable_ops *ops, unsigned long iova,
 764				 void *wd)
 765{
 766	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
 767	struct io_pgtable_walk_data walk_data = {
 768		.data = wd,
 769		.visit = visit_pgtable_walk,
 770		.addr = iova,
 771		.end = iova + 1,
 772	};
 773
 774	return __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level);
 775}
 776
 777static int io_pgtable_visit(struct arm_lpae_io_pgtable *data,
 778			    struct io_pgtable_walk_data *walk_data,
 779			    arm_lpae_iopte *ptep, int lvl)
 780{
 781	struct io_pgtable *iop = &data->iop;
 782	arm_lpae_iopte pte = READ_ONCE(*ptep);
 783
 784	size_t size = ARM_LPAE_BLOCK_SIZE(lvl, data);
 785	int ret = walk_data->visit(walk_data, lvl, ptep, size);
 786	if (ret)
 787		return ret;
 788
 789	if (iopte_leaf(pte, lvl, iop->fmt)) {
 790		walk_data->addr += size;
 791		return 0;
 792	}
 793
 794	if (!iopte_table(pte, lvl)) {
 795		return -EINVAL;
 796	}
 797
 798	ptep = iopte_deref(pte, data);
 799	return __arm_lpae_iopte_walk(data, walk_data, ptep, lvl + 1);
 800}
 801
 802static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
 803				 struct io_pgtable_walk_data *walk_data,
 804				 arm_lpae_iopte *ptep,
 805				 int lvl)
 806{
 807	u32 idx;
 808	int max_entries, ret;
 809
 810	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
 811		return -EINVAL;
 812
 813	if (lvl == data->start_level)
 814		max_entries = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);
 815	else
 816		max_entries = ARM_LPAE_PTES_PER_TABLE(data);
 817
 818	for (idx = ARM_LPAE_LVL_IDX(walk_data->addr, lvl, data);
 819	     (idx < max_entries) && (walk_data->addr < walk_data->end); ++idx) {
 820		ret = io_pgtable_visit(data, walk_data, ptep + idx, lvl);
 821		if (ret)
 822			return ret;
 823	}
 824
 825	return 0;
 826}
 827
 828static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl,
 829		       arm_lpae_iopte *ptep, size_t size)
 830{
 831	struct iommu_dirty_bitmap *dirty = walk_data->data;
 832
 833	if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt))
 834		return 0;
 835
 836	if (iopte_writeable_dirty(*ptep)) {
 837		iommu_dirty_bitmap_record(dirty, walk_data->addr, size);
 838		if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR))
 839			iopte_set_writeable_clean(ptep);
 840	}
 841
 842	return 0;
 843}
 844
 845static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops,
 846					 unsigned long iova, size_t size,
 847					 unsigned long flags,
 848					 struct iommu_dirty_bitmap *dirty)
 849{
 850	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
 851	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 852	struct io_pgtable_walk_data walk_data = {
 853		.iop = &data->iop,
 854		.data = dirty,
 855		.visit = visit_dirty,
 856		.flags = flags,
 857		.addr = iova,
 858		.end = iova + size,
 859	};
 860	arm_lpae_iopte *ptep = data->pgd;
 861	int lvl = data->start_level;
 862
 863	if (WARN_ON(!size))
 864		return -EINVAL;
 865	if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1)))
 866		return -EINVAL;
 867	if (data->iop.fmt != ARM_64_LPAE_S1)
 868		return -EINVAL;
 869
 870	return __arm_lpae_iopte_walk(data, &walk_data, ptep, lvl);
 871}
 872
 873static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
 874{
 875	unsigned long granule, page_sizes;
 876	unsigned int max_addr_bits = 48;
 877
 878	/*
 879	 * We need to restrict the supported page sizes to match the
 880	 * translation regime for a particular granule. Aim to match
 881	 * the CPU page size if possible, otherwise prefer smaller sizes.
 882	 * While we're at it, restrict the block sizes to match the
 883	 * chosen granule.
 884	 */
 885	if (cfg->pgsize_bitmap & PAGE_SIZE)
 886		granule = PAGE_SIZE;
 887	else if (cfg->pgsize_bitmap & ~PAGE_MASK)
 888		granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);
 889	else if (cfg->pgsize_bitmap & PAGE_MASK)
 890		granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);
 891	else
 892		granule = 0;
 893
 894	switch (granule) {
 895	case SZ_4K:
 896		page_sizes = (SZ_4K | SZ_2M | SZ_1G);
 897		break;
 898	case SZ_16K:
 899		page_sizes = (SZ_16K | SZ_32M);
 900		break;
 901	case SZ_64K:
 902		max_addr_bits = 52;
 903		page_sizes = (SZ_64K | SZ_512M);
 904		if (cfg->oas > 48)
 905			page_sizes |= 1ULL << 42; /* 4TB */
 906		break;
 907	default:
 908		page_sizes = 0;
 909	}
 910
 911	cfg->pgsize_bitmap &= page_sizes;
 912	cfg->ias = min(cfg->ias, max_addr_bits);
 913	cfg->oas = min(cfg->oas, max_addr_bits);
 914}
 915
 916static struct arm_lpae_io_pgtable *
 917arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
 918{
 919	struct arm_lpae_io_pgtable *data;
 920	int levels, va_bits, pg_shift;
 921
 922	arm_lpae_restrict_pgsizes(cfg);
 923
 924	if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))
 925		return NULL;
 926
 927	if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)
 928		return NULL;
 929
 930	if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)
 931		return NULL;
 932
 933	data = kmalloc(sizeof(*data), GFP_KERNEL);
 934	if (!data)
 935		return NULL;
 936
 937	pg_shift = __ffs(cfg->pgsize_bitmap);
 938	data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
 939
 940	va_bits = cfg->ias - pg_shift;
 941	levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
 942	data->start_level = ARM_LPAE_MAX_LEVELS - levels;
 943
 944	/* Calculate the actual size of our pgd (without concatenation) */
 945	data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
 946
 947	data->iop.ops = (struct io_pgtable_ops) {
 948		.map_pages	= arm_lpae_map_pages,
 949		.unmap_pages	= arm_lpae_unmap_pages,
 950		.iova_to_phys	= arm_lpae_iova_to_phys,
 951		.read_and_clear_dirty = arm_lpae_read_and_clear_dirty,
 952		.pgtable_walk	= arm_lpae_pgtable_walk,
 953	};
 954
 955	return data;
 956}
 957
 958static struct io_pgtable *
 959arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 960{
 961	u64 reg;
 962	struct arm_lpae_io_pgtable *data;
 963	typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
 964	bool tg1;
 965
 966	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
 967			    IO_PGTABLE_QUIRK_ARM_TTBR1 |
 968			    IO_PGTABLE_QUIRK_ARM_OUTER_WBWA |
 969			    IO_PGTABLE_QUIRK_ARM_HD |
 970			    IO_PGTABLE_QUIRK_NO_WARN))
 971		return NULL;
 972
 973	data = arm_lpae_alloc_pgtable(cfg);
 974	if (!data)
 975		return NULL;
 976
 977	/* TCR */
 978	if (cfg->coherent_walk) {
 979		tcr->sh = ARM_LPAE_TCR_SH_IS;
 980		tcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
 981		tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
 982		if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)
 983			goto out_free_data;
 984	} else {
 985		tcr->sh = ARM_LPAE_TCR_SH_OS;
 986		tcr->irgn = ARM_LPAE_TCR_RGN_NC;
 987		if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))
 988			tcr->orgn = ARM_LPAE_TCR_RGN_NC;
 989		else
 990			tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
 991	}
 992
 993	tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;
 994	switch (ARM_LPAE_GRANULE(data)) {
 995	case SZ_4K:
 996		tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K;
 997		break;
 998	case SZ_16K:
 999		tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K;
1000		break;
1001	case SZ_64K:
1002		tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K;
1003		break;
1004	}
1005
1006	switch (cfg->oas) {
1007	case 32:
1008		tcr->ips = ARM_LPAE_TCR_PS_32_BIT;
1009		break;
1010	case 36:
1011		tcr->ips = ARM_LPAE_TCR_PS_36_BIT;
1012		break;
1013	case 40:
1014		tcr->ips = ARM_LPAE_TCR_PS_40_BIT;
1015		break;
1016	case 42:
1017		tcr->ips = ARM_LPAE_TCR_PS_42_BIT;
1018		break;
1019	case 44:
1020		tcr->ips = ARM_LPAE_TCR_PS_44_BIT;
1021		break;
1022	case 48:
1023		tcr->ips = ARM_LPAE_TCR_PS_48_BIT;
1024		break;
1025	case 52:
1026		tcr->ips = ARM_LPAE_TCR_PS_52_BIT;
1027		break;
1028	default:
1029		goto out_free_data;
1030	}
1031
1032	tcr->tsz = 64ULL - cfg->ias;
1033
1034	/* MAIRs */
1035	reg = (ARM_LPAE_MAIR_ATTR_NC
1036	       << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
1037	      (ARM_LPAE_MAIR_ATTR_WBRWA
1038	       << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
1039	      (ARM_LPAE_MAIR_ATTR_DEVICE
1040	       << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) |
1041	      (ARM_LPAE_MAIR_ATTR_INC_OWBRWA
1042	       << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE));
1043
1044	cfg->arm_lpae_s1_cfg.mair = reg;
1045
1046	/* Looking good; allocate a pgd */
1047	data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
1048					   GFP_KERNEL, cfg, cookie);
1049	if (!data->pgd)
1050		goto out_free_data;
1051
1052	/* Ensure the empty pgd is visible before any actual TTBR write */
1053	wmb();
1054
1055	/* TTBR */
1056	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
1057	return &data->iop;
1058
1059out_free_data:
1060	kfree(data);
1061	return NULL;
1062}
1063
1064static struct io_pgtable *
1065arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
1066{
1067	u64 sl;
1068	struct arm_lpae_io_pgtable *data;
1069	typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;
1070
1071	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB |
1072			    IO_PGTABLE_QUIRK_NO_WARN))
1073		return NULL;
1074
1075	data = arm_lpae_alloc_pgtable(cfg);
1076	if (!data)
1077		return NULL;
1078
1079	if (arm_lpae_concat_mandatory(cfg, data)) {
1080		if (WARN_ON((ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte)) >
1081			    ARM_LPAE_S2_MAX_CONCAT_PAGES))
1082			return NULL;
1083		data->pgd_bits += data->bits_per_level;
1084		data->start_level++;
1085	}
1086
1087	/* VTCR */
1088	if (cfg->coherent_walk) {
1089		vtcr->sh = ARM_LPAE_TCR_SH_IS;
1090		vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
1091		vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
1092	} else {
1093		vtcr->sh = ARM_LPAE_TCR_SH_OS;
1094		vtcr->irgn = ARM_LPAE_TCR_RGN_NC;
1095		vtcr->orgn = ARM_LPAE_TCR_RGN_NC;
1096	}
1097
1098	sl = data->start_level;
1099
1100	switch (ARM_LPAE_GRANULE(data)) {
1101	case SZ_4K:
1102		vtcr->tg = ARM_LPAE_TCR_TG0_4K;
1103		sl++; /* SL0 format is different for 4K granule size */
1104		break;
1105	case SZ_16K:
1106		vtcr->tg = ARM_LPAE_TCR_TG0_16K;
1107		break;
1108	case SZ_64K:
1109		vtcr->tg = ARM_LPAE_TCR_TG0_64K;
1110		break;
1111	}
1112
1113	switch (cfg->oas) {
1114	case 32:
1115		vtcr->ps = ARM_LPAE_TCR_PS_32_BIT;
1116		break;
1117	case 36:
1118		vtcr->ps = ARM_LPAE_TCR_PS_36_BIT;
1119		break;
1120	case 40:
1121		vtcr->ps = ARM_LPAE_TCR_PS_40_BIT;
1122		break;
1123	case 42:
1124		vtcr->ps = ARM_LPAE_TCR_PS_42_BIT;
1125		break;
1126	case 44:
1127		vtcr->ps = ARM_LPAE_TCR_PS_44_BIT;
1128		break;
1129	case 48:
1130		vtcr->ps = ARM_LPAE_TCR_PS_48_BIT;
1131		break;
1132	case 52:
1133		vtcr->ps = ARM_LPAE_TCR_PS_52_BIT;
1134		break;
1135	default:
1136		goto out_free_data;
1137	}
1138
1139	vtcr->tsz = 64ULL - cfg->ias;
1140	vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK;
1141
1142	/* Allocate pgd pages */
1143	data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
1144					   GFP_KERNEL, cfg, cookie);
1145	if (!data->pgd)
1146		goto out_free_data;
1147
1148	/* Ensure the empty pgd is visible before any actual TTBR write */
1149	wmb();
1150
1151	/* VTTBR */
1152	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
1153	return &data->iop;
1154
1155out_free_data:
1156	kfree(data);
1157	return NULL;
1158}
1159
1160static struct io_pgtable *
1161arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
1162{
1163	if (cfg->ias > 32 || cfg->oas > 40)
1164		return NULL;
1165
1166	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1167	return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
1168}
1169
1170static struct io_pgtable *
1171arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
1172{
1173	if (cfg->ias > 40 || cfg->oas > 40)
1174		return NULL;
1175
1176	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1177	return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
1178}
1179
1180static struct io_pgtable *
1181arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
1182{
1183	struct arm_lpae_io_pgtable *data;
1184
1185	/* No quirks for Mali (hopefully) */
1186	if (cfg->quirks)
1187		return NULL;
1188
1189	if (cfg->ias > 48 || cfg->oas > 40)
1190		return NULL;
1191
1192	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1193
1194	data = arm_lpae_alloc_pgtable(cfg);
1195	if (!data)
1196		return NULL;
1197
1198	/* Mali seems to need a full 4-level table regardless of IAS */
1199	if (data->start_level > 0) {
1200		data->start_level = 0;
1201		data->pgd_bits = 0;
1202	}
1203	/*
1204	 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the
1205	 * best we can do is mimic the out-of-tree driver and hope that the
1206	 * "implementation-defined caching policy" is good enough. Similarly,
1207	 * we'll use it for the sake of a valid attribute for our 'device'
1208	 * index, although callers should never request that in practice.
1209	 */
1210	cfg->arm_mali_lpae_cfg.memattr =
1211		(ARM_MALI_LPAE_MEMATTR_IMP_DEF
1212		 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
1213		(ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC
1214		 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
1215		(ARM_MALI_LPAE_MEMATTR_IMP_DEF
1216		 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));
1217
1218	data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
1219					   cfg, cookie);
1220	if (!data->pgd)
1221		goto out_free_data;
1222
1223	/* Ensure the empty pgd is visible before TRANSTAB can be written */
1224	wmb();
1225
1226	cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
1227					  ARM_MALI_LPAE_TTBR_READ_INNER |
1228					  ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
1229	if (cfg->coherent_walk)
1230		cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;
1231
1232	return &data->iop;
1233
1234out_free_data:
1235	kfree(data);
1236	return NULL;
1237}
1238
1239struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
1240	.caps	= IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1241	.alloc	= arm_64_lpae_alloc_pgtable_s1,
1242	.free	= arm_lpae_free_pgtable,
1243};
1244
1245struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
1246	.caps	= IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1247	.alloc	= arm_64_lpae_alloc_pgtable_s2,
1248	.free	= arm_lpae_free_pgtable,
1249};
1250
1251struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
1252	.caps	= IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1253	.alloc	= arm_32_lpae_alloc_pgtable_s1,
1254	.free	= arm_lpae_free_pgtable,
1255};
1256
1257struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
1258	.caps	= IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1259	.alloc	= arm_32_lpae_alloc_pgtable_s2,
1260	.free	= arm_lpae_free_pgtable,
1261};
1262
1263struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
1264	.caps	= IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1265	.alloc	= arm_mali_lpae_alloc_pgtable,
1266	.free	= arm_lpae_free_pgtable,
1267};