msm_iommu.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2013 Red Hat
  4 * Author: Rob Clark <robdclark@gmail.com>
  5 */
  6
  7#include <linux/adreno-smmu-priv.h>
  8#include <linux/io-pgtable.h>
  9#include <linux/kmemleak.h>
 10#include "msm_drv.h"
 11#include "msm_gpu_trace.h"
 12#include "msm_mmu.h"
 13
 14struct msm_iommu {
 15	struct msm_mmu base;
 16	struct iommu_domain *domain;
 17
 18	struct mutex init_lock;  /* protects pagetables counter and prr_page */
 19	int pagetables;
 20	struct page *prr_page;
 21
 22	struct kmem_cache *pt_cache;
 23};
 24
 25#define to_msm_iommu(x) container_of(x, struct msm_iommu, base)
 26
 27struct msm_iommu_pagetable {
 28	struct msm_mmu base;
 29	struct msm_mmu *parent;
 30	struct io_pgtable_ops *pgtbl_ops;
 31	const struct iommu_flush_ops *tlb;
 32	struct device *iommu_dev;
 33	unsigned long pgsize_bitmap;	/* Bitmap of page sizes in use */
 34	phys_addr_t ttbr;
 35	u32 asid;
 36
 37	/** @root_page_table: Stores the root page table pointer. */
 38	void *root_page_table;
 39};
 40static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu)
 41{
 42	return container_of(mmu, struct msm_iommu_pagetable, base);
 43}
 44
 45/* based on iommu_pgsize() in iommu.c: */
 46static size_t calc_pgsize(struct msm_iommu_pagetable *pagetable,
 47			   unsigned long iova, phys_addr_t paddr,
 48			   size_t size, size_t *count)
 49{
 50	unsigned int pgsize_idx, pgsize_idx_next;
 51	unsigned long pgsizes;
 52	size_t offset, pgsize, pgsize_next;
 53	unsigned long addr_merge = paddr | iova;
 54
 55	/* Page sizes supported by the hardware and small enough for @size */
 56	pgsizes = pagetable->pgsize_bitmap & GENMASK(__fls(size), 0);
 57
 58	/* Constrain the page sizes further based on the maximum alignment */
 59	if (likely(addr_merge))
 60		pgsizes &= GENMASK(__ffs(addr_merge), 0);
 61
 62	/* Make sure we have at least one suitable page size */
 63	BUG_ON(!pgsizes);
 64
 65	/* Pick the biggest page size remaining */
 66	pgsize_idx = __fls(pgsizes);
 67	pgsize = BIT(pgsize_idx);
 68	if (!count)
 69		return pgsize;
 70
 71	/* Find the next biggest support page size, if it exists */
 72	pgsizes = pagetable->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
 73	if (!pgsizes)
 74		goto out_set_count;
 75
 76	pgsize_idx_next = __ffs(pgsizes);
 77	pgsize_next = BIT(pgsize_idx_next);
 78
 79	/*
 80	 * There's no point trying a bigger page size unless the virtual
 81	 * and physical addresses are similarly offset within the larger page.
 82	 */
 83	if ((iova ^ paddr) & (pgsize_next - 1))
 84		goto out_set_count;
 85
 86	/* Calculate the offset to the next page size alignment boundary */
 87	offset = pgsize_next - (addr_merge & (pgsize_next - 1));
 88
 89	/*
 90	 * If size is big enough to accommodate the larger page, reduce
 91	 * the number of smaller pages.
 92	 */
 93	if (offset + pgsize_next <= size)
 94		size = offset;
 95
 96out_set_count:
 97	*count = size >> pgsize_idx;
 98	return pgsize;
 99}
100
101static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova,
102		size_t size)
103{
104	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
105	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
106	int ret = 0;
107
108	while (size) {
109		size_t pgsize, count;
110		ssize_t unmapped;
111
112		pgsize = calc_pgsize(pagetable, iova, iova, size, &count);
113
114		unmapped = ops->unmap_pages(ops, iova, pgsize, count, NULL);
115		if (unmapped <= 0) {
116			ret = -EINVAL;
117			/*
118			 * Continue attempting to unamp the remained of the
119			 * range, so we don't end up with some dangling
120			 * mapped pages
121			 */
122			unmapped = PAGE_SIZE;
123		}
124
125		iova += unmapped;
126		size -= unmapped;
127	}
128
129	iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain);
130
131	return ret;
132}
133
134static int msm_iommu_pagetable_map_prr(struct msm_mmu *mmu, u64 iova, size_t len, int prot)
135{
136	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
137	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
138	struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);
139	phys_addr_t phys = page_to_phys(iommu->prr_page);
140	u64 addr = iova;
141
142	while (len) {
143		size_t mapped = 0;
144		size_t size = PAGE_SIZE;
145		int ret;
146
147		ret = ops->map_pages(ops, addr, phys, size, 1, prot, GFP_KERNEL, &mapped);
148
149		/* map_pages could fail after mapping some of the pages,
150		 * so update the counters before error handling.
151		 */
152		addr += mapped;
153		len  -= mapped;
154
155		if (ret) {
156			msm_iommu_pagetable_unmap(mmu, iova, addr - iova);
157			return -EINVAL;
158		}
159	}
160
161	return 0;
162}
163
164static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
165				   struct sg_table *sgt, size_t off, size_t len,
166				   int prot)
167{
168	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
169	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
170	struct scatterlist *sg;
171	u64 addr = iova;
172	unsigned int i;
173
174	if (!sgt)
175		return msm_iommu_pagetable_map_prr(mmu, iova, len, prot);
176
177	for_each_sgtable_sg(sgt, sg, i) {
178		size_t size = sg->length;
179		phys_addr_t phys = sg_phys(sg);
180
181		if (!len)
182			break;
183
184		if (size <= off) {
185			off -= size;
186			continue;
187		}
188
189		phys += off;
190		size -= off;
191		size = min_t(size_t, size, len);
192		off = 0;
193
194		while (size) {
195			size_t pgsize, count, mapped = 0;
196			int ret;
197
198			pgsize = calc_pgsize(pagetable, addr, phys, size, &count);
199
200			ret = ops->map_pages(ops, addr, phys, pgsize, count,
201					     prot, GFP_KERNEL, &mapped);
202
203			/* map_pages could fail after mapping some of the pages,
204			 * so update the counters before error handling.
205			 */
206			phys += mapped;
207			addr += mapped;
208			size -= mapped;
209			len  -= mapped;
210
211			if (ret) {
212				msm_iommu_pagetable_unmap(mmu, iova, addr - iova);
213				return -EINVAL;
214			}
215		}
216	}
217
218	return 0;
219}
220
221static void msm_iommu_pagetable_destroy(struct msm_mmu *mmu)
222{
223	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
224	struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);
225	struct adreno_smmu_priv *adreno_smmu =
226		dev_get_drvdata(pagetable->parent->dev);
227
228	/*
229	 * If this is the last attached pagetable for the parent,
230	 * disable TTBR0 in the arm-smmu driver
231	 */
232	mutex_lock(&iommu->init_lock);
233	if (--iommu->pagetables == 0) {
234		adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, NULL);
235
236		if (adreno_smmu->set_prr_bit) {
237			adreno_smmu->set_prr_bit(adreno_smmu->cookie, false);
238			__free_page(iommu->prr_page);
239			iommu->prr_page = NULL;
240		}
241	}
242	mutex_unlock(&iommu->init_lock);
243
244	free_io_pgtable_ops(pagetable->pgtbl_ops);
245	kfree(pagetable);
246}
247
248int msm_iommu_pagetable_params(struct msm_mmu *mmu,
249		phys_addr_t *ttbr, int *asid)
250{
251	struct msm_iommu_pagetable *pagetable;
252
253	if (mmu->type != MSM_MMU_IOMMU_PAGETABLE)
254		return -EINVAL;
255
256	pagetable = to_pagetable(mmu);
257
258	if (ttbr)
259		*ttbr = pagetable->ttbr;
260
261	if (asid)
262		*asid = pagetable->asid;
263
264	return 0;
265}
266
267struct iommu_domain_geometry *msm_iommu_get_geometry(struct msm_mmu *mmu)
268{
269	struct msm_iommu *iommu = to_msm_iommu(mmu);
270
271	return &iommu->domain->geometry;
272}
273
274int
275msm_iommu_pagetable_walk(struct msm_mmu *mmu, unsigned long iova, uint64_t ptes[4])
276{
277	struct msm_iommu_pagetable *pagetable;
278	struct arm_lpae_io_pgtable_walk_data wd = {};
279
280	if (mmu->type != MSM_MMU_IOMMU_PAGETABLE)
281		return -EINVAL;
282
283	pagetable = to_pagetable(mmu);
284
285	if (!pagetable->pgtbl_ops->pgtable_walk)
286		return -EINVAL;
287
288	pagetable->pgtbl_ops->pgtable_walk(pagetable->pgtbl_ops, iova, &wd);
289
290	for (int i = 0; i < ARRAY_SIZE(wd.ptes); i++)
291		ptes[i] = wd.ptes[i];
292
293	return 0;
294}
295
296static void
297msm_iommu_pagetable_prealloc_count(struct msm_mmu *mmu, struct msm_mmu_prealloc *p,
298				   uint64_t iova, size_t len)
299{
300	u64 pt_count;
301
302	/*
303	 * L1, L2 and L3 page tables.
304	 *
305	 * We could optimize L3 allocation by iterating over the sgt and merging
306	 * 2M contiguous blocks, but it's simpler to over-provision and return
307	 * the pages if they're not used.
308	 *
309	 * The first level descriptor (v8 / v7-lpae page table format) encodes
310	 * 30 bits of address.  The second level encodes 29.  For the 3rd it is
311	 * 39.
312	 *
313	 * https://developer.arm.com/documentation/ddi0406/c/System-Level-Architecture/Virtual-Memory-System-Architecture--VMSA-/Long-descriptor-translation-table-format/Long-descriptor-translation-table-format-descriptors?lang=en#BEIHEFFB
314	 */
315	pt_count = ((ALIGN(iova + len, 1ull << 39) - ALIGN_DOWN(iova, 1ull << 39)) >> 39) +
316		   ((ALIGN(iova + len, 1ull << 30) - ALIGN_DOWN(iova, 1ull << 30)) >> 30) +
317		   ((ALIGN(iova + len, 1ull << 21) - ALIGN_DOWN(iova, 1ull << 21)) >> 21);
318
319	p->count += pt_count;
320}
321
322static struct kmem_cache *
323get_pt_cache(struct msm_mmu *mmu)
324{
325	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
326	return to_msm_iommu(pagetable->parent)->pt_cache;
327}
328
329static int
330msm_iommu_pagetable_prealloc_allocate(struct msm_mmu *mmu, struct msm_mmu_prealloc *p)
331{
332	struct kmem_cache *pt_cache = get_pt_cache(mmu);
333	int ret;
334
335	p->pages = kvmalloc_array(p->count, sizeof(p->pages), GFP_KERNEL);
336	if (!p->pages)
337		return -ENOMEM;
338
339	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages);
340	if (ret != p->count) {
341		kfree(p->pages);
342		p->pages = NULL;
343		p->count = ret;
344		return -ENOMEM;
345	}
346
347	return 0;
348}
349
350static void
351msm_iommu_pagetable_prealloc_cleanup(struct msm_mmu *mmu, struct msm_mmu_prealloc *p)
352{
353	struct kmem_cache *pt_cache = get_pt_cache(mmu);
354	uint32_t remaining_pt_count = p->count - p->ptr;
355
356	if (!p->pages)
357		return;
358
359	if (p->count > 0)
360		trace_msm_mmu_prealloc_cleanup(p->count, remaining_pt_count);
361
362	kmem_cache_free_bulk(pt_cache, remaining_pt_count, &p->pages[p->ptr]);
363	kvfree(p->pages);
364}
365
366/**
367 * msm_iommu_pagetable_alloc_pt() - Custom page table allocator
368 * @cookie: Cookie passed at page table allocation time.
369 * @size: Size of the page table. This size should be fixed,
370 * and determined at creation time based on the granule size.
371 * @gfp: GFP flags.
372 *
373 * We want a custom allocator so we can use a cache for page table
374 * allocations and amortize the cost of the over-reservation that's
375 * done to allow asynchronous VM operations.
376 *
377 * Return: non-NULL on success, NULL if the allocation failed for any
378 * reason.
379 */
380static void *
381msm_iommu_pagetable_alloc_pt(void *cookie, size_t size, gfp_t gfp)
382{
383	struct msm_iommu_pagetable *pagetable = cookie;
384	struct msm_mmu_prealloc *p = pagetable->base.prealloc;
385	void *page;
386
387	/* Allocation of the root page table happening during init. */
388	if (unlikely(!pagetable->root_page_table)) {
389		struct page *p;
390
391		p = alloc_pages_node(dev_to_node(pagetable->iommu_dev),
392				     gfp | __GFP_ZERO, get_order(size));
393		page = p ? page_address(p) : NULL;
394		pagetable->root_page_table = page;
395		return page;
396	}
397
398	if (WARN_ON(!p) || WARN_ON(p->ptr >= p->count))
399		return NULL;
400
401	page = p->pages[p->ptr++];
402	memset(page, 0, size);
403
404	/*
405	 * Page table entries don't use virtual addresses, which trips out
406	 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
407	 * are mixed with other fields, and I fear kmemleak won't detect that
408	 * either.
409	 *
410	 * Let's just ignore memory passed to the page-table driver for now.
411	 */
412	kmemleak_ignore(page);
413
414	return page;
415}
416
417
418/**
419 * msm_iommu_pagetable_free_pt() - Custom page table free function
420 * @cookie: Cookie passed at page table allocation time.
421 * @data: Page table to free.
422 * @size: Size of the page table. This size should be fixed,
423 * and determined at creation time based on the granule size.
424 */
425static void
426msm_iommu_pagetable_free_pt(void *cookie, void *data, size_t size)
427{
428	struct msm_iommu_pagetable *pagetable = cookie;
429
430	if (unlikely(pagetable->root_page_table == data)) {
431		free_pages((unsigned long)data, get_order(size));
432		pagetable->root_page_table = NULL;
433		return;
434	}
435
436	kmem_cache_free(get_pt_cache(&pagetable->base), data);
437}
438
439static const struct msm_mmu_funcs pagetable_funcs = {
440		.prealloc_count = msm_iommu_pagetable_prealloc_count,
441		.prealloc_allocate = msm_iommu_pagetable_prealloc_allocate,
442		.prealloc_cleanup = msm_iommu_pagetable_prealloc_cleanup,
443		.map = msm_iommu_pagetable_map,
444		.unmap = msm_iommu_pagetable_unmap,
445		.destroy = msm_iommu_pagetable_destroy,
446};
447
448static void msm_iommu_tlb_flush_all(void *cookie)
449{
450	struct msm_iommu_pagetable *pagetable = cookie;
451	struct adreno_smmu_priv *adreno_smmu;
452
453	if (!pm_runtime_get_if_in_use(pagetable->iommu_dev))
454		return;
455
456	adreno_smmu = dev_get_drvdata(pagetable->parent->dev);
457
458	pagetable->tlb->tlb_flush_all((void *)adreno_smmu->cookie);
459
460	pm_runtime_put_autosuspend(pagetable->iommu_dev);
461}
462
463static void msm_iommu_tlb_flush_walk(unsigned long iova, size_t size,
464		size_t granule, void *cookie)
465{
466	struct msm_iommu_pagetable *pagetable = cookie;
467	struct adreno_smmu_priv *adreno_smmu;
468
469	if (!pm_runtime_get_if_in_use(pagetable->iommu_dev))
470		return;
471
472	adreno_smmu = dev_get_drvdata(pagetable->parent->dev);
473
474	pagetable->tlb->tlb_flush_walk(iova, size, granule, (void *)adreno_smmu->cookie);
475
476	pm_runtime_put_autosuspend(pagetable->iommu_dev);
477}
478
479static void msm_iommu_tlb_add_page(struct iommu_iotlb_gather *gather,
480		unsigned long iova, size_t granule, void *cookie)
481{
482}
483
484static const struct iommu_flush_ops tlb_ops = {
485	.tlb_flush_all = msm_iommu_tlb_flush_all,
486	.tlb_flush_walk = msm_iommu_tlb_flush_walk,
487	.tlb_add_page = msm_iommu_tlb_add_page,
488};
489
490static int msm_gpu_fault_handler(struct iommu_domain *domain, struct device *dev,
491		unsigned long iova, int flags, void *arg);
492
493static size_t get_tblsz(const struct io_pgtable_cfg *cfg)
494{
495	int pg_shift, bits_per_level;
496
497	pg_shift = __ffs(cfg->pgsize_bitmap);
498	/* arm_lpae_iopte is u64: */
499	bits_per_level = pg_shift - ilog2(sizeof(u64));
500
501	return sizeof(u64) << bits_per_level;
502}
503
504struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent, bool kernel_managed)
505{
506	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(parent->dev);
507	struct msm_iommu *iommu = to_msm_iommu(parent);
508	struct msm_iommu_pagetable *pagetable;
509	const struct io_pgtable_cfg *ttbr1_cfg = NULL;
510	struct io_pgtable_cfg ttbr0_cfg;
511	int ret;
512
513	/* Get the pagetable configuration from the domain */
514	if (adreno_smmu->cookie)
515		ttbr1_cfg = adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie);
516
517	/*
518	 * If you hit this WARN_ONCE() you are probably missing an entry in
519	 * qcom_smmu_impl_of_match[] in arm-smmu-qcom.c
520	 */
521	if (WARN_ONCE(!ttbr1_cfg, "No per-process page tables"))
522		return ERR_PTR(-ENODEV);
523
524	pagetable = kzalloc(sizeof(*pagetable), GFP_KERNEL);
525	if (!pagetable)
526		return ERR_PTR(-ENOMEM);
527
528	msm_mmu_init(&pagetable->base, parent->dev, &pagetable_funcs,
529		MSM_MMU_IOMMU_PAGETABLE);
530
531	/* Clone the TTBR1 cfg as starting point for TTBR0 cfg: */
532	ttbr0_cfg = *ttbr1_cfg;
533
534	/* The incoming cfg will have the TTBR1 quirk enabled */
535	ttbr0_cfg.quirks &= ~IO_PGTABLE_QUIRK_ARM_TTBR1;
536	ttbr0_cfg.tlb = &tlb_ops;
537
538	if (!kernel_managed) {
539		ttbr0_cfg.quirks |= IO_PGTABLE_QUIRK_NO_WARN;
540
541		/*
542		 * With userspace managed VM (aka VM_BIND), we need to pre-
543		 * allocate pages ahead of time for map/unmap operations,
544		 * handing them to io-pgtable via custom alloc/free ops as
545		 * needed:
546		 */
547		ttbr0_cfg.alloc = msm_iommu_pagetable_alloc_pt;
548		ttbr0_cfg.free  = msm_iommu_pagetable_free_pt;
549
550		/*
551		 * Restrict to single page granules.  Otherwise we may run
552		 * into a situation where userspace wants to unmap/remap
553		 * only a part of a larger block mapping, which is not
554		 * possible without unmapping the entire block.  Which in
555		 * turn could cause faults if the GPU is accessing other
556		 * parts of the block mapping.
557		 *
558		 * Note that prior to commit 33729a5fc0ca ("iommu/io-pgtable-arm:
559		 * Remove split on unmap behavior)" this was handled in
560		 * io-pgtable-arm.  But this apparently does not work
561		 * correctly on SMMUv3.
562		 */
563		WARN_ON(!(ttbr0_cfg.pgsize_bitmap & PAGE_SIZE));
564		ttbr0_cfg.pgsize_bitmap = PAGE_SIZE;
565	}
566
567	pagetable->iommu_dev = ttbr1_cfg->iommu_dev;
568	pagetable->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1,
569		&ttbr0_cfg, pagetable);
570
571	if (!pagetable->pgtbl_ops) {
572		kfree(pagetable);
573		return ERR_PTR(-ENOMEM);
574	}
575
576	/*
577	 * If this is the first pagetable that we've allocated, send it back to
578	 * the arm-smmu driver as a trigger to set up TTBR0
579	 */
580	mutex_lock(&iommu->init_lock);
581	if (iommu->pagetables++ == 0) {
582		ret = adreno_smmu->set_ttbr0_cfg(adreno_smmu->cookie, &ttbr0_cfg);
583		if (ret) {
584			iommu->pagetables--;
585			mutex_unlock(&iommu->init_lock);
586			free_io_pgtable_ops(pagetable->pgtbl_ops);
587			kfree(pagetable);
588			return ERR_PTR(ret);
589		}
590
591		BUG_ON(iommu->prr_page);
592		if (adreno_smmu->set_prr_bit) {
593			/*
594			 * We need a zero'd page for two reasons:
595			 *
596			 * 1) Reserve a known physical address to use when
597			 *    mapping NULL / sparsely resident regions
598			 * 2) Read back zero
599			 *
600			 * It appears the hw drops writes to the PRR region
601			 * on the floor, but reads actually return whatever
602			 * is in the PRR page.
603			 */
604			iommu->prr_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
605			adreno_smmu->set_prr_addr(adreno_smmu->cookie,
606						  page_to_phys(iommu->prr_page));
607			adreno_smmu->set_prr_bit(adreno_smmu->cookie, true);
608		}
609	}
610	mutex_unlock(&iommu->init_lock);
611
612	/* Needed later for TLB flush */
613	pagetable->parent = parent;
614	pagetable->tlb = ttbr1_cfg->tlb;
615	pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap;
616	pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr;
617
618	/*
619	 * TODO we would like each set of page tables to have a unique ASID
620	 * to optimize TLB invalidation.  But iommu_flush_iotlb_all() will
621	 * end up flushing the ASID used for TTBR1 pagetables, which is not
622	 * what we want.  So for now just use the same ASID as TTBR1.
623	 */
624	pagetable->asid = 0;
625
626	return &pagetable->base;
627}
628
629static int msm_gpu_fault_handler(struct iommu_domain *domain, struct device *dev,
630		unsigned long iova, int flags, void *arg)
631{
632	struct msm_iommu *iommu = arg;
633	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(iommu->base.dev);
634	struct adreno_smmu_fault_info info, *ptr = NULL;
635
636	if (adreno_smmu->get_fault_info) {
637		adreno_smmu->get_fault_info(adreno_smmu->cookie, &info);
638		ptr = &info;
639	}
640
641	if (iommu->base.handler)
642		return iommu->base.handler(iommu->base.arg, iova, flags, ptr);
643
644	pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n", iova, flags);
645
646	return 0;
647}
648
649static int msm_disp_fault_handler(struct iommu_domain *domain, struct device *dev,
650				  unsigned long iova, int flags, void *arg)
651{
652	struct msm_iommu *iommu = arg;
653
654	if (iommu->base.handler)
655		return iommu->base.handler(iommu->base.arg, iova, flags, NULL);
656
657	return -ENOSYS;
658}
659
660static void msm_iommu_set_stall(struct msm_mmu *mmu, bool enable)
661{
662	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(mmu->dev);
663
664	if (adreno_smmu->set_stall)
665		adreno_smmu->set_stall(adreno_smmu->cookie, enable);
666}
667
668static void msm_iommu_detach(struct msm_mmu *mmu)
669{
670	struct msm_iommu *iommu = to_msm_iommu(mmu);
671
672	iommu_detach_device(iommu->domain, mmu->dev);
673}
674
675static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
676			 struct sg_table *sgt, size_t off, size_t len,
677			 int prot)
678{
679	struct msm_iommu *iommu = to_msm_iommu(mmu);
680	size_t ret;
681
682	WARN_ON(off != 0);
683
684	/* The arm-smmu driver expects the addresses to be sign extended */
685	if (iova & BIT_ULL(48))
686		iova |= GENMASK_ULL(63, 49);
687
688	ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);
689	WARN_ON(!ret);
690
691	return (ret == len) ? 0 : -EINVAL;
692}
693
694static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
695{
696	struct msm_iommu *iommu = to_msm_iommu(mmu);
697
698	if (iova & BIT_ULL(48))
699		iova |= GENMASK_ULL(63, 49);
700
701	iommu_unmap(iommu->domain, iova, len);
702
703	return 0;
704}
705
706static void msm_iommu_destroy(struct msm_mmu *mmu)
707{
708	struct msm_iommu *iommu = to_msm_iommu(mmu);
709	iommu_domain_free(iommu->domain);
710	kmem_cache_destroy(iommu->pt_cache);
711	kfree(iommu);
712}
713
714static const struct msm_mmu_funcs funcs = {
715		.detach = msm_iommu_detach,
716		.map = msm_iommu_map,
717		.unmap = msm_iommu_unmap,
718		.destroy = msm_iommu_destroy,
719		.set_stall = msm_iommu_set_stall,
720};
721
722struct msm_mmu *msm_iommu_new(struct device *dev, unsigned long quirks)
723{
724	struct iommu_domain *domain;
725	struct msm_iommu *iommu;
726	int ret;
727
728	if (!device_iommu_mapped(dev))
729		return ERR_PTR(-ENODEV);
730
731	domain = iommu_paging_domain_alloc(dev);
732	if (IS_ERR(domain))
733		return ERR_CAST(domain);
734
735	iommu_set_pgtable_quirks(domain, quirks);
736
737	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
738	if (!iommu) {
739		iommu_domain_free(domain);
740		return ERR_PTR(-ENOMEM);
741	}
742
743	iommu->domain = domain;
744	msm_mmu_init(&iommu->base, dev, &funcs, MSM_MMU_IOMMU);
745
746	mutex_init(&iommu->init_lock);
747
748	ret = iommu_attach_device(iommu->domain, dev);
749	if (ret) {
750		iommu_domain_free(domain);
751		kfree(iommu);
752		return ERR_PTR(ret);
753	}
754
755	return &iommu->base;
756}
757
758struct msm_mmu *msm_iommu_disp_new(struct device *dev, unsigned long quirks)
759{
760	struct msm_iommu *iommu;
761	struct msm_mmu *mmu;
762
763	mmu = msm_iommu_new(dev, quirks);
764	if (IS_ERR(mmu))
765		return mmu;
766
767	iommu = to_msm_iommu(mmu);
768	iommu_set_fault_handler(iommu->domain, msm_disp_fault_handler, iommu);
769
770	return mmu;
771}
772
773struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsigned long quirks)
774{
775	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev);
776	struct msm_iommu *iommu;
777	struct msm_mmu *mmu;
778
779	mmu = msm_iommu_new(dev, quirks);
780	if (IS_ERR(mmu))
781		return mmu;
782
783	iommu = to_msm_iommu(mmu);
784	if (adreno_smmu->cookie) {
785		const struct io_pgtable_cfg *cfg =
786			adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie);
787		size_t tblsz = get_tblsz(cfg);
788
789		iommu->pt_cache =
790			kmem_cache_create("msm-mmu-pt", tblsz, tblsz, 0, NULL);
791	}
792	iommu_set_fault_handler(iommu->domain, msm_gpu_fault_handler, iommu);
793
794	/* Enable stall on iommu fault: */
795	if (adreno_smmu->set_stall)
796		adreno_smmu->set_stall(adreno_smmu->cookie, true);
797
798	return mmu;
799}