iommu.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * iommu.c:  IOMMU specific routines for memory management.
  4 *
  5 * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
  6 * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
  7 * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
  8 * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
  9 */
 10
 11#include <linux/kernel.h>
 12#include <linux/init.h>
 13#include <linux/mm.h>
 14#include <linux/slab.h>
 15#include <linux/dma-map-ops.h>
 16#include <linux/of.h>
 17#include <linux/of_platform.h>
 18#include <linux/platform_device.h>
 19
 20#include <asm/io.h>
 21#include <asm/mxcc.h>
 22#include <asm/mbus.h>
 23#include <asm/cacheflush.h>
 24#include <asm/tlbflush.h>
 25#include <asm/bitext.h>
 26#include <asm/iommu.h>
 27#include <asm/dma.h>
 28
 29#include "mm_32.h"
 30
 31/*
 32 * This can be sized dynamically, but we will do this
 33 * only when we have a guidance about actual I/O pressures.
 34 */
 35#define IOMMU_RNGE	IOMMU_RNGE_256MB
 36#define IOMMU_START	0xF0000000
 37#define IOMMU_WINSIZE	(256*1024*1024U)
 38#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 256KB */
 39#define IOMMU_ORDER	6				/* 4096 * (1<<6) */
 40
 41static int viking_flush;
 42/* viking.S */
 43extern void viking_flush_page(unsigned long page);
 44extern void viking_mxcc_flush_page(unsigned long page);
 45
 46/*
 47 * Values precomputed according to CPU type.
 48 */
 49static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
 50static pgprot_t dvma_prot;		/* Consistent mapping pte flags */
 51
 52#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
 53#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
 54
 55static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
 56static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
 57
 58static void __init sbus_iommu_init(struct platform_device *op)
 59{
 60	struct iommu_struct *iommu;
 61	unsigned int impl, vers;
 62	unsigned long *bitmap;
 63	unsigned long control;
 64	unsigned long base;
 65	unsigned long tmp;
 66
 67	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
 68	if (!iommu) {
 69		prom_printf("Unable to allocate iommu structure\n");
 70		prom_halt();
 71	}
 72
 73	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
 74				 "iommu_regs");
 75	if (!iommu->regs) {
 76		prom_printf("Cannot map IOMMU registers\n");
 77		prom_halt();
 78	}
 79
 80	control = sbus_readl(&iommu->regs->control);
 81	impl = (control & IOMMU_CTRL_IMPL) >> 28;
 82	vers = (control & IOMMU_CTRL_VERS) >> 24;
 83	control &= ~(IOMMU_CTRL_RNGE);
 84	control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
 85	sbus_writel(control, &iommu->regs->control);
 86
 87	iommu_invalidate(iommu->regs);
 88	iommu->start = IOMMU_START;
 89	iommu->end = 0xffffffff;
 90
 91	/* Allocate IOMMU page table */
 92	/* Stupid alignment constraints give me a headache. 
 93	   We need 256K or 512K or 1M or 2M area aligned to
 94           its size and current gfp will fortunately give
 95           it to us. */
 96        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
 97	if (!tmp) {
 98		prom_printf("Unable to allocate iommu table [0x%lx]\n",
 99			    IOMMU_NPTES * sizeof(iopte_t));
100		prom_halt();
101	}
102	iommu->page_table = (iopte_t *)tmp;
103
104	/* Initialize new table. */
105	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
106	flush_cache_all();
107	flush_tlb_all();
108
109	base = __pa((unsigned long)iommu->page_table) >> 4;
110	sbus_writel(base, &iommu->regs->base);
111	iommu_invalidate(iommu->regs);
112
113	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
114	if (!bitmap) {
115		prom_printf("Unable to allocate iommu bitmap [%d]\n",
116			    (int)(IOMMU_NPTES>>3));
117		prom_halt();
118	}
119	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
120	/* To be coherent on HyperSparc, the page color of DVMA
121	 * and physical addresses must match.
122	 */
123	if (srmmu_modtype == HyperSparc)
124		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
125	else
126		iommu->usemap.num_colors = 1;
127
128	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
129	       impl, vers, iommu->page_table,
130	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
131
132	op->dev.archdata.iommu = iommu;
133
134	if (flush_page_for_dma_global)
135		op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
136	 else
137		op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
138}
139
140static int __init iommu_init(void)
141{
142	struct device_node *dp;
143
144	for_each_node_by_name(dp, "iommu") {
145		struct platform_device *op = of_find_device_by_node(dp);
146
147		sbus_iommu_init(op);
148		of_propagate_archdata(op);
149	}
150
151	return 0;
152}
153
154subsys_initcall(iommu_init);
155
156/* Flush the iotlb entries to ram. */
157/* This could be better if we didn't have to flush whole pages. */
158static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
159{
160	unsigned long start;
161	unsigned long end;
162
163	start = (unsigned long)iopte;
164	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
165	start &= PAGE_MASK;
166	if (viking_mxcc_present) {
167		while(start < end) {
168			viking_mxcc_flush_page(start);
169			start += PAGE_SIZE;
170		}
171	} else if (viking_flush) {
172		while(start < end) {
173			viking_flush_page(start);
174			start += PAGE_SIZE;
175		}
176	} else {
177		while(start < end) {
178			__flush_page_to_ram(start);
179			start += PAGE_SIZE;
180		}
181	}
182}
183
184static dma_addr_t __sbus_iommu_map_phys(struct device *dev, phys_addr_t paddr,
185		size_t len, bool per_page_flush, unsigned long attrs)
186{
187	struct iommu_struct *iommu = dev->archdata.iommu;
188	unsigned long off = offset_in_page(paddr);
189	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
190	unsigned long pfn = __phys_to_pfn(paddr);
191	unsigned int busa, busa0;
192	iopte_t *iopte, *iopte0;
193	int ioptex, i;
194
195	if (unlikely(attrs & DMA_ATTR_MMIO))
196		return DMA_MAPPING_ERROR;
197
198	/* XXX So what is maxphys for us and how do drivers know it? */
199	if (!len || len > 256 * 1024)
200		return DMA_MAPPING_ERROR;
201
202	/*
203	 * We expect unmapped highmem pages to be not in the cache.
204	 * XXX Is this a good assumption?
205	 * XXX What if someone else unmaps it here and races us?
206	 */
207	if (per_page_flush && !PhysHighMem(paddr)) {
208		unsigned long vaddr, p;
209
210		vaddr = (unsigned long)phys_to_virt(paddr);
211		for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
212			flush_page_for_dma(p);
213	}
214
215	/* page color = pfn of page */
216	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
217	if (ioptex < 0)
218		panic("iommu out");
219	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
220	iopte0 = &iommu->page_table[ioptex];
221
222	busa = busa0;
223	iopte = iopte0;
224	for (i = 0; i < npages; i++) {
225		iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
226		iommu_invalidate_page(iommu->regs, busa);
227		busa += PAGE_SIZE;
228		iopte++;
229		pfn++;
230	}
231
232	iommu_flush_iotlb(iopte0, npages);
233	return busa0 + off;
234}
235
236static dma_addr_t sbus_iommu_map_phys_gflush(struct device *dev,
237		phys_addr_t phys, size_t len, enum dma_data_direction dir,
238		unsigned long attrs)
239{
240	flush_page_for_dma(0);
241	return __sbus_iommu_map_phys(dev, phys, len, false, attrs);
242}
243
244static dma_addr_t sbus_iommu_map_phys_pflush(struct device *dev,
245		phys_addr_t phys, size_t len, enum dma_data_direction dir,
246		unsigned long attrs)
247{
248	return __sbus_iommu_map_phys(dev, phys, len, true, attrs);
249}
250
251static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
252		int nents, enum dma_data_direction dir, unsigned long attrs,
253		bool per_page_flush)
254{
255	struct scatterlist *sg;
256	int j;
257
258	for_each_sg(sgl, sg, nents, j) {
259		sg->dma_address = __sbus_iommu_map_phys(dev, sg_phys(sg),
260				sg->length, per_page_flush, attrs);
261		if (sg->dma_address == DMA_MAPPING_ERROR)
262			return -EIO;
263		sg->dma_length = sg->length;
264	}
265
266	return nents;
267}
268
269static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
270		int nents, enum dma_data_direction dir, unsigned long attrs)
271{
272	flush_page_for_dma(0);
273	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
274}
275
276static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
277		int nents, enum dma_data_direction dir, unsigned long attrs)
278{
279	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
280}
281
282static void sbus_iommu_unmap_phys(struct device *dev, dma_addr_t dma_addr,
283		size_t len, enum dma_data_direction dir, unsigned long attrs)
284{
285	struct iommu_struct *iommu = dev->archdata.iommu;
286	unsigned int busa = dma_addr & PAGE_MASK;
287	unsigned long off = dma_addr & ~PAGE_MASK;
288	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
289	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
290	unsigned int i;
291
292	BUG_ON(busa < iommu->start);
293	for (i = 0; i < npages; i++) {
294		iopte_val(iommu->page_table[ioptex + i]) = 0;
295		iommu_invalidate_page(iommu->regs, busa);
296		busa += PAGE_SIZE;
297	}
298	bit_map_clear(&iommu->usemap, ioptex, npages);
299}
300
301static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
302		int nents, enum dma_data_direction dir, unsigned long attrs)
303{
304	struct scatterlist *sg;
305	int i;
306
307	for_each_sg(sgl, sg, nents, i) {
308		sbus_iommu_unmap_phys(dev, sg->dma_address, sg->length, dir,
309				attrs);
310		sg->dma_address = 0x21212121;
311	}
312}
313
314#ifdef CONFIG_SBUS
315static void *sbus_iommu_alloc(struct device *dev, size_t len,
316		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
317{
318	struct iommu_struct *iommu = dev->archdata.iommu;
319	unsigned long va, addr, page, end, ret;
320	iopte_t *iopte = iommu->page_table;
321	iopte_t *first;
322	int ioptex;
323
324	/* XXX So what is maxphys for us and how do drivers know it? */
325	if (!len || len > 256 * 1024)
326		return NULL;
327
328	len = PAGE_ALIGN(len);
329	va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
330	if (va == 0)
331		return NULL;
332
333	addr = ret = sparc_dma_alloc_resource(dev, len);
334	if (!addr)
335		goto out_free_pages;
336
337	BUG_ON((va & ~PAGE_MASK) != 0);
338	BUG_ON((addr & ~PAGE_MASK) != 0);
339	BUG_ON((len & ~PAGE_MASK) != 0);
340
341	/* page color = physical address */
342	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
343		addr >> PAGE_SHIFT);
344	if (ioptex < 0)
345		panic("iommu out");
346
347	iopte += ioptex;
348	first = iopte;
349	end = addr + len;
350	while(addr < end) {
351		page = va;
352		{
353			pmd_t *pmdp;
354			pte_t *ptep;
355
356			if (viking_mxcc_present)
357				viking_mxcc_flush_page(page);
358			else if (viking_flush)
359				viking_flush_page(page);
360			else
361				__flush_page_to_ram(page);
362
363			pmdp = pmd_off_k(addr);
364			ptep = pte_offset_kernel(pmdp, addr);
365
366			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
367		}
368		iopte_val(*iopte++) =
369		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
370		addr += PAGE_SIZE;
371		va += PAGE_SIZE;
372	}
373	/* P3: why do we need this?
374	 *
375	 * DAVEM: Because there are several aspects, none of which
376	 *        are handled by a single interface.  Some cpus are
377	 *        completely not I/O DMA coherent, and some have
378	 *        virtually indexed caches.  The driver DMA flushing
379	 *        methods handle the former case, but here during
380	 *        IOMMU page table modifications, and usage of non-cacheable
381	 *        cpu mappings of pages potentially in the cpu caches, we have
382	 *        to handle the latter case as well.
383	 */
384	flush_cache_all();
385	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
386	flush_tlb_all();
387	iommu_invalidate(iommu->regs);
388
389	*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
390	return (void *)ret;
391
392out_free_pages:
393	free_pages(va, get_order(len));
394	return NULL;
395}
396
397static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
398			       dma_addr_t busa, unsigned long attrs)
399{
400	struct iommu_struct *iommu = dev->archdata.iommu;
401	iopte_t *iopte = iommu->page_table;
402	struct page *page = virt_to_page(cpu_addr);
403	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
404	unsigned long end;
405
406	if (!sparc_dma_free_resource(cpu_addr, len))
407		return;
408
409	BUG_ON((busa & ~PAGE_MASK) != 0);
410	BUG_ON((len & ~PAGE_MASK) != 0);
411
412	iopte += ioptex;
413	end = busa + len;
414	while (busa < end) {
415		iopte_val(*iopte++) = 0;
416		busa += PAGE_SIZE;
417	}
418	flush_tlb_all();
419	iommu_invalidate(iommu->regs);
420	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
421
422	__free_pages(page, get_order(len));
423}
424#endif
425
426static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
427#ifdef CONFIG_SBUS
428	.alloc			= sbus_iommu_alloc,
429	.free			= sbus_iommu_free,
430#endif
431	.map_phys		= sbus_iommu_map_phys_gflush,
432	.unmap_phys		= sbus_iommu_unmap_phys,
433	.map_sg			= sbus_iommu_map_sg_gflush,
434	.unmap_sg		= sbus_iommu_unmap_sg,
435};
436
437static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
438#ifdef CONFIG_SBUS
439	.alloc			= sbus_iommu_alloc,
440	.free			= sbus_iommu_free,
441#endif
442	.map_phys		= sbus_iommu_map_phys_pflush,
443	.unmap_phys		= sbus_iommu_unmap_phys,
444	.map_sg			= sbus_iommu_map_sg_pflush,
445	.unmap_sg		= sbus_iommu_unmap_sg,
446};
447
448void __init ld_mmu_iommu(void)
449{
450	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
451		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
452		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
453	} else {
454		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
455		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
456	}
457}