amd_gart_64.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Dynamic DMA mapping support for AMD Hammer.
  4 *
  5 * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
  6 * This allows to use PCI devices that only support 32bit addresses on systems
  7 * with more than 4GB.
  8 *
  9 * See Documentation/core-api/dma-api-howto.rst for the interface specification.
 10 *
 11 * Copyright 2002 Andi Kleen, SuSE Labs.
 12 */
 13
 14#include <linux/types.h>
 15#include <linux/ctype.h>
 16#include <linux/agp_backend.h>
 17#include <linux/init.h>
 18#include <linux/mm.h>
 19#include <linux/sched.h>
 20#include <linux/sched/debug.h>
 21#include <linux/string.h>
 22#include <linux/spinlock.h>
 23#include <linux/pci.h>
 24#include <linux/topology.h>
 25#include <linux/interrupt.h>
 26#include <linux/bitmap.h>
 27#include <linux/kdebug.h>
 28#include <linux/scatterlist.h>
 29#include <linux/iommu-helper.h>
 30#include <linux/syscore_ops.h>
 31#include <linux/io.h>
 32#include <linux/gfp.h>
 33#include <linux/atomic.h>
 34#include <linux/dma-direct.h>
 35#include <linux/dma-map-ops.h>
 36#include <asm/mtrr.h>
 37#include <asm/proto.h>
 38#include <asm/iommu.h>
 39#include <asm/gart.h>
 40#include <asm/set_memory.h>
 41#include <asm/dma.h>
 42#include <asm/amd/nb.h>
 43#include <asm/x86_init.h>
 44
 45static unsigned long iommu_bus_base;	/* GART remapping area (physical) */
 46static unsigned long iommu_size;	/* size of remapping area bytes */
 47static unsigned long iommu_pages;	/* .. and in pages */
 48
 49static u32 *iommu_gatt_base;		/* Remapping table */
 50
 51/*
 52 * If this is disabled the IOMMU will use an optimized flushing strategy
 53 * of only flushing when an mapping is reused. With it true the GART is
 54 * flushed for every mapping. Problem is that doing the lazy flush seems
 55 * to trigger bugs with some popular PCI cards, in particular 3ware (but
 56 * has been also seen with Qlogic at least).
 57 */
 58static int iommu_fullflush = 1;
 59
 60/* Allocation bitmap for the remapping area: */
 61static DEFINE_SPINLOCK(iommu_bitmap_lock);
 62/* Guarded by iommu_bitmap_lock: */
 63static unsigned long *iommu_gart_bitmap;
 64
 65static u32 gart_unmapped_entry;
 66
 67#define GPTE_VALID    1
 68#define GPTE_COHERENT 2
 69#define GPTE_ENCODE(x) \
 70	(((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT)
 71#define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28))
 72
 73#ifdef CONFIG_AGP
 74#define AGPEXTERN extern
 75#else
 76#define AGPEXTERN
 77#endif
 78
 79/* GART can only remap to physical addresses < 1TB */
 80#define GART_MAX_PHYS_ADDR	(1ULL << 40)
 81
 82/* backdoor interface to AGP driver */
 83AGPEXTERN int agp_memory_reserved;
 84AGPEXTERN __u32 *agp_gatt_table;
 85
 86static unsigned long next_bit;  /* protected by iommu_bitmap_lock */
 87static bool need_flush;		/* global flush state. set for each gart wrap */
 88
 89static unsigned long alloc_iommu(struct device *dev, int size,
 90				 unsigned long align_mask)
 91{
 92	unsigned long offset, flags;
 93	unsigned long boundary_size;
 94	unsigned long base_index;
 95
 96	base_index = ALIGN(iommu_bus_base & dma_get_seg_boundary(dev),
 97			   PAGE_SIZE) >> PAGE_SHIFT;
 98	boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
 99
100	spin_lock_irqsave(&iommu_bitmap_lock, flags);
101	offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, next_bit,
102				  size, base_index, boundary_size, align_mask);
103	if (offset == -1) {
104		need_flush = true;
105		offset = iommu_area_alloc(iommu_gart_bitmap, iommu_pages, 0,
106					  size, base_index, boundary_size,
107					  align_mask);
108	}
109	if (offset != -1) {
110		next_bit = offset+size;
111		if (next_bit >= iommu_pages) {
112			next_bit = 0;
113			need_flush = true;
114		}
115	}
116	if (iommu_fullflush)
117		need_flush = true;
118	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
119
120	return offset;
121}
122
123static void free_iommu(unsigned long offset, int size)
124{
125	unsigned long flags;
126
127	spin_lock_irqsave(&iommu_bitmap_lock, flags);
128	bitmap_clear(iommu_gart_bitmap, offset, size);
129	if (offset >= next_bit)
130		next_bit = offset + size;
131	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
132}
133
134/*
135 * Use global flush state to avoid races with multiple flushers.
136 */
137static void flush_gart(void)
138{
139	unsigned long flags;
140
141	spin_lock_irqsave(&iommu_bitmap_lock, flags);
142	if (need_flush) {
143		amd_flush_garts();
144		need_flush = false;
145	}
146	spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
147}
148
149#ifdef CONFIG_IOMMU_LEAK
150/* Debugging aid for drivers that don't free their IOMMU tables */
151static void dump_leak(void)
152{
153	static int dump;
154
155	if (dump)
156		return;
157	dump = 1;
158
159	show_stack(NULL, NULL, KERN_ERR);
160	debug_dma_dump_mappings(NULL);
161}
162#endif
163
164static void iommu_full(struct device *dev, size_t size, int dir)
165{
166	/*
167	 * Ran out of IOMMU space for this operation. This is very bad.
168	 * Unfortunately the drivers cannot handle this operation properly.
169	 * Return some non mapped prereserved space in the aperture and
170	 * let the Northbridge deal with it. This will result in garbage
171	 * in the IO operation. When the size exceeds the prereserved space
172	 * memory corruption will occur or random memory will be DMAed
173	 * out. Hopefully no network devices use single mappings that big.
174	 */
175
176	dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size);
177#ifdef CONFIG_IOMMU_LEAK
178	dump_leak();
179#endif
180}
181
182static inline int
183need_iommu(struct device *dev, unsigned long addr, size_t size)
184{
185	return force_iommu || !dma_capable(dev, addr, size, true);
186}
187
188static inline int
189nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
190{
191	return !dma_capable(dev, addr, size, true);
192}
193
194/* Map a single continuous physical area into the IOMMU.
195 * Caller needs to check if the iommu is needed and flush.
196 */
197static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
198				size_t size, int dir, unsigned long align_mask)
199{
200	unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
201	unsigned long iommu_page;
202	int i;
203
204	if (unlikely(phys_mem + size > GART_MAX_PHYS_ADDR))
205		return DMA_MAPPING_ERROR;
206
207	iommu_page = alloc_iommu(dev, npages, align_mask);
208	if (iommu_page == -1) {
209		if (!nonforced_iommu(dev, phys_mem, size))
210			return phys_mem;
211		if (panic_on_overflow)
212			panic("dma_map_area overflow %lu bytes\n", size);
213		iommu_full(dev, size, dir);
214		return DMA_MAPPING_ERROR;
215	}
216
217	for (i = 0; i < npages; i++) {
218		iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem);
219		phys_mem += PAGE_SIZE;
220	}
221	return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
222}
223
224/* Map a single area into the IOMMU */
225static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr,
226				size_t size, enum dma_data_direction dir,
227				unsigned long attrs)
228{
229	unsigned long bus;
230
231	if (unlikely(attrs & DMA_ATTR_MMIO))
232		return DMA_MAPPING_ERROR;
233
234	if (!need_iommu(dev, paddr, size))
235		return paddr;
236
237	bus = dma_map_area(dev, paddr, size, dir, 0);
238	flush_gart();
239
240	return bus;
241}
242
243/*
244 * Free a DMA mapping.
245 */
246static void gart_unmap_phys(struct device *dev, dma_addr_t dma_addr,
247			    size_t size, enum dma_data_direction dir,
248			    unsigned long attrs)
249{
250	unsigned long iommu_page;
251	int npages;
252	int i;
253
254	if (WARN_ON_ONCE(dma_addr == DMA_MAPPING_ERROR))
255		return;
256
257	/*
258	 * This driver will not always use a GART mapping, but might have
259	 * created a direct mapping instead.  If that is the case there is
260	 * nothing to unmap here.
261	 */
262	if (dma_addr < iommu_bus_base ||
263	    dma_addr >= iommu_bus_base + iommu_size)
264		return;
265
266	iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;
267	npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
268	for (i = 0; i < npages; i++) {
269		iommu_gatt_base[iommu_page + i] = gart_unmapped_entry;
270	}
271	free_iommu(iommu_page, npages);
272}
273
274/*
275 * Wrapper for pci_unmap_single working with scatterlists.
276 */
277static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
278			  enum dma_data_direction dir, unsigned long attrs)
279{
280	struct scatterlist *s;
281	int i;
282
283	for_each_sg(sg, s, nents, i) {
284		if (!s->dma_length || !s->length)
285			break;
286		gart_unmap_phys(dev, s->dma_address, s->dma_length, dir, 0);
287	}
288}
289
290/* Fallback for dma_map_sg in case of overflow */
291static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
292			       int nents, int dir)
293{
294	struct scatterlist *s;
295	int i;
296
297#ifdef CONFIG_IOMMU_DEBUG
298	pr_debug("dma_map_sg overflow\n");
299#endif
300
301	for_each_sg(sg, s, nents, i) {
302		unsigned long addr = sg_phys(s);
303
304		if (nonforced_iommu(dev, addr, s->length)) {
305			addr = dma_map_area(dev, addr, s->length, dir, 0);
306			if (addr == DMA_MAPPING_ERROR) {
307				if (i > 0)
308					gart_unmap_sg(dev, sg, i, dir, 0);
309				nents = 0;
310				sg[0].dma_length = 0;
311				break;
312			}
313		}
314		s->dma_address = addr;
315		s->dma_length = s->length;
316	}
317	flush_gart();
318
319	return nents;
320}
321
322/* Map multiple scatterlist entries continuous into the first. */
323static int __dma_map_cont(struct device *dev, struct scatterlist *start,
324			  int nelems, struct scatterlist *sout,
325			  unsigned long pages)
326{
327	unsigned long iommu_start = alloc_iommu(dev, pages, 0);
328	unsigned long iommu_page = iommu_start;
329	struct scatterlist *s;
330	int i;
331
332	if (iommu_start == -1)
333		return -ENOMEM;
334
335	for_each_sg(start, s, nelems, i) {
336		unsigned long pages, addr;
337		unsigned long phys_addr = s->dma_address;
338
339		BUG_ON(s != start && s->offset);
340		if (s == start) {
341			sout->dma_address = iommu_bus_base;
342			sout->dma_address += iommu_page*PAGE_SIZE + s->offset;
343			sout->dma_length = s->length;
344		} else {
345			sout->dma_length += s->length;
346		}
347
348		addr = phys_addr;
349		pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE);
350		while (pages--) {
351			iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr);
352			addr += PAGE_SIZE;
353			iommu_page++;
354		}
355	}
356	BUG_ON(iommu_page - iommu_start != pages);
357
358	return 0;
359}
360
361static inline int
362dma_map_cont(struct device *dev, struct scatterlist *start, int nelems,
363	     struct scatterlist *sout, unsigned long pages, int need)
364{
365	if (!need) {
366		BUG_ON(nelems != 1);
367		sout->dma_address = start->dma_address;
368		sout->dma_length = start->length;
369		return 0;
370	}
371	return __dma_map_cont(dev, start, nelems, sout, pages);
372}
373
374/*
375 * DMA map all entries in a scatterlist.
376 * Merge chunks that have page aligned sizes into a continuous mapping.
377 */
378static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
379		       enum dma_data_direction dir, unsigned long attrs)
380{
381	struct scatterlist *s, *ps, *start_sg, *sgmap;
382	int need = 0, nextneed, i, out, start, ret;
383	unsigned long pages = 0;
384	unsigned int seg_size;
385	unsigned int max_seg_size;
386
387	if (nents == 0)
388		return -EINVAL;
389
390	out		= 0;
391	start		= 0;
392	start_sg	= sg;
393	sgmap		= sg;
394	seg_size	= 0;
395	max_seg_size	= dma_get_max_seg_size(dev);
396	ps		= NULL; /* shut up gcc */
397
398	for_each_sg(sg, s, nents, i) {
399		dma_addr_t addr = sg_phys(s);
400
401		s->dma_address = addr;
402		BUG_ON(s->length == 0);
403
404		nextneed = need_iommu(dev, addr, s->length);
405
406		/* Handle the previous not yet processed entries */
407		if (i > start) {
408			/*
409			 * Can only merge when the last chunk ends on a
410			 * page boundary and the new one doesn't have an
411			 * offset.
412			 */
413			if (!iommu_merge || !nextneed || !need || s->offset ||
414			    (s->length + seg_size > max_seg_size) ||
415			    (ps->offset + ps->length) % PAGE_SIZE) {
416				ret = dma_map_cont(dev, start_sg, i - start,
417						   sgmap, pages, need);
418				if (ret < 0)
419					goto error;
420				out++;
421
422				seg_size	= 0;
423				sgmap		= sg_next(sgmap);
424				pages		= 0;
425				start		= i;
426				start_sg	= s;
427			}
428		}
429
430		seg_size += s->length;
431		need = nextneed;
432		pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE);
433		ps = s;
434	}
435	ret = dma_map_cont(dev, start_sg, i - start, sgmap, pages, need);
436	if (ret < 0)
437		goto error;
438	out++;
439	flush_gart();
440	if (out < nents) {
441		sgmap = sg_next(sgmap);
442		sgmap->dma_length = 0;
443	}
444	return out;
445
446error:
447	flush_gart();
448	gart_unmap_sg(dev, sg, out, dir, 0);
449
450	/* When it was forced or merged try again in a dumb way */
451	if (force_iommu || iommu_merge) {
452		out = dma_map_sg_nonforce(dev, sg, nents, dir);
453		if (out > 0)
454			return out;
455	}
456	if (panic_on_overflow)
457		panic("dma_map_sg: overflow on %lu pages\n", pages);
458
459	iommu_full(dev, pages << PAGE_SHIFT, dir);
460	return ret;
461}
462
463/* allocate and map a coherent mapping */
464static void *
465gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
466		    gfp_t flag, unsigned long attrs)
467{
468	void *vaddr;
469
470	vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
471	if (!vaddr ||
472	    !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
473		return vaddr;
474
475	*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
476			DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
477	flush_gart();
478	if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
479		goto out_free;
480	return vaddr;
481out_free:
482	dma_direct_free(dev, size, vaddr, *dma_addr, attrs);
483	return NULL;
484}
485
486/* free a coherent mapping */
487static void
488gart_free_coherent(struct device *dev, size_t size, void *vaddr,
489		   dma_addr_t dma_addr, unsigned long attrs)
490{
491	gart_unmap_phys(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
492	dma_direct_free(dev, size, vaddr, dma_addr, attrs);
493}
494
495static int no_agp;
496
497static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
498{
499	unsigned long a;
500
501	if (!iommu_size) {
502		iommu_size = aper_size;
503		if (!no_agp)
504			iommu_size /= 2;
505	}
506
507	a = aper + iommu_size;
508	iommu_size -= round_up(a, PMD_SIZE) - a;
509
510	if (iommu_size < 64*1024*1024) {
511		pr_warn("PCI-DMA: Warning: Small IOMMU %luMB."
512			" Consider increasing the AGP aperture in BIOS\n",
513			iommu_size >> 20);
514	}
515
516	return iommu_size;
517}
518
519static __init unsigned read_aperture(struct pci_dev *dev, u32 *size)
520{
521	unsigned aper_size = 0, aper_base_32, aper_order;
522	u64 aper_base;
523
524	pci_read_config_dword(dev, AMD64_GARTAPERTUREBASE, &aper_base_32);
525	pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &aper_order);
526	aper_order = (aper_order >> 1) & 7;
527
528	aper_base = aper_base_32 & 0x7fff;
529	aper_base <<= 25;
530
531	aper_size = (32 * 1024 * 1024) << aper_order;
532	if (aper_base + aper_size > 0x100000000UL || !aper_size)
533		aper_base = 0;
534
535	*size = aper_size;
536	return aper_base;
537}
538
539static void enable_gart_translations(void)
540{
541	int i;
542
543	if (!amd_nb_has_feature(AMD_NB_GART))
544		return;
545
546	for (i = 0; i < amd_nb_num(); i++) {
547		struct pci_dev *dev = node_to_amd_nb(i)->misc;
548
549		enable_gart_translation(dev, __pa(agp_gatt_table));
550	}
551
552	/* Flush the GART-TLB to remove stale entries */
553	amd_flush_garts();
554}
555
556/*
557 * If fix_up_north_bridges is set, the north bridges have to be fixed up on
558 * resume in the same way as they are handled in gart_iommu_hole_init().
559 */
560static bool fix_up_north_bridges;
561static u32 aperture_order;
562static u32 aperture_alloc;
563
564void set_up_gart_resume(u32 aper_order, u32 aper_alloc)
565{
566	fix_up_north_bridges = true;
567	aperture_order = aper_order;
568	aperture_alloc = aper_alloc;
569}
570
571static void gart_fixup_northbridges(void)
572{
573	int i;
574
575	if (!fix_up_north_bridges)
576		return;
577
578	if (!amd_nb_has_feature(AMD_NB_GART))
579		return;
580
581	pr_info("PCI-DMA: Restoring GART aperture settings\n");
582
583	for (i = 0; i < amd_nb_num(); i++) {
584		struct pci_dev *dev = node_to_amd_nb(i)->misc;
585
586		/*
587		 * Don't enable translations just yet.  That is the next
588		 * step.  Restore the pre-suspend aperture settings.
589		 */
590		gart_set_size_and_enable(dev, aperture_order);
591		pci_write_config_dword(dev, AMD64_GARTAPERTUREBASE, aperture_alloc >> 25);
592	}
593}
594
595static void gart_resume(void *data)
596{
597	pr_info("PCI-DMA: Resuming GART IOMMU\n");
598
599	gart_fixup_northbridges();
600
601	enable_gart_translations();
602}
603
604static const struct syscore_ops gart_syscore_ops = {
605	.resume		= gart_resume,
606
607};
608
609static struct syscore gart_syscore = {
610	.ops = &gart_syscore_ops,
611};
612
613/*
614 * Private Northbridge GATT initialization in case we cannot use the
615 * AGP driver for some reason.
616 */
617static __init int init_amd_gatt(struct agp_kern_info *info)
618{
619	unsigned aper_size, gatt_size, new_aper_size;
620	unsigned aper_base, new_aper_base;
621	struct pci_dev *dev;
622	void *gatt;
623	int i;
624
625	pr_info("PCI-DMA: Disabling AGP.\n");
626
627	aper_size = aper_base = info->aper_size = 0;
628	dev = NULL;
629	for (i = 0; i < amd_nb_num(); i++) {
630		dev = node_to_amd_nb(i)->misc;
631		new_aper_base = read_aperture(dev, &new_aper_size);
632		if (!new_aper_base)
633			goto nommu;
634
635		if (!aper_base) {
636			aper_size = new_aper_size;
637			aper_base = new_aper_base;
638		}
639		if (aper_size != new_aper_size || aper_base != new_aper_base)
640			goto nommu;
641	}
642	if (!aper_base)
643		goto nommu;
644
645	info->aper_base = aper_base;
646	info->aper_size = aper_size >> 20;
647
648	gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32);
649	gatt = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
650					get_order(gatt_size));
651	if (!gatt)
652		panic("Cannot allocate GATT table");
653	if (set_memory_uc((unsigned long)gatt, gatt_size >> PAGE_SHIFT))
654		panic("Could not set GART PTEs to uncacheable pages");
655
656	agp_gatt_table = gatt;
657
658	register_syscore(&gart_syscore);
659
660	flush_gart();
661
662	pr_info("PCI-DMA: aperture base @ %x size %u KB\n",
663	       aper_base, aper_size>>10);
664
665	return 0;
666
667 nommu:
668	/* Should not happen anymore */
669	pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n");
670	return -1;
671}
672
673static const struct dma_map_ops gart_dma_ops = {
674	.map_sg				= gart_map_sg,
675	.unmap_sg			= gart_unmap_sg,
676	.map_phys			= gart_map_phys,
677	.unmap_phys			= gart_unmap_phys,
678	.alloc				= gart_alloc_coherent,
679	.free				= gart_free_coherent,
680	.mmap				= dma_common_mmap,
681	.get_sgtable			= dma_common_get_sgtable,
682	.dma_supported			= dma_direct_supported,
683	.get_required_mask		= dma_direct_get_required_mask,
684	.alloc_pages_op			= dma_direct_alloc_pages,
685	.free_pages			= dma_direct_free_pages,
686};
687
688static void gart_iommu_shutdown(void)
689{
690	struct pci_dev *dev;
691	int i;
692
693	/* don't shutdown it if there is AGP installed */
694	if (!no_agp)
695		return;
696
697	if (!amd_nb_has_feature(AMD_NB_GART))
698		return;
699
700	for (i = 0; i < amd_nb_num(); i++) {
701		u32 ctl;
702
703		dev = node_to_amd_nb(i)->misc;
704		pci_read_config_dword(dev, AMD64_GARTAPERTURECTL, &ctl);
705
706		ctl &= ~GARTEN;
707
708		pci_write_config_dword(dev, AMD64_GARTAPERTURECTL, ctl);
709	}
710}
711
712int __init gart_iommu_init(void)
713{
714	struct agp_kern_info info;
715	unsigned long iommu_start;
716	unsigned long aper_base, aper_size;
717	unsigned long start_pfn, end_pfn;
718	unsigned long scratch;
719
720	if (!amd_nb_has_feature(AMD_NB_GART))
721		return 0;
722
723#ifndef CONFIG_AGP_AMD64
724	no_agp = 1;
725#else
726	/* Makefile puts PCI initialization via subsys_initcall first. */
727	/* Add other AMD AGP bridge drivers here */
728	no_agp = no_agp ||
729		(agp_amd64_init() < 0) ||
730		(agp_copy_info(agp_bridge, &info) < 0);
731#endif
732
733	if (no_iommu ||
734	    (!force_iommu && max_pfn <= MAX_DMA32_PFN) ||
735	    !gart_iommu_aperture ||
736	    (no_agp && init_amd_gatt(&info) < 0)) {
737		if (max_pfn > MAX_DMA32_PFN) {
738			pr_warn("More than 4GB of memory but GART IOMMU not available.\n");
739			pr_warn("falling back to iommu=soft.\n");
740		}
741		return 0;
742	}
743
744	/* need to map that range */
745	aper_size	= info.aper_size << 20;
746	aper_base	= info.aper_base;
747	end_pfn		= (aper_base>>PAGE_SHIFT) + (aper_size>>PAGE_SHIFT);
748
749	start_pfn = PFN_DOWN(aper_base);
750	if (!pfn_range_is_mapped(start_pfn, end_pfn))
751		init_memory_mapping(start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT,
752				    PAGE_KERNEL);
753
754	pr_info("PCI-DMA: using GART IOMMU.\n");
755	iommu_size = check_iommu_size(info.aper_base, aper_size);
756	iommu_pages = iommu_size >> PAGE_SHIFT;
757
758	iommu_gart_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
759						      get_order(iommu_pages/8));
760	if (!iommu_gart_bitmap)
761		panic("Cannot allocate iommu bitmap\n");
762
763	pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",
764	       iommu_size >> 20);
765
766	agp_memory_reserved	= iommu_size;
767	iommu_start		= aper_size - iommu_size;
768	iommu_bus_base		= info.aper_base + iommu_start;
769	iommu_gatt_base		= agp_gatt_table + (iommu_start>>PAGE_SHIFT);
770
771	/*
772	 * Unmap the IOMMU part of the GART. The alias of the page is
773	 * always mapped with cache enabled and there is no full cache
774	 * coherency across the GART remapping. The unmapping avoids
775	 * automatic prefetches from the CPU allocating cache lines in
776	 * there. All CPU accesses are done via the direct mapping to
777	 * the backing memory. The GART address is only used by PCI
778	 * devices.
779	 */
780	set_memory_np((unsigned long)__va(iommu_bus_base),
781				iommu_size >> PAGE_SHIFT);
782	/*
783	 * Tricky. The GART table remaps the physical memory range,
784	 * so the CPU won't notice potential aliases and if the memory
785	 * is remapped to UC later on, we might surprise the PCI devices
786	 * with a stray writeout of a cacheline. So play it sure and
787	 * do an explicit, full-scale wbinvd() _after_ having marked all
788	 * the pages as Not-Present:
789	 */
790	wbinvd();
791
792	/*
793	 * Now all caches are flushed and we can safely enable
794	 * GART hardware.  Doing it early leaves the possibility
795	 * of stale cache entries that can lead to GART PTE
796	 * errors.
797	 */
798	enable_gart_translations();
799
800	/*
801	 * Try to workaround a bug (thanks to BenH):
802	 * Set unmapped entries to a scratch page instead of 0.
803	 * Any prefetches that hit unmapped entries won't get an bus abort
804	 * then. (P2P bridge may be prefetching on DMA reads).
805	 */
806	scratch = get_zeroed_page(GFP_KERNEL);
807	if (!scratch)
808		panic("Cannot allocate iommu scratch page");
809	gart_unmapped_entry = GPTE_ENCODE(__pa(scratch));
810
811	flush_gart();
812	dma_ops = &gart_dma_ops;
813	x86_platform.iommu_shutdown = gart_iommu_shutdown;
814	x86_swiotlb_enable = false;
815
816	return 0;
817}
818
819void __init gart_parse_options(char *p)
820{
821	int arg;
822
823	if (isdigit(*p) && get_option(&p, &arg))
824		iommu_size = arg;
825	if (!strncmp(p, "fullflush", 9))
826		iommu_fullflush = 1;
827	if (!strncmp(p, "nofullflush", 11))
828		iommu_fullflush = 0;
829	if (!strncmp(p, "noagp", 5))
830		no_agp = 1;
831	if (!strncmp(p, "noaperture", 10))
832		fix_aperture = 0;
833	/* duplicated from pci-dma.c */
834	if (!strncmp(p, "force", 5))
835		gart_iommu_aperture_allowed = 1;
836	if (!strncmp(p, "allowed", 7))
837		gart_iommu_aperture_allowed = 1;
838	if (!strncmp(p, "memaper", 7)) {
839		fallback_aper_force = 1;
840		p += 7;
841		if (*p == '=') {
842			++p;
843			if (get_option(&p, &arg))
844				fallback_aper_order = arg;
845		}
846	}
847}