pci_iommu.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *	linux/arch/alpha/kernel/pci_iommu.c
  4 */
  5
  6#include <linux/kernel.h>
  7#include <linux/mm.h>
  8#include <linux/pci.h>
  9#include <linux/gfp.h>
 10#include <linux/memblock.h>
 11#include <linux/export.h>
 12#include <linux/scatterlist.h>
 13#include <linux/log2.h>
 14#include <linux/dma-map-ops.h>
 15#include <linux/iommu-helper.h>
 16#include <linux/string_choices.h>
 17
 18#include <asm/io.h>
 19#include <asm/hwrpb.h>
 20
 21#include "proto.h"
 22#include "pci_impl.h"
 23
 24
 25#define DEBUG_ALLOC 0
 26#if DEBUG_ALLOC > 0
 27# define DBGA(args...)		printk(KERN_DEBUG args)
 28#else
 29# define DBGA(args...)
 30#endif
 31#if DEBUG_ALLOC > 1
 32# define DBGA2(args...)		printk(KERN_DEBUG args)
 33#else
 34# define DBGA2(args...)
 35#endif
 36
 37#define DEBUG_NODIRECT 0
 38
 39#define ISA_DMA_MASK		0x00ffffff
 40
 41static inline unsigned long
 42mk_iommu_pte(unsigned long paddr)
 43{
 44	return (paddr >> (PAGE_SHIFT-1)) | 1;
 45}
 46
 47/* Return the minimum of MAX or the first power of two larger
 48   than main memory.  */
 49
 50unsigned long
 51size_for_memory(unsigned long max)
 52{
 53	unsigned long mem = max_low_pfn << PAGE_SHIFT;
 54	if (mem < max)
 55		max = roundup_pow_of_two(mem);
 56	return max;
 57}
 58
 59struct pci_iommu_arena * __init
 60iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base,
 61		     unsigned long window_size, unsigned long align)
 62{
 63	unsigned long mem_size;
 64	struct pci_iommu_arena *arena;
 65
 66	mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long));
 67
 68	/* Note that the TLB lookup logic uses bitwise concatenation,
 69	   not addition, so the required arena alignment is based on
 70	   the size of the window.  Retain the align parameter so that
 71	   particular systems can over-align the arena.  */
 72	if (align < mem_size)
 73		align = mem_size;
 74
 75	arena = memblock_alloc_or_panic(sizeof(*arena), SMP_CACHE_BYTES);
 76	arena->ptes = memblock_alloc_or_panic(mem_size, align);
 77
 78	spin_lock_init(&arena->lock);
 79	arena->hose = hose;
 80	arena->dma_base = base;
 81	arena->size = window_size;
 82	arena->next_entry = 0;
 83
 84	/* Align allocations to a multiple of a page size.  Not needed
 85	   unless there are chip bugs.  */
 86	arena->align_entry = 1;
 87
 88	return arena;
 89}
 90
 91struct pci_iommu_arena * __init
 92iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
 93		unsigned long window_size, unsigned long align)
 94{
 95	return iommu_arena_new_node(0, hose, base, window_size, align);
 96}
 97
 98/* Must be called with the arena lock held */
 99static long
100iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
101		       long n, long mask)
102{
103	unsigned long *ptes;
104	long i, p, nent;
105	int pass = 0;
106	unsigned long base;
107	unsigned long boundary_size;
108
109	base = arena->dma_base >> PAGE_SHIFT;
110	boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT);
111
112	/* Search forward for the first mask-aligned sequence of N free ptes */
113	ptes = arena->ptes;
114	nent = arena->size >> PAGE_SHIFT;
115	p = ALIGN(arena->next_entry, mask + 1);
116	i = 0;
117
118again:
119	while (i < n && p+i < nent) {
120		if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) {
121			p = ALIGN(p + 1, mask + 1);
122			goto again;
123		}
124
125		if (ptes[p+i]) {
126			p = ALIGN(p + i + 1, mask + 1);
127			i = 0;
128		} else {
129			i = i + 1;
130		}
131	}
132
133	if (i < n) {
134		if (pass < 1) {
135			/*
136			 * Reached the end.  Flush the TLB and restart
137			 * the search from the beginning.
138			*/
139			alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
140
141			pass++;
142			p = 0;
143			i = 0;
144			goto again;
145		} else
146			return -1;
147	}
148
149	/* Success. It's the responsibility of the caller to mark them
150	   in use before releasing the lock */
151	return p;
152}
153
154static long
155iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
156		  unsigned int align)
157{
158	unsigned long flags;
159	unsigned long *ptes;
160	long i, p, mask;
161
162	spin_lock_irqsave(&arena->lock, flags);
163
164	/* Search for N empty ptes */
165	ptes = arena->ptes;
166	mask = max(align, arena->align_entry) - 1;
167	p = iommu_arena_find_pages(dev, arena, n, mask);
168	if (p < 0) {
169		spin_unlock_irqrestore(&arena->lock, flags);
170		return -1;
171	}
172
173	/* Success.  Mark them all in use, ie not zero and invalid
174	   for the iommu tlb that could load them from under us.
175	   The chip specific bits will fill this in with something
176	   kosher when we return.  */
177	for (i = 0; i < n; ++i)
178		ptes[p+i] = IOMMU_INVALID_PTE;
179
180	arena->next_entry = p + n;
181	spin_unlock_irqrestore(&arena->lock, flags);
182
183	return p;
184}
185
186static void
187iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n)
188{
189	unsigned long *p;
190	long i;
191
192	p = arena->ptes + ofs;
193	for (i = 0; i < n; ++i)
194		p[i] = 0;
195}
196
197/*
198 * True if the machine supports DAC addressing, and DEV can
199 * make use of it given MASK.
200 */
201static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
202{
203	dma_addr_t dac_offset = alpha_mv.pci_dac_offset;
204	int ok = 1;
205
206	/* If this is not set, the machine doesn't support DAC at all.  */
207	if (dac_offset == 0)
208		ok = 0;
209
210	/* The device has to be able to address our DAC bit.  */
211	if ((dac_offset & dev->dma_mask) != dac_offset)
212		ok = 0;
213
214	/* If both conditions above are met, we are fine. */
215	DBGA("pci_dac_dma_supported %s from %ps\n",
216	     str_yes_no(ok), __builtin_return_address(0));
217
218	return ok;
219}
220
221/* Map a single buffer of the indicated size for PCI DMA in streaming
222   mode.  The 32-bit PCI bus mastering address to use is returned.
223   Once the device is given the dma address, the device owns this memory
224   until either pci_unmap_single or pci_dma_sync_single is performed.  */
225
226static dma_addr_t
227pci_map_single_1(struct pci_dev *pdev, phys_addr_t paddr, size_t size,
228		 int dac_allowed)
229{
230	struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
231	dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
232	unsigned long offset = offset_in_page(paddr);
233	struct pci_iommu_arena *arena;
234	long npages, dma_ofs, i;
235	dma_addr_t ret;
236	unsigned int align = 0;
237	struct device *dev = pdev ? &pdev->dev : NULL;
238
239#if !DEBUG_NODIRECT
240	/* First check to see if we can use the direct map window.  */
241	if (paddr + size + __direct_map_base - 1 <= max_dma
242	    && paddr + size <= __direct_map_size) {
243		ret = paddr + __direct_map_base;
244
245		DBGA2("pci_map_single: [%pa,%zx] -> direct %llx from %ps\n",
246		      &paddr, size, ret, __builtin_return_address(0));
247
248		return ret;
249	}
250#endif
251
252	/* Next, use DAC if selected earlier.  */
253	if (dac_allowed) {
254		ret = paddr + alpha_mv.pci_dac_offset;
255
256		DBGA2("pci_map_single: [%pa,%zx] -> DAC %llx from %ps\n",
257		      &paddr, size, ret, __builtin_return_address(0));
258
259		return ret;
260	}
261
262	/* If the machine doesn't define a pci_tbi routine, we have to
263	   assume it doesn't support sg mapping, and, since we tried to
264	   use direct_map above, it now must be considered an error. */
265	if (! alpha_mv.mv_pci_tbi) {
266		printk_once(KERN_WARNING "pci_map_single: no HW sg\n");
267		return DMA_MAPPING_ERROR;
268	}
269
270	arena = hose->sg_pci;
271	if (!arena || arena->dma_base + arena->size - 1 > max_dma)
272		arena = hose->sg_isa;
273
274	npages = iommu_num_pages(paddr, size, PAGE_SIZE);
275
276	/* Force allocation to 64KB boundary for ISA bridges. */
277	if (pdev && pdev == isa_bridge)
278		align = 8;
279	dma_ofs = iommu_arena_alloc(dev, arena, npages, align);
280	if (dma_ofs < 0) {
281		printk(KERN_WARNING "pci_map_single failed: "
282		       "could not allocate dma page tables\n");
283		return DMA_MAPPING_ERROR;
284	}
285
286	paddr &= PAGE_MASK;
287	for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
288		arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr);
289
290	ret = arena->dma_base + dma_ofs * PAGE_SIZE;
291	ret += offset;
292
293	DBGA2("pci_map_single: [%pa,%zx] np %ld -> sg %llx from %ps\n",
294	      &paddr, size, npages, ret, __builtin_return_address(0));
295
296	return ret;
297}
298
299/* Helper for generic DMA-mapping functions. */
300static struct pci_dev *alpha_gendev_to_pci(struct device *dev)
301{
302	if (dev && dev_is_pci(dev))
303		return to_pci_dev(dev);
304
305	/* Assume that non-PCI devices asking for DMA are either ISA or EISA,
306	   BUG() otherwise. */
307	BUG_ON(!isa_bridge);
308
309	/* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA
310	   bridge is bus master then). */
311	if (!dev || !dev->dma_mask || !*dev->dma_mask)
312		return isa_bridge;
313
314	/* For EISA bus masters, return isa_bridge (it might have smaller
315	   dma_mask due to wiring limitations). */
316	if (*dev->dma_mask >= isa_bridge->dma_mask)
317		return isa_bridge;
318
319	/* This assumes ISA bus master with dma_mask 0xffffff. */
320	return NULL;
321}
322
323static dma_addr_t alpha_pci_map_phys(struct device *dev, phys_addr_t phys,
324				     size_t size, enum dma_data_direction dir,
325				     unsigned long attrs)
326{
327	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
328	int dac_allowed;
329
330	if (unlikely(attrs & DMA_ATTR_MMIO))
331		return DMA_MAPPING_ERROR;
332
333	dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
334	return pci_map_single_1(pdev, phys, size, dac_allowed);
335}
336
337/* Unmap a single streaming mode DMA translation.  The DMA_ADDR and
338   SIZE must match what was provided for in a previous pci_map_single
339   call.  All other usages are undefined.  After this call, reads by
340   the cpu to the buffer are guaranteed to see whatever the device
341   wrote there.  */
342
343static void alpha_pci_unmap_phys(struct device *dev, dma_addr_t dma_addr,
344				 size_t size, enum dma_data_direction dir,
345				 unsigned long attrs)
346{
347	unsigned long flags;
348	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
349	struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
350	struct pci_iommu_arena *arena;
351	long dma_ofs, npages;
352
353	if (dma_addr >= __direct_map_base
354	    && dma_addr < __direct_map_base + __direct_map_size) {
355		/* Nothing to do.  */
356
357		DBGA2("pci_unmap_single: direct [%llx,%zx] from %ps\n",
358		      dma_addr, size, __builtin_return_address(0));
359
360		return;
361	}
362
363	if (dma_addr > 0xffffffff) {
364		DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %ps\n",
365		      dma_addr, size, __builtin_return_address(0));
366		return;
367	}
368
369	arena = hose->sg_pci;
370	if (!arena || dma_addr < arena->dma_base)
371		arena = hose->sg_isa;
372
373	dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
374	if (dma_ofs * PAGE_SIZE >= arena->size) {
375		printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx "
376		       " base %llx size %x\n",
377		       dma_addr, arena->dma_base, arena->size);
378		return;
379		BUG();
380	}
381
382	npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
383
384	spin_lock_irqsave(&arena->lock, flags);
385
386	iommu_arena_free(arena, dma_ofs, npages);
387
388        /* If we're freeing ptes above the `next_entry' pointer (they
389           may have snuck back into the TLB since the last wrap flush),
390           we need to flush the TLB before reallocating the latter.  */
391	if (dma_ofs >= arena->next_entry)
392		alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
393
394	spin_unlock_irqrestore(&arena->lock, flags);
395
396	DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %ps\n",
397	      dma_addr, size, npages, __builtin_return_address(0));
398}
399
400/* Allocate and map kernel buffer using consistent mode DMA for PCI
401   device.  Returns non-NULL cpu-view pointer to the buffer if
402   successful and sets *DMA_ADDRP to the pci side dma address as well,
403   else DMA_ADDRP is undefined.  */
404
405static void *alpha_pci_alloc_coherent(struct device *dev, size_t size,
406				      dma_addr_t *dma_addrp, gfp_t gfp,
407				      unsigned long attrs)
408{
409	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
410	void *cpu_addr;
411	long order = get_order(size);
412
413	gfp &= ~GFP_DMA;
414
415try_again:
416	cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order);
417	if (! cpu_addr) {
418		printk(KERN_INFO "pci_alloc_consistent: "
419		       "get_free_pages failed from %ps\n",
420			__builtin_return_address(0));
421		/* ??? Really atomic allocation?  Otherwise we could play
422		   with vmalloc and sg if we can't find contiguous memory.  */
423		return NULL;
424	}
425	memset(cpu_addr, 0, size);
426
427	*dma_addrp = pci_map_single_1(pdev, virt_to_phys(cpu_addr), size, 0);
428	if (*dma_addrp == DMA_MAPPING_ERROR) {
429		free_pages((unsigned long)cpu_addr, order);
430		if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA))
431			return NULL;
432		/* The address doesn't fit required mask and we
433		   do not have iommu. Try again with GFP_DMA. */
434		gfp |= GFP_DMA;
435		goto try_again;
436	}
437
438	DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n",
439	      size, cpu_addr, *dma_addrp, __builtin_return_address(0));
440
441	return cpu_addr;
442}
443
444/* Free and unmap a consistent DMA buffer.  CPU_ADDR and DMA_ADDR must
445   be values that were returned from pci_alloc_consistent.  SIZE must
446   be the same as what as passed into pci_alloc_consistent.
447   References to the memory and mappings associated with CPU_ADDR or
448   DMA_ADDR past this call are illegal.  */
449
450static void alpha_pci_free_coherent(struct device *dev, size_t size,
451				    void *cpu_addr, dma_addr_t dma_addr,
452				    unsigned long attrs)
453{
454	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
455	dma_unmap_single(&pdev->dev, dma_addr, size, DMA_BIDIRECTIONAL);
456	free_pages((unsigned long)cpu_addr, get_order(size));
457
458	DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n",
459	      dma_addr, size, __builtin_return_address(0));
460}
461
462/* Classify the elements of the scatterlist.  Write dma_address
463   of each element with:
464	0   : Followers all physically adjacent.
465	1   : Followers all virtually adjacent.
466	-1  : Not leader, physically adjacent to previous.
467	-2  : Not leader, virtually adjacent to previous.
468   Write dma_length of each leader with the combined lengths of
469   the mergable followers.  */
470
471#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG)))
472#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG))
473
474static void
475sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end,
476	    int virt_ok)
477{
478	unsigned long next_paddr;
479	struct scatterlist *leader;
480	long leader_flag, leader_length;
481	unsigned int max_seg_size;
482
483	leader = sg;
484	leader_flag = 0;
485	leader_length = leader->length;
486	next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
487
488	/* we will not marge sg without device. */
489	max_seg_size = dev ? dma_get_max_seg_size(dev) : 0;
490	for (++sg; sg < end; ++sg) {
491		unsigned long addr, len;
492		addr = SG_ENT_PHYS_ADDRESS(sg);
493		len = sg->length;
494
495		if (leader_length + len > max_seg_size)
496			goto new_segment;
497
498		if (next_paddr == addr) {
499			sg->dma_address = -1;
500			leader_length += len;
501		} else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
502			sg->dma_address = -2;
503			leader_flag = 1;
504			leader_length += len;
505		} else {
506new_segment:
507			leader->dma_address = leader_flag;
508			leader->dma_length = leader_length;
509			leader = sg;
510			leader_flag = 0;
511			leader_length = len;
512		}
513
514		next_paddr = addr + len;
515	}
516
517	leader->dma_address = leader_flag;
518	leader->dma_length = leader_length;
519}
520
521/* Given a scatterlist leader, choose an allocation method and fill
522   in the blanks.  */
523
524static int
525sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end,
526	struct scatterlist *out, struct pci_iommu_arena *arena,
527	dma_addr_t max_dma, int dac_allowed)
528{
529	unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
530	long size = leader->dma_length;
531	struct scatterlist *sg;
532	unsigned long *ptes;
533	long npages, dma_ofs, i;
534
535#if !DEBUG_NODIRECT
536	/* If everything is physically contiguous, and the addresses
537	   fall into the direct-map window, use it.  */
538	if (leader->dma_address == 0
539	    && paddr + size + __direct_map_base - 1 <= max_dma
540	    && paddr + size <= __direct_map_size) {
541		out->dma_address = paddr + __direct_map_base;
542		out->dma_length = size;
543
544		DBGA("    sg_fill: [%p,%lx] -> direct %llx\n",
545		     __va(paddr), size, out->dma_address);
546
547		return 0;
548	}
549#endif
550
551	/* If physically contiguous and DAC is available, use it.  */
552	if (leader->dma_address == 0 && dac_allowed) {
553		out->dma_address = paddr + alpha_mv.pci_dac_offset;
554		out->dma_length = size;
555
556		DBGA("    sg_fill: [%p,%lx] -> DAC %llx\n",
557		     __va(paddr), size, out->dma_address);
558
559		return 0;
560	}
561
562	/* Otherwise, we'll use the iommu to make the pages virtually
563	   contiguous.  */
564
565	paddr &= ~PAGE_MASK;
566	npages = iommu_num_pages(paddr, size, PAGE_SIZE);
567	dma_ofs = iommu_arena_alloc(dev, arena, npages, 0);
568	if (dma_ofs < 0) {
569		/* If we attempted a direct map above but failed, die.  */
570		if (leader->dma_address == 0)
571			return -1;
572
573		/* Otherwise, break up the remaining virtually contiguous
574		   hunks into individual direct maps and retry.  */
575		sg_classify(dev, leader, end, 0);
576		return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed);
577	}
578
579	out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
580	out->dma_length = size;
581
582	DBGA("    sg_fill: [%p,%lx] -> sg %llx np %ld\n",
583	     __va(paddr), size, out->dma_address, npages);
584
585	/* All virtually contiguous.  We need to find the length of each
586	   physically contiguous subsegment to fill in the ptes.  */
587	ptes = &arena->ptes[dma_ofs];
588	sg = leader;
589	do {
590#if DEBUG_ALLOC > 0
591		struct scatterlist *last_sg = sg;
592#endif
593
594		size = sg->length;
595		paddr = SG_ENT_PHYS_ADDRESS(sg);
596
597		while (sg+1 < end && (int) sg[1].dma_address == -1) {
598			size += sg[1].length;
599			sg = sg_next(sg);
600		}
601
602		npages = iommu_num_pages(paddr, size, PAGE_SIZE);
603
604		paddr &= PAGE_MASK;
605		for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
606			*ptes++ = mk_iommu_pte(paddr);
607
608#if DEBUG_ALLOC > 0
609		DBGA("    (%ld) [%p,%x] np %ld\n",
610		     last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
611		     last_sg->length, npages);
612		while (++last_sg <= sg) {
613			DBGA("        (%ld) [%p,%x] cont\n",
614			     last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
615			     last_sg->length);
616		}
617#endif
618	} while (++sg < end && (int) sg->dma_address < 0);
619
620	return 1;
621}
622
623static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg,
624			    int nents, enum dma_data_direction dir,
625			    unsigned long attrs)
626{
627	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
628	struct scatterlist *start, *end, *out;
629	struct pci_controller *hose;
630	struct pci_iommu_arena *arena;
631	dma_addr_t max_dma;
632	int dac_allowed;
633
634	BUG_ON(dir == DMA_NONE);
635
636	dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
637
638	/* Fast path single entry scatterlists.  */
639	if (nents == 1) {
640		sg->dma_length = sg->length;
641		sg->dma_address = pci_map_single_1(pdev, sg_phys(sg),
642						   sg->length, dac_allowed);
643		if (sg->dma_address == DMA_MAPPING_ERROR)
644			return -EIO;
645		return 1;
646	}
647
648	start = sg;
649	end = sg + nents;
650
651	/* First, prepare information about the entries.  */
652	sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0);
653
654	/* Second, figure out where we're going to map things.  */
655	if (alpha_mv.mv_pci_tbi) {
656		hose = pdev ? pdev->sysdata : pci_isa_hose;
657		max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
658		arena = hose->sg_pci;
659		if (!arena || arena->dma_base + arena->size - 1 > max_dma)
660			arena = hose->sg_isa;
661	} else {
662		max_dma = -1;
663		arena = NULL;
664		hose = NULL;
665	}
666
667	/* Third, iterate over the scatterlist leaders and allocate
668	   dma space as needed.  */
669	for (out = sg; sg < end; ++sg) {
670		if ((int) sg->dma_address < 0)
671			continue;
672		if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0)
673			goto error;
674		out++;
675	}
676
677	/* Mark the end of the list for pci_unmap_sg.  */
678	if (out < end)
679		out->dma_length = 0;
680
681	if (out - start == 0) {
682		printk(KERN_WARNING "pci_map_sg failed: no entries?\n");
683		return -ENOMEM;
684	}
685	DBGA("pci_map_sg: %ld entries\n", out - start);
686
687	return out - start;
688
689 error:
690	printk(KERN_WARNING "pci_map_sg failed: "
691	       "could not allocate dma page tables\n");
692
693	/* Some allocation failed while mapping the scatterlist
694	   entries.  Unmap them now.  */
695	if (out > start)
696		dma_unmap_sg(&pdev->dev, start, out - start, dir);
697	return -ENOMEM;
698}
699
700/* Unmap a set of streaming mode DMA translations.  Again, cpu read
701   rules concerning calls here are the same as for pci_unmap_single()
702   above.  */
703
704static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg,
705			       int nents, enum dma_data_direction dir,
706			       unsigned long attrs)
707{
708	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
709	unsigned long flags;
710	struct pci_controller *hose;
711	struct pci_iommu_arena *arena;
712	struct scatterlist *end;
713	dma_addr_t max_dma;
714	dma_addr_t fbeg, fend;
715
716	BUG_ON(dir == DMA_NONE);
717
718	if (! alpha_mv.mv_pci_tbi)
719		return;
720
721	hose = pdev ? pdev->sysdata : pci_isa_hose;
722	max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
723	arena = hose->sg_pci;
724	if (!arena || arena->dma_base + arena->size - 1 > max_dma)
725		arena = hose->sg_isa;
726
727	fbeg = -1, fend = 0;
728
729	spin_lock_irqsave(&arena->lock, flags);
730
731	for (end = sg + nents; sg < end; ++sg) {
732		dma_addr_t addr;
733		size_t size;
734		long npages, ofs;
735		dma_addr_t tend;
736
737		addr = sg->dma_address;
738		size = sg->dma_length;
739		if (!size)
740			break;
741
742		if (addr > 0xffffffff) {
743			/* It's a DAC address -- nothing to do.  */
744			DBGA("    (%ld) DAC [%llx,%zx]\n",
745			      sg - end + nents, addr, size);
746			continue;
747		}
748
749		if (addr >= __direct_map_base
750		    && addr < __direct_map_base + __direct_map_size) {
751			/* Nothing to do.  */
752			DBGA("    (%ld) direct [%llx,%zx]\n",
753			      sg - end + nents, addr, size);
754			continue;
755		}
756
757		DBGA("    (%ld) sg [%llx,%zx]\n",
758		     sg - end + nents, addr, size);
759
760		npages = iommu_num_pages(addr, size, PAGE_SIZE);
761		ofs = (addr - arena->dma_base) >> PAGE_SHIFT;
762		iommu_arena_free(arena, ofs, npages);
763
764		tend = addr + size - 1;
765		if (fbeg > addr) fbeg = addr;
766		if (fend < tend) fend = tend;
767	}
768
769        /* If we're freeing ptes above the `next_entry' pointer (they
770           may have snuck back into the TLB since the last wrap flush),
771           we need to flush the TLB before reallocating the latter.  */
772	if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
773		alpha_mv.mv_pci_tbi(hose, fbeg, fend);
774
775	spin_unlock_irqrestore(&arena->lock, flags);
776
777	DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
778}
779
780/* Return whether the given PCI device DMA address mask can be
781   supported properly.  */
782
783static int alpha_pci_supported(struct device *dev, u64 mask)
784{
785	struct pci_dev *pdev = alpha_gendev_to_pci(dev);
786	struct pci_controller *hose;
787	struct pci_iommu_arena *arena;
788
789	/* If there exists a direct map, and the mask fits either
790	   the entire direct mapped space or the total system memory as
791	   shifted by the map base */
792	if (__direct_map_size != 0
793	    && (__direct_map_base + __direct_map_size - 1 <= mask ||
794		__direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask))
795		return 1;
796
797	/* Check that we have a scatter-gather arena that fits.  */
798	hose = pdev ? pdev->sysdata : pci_isa_hose;
799	arena = hose->sg_isa;
800	if (arena && arena->dma_base + arena->size - 1 <= mask)
801		return 1;
802	arena = hose->sg_pci;
803	if (arena && arena->dma_base + arena->size - 1 <= mask)
804		return 1;
805
806	/* As last resort try ZONE_DMA.  */
807	if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask)
808		return 1;
809
810	return 0;
811}
812
813
814/*
815 * AGP GART extensions to the IOMMU
816 */
817int
818iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask) 
819{
820	unsigned long flags;
821	unsigned long *ptes;
822	long i, p;
823
824	if (!arena) return -EINVAL;
825
826	spin_lock_irqsave(&arena->lock, flags);
827
828	/* Search for N empty ptes.  */
829	ptes = arena->ptes;
830	p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask);
831	if (p < 0) {
832		spin_unlock_irqrestore(&arena->lock, flags);
833		return -1;
834	}
835
836	/* Success.  Mark them all reserved (ie not zero and invalid)
837	   for the iommu tlb that could load them from under us.
838	   They will be filled in with valid bits by _bind() */
839	for (i = 0; i < pg_count; ++i)
840		ptes[p+i] = IOMMU_RESERVED_PTE;
841
842	arena->next_entry = p + pg_count;
843	spin_unlock_irqrestore(&arena->lock, flags);
844
845	return p;
846}
847
848int 
849iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count)
850{
851	unsigned long *ptes;
852	long i;
853
854	if (!arena) return -EINVAL;
855
856	ptes = arena->ptes;
857
858	/* Make sure they're all reserved first... */
859	for(i = pg_start; i < pg_start + pg_count; i++)
860		if (ptes[i] != IOMMU_RESERVED_PTE)
861			return -EBUSY;
862
863	iommu_arena_free(arena, pg_start, pg_count);
864	return 0;
865}
866
867int
868iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count, 
869	   struct page **pages)
870{
871	unsigned long flags;
872	unsigned long *ptes;
873	long i, j;
874
875	if (!arena) return -EINVAL;
876	
877	spin_lock_irqsave(&arena->lock, flags);
878
879	ptes = arena->ptes;
880
881	for(j = pg_start; j < pg_start + pg_count; j++) {
882		if (ptes[j] != IOMMU_RESERVED_PTE) {
883			spin_unlock_irqrestore(&arena->lock, flags);
884			return -EBUSY;
885		}
886	}
887		
888	for(i = 0, j = pg_start; i < pg_count; i++, j++)
889		ptes[j] = mk_iommu_pte(page_to_phys(pages[i]));
890
891	spin_unlock_irqrestore(&arena->lock, flags);
892
893	return 0;
894}
895
896int
897iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count)
898{
899	unsigned long *p;
900	long i;
901
902	if (!arena) return -EINVAL;
903
904	p = arena->ptes + pg_start;
905	for(i = 0; i < pg_count; i++)
906		p[i] = IOMMU_RESERVED_PTE;
907
908	return 0;
909}
910
911const struct dma_map_ops alpha_pci_ops = {
912	.alloc			= alpha_pci_alloc_coherent,
913	.free			= alpha_pci_free_coherent,
914	.map_phys		= alpha_pci_map_phys,
915	.unmap_phys		= alpha_pci_unmap_phys,
916	.map_sg			= alpha_pci_map_sg,
917	.unmap_sg		= alpha_pci_unmap_sg,
918	.dma_supported		= alpha_pci_supported,
919	.mmap			= dma_common_mmap,
920	.get_sgtable		= dma_common_get_sgtable,
921	.alloc_pages_op		= dma_common_alloc_pages,
922	.free_pages		= dma_common_free_pages,
923};
924EXPORT_SYMBOL(alpha_pci_ops);