tlb-r4k.c

  1/*
  2 * This file is subject to the terms and conditions of the GNU General Public
  3 * License.  See the file "COPYING" in the main directory of this archive
  4 * for more details.
  5 *
  6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
  7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
  8 * Carsten Langgaard, carstenl@mips.com
  9 * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
 10 */
 11#include <linux/cpu_pm.h>
 12#include <linux/init.h>
 13#include <linux/sched.h>
 14#include <linux/smp.h>
 15#include <linux/memblock.h>
 16#include <linux/minmax.h>
 17#include <linux/mm.h>
 18#include <linux/hugetlb.h>
 19#include <linux/export.h>
 20#include <linux/sort.h>
 21
 22#include <asm/cpu.h>
 23#include <asm/cpu-type.h>
 24#include <asm/bootinfo.h>
 25#include <asm/hazards.h>
 26#include <asm/mmu_context.h>
 27#include <asm/tlb.h>
 28#include <asm/tlbdebug.h>
 29#include <asm/tlbex.h>
 30#include <asm/tlbmisc.h>
 31#include <asm/setup.h>
 32
 33/*
 34 * LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has
 35 * a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately,
 36 * itlb/dtlb are not totally transparent to software.
 37 */
 38static inline void flush_micro_tlb(void)
 39{
 40	switch (current_cpu_type()) {
 41	case CPU_LOONGSON2EF:
 42		write_c0_diag(LOONGSON_DIAG_ITLB);
 43		break;
 44	case CPU_LOONGSON64:
 45		write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
 46		break;
 47	default:
 48		break;
 49	}
 50}
 51
 52static inline void flush_micro_tlb_vm(struct vm_area_struct *vma)
 53{
 54	if (vma->vm_flags & VM_EXEC)
 55		flush_micro_tlb();
 56}
 57
 58void local_flush_tlb_all(void)
 59{
 60	unsigned long flags;
 61	unsigned long old_ctx;
 62	int entry, ftlbhighset;
 63
 64	local_irq_save(flags);
 65	/* Save old context and create impossible VPN2 value */
 66	old_ctx = read_c0_entryhi();
 67	htw_stop();
 68	write_c0_entrylo0(0);
 69	write_c0_entrylo1(0);
 70
 71	entry = num_wired_entries();
 72
 73	/*
 74	 * Blast 'em all away.
 75	 * If there are any wired entries, fall back to iterating
 76	 */
 77	if (cpu_has_tlbinv && !entry) {
 78		if (current_cpu_data.tlbsizevtlb) {
 79			write_c0_index(0);
 80			mtc0_tlbw_hazard();
 81			tlbinvf();  /* invalidate VTLB */
 82		}
 83		ftlbhighset = current_cpu_data.tlbsizevtlb +
 84			current_cpu_data.tlbsizeftlbsets;
 85		for (entry = current_cpu_data.tlbsizevtlb;
 86		     entry < ftlbhighset;
 87		     entry++) {
 88			write_c0_index(entry);
 89			mtc0_tlbw_hazard();
 90			tlbinvf();  /* invalidate one FTLB set */
 91		}
 92	} else {
 93		while (entry < current_cpu_data.tlbsize) {
 94			/* Make sure all entries differ. */
 95			write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 96			write_c0_index(entry);
 97			mtc0_tlbw_hazard();
 98			tlb_write_indexed();
 99			entry++;
100		}
101	}
102	tlbw_use_hazard();
103	write_c0_entryhi(old_ctx);
104	htw_start();
105	flush_micro_tlb();
106	local_irq_restore(flags);
107}
108EXPORT_SYMBOL(local_flush_tlb_all);
109
110void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
111	unsigned long end)
112{
113	struct mm_struct *mm = vma->vm_mm;
114	int cpu = smp_processor_id();
115
116	if (cpu_context(cpu, mm) != 0) {
117		unsigned long size, flags;
118
119		local_irq_save(flags);
120		start = round_down(start, PAGE_SIZE << 1);
121		end = round_up(end, PAGE_SIZE << 1);
122		size = (end - start) >> (PAGE_SHIFT + 1);
123		if (size <= (current_cpu_data.tlbsizeftlbsets ?
124			     current_cpu_data.tlbsize / 8 :
125			     current_cpu_data.tlbsize / 2)) {
126			unsigned long old_entryhi, old_mmid;
127			int newpid = cpu_asid(cpu, mm);
128
129			old_entryhi = read_c0_entryhi();
130			if (cpu_has_mmid) {
131				old_mmid = read_c0_memorymapid();
132				write_c0_memorymapid(newpid);
133			}
134
135			htw_stop();
136			while (start < end) {
137				int idx;
138
139				if (cpu_has_mmid)
140					write_c0_entryhi(start);
141				else
142					write_c0_entryhi(start | newpid);
143				start += (PAGE_SIZE << 1);
144				mtc0_tlbw_hazard();
145				tlb_probe();
146				tlb_probe_hazard();
147				idx = read_c0_index();
148				write_c0_entrylo0(0);
149				write_c0_entrylo1(0);
150				if (idx < 0)
151					continue;
152				/* Make sure all entries differ. */
153				write_c0_entryhi(UNIQUE_ENTRYHI(idx));
154				mtc0_tlbw_hazard();
155				tlb_write_indexed();
156			}
157			tlbw_use_hazard();
158			write_c0_entryhi(old_entryhi);
159			if (cpu_has_mmid)
160				write_c0_memorymapid(old_mmid);
161			htw_start();
162		} else {
163			drop_mmu_context(mm);
164		}
165		flush_micro_tlb();
166		local_irq_restore(flags);
167	}
168}
169
170void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
171{
172	unsigned long size, flags;
173
174	local_irq_save(flags);
175	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
176	size = (size + 1) >> 1;
177	if (size <= (current_cpu_data.tlbsizeftlbsets ?
178		     current_cpu_data.tlbsize / 8 :
179		     current_cpu_data.tlbsize / 2)) {
180		int pid = read_c0_entryhi();
181
182		start &= (PAGE_MASK << 1);
183		end += ((PAGE_SIZE << 1) - 1);
184		end &= (PAGE_MASK << 1);
185		htw_stop();
186
187		while (start < end) {
188			int idx;
189
190			write_c0_entryhi(start);
191			start += (PAGE_SIZE << 1);
192			mtc0_tlbw_hazard();
193			tlb_probe();
194			tlb_probe_hazard();
195			idx = read_c0_index();
196			write_c0_entrylo0(0);
197			write_c0_entrylo1(0);
198			if (idx < 0)
199				continue;
200			/* Make sure all entries differ. */
201			write_c0_entryhi(UNIQUE_ENTRYHI(idx));
202			mtc0_tlbw_hazard();
203			tlb_write_indexed();
204		}
205		tlbw_use_hazard();
206		write_c0_entryhi(pid);
207		htw_start();
208	} else {
209		local_flush_tlb_all();
210	}
211	flush_micro_tlb();
212	local_irq_restore(flags);
213}
214
215void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
216{
217	int cpu = smp_processor_id();
218
219	if (cpu_context(cpu, vma->vm_mm) != 0) {
220		unsigned long old_mmid;
221		unsigned long flags, old_entryhi;
222		int idx;
223
224		page &= (PAGE_MASK << 1);
225		local_irq_save(flags);
226		old_entryhi = read_c0_entryhi();
227		htw_stop();
228		if (cpu_has_mmid) {
229			old_mmid = read_c0_memorymapid();
230			write_c0_entryhi(page);
231			write_c0_memorymapid(cpu_asid(cpu, vma->vm_mm));
232		} else {
233			write_c0_entryhi(page | cpu_asid(cpu, vma->vm_mm));
234		}
235		mtc0_tlbw_hazard();
236		tlb_probe();
237		tlb_probe_hazard();
238		idx = read_c0_index();
239		write_c0_entrylo0(0);
240		write_c0_entrylo1(0);
241		if (idx < 0)
242			goto finish;
243		/* Make sure all entries differ. */
244		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
245		mtc0_tlbw_hazard();
246		tlb_write_indexed();
247		tlbw_use_hazard();
248
249	finish:
250		write_c0_entryhi(old_entryhi);
251		if (cpu_has_mmid)
252			write_c0_memorymapid(old_mmid);
253		htw_start();
254		flush_micro_tlb_vm(vma);
255		local_irq_restore(flags);
256	}
257}
258
259/*
260 * This one is only used for pages with the global bit set so we don't care
261 * much about the ASID.
262 */
263void local_flush_tlb_one(unsigned long page)
264{
265	unsigned long flags;
266	int oldpid, idx;
267
268	local_irq_save(flags);
269	oldpid = read_c0_entryhi();
270	htw_stop();
271	page &= (PAGE_MASK << 1);
272	write_c0_entryhi(page);
273	mtc0_tlbw_hazard();
274	tlb_probe();
275	tlb_probe_hazard();
276	idx = read_c0_index();
277	write_c0_entrylo0(0);
278	write_c0_entrylo1(0);
279	if (idx >= 0) {
280		/* Make sure all entries differ. */
281		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
282		mtc0_tlbw_hazard();
283		tlb_write_indexed();
284		tlbw_use_hazard();
285	}
286	write_c0_entryhi(oldpid);
287	htw_start();
288	flush_micro_tlb();
289	local_irq_restore(flags);
290}
291
292/*
293 * We will need multiple versions of update_mmu_cache(), one that just
294 * updates the TLB with the new pte(s), and another which also checks
295 * for the R4k "end of page" hardware bug and does the needy.
296 */
297void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
298{
299	unsigned long flags;
300	pgd_t *pgdp;
301	p4d_t *p4dp;
302	pud_t *pudp;
303	pmd_t *pmdp;
304	pte_t *ptep, *ptemap = NULL;
305	int idx, pid;
306
307	/*
308	 * Handle debugger faulting in for debuggee.
309	 */
310	if (current->active_mm != vma->vm_mm)
311		return;
312
313	local_irq_save(flags);
314
315	htw_stop();
316	address &= (PAGE_MASK << 1);
317	if (cpu_has_mmid) {
318		write_c0_entryhi(address);
319	} else {
320		pid = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
321		write_c0_entryhi(address | pid);
322	}
323	pgdp = pgd_offset(vma->vm_mm, address);
324	mtc0_tlbw_hazard();
325	tlb_probe();
326	tlb_probe_hazard();
327	p4dp = p4d_offset(pgdp, address);
328	pudp = pud_offset(p4dp, address);
329	pmdp = pmd_offset(pudp, address);
330	idx = read_c0_index();
331#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
332	/* this could be a huge page  */
333	if (pmd_leaf(*pmdp)) {
334		unsigned long lo;
335		write_c0_pagemask(PM_HUGE_MASK);
336		ptep = (pte_t *)pmdp;
337		lo = pte_to_entrylo(pte_val(*ptep));
338		write_c0_entrylo0(lo);
339		write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
340
341		mtc0_tlbw_hazard();
342		if (idx < 0)
343			tlb_write_random();
344		else
345			tlb_write_indexed();
346		tlbw_use_hazard();
347		write_c0_pagemask(PM_DEFAULT_MASK);
348	} else
349#endif
350	{
351		ptemap = ptep = pte_offset_map(pmdp, address);
352		/*
353		 * update_mmu_cache() is called between pte_offset_map_lock()
354		 * and pte_unmap_unlock(), so we can assume that ptep is not
355		 * NULL here: and what should be done below if it were NULL?
356		 */
357
358#if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
359#ifdef CONFIG_XPA
360		write_c0_entrylo0(pte_to_entrylo(ptep->pte_high));
361		if (cpu_has_xpa)
362			writex_c0_entrylo0(ptep->pte_low & _PFNX_MASK);
363		ptep++;
364		write_c0_entrylo1(pte_to_entrylo(ptep->pte_high));
365		if (cpu_has_xpa)
366			writex_c0_entrylo1(ptep->pte_low & _PFNX_MASK);
367#else
368		write_c0_entrylo0(ptep->pte_high);
369		ptep++;
370		write_c0_entrylo1(ptep->pte_high);
371#endif
372#else
373		write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
374		write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
375#endif
376		mtc0_tlbw_hazard();
377		if (idx < 0)
378			tlb_write_random();
379		else
380			tlb_write_indexed();
381	}
382	tlbw_use_hazard();
383	htw_start();
384	flush_micro_tlb_vm(vma);
385
386	if (ptemap)
387		pte_unmap(ptemap);
388	local_irq_restore(flags);
389}
390
391void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
392		     unsigned long entryhi, unsigned long pagemask)
393{
394#ifdef CONFIG_XPA
395	panic("Broken for XPA kernels");
396#else
397	unsigned int old_mmid;
398	unsigned long flags;
399	unsigned long wired;
400	unsigned long old_pagemask;
401	unsigned long old_ctx;
402
403	local_irq_save(flags);
404	if (cpu_has_mmid) {
405		old_mmid = read_c0_memorymapid();
406		write_c0_memorymapid(MMID_KERNEL_WIRED);
407	}
408	/* Save old context and create impossible VPN2 value */
409	old_ctx = read_c0_entryhi();
410	htw_stop();
411	old_pagemask = read_c0_pagemask();
412	wired = num_wired_entries();
413	write_c0_wired(wired + 1);
414	write_c0_index(wired);
415	tlbw_use_hazard();	/* What is the hazard here? */
416	write_c0_pagemask(pagemask);
417	write_c0_entryhi(entryhi);
418	write_c0_entrylo0(entrylo0);
419	write_c0_entrylo1(entrylo1);
420	mtc0_tlbw_hazard();
421	tlb_write_indexed();
422	tlbw_use_hazard();
423
424	write_c0_entryhi(old_ctx);
425	if (cpu_has_mmid)
426		write_c0_memorymapid(old_mmid);
427	tlbw_use_hazard();	/* What is the hazard here? */
428	htw_start();
429	write_c0_pagemask(old_pagemask);
430	local_flush_tlb_all();
431	local_irq_restore(flags);
432#endif
433}
434
435#ifdef CONFIG_TRANSPARENT_HUGEPAGE
436
437int has_transparent_hugepage(void)
438{
439	static unsigned int mask = -1;
440
441	if (mask == -1) {	/* first call comes during __init */
442		unsigned long flags;
443
444		local_irq_save(flags);
445		write_c0_pagemask(PM_HUGE_MASK);
446		back_to_back_c0_hazard();
447		mask = read_c0_pagemask();
448		write_c0_pagemask(PM_DEFAULT_MASK);
449		local_irq_restore(flags);
450	}
451	return mask == PM_HUGE_MASK;
452}
453EXPORT_SYMBOL(has_transparent_hugepage);
454
455#endif /* CONFIG_TRANSPARENT_HUGEPAGE  */
456
457/*
458 * Used for loading TLB entries before trap_init() has started, when we
459 * don't actually want to add a wired entry which remains throughout the
460 * lifetime of the system
461 */
462
463int temp_tlb_entry;
464
465#ifndef CONFIG_64BIT
466__init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
467			       unsigned long entryhi, unsigned long pagemask)
468{
469	int ret = 0;
470	unsigned long flags;
471	unsigned long wired;
472	unsigned long old_pagemask;
473	unsigned long old_ctx;
474
475	local_irq_save(flags);
476	/* Save old context and create impossible VPN2 value */
477	htw_stop();
478	old_ctx = read_c0_entryhi();
479	old_pagemask = read_c0_pagemask();
480	wired = num_wired_entries();
481	if (--temp_tlb_entry < wired) {
482		printk(KERN_WARNING
483		       "No TLB space left for add_temporary_entry\n");
484		ret = -ENOSPC;
485		goto out;
486	}
487
488	write_c0_index(temp_tlb_entry);
489	write_c0_pagemask(pagemask);
490	write_c0_entryhi(entryhi);
491	write_c0_entrylo0(entrylo0);
492	write_c0_entrylo1(entrylo1);
493	mtc0_tlbw_hazard();
494	tlb_write_indexed();
495	tlbw_use_hazard();
496
497	write_c0_entryhi(old_ctx);
498	write_c0_pagemask(old_pagemask);
499	htw_start();
500out:
501	local_irq_restore(flags);
502	return ret;
503}
504#endif
505
506static int ntlb;
507static int __init set_ntlb(char *str)
508{
509	get_option(&str, &ntlb);
510	return 1;
511}
512
513__setup("ntlb=", set_ntlb);
514
515
516/* The start bit position of VPN2 and Mask in EntryHi/PageMask registers.  */
517#define VPN2_SHIFT 13
518
519/* Read full EntryHi even with CONFIG_32BIT.  */
520static inline unsigned long long read_c0_entryhi_native(void)
521{
522	return cpu_has_64bits ? read_c0_entryhi_64() : read_c0_entryhi();
523}
524
525/* Write full EntryHi even with CONFIG_32BIT.  */
526static inline void write_c0_entryhi_native(unsigned long long v)
527{
528	if (cpu_has_64bits)
529		write_c0_entryhi_64(v);
530	else
531		write_c0_entryhi(v);
532}
533
534/* TLB entry state for uniquification.  */
535struct tlbent {
536	unsigned long long wired:1;
537	unsigned long long global:1;
538	unsigned long long asid:10;
539	unsigned long long vpn:51;
540	unsigned long long pagesz:5;
541	unsigned long long index:14;
542};
543
544/*
545 * Comparison function for TLB entry sorting.  Place wired entries first,
546 * then global entries, then order by the increasing VPN/ASID and the
547 * decreasing page size.  This lets us avoid clashes with wired entries
548 * easily and get entries for larger pages out of the way first.
549 *
550 * We could group bits so as to reduce the number of comparisons, but this
551 * is seldom executed and not performance-critical, so prefer legibility.
552 */
553static int r4k_entry_cmp(const void *a, const void *b)
554{
555	struct tlbent ea = *(struct tlbent *)a, eb = *(struct tlbent *)b;
556
557	if (ea.wired > eb.wired)
558		return -1;
559	else if (ea.wired < eb.wired)
560		return 1;
561	else if (ea.global > eb.global)
562		return -1;
563	else if (ea.global < eb.global)
564		return 1;
565	else if (ea.vpn < eb.vpn)
566		return -1;
567	else if (ea.vpn > eb.vpn)
568		return 1;
569	else if (ea.asid < eb.asid)
570		return -1;
571	else if (ea.asid > eb.asid)
572		return 1;
573	else if (ea.pagesz > eb.pagesz)
574		return -1;
575	else if (ea.pagesz < eb.pagesz)
576		return 1;
577	else
578		return 0;
579}
580
581/*
582 * Fetch all the TLB entries.  Mask individual VPN values retrieved with
583 * the corresponding page mask and ignoring any 1KiB extension as we'll
584 * be using 4KiB pages for uniquification.
585 */
586static void __ref r4k_tlb_uniquify_read(struct tlbent *tlb_vpns, int tlbsize)
587{
588	int start = num_wired_entries();
589	unsigned long long vpn_mask;
590	bool global;
591	int i;
592
593	vpn_mask = GENMASK(current_cpu_data.vmbits - 1, VPN2_SHIFT);
594	vpn_mask |= cpu_has_64bits ? 3ULL << 62 : 1 << 31;
595
596	for (i = 0; i < tlbsize; i++) {
597		unsigned long long entryhi, vpn, mask, asid;
598		unsigned int pagesz;
599
600		write_c0_index(i);
601		mtc0_tlbr_hazard();
602		tlb_read();
603		tlb_read_hazard();
604
605		global = !!(read_c0_entrylo0() & ENTRYLO_G);
606		entryhi = read_c0_entryhi_native();
607		mask = read_c0_pagemask();
608
609		asid = entryhi & cpu_asid_mask(&current_cpu_data);
610		vpn = (entryhi & vpn_mask & ~mask) >> VPN2_SHIFT;
611		pagesz = ilog2((mask >> VPN2_SHIFT) + 1);
612
613		tlb_vpns[i].global = global;
614		tlb_vpns[i].asid = global ? 0 : asid;
615		tlb_vpns[i].vpn = vpn;
616		tlb_vpns[i].pagesz = pagesz;
617		tlb_vpns[i].wired = i < start;
618		tlb_vpns[i].index = i;
619	}
620}
621
622/*
623 * Write unique values to all but the wired TLB entries each, using
624 * the 4KiB page size.  This size might not be supported with R6, but
625 * EHINV is mandatory for R6, so we won't ever be called in that case.
626 *
627 * A sorted table is supplied with any wired entries at the beginning,
628 * followed by any global entries, and then finally regular entries.
629 * We start at the VPN and ASID values of zero and only assign user
630 * addresses, therefore guaranteeing no clash with addresses produced
631 * by UNIQUE_ENTRYHI.  We avoid any VPN values used by wired or global
632 * entries, by increasing the VPN value beyond the span of such entry.
633 *
634 * When a VPN/ASID clash is found with a regular entry we increment the
635 * ASID instead until no VPN/ASID clash has been found or the ASID space
636 * has been exhausted, in which case we increase the VPN value beyond
637 * the span of the largest clashing entry.
638 *
639 * We do not need to be concerned about FTLB or MMID configurations as
640 * those are required to implement the EHINV feature.
641 */
642static void __ref r4k_tlb_uniquify_write(struct tlbent *tlb_vpns, int tlbsize)
643{
644	unsigned long long asid, vpn, vpn_size, pagesz;
645	int widx, gidx, idx, sidx, lidx, i;
646
647	vpn_size = 1ULL << (current_cpu_data.vmbits - VPN2_SHIFT);
648	pagesz = ilog2((PM_4K >> VPN2_SHIFT) + 1);
649
650	write_c0_pagemask(PM_4K);
651	write_c0_entrylo0(0);
652	write_c0_entrylo1(0);
653
654	asid = 0;
655	vpn = 0;
656	widx = 0;
657	gidx = 0;
658	for (sidx = 0; sidx < tlbsize && tlb_vpns[sidx].wired; sidx++)
659		;
660	for (lidx = sidx; lidx < tlbsize && tlb_vpns[lidx].global; lidx++)
661		;
662	idx = gidx = sidx + 1;
663	for (i = sidx; i < tlbsize; i++) {
664		unsigned long long entryhi, vpn_pagesz = 0;
665
666		while (1) {
667			if (WARN_ON(vpn >= vpn_size)) {
668				dump_tlb_all();
669				/* Pray local_flush_tlb_all() will cope.  */
670				return;
671			}
672
673			/* VPN must be below the next wired entry.  */
674			if (widx < sidx && vpn >= tlb_vpns[widx].vpn) {
675				vpn = max(vpn,
676					  (tlb_vpns[widx].vpn +
677					   (1ULL << tlb_vpns[widx].pagesz)));
678				asid = 0;
679				widx++;
680				continue;
681			}
682			/* VPN must be below the next global entry.  */
683			if (gidx < lidx && vpn >= tlb_vpns[gidx].vpn) {
684				vpn = max(vpn,
685					  (tlb_vpns[gidx].vpn +
686					   (1ULL << tlb_vpns[gidx].pagesz)));
687				asid = 0;
688				gidx++;
689				continue;
690			}
691			/* Try to find a free ASID so as to conserve VPNs.  */
692			if (idx < tlbsize && vpn == tlb_vpns[idx].vpn &&
693			    asid == tlb_vpns[idx].asid) {
694				unsigned long long idx_pagesz;
695
696				idx_pagesz = tlb_vpns[idx].pagesz;
697				vpn_pagesz = max(vpn_pagesz, idx_pagesz);
698				do
699					idx++;
700				while (idx < tlbsize &&
701				       vpn == tlb_vpns[idx].vpn &&
702				       asid == tlb_vpns[idx].asid);
703				asid++;
704				if (asid > cpu_asid_mask(&current_cpu_data)) {
705					vpn += vpn_pagesz;
706					asid = 0;
707					vpn_pagesz = 0;
708				}
709				continue;
710			}
711			/* VPN mustn't be above the next regular entry.  */
712			if (idx < tlbsize && vpn > tlb_vpns[idx].vpn) {
713				vpn = max(vpn,
714					  (tlb_vpns[idx].vpn +
715					   (1ULL << tlb_vpns[idx].pagesz)));
716				asid = 0;
717				idx++;
718				continue;
719			}
720			break;
721		}
722
723		entryhi = (vpn << VPN2_SHIFT) | asid;
724		write_c0_entryhi_native(entryhi);
725		write_c0_index(tlb_vpns[i].index);
726		mtc0_tlbw_hazard();
727		tlb_write_indexed();
728
729		tlb_vpns[i].asid = asid;
730		tlb_vpns[i].vpn = vpn;
731		tlb_vpns[i].pagesz = pagesz;
732
733		asid++;
734		if (asid > cpu_asid_mask(&current_cpu_data)) {
735			vpn += 1ULL << pagesz;
736			asid = 0;
737		}
738	}
739}
740
741/*
742 * Initialise all TLB entries with unique values that do not clash with
743 * what we have been handed over and what we'll be using ourselves.
744 */
745static void __ref r4k_tlb_uniquify(void)
746{
747	int tlbsize = current_cpu_data.tlbsize;
748	bool use_slab = slab_is_available();
749	phys_addr_t tlb_vpn_size;
750	struct tlbent *tlb_vpns;
751
752	tlb_vpn_size = tlbsize * sizeof(*tlb_vpns);
753	tlb_vpns = (use_slab ?
754		    kmalloc(tlb_vpn_size, GFP_ATOMIC) :
755		    memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns)));
756	if (WARN_ON(!tlb_vpns))
757		return; /* Pray local_flush_tlb_all() is good enough. */
758
759	htw_stop();
760
761	r4k_tlb_uniquify_read(tlb_vpns, tlbsize);
762
763	sort(tlb_vpns, tlbsize, sizeof(*tlb_vpns), r4k_entry_cmp, NULL);
764
765	r4k_tlb_uniquify_write(tlb_vpns, tlbsize);
766
767	write_c0_pagemask(PM_DEFAULT_MASK);
768
769	tlbw_use_hazard();
770	htw_start();
771	flush_micro_tlb();
772	if (use_slab)
773		kfree(tlb_vpns);
774	else
775		memblock_free(tlb_vpns, tlb_vpn_size);
776}
777
778/*
779 * Configure TLB (for init or after a CPU has been powered off).
780 */
781static void r4k_tlb_configure(void)
782{
783	/*
784	 * You should never change this register:
785	 *   - On R4600 1.7 the tlbp never hits for pages smaller than
786	 *     the value in the c0_pagemask register.
787	 *   - The entire mm handling assumes the c0_pagemask register to
788	 *     be set to fixed-size pages.
789	 */
790	write_c0_pagemask(PM_DEFAULT_MASK);
791	back_to_back_c0_hazard();
792	if (read_c0_pagemask() != PM_DEFAULT_MASK)
793		panic("MMU doesn't support PAGE_SIZE=0x%lx", PAGE_SIZE);
794
795	write_c0_wired(0);
796	if (current_cpu_type() == CPU_R10000 ||
797	    current_cpu_type() == CPU_R12000 ||
798	    current_cpu_type() == CPU_R14000 ||
799	    current_cpu_type() == CPU_R16000)
800		write_c0_framemask(0);
801
802	if (cpu_has_rixi) {
803		/*
804		 * Enable the no read, no exec bits, and enable large physical
805		 * address.
806		 */
807#ifdef CONFIG_64BIT
808		set_c0_pagegrain(PG_RIE | PG_XIE | PG_ELPA);
809#else
810		set_c0_pagegrain(PG_RIE | PG_XIE);
811#endif
812	}
813
814	temp_tlb_entry = current_cpu_data.tlbsize - 1;
815
816	/* From this point on the ARC firmware is dead.	 */
817	if (!cpu_has_tlbinv)
818		r4k_tlb_uniquify();
819	local_flush_tlb_all();
820
821	/* Did I tell you that ARC SUCKS?  */
822}
823
824void tlb_init(void)
825{
826	r4k_tlb_configure();
827
828	if (ntlb) {
829		if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
830			int wired = current_cpu_data.tlbsize - ntlb;
831			write_c0_wired(wired);
832			write_c0_index(wired-1);
833			printk("Restricting TLB to %d entries\n", ntlb);
834		} else
835			printk("Ignoring invalid argument ntlb=%d\n", ntlb);
836	}
837
838	build_tlb_refill_handler();
839}
840
841static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd,
842			       void *v)
843{
844	switch (cmd) {
845	case CPU_PM_ENTER_FAILED:
846	case CPU_PM_EXIT:
847		r4k_tlb_configure();
848		break;
849	}
850
851	return NOTIFY_OK;
852}
853
854static struct notifier_block r4k_tlb_pm_notifier_block = {
855	.notifier_call = r4k_tlb_pm_notifier,
856};
857
858static int __init r4k_tlb_init_pm(void)
859{
860	return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block);
861}
862arch_initcall(r4k_tlb_init_pm);