hash_native.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * native hashtable management.
  4 *
  5 * SMP scalability work:
  6 *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
  7 */
  8
  9#undef DEBUG_LOW
 10
 11#include <linux/spinlock.h>
 12#include <linux/bitops.h>
 13#include <linux/of.h>
 14#include <linux/processor.h>
 15#include <linux/threads.h>
 16#include <linux/smp.h>
 17#include <linux/pgtable.h>
 18
 19#include <asm/machdep.h>
 20#include <asm/mmu.h>
 21#include <asm/mmu_context.h>
 22#include <asm/trace.h>
 23#include <asm/tlb.h>
 24#include <asm/cputable.h>
 25#include <asm/udbg.h>
 26#include <asm/kexec.h>
 27#include <asm/ppc-opcode.h>
 28#include <asm/feature-fixups.h>
 29
 30#ifdef DEBUG_LOW
 31#define DBG_LOW(fmt...) udbg_printf(fmt)
 32#else
 33#define DBG_LOW(fmt...)
 34#endif
 35
 36#ifdef __BIG_ENDIAN__
 37#define HPTE_LOCK_BIT 3
 38#else
 39#define HPTE_LOCK_BIT (56+3)
 40#endif
 41
 42static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
 43
 44#ifdef CONFIG_LOCKDEP
 45static struct lockdep_map hpte_lock_map =
 46	STATIC_LOCKDEP_MAP_INIT("hpte_lock", &hpte_lock_map);
 47
 48static void acquire_hpte_lock(void)
 49{
 50	lock_map_acquire(&hpte_lock_map);
 51}
 52
 53static void release_hpte_lock(void)
 54{
 55	lock_map_release(&hpte_lock_map);
 56}
 57#else
 58static void acquire_hpte_lock(void)
 59{
 60}
 61
 62static void release_hpte_lock(void)
 63{
 64}
 65#endif
 66
 67static inline unsigned long  ___tlbie(unsigned long vpn, int psize,
 68						int apsize, int ssize)
 69{
 70	unsigned long va;
 71	unsigned int penc;
 72	unsigned long sllp;
 73
 74	/*
 75	 * We need 14 to 65 bits of va for a tlibe of 4K page
 76	 * With vpn we ignore the lower VPN_SHIFT bits already.
 77	 * And top two bits are already ignored because we can
 78	 * only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT
 79	 * of 12.
 80	 */
 81	va = vpn << VPN_SHIFT;
 82	/*
 83	 * clear top 16 bits of 64bit va, non SLS segment
 84	 * Older versions of the architecture (2.02 and earler) require the
 85	 * masking of the top 16 bits.
 86	 */
 87	if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
 88		va &= ~(0xffffULL << 48);
 89
 90	switch (psize) {
 91	case MMU_PAGE_4K:
 92		/* clear out bits after (52) [0....52.....63] */
 93		va &= ~((1ul << (64 - 52)) - 1);
 94		va |= ssize << 8;
 95		sllp = get_sllp_encoding(apsize);
 96		va |= sllp << 5;
 97		asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
 98			     : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
 99			     : "memory");
100		break;
101	default:
102		/* We need 14 to 14 + i bits of va */
103		penc = mmu_psize_defs[psize].penc[apsize];
104		va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
105		va |= penc << 12;
106		va |= ssize << 8;
107		/*
108		 * AVAL bits:
109		 * We don't need all the bits, but rest of the bits
110		 * must be ignored by the processor.
111		 * vpn cover upto 65 bits of va. (0...65) and we need
112		 * 58..64 bits of va.
113		 */
114		va |= (vpn & 0xfe); /* AVAL */
115		va |= 1; /* L */
116		asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
117			     : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
118			     : "memory");
119		break;
120	}
121	return va;
122}
123
124static inline void fixup_tlbie_vpn(unsigned long vpn, int psize,
125				   int apsize, int ssize)
126{
127	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
128		/* Radix flush for a hash guest */
129
130		unsigned long rb,rs,prs,r,ric;
131
132		rb = PPC_BIT(52); /* IS = 2 */
133		rs = 0;  /* lpid = 0 */
134		prs = 0; /* partition scoped */
135		r = 1;   /* radix format */
136		ric = 0; /* RIC_FLSUH_TLB */
137
138		/*
139		 * Need the extra ptesync to make sure we don't
140		 * re-order the tlbie
141		 */
142		asm volatile("ptesync": : :"memory");
143		asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
144			     : : "r"(rb), "i"(r), "i"(prs),
145			       "i"(ric), "r"(rs) : "memory");
146	}
147
148
149	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
150		/* Need the extra ptesync to ensure we don't reorder tlbie*/
151		asm volatile("ptesync": : :"memory");
152		___tlbie(vpn, psize, apsize, ssize);
153	}
154}
155
156static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
157{
158	unsigned long rb;
159
160	rb = ___tlbie(vpn, psize, apsize, ssize);
161	trace_tlbie(0, 0, rb, 0, 0, 0, 0);
162}
163
164static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
165{
166	unsigned long va;
167	unsigned int penc;
168	unsigned long sllp;
169
170	/* VPN_SHIFT can be atmost 12 */
171	va = vpn << VPN_SHIFT;
172	/*
173	 * clear top 16 bits of 64 bit va, non SLS segment
174	 * Older versions of the architecture (2.02 and earler) require the
175	 * masking of the top 16 bits.
176	 */
177	if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
178		va &= ~(0xffffULL << 48);
179
180	switch (psize) {
181	case MMU_PAGE_4K:
182		/* clear out bits after(52) [0....52.....63] */
183		va &= ~((1ul << (64 - 52)) - 1);
184		va |= ssize << 8;
185		sllp = get_sllp_encoding(apsize);
186		va |= sllp << 5;
187		asm volatile(ASM_FTR_IFSET("tlbiel %0", PPC_TLBIEL_v205(%0, 0), %1)
188			     : : "r" (va), "i" (CPU_FTR_ARCH_206)
189			     : "memory");
190		break;
191	default:
192		/* We need 14 to 14 + i bits of va */
193		penc = mmu_psize_defs[psize].penc[apsize];
194		va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
195		va |= penc << 12;
196		va |= ssize << 8;
197		/*
198		 * AVAL bits:
199		 * We don't need all the bits, but rest of the bits
200		 * must be ignored by the processor.
201		 * vpn cover upto 65 bits of va. (0...65) and we need
202		 * 58..64 bits of va.
203		 */
204		va |= (vpn & 0xfe);
205		va |= 1; /* L */
206		asm volatile(ASM_FTR_IFSET("tlbiel %0", PPC_TLBIEL_v205(%0, 1), %1)
207			     : : "r" (va), "i" (CPU_FTR_ARCH_206)
208			     : "memory");
209		break;
210	}
211	trace_tlbie(0, 1, va, 0, 0, 0, 0);
212
213}
214
215static inline void tlbie(unsigned long vpn, int psize, int apsize,
216			 int ssize, int local)
217{
218	unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL);
219	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
220
221	if (use_local)
222		use_local = mmu_psize_defs[psize].tlbiel;
223	if (lock_tlbie && !use_local)
224		raw_spin_lock(&native_tlbie_lock);
225	asm volatile("ptesync": : :"memory");
226	if (use_local) {
227		__tlbiel(vpn, psize, apsize, ssize);
228		ppc_after_tlbiel_barrier();
229	} else {
230		__tlbie(vpn, psize, apsize, ssize);
231		fixup_tlbie_vpn(vpn, psize, apsize, ssize);
232		asm volatile("eieio; tlbsync; ptesync": : :"memory");
233	}
234	if (lock_tlbie && !use_local)
235		raw_spin_unlock(&native_tlbie_lock);
236}
237
238static inline void native_lock_hpte(struct hash_pte *hptep)
239{
240	unsigned long *word = (unsigned long *)&hptep->v;
241
242	acquire_hpte_lock();
243	while (1) {
244		if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
245			break;
246		spin_begin();
247		while(test_bit(HPTE_LOCK_BIT, word))
248			spin_cpu_relax();
249		spin_end();
250	}
251}
252
253static inline void native_unlock_hpte(struct hash_pte *hptep)
254{
255	unsigned long *word = (unsigned long *)&hptep->v;
256
257	release_hpte_lock();
258	clear_bit_unlock(HPTE_LOCK_BIT, word);
259}
260
261static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
262			unsigned long pa, unsigned long rflags,
263			unsigned long vflags, int psize, int apsize, int ssize)
264{
265	struct hash_pte *hptep = htab_address + hpte_group;
266	unsigned long hpte_v, hpte_r;
267	unsigned long flags;
268	int i;
269
270	local_irq_save(flags);
271
272	if (!(vflags & HPTE_V_BOLTED)) {
273		DBG_LOW("    insert(group=%lx, vpn=%016lx, pa=%016lx,"
274			" rflags=%lx, vflags=%lx, psize=%d)\n",
275			hpte_group, vpn, pa, rflags, vflags, psize);
276	}
277
278	for (i = 0; i < HPTES_PER_GROUP; i++) {
279		if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
280			/* retry with lock held */
281			native_lock_hpte(hptep);
282			if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
283				break;
284			native_unlock_hpte(hptep);
285		}
286
287		hptep++;
288	}
289
290	if (i == HPTES_PER_GROUP) {
291		local_irq_restore(flags);
292		return -1;
293	}
294
295	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
296	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
297
298	if (!(vflags & HPTE_V_BOLTED)) {
299		DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
300			i, hpte_v, hpte_r);
301	}
302
303	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
304		hpte_r = hpte_old_to_new_r(hpte_v, hpte_r);
305		hpte_v = hpte_old_to_new_v(hpte_v);
306	}
307
308	hptep->r = cpu_to_be64(hpte_r);
309	/* Guarantee the second dword is visible before the valid bit */
310	eieio();
311	/*
312	 * Now set the first dword including the valid bit
313	 * NOTE: this also unlocks the hpte
314	 */
315	release_hpte_lock();
316	hptep->v = cpu_to_be64(hpte_v);
317
318	__asm__ __volatile__ ("ptesync" : : : "memory");
319
320	local_irq_restore(flags);
321
322	return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
323}
324
325static long native_hpte_remove(unsigned long hpte_group)
326{
327	unsigned long hpte_v, flags;
328	struct hash_pte *hptep;
329	int i;
330	int slot_offset;
331
332	local_irq_save(flags);
333
334	DBG_LOW("    remove(group=%lx)\n", hpte_group);
335
336	/* pick a random entry to start at */
337	slot_offset = mftb() & 0x7;
338
339	for (i = 0; i < HPTES_PER_GROUP; i++) {
340		hptep = htab_address + hpte_group + slot_offset;
341		hpte_v = be64_to_cpu(hptep->v);
342
343		if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
344			/* retry with lock held */
345			native_lock_hpte(hptep);
346			hpte_v = be64_to_cpu(hptep->v);
347			if ((hpte_v & HPTE_V_VALID)
348			    && !(hpte_v & HPTE_V_BOLTED))
349				break;
350			native_unlock_hpte(hptep);
351		}
352
353		slot_offset++;
354		slot_offset &= 0x7;
355	}
356
357	if (i == HPTES_PER_GROUP) {
358		i = -1;
359		goto out;
360	}
361
362	/* Invalidate the hpte. NOTE: this also unlocks it */
363	release_hpte_lock();
364	hptep->v = 0;
365out:
366	local_irq_restore(flags);
367	return i;
368}
369
370static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
371				 unsigned long vpn, int bpsize,
372				 int apsize, int ssize, unsigned long flags)
373{
374	struct hash_pte *hptep = htab_address + slot;
375	unsigned long hpte_v, want_v;
376	int ret = 0, local = 0;
377	unsigned long irqflags;
378
379	local_irq_save(irqflags);
380
381	want_v = hpte_encode_avpn(vpn, bpsize, ssize);
382
383	DBG_LOW("    update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
384		vpn, want_v & HPTE_V_AVPN, slot, newpp);
385
386	hpte_v = hpte_get_old_v(hptep);
387	/*
388	 * We need to invalidate the TLB always because hpte_remove doesn't do
389	 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
390	 * random entry from it. When we do that we don't invalidate the TLB
391	 * (hpte_remove) because we assume the old translation is still
392	 * technically "valid".
393	 */
394	if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
395		DBG_LOW(" -> miss\n");
396		ret = -1;
397	} else {
398		native_lock_hpte(hptep);
399		/* recheck with locks held */
400		hpte_v = hpte_get_old_v(hptep);
401		if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
402			     !(hpte_v & HPTE_V_VALID))) {
403			ret = -1;
404		} else {
405			DBG_LOW(" -> hit\n");
406			/* Update the HPTE */
407			hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
408						~(HPTE_R_PPP | HPTE_R_N)) |
409					       (newpp & (HPTE_R_PPP | HPTE_R_N |
410							 HPTE_R_C)));
411		}
412		native_unlock_hpte(hptep);
413	}
414
415	if (flags & HPTE_LOCAL_UPDATE)
416		local = 1;
417	/*
418	 * Ensure it is out of the tlb too if it is not a nohpte fault
419	 */
420	if (!(flags & HPTE_NOHPTE_UPDATE))
421		tlbie(vpn, bpsize, apsize, ssize, local);
422
423	local_irq_restore(irqflags);
424
425	return ret;
426}
427
428static long __native_hpte_find(unsigned long want_v, unsigned long slot)
429{
430	struct hash_pte *hptep;
431	unsigned long hpte_v;
432	unsigned long i;
433
434	for (i = 0; i < HPTES_PER_GROUP; i++) {
435
436		hptep = htab_address + slot;
437		hpte_v = hpte_get_old_v(hptep);
438		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
439			/* HPTE matches */
440			return slot;
441		++slot;
442	}
443
444	return -1;
445}
446
447static long native_hpte_find(unsigned long vpn, int psize, int ssize)
448{
449	unsigned long hpte_group;
450	unsigned long want_v;
451	unsigned long hash;
452	long slot;
453
454	hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
455	want_v = hpte_encode_avpn(vpn, psize, ssize);
456
457	/*
458	 * We try to keep bolted entries always in primary hash
459	 * But in some case we can find them in secondary too.
460	 */
461	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
462	slot = __native_hpte_find(want_v, hpte_group);
463	if (slot < 0) {
464		/* Try in secondary */
465		hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
466		slot = __native_hpte_find(want_v, hpte_group);
467		if (slot < 0)
468			return -1;
469	}
470
471	return slot;
472}
473
474/*
475 * Update the page protection bits. Intended to be used to create
476 * guard pages for kernel data structures on pages which are bolted
477 * in the HPT. Assumes pages being operated on will not be stolen.
478 *
479 * No need to lock here because we should be the only user.
480 */
481static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
482				       int psize, int ssize)
483{
484	unsigned long vpn;
485	unsigned long vsid;
486	long slot;
487	struct hash_pte *hptep;
488	unsigned long flags;
489
490	local_irq_save(flags);
491
492	vsid = get_kernel_vsid(ea, ssize);
493	vpn = hpt_vpn(ea, vsid, ssize);
494
495	slot = native_hpte_find(vpn, psize, ssize);
496	if (slot == -1)
497		panic("could not find page to bolt\n");
498	hptep = htab_address + slot;
499
500	/* Update the HPTE */
501	hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
502				~(HPTE_R_PPP | HPTE_R_N)) |
503			       (newpp & (HPTE_R_PPP | HPTE_R_N)));
504	/*
505	 * Ensure it is out of the tlb too. Bolted entries base and
506	 * actual page size will be same.
507	 */
508	tlbie(vpn, psize, psize, ssize, 0);
509
510	local_irq_restore(flags);
511}
512
513/*
514 * Remove a bolted kernel entry. Memory hotplug uses this.
515 *
516 * No need to lock here because we should be the only user.
517 */
518static int native_hpte_removebolted(unsigned long ea, int psize, int ssize)
519{
520	unsigned long vpn;
521	unsigned long vsid;
522	long slot;
523	struct hash_pte *hptep;
524	unsigned long flags;
525
526	local_irq_save(flags);
527
528	vsid = get_kernel_vsid(ea, ssize);
529	vpn = hpt_vpn(ea, vsid, ssize);
530
531	slot = native_hpte_find(vpn, psize, ssize);
532	if (slot == -1)
533		return -ENOENT;
534
535	hptep = htab_address + slot;
536
537	VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED));
538
539	/* Invalidate the hpte */
540	hptep->v = 0;
541
542	/* Invalidate the TLB */
543	tlbie(vpn, psize, psize, ssize, 0);
544
545	local_irq_restore(flags);
546
547	return 0;
548}
549
550
551static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
552				   int bpsize, int apsize, int ssize, int local)
553{
554	struct hash_pte *hptep = htab_address + slot;
555	unsigned long hpte_v;
556	unsigned long want_v;
557	unsigned long flags;
558
559	local_irq_save(flags);
560
561	DBG_LOW("    invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
562
563	want_v = hpte_encode_avpn(vpn, bpsize, ssize);
564	hpte_v = hpte_get_old_v(hptep);
565
566	if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
567		native_lock_hpte(hptep);
568		/* recheck with locks held */
569		hpte_v = hpte_get_old_v(hptep);
570
571		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
572			/* Invalidate the hpte. NOTE: this also unlocks it */
573			release_hpte_lock();
574			hptep->v = 0;
575		} else
576			native_unlock_hpte(hptep);
577	}
578	/*
579	 * We need to invalidate the TLB always because hpte_remove doesn't do
580	 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
581	 * random entry from it. When we do that we don't invalidate the TLB
582	 * (hpte_remove) because we assume the old translation is still
583	 * technically "valid".
584	 */
585	tlbie(vpn, bpsize, apsize, ssize, local);
586
587	local_irq_restore(flags);
588}
589
590#ifdef CONFIG_TRANSPARENT_HUGEPAGE
591static void native_hugepage_invalidate(unsigned long vsid,
592				       unsigned long addr,
593				       unsigned char *hpte_slot_array,
594				       int psize, int ssize, int local)
595{
596	int i;
597	struct hash_pte *hptep;
598	int actual_psize = MMU_PAGE_16M;
599	unsigned int max_hpte_count, valid;
600	unsigned long flags, s_addr = addr;
601	unsigned long hpte_v, want_v, shift;
602	unsigned long hidx, vpn = 0, hash, slot;
603
604	shift = mmu_psize_defs[psize].shift;
605	max_hpte_count = 1U << (PMD_SHIFT - shift);
606
607	local_irq_save(flags);
608	for (i = 0; i < max_hpte_count; i++) {
609		valid = hpte_valid(hpte_slot_array, i);
610		if (!valid)
611			continue;
612		hidx =  hpte_hash_index(hpte_slot_array, i);
613
614		/* get the vpn */
615		addr = s_addr + (i * (1ul << shift));
616		vpn = hpt_vpn(addr, vsid, ssize);
617		hash = hpt_hash(vpn, shift, ssize);
618		if (hidx & _PTEIDX_SECONDARY)
619			hash = ~hash;
620
621		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
622		slot += hidx & _PTEIDX_GROUP_IX;
623
624		hptep = htab_address + slot;
625		want_v = hpte_encode_avpn(vpn, psize, ssize);
626		hpte_v = hpte_get_old_v(hptep);
627
628		/* Even if we miss, we need to invalidate the TLB */
629		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
630			/* recheck with locks held */
631			native_lock_hpte(hptep);
632			hpte_v = hpte_get_old_v(hptep);
633
634			if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
635				/* Invalidate the hpte. NOTE: this also unlocks it */
636				release_hpte_lock();
637				hptep->v = 0;
638			} else
639				native_unlock_hpte(hptep);
640		}
641		/*
642		 * We need to do tlb invalidate for all the address, tlbie
643		 * instruction compares entry_VA in tlb with the VA specified
644		 * here
645		 */
646		tlbie(vpn, psize, actual_psize, ssize, local);
647	}
648	local_irq_restore(flags);
649}
650#else
651static void native_hugepage_invalidate(unsigned long vsid,
652				       unsigned long addr,
653				       unsigned char *hpte_slot_array,
654				       int psize, int ssize, int local)
655{
656	WARN(1, "%s called without THP support\n", __func__);
657}
658#endif
659
660static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
661			int *psize, int *apsize, int *ssize, unsigned long *vpn)
662{
663	unsigned long avpn, pteg, vpi;
664	unsigned long hpte_v = be64_to_cpu(hpte->v);
665	unsigned long hpte_r = be64_to_cpu(hpte->r);
666	unsigned long vsid, seg_off;
667	int size, a_size, shift;
668	/* Look at the 8 bit LP value */
669	unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
670
671	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
672		hpte_v = hpte_new_to_old_v(hpte_v, hpte_r);
673		hpte_r = hpte_new_to_old_r(hpte_r);
674	}
675	if (!(hpte_v & HPTE_V_LARGE)) {
676		size   = MMU_PAGE_4K;
677		a_size = MMU_PAGE_4K;
678	} else {
679		size = hpte_page_sizes[lp] & 0xf;
680		a_size = hpte_page_sizes[lp] >> 4;
681	}
682	/* This works for all page sizes, and for 256M and 1T segments */
683	*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
684	shift = mmu_psize_defs[size].shift;
685
686	avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
687	pteg = slot / HPTES_PER_GROUP;
688	if (hpte_v & HPTE_V_SECONDARY)
689		pteg = ~pteg;
690
691	switch (*ssize) {
692	case MMU_SEGSIZE_256M:
693		/* We only have 28 - 23 bits of seg_off in avpn */
694		seg_off = (avpn & 0x1f) << 23;
695		vsid    =  avpn >> 5;
696		/* We can find more bits from the pteg value */
697		if (shift < 23) {
698			vpi = (vsid ^ pteg) & htab_hash_mask;
699			seg_off |= vpi << shift;
700		}
701		*vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
702		break;
703	case MMU_SEGSIZE_1T:
704		/* We only have 40 - 23 bits of seg_off in avpn */
705		seg_off = (avpn & 0x1ffff) << 23;
706		vsid    = avpn >> 17;
707		if (shift < 23) {
708			vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
709			seg_off |= vpi << shift;
710		}
711		*vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
712		break;
713	default:
714		*vpn = size = 0;
715	}
716	*psize  = size;
717	*apsize = a_size;
718}
719
720/*
721 * clear all mappings on kexec.  All cpus are in real mode (or they will
722 * be when they isi), and we are the only one left.  We rely on our kernel
723 * mapping being 0xC0's and the hardware ignoring those two real bits.
724 *
725 * This must be called with interrupts disabled.
726 *
727 * Taking the native_tlbie_lock is unsafe here due to the possibility of
728 * lockdep being on. On pre POWER5 hardware, not taking the lock could
729 * cause deadlock. POWER5 and newer not taking the lock is fine. This only
730 * gets called during boot before secondary CPUs have come up and during
731 * crashdump and all bets are off anyway.
732 *
733 * TODO: add batching support when enabled.  remember, no dynamic memory here,
734 * although there is the control page available...
735 */
736static notrace void native_hpte_clear(void)
737{
738	unsigned long vpn = 0;
739	unsigned long slot, slots;
740	struct hash_pte *hptep = htab_address;
741	unsigned long hpte_v;
742	unsigned long pteg_count;
743	int psize, apsize, ssize;
744
745	pteg_count = htab_hash_mask + 1;
746
747	slots = pteg_count * HPTES_PER_GROUP;
748
749	for (slot = 0; slot < slots; slot++, hptep++) {
750		/*
751		 * we could lock the pte here, but we are the only cpu
752		 * running,  right?  and for crash dump, we probably
753		 * don't want to wait for a maybe bad cpu.
754		 */
755		hpte_v = be64_to_cpu(hptep->v);
756
757		/*
758		 * Call __tlbie() here rather than tlbie() since we can't take the
759		 * native_tlbie_lock.
760		 */
761		if (hpte_v & HPTE_V_VALID) {
762			hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
763			hptep->v = 0;
764			___tlbie(vpn, psize, apsize, ssize);
765		}
766	}
767
768	asm volatile("eieio; tlbsync; ptesync":::"memory");
769}
770
771/*
772 * Batched hash table flush, we batch the tlbie's to avoid taking/releasing
773 * the lock all the time
774 */
775static void native_flush_hash_range(unsigned long number, int local)
776{
777	unsigned long vpn = 0;
778	unsigned long hash, index, hidx, shift, slot;
779	struct hash_pte *hptep;
780	unsigned long hpte_v;
781	unsigned long want_v;
782	unsigned long flags;
783	real_pte_t pte;
784	struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
785	unsigned long psize = batch->psize;
786	int ssize = batch->ssize;
787	int i;
788
789	local_irq_save(flags);
790
791	for (i = 0; i < number; i++) {
792		vpn = batch->vpn[i];
793		pte = batch->pte[i];
794
795		pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
796			hash = hpt_hash(vpn, shift, ssize);
797			hidx = __rpte_to_hidx(pte, index);
798			if (hidx & _PTEIDX_SECONDARY)
799				hash = ~hash;
800			slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
801			slot += hidx & _PTEIDX_GROUP_IX;
802			hptep = htab_address + slot;
803			want_v = hpte_encode_avpn(vpn, psize, ssize);
804			hpte_v = hpte_get_old_v(hptep);
805
806			if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
807				continue;
808			/* lock and try again */
809			native_lock_hpte(hptep);
810			hpte_v = hpte_get_old_v(hptep);
811
812			if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
813				native_unlock_hpte(hptep);
814			else {
815				release_hpte_lock();
816				hptep->v = 0;
817			}
818
819		} pte_iterate_hashed_end();
820	}
821
822	if (mmu_has_feature(MMU_FTR_TLBIEL) &&
823	    mmu_psize_defs[psize].tlbiel && local) {
824		asm volatile("ptesync":::"memory");
825		for (i = 0; i < number; i++) {
826			vpn = batch->vpn[i];
827			pte = batch->pte[i];
828
829			pte_iterate_hashed_subpages(pte, psize,
830						    vpn, index, shift) {
831				__tlbiel(vpn, psize, psize, ssize);
832			} pte_iterate_hashed_end();
833		}
834		ppc_after_tlbiel_barrier();
835	} else {
836		int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
837
838		if (lock_tlbie)
839			raw_spin_lock(&native_tlbie_lock);
840
841		asm volatile("ptesync":::"memory");
842		for (i = 0; i < number; i++) {
843			vpn = batch->vpn[i];
844			pte = batch->pte[i];
845
846			pte_iterate_hashed_subpages(pte, psize,
847						    vpn, index, shift) {
848				__tlbie(vpn, psize, psize, ssize);
849			} pte_iterate_hashed_end();
850		}
851		/*
852		 * Just do one more with the last used values.
853		 */
854		fixup_tlbie_vpn(vpn, psize, psize, ssize);
855		asm volatile("eieio; tlbsync; ptesync":::"memory");
856
857		if (lock_tlbie)
858			raw_spin_unlock(&native_tlbie_lock);
859	}
860
861	local_irq_restore(flags);
862}
863
864void __init hpte_init_native(void)
865{
866	mmu_hash_ops.hpte_invalidate	= native_hpte_invalidate;
867	mmu_hash_ops.hpte_updatepp	= native_hpte_updatepp;
868	mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
869	mmu_hash_ops.hpte_removebolted = native_hpte_removebolted;
870	mmu_hash_ops.hpte_insert	= native_hpte_insert;
871	mmu_hash_ops.hpte_remove	= native_hpte_remove;
872	mmu_hash_ops.hpte_clear_all	= native_hpte_clear;
873	mmu_hash_ops.flush_hash_range = native_flush_hash_range;
874	mmu_hash_ops.hugepage_invalidate   = native_hugepage_invalidate;
875}