tlbflush.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Based on arch/arm/include/asm/tlbflush.h
  4 *
  5 * Copyright (C) 1999-2003 Russell King
  6 * Copyright (C) 2012 ARM Ltd.
  7 */
  8#ifndef __ASM_TLBFLUSH_H
  9#define __ASM_TLBFLUSH_H
 10
 11#ifndef __ASSEMBLER__
 12
 13#include <linux/bitfield.h>
 14#include <linux/mm_types.h>
 15#include <linux/sched.h>
 16#include <linux/mmu_notifier.h>
 17#include <asm/cputype.h>
 18#include <asm/mmu.h>
 19
 20/*
 21 * Raw TLBI operations.
 22 *
 23 * Where necessary, use the __tlbi() macro to avoid asm()
 24 * boilerplate. Drivers and most kernel code should use the TLB
 25 * management routines in preference to the macro below.
 26 *
 27 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending
 28 * on whether a particular TLBI operation takes an argument or
 29 * not. The macros handles invoking the asm with or without the
 30 * register argument as appropriate.
 31 */
 32#define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE			       \
 33			       "tlbi " #op "\n"				       \
 34		   ALTERNATIVE("nop\n			nop",		       \
 35			       "dsb ish\n		tlbi " #op,	       \
 36			       ARM64_WORKAROUND_REPEAT_TLBI,		       \
 37			       CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)	       \
 38			    : : )
 39
 40#define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE			       \
 41			       "tlbi " #op ", %0\n"			       \
 42		   ALTERNATIVE("nop\n			nop",		       \
 43			       "dsb ish\n		tlbi " #op ", %0",     \
 44			       ARM64_WORKAROUND_REPEAT_TLBI,		       \
 45			       CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)	       \
 46			    : : "r" (arg))
 47
 48#define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg)
 49
 50#define __tlbi(op, ...)		__TLBI_N(op, ##__VA_ARGS__, 1, 0)
 51
 52#define __tlbi_user(op, arg) do {						\
 53	if (arm64_kernel_unmapped_at_el0())					\
 54		__tlbi(op, (arg) | USER_ASID_FLAG);				\
 55} while (0)
 56
 57/* This macro creates a properly formatted VA operand for the TLBI */
 58#define __TLBI_VADDR(addr, asid)				\
 59	({							\
 60		unsigned long __ta = (addr) >> 12;		\
 61		__ta &= GENMASK_ULL(43, 0);			\
 62		__ta |= (unsigned long)(asid) << 48;		\
 63		__ta;						\
 64	})
 65
 66/*
 67 * Get translation granule of the system, which is decided by
 68 * PAGE_SIZE.  Used by TTL.
 69 *  - 4KB	: 1
 70 *  - 16KB	: 2
 71 *  - 64KB	: 3
 72 */
 73#define TLBI_TTL_TG_4K		1
 74#define TLBI_TTL_TG_16K		2
 75#define TLBI_TTL_TG_64K		3
 76
 77static inline unsigned long get_trans_granule(void)
 78{
 79	switch (PAGE_SIZE) {
 80	case SZ_4K:
 81		return TLBI_TTL_TG_4K;
 82	case SZ_16K:
 83		return TLBI_TTL_TG_16K;
 84	case SZ_64K:
 85		return TLBI_TTL_TG_64K;
 86	default:
 87		return 0;
 88	}
 89}
 90
 91/*
 92 * Level-based TLBI operations.
 93 *
 94 * When ARMv8.4-TTL exists, TLBI operations take an additional hint for
 95 * the level at which the invalidation must take place. If the level is
 96 * wrong, no invalidation may take place. In the case where the level
 97 * cannot be easily determined, the value TLBI_TTL_UNKNOWN will perform
 98 * a non-hinted invalidation. Any provided level outside the hint range
 99 * will also cause fall-back to non-hinted invalidation.
100 *
101 * For Stage-2 invalidation, use the level values provided to that effect
102 * in asm/stage2_pgtable.h.
103 */
104#define TLBI_TTL_MASK		GENMASK_ULL(47, 44)
105
106#define TLBI_TTL_UNKNOWN	INT_MAX
107
108#define __tlbi_level(op, addr, level) do {				\
109	u64 arg = addr;							\
110									\
111	if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) &&	\
112	    level >= 0 && level <= 3) {					\
113		u64 ttl = level & 3;					\
114		ttl |= get_trans_granule() << 2;			\
115		arg &= ~TLBI_TTL_MASK;					\
116		arg |= FIELD_PREP(TLBI_TTL_MASK, ttl);			\
117	}								\
118									\
119	__tlbi(op, arg);						\
120} while(0)
121
122#define __tlbi_user_level(op, arg, level) do {				\
123	if (arm64_kernel_unmapped_at_el0())				\
124		__tlbi_level(op, (arg | USER_ASID_FLAG), level);	\
125} while (0)
126
127/*
128 * This macro creates a properly formatted VA operand for the TLB RANGE. The
129 * value bit assignments are:
130 *
131 * +----------+------+-------+-------+-------+----------------------+
132 * |   ASID   |  TG  | SCALE |  NUM  |  TTL  |        BADDR         |
133 * +-----------------+-------+-------+-------+----------------------+
134 * |63      48|47  46|45   44|43   39|38   37|36                   0|
135 *
136 * The address range is determined by below formula: [BADDR, BADDR + (NUM + 1) *
137 * 2^(5*SCALE + 1) * PAGESIZE)
138 *
139 * Note that the first argument, baddr, is pre-shifted; If LPA2 is in use, BADDR
140 * holds addr[52:16]. Else BADDR holds page number. See for example ARM DDI
141 * 0487J.a section C5.5.60 "TLBI VAE1IS, TLBI VAE1ISNXS, TLB Invalidate by VA,
142 * EL1, Inner Shareable".
143 *
144 */
145#define TLBIR_ASID_MASK		GENMASK_ULL(63, 48)
146#define TLBIR_TG_MASK		GENMASK_ULL(47, 46)
147#define TLBIR_SCALE_MASK	GENMASK_ULL(45, 44)
148#define TLBIR_NUM_MASK		GENMASK_ULL(43, 39)
149#define TLBIR_TTL_MASK		GENMASK_ULL(38, 37)
150#define TLBIR_BADDR_MASK	GENMASK_ULL(36,  0)
151
152#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl)		\
153	({								\
154		unsigned long __ta = 0;					\
155		unsigned long __ttl = (ttl >= 1 && ttl <= 3) ? ttl : 0;	\
156		__ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr);		\
157		__ta |= FIELD_PREP(TLBIR_TTL_MASK, __ttl);		\
158		__ta |= FIELD_PREP(TLBIR_NUM_MASK, num);		\
159		__ta |= FIELD_PREP(TLBIR_SCALE_MASK, scale);		\
160		__ta |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule());	\
161		__ta |= FIELD_PREP(TLBIR_ASID_MASK, asid);		\
162		__ta;							\
163	})
164
165/* These macros are used by the TLBI RANGE feature. */
166#define __TLBI_RANGE_PAGES(num, scale)	\
167	((unsigned long)((num) + 1) << (5 * (scale) + 1))
168#define MAX_TLBI_RANGE_PAGES		__TLBI_RANGE_PAGES(31, 3)
169
170/*
171 * Generate 'num' values from -1 to 31 with -1 rejected by the
172 * __flush_tlb_range() loop below. Its return value is only
173 * significant for a maximum of MAX_TLBI_RANGE_PAGES pages. If
174 * 'pages' is more than that, you must iterate over the overall
175 * range.
176 */
177#define __TLBI_RANGE_NUM(pages, scale)					\
178	({								\
179		int __pages = min((pages),				\
180				  __TLBI_RANGE_PAGES(31, (scale)));	\
181		(__pages >> (5 * (scale) + 1)) - 1;			\
182	})
183
184/*
185 *	TLB Invalidation
186 *	================
187 *
188 * 	This header file implements the low-level TLB invalidation routines
189 *	(sometimes referred to as "flushing" in the kernel) for arm64.
190 *
191 *	Every invalidation operation uses the following template:
192 *
193 *	DSB ISHST	// Ensure prior page-table updates have completed
194 *	TLBI ...	// Invalidate the TLB
195 *	DSB ISH		// Ensure the TLB invalidation has completed
196 *      if (invalidated kernel mappings)
197 *		ISB	// Discard any instructions fetched from the old mapping
198 *
199 *
200 *	The following functions form part of the "core" TLB invalidation API,
201 *	as documented in Documentation/core-api/cachetlb.rst:
202 *
203 *	flush_tlb_all()
204 *		Invalidate the entire TLB (kernel + user) on all CPUs
205 *
206 *	flush_tlb_mm(mm)
207 *		Invalidate an entire user address space on all CPUs.
208 *		The 'mm' argument identifies the ASID to invalidate.
209 *
210 *	flush_tlb_range(vma, start, end)
211 *		Invalidate the virtual-address range '[start, end)' on all
212 *		CPUs for the user address space corresponding to 'vma->mm'.
213 *		Note that this operation also invalidates any walk-cache
214 *		entries associated with translations for the specified address
215 *		range.
216 *
217 *	flush_tlb_kernel_range(start, end)
218 *		Same as flush_tlb_range(..., start, end), but applies to
219 * 		kernel mappings rather than a particular user address space.
220 *		Whilst not explicitly documented, this function is used when
221 *		unmapping pages from vmalloc/io space.
222 *
223 *	flush_tlb_page(vma, addr)
224 *		Invalidate a single user mapping for address 'addr' in the
225 *		address space corresponding to 'vma->mm'.  Note that this
226 *		operation only invalidates a single, last-level page-table
227 *		entry and therefore does not affect any walk-caches.
228 *
229 *
230 *	Next, we have some undocumented invalidation routines that you probably
231 *	don't want to call unless you know what you're doing:
232 *
233 *	local_flush_tlb_all()
234 *		Same as flush_tlb_all(), but only applies to the calling CPU.
235 *
236 *	__flush_tlb_kernel_pgtable(addr)
237 *		Invalidate a single kernel mapping for address 'addr' on all
238 *		CPUs, ensuring that any walk-cache entries associated with the
239 *		translation are also invalidated.
240 *
241 *	__flush_tlb_range(vma, start, end, stride, last_level, tlb_level)
242 *		Invalidate the virtual-address range '[start, end)' on all
243 *		CPUs for the user address space corresponding to 'vma->mm'.
244 *		The invalidation operations are issued at a granularity
245 *		determined by 'stride' and only affect any walk-cache entries
246 *		if 'last_level' is equal to false. tlb_level is the level at
247 *		which the invalidation must take place. If the level is wrong,
248 *		no invalidation may take place. In the case where the level
249 *		cannot be easily determined, the value TLBI_TTL_UNKNOWN will
250 *		perform a non-hinted invalidation.
251 *
252 *	local_flush_tlb_page(vma, addr)
253 *		Local variant of flush_tlb_page().  Stale TLB entries may
254 *		remain in remote CPUs.
255 *
256 *	local_flush_tlb_page_nonotify(vma, addr)
257 *		Same as local_flush_tlb_page() except MMU notifier will not be
258 *		called.
259 *
260 *	local_flush_tlb_contpte(vma, addr)
261 *		Invalidate the virtual-address range
262 *		'[addr, addr+CONT_PTE_SIZE)' mapped with contpte on local CPU
263 *		for the user address space corresponding to 'vma->mm'.  Stale
264 *		TLB entries may remain in remote CPUs.
265 *
266 *	Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
267 *	on top of these routines, since that is our interface to the mmu_gather
268 *	API as used by munmap() and friends.
269 */
270static inline void local_flush_tlb_all(void)
271{
272	dsb(nshst);
273	__tlbi(vmalle1);
274	dsb(nsh);
275	isb();
276}
277
278static inline void flush_tlb_all(void)
279{
280	dsb(ishst);
281	__tlbi(vmalle1is);
282	dsb(ish);
283	isb();
284}
285
286static inline void flush_tlb_mm(struct mm_struct *mm)
287{
288	unsigned long asid;
289
290	dsb(ishst);
291	asid = __TLBI_VADDR(0, ASID(mm));
292	__tlbi(aside1is, asid);
293	__tlbi_user(aside1is, asid);
294	dsb(ish);
295	mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
296}
297
298static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct *mm,
299							  unsigned long uaddr)
300{
301	unsigned long addr;
302
303	dsb(nshst);
304	addr = __TLBI_VADDR(uaddr, ASID(mm));
305	__tlbi(vale1, addr);
306	__tlbi_user(vale1, addr);
307}
308
309static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vma,
310						 unsigned long uaddr)
311{
312	__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
313	dsb(nsh);
314}
315
316static inline void local_flush_tlb_page(struct vm_area_struct *vma,
317					unsigned long uaddr)
318{
319	__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
320	mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK,
321						(uaddr & PAGE_MASK) + PAGE_SIZE);
322	dsb(nsh);
323}
324
325static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
326					   unsigned long uaddr)
327{
328	unsigned long addr;
329
330	dsb(ishst);
331	addr = __TLBI_VADDR(uaddr, ASID(mm));
332	__tlbi(vale1is, addr);
333	__tlbi_user(vale1is, addr);
334	mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
335						(uaddr & PAGE_MASK) + PAGE_SIZE);
336}
337
338static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
339					 unsigned long uaddr)
340{
341	return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
342}
343
344static inline void flush_tlb_page(struct vm_area_struct *vma,
345				  unsigned long uaddr)
346{
347	flush_tlb_page_nosync(vma, uaddr);
348	dsb(ish);
349}
350
351static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
352{
353	/*
354	 * TLB flush deferral is not required on systems which are affected by
355	 * ARM64_WORKAROUND_REPEAT_TLBI, as __tlbi()/__tlbi_user() implementation
356	 * will have two consecutive TLBI instructions with a dsb(ish) in between
357	 * defeating the purpose (i.e save overall 'dsb ish' cost).
358	 */
359	if (alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI))
360		return false;
361
362	return true;
363}
364
365/*
366 * To support TLB batched flush for multiple pages unmapping, we only send
367 * the TLBI for each page in arch_tlbbatch_add_pending() and wait for the
368 * completion at the end in arch_tlbbatch_flush(). Since we've already issued
369 * TLBI for each page so only a DSB is needed to synchronise its effect on the
370 * other CPUs.
371 *
372 * This will save the time waiting on DSB comparing issuing a TLBI;DSB sequence
373 * for each page.
374 */
375static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
376{
377	dsb(ish);
378}
379
380/*
381 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
382 * necessarily a performance improvement.
383 */
384#define MAX_DVM_OPS	PTRS_PER_PTE
385
386/*
387 * __flush_tlb_range_op - Perform TLBI operation upon a range
388 *
389 * @op:	TLBI instruction that operates on a range (has 'r' prefix)
390 * @start:	The start address of the range
391 * @pages:	Range as the number of pages from 'start'
392 * @stride:	Flush granularity
393 * @asid:	The ASID of the task (0 for IPA instructions)
394 * @tlb_level:	Translation Table level hint, if known
395 * @tlbi_user:	If 'true', call an additional __tlbi_user()
396 *              (typically for user ASIDs). 'flase' for IPA instructions
397 * @lpa2:	If 'true', the lpa2 scheme is used as set out below
398 *
399 * When the CPU does not support TLB range operations, flush the TLB
400 * entries one by one at the granularity of 'stride'. If the TLB
401 * range ops are supported, then:
402 *
403 * 1. If FEAT_LPA2 is in use, the start address of a range operation must be
404 *    64KB aligned, so flush pages one by one until the alignment is reached
405 *    using the non-range operations. This step is skipped if LPA2 is not in
406 *    use.
407 *
408 * 2. The minimum range granularity is decided by 'scale', so multiple range
409 *    TLBI operations may be required. Start from scale = 3, flush the largest
410 *    possible number of pages ((num+1)*2^(5*scale+1)) that fit into the
411 *    requested range, then decrement scale and continue until one or zero pages
412 *    are left. We must start from highest scale to ensure 64KB start alignment
413 *    is maintained in the LPA2 case.
414 *
415 * 3. If there is 1 page remaining, flush it through non-range operations. Range
416 *    operations can only span an even number of pages. We save this for last to
417 *    ensure 64KB start alignment is maintained for the LPA2 case.
418 */
419#define __flush_tlb_range_op(op, start, pages, stride,			\
420				asid, tlb_level, tlbi_user, lpa2)	\
421do {									\
422	typeof(start) __flush_start = start;				\
423	typeof(pages) __flush_pages = pages;				\
424	int num = 0;							\
425	int scale = 3;							\
426	int shift = lpa2 ? 16 : PAGE_SHIFT;				\
427	unsigned long addr;						\
428									\
429	while (__flush_pages > 0) {					\
430		if (!system_supports_tlb_range() ||			\
431		    __flush_pages == 1 ||				\
432		    (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) {	\
433			addr = __TLBI_VADDR(__flush_start, asid);	\
434			__tlbi_level(op, addr, tlb_level);		\
435			if (tlbi_user)					\
436				__tlbi_user_level(op, addr, tlb_level);	\
437			__flush_start += stride;			\
438			__flush_pages -= stride >> PAGE_SHIFT;		\
439			continue;					\
440		}							\
441									\
442		num = __TLBI_RANGE_NUM(__flush_pages, scale);		\
443		if (num >= 0) {						\
444			addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \
445						scale, num, tlb_level);	\
446			__tlbi(r##op, addr);				\
447			if (tlbi_user)					\
448				__tlbi_user(r##op, addr);		\
449			__flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
450			__flush_pages -= __TLBI_RANGE_PAGES(num, scale);\
451		}							\
452		scale--;						\
453	}								\
454} while (0)
455
456#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
457	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_lpa2_is_enabled());
458
459static inline bool __flush_tlb_range_limit_excess(unsigned long start,
460		unsigned long end, unsigned long pages, unsigned long stride)
461{
462	/*
463	 * When the system does not support TLB range based flush
464	 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
465	 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
466	 * pages can be handled.
467	 */
468	if ((!system_supports_tlb_range() &&
469	     (end - start) >= (MAX_DVM_OPS * stride)) ||
470	    pages > MAX_TLBI_RANGE_PAGES)
471		return true;
472
473	return false;
474}
475
476static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
477				     unsigned long start, unsigned long end,
478				     unsigned long stride, bool last_level,
479				     int tlb_level)
480{
481	unsigned long asid, pages;
482
483	start = round_down(start, stride);
484	end = round_up(end, stride);
485	pages = (end - start) >> PAGE_SHIFT;
486
487	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
488		flush_tlb_mm(mm);
489		return;
490	}
491
492	dsb(ishst);
493	asid = ASID(mm);
494
495	if (last_level)
496		__flush_tlb_range_op(vale1is, start, pages, stride, asid,
497				     tlb_level, true, lpa2_is_enabled());
498	else
499		__flush_tlb_range_op(vae1is, start, pages, stride, asid,
500				     tlb_level, true, lpa2_is_enabled());
501
502	mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
503}
504
505static inline void __flush_tlb_range(struct vm_area_struct *vma,
506				     unsigned long start, unsigned long end,
507				     unsigned long stride, bool last_level,
508				     int tlb_level)
509{
510	__flush_tlb_range_nosync(vma->vm_mm, start, end, stride,
511				 last_level, tlb_level);
512	dsb(ish);
513}
514
515static inline void local_flush_tlb_contpte(struct vm_area_struct *vma,
516					   unsigned long addr)
517{
518	unsigned long asid;
519
520	addr = round_down(addr, CONT_PTE_SIZE);
521
522	dsb(nshst);
523	asid = ASID(vma->vm_mm);
524	__flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid,
525			     3, true, lpa2_is_enabled());
526	mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr,
527						    addr + CONT_PTE_SIZE);
528	dsb(nsh);
529}
530
531static inline void flush_tlb_range(struct vm_area_struct *vma,
532				   unsigned long start, unsigned long end)
533{
534	/*
535	 * We cannot use leaf-only invalidation here, since we may be invalidating
536	 * table entries as part of collapsing hugepages or moving page tables.
537	 * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough
538	 * information here.
539	 */
540	__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
541}
542
543static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
544{
545	const unsigned long stride = PAGE_SIZE;
546	unsigned long pages;
547
548	start = round_down(start, stride);
549	end = round_up(end, stride);
550	pages = (end - start) >> PAGE_SHIFT;
551
552	if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
553		flush_tlb_all();
554		return;
555	}
556
557	dsb(ishst);
558	__flush_tlb_range_op(vaale1is, start, pages, stride, 0,
559			     TLBI_TTL_UNKNOWN, false, lpa2_is_enabled());
560	dsb(ish);
561	isb();
562}
563
564/*
565 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
566 * table levels (pgd/pud/pmd).
567 */
568static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
569{
570	unsigned long addr = __TLBI_VADDR(kaddr, 0);
571
572	dsb(ishst);
573	__tlbi(vaae1is, addr);
574	dsb(ish);
575	isb();
576}
577
578static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
579		struct mm_struct *mm, unsigned long start, unsigned long end)
580{
581	__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
582}
583
584static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)
585{
586	ptdesc_t diff = oldval ^ newval;
587
588	/* invalid to valid transition requires no flush */
589	if (!(oldval & PTE_VALID))
590		return false;
591
592	/* Transition in the SW bits requires no flush */
593	diff &= ~PTE_SWBITS_MASK;
594
595	return diff;
596}
597
598static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
599{
600	return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte));
601}
602#define pte_needs_flush pte_needs_flush
603
604static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
605{
606	return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd));
607}
608#define huge_pmd_needs_flush huge_pmd_needs_flush
609
610#endif
611
612#endif