tlb.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Based on arch/arm/include/asm/tlb.h
  4 *
  5 * Copyright (C) 2002 Russell King
  6 * Copyright (C) 2012 ARM Ltd.
  7 */
  8#ifndef __ASM_TLB_H
  9#define __ASM_TLB_H
 10
 11#include <linux/pagemap.h>
 12
 13
 14#define tlb_flush tlb_flush
 15static void tlb_flush(struct mmu_gather *tlb);
 16
 17#include <asm-generic/tlb.h>
 18
 19/*
 20 * get the tlbi levels in arm64.  Default value is TLBI_TTL_UNKNOWN if more than
 21 * one of cleared_* is set or neither is set - this elides the level hinting to
 22 * the hardware.
 23 */
 24static inline int tlb_get_level(struct mmu_gather *tlb)
 25{
 26	/* The TTL field is only valid for the leaf entry. */
 27	if (tlb->freed_tables)
 28		return TLBI_TTL_UNKNOWN;
 29
 30	if (tlb->cleared_ptes && !(tlb->cleared_pmds ||
 31				   tlb->cleared_puds ||
 32				   tlb->cleared_p4ds))
 33		return 3;
 34
 35	if (tlb->cleared_pmds && !(tlb->cleared_ptes ||
 36				   tlb->cleared_puds ||
 37				   tlb->cleared_p4ds))
 38		return 2;
 39
 40	if (tlb->cleared_puds && !(tlb->cleared_ptes ||
 41				   tlb->cleared_pmds ||
 42				   tlb->cleared_p4ds))
 43		return 1;
 44
 45	if (tlb->cleared_p4ds && !(tlb->cleared_ptes ||
 46				   tlb->cleared_pmds ||
 47				   tlb->cleared_puds))
 48		return 0;
 49
 50	return TLBI_TTL_UNKNOWN;
 51}
 52
 53static inline void tlb_flush(struct mmu_gather *tlb)
 54{
 55	struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
 56	bool last_level = !tlb->freed_tables;
 57	unsigned long stride = tlb_get_unmap_size(tlb);
 58	int tlb_level = tlb_get_level(tlb);
 59
 60	/*
 61	 * If we're tearing down the address space then we only care about
 62	 * invalidating the walk-cache, since the ASID allocator won't
 63	 * reallocate our ASID without invalidating the entire TLB.
 64	 */
 65	if (tlb->fullmm) {
 66		if (!last_level)
 67			flush_tlb_mm(tlb->mm);
 68		return;
 69	}
 70
 71	__flush_tlb_range(&vma, tlb->start, tlb->end, stride,
 72			  last_level, tlb_level);
 73}
 74
 75static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 76				  unsigned long addr)
 77{
 78	struct ptdesc *ptdesc = page_ptdesc(pte);
 79
 80	tlb_remove_ptdesc(tlb, ptdesc);
 81}
 82
 83#if CONFIG_PGTABLE_LEVELS > 2
 84static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 85				  unsigned long addr)
 86{
 87	struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
 88
 89	tlb_remove_ptdesc(tlb, ptdesc);
 90}
 91#endif
 92
 93#if CONFIG_PGTABLE_LEVELS > 3
 94static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
 95				  unsigned long addr)
 96{
 97	struct ptdesc *ptdesc = virt_to_ptdesc(pudp);
 98
 99	if (!pgtable_l4_enabled())
100		return;
101
102	tlb_remove_ptdesc(tlb, ptdesc);
103}
104#endif
105
106#if CONFIG_PGTABLE_LEVELS > 4
107static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp,
108				  unsigned long addr)
109{
110	struct ptdesc *ptdesc = virt_to_ptdesc(p4dp);
111
112	if (!pgtable_l5_enabled())
113		return;
114
115	tlb_remove_ptdesc(tlb, ptdesc);
116}
117#endif
118
119#endif