tlbflush.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <linux/mm.h>
  4#include <linux/smp.h>
  5#include <linux/sched.h>
  6#include <linux/hugetlb.h>
  7#include <linux/mmu_notifier.h>
  8#include <asm/sbi.h>
  9#include <asm/mmu_context.h>
 10#include <asm/cpufeature.h>
 11
 12#define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 13
 14static inline void local_sfence_inval_ir(void)
 15{
 16	asm volatile(SFENCE_INVAL_IR() ::: "memory");
 17}
 18
 19static inline void local_sfence_w_inval(void)
 20{
 21	asm volatile(SFENCE_W_INVAL() ::: "memory");
 22}
 23
 24static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
 25{
 26	if (asid != FLUSH_TLB_NO_ASID)
 27		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
 28	else
 29		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
 30}
 31
 32/*
 33 * Flush entire TLB if number of entries to be flushed is greater
 34 * than the threshold below.
 35 */
 36unsigned long tlb_flush_all_threshold __read_mostly = 64;
 37
 38static void local_flush_tlb_range_threshold_asid(unsigned long start,
 39						 unsigned long size,
 40						 unsigned long stride,
 41						 unsigned long asid)
 42{
 43	unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride);
 44	int i;
 45
 46	if (nr_ptes_in_range > tlb_flush_all_threshold) {
 47		local_flush_tlb_all_asid(asid);
 48		return;
 49	}
 50
 51	if (has_svinval()) {
 52		local_sfence_w_inval();
 53		for (i = 0; i < nr_ptes_in_range; ++i) {
 54			local_sinval_vma(start, asid);
 55			start += stride;
 56		}
 57		local_sfence_inval_ir();
 58		return;
 59	}
 60
 61	for (i = 0; i < nr_ptes_in_range; ++i) {
 62		local_flush_tlb_page_asid(start, asid);
 63		start += stride;
 64	}
 65}
 66
 67static inline void local_flush_tlb_range_asid(unsigned long start,
 68		unsigned long size, unsigned long stride, unsigned long asid)
 69{
 70	if (size <= stride)
 71		local_flush_tlb_page_asid(start, asid);
 72	else if (size == FLUSH_TLB_MAX_SIZE)
 73		local_flush_tlb_all_asid(asid);
 74	else
 75		local_flush_tlb_range_threshold_asid(start, size, stride, asid);
 76}
 77
 78/* Flush a range of kernel pages without broadcasting */
 79void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 80{
 81	local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
 82}
 83
 84static void __ipi_flush_tlb_all(void *info)
 85{
 86	local_flush_tlb_all();
 87}
 88
 89void flush_tlb_all(void)
 90{
 91	if (num_online_cpus() < 2)
 92		local_flush_tlb_all();
 93	else if (riscv_use_sbi_for_rfence())
 94		sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID);
 95	else
 96		on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
 97}
 98
 99struct flush_tlb_range_data {
100	unsigned long asid;
101	unsigned long start;
102	unsigned long size;
103	unsigned long stride;
104};
105
106static void __ipi_flush_tlb_range_asid(void *info)
107{
108	struct flush_tlb_range_data *d = info;
109
110	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
111}
112
113static inline unsigned long get_mm_asid(struct mm_struct *mm)
114{
115	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
116}
117
118static void __flush_tlb_range(struct mm_struct *mm,
119			      const struct cpumask *cmask,
120			      unsigned long start, unsigned long size,
121			      unsigned long stride)
122{
123	unsigned long asid = get_mm_asid(mm);
124	unsigned int cpu;
125
126	if (cpumask_empty(cmask))
127		return;
128
129	cpu = get_cpu();
130
131	/* Check if the TLB flush needs to be sent to other CPUs. */
132	if (cpumask_any_but(cmask, cpu) >= nr_cpu_ids) {
133		local_flush_tlb_range_asid(start, size, stride, asid);
134	} else if (riscv_use_sbi_for_rfence()) {
135		sbi_remote_sfence_vma_asid(cmask, start, size, asid);
136	} else {
137		struct flush_tlb_range_data ftd;
138
139		ftd.asid = asid;
140		ftd.start = start;
141		ftd.size = size;
142		ftd.stride = stride;
143		on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1);
144	}
145
146	put_cpu();
147
148	if (mm)
149		mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size);
150}
151
152void flush_tlb_mm(struct mm_struct *mm)
153{
154	__flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
155}
156
157void flush_tlb_mm_range(struct mm_struct *mm,
158			unsigned long start, unsigned long end,
159			unsigned int page_size)
160{
161	__flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size);
162}
163
164void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
165{
166	__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
167			  addr, PAGE_SIZE, PAGE_SIZE);
168}
169
170void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
171		     unsigned long end)
172{
173	unsigned long stride_size;
174
175	if (!is_vm_hugetlb_page(vma)) {
176		stride_size = PAGE_SIZE;
177	} else {
178		stride_size = huge_page_size(hstate_vma(vma));
179
180		/*
181		 * As stated in the privileged specification, every PTE in a
182		 * NAPOT region must be invalidated, so reset the stride in that
183		 * case.
184		 */
185		if (has_svnapot()) {
186			if (stride_size >= PGDIR_SIZE)
187				stride_size = PGDIR_SIZE;
188			else if (stride_size >= P4D_SIZE)
189				stride_size = P4D_SIZE;
190			else if (stride_size >= PUD_SIZE)
191				stride_size = PUD_SIZE;
192			else if (stride_size >= PMD_SIZE)
193				stride_size = PMD_SIZE;
194			else
195				stride_size = PAGE_SIZE;
196		}
197	}
198
199	__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
200			  start, end - start, stride_size);
201}
202
203void flush_tlb_kernel_range(unsigned long start, unsigned long end)
204{
205	__flush_tlb_range(NULL, cpu_online_mask,
206			  start, end - start, PAGE_SIZE);
207}
208
209#ifdef CONFIG_TRANSPARENT_HUGEPAGE
210void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
211			unsigned long end)
212{
213	__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
214			  start, end - start, PMD_SIZE);
215}
216
217void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start,
218			 unsigned long end)
219{
220	__flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
221			  start, end - start, PUD_SIZE);
222}
223#endif
224
225bool arch_tlbbatch_should_defer(struct mm_struct *mm)
226{
227	return true;
228}
229
230void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
231		struct mm_struct *mm, unsigned long start, unsigned long end)
232{
233	cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
234	mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
235}
236
237void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
238{
239	__flush_tlb_range(NULL, &batch->cpumask,
240			  0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
241	cpumask_clear(&batch->cpumask);
242}