tlb.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* arch/sparc64/mm/tlb.c
  3 *
  4 * Copyright (C) 2004 David S. Miller <davem@redhat.com>
  5 */
  6
  7#include <linux/kernel.h>
  8#include <linux/percpu.h>
  9#include <linux/mm.h>
 10#include <linux/swap.h>
 11#include <linux/preempt.h>
 12#include <linux/pagemap.h>
 13
 14#include <asm/tlbflush.h>
 15#include <asm/cacheflush.h>
 16#include <asm/mmu_context.h>
 17#include <asm/tlb.h>
 18
 19/* Heavily inspired by the ppc64 code.  */
 20
 21static DEFINE_PER_CPU(struct tlb_batch, tlb_batch);
 22
 23void flush_tlb_pending(void)
 24{
 25	struct tlb_batch *tb = &get_cpu_var(tlb_batch);
 26	struct mm_struct *mm = tb->mm;
 27
 28	if (!tb->tlb_nr)
 29		goto out;
 30
 31	flush_tsb_user(tb);
 32
 33	if (CTX_VALID(mm->context)) {
 34		if (tb->tlb_nr == 1) {
 35			global_flush_tlb_page(mm, tb->vaddrs[0]);
 36		} else {
 37#ifdef CONFIG_SMP
 38			smp_flush_tlb_pending(tb->mm, tb->tlb_nr,
 39					      &tb->vaddrs[0]);
 40#else
 41			__flush_tlb_pending(CTX_HWBITS(tb->mm->context),
 42					    tb->tlb_nr, &tb->vaddrs[0]);
 43#endif
 44		}
 45	}
 46
 47	tb->tlb_nr = 0;
 48
 49out:
 50	put_cpu_var(tlb_batch);
 51}
 52
 53void arch_enter_lazy_mmu_mode(void)
 54{
 55	struct tlb_batch *tb;
 56
 57	preempt_disable();
 58	tb = this_cpu_ptr(&tlb_batch);
 59	tb->active = 1;
 60}
 61
 62void arch_leave_lazy_mmu_mode(void)
 63{
 64	struct tlb_batch *tb = this_cpu_ptr(&tlb_batch);
 65
 66	if (tb->tlb_nr)
 67		flush_tlb_pending();
 68	tb->active = 0;
 69	preempt_enable();
 70}
 71
 72static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
 73			      bool exec, unsigned int hugepage_shift)
 74{
 75	struct tlb_batch *tb = &get_cpu_var(tlb_batch);
 76	unsigned long nr;
 77
 78	vaddr &= PAGE_MASK;
 79	if (exec)
 80		vaddr |= 0x1UL;
 81
 82	nr = tb->tlb_nr;
 83
 84	if (unlikely(nr != 0 && mm != tb->mm)) {
 85		flush_tlb_pending();
 86		nr = 0;
 87	}
 88
 89	if (!tb->active) {
 90		flush_tsb_user_page(mm, vaddr, hugepage_shift);
 91		global_flush_tlb_page(mm, vaddr);
 92		goto out;
 93	}
 94
 95	if (nr == 0) {
 96		tb->mm = mm;
 97		tb->hugepage_shift = hugepage_shift;
 98	}
 99
100	if (tb->hugepage_shift != hugepage_shift) {
101		flush_tlb_pending();
102		tb->hugepage_shift = hugepage_shift;
103		nr = 0;
104	}
105
106	tb->vaddrs[nr] = vaddr;
107	tb->tlb_nr = ++nr;
108	if (nr >= TLB_BATCH_NR)
109		flush_tlb_pending();
110
111out:
112	put_cpu_var(tlb_batch);
113}
114
115void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
116		   pte_t *ptep, pte_t orig, int fullmm,
117		   unsigned int hugepage_shift)
118{
119	if (tlb_type != hypervisor &&
120	    pte_dirty(orig)) {
121		unsigned long paddr, pfn = pte_pfn(orig);
122		struct address_space *mapping;
123		struct page *page;
124		struct folio *folio;
125
126		if (!pfn_valid(pfn))
127			goto no_cache_flush;
128
129		page = pfn_to_page(pfn);
130		if (PageReserved(page))
131			goto no_cache_flush;
132
133		/* A real file page? */
134		folio = page_folio(page);
135		mapping = folio_flush_mapping(folio);
136		if (!mapping)
137			goto no_cache_flush;
138
139		paddr = (unsigned long) page_address(page);
140		if ((paddr ^ vaddr) & (1 << 13))
141			flush_dcache_folio_all(mm, folio);
142	}
143
144no_cache_flush:
145	if (!fullmm)
146		tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift);
147}
148
149#ifdef CONFIG_TRANSPARENT_HUGEPAGE
150static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
151			       pmd_t pmd)
152{
153	unsigned long end;
154	pte_t *pte;
155
156	pte = pte_offset_map(&pmd, vaddr);
157	if (!pte)
158		return;
159	end = vaddr + HPAGE_SIZE;
160	while (vaddr < end) {
161		if (pte_val(*pte) & _PAGE_VALID) {
162			bool exec = pte_exec(*pte);
163
164			tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT);
165		}
166		pte++;
167		vaddr += PAGE_SIZE;
168	}
169	pte_unmap(pte);
170}
171
172
173static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr,
174			   pmd_t orig, pmd_t pmd)
175{
176	if (mm == &init_mm)
177		return;
178
179	if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
180		/*
181		 * Note that this routine only sets pmds for THP pages.
182		 * Hugetlb pages are handled elsewhere.  We need to check
183		 * for huge zero page.  Huge zero pages are like hugetlb
184		 * pages in that there is no RSS, but there is the need
185		 * for TSB entries.  So, huge zero page counts go into
186		 * hugetlb_pte_count.
187		 */
188		if (pmd_val(pmd) & _PAGE_PMD_HUGE) {
189			if (is_huge_zero_pmd(pmd))
190				mm->context.hugetlb_pte_count++;
191			else
192				mm->context.thp_pte_count++;
193		} else {
194			if (is_huge_zero_pmd(orig))
195				mm->context.hugetlb_pte_count--;
196			else
197				mm->context.thp_pte_count--;
198		}
199
200		/* Do not try to allocate the TSB hash table if we
201		 * don't have one already.  We have various locks held
202		 * and thus we'll end up doing a GFP_KERNEL allocation
203		 * in an atomic context.
204		 *
205		 * Instead, we let the first TLB miss on a hugepage
206		 * take care of this.
207		 */
208	}
209
210	if (!pmd_none(orig)) {
211		addr &= HPAGE_MASK;
212		if (pmd_trans_huge(orig)) {
213			pte_t orig_pte = __pte(pmd_val(orig));
214			bool exec = pte_exec(orig_pte);
215
216			tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);
217			tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
218					  REAL_HPAGE_SHIFT);
219		} else {
220			tlb_batch_pmd_scan(mm, addr, orig);
221		}
222	}
223}
224
225void set_pmd_at(struct mm_struct *mm, unsigned long addr,
226		pmd_t *pmdp, pmd_t pmd)
227{
228	pmd_t orig = *pmdp;
229
230	*pmdp = pmd;
231	__set_pmd_acct(mm, addr, orig, pmd);
232}
233
234static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
235		unsigned long address, pmd_t *pmdp, pmd_t pmd)
236{
237	pmd_t old;
238
239	do {
240		old = *pmdp;
241	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
242	__set_pmd_acct(vma->vm_mm, address, old, pmd);
243
244	return old;
245}
246
247/*
248 * This routine is only called when splitting a THP
249 */
250pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
251		     pmd_t *pmdp)
252{
253	pmd_t old, entry;
254
255	VM_WARN_ON_ONCE(!pmd_present(*pmdp));
256	entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID);
257	old = pmdp_establish(vma, address, pmdp, entry);
258	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
259
260	/*
261	 * set_pmd_at() will not be called in a way to decrement
262	 * thp_pte_count when splitting a THP, so do it now.
263	 * Sanity check pmd before doing the actual decrement.
264	 */
265	if ((pmd_val(entry) & _PAGE_PMD_HUGE) &&
266	    !is_huge_zero_pmd(entry))
267		(vma->vm_mm)->context.thp_pte_count--;
268
269	return old;
270}
271
272void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
273				pgtable_t pgtable)
274{
275	struct list_head *lh = (struct list_head *) pgtable;
276
277	assert_spin_locked(&mm->page_table_lock);
278
279	/* FIFO */
280	if (!pmd_huge_pte(mm, pmdp))
281		INIT_LIST_HEAD(lh);
282	else
283		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
284	pmd_huge_pte(mm, pmdp) = pgtable;
285}
286
287pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
288{
289	struct list_head *lh;
290	pgtable_t pgtable;
291
292	assert_spin_locked(&mm->page_table_lock);
293
294	/* FIFO */
295	pgtable = pmd_huge_pte(mm, pmdp);
296	lh = (struct list_head *) pgtable;
297	if (list_empty(lh))
298		pmd_huge_pte(mm, pmdp) = NULL;
299	else {
300		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
301		list_del(lh);
302	}
303	pte_val(pgtable[0]) = 0;
304	pte_val(pgtable[1]) = 0;
305
306	return pgtable;
307}
308#endif /* CONFIG_TRANSPARENT_HUGEPAGE */