pgtable-generic.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *  mm/pgtable-generic.c
  4 *
  5 *  Generic pgtable methods declared in linux/pgtable.h
  6 *
  7 *  Copyright (C) 2010  Linus Torvalds
  8 */
  9
 10#include <linux/pagemap.h>
 11#include <linux/hugetlb.h>
 12#include <linux/pgtable.h>
 13#include <linux/swap.h>
 14#include <linux/swapops.h>
 15#include <linux/mm_inline.h>
 16#include <linux/iommu.h>
 17#include <linux/pgalloc.h>
 18
 19#include <asm/tlb.h>
 20
 21/*
 22 * If a p?d_bad entry is found while walking page tables, report
 23 * the error, before resetting entry to p?d_none.  Usually (but
 24 * very seldom) called out from the p?d_none_or_clear_bad macros.
 25 */
 26
 27void pgd_clear_bad(pgd_t *pgd)
 28{
 29	pgd_ERROR(*pgd);
 30	pgd_clear(pgd);
 31}
 32
 33#ifndef __PAGETABLE_P4D_FOLDED
 34void p4d_clear_bad(p4d_t *p4d)
 35{
 36	p4d_ERROR(*p4d);
 37	p4d_clear(p4d);
 38}
 39#endif
 40
 41#ifndef __PAGETABLE_PUD_FOLDED
 42void pud_clear_bad(pud_t *pud)
 43{
 44	pud_ERROR(*pud);
 45	pud_clear(pud);
 46}
 47#endif
 48
 49/*
 50 * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
 51 * above. pmd folding is special and typically pmd_* macros refer to upper
 52 * level even when folded
 53 */
 54void pmd_clear_bad(pmd_t *pmd)
 55{
 56	pmd_ERROR(*pmd);
 57	pmd_clear(pmd);
 58}
 59
 60#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
 61/*
 62 * Only sets the access flags (dirty, accessed), as well as write
 63 * permission. Furthermore, we know it always gets set to a "more
 64 * permissive" setting, which allows most architectures to optimize
 65 * this. We return whether the PTE actually changed, which in turn
 66 * instructs the caller to do things like update__mmu_cache.  This
 67 * used to be done in the caller, but sparc needs minor faults to
 68 * force that call on sun4c so we changed this macro slightly
 69 */
 70int ptep_set_access_flags(struct vm_area_struct *vma,
 71			  unsigned long address, pte_t *ptep,
 72			  pte_t entry, int dirty)
 73{
 74	int changed = !pte_same(ptep_get(ptep), entry);
 75	if (changed) {
 76		set_pte_at(vma->vm_mm, address, ptep, entry);
 77		flush_tlb_fix_spurious_fault(vma, address, ptep);
 78	}
 79	return changed;
 80}
 81#endif
 82
 83#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
 84int ptep_clear_flush_young(struct vm_area_struct *vma,
 85			   unsigned long address, pte_t *ptep)
 86{
 87	int young;
 88	young = ptep_test_and_clear_young(vma, address, ptep);
 89	if (young)
 90		flush_tlb_page(vma, address);
 91	return young;
 92}
 93#endif
 94
 95#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
 96pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
 97		       pte_t *ptep)
 98{
 99	struct mm_struct *mm = (vma)->vm_mm;
100	pte_t pte;
101	pte = ptep_get_and_clear(mm, address, ptep);
102	if (pte_accessible(mm, pte))
103		flush_tlb_page(vma, address);
104	return pte;
105}
106#endif
107
108#ifdef CONFIG_TRANSPARENT_HUGEPAGE
109
110#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
111int pmdp_set_access_flags(struct vm_area_struct *vma,
112			  unsigned long address, pmd_t *pmdp,
113			  pmd_t entry, int dirty)
114{
115	int changed = !pmd_same(*pmdp, entry);
116	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
117	if (changed) {
118		set_pmd_at(vma->vm_mm, address, pmdp, entry);
119		flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
120	}
121	return changed;
122}
123#endif
124
125#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
126int pmdp_clear_flush_young(struct vm_area_struct *vma,
127			   unsigned long address, pmd_t *pmdp)
128{
129	int young;
130	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
131	young = pmdp_test_and_clear_young(vma, address, pmdp);
132	if (young)
133		flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
134	return young;
135}
136#endif
137
138#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
139pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
140			    pmd_t *pmdp)
141{
142	pmd_t pmd;
143	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
144	VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp));
145	pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
146	flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
147	return pmd;
148}
149
150#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
151pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
152			    pud_t *pudp)
153{
154	pud_t pud;
155
156	VM_BUG_ON(address & ~HPAGE_PUD_MASK);
157	VM_BUG_ON(!pud_trans_huge(*pudp));
158	pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
159	flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
160	return pud;
161}
162#endif
163#endif
164
165#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
166void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
167				pgtable_t pgtable)
168{
169	assert_spin_locked(pmd_lockptr(mm, pmdp));
170
171	/* FIFO */
172	if (!pmd_huge_pte(mm, pmdp))
173		INIT_LIST_HEAD(&pgtable->lru);
174	else
175		list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
176	pmd_huge_pte(mm, pmdp) = pgtable;
177}
178#endif
179
180#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
181/* no "address" argument so destroys page coloring of some arch */
182pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
183{
184	pgtable_t pgtable;
185
186	assert_spin_locked(pmd_lockptr(mm, pmdp));
187
188	/* FIFO */
189	pgtable = pmd_huge_pte(mm, pmdp);
190	pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
191							  struct page, lru);
192	if (pmd_huge_pte(mm, pmdp))
193		list_del(&pgtable->lru);
194	return pgtable;
195}
196#endif
197
198#ifndef __HAVE_ARCH_PMDP_INVALIDATE
199pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
200		     pmd_t *pmdp)
201{
202	VM_WARN_ON_ONCE(!pmd_present(*pmdp));
203	pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
204	flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
205	return old;
206}
207#endif
208
209#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
210pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
211			 pmd_t *pmdp)
212{
213	VM_WARN_ON_ONCE(!pmd_present(*pmdp));
214	return pmdp_invalidate(vma, address, pmdp);
215}
216#endif
217
218#ifndef pmdp_collapse_flush
219pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
220			  pmd_t *pmdp)
221{
222	/*
223	 * pmd and hugepage pte format are same. So we could
224	 * use the same function.
225	 */
226	pmd_t pmd;
227
228	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
229	VM_BUG_ON(pmd_trans_huge(*pmdp));
230	pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
231
232	/* collapse entails shooting down ptes not pmd */
233	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
234	return pmd;
235}
236#endif
237
238/* arch define pte_free_defer in asm/pgalloc.h for its own implementation */
239#ifndef pte_free_defer
240static void pte_free_now(struct rcu_head *head)
241{
242	struct page *page;
243
244	page = container_of(head, struct page, rcu_head);
245	pte_free(NULL /* mm not passed and not used */, (pgtable_t)page);
246}
247
248void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
249{
250	struct page *page;
251
252	page = pgtable;
253	call_rcu(&page->rcu_head, pte_free_now);
254}
255#endif /* pte_free_defer */
256#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
257
258#if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \
259	(defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RCU))
260/*
261 * See the comment above ptep_get_lockless() in include/linux/pgtable.h:
262 * the barriers in pmdp_get_lockless() cannot guarantee that the value in
263 * pmd_high actually belongs with the value in pmd_low; but holding interrupts
264 * off blocks the TLB flush between present updates, which guarantees that a
265 * successful __pte_offset_map() points to a page from matched halves.
266 */
267static unsigned long pmdp_get_lockless_start(void)
268{
269	unsigned long irqflags;
270
271	local_irq_save(irqflags);
272	return irqflags;
273}
274static void pmdp_get_lockless_end(unsigned long irqflags)
275{
276	local_irq_restore(irqflags);
277}
278#else
279static unsigned long pmdp_get_lockless_start(void) { return 0; }
280static void pmdp_get_lockless_end(unsigned long irqflags) { }
281#endif
282
283pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
284{
285	unsigned long irqflags;
286	pmd_t pmdval;
287
288	rcu_read_lock();
289	irqflags = pmdp_get_lockless_start();
290	pmdval = pmdp_get_lockless(pmd);
291	pmdp_get_lockless_end(irqflags);
292
293	if (pmdvalp)
294		*pmdvalp = pmdval;
295	if (unlikely(pmd_none(pmdval) || !pmd_present(pmdval)))
296		goto nomap;
297	if (unlikely(pmd_trans_huge(pmdval)))
298		goto nomap;
299	if (unlikely(pmd_bad(pmdval))) {
300		pmd_clear_bad(pmd);
301		goto nomap;
302	}
303	return __pte_map(&pmdval, addr);
304nomap:
305	rcu_read_unlock();
306	return NULL;
307}
308
309pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
310				unsigned long addr, spinlock_t **ptlp)
311{
312	pmd_t pmdval;
313	pte_t *pte;
314
315	pte = __pte_offset_map(pmd, addr, &pmdval);
316	if (likely(pte))
317		*ptlp = pte_lockptr(mm, &pmdval);
318	return pte;
319}
320
321pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
322				unsigned long addr, pmd_t *pmdvalp,
323				spinlock_t **ptlp)
324{
325	pte_t *pte;
326
327	VM_WARN_ON_ONCE(!pmdvalp);
328	pte = __pte_offset_map(pmd, addr, pmdvalp);
329	if (likely(pte))
330		*ptlp = pte_lockptr(mm, pmdvalp);
331	return pte;
332}
333
334/*
335 * pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
336 * __pte_offset_map_lock() below, is usually called with the pmd pointer for
337 * addr, reached by walking down the mm's pgd, p4d, pud for addr: either while
338 * holding mmap_lock or vma lock for read or for write; or in truncate or rmap
339 * context, while holding file's i_mmap_lock or anon_vma lock for read (or for
340 * write). In a few cases, it may be used with pmd pointing to a pmd_t already
341 * copied to or constructed on the stack.
342 *
343 * When successful, it returns the pte pointer for addr, with its page table
344 * kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent
345 * modification by software, with a pointer to that spinlock in ptlp (in some
346 * configs mm->page_table_lock, in SPLIT_PTLOCK configs a spinlock in table's
347 * struct page).  pte_unmap_unlock(pte, ptl) to unlock and unmap afterwards.
348 *
349 * But it is unsuccessful, returning NULL with *ptlp unchanged, if there is no
350 * page table at *pmd: if, for example, the page table has just been removed,
351 * or replaced by the huge pmd of a THP.  (When successful, *pmd is rechecked
352 * after acquiring the ptlock, and retried internally if it changed: so that a
353 * page table can be safely removed or replaced by THP while holding its lock.)
354 *
355 * pte_offset_map(pmd, addr), and its internal helper __pte_offset_map() above,
356 * just returns the pte pointer for addr, its page table kmapped if necessary;
357 * or NULL if there is no page table at *pmd.  It does not attempt to lock the
358 * page table, so cannot normally be used when the page table is to be updated,
359 * or when entries read must be stable.  But it does take rcu_read_lock(): so
360 * that even when page table is racily removed, it remains a valid though empty
361 * and disconnected table.  Until pte_unmap(pte) unmaps and rcu_read_unlock()s
362 * afterwards.
363 *
364 * pte_offset_map_ro_nolock(mm, pmd, addr, ptlp), above, is like pte_offset_map();
365 * but when successful, it also outputs a pointer to the spinlock in ptlp - as
366 * pte_offset_map_lock() does, but in this case without locking it.  This helps
367 * the caller to avoid a later pte_lockptr(mm, *pmd), which might by that time
368 * act on a changed *pmd: pte_offset_map_ro_nolock() provides the correct spinlock
369 * pointer for the page table that it returns. Even after grabbing the spinlock,
370 * we might be looking either at a page table that is still mapped or one that
371 * was unmapped and is about to get freed. But for R/O access this is sufficient.
372 * So it is only applicable for read-only cases where any modification operations
373 * to the page table are not allowed even if the corresponding spinlock is held
374 * afterwards.
375 *
376 * pte_offset_map_rw_nolock(mm, pmd, addr, pmdvalp, ptlp), above, is like
377 * pte_offset_map_ro_nolock(); but when successful, it also outputs the pdmval.
378 * It is applicable for may-write cases where any modification operations to the
379 * page table may happen after the corresponding spinlock is held afterwards.
380 * But the users should make sure the page table is stable like checking pte_same()
381 * or checking pmd_same() by using the output pmdval before performing the write
382 * operations.
383 *
384 * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will
385 * be read-only/read-write protected.
386 *
387 * Note that free_pgtables(), used after unmapping detached vmas, or when
388 * exiting the whole mm, does not take page table lock before freeing a page
389 * table, and may not use RCU at all: "outsiders" like khugepaged should avoid
390 * pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
391 */
392pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
393			     unsigned long addr, spinlock_t **ptlp)
394{
395	spinlock_t *ptl;
396	pmd_t pmdval;
397	pte_t *pte;
398again:
399	pte = __pte_offset_map(pmd, addr, &pmdval);
400	if (unlikely(!pte))
401		return pte;
402	ptl = pte_lockptr(mm, &pmdval);
403	spin_lock(ptl);
404	if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
405		*ptlp = ptl;
406		return pte;
407	}
408	pte_unmap_unlock(pte, ptl);
409	goto again;
410}
411
412#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE
413static void kernel_pgtable_work_func(struct work_struct *work);
414
415static struct {
416	struct list_head list;
417	/* protect above ptdesc lists */
418	spinlock_t lock;
419	struct work_struct work;
420} kernel_pgtable_work = {
421	.list = LIST_HEAD_INIT(kernel_pgtable_work.list),
422	.lock = __SPIN_LOCK_UNLOCKED(kernel_pgtable_work.lock),
423	.work = __WORK_INITIALIZER(kernel_pgtable_work.work, kernel_pgtable_work_func),
424};
425
426static void kernel_pgtable_work_func(struct work_struct *work)
427{
428	struct ptdesc *pt, *next;
429	LIST_HEAD(page_list);
430
431	spin_lock(&kernel_pgtable_work.lock);
432	list_splice_tail_init(&kernel_pgtable_work.list, &page_list);
433	spin_unlock(&kernel_pgtable_work.lock);
434
435	iommu_sva_invalidate_kva_range(PAGE_OFFSET, TLB_FLUSH_ALL);
436	list_for_each_entry_safe(pt, next, &page_list, pt_list)
437		__pagetable_free(pt);
438}
439
440void pagetable_free_kernel(struct ptdesc *pt)
441{
442	spin_lock(&kernel_pgtable_work.lock);
443	list_add(&pt->pt_list, &kernel_pgtable_work.list);
444	spin_unlock(&kernel_pgtable_work.lock);
445
446	schedule_work(&kernel_pgtable_work.work);
447}
448#endif