pt_reclaim.c

 1// SPDX-License-Identifier: GPL-2.0
 2#include <linux/hugetlb.h>
 3#include <linux/pgalloc.h>
 4
 5#include <asm-generic/tlb.h>
 6
 7#include "internal.h"
 8
 9bool reclaim_pt_is_enabled(unsigned long start, unsigned long end,
10			   struct zap_details *details)
11{
12	return details && details->reclaim_pt && (end - start >= PMD_SIZE);
13}
14
15bool try_get_and_clear_pmd(struct mm_struct *mm, pmd_t *pmd, pmd_t *pmdval)
16{
17	spinlock_t *pml = pmd_lockptr(mm, pmd);
18
19	if (!spin_trylock(pml))
20		return false;
21
22	*pmdval = pmdp_get_lockless(pmd);
23	pmd_clear(pmd);
24	spin_unlock(pml);
25
26	return true;
27}
28
29void free_pte(struct mm_struct *mm, unsigned long addr, struct mmu_gather *tlb,
30	      pmd_t pmdval)
31{
32	pte_free_tlb(tlb, pmd_pgtable(pmdval), addr);
33	mm_dec_nr_ptes(mm);
34}
35
36void try_to_free_pte(struct mm_struct *mm, pmd_t *pmd, unsigned long addr,
37		     struct mmu_gather *tlb)
38{
39	pmd_t pmdval;
40	spinlock_t *pml, *ptl = NULL;
41	pte_t *start_pte, *pte;
42	int i;
43
44	pml = pmd_lock(mm, pmd);
45	start_pte = pte_offset_map_rw_nolock(mm, pmd, addr, &pmdval, &ptl);
46	if (!start_pte)
47		goto out_ptl;
48	if (ptl != pml)
49		spin_lock_nested(ptl, SINGLE_DEPTH_NESTING);
50
51	/* Check if it is empty PTE page */
52	for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) {
53		if (!pte_none(ptep_get(pte)))
54			goto out_ptl;
55	}
56	pte_unmap(start_pte);
57
58	pmd_clear(pmd);
59
60	if (ptl != pml)
61		spin_unlock(ptl);
62	spin_unlock(pml);
63
64	free_pte(mm, addr, tlb, pmdval);
65
66	return;
67out_ptl:
68	if (start_pte)
69		pte_unmap_unlock(start_pte, ptl);
70	if (ptl != pml)
71		spin_unlock(pml);
72}