pgtable-frag.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 *  Handling Page Tables through page fragments
  5 *
  6 */
  7
  8#include <linux/kernel.h>
  9#include <linux/gfp.h>
 10#include <linux/mm.h>
 11#include <linux/percpu.h>
 12#include <linux/hardirq.h>
 13#include <linux/hugetlb.h>
 14#include <asm/pgalloc.h>
 15#include <asm/tlbflush.h>
 16#include <asm/tlb.h>
 17
 18void pte_frag_destroy(void *pte_frag)
 19{
 20	int count;
 21	struct ptdesc *ptdesc;
 22
 23	ptdesc = virt_to_ptdesc(pte_frag);
 24	/* drop all the pending references */
 25	count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
 26	/* We allow PTE_FRAG_NR fragments from a PTE page */
 27	if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
 28		pagetable_dtor(ptdesc);
 29		pagetable_free(ptdesc);
 30	}
 31}
 32
 33static pte_t *get_pte_from_cache(struct mm_struct *mm)
 34{
 35	void *pte_frag, *ret;
 36
 37	if (PTE_FRAG_NR == 1)
 38		return NULL;
 39
 40	spin_lock(&mm->page_table_lock);
 41	ret = pte_frag_get(&mm->context);
 42	if (ret) {
 43		pte_frag = ret + PTE_FRAG_SIZE;
 44		/*
 45		 * If we have taken up all the fragments mark PTE page NULL
 46		 */
 47		if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
 48			pte_frag = NULL;
 49		pte_frag_set(&mm->context, pte_frag);
 50	}
 51	spin_unlock(&mm->page_table_lock);
 52	return (pte_t *)ret;
 53}
 54
 55static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel)
 56{
 57	void *ret = NULL;
 58	struct ptdesc *ptdesc;
 59	gfp_t gfp = PGALLOC_GFP;
 60
 61	if (!kernel)
 62		gfp |= __GFP_ACCOUNT;
 63
 64	ptdesc = pagetable_alloc(gfp, 0);
 65	if (!ptdesc)
 66		return NULL;
 67	if (!pagetable_pte_ctor(mm, ptdesc)) {
 68		pagetable_free(ptdesc);
 69		return NULL;
 70	}
 71
 72	atomic_set(&ptdesc->pt_frag_refcount, 1);
 73
 74	ret = ptdesc_address(ptdesc);
 75	/*
 76	 * if we support only one fragment just return the
 77	 * allocated page.
 78	 */
 79	if (PTE_FRAG_NR == 1)
 80		return ret;
 81	spin_lock(&mm->page_table_lock);
 82	/*
 83	 * If we find ptdesc_page set, we return
 84	 * the allocated page with single fragment
 85	 * count.
 86	 */
 87	if (likely(!pte_frag_get(&mm->context))) {
 88		atomic_set(&ptdesc->pt_frag_refcount, PTE_FRAG_NR);
 89		pte_frag_set(&mm->context, ret + PTE_FRAG_SIZE);
 90	}
 91	spin_unlock(&mm->page_table_lock);
 92
 93	return (pte_t *)ret;
 94}
 95
 96pte_t *pte_fragment_alloc(struct mm_struct *mm, int kernel)
 97{
 98	pte_t *pte;
 99
100	pte = get_pte_from_cache(mm);
101	if (pte)
102		return pte;
103
104	return __alloc_for_ptecache(mm, kernel);
105}
106
107static void pte_free_now(struct rcu_head *head)
108{
109	struct ptdesc *ptdesc;
110
111	ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
112	pagetable_dtor(ptdesc);
113	pagetable_free(ptdesc);
114}
115
116void pte_fragment_free(unsigned long *table, int kernel)
117{
118	struct ptdesc *ptdesc = virt_to_ptdesc(table);
119
120	if (pagetable_is_reserved(ptdesc))
121		return free_reserved_ptdesc(ptdesc);
122
123	BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0);
124	if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) {
125		if (kernel || !folio_test_clear_active(ptdesc_folio(ptdesc)))
126			pte_free_now(&ptdesc->pt_rcu_head);
127		else
128			call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
129	}
130}
131
132#ifdef CONFIG_TRANSPARENT_HUGEPAGE
133void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
134{
135	struct folio *folio;
136
137	folio = virt_to_folio(pgtable);
138	folio_set_active(folio);
139	pte_fragment_free((unsigned long *)pgtable, 0);
140}
141#endif /* CONFIG_TRANSPARENT_HUGEPAGE */