page_table_check.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright (c) 2021, Google LLC.
  5 * Pasha Tatashin <pasha.tatashin@soleen.com>
  6 */
  7#include <linux/kstrtox.h>
  8#include <linux/mm.h>
  9#include <linux/page_table_check.h>
 10#include <linux/swap.h>
 11#include <linux/leafops.h>
 12
 13#undef pr_fmt
 14#define pr_fmt(fmt)	"page_table_check: " fmt
 15
 16struct page_table_check {
 17	atomic_t anon_map_count;
 18	atomic_t file_map_count;
 19};
 20
 21static bool __page_table_check_enabled __initdata =
 22				IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
 23
 24DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
 25EXPORT_SYMBOL(page_table_check_disabled);
 26
 27static int __init early_page_table_check_param(char *buf)
 28{
 29	return kstrtobool(buf, &__page_table_check_enabled);
 30}
 31
 32early_param("page_table_check", early_page_table_check_param);
 33
 34static bool __init need_page_table_check(void)
 35{
 36	return __page_table_check_enabled;
 37}
 38
 39static void __init init_page_table_check(void)
 40{
 41	if (!__page_table_check_enabled)
 42		return;
 43	static_branch_disable(&page_table_check_disabled);
 44}
 45
 46struct page_ext_operations page_table_check_ops = {
 47	.size = sizeof(struct page_table_check),
 48	.need = need_page_table_check,
 49	.init = init_page_table_check,
 50	.need_shared_flags = false,
 51};
 52
 53static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
 54{
 55	BUG_ON(!page_ext);
 56	return page_ext_data(page_ext, &page_table_check_ops);
 57}
 58
 59/*
 60 * An entry is removed from the page table, decrement the counters for that page
 61 * verify that it is of correct type and counters do not become negative.
 62 */
 63static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
 64{
 65	struct page_ext_iter iter;
 66	struct page_ext *page_ext;
 67	struct page *page;
 68	bool anon;
 69
 70	if (!pfn_valid(pfn))
 71		return;
 72
 73	page = pfn_to_page(pfn);
 74	BUG_ON(PageSlab(page));
 75	anon = PageAnon(page);
 76
 77	rcu_read_lock();
 78	for_each_page_ext(page, pgcnt, page_ext, iter) {
 79		struct page_table_check *ptc = get_page_table_check(page_ext);
 80
 81		if (anon) {
 82			BUG_ON(atomic_read(&ptc->file_map_count));
 83			BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
 84		} else {
 85			BUG_ON(atomic_read(&ptc->anon_map_count));
 86			BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
 87		}
 88	}
 89	rcu_read_unlock();
 90}
 91
 92/*
 93 * A new entry is added to the page table, increment the counters for that page
 94 * verify that it is of correct type and is not being mapped with a different
 95 * type to a different process.
 96 */
 97static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
 98				 bool rw)
 99{
100	struct page_ext_iter iter;
101	struct page_ext *page_ext;
102	struct page *page;
103	bool anon;
104
105	if (!pfn_valid(pfn))
106		return;
107
108	page = pfn_to_page(pfn);
109	BUG_ON(PageSlab(page));
110	anon = PageAnon(page);
111
112	rcu_read_lock();
113	for_each_page_ext(page, pgcnt, page_ext, iter) {
114		struct page_table_check *ptc = get_page_table_check(page_ext);
115
116		if (anon) {
117			BUG_ON(atomic_read(&ptc->file_map_count));
118			BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
119		} else {
120			BUG_ON(atomic_read(&ptc->anon_map_count));
121			BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
122		}
123	}
124	rcu_read_unlock();
125}
126
127/*
128 * page is on free list, or is being allocated, verify that counters are zeroes
129 * crash if they are not.
130 */
131void __page_table_check_zero(struct page *page, unsigned int order)
132{
133	struct page_ext_iter iter;
134	struct page_ext *page_ext;
135
136	BUG_ON(PageSlab(page));
137
138	rcu_read_lock();
139	for_each_page_ext(page, 1 << order, page_ext, iter) {
140		struct page_table_check *ptc = get_page_table_check(page_ext);
141
142		BUG_ON(atomic_read(&ptc->anon_map_count));
143		BUG_ON(atomic_read(&ptc->file_map_count));
144	}
145	rcu_read_unlock();
146}
147
148void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
149{
150	if (&init_mm == mm)
151		return;
152
153	if (pte_user_accessible_page(pte)) {
154		page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
155	}
156}
157EXPORT_SYMBOL(__page_table_check_pte_clear);
158
159void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
160{
161	if (&init_mm == mm)
162		return;
163
164	if (pmd_user_accessible_page(pmd)) {
165		page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
166	}
167}
168EXPORT_SYMBOL(__page_table_check_pmd_clear);
169
170void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
171{
172	if (&init_mm == mm)
173		return;
174
175	if (pud_user_accessible_page(pud)) {
176		page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
177	}
178}
179EXPORT_SYMBOL(__page_table_check_pud_clear);
180
181/* Whether the swap entry cached writable information */
182static inline bool softleaf_cached_writable(softleaf_t entry)
183{
184	return softleaf_is_device_private_write(entry) ||
185		softleaf_is_migration_write(entry);
186}
187
188static void page_table_check_pte_flags(pte_t pte)
189{
190	if (pte_present(pte)) {
191		WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte));
192	} else if (pte_swp_uffd_wp(pte)) {
193		const softleaf_t entry = softleaf_from_pte(pte);
194
195		WARN_ON_ONCE(softleaf_cached_writable(entry));
196	}
197}
198
199void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
200		unsigned int nr)
201{
202	unsigned int i;
203
204	if (&init_mm == mm)
205		return;
206
207	page_table_check_pte_flags(pte);
208
209	for (i = 0; i < nr; i++)
210		__page_table_check_pte_clear(mm, ptep_get(ptep + i));
211	if (pte_user_accessible_page(pte))
212		page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
213}
214EXPORT_SYMBOL(__page_table_check_ptes_set);
215
216static inline void page_table_check_pmd_flags(pmd_t pmd)
217{
218	if (pmd_present(pmd)) {
219		if (pmd_uffd_wp(pmd))
220			WARN_ON_ONCE(pmd_write(pmd));
221	} else if (pmd_swp_uffd_wp(pmd)) {
222		const softleaf_t entry = softleaf_from_pmd(pmd);
223
224		WARN_ON_ONCE(softleaf_cached_writable(entry));
225	}
226}
227
228void __page_table_check_pmds_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd,
229		unsigned int nr)
230{
231	unsigned long stride = PMD_SIZE >> PAGE_SHIFT;
232	unsigned int i;
233
234	if (&init_mm == mm)
235		return;
236
237	page_table_check_pmd_flags(pmd);
238
239	for (i = 0; i < nr; i++)
240		__page_table_check_pmd_clear(mm, *(pmdp + i));
241	if (pmd_user_accessible_page(pmd))
242		page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd));
243}
244EXPORT_SYMBOL(__page_table_check_pmds_set);
245
246void __page_table_check_puds_set(struct mm_struct *mm, pud_t *pudp, pud_t pud,
247		unsigned int nr)
248{
249	unsigned long stride = PUD_SIZE >> PAGE_SHIFT;
250	unsigned int i;
251
252	if (&init_mm == mm)
253		return;
254
255	for (i = 0; i < nr; i++)
256		__page_table_check_pud_clear(mm, *(pudp + i));
257	if (pud_user_accessible_page(pud))
258		page_table_check_set(pud_pfn(pud), stride * nr, pud_write(pud));
259}
260EXPORT_SYMBOL(__page_table_check_puds_set);
261
262void __page_table_check_pte_clear_range(struct mm_struct *mm,
263					unsigned long addr,
264					pmd_t pmd)
265{
266	if (&init_mm == mm)
267		return;
268
269	if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
270		pte_t *ptep = pte_offset_map(&pmd, addr);
271		unsigned long i;
272
273		if (WARN_ON(!ptep))
274			return;
275		for (i = 0; i < PTRS_PER_PTE; i++) {
276			__page_table_check_pte_clear(mm, ptep_get(ptep));
277			addr += PAGE_SIZE;
278			ptep++;
279		}
280		pte_unmap(ptep - PTRS_PER_PTE);
281	}
282}