mte.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright (C) 2020 ARM Ltd.
  4 */
  5#ifndef __ASM_MTE_H
  6#define __ASM_MTE_H
  7
  8#include <asm/compiler.h>
  9#include <asm/mte-def.h>
 10
 11#ifndef __ASSEMBLER__
 12
 13#include <linux/bitfield.h>
 14#include <linux/kasan-enabled.h>
 15#include <linux/page-flags.h>
 16#include <linux/sched.h>
 17#include <linux/types.h>
 18
 19#include <asm/pgtable-types.h>
 20
 21void mte_clear_page_tags(void *addr);
 22unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
 23				      unsigned long n);
 24unsigned long mte_copy_tags_to_user(void __user *to, void *from,
 25				    unsigned long n);
 26int mte_save_tags(struct page *page);
 27void mte_save_page_tags(const void *page_addr, void *tag_storage);
 28void mte_restore_tags(swp_entry_t entry, struct page *page);
 29void mte_restore_page_tags(void *page_addr, const void *tag_storage);
 30void mte_invalidate_tags(int type, pgoff_t offset);
 31void mte_invalidate_tags_area(int type);
 32void *mte_allocate_tag_storage(void);
 33void mte_free_tag_storage(char *storage);
 34
 35#ifdef CONFIG_ARM64_MTE
 36
 37/* track which pages have valid allocation tags */
 38#define PG_mte_tagged	PG_arch_2
 39/* simple lock to avoid multiple threads tagging the same page */
 40#define PG_mte_lock	PG_arch_3
 41
 42static inline void set_page_mte_tagged(struct page *page)
 43{
 44	VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));
 45
 46	/*
 47	 * Ensure that the tags written prior to this function are visible
 48	 * before the page flags update.
 49	 */
 50	smp_wmb();
 51	set_bit(PG_mte_tagged, &page->flags.f);
 52}
 53
 54static inline bool page_mte_tagged(struct page *page)
 55{
 56	bool ret = test_bit(PG_mte_tagged, &page->flags.f);
 57
 58	VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));
 59
 60	/*
 61	 * If the page is tagged, ensure ordering with a likely subsequent
 62	 * read of the tags.
 63	 */
 64	if (ret)
 65		smp_rmb();
 66	return ret;
 67}
 68
 69/*
 70 * Lock the page for tagging and return 'true' if the page can be tagged,
 71 * 'false' if already tagged. PG_mte_tagged is never cleared and therefore the
 72 * locking only happens once for page initialisation.
 73 *
 74 * The page MTE lock state:
 75 *
 76 *   Locked:	PG_mte_lock && !PG_mte_tagged
 77 *   Unlocked:	!PG_mte_lock || PG_mte_tagged
 78 *
 79 * Acquire semantics only if the page is tagged (returning 'false').
 80 */
 81static inline bool try_page_mte_tagging(struct page *page)
 82{
 83	VM_WARN_ON_ONCE(folio_test_hugetlb(page_folio(page)));
 84
 85	if (!test_and_set_bit(PG_mte_lock, &page->flags.f))
 86		return true;
 87
 88	/*
 89	 * The tags are either being initialised or may have been initialised
 90	 * already. Check if the PG_mte_tagged flag has been set or wait
 91	 * otherwise.
 92	 */
 93	smp_cond_load_acquire(&page->flags.f, VAL & (1UL << PG_mte_tagged));
 94
 95	return false;
 96}
 97
 98void mte_zero_clear_page_tags(void *addr);
 99void mte_sync_tags(pte_t pte, unsigned int nr_pages);
100void mte_copy_page_tags(void *kto, const void *kfrom);
101void mte_thread_init_user(void);
102void mte_thread_switch(struct task_struct *next);
103void mte_cpu_setup(void);
104void mte_suspend_enter(void);
105void mte_suspend_exit(void);
106long set_mte_ctrl(struct task_struct *task, unsigned long arg);
107long get_mte_ctrl(struct task_struct *task);
108int mte_ptrace_copy_tags(struct task_struct *child, long request,
109			 unsigned long addr, unsigned long data);
110size_t mte_probe_user_range(const char __user *uaddr, size_t size);
111
112#else /* CONFIG_ARM64_MTE */
113
114/* unused if !CONFIG_ARM64_MTE, silence the compiler */
115#define PG_mte_tagged	0
116
117static inline void set_page_mte_tagged(struct page *page)
118{
119}
120static inline bool page_mte_tagged(struct page *page)
121{
122	return false;
123}
124static inline bool try_page_mte_tagging(struct page *page)
125{
126	return false;
127}
128static inline void mte_zero_clear_page_tags(void *addr)
129{
130}
131static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages)
132{
133}
134static inline void mte_copy_page_tags(void *kto, const void *kfrom)
135{
136}
137static inline void mte_thread_init_user(void)
138{
139}
140static inline void mte_thread_switch(struct task_struct *next)
141{
142}
143static inline void mte_suspend_enter(void)
144{
145}
146static inline void mte_suspend_exit(void)
147{
148}
149static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
150{
151	return 0;
152}
153static inline long get_mte_ctrl(struct task_struct *task)
154{
155	return 0;
156}
157static inline int mte_ptrace_copy_tags(struct task_struct *child,
158				       long request, unsigned long addr,
159				       unsigned long data)
160{
161	return -EIO;
162}
163
164#endif /* CONFIG_ARM64_MTE */
165
166#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_ARM64_MTE)
167static inline void folio_set_hugetlb_mte_tagged(struct folio *folio)
168{
169	VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));
170
171	/*
172	 * Ensure that the tags written prior to this function are visible
173	 * before the folio flags update.
174	 */
175	smp_wmb();
176	set_bit(PG_mte_tagged, &folio->flags.f);
177
178}
179
180static inline bool folio_test_hugetlb_mte_tagged(struct folio *folio)
181{
182	bool ret = test_bit(PG_mte_tagged, &folio->flags.f);
183
184	VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));
185
186	/*
187	 * If the folio is tagged, ensure ordering with a likely subsequent
188	 * read of the tags.
189	 */
190	if (ret)
191		smp_rmb();
192	return ret;
193}
194
195static inline bool folio_try_hugetlb_mte_tagging(struct folio *folio)
196{
197	VM_WARN_ON_ONCE(!folio_test_hugetlb(folio));
198
199	if (!test_and_set_bit(PG_mte_lock, &folio->flags.f))
200		return true;
201
202	/*
203	 * The tags are either being initialised or may have been initialised
204	 * already. Check if the PG_mte_tagged flag has been set or wait
205	 * otherwise.
206	 */
207	smp_cond_load_acquire(&folio->flags.f, VAL & (1UL << PG_mte_tagged));
208
209	return false;
210}
211#else
212static inline void folio_set_hugetlb_mte_tagged(struct folio *folio)
213{
214}
215
216static inline bool folio_test_hugetlb_mte_tagged(struct folio *folio)
217{
218	return false;
219}
220
221static inline bool folio_try_hugetlb_mte_tagging(struct folio *folio)
222{
223	return false;
224}
225#endif
226
227static inline void mte_disable_tco_entry(struct task_struct *task)
228{
229	if (!system_supports_mte())
230		return;
231
232	/*
233	 * Re-enable tag checking (TCO set on exception entry). This is only
234	 * necessary if MTE is enabled in either the kernel or the userspace
235	 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set
236	 * for both). With MTE disabled in the kernel and disabled or
237	 * asynchronous in userspace, tag check faults (including in uaccesses)
238	 * are not reported, therefore there is no need to re-enable checking.
239	 * This is beneficial on microarchitectures where re-enabling TCO is
240	 * expensive.
241	 */
242	if (kasan_hw_tags_enabled() ||
243	    (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT)))
244		asm volatile(SET_PSTATE_TCO(0));
245}
246
247#ifdef CONFIG_KASAN_HW_TAGS
248void mte_check_tfsr_el1(void);
249
250static inline void mte_check_tfsr_entry(void)
251{
252	if (!kasan_hw_tags_enabled())
253		return;
254
255	mte_check_tfsr_el1();
256}
257
258static inline void mte_check_tfsr_exit(void)
259{
260	if (!kasan_hw_tags_enabled())
261		return;
262
263	/*
264	 * The asynchronous faults are sync'ed automatically with
265	 * TFSR_EL1 on kernel entry but for exit an explicit dsb()
266	 * is required.
267	 */
268	dsb(nsh);
269	isb();
270
271	mte_check_tfsr_el1();
272}
273#else
274static inline void mte_check_tfsr_el1(void)
275{
276}
277static inline void mte_check_tfsr_entry(void)
278{
279}
280static inline void mte_check_tfsr_exit(void)
281{
282}
283#endif /* CONFIG_KASAN_HW_TAGS */
284
285#endif /* __ASSEMBLER__ */
286#endif /* __ASM_MTE_H  */