mte.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2020 ARM Ltd.
  4 */
  5
  6#include <linux/bitops.h>
  7#include <linux/cpu.h>
  8#include <linux/kernel.h>
  9#include <linux/mm.h>
 10#include <linux/prctl.h>
 11#include <linux/sched.h>
 12#include <linux/sched/mm.h>
 13#include <linux/string.h>
 14#include <linux/swap.h>
 15#include <linux/swapops.h>
 16#include <linux/thread_info.h>
 17#include <linux/types.h>
 18#include <linux/uaccess.h>
 19#include <linux/uio.h>
 20
 21#include <asm/barrier.h>
 22#include <asm/cpufeature.h>
 23#include <asm/mte.h>
 24#include <asm/ptrace.h>
 25#include <asm/sysreg.h>
 26
 27static DEFINE_PER_CPU_READ_MOSTLY(u64, mte_tcf_preferred);
 28
 29#ifdef CONFIG_KASAN_HW_TAGS
 30/*
 31 * The asynchronous and asymmetric MTE modes have the same behavior for
 32 * store operations. This flag is set when either of these modes is enabled.
 33 */
 34DEFINE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
 35EXPORT_SYMBOL_GPL(mte_async_or_asymm_mode);
 36#endif
 37
 38void mte_sync_tags(pte_t pte, unsigned int nr_pages)
 39{
 40	struct page *page = pte_page(pte);
 41	struct folio *folio = page_folio(page);
 42	unsigned long i;
 43
 44	if (folio_test_hugetlb(folio)) {
 45		unsigned long nr = folio_nr_pages(folio);
 46
 47		/* Hugetlb MTE flags are set for head page only */
 48		if (folio_try_hugetlb_mte_tagging(folio)) {
 49			for (i = 0; i < nr; i++, page++)
 50				mte_clear_page_tags(page_address(page));
 51			folio_set_hugetlb_mte_tagged(folio);
 52		}
 53
 54		/* ensure the tags are visible before the PTE is set */
 55		smp_wmb();
 56
 57		return;
 58	}
 59
 60	/* if PG_mte_tagged is set, tags have already been initialised */
 61	for (i = 0; i < nr_pages; i++, page++) {
 62		if (try_page_mte_tagging(page)) {
 63			mte_clear_page_tags(page_address(page));
 64			set_page_mte_tagged(page);
 65		}
 66	}
 67
 68	/* ensure the tags are visible before the PTE is set */
 69	smp_wmb();
 70}
 71
 72int memcmp_pages(struct page *page1, struct page *page2)
 73{
 74	char *addr1, *addr2;
 75	int ret;
 76
 77	addr1 = page_address(page1);
 78	addr2 = page_address(page2);
 79	ret = memcmp(addr1, addr2, PAGE_SIZE);
 80
 81	if (!system_supports_mte() || ret)
 82		return ret;
 83
 84	/*
 85	 * If the page content is identical but at least one of the pages is
 86	 * tagged, return non-zero to avoid KSM merging. If only one of the
 87	 * pages is tagged, __set_ptes() may zero or change the tags of the
 88	 * other page via mte_sync_tags().
 89	 */
 90	if (page_mte_tagged(page1) || page_mte_tagged(page2))
 91		return addr1 != addr2;
 92
 93	return ret;
 94}
 95
 96static inline void __mte_enable_kernel(const char *mode, unsigned long tcf)
 97{
 98	/* Enable MTE Sync Mode for EL1. */
 99	sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK,
100			 SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf));
101	isb();
102
103	pr_info_once("MTE: enabled in %s mode at EL1\n", mode);
104}
105
106#ifdef CONFIG_KASAN_HW_TAGS
107void mte_enable_kernel_sync(void)
108{
109	/*
110	 * Make sure we enter this function when no PE has set
111	 * async mode previously.
112	 */
113	WARN_ONCE(system_uses_mte_async_or_asymm_mode(),
114			"MTE async mode enabled system wide!");
115
116	__mte_enable_kernel("synchronous", SCTLR_EL1_TCF_SYNC);
117}
118
119void mte_enable_kernel_async(void)
120{
121	__mte_enable_kernel("asynchronous", SCTLR_EL1_TCF_ASYNC);
122
123	/*
124	 * MTE async mode is set system wide by the first PE that
125	 * executes this function.
126	 *
127	 * Note: If in future KASAN acquires a runtime switching
128	 * mode in between sync and async, this strategy needs
129	 * to be reviewed.
130	 */
131	if (!system_uses_mte_async_or_asymm_mode())
132		static_branch_enable(&mte_async_or_asymm_mode);
133}
134
135void mte_enable_kernel_asymm(void)
136{
137	if (cpus_have_cap(ARM64_MTE_ASYMM)) {
138		__mte_enable_kernel("asymmetric", SCTLR_EL1_TCF_ASYMM);
139
140		/*
141		 * MTE asymm mode behaves as async mode for store
142		 * operations. The mode is set system wide by the
143		 * first PE that executes this function.
144		 *
145		 * Note: If in future KASAN acquires a runtime switching
146		 * mode in between sync and async, this strategy needs
147		 * to be reviewed.
148		 */
149		if (!system_uses_mte_async_or_asymm_mode())
150			static_branch_enable(&mte_async_or_asymm_mode);
151	} else {
152		/*
153		 * If the CPU does not support MTE asymmetric mode the
154		 * kernel falls back on synchronous mode which is the
155		 * default for kasan=on.
156		 */
157		mte_enable_kernel_sync();
158	}
159}
160
161int mte_enable_kernel_store_only(void)
162{
163	/*
164	 * If the CPU does not support MTE store only,
165	 * the kernel checks all operations.
166	 */
167	if (!cpus_have_cap(ARM64_MTE_STORE_ONLY))
168		return -EINVAL;
169
170	sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCSO_MASK,
171			 SYS_FIELD_PREP(SCTLR_EL1, TCSO, 1));
172	isb();
173
174	pr_info_once("MTE: enabled store only mode at EL1\n");
175
176	return 0;
177}
178#endif
179
180#ifdef CONFIG_KASAN_HW_TAGS
181void mte_check_tfsr_el1(void)
182{
183	u64 tfsr_el1 = read_sysreg_s(SYS_TFSR_EL1);
184
185	if (unlikely(tfsr_el1 & SYS_TFSR_EL1_TF1)) {
186		/*
187		 * Note: isb() is not required after this direct write
188		 * because there is no indirect read subsequent to it
189		 * (per ARM DDI 0487F.c table D13-1).
190		 */
191		write_sysreg_s(0, SYS_TFSR_EL1);
192
193		kasan_report_async();
194	}
195}
196#endif
197
198/*
199 * This is where we actually resolve the system and process MTE mode
200 * configuration into an actual value in SCTLR_EL1 that affects
201 * userspace.
202 */
203static void mte_update_sctlr_user(struct task_struct *task)
204{
205	/*
206	 * This must be called with preemption disabled and can only be called
207	 * on the current or next task since the CPU must match where the thread
208	 * is going to run. The caller is responsible for calling
209	 * update_sctlr_el1() later in the same preemption disabled block.
210	 */
211	unsigned long sctlr = task->thread.sctlr_user;
212	unsigned long mte_ctrl = task->thread.mte_ctrl;
213	unsigned long pref, resolved_mte_tcf;
214
215	pref = __this_cpu_read(mte_tcf_preferred);
216	/*
217	 * If there is no overlap between the system preferred and
218	 * program requested values go with what was requested.
219	 */
220	resolved_mte_tcf = (mte_ctrl & pref) ? pref : mte_ctrl;
221	sctlr &= ~(SCTLR_EL1_TCF0_MASK | SCTLR_EL1_TCSO0_MASK);
222	/*
223	 * Pick an actual setting. The order in which we check for
224	 * set bits and map into register values determines our
225	 * default order.
226	 */
227	if (resolved_mte_tcf & MTE_CTRL_TCF_ASYMM)
228		sctlr |= SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF0, ASYMM);
229	else if (resolved_mte_tcf & MTE_CTRL_TCF_ASYNC)
230		sctlr |= SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF0, ASYNC);
231	else if (resolved_mte_tcf & MTE_CTRL_TCF_SYNC)
232		sctlr |= SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF0, SYNC);
233
234	if (mte_ctrl & MTE_CTRL_STORE_ONLY)
235		sctlr |= SYS_FIELD_PREP(SCTLR_EL1, TCSO0, 1);
236
237	task->thread.sctlr_user = sctlr;
238}
239
240static void mte_update_gcr_excl(struct task_struct *task)
241{
242	/*
243	 * SYS_GCR_EL1 will be set to current->thread.mte_ctrl value by
244	 * mte_set_user_gcr() in kernel_exit, but only if KASAN is enabled.
245	 */
246	if (kasan_hw_tags_enabled())
247		return;
248
249	write_sysreg_s(
250		((task->thread.mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
251		 SYS_GCR_EL1_EXCL_MASK) | SYS_GCR_EL1_RRND,
252		SYS_GCR_EL1);
253}
254
255#ifdef CONFIG_KASAN_HW_TAGS
256/* Only called from assembly, silence sparse */
257void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
258				 __le32 *updptr, int nr_inst);
259
260void __init kasan_hw_tags_enable(struct alt_instr *alt, __le32 *origptr,
261				 __le32 *updptr, int nr_inst)
262{
263	BUG_ON(nr_inst != 1); /* Branch -> NOP */
264
265	if (kasan_hw_tags_enabled())
266		*updptr = cpu_to_le32(aarch64_insn_gen_nop());
267}
268#endif
269
270void mte_thread_init_user(void)
271{
272	if (!system_supports_mte())
273		return;
274
275	/* clear any pending asynchronous tag fault */
276	dsb(ish);
277	write_sysreg_s(0, SYS_TFSRE0_EL1);
278	clear_thread_flag(TIF_MTE_ASYNC_FAULT);
279	/* disable tag checking and reset tag generation mask */
280	set_mte_ctrl(current, 0);
281}
282
283void mte_thread_switch(struct task_struct *next)
284{
285	if (!system_supports_mte())
286		return;
287
288	mte_update_sctlr_user(next);
289	mte_update_gcr_excl(next);
290
291	/* TCO may not have been disabled on exception entry for the current task. */
292	mte_disable_tco_entry(next);
293
294	/*
295	 * Check if an async tag exception occurred at EL1.
296	 *
297	 * Note: On the context switch path we rely on the dsb() present
298	 * in __switch_to() to guarantee that the indirect writes to TFSR_EL1
299	 * are synchronized before this point.
300	 */
301	isb();
302	mte_check_tfsr_el1();
303}
304
305void mte_cpu_setup(void)
306{
307	u64 rgsr;
308
309	/*
310	 * CnP must be enabled only after the MAIR_EL1 register has been set
311	 * up. Inconsistent MAIR_EL1 between CPUs sharing the same TLB may
312	 * lead to the wrong memory type being used for a brief window during
313	 * CPU power-up.
314	 *
315	 * CnP is not a boot feature so MTE gets enabled before CnP, but let's
316	 * make sure that is the case.
317	 */
318	BUG_ON(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT);
319	BUG_ON(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT);
320
321	/* Normal Tagged memory type at the corresponding MAIR index */
322	sysreg_clear_set(mair_el1,
323			 MAIR_ATTRIDX(MAIR_ATTR_MASK, MT_NORMAL_TAGGED),
324			 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_TAGGED,
325				      MT_NORMAL_TAGGED));
326
327	write_sysreg_s(KERNEL_GCR_EL1, SYS_GCR_EL1);
328
329	/*
330	 * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
331	 * RGSR_EL1.SEED must be non-zero for IRG to produce
332	 * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
333	 * must initialize it.
334	 */
335	rgsr = (read_sysreg(CNTVCT_EL0) & SYS_RGSR_EL1_SEED_MASK) <<
336	       SYS_RGSR_EL1_SEED_SHIFT;
337	if (rgsr == 0)
338		rgsr = 1 << SYS_RGSR_EL1_SEED_SHIFT;
339	write_sysreg_s(rgsr, SYS_RGSR_EL1);
340
341	/* clear any pending tag check faults in TFSR*_EL1 */
342	write_sysreg_s(0, SYS_TFSR_EL1);
343	write_sysreg_s(0, SYS_TFSRE0_EL1);
344
345	local_flush_tlb_all();
346}
347
348void mte_suspend_enter(void)
349{
350	if (!system_supports_mte())
351		return;
352
353	/*
354	 * The barriers are required to guarantee that the indirect writes
355	 * to TFSR_EL1 are synchronized before we report the state.
356	 */
357	dsb(nsh);
358	isb();
359
360	/* Report SYS_TFSR_EL1 before suspend entry */
361	mte_check_tfsr_el1();
362}
363
364void mte_suspend_exit(void)
365{
366	if (!system_supports_mte())
367		return;
368
369	mte_cpu_setup();
370}
371
372long set_mte_ctrl(struct task_struct *task, unsigned long arg)
373{
374	u64 mte_ctrl = (~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) &
375			SYS_GCR_EL1_EXCL_MASK) << MTE_CTRL_GCR_USER_EXCL_SHIFT;
376
377	if (!system_supports_mte())
378		return 0;
379
380	if (arg & PR_MTE_TCF_ASYNC)
381		mte_ctrl |= MTE_CTRL_TCF_ASYNC;
382	if (arg & PR_MTE_TCF_SYNC)
383		mte_ctrl |= MTE_CTRL_TCF_SYNC;
384
385	/*
386	 * If the system supports it and both sync and async modes are
387	 * specified then implicitly enable asymmetric mode.
388	 * Userspace could see a mix of both sync and async anyway due
389	 * to differing or changing defaults on CPUs.
390	 */
391	if (cpus_have_cap(ARM64_MTE_ASYMM) &&
392	    (arg & PR_MTE_TCF_ASYNC) &&
393	    (arg & PR_MTE_TCF_SYNC))
394		mte_ctrl |= MTE_CTRL_TCF_ASYMM;
395
396	if (arg & PR_MTE_STORE_ONLY)
397		mte_ctrl |= MTE_CTRL_STORE_ONLY;
398
399	task->thread.mte_ctrl = mte_ctrl;
400	if (task == current) {
401		preempt_disable();
402		mte_update_sctlr_user(task);
403		mte_update_gcr_excl(task);
404		update_sctlr_el1(task->thread.sctlr_user);
405		preempt_enable();
406	}
407
408	return 0;
409}
410
411long get_mte_ctrl(struct task_struct *task)
412{
413	unsigned long ret;
414	u64 mte_ctrl = task->thread.mte_ctrl;
415	u64 incl = (~mte_ctrl >> MTE_CTRL_GCR_USER_EXCL_SHIFT) &
416		   SYS_GCR_EL1_EXCL_MASK;
417
418	if (!system_supports_mte())
419		return 0;
420
421	ret = incl << PR_MTE_TAG_SHIFT;
422	if (mte_ctrl & MTE_CTRL_TCF_ASYNC)
423		ret |= PR_MTE_TCF_ASYNC;
424	if (mte_ctrl & MTE_CTRL_TCF_SYNC)
425		ret |= PR_MTE_TCF_SYNC;
426	if (mte_ctrl & MTE_CTRL_STORE_ONLY)
427		ret |= PR_MTE_STORE_ONLY;
428
429	return ret;
430}
431
432/*
433 * Access MTE tags in another process' address space as given in mm. Update
434 * the number of tags copied. Return 0 if any tags copied, error otherwise.
435 * Inspired by __access_remote_vm().
436 */
437static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,
438				struct iovec *kiov, unsigned int gup_flags)
439{
440	void __user *buf = kiov->iov_base;
441	size_t len = kiov->iov_len;
442	int err = 0;
443	int write = gup_flags & FOLL_WRITE;
444
445	if (!access_ok(buf, len))
446		return -EFAULT;
447
448	if (mmap_read_lock_killable(mm))
449		return -EIO;
450
451	while (len) {
452		struct vm_area_struct *vma;
453		unsigned long tags, offset;
454		void *maddr;
455		struct page *page = get_user_page_vma_remote(mm, addr,
456							     gup_flags, &vma);
457		struct folio *folio;
458
459		if (IS_ERR(page)) {
460			err = PTR_ERR(page);
461			break;
462		}
463
464		/*
465		 * Only copy tags if the page has been mapped as PROT_MTE
466		 * (PG_mte_tagged set). Otherwise the tags are not valid and
467		 * not accessible to user. Moreover, an mprotect(PROT_MTE)
468		 * would cause the existing tags to be cleared if the page
469		 * was never mapped with PROT_MTE.
470		 */
471		if (!(vma->vm_flags & VM_MTE)) {
472			err = -EOPNOTSUPP;
473			put_page(page);
474			break;
475		}
476
477		folio = page_folio(page);
478		if (folio_test_hugetlb(folio))
479			WARN_ON_ONCE(!folio_test_hugetlb_mte_tagged(folio) &&
480				     !is_huge_zero_folio(folio));
481		else
482			WARN_ON_ONCE(!page_mte_tagged(page) && !is_zero_page(page));
483
484		/* limit access to the end of the page */
485		offset = offset_in_page(addr);
486		tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE);
487
488		maddr = page_address(page);
489		if (write) {
490			tags = mte_copy_tags_from_user(maddr + offset, buf, tags);
491			set_page_dirty_lock(page);
492		} else {
493			tags = mte_copy_tags_to_user(buf, maddr + offset, tags);
494		}
495		put_page(page);
496
497		/* error accessing the tracer's buffer */
498		if (!tags)
499			break;
500
501		len -= tags;
502		buf += tags;
503		addr += tags * MTE_GRANULE_SIZE;
504	}
505	mmap_read_unlock(mm);
506
507	/* return an error if no tags copied */
508	kiov->iov_len = buf - kiov->iov_base;
509	if (!kiov->iov_len) {
510		/* check for error accessing the tracee's address space */
511		if (err)
512			return -EIO;
513		else
514			return -EFAULT;
515	}
516
517	return 0;
518}
519
520/*
521 * Copy MTE tags in another process' address space at 'addr' to/from tracer's
522 * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm().
523 */
524static int access_remote_tags(struct task_struct *tsk, unsigned long addr,
525			      struct iovec *kiov, unsigned int gup_flags)
526{
527	struct mm_struct *mm;
528	int ret;
529
530	mm = get_task_mm(tsk);
531	if (!mm)
532		return -EPERM;
533
534	if (!tsk->ptrace || (current != tsk->parent) ||
535	    ((get_dumpable(mm) != SUID_DUMP_USER) &&
536	     !ptracer_capable(tsk, mm->user_ns))) {
537		mmput(mm);
538		return -EPERM;
539	}
540
541	ret = __access_remote_tags(mm, addr, kiov, gup_flags);
542	mmput(mm);
543
544	return ret;
545}
546
547int mte_ptrace_copy_tags(struct task_struct *child, long request,
548			 unsigned long addr, unsigned long data)
549{
550	int ret;
551	struct iovec kiov;
552	struct iovec __user *uiov = (void __user *)data;
553	unsigned int gup_flags = FOLL_FORCE;
554
555	if (!system_supports_mte())
556		return -EIO;
557
558	if (get_user(kiov.iov_base, &uiov->iov_base) ||
559	    get_user(kiov.iov_len, &uiov->iov_len))
560		return -EFAULT;
561
562	if (request == PTRACE_POKEMTETAGS)
563		gup_flags |= FOLL_WRITE;
564
565	/* align addr to the MTE tag granule */
566	addr &= MTE_GRANULE_MASK;
567
568	ret = access_remote_tags(child, addr, &kiov, gup_flags);
569	if (!ret)
570		ret = put_user(kiov.iov_len, &uiov->iov_len);
571
572	return ret;
573}
574
575static ssize_t mte_tcf_preferred_show(struct device *dev,
576				      struct device_attribute *attr, char *buf)
577{
578	switch (per_cpu(mte_tcf_preferred, dev->id)) {
579	case MTE_CTRL_TCF_ASYNC:
580		return sysfs_emit(buf, "async\n");
581	case MTE_CTRL_TCF_SYNC:
582		return sysfs_emit(buf, "sync\n");
583	case MTE_CTRL_TCF_ASYMM:
584		return sysfs_emit(buf, "asymm\n");
585	default:
586		return sysfs_emit(buf, "???\n");
587	}
588}
589
590static ssize_t mte_tcf_preferred_store(struct device *dev,
591				       struct device_attribute *attr,
592				       const char *buf, size_t count)
593{
594	u64 tcf;
595
596	if (sysfs_streq(buf, "async"))
597		tcf = MTE_CTRL_TCF_ASYNC;
598	else if (sysfs_streq(buf, "sync"))
599		tcf = MTE_CTRL_TCF_SYNC;
600	else if (cpus_have_cap(ARM64_MTE_ASYMM) && sysfs_streq(buf, "asymm"))
601		tcf = MTE_CTRL_TCF_ASYMM;
602	else
603		return -EINVAL;
604
605	device_lock(dev);
606	per_cpu(mte_tcf_preferred, dev->id) = tcf;
607	device_unlock(dev);
608
609	return count;
610}
611static DEVICE_ATTR_RW(mte_tcf_preferred);
612
613static int register_mte_tcf_preferred_sysctl(void)
614{
615	unsigned int cpu;
616
617	if (!system_supports_mte())
618		return 0;
619
620	for_each_possible_cpu(cpu) {
621		per_cpu(mte_tcf_preferred, cpu) = MTE_CTRL_TCF_ASYNC;
622		device_create_file(get_cpu_device(cpu),
623				   &dev_attr_mte_tcf_preferred);
624	}
625
626	return 0;
627}
628subsys_initcall(register_mte_tcf_preferred_sysctl);
629
630/*
631 * Return 0 on success, the number of bytes not probed otherwise.
632 */
633size_t mte_probe_user_range(const char __user *uaddr, size_t size)
634{
635	const char __user *end = uaddr + size;
636	char val;
637
638	__raw_get_user(val, uaddr, efault);
639
640	uaddr = PTR_ALIGN(uaddr, MTE_GRANULE_SIZE);
641	while (uaddr < end) {
642		/*
643		 * A read is sufficient for mte, the caller should have probed
644		 * for the pte write permission if required.
645		 */
646		__raw_get_user(val, uaddr, efault);
647		uaddr += MTE_GRANULE_SIZE;
648	}
649	(void)val;
650
651	return 0;
652
653efault:
654	return end - uaddr;
655}