fault.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *  linux/arch/arm/mm/fault.c
  4 *
  5 *  Copyright (C) 1995  Linus Torvalds
  6 *  Modifications for ARM processor (c) 1995-2004 Russell King
  7 */
  8#include <linux/extable.h>
  9#include <linux/signal.h>
 10#include <linux/mm.h>
 11#include <linux/hardirq.h>
 12#include <linux/init.h>
 13#include <linux/kprobes.h>
 14#include <linux/uaccess.h>
 15#include <linux/page-flags.h>
 16#include <linux/sched/signal.h>
 17#include <linux/sched/debug.h>
 18#include <linux/highmem.h>
 19#include <linux/perf_event.h>
 20#include <linux/kfence.h>
 21
 22#include <asm/system_misc.h>
 23#include <asm/system_info.h>
 24#include <asm/tlbflush.h>
 25
 26#include "fault.h"
 27
 28#ifdef CONFIG_MMU
 29
 30bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
 31{
 32	unsigned long addr = (unsigned long)unsafe_src;
 33
 34	return addr >= TASK_SIZE && ULONG_MAX - addr >= size;
 35}
 36
 37/*
 38 * This is useful to dump out the page tables associated with
 39 * 'addr' in mm 'mm'.
 40 */
 41void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
 42{
 43	pgd_t *pgd;
 44
 45	if (!mm)
 46		mm = &init_mm;
 47
 48	pgd = pgd_offset(mm, addr);
 49	printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
 50
 51	do {
 52		p4d_t *p4d;
 53		pud_t *pud;
 54		pmd_t *pmd;
 55		pte_t *pte;
 56
 57		p4d = p4d_offset(pgd, addr);
 58		if (p4d_none(*p4d))
 59			break;
 60
 61		if (p4d_bad(*p4d)) {
 62			pr_cont("(bad)");
 63			break;
 64		}
 65
 66		pud = pud_offset(p4d, addr);
 67		if (PTRS_PER_PUD != 1)
 68			pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
 69
 70		if (pud_none(*pud))
 71			break;
 72
 73		if (pud_bad(*pud)) {
 74			pr_cont("(bad)");
 75			break;
 76		}
 77
 78		pmd = pmd_offset(pud, addr);
 79		if (PTRS_PER_PMD != 1)
 80			pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd));
 81
 82		if (pmd_none(*pmd))
 83			break;
 84
 85		if (pmd_bad(*pmd)) {
 86			pr_cont("(bad)");
 87			break;
 88		}
 89
 90		/* We must not map this if we have highmem enabled */
 91		if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
 92			break;
 93
 94		pte = pte_offset_map(pmd, addr);
 95		if (!pte)
 96			break;
 97
 98		pr_cont(", *pte=%08llx", (long long)pte_val(*pte));
 99#ifndef CONFIG_ARM_LPAE
100		pr_cont(", *ppte=%08llx",
101		       (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
102#endif
103		pte_unmap(pte);
104	} while(0);
105
106	pr_cont("\n");
107}
108#else					/* CONFIG_MMU */
109void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
110{ }
111#endif					/* CONFIG_MMU */
112
113static inline bool is_write_fault(unsigned int fsr)
114{
115	return (fsr & FSR_WRITE) && !(fsr & FSR_CM);
116}
117
118static inline bool is_translation_fault(unsigned int fsr)
119{
120	int fs = fsr_fs(fsr);
121#ifdef CONFIG_ARM_LPAE
122	if ((fs & FS_MMU_NOLL_MASK) == FS_TRANS_NOLL)
123		return true;
124#else
125	if (fs == FS_L1_TRANS || fs == FS_L2_TRANS)
126		return true;
127#endif
128	return false;
129}
130
131static inline bool is_permission_fault(unsigned int fsr)
132{
133	int fs = fsr_fs(fsr);
134#ifdef CONFIG_ARM_LPAE
135	if ((fs & FS_MMU_NOLL_MASK) == FS_PERM_NOLL)
136		return true;
137#else
138	if (fs == FS_L1_PERM || fs == FS_L2_PERM)
139		return true;
140#endif
141	return false;
142}
143
144static void die_kernel_fault(const char *msg, struct mm_struct *mm,
145			     unsigned long addr, unsigned int fsr,
146			     struct pt_regs *regs)
147{
148	bust_spinlocks(1);
149	pr_alert("8<--- cut here ---\n");
150	pr_alert("Unable to handle kernel %s at virtual address %08lx when %s\n",
151		 msg, addr, fsr & FSR_LNX_PF ? "execute" : str_write_read(fsr & FSR_WRITE));
152
153	show_pte(KERN_ALERT, mm, addr);
154	die("Oops", regs, fsr);
155	bust_spinlocks(0);
156	make_task_dead(SIGKILL);
157}
158
159/*
160 * Oops.  The kernel tried to access some page that wasn't present.
161 */
162static void
163__do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
164		  struct pt_regs *regs)
165{
166	const char *msg;
167	/*
168	 * Are we prepared to handle this kernel fault?
169	 */
170	if (fixup_exception(regs))
171		return;
172
173	/*
174	 * No handler, we'll have to terminate things with extreme prejudice.
175	 */
176	if (addr < PAGE_SIZE) {
177		msg = "NULL pointer dereference";
178	} else if (is_permission_fault(fsr) && fsr & FSR_LNX_PF) {
179		msg = "execution of memory";
180	} else {
181		if (is_translation_fault(fsr) &&
182		    kfence_handle_page_fault(addr, is_write_fault(fsr), regs))
183			return;
184
185		msg = "paging request";
186	}
187
188	die_kernel_fault(msg, mm, addr, fsr, regs);
189}
190
191/*
192 * Something tried to access memory that isn't in our memory map..
193 * User mode accesses just cause a SIGSEGV
194 */
195static void
196__do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
197		int code, struct pt_regs *regs)
198{
199	struct task_struct *tsk = current;
200
201#ifdef CONFIG_DEBUG_USER
202	if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
203	    ((user_debug & UDBG_BUS)  && (sig == SIGBUS))) {
204		pr_err("8<--- cut here ---\n");
205		pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
206		       tsk->comm, sig, addr, fsr);
207		show_pte(KERN_ERR, tsk->mm, addr);
208		show_regs(regs);
209	}
210#endif
211#ifndef CONFIG_KUSER_HELPERS
212	if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))
213		printk_ratelimited(KERN_DEBUG
214				   "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n",
215				   tsk->comm, addr);
216#endif
217
218	tsk->thread.address = addr;
219	tsk->thread.error_code = fsr;
220	tsk->thread.trap_no = 14;
221	force_sig_fault(sig, code, (void __user *)addr);
222}
223
224void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
225{
226	struct task_struct *tsk = current;
227	struct mm_struct *mm = tsk->active_mm;
228
229	/*
230	 * If we are in kernel mode at this point, we
231	 * have no context to handle this fault with.
232	 */
233	if (user_mode(regs))
234		__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
235	else
236		__do_kernel_fault(mm, addr, fsr, regs);
237}
238
239#ifdef CONFIG_MMU
240#ifdef CONFIG_CPU_TTBR0_PAN
241static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
242{
243	struct svc_pt_regs *svcregs;
244
245	/* If we are in user mode: permission granted */
246	if (user_mode(regs))
247		return true;
248
249	/* uaccess state saved above pt_regs on SVC exception entry */
250	svcregs = to_svc_pt_regs(regs);
251
252	return !(svcregs->ttbcr & TTBCR_EPD0);
253}
254#else
255static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
256{
257	return true;
258}
259#endif
260
261static int __kprobes
262do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr,
263			     unsigned int fsr, struct pt_regs *regs)
264{
265	if (user_mode(regs)) {
266		/*
267		 * Fault from user mode for a kernel space address. User mode
268		 * should not be faulting in kernel space, which includes the
269		 * vector/khelper page. Handle the branch predictor hardening
270		 * while interrupts are still disabled, then send a SIGSEGV.
271		 */
272		harden_branch_predictor();
273		__do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
274	} else {
275		/*
276		 * Fault from kernel mode. Enable interrupts if they were
277		 * enabled in the parent context. Section (upper page table)
278		 * translation faults are handled via do_translation_fault(),
279		 * so we will only get here for a non-present kernel space
280		 * PTE or PTE permission fault. This may happen in exceptional
281		 * circumstances and need the fixup tables to be walked.
282		 */
283		if (interrupts_enabled(regs))
284			local_irq_enable();
285
286		__do_kernel_fault(mm, addr, fsr, regs);
287	}
288
289	return 0;
290}
291
292static int __kprobes
293do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
294{
295	struct mm_struct *mm = current->mm;
296	struct vm_area_struct *vma;
297	int sig, code;
298	vm_fault_t fault;
299	unsigned int flags = FAULT_FLAG_DEFAULT;
300	vm_flags_t vm_flags = VM_ACCESS_FLAGS;
301
302	if (kprobe_page_fault(regs, fsr))
303		return 0;
304
305	/*
306	 * Handle kernel addresses faults separately, which avoids touching
307	 * the mmap lock from contexts that are not able to sleep.
308	 */
309	if (addr >= TASK_SIZE)
310		return do_kernel_address_page_fault(mm, addr, fsr, regs);
311
312	/* Enable interrupts if they were enabled in the parent context. */
313	if (interrupts_enabled(regs))
314		local_irq_enable();
315
316	/*
317	 * If we're in an interrupt or have no user
318	 * context, we must not take the fault..
319	 */
320	if (faulthandler_disabled() || !mm)
321		goto no_context;
322
323	if (user_mode(regs))
324		flags |= FAULT_FLAG_USER;
325
326	if (is_write_fault(fsr)) {
327		flags |= FAULT_FLAG_WRITE;
328		vm_flags = VM_WRITE;
329	}
330
331	if (fsr & FSR_LNX_PF) {
332		vm_flags = VM_EXEC;
333
334		if (is_permission_fault(fsr) && !user_mode(regs))
335			die_kernel_fault("execution of memory",
336					 mm, addr, fsr, regs);
337	}
338
339	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
340
341	/*
342	 * Privileged access aborts with CONFIG_CPU_TTBR0_PAN enabled are
343	 * routed via the translation fault mechanism. Check whether uaccess
344	 * is disabled while in kernel mode.
345	 */
346	if (!ttbr0_usermode_access_allowed(regs))
347		goto no_context;
348
349	if (!(flags & FAULT_FLAG_USER))
350		goto lock_mmap;
351
352	vma = lock_vma_under_rcu(mm, addr);
353	if (!vma)
354		goto lock_mmap;
355
356	if (!(vma->vm_flags & vm_flags)) {
357		vma_end_read(vma);
358		count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
359		fault = 0;
360		code = SEGV_ACCERR;
361		goto bad_area;
362	}
363	fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs);
364	if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
365		vma_end_read(vma);
366
367	if (!(fault & VM_FAULT_RETRY)) {
368		count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
369		goto done;
370	}
371	count_vm_vma_lock_event(VMA_LOCK_RETRY);
372	if (fault & VM_FAULT_MAJOR)
373		flags |= FAULT_FLAG_TRIED;
374
375	/* Quick path to respond to signals */
376	if (fault_signal_pending(fault, regs)) {
377		if (!user_mode(regs))
378			goto no_context;
379		return 0;
380	}
381lock_mmap:
382
383retry:
384	vma = lock_mm_and_find_vma(mm, addr, regs);
385	if (unlikely(!vma)) {
386		fault = 0;
387		code = SEGV_MAPERR;
388		goto bad_area;
389	}
390
391	/*
392	 * ok, we have a good vm_area for this memory access, check the
393	 * permissions on the VMA allow for the fault which occurred.
394	 */
395	if (!(vma->vm_flags & vm_flags)) {
396		mmap_read_unlock(mm);
397		fault = 0;
398		code = SEGV_ACCERR;
399		goto bad_area;
400	}
401
402	fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, regs);
403
404	/* If we need to retry but a fatal signal is pending, handle the
405	 * signal first. We do not need to release the mmap_lock because
406	 * it would already be released in __lock_page_or_retry in
407	 * mm/filemap.c. */
408	if (fault_signal_pending(fault, regs)) {
409		if (!user_mode(regs))
410			goto no_context;
411		return 0;
412	}
413
414	/* The fault is fully completed (including releasing mmap lock) */
415	if (fault & VM_FAULT_COMPLETED)
416		return 0;
417
418	if (!(fault & VM_FAULT_ERROR)) {
419		if (fault & VM_FAULT_RETRY) {
420			flags |= FAULT_FLAG_TRIED;
421			goto retry;
422		}
423	}
424
425	mmap_read_unlock(mm);
426done:
427
428	/* Handle the "normal" case first */
429	if (likely(!(fault & VM_FAULT_ERROR)))
430		return 0;
431
432	code = SEGV_MAPERR;
433bad_area:
434	/*
435	 * If we are in kernel mode at this point, we
436	 * have no context to handle this fault with.
437	 */
438	if (!user_mode(regs))
439		goto no_context;
440
441	if (fault & VM_FAULT_OOM) {
442		/*
443		 * We ran out of memory, call the OOM killer, and return to
444		 * userspace (which will retry the fault, or kill us if we
445		 * got oom-killed)
446		 */
447		pagefault_out_of_memory();
448		return 0;
449	}
450
451	if (fault & VM_FAULT_SIGBUS) {
452		/*
453		 * We had some memory, but were unable to
454		 * successfully fix up this page fault.
455		 */
456		sig = SIGBUS;
457		code = BUS_ADRERR;
458	} else {
459		/*
460		 * Something tried to access memory that
461		 * isn't in our memory map..
462		 */
463		sig = SIGSEGV;
464	}
465
466	__do_user_fault(addr, fsr, sig, code, regs);
467	return 0;
468
469no_context:
470	__do_kernel_fault(mm, addr, fsr, regs);
471	return 0;
472}
473#else					/* CONFIG_MMU */
474static int
475do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
476{
477	return 0;
478}
479#endif					/* CONFIG_MMU */
480
481/*
482 * First Level Translation Fault Handler
483 *
484 * We enter here because the first level page table doesn't contain
485 * a valid entry for the address.
486 *
487 * If this is a user address (addr < TASK_SIZE), we handle this as a
488 * normal page fault. This leaves the remainder of the function to handle
489 * kernel address translation faults.
490 *
491 * Since user mode is not permitted to access kernel addresses, pass these
492 * directly to do_kernel_address_page_fault() to handle.
493 *
494 * Otherwise, we're probably faulting in the vmalloc() area, so try to fix
495 * that up. Note that we must not take any locks or enable interrupts in
496 * this case.
497 *
498 * If vmalloc() fixup fails, that means the non-leaf page tables did not
499 * contain an entry for this address, so handle this via
500 * do_kernel_address_page_fault().
501 */
502#ifdef CONFIG_MMU
503static int __kprobes
504do_translation_fault(unsigned long addr, unsigned int fsr,
505		     struct pt_regs *regs)
506{
507	unsigned int index;
508	pgd_t *pgd, *pgd_k;
509	p4d_t *p4d, *p4d_k;
510	pud_t *pud, *pud_k;
511	pmd_t *pmd, *pmd_k;
512
513	if (addr < TASK_SIZE)
514		return do_page_fault(addr, fsr, regs);
515
516	if (user_mode(regs))
517		goto bad_area;
518
519	index = pgd_index(addr);
520
521	pgd = cpu_get_pgd() + index;
522	pgd_k = init_mm.pgd + index;
523
524	p4d = p4d_offset(pgd, addr);
525	p4d_k = p4d_offset(pgd_k, addr);
526
527	if (p4d_none(*p4d_k))
528		goto bad_area;
529	if (!p4d_present(*p4d))
530		set_p4d(p4d, *p4d_k);
531
532	pud = pud_offset(p4d, addr);
533	pud_k = pud_offset(p4d_k, addr);
534
535	if (pud_none(*pud_k))
536		goto bad_area;
537	if (!pud_present(*pud))
538		set_pud(pud, *pud_k);
539
540	pmd = pmd_offset(pud, addr);
541	pmd_k = pmd_offset(pud_k, addr);
542
543#ifdef CONFIG_ARM_LPAE
544	/*
545	 * Only one hardware entry per PMD with LPAE.
546	 */
547	index = 0;
548#else
549	/*
550	 * On ARM one Linux PGD entry contains two hardware entries (see page
551	 * tables layout in pgtable.h). We normally guarantee that we always
552	 * fill both L1 entries. But create_mapping() doesn't follow the rule.
553	 * It can create inidividual L1 entries, so here we have to call
554	 * pmd_none() check for the entry really corresponded to address, not
555	 * for the first of pair.
556	 */
557	index = (addr >> SECTION_SHIFT) & 1;
558#endif
559	if (pmd_none(pmd_k[index]))
560		goto bad_area;
561
562	copy_pmd(pmd, pmd_k);
563	return 0;
564
565bad_area:
566	do_kernel_address_page_fault(current->mm, addr, fsr, regs);
567
568	return 0;
569}
570#else					/* CONFIG_MMU */
571static int
572do_translation_fault(unsigned long addr, unsigned int fsr,
573		     struct pt_regs *regs)
574{
575	return 0;
576}
577#endif					/* CONFIG_MMU */
578
579/*
580 * Some section permission faults need to be handled gracefully.
581 * They can happen due to a __{get,put}_user during an oops.
582 */
583#ifndef CONFIG_ARM_LPAE
584static int
585do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
586{
587	/*
588	 * If this is a kernel address, but from user mode, then userspace
589	 * is trying bad stuff. Invoke the branch predictor handling.
590	 * Interrupts are disabled here.
591	 */
592	if (addr >= TASK_SIZE && user_mode(regs))
593		harden_branch_predictor();
594
595	do_bad_area(addr, fsr, regs);
596
597	return 0;
598}
599#endif /* CONFIG_ARM_LPAE */
600
601/*
602 * This abort handler always returns "fault".
603 */
604static int
605do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
606{
607	return 1;
608}
609
610struct fsr_info {
611	int	(*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
612	int	sig;
613	int	code;
614	const char *name;
615};
616
617/* FSR definition */
618#ifdef CONFIG_ARM_LPAE
619#include "fsr-3level.c"
620#else
621#include "fsr-2level.c"
622#endif
623
624void __init
625hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
626		int sig, int code, const char *name)
627{
628	if (nr < 0 || nr >= ARRAY_SIZE(fsr_info))
629		BUG();
630
631	fsr_info[nr].fn   = fn;
632	fsr_info[nr].sig  = sig;
633	fsr_info[nr].code = code;
634	fsr_info[nr].name = name;
635}
636
637/*
638 * Dispatch a data abort to the relevant handler.
639 */
640asmlinkage void
641do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
642{
643	const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
644
645	if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
646		return;
647
648	pr_alert("8<--- cut here ---\n");
649	pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
650		inf->name, fsr, addr);
651	show_pte(KERN_ALERT, current->mm, addr);
652
653	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
654		       fsr, 0);
655}
656
657void __init
658hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
659		 int sig, int code, const char *name)
660{
661	if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
662		BUG();
663
664	ifsr_info[nr].fn   = fn;
665	ifsr_info[nr].sig  = sig;
666	ifsr_info[nr].code = code;
667	ifsr_info[nr].name = name;
668}
669
670asmlinkage void
671do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
672{
673	const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
674
675	if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
676		return;
677
678	pr_alert("8<--- cut here ---\n");
679	pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
680		inf->name, ifsr, addr);
681
682	arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
683		       ifsr, 0);
684}
685
686/*
687 * Abort handler to be used only during first unmasking of asynchronous aborts
688 * on the boot CPU. This makes sure that the machine will not die if the
689 * firmware/bootloader left an imprecise abort pending for us to trip over.
690 */
691static int __init early_abort_handler(unsigned long addr, unsigned int fsr,
692				      struct pt_regs *regs)
693{
694	pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during "
695		"first unmask, this is most likely caused by a "
696		"firmware/bootloader bug.\n", fsr);
697
698	return 0;
699}
700
701void __init early_abt_enable(void)
702{
703	fsr_info[FSR_FS_AEA].fn = early_abort_handler;
704	local_abt_enable();
705	fsr_info[FSR_FS_AEA].fn = do_bad;
706}
707
708#ifndef CONFIG_ARM_LPAE
709static int __init exceptions_init(void)
710{
711	if (cpu_architecture() >= CPU_ARCH_ARMv6) {
712		hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR,
713				"I-cache maintenance fault");
714	}
715
716	if (cpu_architecture() >= CPU_ARCH_ARMv7) {
717		/*
718		 * TODO: Access flag faults introduced in ARMv6K.
719		 * Runtime check for 'K' extension is needed
720		 */
721		hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR,
722				"section access flag fault");
723		hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR,
724				"section access flag fault");
725	}
726
727	return 0;
728}
729
730arch_initcall(exceptions_init);
731#endif