fault.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 *
  5 * Derived from MIPS:
  6 * Copyright (C) 1995 - 2000 by Ralf Baechle
  7 */
  8#include <linux/context_tracking.h>
  9#include <linux/signal.h>
 10#include <linux/sched.h>
 11#include <linux/interrupt.h>
 12#include <linux/kernel.h>
 13#include <linux/entry-common.h>
 14#include <linux/errno.h>
 15#include <linux/string.h>
 16#include <linux/types.h>
 17#include <linux/ptrace.h>
 18#include <linux/ratelimit.h>
 19#include <linux/mman.h>
 20#include <linux/mm.h>
 21#include <linux/smp.h>
 22#include <linux/kdebug.h>
 23#include <linux/perf_event.h>
 24#include <linux/uaccess.h>
 25#include <linux/kfence.h>
 26
 27#include <asm/branch.h>
 28#include <asm/exception.h>
 29#include <asm/mmu_context.h>
 30#include <asm/ptrace.h>
 31
 32int show_unhandled_signals = 1;
 33
 34static int __kprobes spurious_fault(unsigned long write, unsigned long address)
 35{
 36	pgd_t *pgd;
 37	p4d_t *p4d;
 38	pud_t *pud;
 39	pmd_t *pmd;
 40	pte_t *pte;
 41
 42	if (!(address & __UA_LIMIT))
 43		return 0;
 44
 45	pgd = pgd_offset_k(address);
 46	if (!pgd_present(pgdp_get(pgd)))
 47		return 0;
 48
 49	p4d = p4d_offset(pgd, address);
 50	if (!p4d_present(p4dp_get(p4d)))
 51		return 0;
 52
 53	pud = pud_offset(p4d, address);
 54	if (!pud_present(pudp_get(pud)))
 55		return 0;
 56
 57	pmd = pmd_offset(pud, address);
 58	if (!pmd_present(pmdp_get(pmd)))
 59		return 0;
 60
 61	if (pmd_leaf(*pmd)) {
 62		return write ? pmd_write(pmdp_get(pmd)) : 1;
 63	} else {
 64		pte = pte_offset_kernel(pmd, address);
 65		if (!pte_present(ptep_get(pte)))
 66			return 0;
 67
 68		return write ? pte_write(ptep_get(pte)) : 1;
 69	}
 70}
 71
 72static void __kprobes no_context(struct pt_regs *regs,
 73			unsigned long write, unsigned long address)
 74{
 75	const int field = sizeof(unsigned long) * 2;
 76
 77	if (spurious_fault(write, address))
 78		return;
 79
 80	/* Are we prepared to handle this kernel fault?	 */
 81	if (fixup_exception(regs))
 82		return;
 83
 84	if (kfence_handle_page_fault(address, write, regs))
 85		return;
 86
 87	/*
 88	 * Oops. The kernel tried to access some bad page. We'll have to
 89	 * terminate things with extreme prejudice.
 90	 */
 91	bust_spinlocks(1);
 92
 93	pr_alert("CPU %d Unable to handle kernel paging request at "
 94	       "virtual address %0*lx, era == %0*lx, ra == %0*lx\n",
 95	       raw_smp_processor_id(), field, address, field, regs->csr_era,
 96	       field,  regs->regs[1]);
 97	die("Oops", regs);
 98}
 99
100static void __kprobes do_out_of_memory(struct pt_regs *regs,
101			unsigned long write, unsigned long address)
102{
103	/*
104	 * We ran out of memory, call the OOM killer, and return the userspace
105	 * (which will retry the fault, or kill us if we got oom-killed).
106	 */
107	if (!user_mode(regs)) {
108		no_context(regs, write, address);
109		return;
110	}
111	pagefault_out_of_memory();
112}
113
114static void __kprobes do_sigbus(struct pt_regs *regs,
115		unsigned long write, unsigned long address, int si_code)
116{
117	/* Kernel mode? Handle exceptions or die */
118	if (!user_mode(regs)) {
119		no_context(regs, write, address);
120		return;
121	}
122
123	/*
124	 * Send a sigbus, regardless of whether we were in kernel
125	 * or user mode.
126	 */
127	current->thread.csr_badvaddr = address;
128	current->thread.trap_nr = read_csr_excode();
129	force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
130}
131
132static void __kprobes do_sigsegv(struct pt_regs *regs,
133		unsigned long write, unsigned long address, int si_code)
134{
135	const int field = sizeof(unsigned long) * 2;
136	static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
137
138	/* Kernel mode? Handle exceptions or die */
139	if (!user_mode(regs)) {
140		no_context(regs, write, address);
141		return;
142	}
143
144	/* User mode accesses just cause a SIGSEGV */
145	current->thread.csr_badvaddr = address;
146	if (!write)
147		current->thread.error_code = 1;
148	else
149		current->thread.error_code = 2;
150	current->thread.trap_nr = read_csr_excode();
151
152	if (show_unhandled_signals &&
153	    unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) {
154		pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n",
155			current->comm,
156			write ? "write access to" : "read access from",
157			field, address);
158		pr_info("era = %0*lx in", field,
159			(unsigned long) regs->csr_era);
160		print_vma_addr(KERN_CONT " ", regs->csr_era);
161		pr_cont("\n");
162		pr_info("ra  = %0*lx in", field,
163			(unsigned long) regs->regs[1]);
164		print_vma_addr(KERN_CONT " ", regs->regs[1]);
165		pr_cont("\n");
166	}
167	force_sig_fault(SIGSEGV, si_code, (void __user *)address);
168}
169
170/*
171 * This routine handles page faults.  It determines the address,
172 * and the problem, and then passes it off to one of the appropriate
173 * routines.
174 */
175static void __kprobes __do_page_fault(struct pt_regs *regs,
176			unsigned long write, unsigned long address)
177{
178	int si_code = SEGV_MAPERR;
179	unsigned int flags = FAULT_FLAG_DEFAULT;
180	struct task_struct *tsk = current;
181	struct mm_struct *mm = tsk->mm;
182	struct vm_area_struct *vma = NULL;
183	vm_fault_t fault;
184
185	if (kprobe_page_fault(regs, current->thread.trap_nr))
186		return;
187
188	/*
189	 * We fault-in kernel-space virtual memory on-demand. The
190	 * 'reference' page table is init_mm.pgd.
191	 *
192	 * NOTE! We MUST NOT take any locks for this case. We may
193	 * be in an interrupt or a critical region, and should
194	 * only copy the information from the master page table,
195	 * nothing more.
196	 */
197	if (address & __UA_LIMIT) {
198		if (!user_mode(regs))
199			no_context(regs, write, address);
200		else
201			do_sigsegv(regs, write, address, si_code);
202		return;
203	}
204
205	/*
206	 * If we're in an interrupt or have no user
207	 * context, we must not take the fault..
208	 */
209	if (faulthandler_disabled() || !mm) {
210		do_sigsegv(regs, write, address, si_code);
211		return;
212	}
213
214	if (user_mode(regs))
215		flags |= FAULT_FLAG_USER;
216
217	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
218
219	if (!(flags & FAULT_FLAG_USER))
220		goto lock_mmap;
221
222	vma = lock_vma_under_rcu(mm, address);
223	if (!vma)
224		goto lock_mmap;
225
226	if (write) {
227		flags |= FAULT_FLAG_WRITE;
228		if (!(vma->vm_flags & VM_WRITE)) {
229			vma_end_read(vma);
230			si_code = SEGV_ACCERR;
231			count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
232			goto bad_area_nosemaphore;
233		}
234	} else {
235		if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) {
236			vma_end_read(vma);
237			si_code = SEGV_ACCERR;
238			count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
239			goto bad_area_nosemaphore;
240		}
241		if (!(vma->vm_flags & (VM_READ | VM_WRITE)) && address != exception_era(regs)) {
242			vma_end_read(vma);
243			si_code = SEGV_ACCERR;
244			count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
245			goto bad_area_nosemaphore;
246		}
247	}
248
249	fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
250	if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
251		vma_end_read(vma);
252
253	if (!(fault & VM_FAULT_RETRY)) {
254		count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
255		goto done;
256	}
257
258	count_vm_vma_lock_event(VMA_LOCK_RETRY);
259	if (fault & VM_FAULT_MAJOR)
260		flags |= FAULT_FLAG_TRIED;
261
262	/* Quick path to respond to signals */
263	if (fault_signal_pending(fault, regs)) {
264		if (!user_mode(regs))
265			no_context(regs, write, address);
266		return;
267	}
268lock_mmap:
269
270retry:
271	vma = lock_mm_and_find_vma(mm, address, regs);
272	if (unlikely(!vma))
273		goto bad_area_nosemaphore;
274	goto good_area;
275
276/*
277 * Something tried to access memory that isn't in our memory map..
278 * Fix it, but check if it's kernel or user first..
279 */
280bad_area:
281	mmap_read_unlock(mm);
282bad_area_nosemaphore:
283	do_sigsegv(regs, write, address, si_code);
284	return;
285
286/*
287 * Ok, we have a good vm_area for this memory access, so
288 * we can handle it..
289 */
290good_area:
291	si_code = SEGV_ACCERR;
292
293	if (write) {
294		flags |= FAULT_FLAG_WRITE;
295		if (!(vma->vm_flags & VM_WRITE))
296			goto bad_area;
297	} else {
298		if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs))
299			goto bad_area;
300		if (!(vma->vm_flags & (VM_READ | VM_WRITE)) && address != exception_era(regs))
301			goto bad_area;
302	}
303
304	/*
305	 * If for any reason at all we couldn't handle the fault,
306	 * make sure we exit gracefully rather than endlessly redo
307	 * the fault.
308	 */
309	fault = handle_mm_fault(vma, address, flags, regs);
310
311	if (fault_signal_pending(fault, regs)) {
312		if (!user_mode(regs))
313			no_context(regs, write, address);
314		return;
315	}
316
317	/* The fault is fully completed (including releasing mmap lock) */
318	if (fault & VM_FAULT_COMPLETED)
319		return;
320
321	if (unlikely(fault & VM_FAULT_RETRY)) {
322		flags |= FAULT_FLAG_TRIED;
323
324		/*
325		 * No need to mmap_read_unlock(mm) as we would
326		 * have already released it in __lock_page_or_retry
327		 * in mm/filemap.c.
328		 */
329		goto retry;
330	}
331	mmap_read_unlock(mm);
332
333done:
334	if (unlikely(fault & VM_FAULT_ERROR)) {
335		if (fault & VM_FAULT_OOM) {
336			do_out_of_memory(regs, write, address);
337			return;
338		} else if (fault & VM_FAULT_SIGSEGV) {
339			do_sigsegv(regs, write, address, si_code);
340			return;
341		} else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
342			do_sigbus(regs, write, address, si_code);
343			return;
344		}
345		BUG();
346	}
347}
348
349asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
350			unsigned long write, unsigned long address)
351{
352	irqentry_state_t state = irqentry_enter(regs);
353
354	/* Enable interrupt if enabled in parent context */
355	if (likely(regs->csr_prmd & CSR_PRMD_PIE))
356		local_irq_enable();
357
358	__do_page_fault(regs, write, address);
359
360	local_irq_disable();
361
362	irqentry_exit(regs, state);
363}