kprobes.c

  1// SPDX-License-Identifier: GPL-2.0+
  2
  3#define pr_fmt(fmt) "kprobes: " fmt
  4
  5#include <linux/kprobes.h>
  6#include <linux/extable.h>
  7#include <linux/slab.h>
  8#include <linux/stop_machine.h>
  9#include <linux/vmalloc.h>
 10#include <asm/ptrace.h>
 11#include <linux/uaccess.h>
 12#include <asm/sections.h>
 13#include <asm/cacheflush.h>
 14#include <asm/bug.h>
 15#include <asm/text-patching.h>
 16
 17#include "decode-insn.h"
 18
 19DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
 20DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
 21
 22static void __kprobes
 23post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *);
 24
 25static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
 26{
 27	size_t len = GET_INSN_LENGTH(p->opcode);
 28	u32 insn = __BUG_INSN_32;
 29
 30	p->ainsn.api.restore = (unsigned long)p->addr + len;
 31
 32	patch_text_nosync(p->ainsn.api.insn, &p->opcode, len);
 33	patch_text_nosync((void *)p->ainsn.api.insn + len, &insn, GET_INSN_LENGTH(insn));
 34}
 35
 36static void __kprobes arch_prepare_simulate(struct kprobe *p)
 37{
 38	p->ainsn.api.restore = 0;
 39}
 40
 41static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
 42{
 43	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 44
 45	if (p->ainsn.api.handler)
 46		p->ainsn.api.handler((u32)p->opcode,
 47					(unsigned long)p->addr, regs);
 48
 49	post_kprobe_handler(p, kcb, regs);
 50}
 51
 52static bool __kprobes arch_check_kprobe(unsigned long addr)
 53{
 54	unsigned long tmp, offset;
 55
 56	/* start iterating at the closest preceding symbol */
 57	if (!kallsyms_lookup_size_offset(addr, NULL, &offset))
 58		return false;
 59
 60	tmp = addr - offset;
 61
 62	while (tmp <= addr) {
 63		if (tmp == addr)
 64			return true;
 65
 66		tmp += GET_INSN_LENGTH(*(u16 *)tmp);
 67	}
 68
 69	return false;
 70}
 71
 72int __kprobes arch_prepare_kprobe(struct kprobe *p)
 73{
 74	u16 *insn = (u16 *)p->addr;
 75
 76	if ((unsigned long)insn & 0x1)
 77		return -EILSEQ;
 78
 79	if (!arch_check_kprobe((unsigned long)p->addr))
 80		return -EILSEQ;
 81
 82	/* copy instruction */
 83	p->opcode = (kprobe_opcode_t)(*insn++);
 84	if (GET_INSN_LENGTH(p->opcode) == 4)
 85		p->opcode |= (kprobe_opcode_t)(*insn) << 16;
 86
 87	/* decode instruction */
 88	switch (riscv_probe_decode_insn(p->addr, &p->ainsn.api)) {
 89	case INSN_REJECTED:	/* insn not supported */
 90		return -EINVAL;
 91
 92	case INSN_GOOD_NO_SLOT:	/* insn need simulation */
 93		p->ainsn.api.insn = NULL;
 94		break;
 95
 96	case INSN_GOOD:	/* instruction uses slot */
 97		p->ainsn.api.insn = get_insn_slot();
 98		if (!p->ainsn.api.insn)
 99			return -ENOMEM;
100		break;
101	}
102
103	/* prepare the instruction */
104	if (p->ainsn.api.insn)
105		arch_prepare_ss_slot(p);
106	else
107		arch_prepare_simulate(p);
108
109	return 0;
110}
111
112/* install breakpoint in text */
113void __kprobes arch_arm_kprobe(struct kprobe *p)
114{
115	size_t len = GET_INSN_LENGTH(p->opcode);
116	u32 insn = len == 4 ? __BUG_INSN_32 : __BUG_INSN_16;
117
118	patch_text(p->addr, &insn, len);
119}
120
121/* remove breakpoint from text */
122void __kprobes arch_disarm_kprobe(struct kprobe *p)
123{
124	size_t len = GET_INSN_LENGTH(p->opcode);
125
126	patch_text(p->addr, &p->opcode, len);
127}
128
129void __kprobes arch_remove_kprobe(struct kprobe *p)
130{
131}
132
133static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
134{
135	kcb->prev_kprobe.kp = kprobe_running();
136	kcb->prev_kprobe.status = kcb->kprobe_status;
137}
138
139static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
140{
141	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
142	kcb->kprobe_status = kcb->prev_kprobe.status;
143}
144
145static void __kprobes set_current_kprobe(struct kprobe *p)
146{
147	__this_cpu_write(current_kprobe, p);
148}
149
150/*
151 * Interrupts need to be disabled before single-step mode is set, and not
152 * reenabled until after single-step mode ends.
153 * Without disabling interrupt on local CPU, there is a chance of
154 * interrupt occurrence in the period of exception return and  start of
155 * out-of-line single-step, that result in wrongly single stepping
156 * into the interrupt handler.
157 */
158static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
159						struct pt_regs *regs)
160{
161	kcb->saved_status = regs->status;
162	regs->status &= ~SR_SPIE;
163}
164
165static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
166						struct pt_regs *regs)
167{
168	regs->status = kcb->saved_status;
169}
170
171static void __kprobes setup_singlestep(struct kprobe *p,
172				       struct pt_regs *regs,
173				       struct kprobe_ctlblk *kcb, int reenter)
174{
175	unsigned long slot;
176
177	if (reenter) {
178		save_previous_kprobe(kcb);
179		set_current_kprobe(p);
180		kcb->kprobe_status = KPROBE_REENTER;
181	} else {
182		kcb->kprobe_status = KPROBE_HIT_SS;
183	}
184
185	if (p->ainsn.api.insn) {
186		/* prepare for single stepping */
187		slot = (unsigned long)p->ainsn.api.insn;
188
189		/* IRQs and single stepping do not mix well. */
190		kprobes_save_local_irqflag(kcb, regs);
191
192		instruction_pointer_set(regs, slot);
193	} else {
194		/* insn simulation */
195		arch_simulate_insn(p, regs);
196	}
197}
198
199static int __kprobes reenter_kprobe(struct kprobe *p,
200				    struct pt_regs *regs,
201				    struct kprobe_ctlblk *kcb)
202{
203	switch (kcb->kprobe_status) {
204	case KPROBE_HIT_SSDONE:
205	case KPROBE_HIT_ACTIVE:
206		kprobes_inc_nmissed_count(p);
207		setup_singlestep(p, regs, kcb, 1);
208		break;
209	case KPROBE_HIT_SS:
210	case KPROBE_REENTER:
211		pr_warn("Failed to recover from reentered kprobes.\n");
212		dump_kprobe(p);
213		BUG();
214		break;
215	default:
216		WARN_ON(1);
217		return 0;
218	}
219
220	return 1;
221}
222
223static void __kprobes
224post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regs *regs)
225{
226	/* return addr restore if non-branching insn */
227	if (cur->ainsn.api.restore != 0)
228		regs->epc = cur->ainsn.api.restore;
229
230	/* restore back original saved kprobe variables and continue */
231	if (kcb->kprobe_status == KPROBE_REENTER) {
232		restore_previous_kprobe(kcb);
233		return;
234	}
235
236	/* call post handler */
237	kcb->kprobe_status = KPROBE_HIT_SSDONE;
238	if (cur->post_handler)	{
239		/* post_handler can hit breakpoint and single step
240		 * again, so we enable D-flag for recursive exception.
241		 */
242		cur->post_handler(cur, regs, 0);
243	}
244
245	reset_current_kprobe();
246}
247
248int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr)
249{
250	struct kprobe *cur = kprobe_running();
251	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
252
253	switch (kcb->kprobe_status) {
254	case KPROBE_HIT_SS:
255	case KPROBE_REENTER:
256		/*
257		 * We are here because the instruction being single
258		 * stepped caused a page fault. We reset the current
259		 * kprobe and the ip points back to the probe address
260		 * and allow the page fault handler to continue as a
261		 * normal page fault.
262		 */
263		regs->epc = (unsigned long) cur->addr;
264		BUG_ON(!instruction_pointer(regs));
265
266		if (kcb->kprobe_status == KPROBE_REENTER)
267			restore_previous_kprobe(kcb);
268		else {
269			kprobes_restore_local_irqflag(kcb, regs);
270			reset_current_kprobe();
271		}
272
273		break;
274	case KPROBE_HIT_ACTIVE:
275	case KPROBE_HIT_SSDONE:
276		/*
277		 * In case the user-specified fault handler returned
278		 * zero, try to fix up.
279		 */
280		if (fixup_exception(regs))
281			return 1;
282	}
283	return 0;
284}
285
286bool __kprobes
287kprobe_breakpoint_handler(struct pt_regs *regs)
288{
289	struct kprobe *p, *cur_kprobe;
290	struct kprobe_ctlblk *kcb;
291	unsigned long addr = instruction_pointer(regs);
292
293	kcb = get_kprobe_ctlblk();
294	cur_kprobe = kprobe_running();
295
296	p = get_kprobe((kprobe_opcode_t *) addr);
297
298	if (p) {
299		if (cur_kprobe) {
300			if (reenter_kprobe(p, regs, kcb))
301				return true;
302		} else {
303			/* Probe hit */
304			set_current_kprobe(p);
305			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
306
307			/*
308			 * If we have no pre-handler or it returned 0, we
309			 * continue with normal processing.  If we have a
310			 * pre-handler and it returned non-zero, it will
311			 * modify the execution path and no need to single
312			 * stepping. Let's just reset current kprobe and exit.
313			 *
314			 * pre_handler can hit a breakpoint and can step thru
315			 * before return.
316			 */
317			if (!p->pre_handler || !p->pre_handler(p, regs))
318				setup_singlestep(p, regs, kcb, 0);
319			else
320				reset_current_kprobe();
321		}
322		return true;
323	}
324
325	/*
326	 * The breakpoint instruction was removed right
327	 * after we hit it.  Another cpu has removed
328	 * either a probepoint or a debugger breakpoint
329	 * at this address.  In either case, no further
330	 * handling of this interrupt is appropriate.
331	 * Return back to original instruction, and continue.
332	 */
333	return false;
334}
335
336bool __kprobes
337kprobe_single_step_handler(struct pt_regs *regs)
338{
339	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
340	unsigned long addr = instruction_pointer(regs);
341	struct kprobe *cur = kprobe_running();
342
343	if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) &&
344	    ((unsigned long)&cur->ainsn.api.insn[0] + GET_INSN_LENGTH(cur->opcode) == addr)) {
345		kprobes_restore_local_irqflag(kcb, regs);
346		post_kprobe_handler(cur, kcb, regs);
347		return true;
348	}
349	/* not ours, kprobes should ignore it */
350	return false;
351}
352
353/*
354 * Provide a blacklist of symbols identifying ranges which cannot be kprobed.
355 * This blacklist is exposed to userspace via debugfs (kprobes/blacklist).
356 */
357int __init arch_populate_kprobe_blacklist(void)
358{
359	int ret;
360
361	ret = kprobe_add_area_blacklist((unsigned long)__irqentry_text_start,
362					(unsigned long)__irqentry_text_end);
363	return ret;
364}
365
366int __kprobes arch_trampoline_kprobe(struct kprobe *p)
367{
368	return 0;
369}
370
371int __init arch_init_kprobes(void)
372{
373	return 0;
374}