FILE
core.c
File preview with syntax highlighting.
linux-6.19.14/arch/x86/kernel/kprobes/core.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation ( includes contributions from 9 * Rusty Russell). 10 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 11 * interface to access function arguments. 12 * 2004-Oct Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 13 * <prasanna@in.ibm.com> adapted for x86_64 from i386. 14 * 2005-Mar Roland McGrath <roland@redhat.com> 15 * Fixed to handle %rip-relative addressing mode correctly. 16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 18 * <prasanna@in.ibm.com> added function-return probes. 19 * 2005-May Rusty Lynch <rusty.lynch@intel.com> 20 * Added function return probes functionality 21 * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added 22 * kprobe-booster and kretprobe-booster for i386. 23 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster 24 * and kretprobe-booster for x86-64 25 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven 26 * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com> 27 * unified x86 kprobes code. 28 */ 29#include <linux/kprobes.h> 30#include <linux/ptrace.h> 31#include <linux/string.h> 32#include <linux/slab.h> 33#include <linux/hardirq.h> 34#include <linux/preempt.h> 35#include <linux/sched/debug.h> 36#include <linux/perf_event.h> 37#include <linux/extable.h> 38#include <linux/kdebug.h> 39#include <linux/kallsyms.h> 40#include <linux/kgdb.h> 41#include <linux/ftrace.h> 42#include <linux/kasan.h> 43#include <linux/objtool.h> 44#include <linux/vmalloc.h> 45#include <linux/pgtable.h> 46#include <linux/set_memory.h> 47#include <linux/cfi.h> 48#include <linux/execmem.h> 49 50#include <asm/text-patching.h> 51#include <asm/cacheflush.h> 52#include <asm/desc.h> 53#include <linux/uaccess.h> 54#include <asm/alternative.h> 55#include <asm/insn.h> 56#include <asm/debugreg.h> 57#include <asm/ibt.h> 58 59#include "common.h" 60 61DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 62DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 63 64#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ 65 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \ 66 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \ 67 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \ 68 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \ 69 << (row % 32)) 70 /* 71 * Undefined/reserved opcodes, conditional jump, Opcode Extension 72 * Groups, and some special opcodes can not boost. 73 * This is non-const and volatile to keep gcc from statically 74 * optimizing it out, as variable_test_bit makes gcc think only 75 * *(unsigned long*) is used. 76 */ 77static volatile u32 twobyte_is_boostable[256 / 32] = { 78 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 79 /* ---------------------------------------------- */ 80 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */ 81 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */ 82 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */ 83 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */ 84 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */ 85 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */ 86 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */ 87 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */ 88 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */ 89 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */ 90 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */ 91 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */ 92 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */ 93 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */ 94 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */ 95 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */ 96 /* ----------------------------------------------- */ 97 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 98}; 99#undef W 100 101struct kretprobe_blackpoint kretprobe_blacklist[] = { 102 {"__switch_to", }, /* This function switches only current task, but 103 doesn't switch kernel stack.*/ 104 {NULL, NULL} /* Terminator */ 105}; 106 107const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); 108 109static nokprobe_inline void 110__synthesize_relative_insn(void *dest, void *from, void *to, u8 op) 111{ 112 struct __arch_relative_insn { 113 u8 op; 114 s32 raddr; 115 } __packed *insn; 116 117 insn = (struct __arch_relative_insn *)dest; 118 insn->raddr = (s32)((long)(to) - ((long)(from) + 5)); 119 insn->op = op; 120} 121 122/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ 123void synthesize_reljump(void *dest, void *from, void *to) 124{ 125 __synthesize_relative_insn(dest, from, to, JMP32_INSN_OPCODE); 126} 127NOKPROBE_SYMBOL(synthesize_reljump); 128 129/* Insert a call instruction at address 'from', which calls address 'to'.*/ 130void synthesize_relcall(void *dest, void *from, void *to) 131{ 132 __synthesize_relative_insn(dest, from, to, CALL_INSN_OPCODE); 133} 134NOKPROBE_SYMBOL(synthesize_relcall); 135 136/* 137 * Returns non-zero if INSN is boostable. 138 * RIP relative instructions are adjusted at copying time in 64 bits mode 139 */ 140bool can_boost(struct insn *insn, void *addr) 141{ 142 kprobe_opcode_t opcode; 143 insn_byte_t prefix; 144 145 if (search_exception_tables((unsigned long)addr)) 146 return false; /* Page fault may occur on this address. */ 147 148 /* 2nd-byte opcode */ 149 if (insn->opcode.nbytes == 2) 150 return test_bit(insn->opcode.bytes[1], 151 (unsigned long *)twobyte_is_boostable); 152 153 if (insn->opcode.nbytes != 1) 154 return false; 155 156 for_each_insn_prefix(insn, prefix) { 157 insn_attr_t attr; 158 159 attr = inat_get_opcode_attribute(prefix); 160 /* Can't boost Address-size override prefix and CS override prefix */ 161 if (prefix == 0x2e || inat_is_address_size_prefix(attr)) 162 return false; 163 } 164 165 opcode = insn->opcode.bytes[0]; 166 167 switch (opcode) { 168 case 0x62: /* bound */ 169 case 0x70 ... 0x7f: /* Conditional jumps */ 170 case 0x9a: /* Call far */ 171 case 0xcc ... 0xce: /* software exceptions */ 172 case 0xd6: /* (UD) */ 173 case 0xd8 ... 0xdf: /* ESC */ 174 case 0xe0 ... 0xe3: /* LOOP*, JCXZ */ 175 case 0xe8 ... 0xe9: /* near Call, JMP */ 176 case 0xeb: /* Short JMP */ 177 case 0xf0 ... 0xf4: /* LOCK/REP, HLT */ 178 /* ... are not boostable */ 179 return false; 180 case 0xc0 ... 0xc1: /* Grp2 */ 181 case 0xd0 ... 0xd3: /* Grp2 */ 182 /* 183 * AMD uses nnn == 110 as SHL/SAL, but Intel makes it reserved. 184 */ 185 return X86_MODRM_REG(insn->modrm.bytes[0]) != 0b110; 186 case 0xf6 ... 0xf7: /* Grp3 */ 187 /* AMD uses nnn == 001 as TEST, but Intel makes it reserved. */ 188 return X86_MODRM_REG(insn->modrm.bytes[0]) != 0b001; 189 case 0xfe: /* Grp4 */ 190 /* Only INC and DEC are boostable */ 191 return X86_MODRM_REG(insn->modrm.bytes[0]) == 0b000 || 192 X86_MODRM_REG(insn->modrm.bytes[0]) == 0b001; 193 case 0xff: /* Grp5 */ 194 /* Only INC, DEC, and indirect JMP are boostable */ 195 return X86_MODRM_REG(insn->modrm.bytes[0]) == 0b000 || 196 X86_MODRM_REG(insn->modrm.bytes[0]) == 0b001 || 197 X86_MODRM_REG(insn->modrm.bytes[0]) == 0b100; 198 default: 199 return true; 200 } 201} 202 203static unsigned long 204__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) 205{ 206 struct kprobe *kp; 207 bool faddr; 208 209 kp = get_kprobe((void *)addr); 210 faddr = ftrace_location(addr) == addr; 211 /* 212 * Use the current code if it is not modified by Kprobe 213 * and it cannot be modified by ftrace. 214 */ 215 if (!kp && !faddr) 216 return addr; 217 218 /* 219 * Basically, kp->ainsn.insn has an original instruction. 220 * However, RIP-relative instruction can not do single-stepping 221 * at different place, __copy_instruction() tweaks the displacement of 222 * that instruction. In that case, we can't recover the instruction 223 * from the kp->ainsn.insn. 224 * 225 * On the other hand, in case on normal Kprobe, kp->opcode has a copy 226 * of the first byte of the probed instruction, which is overwritten 227 * by int3. And the instruction at kp->addr is not modified by kprobes 228 * except for the first byte, we can recover the original instruction 229 * from it and kp->opcode. 230 * 231 * In case of Kprobes using ftrace, we do not have a copy of 232 * the original instruction. In fact, the ftrace location might 233 * be modified at anytime and even could be in an inconsistent state. 234 * Fortunately, we know that the original code is the ideal 5-byte 235 * long NOP. 236 */ 237 if (copy_from_kernel_nofault(buf, (void *)addr, 238 MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 239 return 0UL; 240 241 if (faddr) 242 memcpy(buf, x86_nops[5], 5); 243 else 244 buf[0] = kp->opcode; 245 return (unsigned long)buf; 246} 247 248/* 249 * Recover the probed instruction at addr for further analysis. 250 * Caller must lock kprobes by kprobe_mutex, or disable preemption 251 * for preventing to release referencing kprobes. 252 * Returns zero if the instruction can not get recovered (or access failed). 253 */ 254unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) 255{ 256 unsigned long __addr; 257 258 __addr = __recover_optprobed_insn(buf, addr); 259 if (__addr != addr) 260 return __addr; 261 262 return __recover_probed_insn(buf, addr); 263} 264 265/* Check if insn is INT or UD */ 266static inline bool is_exception_insn(struct insn *insn) 267{ 268 /* UD uses 0f escape */ 269 if (insn->opcode.bytes[0] == 0x0f) { 270 /* UD0 / UD1 / UD2 */ 271 return insn->opcode.bytes[1] == 0xff || 272 insn->opcode.bytes[1] == 0xb9 || 273 insn->opcode.bytes[1] == 0x0b; 274 } 275 276 /* INT3 / INT n / INTO / INT1 */ 277 return insn->opcode.bytes[0] == 0xcc || 278 insn->opcode.bytes[0] == 0xcd || 279 insn->opcode.bytes[0] == 0xce || 280 insn->opcode.bytes[0] == 0xf1; 281} 282 283/* 284 * Check if paddr is at an instruction boundary and that instruction can 285 * be probed 286 */ 287static bool can_probe(unsigned long paddr) 288{ 289 unsigned long addr, __addr, offset = 0; 290 struct insn insn; 291 kprobe_opcode_t buf[MAX_INSN_SIZE]; 292 293 if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) 294 return false; 295 296 /* Decode instructions */ 297 addr = paddr - offset; 298 while (addr < paddr) { 299 /* 300 * Check if the instruction has been modified by another 301 * kprobe, in which case we replace the breakpoint by the 302 * original instruction in our buffer. 303 * Also, jump optimization will change the breakpoint to 304 * relative-jump. Since the relative-jump itself is 305 * normally used, we just go through if there is no kprobe. 306 */ 307 __addr = recover_probed_instruction(buf, addr); 308 if (!__addr) 309 return false; 310 311 if (insn_decode_kernel(&insn, (void *)__addr) < 0) 312 return false; 313 314#ifdef CONFIG_KGDB 315 /* 316 * If there is a dynamically installed kgdb sw breakpoint, 317 * this function should not be probed. 318 */ 319 if (insn.opcode.bytes[0] == INT3_INSN_OPCODE && 320 kgdb_has_hit_break(addr)) 321 return false; 322#endif 323 addr += insn.length; 324 } 325 326 /* Check if paddr is at an instruction boundary */ 327 if (addr != paddr) 328 return false; 329 330 __addr = recover_probed_instruction(buf, addr); 331 if (!__addr) 332 return false; 333 334 if (insn_decode_kernel(&insn, (void *)__addr) < 0) 335 return false; 336 337 /* INT and UD are special and should not be kprobed */ 338 if (is_exception_insn(&insn)) 339 return false; 340 341 if (IS_ENABLED(CONFIG_CFI)) { 342 /* 343 * The compiler generates the following instruction sequence 344 * for indirect call checks and cfi.c decodes this; 345 * 346 *Â movl -<id>, %r10d ; 6 bytes 347 * addl -4(%reg), %r10d ; 4 bytes 348 * je .Ltmp1 ; 2 bytes 349 * ud2 ; <- regs->ip 350 * .Ltmp1: 351 * 352 * Also, these movl and addl are used for showing expected 353 * type. So those must not be touched. 354 */ 355 if (insn.opcode.value == 0xBA) 356 offset = 12; 357 else if (insn.opcode.value == 0x3) 358 offset = 6; 359 else 360 goto out; 361 362 /* This movl/addl is used for decoding CFI. */ 363 if (is_cfi_trap(addr + offset)) 364 return false; 365 } 366 367out: 368 return true; 369} 370 371/* If x86 supports IBT (ENDBR) it must be skipped. */ 372kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, 373 bool *on_func_entry) 374{ 375 if (is_endbr((u32 *)addr)) { 376 *on_func_entry = !offset || offset == 4; 377 if (*on_func_entry) 378 offset = 4; 379 380 } else { 381 *on_func_entry = !offset; 382 } 383 384 return (kprobe_opcode_t *)(addr + offset); 385} 386 387/* 388 * Copy an instruction with recovering modified instruction by kprobes 389 * and adjust the displacement if the instruction uses the %rip-relative 390 * addressing mode. Note that since @real will be the final place of copied 391 * instruction, displacement must be adjust by @real, not @dest. 392 * This returns the length of copied instruction, or 0 if it has an error. 393 */ 394int __copy_instruction(u8 *dest, u8 *src, u8 *real, struct insn *insn) 395{ 396 kprobe_opcode_t buf[MAX_INSN_SIZE]; 397 unsigned long recovered_insn = recover_probed_instruction(buf, (unsigned long)src); 398 int ret; 399 400 if (!recovered_insn || !insn) 401 return 0; 402 403 /* This can access kernel text if given address is not recovered */ 404 if (copy_from_kernel_nofault(dest, (void *)recovered_insn, 405 MAX_INSN_SIZE)) 406 return 0; 407 408 ret = insn_decode_kernel(insn, dest); 409 if (ret < 0) 410 return 0; 411 412 /* We can not probe force emulate prefixed instruction */ 413 if (insn_has_emulate_prefix(insn)) 414 return 0; 415 416 /* Another subsystem puts a breakpoint, failed to recover */ 417 if (insn->opcode.bytes[0] == INT3_INSN_OPCODE) 418 return 0; 419 420 /* We should not singlestep on the exception masking instructions */ 421 if (insn_masking_exception(insn)) 422 return 0; 423 424#ifdef CONFIG_X86_64 425 /* Only x86_64 has RIP relative instructions */ 426 if (insn_rip_relative(insn)) { 427 s64 newdisp; 428 u8 *disp; 429 /* 430 * The copied instruction uses the %rip-relative addressing 431 * mode. Adjust the displacement for the difference between 432 * the original location of this instruction and the location 433 * of the copy that will actually be run. The tricky bit here 434 * is making sure that the sign extension happens correctly in 435 * this calculation, since we need a signed 32-bit result to 436 * be sign-extended to 64 bits when it's added to the %rip 437 * value and yield the same 64-bit result that the sign- 438 * extension of the original signed 32-bit displacement would 439 * have given. 440 */ 441 newdisp = (u8 *) src + (s64) insn->displacement.value 442 - (u8 *) real; 443 if ((s64) (s32) newdisp != newdisp) { 444 pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp); 445 return 0; 446 } 447 disp = (u8 *) dest + insn_offset_displacement(insn); 448 *(s32 *) disp = (s32) newdisp; 449 } 450#endif 451 return insn->length; 452} 453 454/* Prepare reljump or int3 right after instruction */ 455static int prepare_singlestep(kprobe_opcode_t *buf, struct kprobe *p, 456 struct insn *insn) 457{ 458 int len = insn->length; 459 460 if (!IS_ENABLED(CONFIG_PREEMPTION) && 461 !p->post_handler && can_boost(insn, p->addr) && 462 MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) { 463 /* 464 * These instructions can be executed directly if it 465 * jumps back to correct address. 466 */ 467 synthesize_reljump(buf + len, p->ainsn.insn + len, 468 p->addr + insn->length); 469 len += JMP32_INSN_SIZE; 470 p->ainsn.boostable = 1; 471 } else { 472 /* Otherwise, put an int3 for trapping singlestep */ 473 if (MAX_INSN_SIZE - len < INT3_INSN_SIZE) 474 return -ENOSPC; 475 476 buf[len] = INT3_INSN_OPCODE; 477 len += INT3_INSN_SIZE; 478 } 479 480 return len; 481} 482 483/* Kprobe x86 instruction emulation - only regs->ip or IF flag modifiers */ 484 485static void kprobe_emulate_ifmodifiers(struct kprobe *p, struct pt_regs *regs) 486{ 487 switch (p->ainsn.opcode) { 488 case 0xfa: /* cli */ 489 regs->flags &= ~(X86_EFLAGS_IF); 490 break; 491 case 0xfb: /* sti */ 492 regs->flags |= X86_EFLAGS_IF; 493 break; 494 case 0x9c: /* pushf */ 495 int3_emulate_push(regs, regs->flags); 496 break; 497 case 0x9d: /* popf */ 498 regs->flags = int3_emulate_pop(regs); 499 break; 500 } 501 regs->ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 502} 503NOKPROBE_SYMBOL(kprobe_emulate_ifmodifiers); 504 505static void kprobe_emulate_ret(struct kprobe *p, struct pt_regs *regs) 506{ 507 int3_emulate_ret(regs); 508} 509NOKPROBE_SYMBOL(kprobe_emulate_ret); 510 511static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs) 512{ 513 unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 514 515 func += p->ainsn.rel32; 516 int3_emulate_call(regs, func); 517} 518NOKPROBE_SYMBOL(kprobe_emulate_call); 519 520static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs) 521{ 522 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 523 524 ip += p->ainsn.rel32; 525 int3_emulate_jmp(regs, ip); 526} 527NOKPROBE_SYMBOL(kprobe_emulate_jmp); 528 529static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs) 530{ 531 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 532 533 int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32); 534} 535NOKPROBE_SYMBOL(kprobe_emulate_jcc); 536 537static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs) 538{ 539 unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size; 540 bool match; 541 542 if (p->ainsn.loop.type != 3) { /* LOOP* */ 543 if (p->ainsn.loop.asize == 32) 544 match = ((*(u32 *)®s->cx)--) != 0; 545#ifdef CONFIG_X86_64 546 else if (p->ainsn.loop.asize == 64) 547 match = ((*(u64 *)®s->cx)--) != 0; 548#endif 549 else 550 match = ((*(u16 *)®s->cx)--) != 0; 551 } else { /* JCXZ */ 552 if (p->ainsn.loop.asize == 32) 553 match = *(u32 *)(®s->cx) == 0; 554#ifdef CONFIG_X86_64 555 else if (p->ainsn.loop.asize == 64) 556 match = *(u64 *)(®s->cx) == 0; 557#endif 558 else 559 match = *(u16 *)(®s->cx) == 0; 560 } 561 562 if (p->ainsn.loop.type == 0) /* LOOPNE */ 563 match = match && !(regs->flags & X86_EFLAGS_ZF); 564 else if (p->ainsn.loop.type == 1) /* LOOPE */ 565 match = match && (regs->flags & X86_EFLAGS_ZF); 566 567 if (match) 568 ip += p->ainsn.rel32; 569 int3_emulate_jmp(regs, ip); 570} 571NOKPROBE_SYMBOL(kprobe_emulate_loop); 572 573static const int addrmode_regoffs[] = { 574 offsetof(struct pt_regs, ax), 575 offsetof(struct pt_regs, cx), 576 offsetof(struct pt_regs, dx), 577 offsetof(struct pt_regs, bx), 578 offsetof(struct pt_regs, sp), 579 offsetof(struct pt_regs, bp), 580 offsetof(struct pt_regs, si), 581 offsetof(struct pt_regs, di), 582#ifdef CONFIG_X86_64 583 offsetof(struct pt_regs, r8), 584 offsetof(struct pt_regs, r9), 585 offsetof(struct pt_regs, r10), 586 offsetof(struct pt_regs, r11), 587 offsetof(struct pt_regs, r12), 588 offsetof(struct pt_regs, r13), 589 offsetof(struct pt_regs, r14), 590 offsetof(struct pt_regs, r15), 591#endif 592}; 593 594static void kprobe_emulate_call_indirect(struct kprobe *p, struct pt_regs *regs) 595{ 596 unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg]; 597 598 int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + p->ainsn.size); 599 int3_emulate_jmp(regs, regs_get_register(regs, offs)); 600} 601NOKPROBE_SYMBOL(kprobe_emulate_call_indirect); 602 603static void kprobe_emulate_jmp_indirect(struct kprobe *p, struct pt_regs *regs) 604{ 605 unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg]; 606 607 int3_emulate_jmp(regs, regs_get_register(regs, offs)); 608} 609NOKPROBE_SYMBOL(kprobe_emulate_jmp_indirect); 610 611static int prepare_emulation(struct kprobe *p, struct insn *insn) 612{ 613 insn_byte_t opcode = insn->opcode.bytes[0]; 614 615 switch (opcode) { 616 case 0xfa: /* cli */ 617 case 0xfb: /* sti */ 618 case 0x9c: /* pushfl */ 619 case 0x9d: /* popf/popfd */ 620 /* 621 * IF modifiers must be emulated since it will enable interrupt while 622 * int3 single stepping. 623 */ 624 p->ainsn.emulate_op = kprobe_emulate_ifmodifiers; 625 p->ainsn.opcode = opcode; 626 break; 627 case 0xc2: /* ret/lret */ 628 case 0xc3: 629 case 0xca: 630 case 0xcb: 631 p->ainsn.emulate_op = kprobe_emulate_ret; 632 break; 633 case 0x9a: /* far call absolute -- segment is not supported */ 634 case 0xea: /* far jmp absolute -- segment is not supported */ 635 case 0xcc: /* int3 */ 636 case 0xcf: /* iret -- in-kernel IRET is not supported */ 637 return -EOPNOTSUPP; 638 break; 639 case 0xe8: /* near call relative */ 640 p->ainsn.emulate_op = kprobe_emulate_call; 641 if (insn->immediate.nbytes == 2) 642 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 643 else 644 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 645 break; 646 case 0xeb: /* short jump relative */ 647 case 0xe9: /* near jump relative */ 648 p->ainsn.emulate_op = kprobe_emulate_jmp; 649 if (insn->immediate.nbytes == 1) 650 p->ainsn.rel32 = *(s8 *)&insn->immediate.value; 651 else if (insn->immediate.nbytes == 2) 652 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 653 else 654 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 655 break; 656 case 0x70 ... 0x7f: 657 /* 1 byte conditional jump */ 658 p->ainsn.emulate_op = kprobe_emulate_jcc; 659 p->ainsn.jcc.type = opcode & 0xf; 660 p->ainsn.rel32 = insn->immediate.value; 661 break; 662 case 0x0f: 663 opcode = insn->opcode.bytes[1]; 664 if ((opcode & 0xf0) == 0x80) { 665 /* 2 bytes Conditional Jump */ 666 p->ainsn.emulate_op = kprobe_emulate_jcc; 667 p->ainsn.jcc.type = opcode & 0xf; 668 if (insn->immediate.nbytes == 2) 669 p->ainsn.rel32 = *(s16 *)&insn->immediate.value; 670 else 671 p->ainsn.rel32 = *(s32 *)&insn->immediate.value; 672 } else if (opcode == 0x01 && 673 X86_MODRM_REG(insn->modrm.bytes[0]) == 0 && 674 X86_MODRM_MOD(insn->modrm.bytes[0]) == 3) { 675 /* VM extensions - not supported */ 676 return -EOPNOTSUPP; 677 } 678 break; 679 case 0xe0: /* Loop NZ */ 680 case 0xe1: /* Loop */ 681 case 0xe2: /* Loop */ 682 case 0xe3: /* J*CXZ */ 683 p->ainsn.emulate_op = kprobe_emulate_loop; 684 p->ainsn.loop.type = opcode & 0x3; 685 p->ainsn.loop.asize = insn->addr_bytes * 8; 686 p->ainsn.rel32 = *(s8 *)&insn->immediate.value; 687 break; 688 case 0xff: 689 /* 690 * Since the 0xff is an extended group opcode, the instruction 691 * is determined by the MOD/RM byte. 692 */ 693 opcode = insn->modrm.bytes[0]; 694 switch (X86_MODRM_REG(opcode)) { 695 case 0b010: /* FF /2, call near, absolute indirect */ 696 p->ainsn.emulate_op = kprobe_emulate_call_indirect; 697 break; 698 case 0b100: /* FF /4, jmp near, absolute indirect */ 699 p->ainsn.emulate_op = kprobe_emulate_jmp_indirect; 700 break; 701 case 0b011: /* FF /3, call far, absolute indirect */ 702 case 0b101: /* FF /5, jmp far, absolute indirect */ 703 return -EOPNOTSUPP; 704 } 705 706 if (!p->ainsn.emulate_op) 707 break; 708 709 if (insn->addr_bytes != sizeof(unsigned long)) 710 return -EOPNOTSUPP; /* Don't support different size */ 711 if (X86_MODRM_MOD(opcode) != 3) 712 return -EOPNOTSUPP; /* TODO: support memory addressing */ 713 714 p->ainsn.indirect.reg = X86_MODRM_RM(opcode); 715#ifdef CONFIG_X86_64 716 if (X86_REX_B(insn->rex_prefix.value)) 717 p->ainsn.indirect.reg += 8; 718#endif 719 break; 720 default: 721 break; 722 } 723 p->ainsn.size = insn->length; 724 725 return 0; 726} 727 728static int arch_copy_kprobe(struct kprobe *p) 729{ 730 struct insn insn; 731 kprobe_opcode_t buf[MAX_INSN_SIZE]; 732 int ret, len; 733 734 /* Copy an instruction with recovering if other optprobe modifies it.*/ 735 len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn); 736 if (!len) 737 return -EINVAL; 738 739 /* Analyze the opcode and setup emulate functions */ 740 ret = prepare_emulation(p, &insn); 741 if (ret < 0) 742 return ret; 743 744 /* Add int3 for single-step or booster jmp */ 745 len = prepare_singlestep(buf, p, &insn); 746 if (len < 0) 747 return len; 748 749 /* Also, displacement change doesn't affect the first byte */ 750 p->opcode = buf[0]; 751 752 p->ainsn.tp_len = len; 753 perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len); 754 755 /* OK, write back the instruction(s) into ROX insn buffer */ 756 text_poke(p->ainsn.insn, buf, len); 757 758 return 0; 759} 760 761int arch_prepare_kprobe(struct kprobe *p) 762{ 763 int ret; 764 765 if (alternatives_text_reserved(p->addr, p->addr)) 766 return -EINVAL; 767 768 if (!can_probe((unsigned long)p->addr)) 769 return -EILSEQ; 770 771 memset(&p->ainsn, 0, sizeof(p->ainsn)); 772 773 /* insn: must be on special executable page on x86. */ 774 p->ainsn.insn = get_insn_slot(); 775 if (!p->ainsn.insn) 776 return -ENOMEM; 777 778 ret = arch_copy_kprobe(p); 779 if (ret) { 780 free_insn_slot(p->ainsn.insn, 0); 781 p->ainsn.insn = NULL; 782 } 783 784 return ret; 785} 786 787void arch_arm_kprobe(struct kprobe *p) 788{ 789 u8 int3 = INT3_INSN_OPCODE; 790 791 text_poke(p->addr, &int3, 1); 792 smp_text_poke_sync_each_cpu(); 793 perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1); 794} 795 796void arch_disarm_kprobe(struct kprobe *p) 797{ 798 u8 int3 = INT3_INSN_OPCODE; 799 800 perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1); 801 text_poke(p->addr, &p->opcode, 1); 802 smp_text_poke_sync_each_cpu(); 803} 804 805void arch_remove_kprobe(struct kprobe *p) 806{ 807 if (p->ainsn.insn) { 808 /* Record the perf event before freeing the slot */ 809 perf_event_text_poke(p->ainsn.insn, p->ainsn.insn, 810 p->ainsn.tp_len, NULL, 0); 811 free_insn_slot(p->ainsn.insn, p->ainsn.boostable); 812 p->ainsn.insn = NULL; 813 } 814} 815 816static nokprobe_inline void 817save_previous_kprobe(struct kprobe_ctlblk *kcb) 818{ 819 kcb->prev_kprobe.kp = kprobe_running(); 820 kcb->prev_kprobe.status = kcb->kprobe_status; 821 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags; 822 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; 823} 824 825static nokprobe_inline void 826restore_previous_kprobe(struct kprobe_ctlblk *kcb) 827{ 828 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 829 kcb->kprobe_status = kcb->prev_kprobe.status; 830 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags; 831 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; 832} 833 834static nokprobe_inline void 835set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 836 struct kprobe_ctlblk *kcb) 837{ 838 __this_cpu_write(current_kprobe, p); 839 kcb->kprobe_saved_flags = kcb->kprobe_old_flags 840 = (regs->flags & X86_EFLAGS_IF); 841} 842 843static void kprobe_post_process(struct kprobe *cur, struct pt_regs *regs, 844 struct kprobe_ctlblk *kcb) 845{ 846 /* Restore back the original saved kprobes variables and continue. */ 847 if (kcb->kprobe_status == KPROBE_REENTER) { 848 /* This will restore both kcb and current_kprobe */ 849 restore_previous_kprobe(kcb); 850 } else { 851 /* 852 * Always update the kcb status because 853 * reset_curent_kprobe() doesn't update kcb. 854 */ 855 kcb->kprobe_status = KPROBE_HIT_SSDONE; 856 if (cur->post_handler) 857 cur->post_handler(cur, regs, 0); 858 reset_current_kprobe(); 859 } 860} 861NOKPROBE_SYMBOL(kprobe_post_process); 862 863static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, 864 struct kprobe_ctlblk *kcb, int reenter) 865{ 866 if (setup_detour_execution(p, regs, reenter)) 867 return; 868 869#if !defined(CONFIG_PREEMPTION) 870 if (p->ainsn.boostable) { 871 /* Boost up -- we can execute copied instructions directly */ 872 if (!reenter) 873 reset_current_kprobe(); 874 /* 875 * Reentering boosted probe doesn't reset current_kprobe, 876 * nor set current_kprobe, because it doesn't use single 877 * stepping. 878 */ 879 regs->ip = (unsigned long)p->ainsn.insn; 880 return; 881 } 882#endif 883 if (reenter) { 884 save_previous_kprobe(kcb); 885 set_current_kprobe(p, regs, kcb); 886 kcb->kprobe_status = KPROBE_REENTER; 887 } else 888 kcb->kprobe_status = KPROBE_HIT_SS; 889 890 if (p->ainsn.emulate_op) { 891 p->ainsn.emulate_op(p, regs); 892 kprobe_post_process(p, regs, kcb); 893 return; 894 } 895 896 /* Disable interrupt, and set ip register on trampoline */ 897 regs->flags &= ~X86_EFLAGS_IF; 898 regs->ip = (unsigned long)p->ainsn.insn; 899} 900NOKPROBE_SYMBOL(setup_singlestep); 901 902/* 903 * Called after single-stepping. p->addr is the address of the 904 * instruction whose first byte has been replaced by the "int3" 905 * instruction. To avoid the SMP problems that can occur when we 906 * temporarily put back the original opcode to single-step, we 907 * single-stepped a copy of the instruction. The address of this 908 * copy is p->ainsn.insn. We also doesn't use trap, but "int3" again 909 * right after the copied instruction. 910 * Different from the trap single-step, "int3" single-step can not 911 * handle the instruction which changes the ip register, e.g. jmp, 912 * call, conditional jmp, and the instructions which changes the IF 913 * flags because interrupt must be disabled around the single-stepping. 914 * Such instructions are software emulated, but others are single-stepped 915 * using "int3". 916 * 917 * When the 2nd "int3" handled, the regs->ip and regs->flags needs to 918 * be adjusted, so that we can resume execution on correct code. 919 */ 920static void resume_singlestep(struct kprobe *p, struct pt_regs *regs, 921 struct kprobe_ctlblk *kcb) 922{ 923 unsigned long copy_ip = (unsigned long)p->ainsn.insn; 924 unsigned long orig_ip = (unsigned long)p->addr; 925 926 /* Restore saved interrupt flag and ip register */ 927 regs->flags |= kcb->kprobe_saved_flags; 928 /* Note that regs->ip is executed int3 so must be a step back */ 929 regs->ip += (orig_ip - copy_ip) - INT3_INSN_SIZE; 930} 931NOKPROBE_SYMBOL(resume_singlestep); 932 933/* 934 * We have reentered the kprobe_handler(), since another probe was hit while 935 * within the handler. We save the original kprobes variables and just single 936 * step on the instruction of the new probe without calling any user handlers. 937 */ 938static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, 939 struct kprobe_ctlblk *kcb) 940{ 941 switch (kcb->kprobe_status) { 942 case KPROBE_HIT_SSDONE: 943 case KPROBE_HIT_ACTIVE: 944 case KPROBE_HIT_SS: 945 kprobes_inc_nmissed_count(p); 946 setup_singlestep(p, regs, kcb, 1); 947 break; 948 case KPROBE_REENTER: 949 /* A probe has been hit in the codepath leading up to, or just 950 * after, single-stepping of a probed instruction. This entire 951 * codepath should strictly reside in .kprobes.text section. 952 * Raise a BUG or we'll continue in an endless reentering loop 953 * and eventually a stack overflow. 954 */ 955 pr_err("Unrecoverable kprobe detected.\n"); 956 dump_kprobe(p); 957 BUG(); 958 default: 959 /* impossible cases */ 960 WARN_ON(1); 961 return 0; 962 } 963 964 return 1; 965} 966NOKPROBE_SYMBOL(reenter_kprobe); 967 968static nokprobe_inline int kprobe_is_ss(struct kprobe_ctlblk *kcb) 969{ 970 return (kcb->kprobe_status == KPROBE_HIT_SS || 971 kcb->kprobe_status == KPROBE_REENTER); 972} 973 974/* 975 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 976 * remain disabled throughout this function. 977 */ 978int kprobe_int3_handler(struct pt_regs *regs) 979{ 980 kprobe_opcode_t *addr; 981 struct kprobe *p; 982 struct kprobe_ctlblk *kcb; 983 984 if (user_mode(regs)) 985 return 0; 986 987 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t)); 988 /* 989 * We don't want to be preempted for the entire duration of kprobe 990 * processing. Since int3 and debug trap disables irqs and we clear 991 * IF while singlestepping, it must be no preemptible. 992 */ 993 994 kcb = get_kprobe_ctlblk(); 995 p = get_kprobe(addr); 996 997 if (p) { 998 if (kprobe_running()) { 999 if (reenter_kprobe(p, regs, kcb)) 1000 return 1; 1001 } else { 1002 set_current_kprobe(p, regs, kcb); 1003 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 1004 1005 /* 1006 * If we have no pre-handler or it returned 0, we 1007 * continue with normal processing. If we have a 1008 * pre-handler and it returned non-zero, that means 1009 * user handler setup registers to exit to another 1010 * instruction, we must skip the single stepping. 1011 */ 1012 if (!p->pre_handler || !p->pre_handler(p, regs)) 1013 setup_singlestep(p, regs, kcb, 0); 1014 else 1015 reset_current_kprobe(); 1016 return 1; 1017 } 1018 } else if (kprobe_is_ss(kcb)) { 1019 p = kprobe_running(); 1020 if ((unsigned long)p->ainsn.insn < regs->ip && 1021 (unsigned long)p->ainsn.insn + MAX_INSN_SIZE > regs->ip) { 1022 /* Most provably this is the second int3 for singlestep */ 1023 resume_singlestep(p, regs, kcb); 1024 kprobe_post_process(p, regs, kcb); 1025 return 1; 1026 } 1027 } /* else: not a kprobe fault; let the kernel handle it */ 1028 1029 return 0; 1030} 1031NOKPROBE_SYMBOL(kprobe_int3_handler); 1032 1033int kprobe_fault_handler(struct pt_regs *regs, int trapnr) 1034{ 1035 struct kprobe *cur = kprobe_running(); 1036 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1037 1038 if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) { 1039 /* This must happen on single-stepping */ 1040 WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS && 1041 kcb->kprobe_status != KPROBE_REENTER); 1042 /* 1043 * We are here because the instruction being single 1044 * stepped caused a page fault. We reset the current 1045 * kprobe and the ip points back to the probe address 1046 * and allow the page fault handler to continue as a 1047 * normal page fault. 1048 */ 1049 regs->ip = (unsigned long)cur->addr; 1050 1051 /* 1052 * If the IF flag was set before the kprobe hit, 1053 * don't touch it: 1054 */ 1055 regs->flags |= kcb->kprobe_old_flags; 1056 1057 if (kcb->kprobe_status == KPROBE_REENTER) 1058 restore_previous_kprobe(kcb); 1059 else 1060 reset_current_kprobe(); 1061 } 1062 1063 return 0; 1064} 1065NOKPROBE_SYMBOL(kprobe_fault_handler); 1066 1067int __init arch_populate_kprobe_blacklist(void) 1068{ 1069 return kprobe_add_area_blacklist((unsigned long)__entry_text_start, 1070 (unsigned long)__entry_text_end); 1071} 1072 1073int __init arch_init_kprobes(void) 1074{ 1075 return 0; 1076} 1077 1078int arch_trampoline_kprobe(struct kprobe *p) 1079{ 1080 return 0; 1081}
~/
Read-only file preview.