FILE
check.c
File preview with syntax highlighting.
linux-6.19.14/tools/objtool/check.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com> 4 */ 5 6#define _GNU_SOURCE /* memmem() */ 7#include <fnmatch.h> 8#include <string.h> 9#include <stdlib.h> 10#include <inttypes.h> 11#include <sys/mman.h> 12 13#include <objtool/builtin.h> 14#include <objtool/cfi.h> 15#include <objtool/arch.h> 16#include <objtool/disas.h> 17#include <objtool/check.h> 18#include <objtool/special.h> 19#include <objtool/trace.h> 20#include <objtool/warn.h> 21#include <objtool/checksum.h> 22#include <objtool/util.h> 23 24#include <linux/objtool_types.h> 25#include <linux/hashtable.h> 26#include <linux/kernel.h> 27#include <linux/static_call_types.h> 28#include <linux/string.h> 29 30static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache; 31 32static struct cfi_init_state initial_func_cfi; 33static struct cfi_state init_cfi; 34static struct cfi_state func_cfi; 35static struct cfi_state force_undefined_cfi; 36 37struct disas_context *objtool_disas_ctx; 38 39size_t sym_name_max_len; 40 41struct instruction *find_insn(struct objtool_file *file, 42 struct section *sec, unsigned long offset) 43{ 44 struct instruction *insn; 45 46 hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) { 47 if (insn->sec == sec && insn->offset == offset) 48 return insn; 49 } 50 51 return NULL; 52} 53 54struct instruction *next_insn_same_sec(struct objtool_file *file, 55 struct instruction *insn) 56{ 57 if (insn->idx == INSN_CHUNK_MAX) 58 return find_insn(file, insn->sec, insn->offset + insn->len); 59 60 insn++; 61 if (!insn->len) 62 return NULL; 63 64 return insn; 65} 66 67static struct instruction *next_insn_same_func(struct objtool_file *file, 68 struct instruction *insn) 69{ 70 struct instruction *next = next_insn_same_sec(file, insn); 71 struct symbol *func = insn_func(insn); 72 73 if (!func) 74 return NULL; 75 76 if (next && insn_func(next) == func) 77 return next; 78 79 /* Check if we're already in the subfunction: */ 80 if (func == func->cfunc) 81 return NULL; 82 83 /* Move to the subfunction: */ 84 return find_insn(file, func->cfunc->sec, func->cfunc->offset); 85} 86 87static struct instruction *prev_insn_same_sec(struct objtool_file *file, 88 struct instruction *insn) 89{ 90 if (insn->idx == 0) { 91 if (insn->prev_len) 92 return find_insn(file, insn->sec, insn->offset - insn->prev_len); 93 return NULL; 94 } 95 96 return insn - 1; 97} 98 99static struct instruction *prev_insn_same_sym(struct objtool_file *file, 100 struct instruction *insn) 101{ 102 struct instruction *prev = prev_insn_same_sec(file, insn); 103 104 if (prev && insn_func(prev) == insn_func(insn)) 105 return prev; 106 107 return NULL; 108} 109 110#define for_each_insn(file, insn) \ 111 for (struct section *__sec, *__fake = (struct section *)1; \ 112 __fake; __fake = NULL) \ 113 for_each_sec(file->elf, __sec) \ 114 sec_for_each_insn(file, __sec, insn) 115 116#define func_for_each_insn(file, func, insn) \ 117 for (insn = find_insn(file, func->sec, func->offset); \ 118 insn; \ 119 insn = next_insn_same_func(file, insn)) 120 121#define sym_for_each_insn(file, sym, insn) \ 122 for (insn = find_insn(file, sym->sec, sym->offset); \ 123 insn && insn->offset < sym->offset + sym->len; \ 124 insn = next_insn_same_sec(file, insn)) 125 126#define sym_for_each_insn_continue_reverse(file, sym, insn) \ 127 for (insn = prev_insn_same_sec(file, insn); \ 128 insn && insn->offset >= sym->offset; \ 129 insn = prev_insn_same_sec(file, insn)) 130 131#define sec_for_each_insn_from(file, insn) \ 132 for (; insn; insn = next_insn_same_sec(file, insn)) 133 134#define sec_for_each_insn_continue(file, insn) \ 135 for (insn = next_insn_same_sec(file, insn); insn; \ 136 insn = next_insn_same_sec(file, insn)) 137 138static inline struct reloc *insn_jump_table(struct instruction *insn) 139{ 140 if (insn->type == INSN_JUMP_DYNAMIC || 141 insn->type == INSN_CALL_DYNAMIC) 142 return insn->_jump_table; 143 144 return NULL; 145} 146 147static inline unsigned long insn_jump_table_size(struct instruction *insn) 148{ 149 if (insn->type == INSN_JUMP_DYNAMIC || 150 insn->type == INSN_CALL_DYNAMIC) 151 return insn->_jump_table_size; 152 153 return 0; 154} 155 156static bool is_jump_table_jump(struct instruction *insn) 157{ 158 struct alt_group *alt_group = insn->alt_group; 159 160 if (insn_jump_table(insn)) 161 return true; 162 163 /* Retpoline alternative for a jump table? */ 164 return alt_group && alt_group->orig_group && 165 insn_jump_table(alt_group->orig_group->first_insn); 166} 167 168static bool is_sibling_call(struct instruction *insn) 169{ 170 /* 171 * Assume only STT_FUNC calls have jump-tables. 172 */ 173 if (insn_func(insn)) { 174 /* An indirect jump is either a sibling call or a jump to a table. */ 175 if (insn->type == INSN_JUMP_DYNAMIC) 176 return !is_jump_table_jump(insn); 177 } 178 179 /* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */ 180 return (is_static_jump(insn) && insn_call_dest(insn)); 181} 182 183/* 184 * Checks if a function is a Rust "noreturn" one. 185 */ 186static bool is_rust_noreturn(const struct symbol *func) 187{ 188 /* 189 * If it does not start with "_R", then it is not a Rust symbol. 190 */ 191 if (strncmp(func->name, "_R", 2)) 192 return false; 193 194 /* 195 * These are just heuristics -- we do not control the precise symbol 196 * name, due to the crate disambiguators (which depend on the compiler) 197 * as well as changes to the source code itself between versions (since 198 * these come from the Rust standard library). 199 */ 200 return str_ends_with(func->name, "_4core3num20from_str_radix_panic") || 201 str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || 202 str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || 203 str_ends_with(func->name, "_4core6option13expect_failed") || 204 str_ends_with(func->name, "_4core6option13unwrap_failed") || 205 str_ends_with(func->name, "_4core6result13unwrap_failed") || 206 str_ends_with(func->name, "_4core9panicking5panic") || 207 str_ends_with(func->name, "_4core9panicking9panic_fmt") || 208 str_ends_with(func->name, "_4core9panicking14panic_explicit") || 209 str_ends_with(func->name, "_4core9panicking14panic_nounwind") || 210 str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || 211 str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || 212 str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || 213 str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || 214 str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || 215 str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || 216 strstr(func->name, "_4core9panicking13assert_failed") || 217 strstr(func->name, "_4core9panicking11panic_const24panic_const_") || 218 (strstr(func->name, "_4core5slice5index") && 219 strstr(func->name, "slice_") && 220 str_ends_with(func->name, "_fail")); 221} 222 223/* 224 * This checks to see if the given function is a "noreturn" function. 225 * 226 * For global functions which are outside the scope of this object file, we 227 * have to keep a manual list of them. 228 * 229 * For local functions, we have to detect them manually by simply looking for 230 * the lack of a return instruction. 231 */ 232static bool __dead_end_function(struct objtool_file *file, struct symbol *func, 233 int recursion) 234{ 235 int i; 236 struct instruction *insn; 237 bool empty = true; 238 239#define NORETURN(func) __stringify(func), 240 static const char * const global_noreturns[] = { 241#include "noreturns.h" 242 }; 243#undef NORETURN 244 245 if (!func) 246 return false; 247 248 if (!is_local_sym(func)) { 249 if (is_rust_noreturn(func)) 250 return true; 251 252 for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) 253 if (!strcmp(func->name, global_noreturns[i])) 254 return true; 255 } 256 257 if (is_weak_sym(func)) 258 return false; 259 260 if (!func->len) 261 return false; 262 263 insn = find_insn(file, func->sec, func->offset); 264 if (!insn || !insn_func(insn)) 265 return false; 266 267 func_for_each_insn(file, func, insn) { 268 empty = false; 269 270 if (insn->type == INSN_RETURN) 271 return false; 272 } 273 274 if (empty) 275 return false; 276 277 /* 278 * A function can have a sibling call instead of a return. In that 279 * case, the function's dead-end status depends on whether the target 280 * of the sibling call returns. 281 */ 282 func_for_each_insn(file, func, insn) { 283 if (is_sibling_call(insn)) { 284 struct instruction *dest = insn->jump_dest; 285 286 if (!dest) 287 /* sibling call to another file */ 288 return false; 289 290 /* local sibling call */ 291 if (recursion == 5) { 292 /* 293 * Infinite recursion: two functions have 294 * sibling calls to each other. This is a very 295 * rare case. It means they aren't dead ends. 296 */ 297 return false; 298 } 299 300 return __dead_end_function(file, insn_func(dest), recursion+1); 301 } 302 } 303 304 return true; 305} 306 307static bool dead_end_function(struct objtool_file *file, struct symbol *func) 308{ 309 return __dead_end_function(file, func, 0); 310} 311 312static void init_cfi_state(struct cfi_state *cfi) 313{ 314 int i; 315 316 for (i = 0; i < CFI_NUM_REGS; i++) { 317 cfi->regs[i].base = CFI_UNDEFINED; 318 cfi->vals[i].base = CFI_UNDEFINED; 319 } 320 cfi->cfa.base = CFI_UNDEFINED; 321 cfi->drap_reg = CFI_UNDEFINED; 322 cfi->drap_offset = -1; 323} 324 325static void init_insn_state(struct objtool_file *file, struct insn_state *state, 326 struct section *sec) 327{ 328 memset(state, 0, sizeof(*state)); 329 init_cfi_state(&state->cfi); 330 331 if (opts.noinstr && sec) 332 state->noinstr = sec->noinstr; 333} 334 335static struct cfi_state *cfi_alloc(void) 336{ 337 struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state)); 338 if (!cfi) { 339 ERROR_GLIBC("calloc"); 340 exit(1); 341 } 342 nr_cfi++; 343 return cfi; 344} 345 346static int cfi_bits; 347static struct hlist_head *cfi_hash; 348 349static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2) 350{ 351 return memcmp((void *)cfi1 + sizeof(cfi1->hash), 352 (void *)cfi2 + sizeof(cfi2->hash), 353 sizeof(struct cfi_state) - sizeof(struct hlist_node)); 354} 355 356static inline u32 cfi_key(struct cfi_state *cfi) 357{ 358 return jhash((void *)cfi + sizeof(cfi->hash), 359 sizeof(*cfi) - sizeof(cfi->hash), 0); 360} 361 362static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi) 363{ 364 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 365 struct cfi_state *obj; 366 367 hlist_for_each_entry(obj, head, hash) { 368 if (!cficmp(cfi, obj)) { 369 nr_cfi_cache++; 370 return obj; 371 } 372 } 373 374 obj = cfi_alloc(); 375 *obj = *cfi; 376 hlist_add_head(&obj->hash, head); 377 378 return obj; 379} 380 381static void cfi_hash_add(struct cfi_state *cfi) 382{ 383 struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)]; 384 385 hlist_add_head(&cfi->hash, head); 386} 387 388static void *cfi_hash_alloc(unsigned long size) 389{ 390 cfi_bits = max(10, ilog2(size)); 391 cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits, 392 PROT_READ|PROT_WRITE, 393 MAP_PRIVATE|MAP_ANON, -1, 0); 394 if (cfi_hash == (void *)-1L) { 395 ERROR_GLIBC("mmap fail cfi_hash"); 396 cfi_hash = NULL; 397 } else if (opts.stats) { 398 printf("cfi_bits: %d\n", cfi_bits); 399 } 400 401 return cfi_hash; 402} 403 404static unsigned long nr_insns; 405static unsigned long nr_insns_visited; 406 407/* 408 * Call the arch-specific instruction decoder for all the instructions and add 409 * them to the global instruction list. 410 */ 411static int decode_instructions(struct objtool_file *file) 412{ 413 struct section *sec; 414 struct symbol *func; 415 unsigned long offset; 416 struct instruction *insn; 417 418 for_each_sec(file->elf, sec) { 419 struct instruction *insns = NULL; 420 u8 prev_len = 0; 421 u8 idx = 0; 422 423 if (!is_text_sec(sec)) 424 continue; 425 426 if (strcmp(sec->name, ".altinstr_replacement") && 427 strcmp(sec->name, ".altinstr_aux") && 428 strncmp(sec->name, ".discard.", 9)) 429 sec->text = true; 430 431 if (!strcmp(sec->name, ".noinstr.text") || 432 !strcmp(sec->name, ".entry.text") || 433 !strcmp(sec->name, ".cpuidle.text") || 434 !strncmp(sec->name, ".text..__x86.", 13)) 435 sec->noinstr = true; 436 437 /* 438 * .init.text code is ran before userspace and thus doesn't 439 * strictly need retpolines, except for modules which are 440 * loaded late, they very much do need retpoline in their 441 * .init.text 442 */ 443 if (!strcmp(sec->name, ".init.text") && !opts.module) 444 sec->init = true; 445 446 for (offset = 0; offset < sec_size(sec); offset += insn->len) { 447 if (!insns || idx == INSN_CHUNK_MAX) { 448 insns = calloc(INSN_CHUNK_SIZE, sizeof(*insn)); 449 if (!insns) { 450 ERROR_GLIBC("calloc"); 451 return -1; 452 } 453 idx = 0; 454 } else { 455 idx++; 456 } 457 insn = &insns[idx]; 458 insn->idx = idx; 459 460 INIT_LIST_HEAD(&insn->call_node); 461 insn->sec = sec; 462 insn->offset = offset; 463 insn->prev_len = prev_len; 464 465 if (arch_decode_instruction(file, sec, offset, sec_size(sec) - offset, insn)) 466 return -1; 467 468 prev_len = insn->len; 469 470 /* 471 * By default, "ud2" is a dead end unless otherwise 472 * annotated, because GCC 7 inserts it for certain 473 * divide-by-zero cases. 474 */ 475 if (insn->type == INSN_BUG) 476 insn->dead_end = true; 477 478 hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset)); 479 nr_insns++; 480 } 481 482 sec_for_each_sym(sec, func) { 483 if (!is_notype_sym(func) && !is_func_sym(func)) 484 continue; 485 486 if (func->offset == sec_size(sec)) { 487 /* Heuristic: likely an "end" symbol */ 488 if (is_notype_sym(func)) 489 continue; 490 ERROR("%s(): STT_FUNC at end of section", func->name); 491 return -1; 492 } 493 494 if (func->embedded_insn || func->alias != func) 495 continue; 496 497 if (!find_insn(file, sec, func->offset)) { 498 ERROR("%s(): can't find starting instruction", func->name); 499 return -1; 500 } 501 502 sym_for_each_insn(file, func, insn) { 503 insn->sym = func; 504 if (is_func_sym(func) && 505 insn->type == INSN_ENDBR && 506 list_empty(&insn->call_node)) { 507 if (insn->offset == func->offset) { 508 list_add_tail(&insn->call_node, &file->endbr_list); 509 file->nr_endbr++; 510 } else { 511 file->nr_endbr_int++; 512 } 513 } 514 } 515 } 516 } 517 518 if (opts.stats) 519 printf("nr_insns: %lu\n", nr_insns); 520 521 return 0; 522} 523 524/* 525 * Read the pv_ops[] .data table to find the static initialized values. 526 */ 527static int add_pv_ops(struct objtool_file *file, const char *symname) 528{ 529 struct symbol *sym, *func; 530 unsigned long off, end; 531 struct reloc *reloc; 532 int idx; 533 534 sym = find_symbol_by_name(file->elf, symname); 535 if (!sym) 536 return 0; 537 538 off = sym->offset; 539 end = off + sym->len; 540 for (;;) { 541 reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off); 542 if (!reloc) 543 break; 544 545 idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long); 546 547 func = reloc->sym; 548 if (is_sec_sym(func)) 549 func = find_symbol_by_offset(reloc->sym->sec, 550 reloc_addend(reloc)); 551 if (!func) { 552 ERROR_FUNC(reloc->sym->sec, reloc_addend(reloc), 553 "can't find func at %s[%d]", symname, idx); 554 return -1; 555 } 556 557 if (objtool_pv_add(file, idx, func)) 558 return -1; 559 560 off = reloc_offset(reloc) + 1; 561 if (off > end) 562 break; 563 } 564 565 return 0; 566} 567 568/* 569 * Allocate and initialize file->pv_ops[]. 570 */ 571static int init_pv_ops(struct objtool_file *file) 572{ 573 static const char *pv_ops_tables[] = { 574 "pv_ops", 575 "xen_cpu_ops", 576 "xen_irq_ops", 577 "xen_mmu_ops", 578 NULL, 579 }; 580 const char *pv_ops; 581 struct symbol *sym; 582 int idx, nr; 583 584 if (!opts.noinstr) 585 return 0; 586 587 file->pv_ops = NULL; 588 589 sym = find_symbol_by_name(file->elf, "pv_ops"); 590 if (!sym) 591 return 0; 592 593 nr = sym->len / sizeof(unsigned long); 594 file->pv_ops = calloc(nr, sizeof(struct pv_state)); 595 if (!file->pv_ops) { 596 ERROR_GLIBC("calloc"); 597 return -1; 598 } 599 600 for (idx = 0; idx < nr; idx++) 601 INIT_LIST_HEAD(&file->pv_ops[idx].targets); 602 603 for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++) { 604 if (add_pv_ops(file, pv_ops)) 605 return -1; 606 } 607 608 return 0; 609} 610 611static bool is_livepatch_module(struct objtool_file *file) 612{ 613 struct section *sec; 614 615 if (!opts.module) 616 return false; 617 618 sec = find_section_by_name(file->elf, ".modinfo"); 619 if (!sec) 620 return false; 621 622 return memmem(sec->data->d_buf, sec_size(sec), "\0livepatch=Y", 12); 623} 624 625static int create_static_call_sections(struct objtool_file *file) 626{ 627 struct static_call_site *site; 628 struct section *sec; 629 struct instruction *insn; 630 struct symbol *key_sym; 631 char *key_name, *tmp; 632 int idx; 633 634 sec = find_section_by_name(file->elf, ".static_call_sites"); 635 if (sec) { 636 /* 637 * Livepatch modules may have already extracted the static call 638 * site entries to take advantage of vmlinux static call 639 * privileges. 640 */ 641 if (!file->klp) 642 WARN("file already has .static_call_sites section, skipping"); 643 644 return 0; 645 } 646 647 if (list_empty(&file->static_call_list)) 648 return 0; 649 650 idx = 0; 651 list_for_each_entry(insn, &file->static_call_list, call_node) 652 idx++; 653 654 sec = elf_create_section_pair(file->elf, ".static_call_sites", 655 sizeof(*site), idx, idx * 2); 656 if (!sec) 657 return -1; 658 659 /* Allow modules to modify the low bits of static_call_site::key */ 660 sec->sh.sh_flags |= SHF_WRITE; 661 662 idx = 0; 663 list_for_each_entry(insn, &file->static_call_list, call_node) { 664 665 /* populate reloc for 'addr' */ 666 if (!elf_init_reloc_text_sym(file->elf, sec, 667 idx * sizeof(*site), idx * 2, 668 insn->sec, insn->offset)) 669 return -1; 670 671 /* find key symbol */ 672 key_name = strdup(insn_call_dest(insn)->name); 673 if (!key_name) { 674 ERROR_GLIBC("strdup"); 675 return -1; 676 } 677 if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR, 678 STATIC_CALL_TRAMP_PREFIX_LEN)) { 679 ERROR("static_call: trampoline name malformed: %s", key_name); 680 return -1; 681 } 682 tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN; 683 memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN); 684 685 key_sym = find_symbol_by_name(file->elf, tmp); 686 if (!key_sym) { 687 if (!opts.module) { 688 ERROR("static_call: can't find static_call_key symbol: %s", tmp); 689 return -1; 690 } 691 692 /* 693 * For modules(), the key might not be exported, which 694 * means the module can make static calls but isn't 695 * allowed to change them. 696 * 697 * In that case we temporarily set the key to be the 698 * trampoline address. This is fixed up in 699 * static_call_add_module(). 700 */ 701 key_sym = insn_call_dest(insn); 702 } 703 704 /* populate reloc for 'key' */ 705 if (!elf_init_reloc_data_sym(file->elf, sec, 706 idx * sizeof(*site) + 4, 707 (idx * 2) + 1, key_sym, 708 is_sibling_call(insn) * STATIC_CALL_SITE_TAIL)) 709 return -1; 710 711 idx++; 712 } 713 714 return 0; 715} 716 717static int create_retpoline_sites_sections(struct objtool_file *file) 718{ 719 struct instruction *insn; 720 struct section *sec; 721 int idx; 722 723 sec = find_section_by_name(file->elf, ".retpoline_sites"); 724 if (sec) { 725 WARN("file already has .retpoline_sites, skipping"); 726 return 0; 727 } 728 729 idx = 0; 730 list_for_each_entry(insn, &file->retpoline_call_list, call_node) 731 idx++; 732 733 if (!idx) 734 return 0; 735 736 sec = elf_create_section_pair(file->elf, ".retpoline_sites", 737 sizeof(int), idx, idx); 738 if (!sec) 739 return -1; 740 741 idx = 0; 742 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 743 744 if (!elf_init_reloc_text_sym(file->elf, sec, 745 idx * sizeof(int), idx, 746 insn->sec, insn->offset)) 747 return -1; 748 749 idx++; 750 } 751 752 return 0; 753} 754 755static int create_return_sites_sections(struct objtool_file *file) 756{ 757 struct instruction *insn; 758 struct section *sec; 759 int idx; 760 761 sec = find_section_by_name(file->elf, ".return_sites"); 762 if (sec) { 763 WARN("file already has .return_sites, skipping"); 764 return 0; 765 } 766 767 idx = 0; 768 list_for_each_entry(insn, &file->return_thunk_list, call_node) 769 idx++; 770 771 if (!idx) 772 return 0; 773 774 sec = elf_create_section_pair(file->elf, ".return_sites", 775 sizeof(int), idx, idx); 776 if (!sec) 777 return -1; 778 779 idx = 0; 780 list_for_each_entry(insn, &file->return_thunk_list, call_node) { 781 782 if (!elf_init_reloc_text_sym(file->elf, sec, 783 idx * sizeof(int), idx, 784 insn->sec, insn->offset)) 785 return -1; 786 787 idx++; 788 } 789 790 return 0; 791} 792 793static int create_ibt_endbr_seal_sections(struct objtool_file *file) 794{ 795 struct instruction *insn; 796 struct section *sec; 797 int idx; 798 799 sec = find_section_by_name(file->elf, ".ibt_endbr_seal"); 800 if (sec) { 801 WARN("file already has .ibt_endbr_seal, skipping"); 802 return 0; 803 } 804 805 idx = 0; 806 list_for_each_entry(insn, &file->endbr_list, call_node) 807 idx++; 808 809 if (opts.stats) { 810 printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); 811 printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); 812 printf("ibt: superfluous ENDBR: %d\n", idx); 813 } 814 815 if (!idx) 816 return 0; 817 818 sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal", 819 sizeof(int), idx, idx); 820 if (!sec) 821 return -1; 822 823 idx = 0; 824 list_for_each_entry(insn, &file->endbr_list, call_node) { 825 826 int *site = (int *)sec->data->d_buf + idx; 827 struct symbol *sym = insn->sym; 828 *site = 0; 829 830 if (opts.module && sym && is_func_sym(sym) && 831 insn->offset == sym->offset && 832 (!strcmp(sym->name, "init_module") || 833 !strcmp(sym->name, "cleanup_module"))) { 834 ERROR("%s(): Magic init_module() function name is deprecated, use module_init(fn) instead", 835 sym->name); 836 return -1; 837 } 838 839 if (!elf_init_reloc_text_sym(file->elf, sec, 840 idx * sizeof(int), idx, 841 insn->sec, insn->offset)) 842 return -1; 843 844 idx++; 845 } 846 847 return 0; 848} 849 850static int create_cfi_sections(struct objtool_file *file) 851{ 852 struct section *sec; 853 struct symbol *sym; 854 int idx; 855 856 sec = find_section_by_name(file->elf, ".cfi_sites"); 857 if (sec) { 858 WARN("file already has .cfi_sites section, skipping"); 859 return 0; 860 } 861 862 idx = 0; 863 for_each_sym(file->elf, sym) { 864 if (!is_func_sym(sym)) 865 continue; 866 867 if (strncmp(sym->name, "__cfi_", 6)) 868 continue; 869 870 idx++; 871 } 872 873 sec = elf_create_section_pair(file->elf, ".cfi_sites", 874 sizeof(unsigned int), idx, idx); 875 if (!sec) 876 return -1; 877 878 idx = 0; 879 for_each_sym(file->elf, sym) { 880 if (!is_func_sym(sym)) 881 continue; 882 883 if (strncmp(sym->name, "__cfi_", 6)) 884 continue; 885 886 if (!elf_init_reloc_text_sym(file->elf, sec, 887 idx * sizeof(unsigned int), idx, 888 sym->sec, sym->offset)) 889 return -1; 890 891 idx++; 892 } 893 894 return 0; 895} 896 897static int create_mcount_loc_sections(struct objtool_file *file) 898{ 899 size_t addr_size = elf_addr_size(file->elf); 900 struct instruction *insn; 901 struct section *sec; 902 int idx; 903 904 sec = find_section_by_name(file->elf, "__mcount_loc"); 905 if (sec) { 906 /* 907 * Livepatch modules have already extracted their __mcount_loc 908 * entries to cover the !CONFIG_FTRACE_MCOUNT_USE_OBJTOOL case. 909 */ 910 if (!file->klp) 911 WARN("file already has __mcount_loc section, skipping"); 912 913 return 0; 914 } 915 916 if (list_empty(&file->mcount_loc_list)) 917 return 0; 918 919 idx = 0; 920 list_for_each_entry(insn, &file->mcount_loc_list, call_node) 921 idx++; 922 923 sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size, 924 idx, idx); 925 if (!sec) 926 return -1; 927 928 sec->sh.sh_addralign = addr_size; 929 930 idx = 0; 931 list_for_each_entry(insn, &file->mcount_loc_list, call_node) { 932 933 struct reloc *reloc; 934 935 reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx, 936 insn->sec, insn->offset); 937 if (!reloc) 938 return -1; 939 940 set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32); 941 942 idx++; 943 } 944 945 return 0; 946} 947 948static int create_direct_call_sections(struct objtool_file *file) 949{ 950 struct instruction *insn; 951 struct section *sec; 952 int idx; 953 954 sec = find_section_by_name(file->elf, ".call_sites"); 955 if (sec) { 956 WARN("file already has .call_sites section, skipping"); 957 return 0; 958 } 959 960 if (list_empty(&file->call_list)) 961 return 0; 962 963 idx = 0; 964 list_for_each_entry(insn, &file->call_list, call_node) 965 idx++; 966 967 sec = elf_create_section_pair(file->elf, ".call_sites", 968 sizeof(unsigned int), idx, idx); 969 if (!sec) 970 return -1; 971 972 idx = 0; 973 list_for_each_entry(insn, &file->call_list, call_node) { 974 975 if (!elf_init_reloc_text_sym(file->elf, sec, 976 idx * sizeof(unsigned int), idx, 977 insn->sec, insn->offset)) 978 return -1; 979 980 idx++; 981 } 982 983 return 0; 984} 985 986#ifdef BUILD_KLP 987static int create_sym_checksum_section(struct objtool_file *file) 988{ 989 struct section *sec; 990 struct symbol *sym; 991 unsigned int idx = 0; 992 struct sym_checksum *checksum; 993 size_t entsize = sizeof(struct sym_checksum); 994 995 sec = find_section_by_name(file->elf, ".discard.sym_checksum"); 996 if (sec) { 997 if (!opts.dryrun) 998 WARN("file already has .discard.sym_checksum section, skipping"); 999 1000 return 0; 1001 } 1002 1003 for_each_sym(file->elf, sym) 1004 if (sym->csum.checksum) 1005 idx++; 1006 1007 if (!idx) 1008 return 0; 1009 1010 sec = elf_create_section_pair(file->elf, ".discard.sym_checksum", entsize, 1011 idx, idx); 1012 if (!sec) 1013 return -1; 1014 1015 idx = 0; 1016 for_each_sym(file->elf, sym) { 1017 if (!sym->csum.checksum) 1018 continue; 1019 1020 if (!elf_init_reloc(file->elf, sec->rsec, idx, idx * entsize, 1021 sym, 0, R_TEXT64)) 1022 return -1; 1023 1024 checksum = (struct sym_checksum *)sec->data->d_buf + idx; 1025 checksum->addr = 0; /* reloc */ 1026 checksum->checksum = sym->csum.checksum; 1027 1028 mark_sec_changed(file->elf, sec, true); 1029 1030 idx++; 1031 } 1032 1033 return 0; 1034} 1035#else 1036static int create_sym_checksum_section(struct objtool_file *file) { return -EINVAL; } 1037#endif 1038 1039/* 1040 * Warnings shouldn't be reported for ignored functions. 1041 */ 1042static int add_ignores(struct objtool_file *file) 1043{ 1044 struct section *rsec; 1045 struct symbol *func; 1046 struct reloc *reloc; 1047 1048 rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard"); 1049 if (!rsec) 1050 return 0; 1051 1052 for_each_reloc(rsec, reloc) { 1053 switch (reloc->sym->type) { 1054 case STT_FUNC: 1055 func = reloc->sym; 1056 break; 1057 1058 case STT_SECTION: 1059 func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc)); 1060 if (!func) 1061 continue; 1062 break; 1063 1064 default: 1065 ERROR("unexpected relocation symbol type in %s: %d", 1066 rsec->name, reloc->sym->type); 1067 return -1; 1068 } 1069 1070 func->ignore = true; 1071 if (func->cfunc) 1072 func->cfunc->ignore = true; 1073 } 1074 1075 return 0; 1076} 1077 1078/* 1079 * This is a whitelist of functions that is allowed to be called with AC set. 1080 * The list is meant to be minimal and only contains compiler instrumentation 1081 * ABI and a few functions used to implement *_{to,from}_user() functions. 1082 * 1083 * These functions must not directly change AC, but may PUSHF/POPF. 1084 */ 1085static const char *uaccess_safe_builtin[] = { 1086 /* KASAN */ 1087 "kasan_report", 1088 "kasan_check_range", 1089 /* KASAN out-of-line */ 1090 "__asan_loadN_noabort", 1091 "__asan_load1_noabort", 1092 "__asan_load2_noabort", 1093 "__asan_load4_noabort", 1094 "__asan_load8_noabort", 1095 "__asan_load16_noabort", 1096 "__asan_storeN_noabort", 1097 "__asan_store1_noabort", 1098 "__asan_store2_noabort", 1099 "__asan_store4_noabort", 1100 "__asan_store8_noabort", 1101 "__asan_store16_noabort", 1102 "__kasan_check_read", 1103 "__kasan_check_write", 1104 /* KASAN in-line */ 1105 "__asan_report_load_n_noabort", 1106 "__asan_report_load1_noabort", 1107 "__asan_report_load2_noabort", 1108 "__asan_report_load4_noabort", 1109 "__asan_report_load8_noabort", 1110 "__asan_report_load16_noabort", 1111 "__asan_report_store_n_noabort", 1112 "__asan_report_store1_noabort", 1113 "__asan_report_store2_noabort", 1114 "__asan_report_store4_noabort", 1115 "__asan_report_store8_noabort", 1116 "__asan_report_store16_noabort", 1117 /* KCSAN */ 1118 "__kcsan_check_access", 1119 "__kcsan_mb", 1120 "__kcsan_wmb", 1121 "__kcsan_rmb", 1122 "__kcsan_release", 1123 "kcsan_found_watchpoint", 1124 "kcsan_setup_watchpoint", 1125 "kcsan_check_scoped_accesses", 1126 "kcsan_disable_current", 1127 "kcsan_enable_current_nowarn", 1128 /* KCSAN/TSAN */ 1129 "__tsan_func_entry", 1130 "__tsan_func_exit", 1131 "__tsan_read_range", 1132 "__tsan_write_range", 1133 "__tsan_read1", 1134 "__tsan_read2", 1135 "__tsan_read4", 1136 "__tsan_read8", 1137 "__tsan_read16", 1138 "__tsan_write1", 1139 "__tsan_write2", 1140 "__tsan_write4", 1141 "__tsan_write8", 1142 "__tsan_write16", 1143 "__tsan_read_write1", 1144 "__tsan_read_write2", 1145 "__tsan_read_write4", 1146 "__tsan_read_write8", 1147 "__tsan_read_write16", 1148 "__tsan_volatile_read1", 1149 "__tsan_volatile_read2", 1150 "__tsan_volatile_read4", 1151 "__tsan_volatile_read8", 1152 "__tsan_volatile_read16", 1153 "__tsan_volatile_write1", 1154 "__tsan_volatile_write2", 1155 "__tsan_volatile_write4", 1156 "__tsan_volatile_write8", 1157 "__tsan_volatile_write16", 1158 "__tsan_atomic8_load", 1159 "__tsan_atomic16_load", 1160 "__tsan_atomic32_load", 1161 "__tsan_atomic64_load", 1162 "__tsan_atomic8_store", 1163 "__tsan_atomic16_store", 1164 "__tsan_atomic32_store", 1165 "__tsan_atomic64_store", 1166 "__tsan_atomic8_exchange", 1167 "__tsan_atomic16_exchange", 1168 "__tsan_atomic32_exchange", 1169 "__tsan_atomic64_exchange", 1170 "__tsan_atomic8_fetch_add", 1171 "__tsan_atomic16_fetch_add", 1172 "__tsan_atomic32_fetch_add", 1173 "__tsan_atomic64_fetch_add", 1174 "__tsan_atomic8_fetch_sub", 1175 "__tsan_atomic16_fetch_sub", 1176 "__tsan_atomic32_fetch_sub", 1177 "__tsan_atomic64_fetch_sub", 1178 "__tsan_atomic8_fetch_and", 1179 "__tsan_atomic16_fetch_and", 1180 "__tsan_atomic32_fetch_and", 1181 "__tsan_atomic64_fetch_and", 1182 "__tsan_atomic8_fetch_or", 1183 "__tsan_atomic16_fetch_or", 1184 "__tsan_atomic32_fetch_or", 1185 "__tsan_atomic64_fetch_or", 1186 "__tsan_atomic8_fetch_xor", 1187 "__tsan_atomic16_fetch_xor", 1188 "__tsan_atomic32_fetch_xor", 1189 "__tsan_atomic64_fetch_xor", 1190 "__tsan_atomic8_fetch_nand", 1191 "__tsan_atomic16_fetch_nand", 1192 "__tsan_atomic32_fetch_nand", 1193 "__tsan_atomic64_fetch_nand", 1194 "__tsan_atomic8_compare_exchange_strong", 1195 "__tsan_atomic16_compare_exchange_strong", 1196 "__tsan_atomic32_compare_exchange_strong", 1197 "__tsan_atomic64_compare_exchange_strong", 1198 "__tsan_atomic8_compare_exchange_weak", 1199 "__tsan_atomic16_compare_exchange_weak", 1200 "__tsan_atomic32_compare_exchange_weak", 1201 "__tsan_atomic64_compare_exchange_weak", 1202 "__tsan_atomic8_compare_exchange_val", 1203 "__tsan_atomic16_compare_exchange_val", 1204 "__tsan_atomic32_compare_exchange_val", 1205 "__tsan_atomic64_compare_exchange_val", 1206 "__tsan_atomic_thread_fence", 1207 "__tsan_atomic_signal_fence", 1208 "__tsan_unaligned_read16", 1209 "__tsan_unaligned_write16", 1210 /* KCOV */ 1211 "write_comp_data", 1212 "check_kcov_mode", 1213 "__sanitizer_cov_trace_pc", 1214 "__sanitizer_cov_trace_const_cmp1", 1215 "__sanitizer_cov_trace_const_cmp2", 1216 "__sanitizer_cov_trace_const_cmp4", 1217 "__sanitizer_cov_trace_const_cmp8", 1218 "__sanitizer_cov_trace_cmp1", 1219 "__sanitizer_cov_trace_cmp2", 1220 "__sanitizer_cov_trace_cmp4", 1221 "__sanitizer_cov_trace_cmp8", 1222 "__sanitizer_cov_trace_switch", 1223 /* KMSAN */ 1224 "kmsan_copy_to_user", 1225 "kmsan_disable_current", 1226 "kmsan_enable_current", 1227 "kmsan_report", 1228 "kmsan_unpoison_entry_regs", 1229 "kmsan_unpoison_memory", 1230 "__msan_chain_origin", 1231 "__msan_get_context_state", 1232 "__msan_instrument_asm_store", 1233 "__msan_metadata_ptr_for_load_1", 1234 "__msan_metadata_ptr_for_load_2", 1235 "__msan_metadata_ptr_for_load_4", 1236 "__msan_metadata_ptr_for_load_8", 1237 "__msan_metadata_ptr_for_load_n", 1238 "__msan_metadata_ptr_for_store_1", 1239 "__msan_metadata_ptr_for_store_2", 1240 "__msan_metadata_ptr_for_store_4", 1241 "__msan_metadata_ptr_for_store_8", 1242 "__msan_metadata_ptr_for_store_n", 1243 "__msan_poison_alloca", 1244 "__msan_warning", 1245 /* UBSAN */ 1246 "ubsan_type_mismatch_common", 1247 "__ubsan_handle_type_mismatch", 1248 "__ubsan_handle_type_mismatch_v1", 1249 "__ubsan_handle_shift_out_of_bounds", 1250 "__ubsan_handle_load_invalid_value", 1251 /* KSTACK_ERASE */ 1252 "__sanitizer_cov_stack_depth", 1253 /* TRACE_BRANCH_PROFILING */ 1254 "ftrace_likely_update", 1255 /* STACKPROTECTOR */ 1256 "__stack_chk_fail", 1257 /* misc */ 1258 "csum_partial_copy_generic", 1259 "copy_mc_fragile", 1260 "copy_mc_fragile_handle_tail", 1261 "copy_mc_enhanced_fast_string", 1262 "rep_stos_alternative", 1263 "rep_movs_alternative", 1264 "copy_to_nontemporal", 1265 NULL 1266}; 1267 1268static void add_uaccess_safe(struct objtool_file *file) 1269{ 1270 struct symbol *func; 1271 const char **name; 1272 1273 if (!opts.uaccess) 1274 return; 1275 1276 for (name = uaccess_safe_builtin; *name; name++) { 1277 func = find_symbol_by_name(file->elf, *name); 1278 if (!func) 1279 continue; 1280 1281 func->uaccess_safe = true; 1282 } 1283} 1284 1285/* 1286 * Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol 1287 * will be added to the .retpoline_sites section. 1288 */ 1289__weak bool arch_is_retpoline(struct symbol *sym) 1290{ 1291 return false; 1292} 1293 1294/* 1295 * Symbols that replace INSN_RETURN, every (tail) call to such a symbol 1296 * will be added to the .return_sites section. 1297 */ 1298__weak bool arch_is_rethunk(struct symbol *sym) 1299{ 1300 return false; 1301} 1302 1303/* 1304 * Symbols that are embedded inside other instructions, because sometimes crazy 1305 * code exists. These are mostly ignored for validation purposes. 1306 */ 1307__weak bool arch_is_embedded_insn(struct symbol *sym) 1308{ 1309 return false; 1310} 1311 1312static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) 1313{ 1314 struct reloc *reloc; 1315 1316 if (insn->no_reloc) 1317 return NULL; 1318 1319 if (!file) 1320 return NULL; 1321 1322 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 1323 insn->offset, insn->len); 1324 if (!reloc) { 1325 insn->no_reloc = 1; 1326 return NULL; 1327 } 1328 1329 return reloc; 1330} 1331 1332static void remove_insn_ops(struct instruction *insn) 1333{ 1334 struct stack_op *op, *next; 1335 1336 for (op = insn->stack_ops; op; op = next) { 1337 next = op->next; 1338 free(op); 1339 } 1340 insn->stack_ops = NULL; 1341} 1342 1343static int annotate_call_site(struct objtool_file *file, 1344 struct instruction *insn, bool sibling) 1345{ 1346 struct reloc *reloc = insn_reloc(file, insn); 1347 struct symbol *sym = insn_call_dest(insn); 1348 1349 if (!sym) 1350 sym = reloc->sym; 1351 1352 if (sym->static_call_tramp) { 1353 list_add_tail(&insn->call_node, &file->static_call_list); 1354 return 0; 1355 } 1356 1357 if (sym->retpoline_thunk) { 1358 list_add_tail(&insn->call_node, &file->retpoline_call_list); 1359 return 0; 1360 } 1361 1362 /* 1363 * Many compilers cannot disable KCOV or sanitizer calls with a function 1364 * attribute so they need a little help, NOP out any such calls from 1365 * noinstr text. 1366 */ 1367 if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { 1368 if (reloc) 1369 set_reloc_type(file->elf, reloc, R_NONE); 1370 1371 if (elf_write_insn(file->elf, insn->sec, 1372 insn->offset, insn->len, 1373 sibling ? arch_ret_insn(insn->len) 1374 : arch_nop_insn(insn->len))) { 1375 return -1; 1376 } 1377 1378 insn->type = sibling ? INSN_RETURN : INSN_NOP; 1379 1380 if (sibling) { 1381 /* 1382 * We've replaced the tail-call JMP insn by two new 1383 * insn: RET; INT3, except we only have a single struct 1384 * insn here. Mark it retpoline_safe to avoid the SLS 1385 * warning, instead of adding another insn. 1386 */ 1387 insn->retpoline_safe = true; 1388 } 1389 1390 return 0; 1391 } 1392 1393 if (opts.mcount && sym->fentry) { 1394 if (sibling) 1395 WARN_INSN(insn, "tail call to __fentry__ !?!?"); 1396 if (opts.mnop) { 1397 if (reloc) 1398 set_reloc_type(file->elf, reloc, R_NONE); 1399 1400 if (elf_write_insn(file->elf, insn->sec, 1401 insn->offset, insn->len, 1402 arch_nop_insn(insn->len))) { 1403 return -1; 1404 } 1405 1406 insn->type = INSN_NOP; 1407 } 1408 1409 list_add_tail(&insn->call_node, &file->mcount_loc_list); 1410 return 0; 1411 } 1412 1413 if (insn->type == INSN_CALL && !insn->sec->init && 1414 !insn->_call_dest->embedded_insn) 1415 list_add_tail(&insn->call_node, &file->call_list); 1416 1417 if (!sibling && dead_end_function(file, sym)) 1418 insn->dead_end = true; 1419 1420 return 0; 1421} 1422 1423static int add_call_dest(struct objtool_file *file, struct instruction *insn, 1424 struct symbol *dest, bool sibling) 1425{ 1426 insn->_call_dest = dest; 1427 if (!dest) 1428 return 0; 1429 1430 /* 1431 * Whatever stack impact regular CALLs have, should be undone 1432 * by the RETURN of the called function. 1433 * 1434 * Annotated intra-function calls retain the stack_ops but 1435 * are converted to JUMP, see read_intra_function_calls(). 1436 */ 1437 remove_insn_ops(insn); 1438 1439 return annotate_call_site(file, insn, sibling); 1440} 1441 1442static int add_retpoline_call(struct objtool_file *file, struct instruction *insn) 1443{ 1444 /* 1445 * Retpoline calls/jumps are really dynamic calls/jumps in disguise, 1446 * so convert them accordingly. 1447 */ 1448 switch (insn->type) { 1449 case INSN_CALL: 1450 insn->type = INSN_CALL_DYNAMIC; 1451 break; 1452 case INSN_JUMP_UNCONDITIONAL: 1453 insn->type = INSN_JUMP_DYNAMIC; 1454 break; 1455 case INSN_JUMP_CONDITIONAL: 1456 insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL; 1457 break; 1458 default: 1459 return 0; 1460 } 1461 1462 insn->retpoline_safe = true; 1463 1464 /* 1465 * Whatever stack impact regular CALLs have, should be undone 1466 * by the RETURN of the called function. 1467 * 1468 * Annotated intra-function calls retain the stack_ops but 1469 * are converted to JUMP, see read_intra_function_calls(). 1470 */ 1471 remove_insn_ops(insn); 1472 1473 return annotate_call_site(file, insn, false); 1474} 1475 1476static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) 1477{ 1478 /* 1479 * Return thunk tail calls are really just returns in disguise, 1480 * so convert them accordingly. 1481 */ 1482 insn->type = INSN_RETURN; 1483 insn->retpoline_safe = true; 1484 1485 if (add) 1486 list_add_tail(&insn->call_node, &file->return_thunk_list); 1487} 1488 1489static bool is_first_func_insn(struct objtool_file *file, 1490 struct instruction *insn) 1491{ 1492 struct symbol *func = insn_func(insn); 1493 1494 if (!func) 1495 return false; 1496 1497 if (insn->offset == func->offset) 1498 return true; 1499 1500 /* Allow direct CALL/JMP past ENDBR */ 1501 if (opts.ibt) { 1502 struct instruction *prev = prev_insn_same_sym(file, insn); 1503 1504 if (prev && prev->type == INSN_ENDBR && 1505 insn->offset == func->offset + prev->len) 1506 return true; 1507 } 1508 1509 return false; 1510} 1511 1512/* 1513 * Find the destination instructions for all jumps. 1514 */ 1515static int add_jump_destinations(struct objtool_file *file) 1516{ 1517 struct instruction *insn; 1518 struct reloc *reloc; 1519 1520 for_each_insn(file, insn) { 1521 struct symbol *func = insn_func(insn); 1522 struct instruction *dest_insn; 1523 struct section *dest_sec; 1524 struct symbol *dest_sym; 1525 unsigned long dest_off; 1526 1527 if (!is_static_jump(insn)) 1528 continue; 1529 1530 if (insn->jump_dest) { 1531 /* 1532 * handle_group_alt() may have previously set 1533 * 'jump_dest' for some alternatives. 1534 */ 1535 continue; 1536 } 1537 1538 reloc = insn_reloc(file, insn); 1539 if (!reloc) { 1540 dest_sec = insn->sec; 1541 dest_off = arch_jump_destination(insn); 1542 dest_sym = dest_sec->sym; 1543 } else { 1544 dest_sym = reloc->sym; 1545 if (is_undef_sym(dest_sym)) { 1546 if (dest_sym->retpoline_thunk) { 1547 if (add_retpoline_call(file, insn)) 1548 return -1; 1549 continue; 1550 } 1551 1552 if (dest_sym->return_thunk) { 1553 add_return_call(file, insn, true); 1554 continue; 1555 } 1556 1557 /* External symbol */ 1558 if (func) { 1559 /* External sibling call */ 1560 if (add_call_dest(file, insn, dest_sym, true)) 1561 return -1; 1562 continue; 1563 } 1564 1565 /* Non-func asm code jumping to external symbol */ 1566 continue; 1567 } 1568 1569 dest_sec = dest_sym->sec; 1570 dest_off = dest_sym->offset + arch_insn_adjusted_addend(insn, reloc); 1571 } 1572 1573 dest_insn = find_insn(file, dest_sec, dest_off); 1574 if (!dest_insn) { 1575 struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); 1576 1577 /* 1578 * retbleed_untrain_ret() jumps to 1579 * __x86_return_thunk(), but objtool can't find 1580 * the thunk's starting RET instruction, 1581 * because the RET is also in the middle of 1582 * another instruction. Objtool only knows 1583 * about the outer instruction. 1584 */ 1585 if (sym && sym->embedded_insn) { 1586 add_return_call(file, insn, false); 1587 continue; 1588 } 1589 1590 /* 1591 * GCOV/KCOV dead code can jump to the end of 1592 * the function/section. 1593 */ 1594 if (file->ignore_unreachables && func && 1595 dest_sec == insn->sec && 1596 dest_off == func->offset + func->len) 1597 continue; 1598 1599 ERROR_INSN(insn, "can't find jump dest instruction at %s", 1600 offstr(dest_sec, dest_off)); 1601 return -1; 1602 } 1603 1604 if (!dest_sym || is_sec_sym(dest_sym)) { 1605 dest_sym = dest_insn->sym; 1606 if (!dest_sym) 1607 goto set_jump_dest; 1608 } 1609 1610 if (dest_sym->retpoline_thunk && dest_insn->offset == dest_sym->offset) { 1611 if (add_retpoline_call(file, insn)) 1612 return -1; 1613 continue; 1614 } 1615 1616 if (dest_sym->return_thunk && dest_insn->offset == dest_sym->offset) { 1617 add_return_call(file, insn, true); 1618 continue; 1619 } 1620 1621 if (!insn->sym || insn->sym->pfunc == dest_sym->pfunc) 1622 goto set_jump_dest; 1623 1624 /* 1625 * Internal cross-function jump. 1626 */ 1627 1628 if (is_first_func_insn(file, dest_insn)) { 1629 /* Internal sibling call */ 1630 if (add_call_dest(file, insn, dest_sym, true)) 1631 return -1; 1632 continue; 1633 } 1634 1635set_jump_dest: 1636 insn->jump_dest = dest_insn; 1637 } 1638 1639 return 0; 1640} 1641 1642static struct symbol *find_call_destination(struct section *sec, unsigned long offset) 1643{ 1644 struct symbol *call_dest; 1645 1646 call_dest = find_func_by_offset(sec, offset); 1647 if (!call_dest) 1648 call_dest = find_symbol_by_offset(sec, offset); 1649 1650 return call_dest; 1651} 1652 1653/* 1654 * Find the destination instructions for all calls. 1655 */ 1656static int add_call_destinations(struct objtool_file *file) 1657{ 1658 struct instruction *insn; 1659 unsigned long dest_off; 1660 struct symbol *dest; 1661 struct reloc *reloc; 1662 1663 for_each_insn(file, insn) { 1664 struct symbol *func = insn_func(insn); 1665 if (insn->type != INSN_CALL) 1666 continue; 1667 1668 reloc = insn_reloc(file, insn); 1669 if (!reloc) { 1670 dest_off = arch_jump_destination(insn); 1671 dest = find_call_destination(insn->sec, dest_off); 1672 1673 if (add_call_dest(file, insn, dest, false)) 1674 return -1; 1675 1676 if (func && func->ignore) 1677 continue; 1678 1679 if (!insn_call_dest(insn)) { 1680 ERROR_INSN(insn, "unannotated intra-function call"); 1681 return -1; 1682 } 1683 1684 if (func && !is_func_sym(insn_call_dest(insn))) { 1685 ERROR_INSN(insn, "unsupported call to non-function"); 1686 return -1; 1687 } 1688 1689 } else if (is_sec_sym(reloc->sym)) { 1690 dest_off = arch_insn_adjusted_addend(insn, reloc); 1691 dest = find_call_destination(reloc->sym->sec, dest_off); 1692 if (!dest) { 1693 ERROR_INSN(insn, "can't find call dest symbol at %s+0x%lx", 1694 reloc->sym->sec->name, dest_off); 1695 return -1; 1696 } 1697 1698 if (add_call_dest(file, insn, dest, false)) 1699 return -1; 1700 1701 } else if (reloc->sym->retpoline_thunk) { 1702 if (add_retpoline_call(file, insn)) 1703 return -1; 1704 1705 } else { 1706 if (add_call_dest(file, insn, reloc->sym, false)) 1707 return -1; 1708 } 1709 } 1710 1711 return 0; 1712} 1713 1714/* 1715 * The .alternatives section requires some extra special care over and above 1716 * other special sections because alternatives are patched in place. 1717 */ 1718static int handle_group_alt(struct objtool_file *file, 1719 struct special_alt *special_alt, 1720 struct instruction *orig_insn, 1721 struct instruction **new_insn) 1722{ 1723 struct instruction *last_new_insn = NULL, *insn, *nop = NULL; 1724 struct alt_group *orig_alt_group, *new_alt_group; 1725 unsigned long dest_off; 1726 1727 orig_alt_group = orig_insn->alt_group; 1728 if (!orig_alt_group) { 1729 struct instruction *last_orig_insn = NULL; 1730 1731 orig_alt_group = calloc(1, sizeof(*orig_alt_group)); 1732 if (!orig_alt_group) { 1733 ERROR_GLIBC("calloc"); 1734 return -1; 1735 } 1736 orig_alt_group->cfi = calloc(special_alt->orig_len, 1737 sizeof(struct cfi_state *)); 1738 if (!orig_alt_group->cfi) { 1739 ERROR_GLIBC("calloc"); 1740 return -1; 1741 } 1742 1743 insn = orig_insn; 1744 sec_for_each_insn_from(file, insn) { 1745 if (insn->offset >= special_alt->orig_off + special_alt->orig_len) 1746 break; 1747 1748 insn->alt_group = orig_alt_group; 1749 last_orig_insn = insn; 1750 } 1751 orig_alt_group->orig_group = NULL; 1752 orig_alt_group->first_insn = orig_insn; 1753 orig_alt_group->last_insn = last_orig_insn; 1754 orig_alt_group->nop = NULL; 1755 orig_alt_group->ignore = orig_insn->ignore_alts; 1756 orig_alt_group->feature = 0; 1757 } else { 1758 if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - 1759 orig_alt_group->first_insn->offset != special_alt->orig_len) { 1760 ERROR_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", 1761 orig_alt_group->last_insn->offset + 1762 orig_alt_group->last_insn->len - 1763 orig_alt_group->first_insn->offset, 1764 special_alt->orig_len); 1765 return -1; 1766 } 1767 } 1768 1769 new_alt_group = calloc(1, sizeof(*new_alt_group)); 1770 if (!new_alt_group) { 1771 ERROR_GLIBC("calloc"); 1772 return -1; 1773 } 1774 1775 if (special_alt->new_len < special_alt->orig_len) { 1776 /* 1777 * Insert a fake nop at the end to make the replacement 1778 * alt_group the same size as the original. This is needed to 1779 * allow propagate_alt_cfi() to do its magic. When the last 1780 * instruction affects the stack, the instruction after it (the 1781 * nop) will propagate the new state to the shared CFI array. 1782 */ 1783 nop = calloc(1, sizeof(*nop)); 1784 if (!nop) { 1785 ERROR_GLIBC("calloc"); 1786 return -1; 1787 } 1788 memset(nop, 0, sizeof(*nop)); 1789 1790 nop->sec = special_alt->new_sec; 1791 nop->offset = special_alt->new_off + special_alt->new_len; 1792 nop->len = special_alt->orig_len - special_alt->new_len; 1793 nop->type = INSN_NOP; 1794 nop->sym = orig_insn->sym; 1795 nop->alt_group = new_alt_group; 1796 nop->fake = 1; 1797 } 1798 1799 if (!special_alt->new_len) { 1800 *new_insn = nop; 1801 goto end; 1802 } 1803 1804 insn = *new_insn; 1805 sec_for_each_insn_from(file, insn) { 1806 struct reloc *alt_reloc; 1807 1808 if (insn->offset >= special_alt->new_off + special_alt->new_len) 1809 break; 1810 1811 last_new_insn = insn; 1812 1813 insn->sym = orig_insn->sym; 1814 insn->alt_group = new_alt_group; 1815 1816 /* 1817 * Since alternative replacement code is copy/pasted by the 1818 * kernel after applying relocations, generally such code can't 1819 * have relative-address relocation references to outside the 1820 * .altinstr_replacement section, unless the arch's 1821 * alternatives code can adjust the relative offsets 1822 * accordingly. 1823 */ 1824 alt_reloc = insn_reloc(file, insn); 1825 if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && 1826 !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { 1827 1828 ERROR_INSN(insn, "unsupported relocation in alternatives section"); 1829 return -1; 1830 } 1831 1832 if (!is_static_jump(insn)) 1833 continue; 1834 1835 if (!insn->immediate) 1836 continue; 1837 1838 dest_off = arch_jump_destination(insn); 1839 if (dest_off == special_alt->new_off + special_alt->new_len) { 1840 insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); 1841 if (!insn->jump_dest) { 1842 ERROR_INSN(insn, "can't find alternative jump destination"); 1843 return -1; 1844 } 1845 } 1846 } 1847 1848 if (!last_new_insn) { 1849 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1850 "can't find last new alternative instruction"); 1851 return -1; 1852 } 1853 1854end: 1855 new_alt_group->orig_group = orig_alt_group; 1856 new_alt_group->first_insn = *new_insn; 1857 new_alt_group->last_insn = last_new_insn; 1858 new_alt_group->nop = nop; 1859 new_alt_group->ignore = (*new_insn)->ignore_alts; 1860 new_alt_group->cfi = orig_alt_group->cfi; 1861 new_alt_group->feature = special_alt->feature; 1862 return 0; 1863} 1864 1865/* 1866 * A jump table entry can either convert a nop to a jump or a jump to a nop. 1867 * If the original instruction is a jump, make the alt entry an effective nop 1868 * by just skipping the original instruction. 1869 */ 1870static int handle_jump_alt(struct objtool_file *file, 1871 struct special_alt *special_alt, 1872 struct instruction *orig_insn, 1873 struct instruction **new_insn) 1874{ 1875 if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && 1876 orig_insn->type != INSN_NOP) { 1877 1878 ERROR_INSN(orig_insn, "unsupported instruction at jump label"); 1879 return -1; 1880 } 1881 1882 if (opts.hack_jump_label && special_alt->key_addend & 2) { 1883 struct reloc *reloc = insn_reloc(file, orig_insn); 1884 1885 if (reloc) 1886 set_reloc_type(file->elf, reloc, R_NONE); 1887 1888 if (elf_write_insn(file->elf, orig_insn->sec, 1889 orig_insn->offset, orig_insn->len, 1890 arch_nop_insn(orig_insn->len))) { 1891 return -1; 1892 } 1893 1894 orig_insn->type = INSN_NOP; 1895 } 1896 1897 if (orig_insn->type == INSN_NOP) { 1898 if (orig_insn->len == 2) 1899 file->jl_nop_short++; 1900 else 1901 file->jl_nop_long++; 1902 1903 return 0; 1904 } 1905 1906 if (orig_insn->len == 2) 1907 file->jl_short++; 1908 else 1909 file->jl_long++; 1910 1911 *new_insn = next_insn_same_sec(file, orig_insn); 1912 return 0; 1913} 1914 1915/* 1916 * Read all the special sections which have alternate instructions which can be 1917 * patched in or redirected to at runtime. Each instruction having alternate 1918 * instruction(s) has them added to its insn->alts list, which will be 1919 * traversed in validate_branch(). 1920 */ 1921static int add_special_section_alts(struct objtool_file *file) 1922{ 1923 struct list_head special_alts; 1924 struct instruction *orig_insn, *new_insn; 1925 struct special_alt *special_alt, *tmp; 1926 enum alternative_type alt_type; 1927 struct alternative *alt; 1928 struct alternative *a; 1929 1930 if (special_get_alts(file->elf, &special_alts)) 1931 return -1; 1932 1933 list_for_each_entry_safe(special_alt, tmp, &special_alts, list) { 1934 1935 orig_insn = find_insn(file, special_alt->orig_sec, 1936 special_alt->orig_off); 1937 if (!orig_insn) { 1938 ERROR_FUNC(special_alt->orig_sec, special_alt->orig_off, 1939 "special: can't find orig instruction"); 1940 return -1; 1941 } 1942 1943 new_insn = NULL; 1944 if (!special_alt->group || special_alt->new_len) { 1945 new_insn = find_insn(file, special_alt->new_sec, 1946 special_alt->new_off); 1947 if (!new_insn) { 1948 ERROR_FUNC(special_alt->new_sec, special_alt->new_off, 1949 "special: can't find new instruction"); 1950 return -1; 1951 } 1952 } 1953 1954 if (special_alt->group) { 1955 if (!special_alt->orig_len) { 1956 ERROR_INSN(orig_insn, "empty alternative entry"); 1957 continue; 1958 } 1959 1960 if (handle_group_alt(file, special_alt, orig_insn, &new_insn)) 1961 return -1; 1962 1963 alt_type = ALT_TYPE_INSTRUCTIONS; 1964 1965 } else if (special_alt->jump_or_nop) { 1966 if (handle_jump_alt(file, special_alt, orig_insn, &new_insn)) 1967 return -1; 1968 1969 alt_type = ALT_TYPE_JUMP_TABLE; 1970 } else { 1971 alt_type = ALT_TYPE_EX_TABLE; 1972 } 1973 1974 alt = calloc(1, sizeof(*alt)); 1975 if (!alt) { 1976 ERROR_GLIBC("calloc"); 1977 return -1; 1978 } 1979 1980 alt->insn = new_insn; 1981 alt->type = alt_type; 1982 alt->next = NULL; 1983 1984 /* 1985 * Store alternatives in the same order they have been 1986 * defined. 1987 */ 1988 if (!orig_insn->alts) { 1989 orig_insn->alts = alt; 1990 } else { 1991 for (a = orig_insn->alts; a->next; a = a->next) 1992 ; 1993 a->next = alt; 1994 } 1995 1996 list_del(&special_alt->list); 1997 free(special_alt); 1998 } 1999 2000 if (opts.stats) { 2001 printf("jl\\\tNOP\tJMP\n"); 2002 printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); 2003 printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); 2004 } 2005 2006 return 0; 2007} 2008 2009__weak unsigned long arch_jump_table_sym_offset(struct reloc *reloc, struct reloc *table) 2010{ 2011 return reloc->sym->offset + reloc_addend(reloc); 2012} 2013 2014static int add_jump_table(struct objtool_file *file, struct instruction *insn) 2015{ 2016 unsigned long table_size = insn_jump_table_size(insn); 2017 struct symbol *pfunc = insn_func(insn)->pfunc; 2018 struct reloc *table = insn_jump_table(insn); 2019 struct instruction *dest_insn; 2020 unsigned int prev_offset = 0; 2021 struct reloc *reloc = table; 2022 struct alternative *alt; 2023 unsigned long sym_offset; 2024 2025 /* 2026 * Each @reloc is a switch table relocation which points to the target 2027 * instruction. 2028 */ 2029 for_each_reloc_from(table->sec, reloc) { 2030 2031 /* Check for the end of the table: */ 2032 if (table_size && reloc_offset(reloc) - reloc_offset(table) >= table_size) 2033 break; 2034 if (reloc != table && is_jump_table(reloc)) 2035 break; 2036 2037 /* Make sure the table entries are consecutive: */ 2038 if (prev_offset && reloc_offset(reloc) != prev_offset + arch_reloc_size(reloc)) 2039 break; 2040 2041 sym_offset = arch_jump_table_sym_offset(reloc, table); 2042 2043 /* Detect function pointers from contiguous objects: */ 2044 if (reloc->sym->sec == pfunc->sec && sym_offset == pfunc->offset) 2045 break; 2046 2047 /* 2048 * Clang sometimes leaves dangling unused jump table entries 2049 * which point to the end of the function. Ignore them. 2050 */ 2051 if (reloc->sym->sec == pfunc->sec && 2052 sym_offset == pfunc->offset + pfunc->len) 2053 goto next; 2054 2055 dest_insn = find_insn(file, reloc->sym->sec, sym_offset); 2056 if (!dest_insn) 2057 break; 2058 2059 /* Make sure the destination is in the same function: */ 2060 if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc) 2061 break; 2062 2063 alt = calloc(1, sizeof(*alt)); 2064 if (!alt) { 2065 ERROR_GLIBC("calloc"); 2066 return -1; 2067 } 2068 2069 alt->insn = dest_insn; 2070 alt->next = insn->alts; 2071 insn->alts = alt; 2072next: 2073 prev_offset = reloc_offset(reloc); 2074 } 2075 2076 if (!prev_offset) { 2077 ERROR_INSN(insn, "can't find switch jump table"); 2078 return -1; 2079 } 2080 2081 return 0; 2082} 2083 2084/* 2085 * find_jump_table() - Given a dynamic jump, find the switch jump table 2086 * associated with it. 2087 */ 2088static void find_jump_table(struct objtool_file *file, struct symbol *func, 2089 struct instruction *insn) 2090{ 2091 struct reloc *table_reloc; 2092 struct instruction *dest_insn, *orig_insn = insn; 2093 unsigned long table_size; 2094 unsigned long sym_offset; 2095 2096 /* 2097 * Backward search using the @first_jump_src links, these help avoid 2098 * much of the 'in between' code. Which avoids us getting confused by 2099 * it. 2100 */ 2101 for (; 2102 insn && insn_func(insn) && insn_func(insn)->pfunc == func; 2103 insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) { 2104 2105 if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC) 2106 break; 2107 2108 /* allow small jumps within the range */ 2109 if (insn->type == INSN_JUMP_UNCONDITIONAL && 2110 insn->jump_dest && 2111 (insn->jump_dest->offset <= insn->offset || 2112 insn->jump_dest->offset > orig_insn->offset)) 2113 break; 2114 2115 table_reloc = arch_find_switch_table(file, insn, &table_size); 2116 if (!table_reloc) 2117 continue; 2118 2119 sym_offset = table_reloc->sym->offset + reloc_addend(table_reloc); 2120 2121 dest_insn = find_insn(file, table_reloc->sym->sec, sym_offset); 2122 if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func) 2123 continue; 2124 2125 set_jump_table(table_reloc); 2126 orig_insn->_jump_table = table_reloc; 2127 orig_insn->_jump_table_size = table_size; 2128 2129 break; 2130 } 2131} 2132 2133/* 2134 * First pass: Mark the head of each jump table so that in the next pass, 2135 * we know when a given jump table ends and the next one starts. 2136 */ 2137static void mark_func_jump_tables(struct objtool_file *file, 2138 struct symbol *func) 2139{ 2140 struct instruction *insn, *last = NULL; 2141 2142 func_for_each_insn(file, func, insn) { 2143 if (!last) 2144 last = insn; 2145 2146 /* 2147 * Store back-pointers for forward jumps such 2148 * that find_jump_table() can back-track using those and 2149 * avoid some potentially confusing code. 2150 */ 2151 if (insn->jump_dest && 2152 insn->jump_dest->offset > insn->offset && 2153 !insn->jump_dest->first_jump_src) { 2154 2155 insn->jump_dest->first_jump_src = insn; 2156 last = insn->jump_dest; 2157 } 2158 2159 if (insn->type != INSN_JUMP_DYNAMIC) 2160 continue; 2161 2162 find_jump_table(file, func, insn); 2163 } 2164} 2165 2166static int add_func_jump_tables(struct objtool_file *file, 2167 struct symbol *func) 2168{ 2169 struct instruction *insn; 2170 2171 func_for_each_insn(file, func, insn) { 2172 if (!insn_jump_table(insn)) 2173 continue; 2174 2175 if (add_jump_table(file, insn)) 2176 return -1; 2177 } 2178 2179 return 0; 2180} 2181 2182/* 2183 * For some switch statements, gcc generates a jump table in the .rodata 2184 * section which contains a list of addresses within the function to jump to. 2185 * This finds these jump tables and adds them to the insn->alts lists. 2186 */ 2187static int add_jump_table_alts(struct objtool_file *file) 2188{ 2189 struct symbol *func; 2190 2191 if (!file->rodata) 2192 return 0; 2193 2194 for_each_sym(file->elf, func) { 2195 if (!is_func_sym(func) || func->alias != func) 2196 continue; 2197 2198 mark_func_jump_tables(file, func); 2199 if (add_func_jump_tables(file, func)) 2200 return -1; 2201 } 2202 2203 return 0; 2204} 2205 2206static void set_func_state(struct cfi_state *state) 2207{ 2208 state->cfa = initial_func_cfi.cfa; 2209 memcpy(&state->regs, &initial_func_cfi.regs, 2210 CFI_NUM_REGS * sizeof(struct cfi_reg)); 2211 state->stack_size = initial_func_cfi.cfa.offset; 2212 state->type = UNWIND_HINT_TYPE_CALL; 2213} 2214 2215static int read_unwind_hints(struct objtool_file *file) 2216{ 2217 struct cfi_state cfi = init_cfi; 2218 struct section *sec; 2219 struct unwind_hint *hint; 2220 struct instruction *insn; 2221 struct reloc *reloc; 2222 unsigned long offset; 2223 int i; 2224 2225 sec = find_section_by_name(file->elf, ".discard.unwind_hints"); 2226 if (!sec) 2227 return 0; 2228 2229 if (!sec->rsec) { 2230 ERROR("missing .rela.discard.unwind_hints section"); 2231 return -1; 2232 } 2233 2234 if (sec_size(sec) % sizeof(struct unwind_hint)) { 2235 ERROR("struct unwind_hint size mismatch"); 2236 return -1; 2237 } 2238 2239 file->hints = true; 2240 2241 for (i = 0; i < sec_size(sec) / sizeof(struct unwind_hint); i++) { 2242 hint = (struct unwind_hint *)sec->data->d_buf + i; 2243 2244 reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint)); 2245 if (!reloc) { 2246 ERROR("can't find reloc for unwind_hints[%d]", i); 2247 return -1; 2248 } 2249 2250 offset = reloc->sym->offset + reloc_addend(reloc); 2251 2252 insn = find_insn(file, reloc->sym->sec, offset); 2253 if (!insn) { 2254 ERROR("can't find insn for unwind_hints[%d]", i); 2255 return -1; 2256 } 2257 2258 insn->hint = true; 2259 2260 if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) { 2261 insn->cfi = &force_undefined_cfi; 2262 continue; 2263 } 2264 2265 if (hint->type == UNWIND_HINT_TYPE_SAVE) { 2266 insn->hint = false; 2267 insn->save = true; 2268 continue; 2269 } 2270 2271 if (hint->type == UNWIND_HINT_TYPE_RESTORE) { 2272 insn->restore = true; 2273 continue; 2274 } 2275 2276 if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { 2277 struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); 2278 2279 if (sym && is_global_sym(sym)) { 2280 if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { 2281 ERROR_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); 2282 return -1; 2283 } 2284 } 2285 } 2286 2287 if (hint->type == UNWIND_HINT_TYPE_FUNC) { 2288 insn->cfi = &func_cfi; 2289 continue; 2290 } 2291 2292 if (insn->cfi) 2293 cfi = *(insn->cfi); 2294 2295 if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { 2296 ERROR_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); 2297 return -1; 2298 } 2299 2300 cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); 2301 cfi.type = hint->type; 2302 cfi.signal = hint->signal; 2303 2304 insn->cfi = cfi_hash_find_or_add(&cfi); 2305 } 2306 2307 return 0; 2308} 2309 2310static int read_annotate(struct objtool_file *file, 2311 int (*func)(struct objtool_file *file, int type, struct instruction *insn)) 2312{ 2313 struct section *sec; 2314 struct instruction *insn; 2315 struct reloc *reloc; 2316 uint64_t offset; 2317 int type; 2318 2319 sec = find_section_by_name(file->elf, ".discard.annotate_insn"); 2320 if (!sec) 2321 return 0; 2322 2323 if (!sec->rsec) 2324 return 0; 2325 2326 if (sec->sh.sh_entsize != 8) { 2327 static bool warned = false; 2328 if (!warned && opts.verbose) { 2329 WARN("%s: dodgy linker, sh_entsize != 8", sec->name); 2330 warned = true; 2331 } 2332 sec->sh.sh_entsize = 8; 2333 } 2334 2335 if (sec_num_entries(sec) != sec_num_entries(sec->rsec)) { 2336 ERROR("bad .discard.annotate_insn section: missing relocs"); 2337 return -1; 2338 } 2339 2340 for_each_reloc(sec->rsec, reloc) { 2341 type = annotype(file->elf, sec, reloc); 2342 offset = reloc->sym->offset + reloc_addend(reloc); 2343 insn = find_insn(file, reloc->sym->sec, offset); 2344 2345 if (!insn) { 2346 ERROR("bad .discard.annotate_insn entry: %d of type %d", reloc_idx(reloc), type); 2347 return -1; 2348 } 2349 2350 if (func(file, type, insn)) 2351 return -1; 2352 } 2353 2354 return 0; 2355} 2356 2357static int __annotate_early(struct objtool_file *file, int type, struct instruction *insn) 2358{ 2359 switch (type) { 2360 2361 /* Must be before add_special_section_alts() */ 2362 case ANNOTYPE_IGNORE_ALTS: 2363 insn->ignore_alts = true; 2364 break; 2365 2366 /* 2367 * Must be before read_unwind_hints() since that needs insn->noendbr. 2368 */ 2369 case ANNOTYPE_NOENDBR: 2370 insn->noendbr = 1; 2371 break; 2372 2373 default: 2374 break; 2375 } 2376 2377 return 0; 2378} 2379 2380static int __annotate_ifc(struct objtool_file *file, int type, struct instruction *insn) 2381{ 2382 unsigned long dest_off; 2383 2384 if (type != ANNOTYPE_INTRA_FUNCTION_CALL) 2385 return 0; 2386 2387 if (insn->type != INSN_CALL) { 2388 ERROR_INSN(insn, "intra_function_call not a direct call"); 2389 return -1; 2390 } 2391 2392 /* 2393 * Treat intra-function CALLs as JMPs, but with a stack_op. 2394 * See add_call_destinations(), which strips stack_ops from 2395 * normal CALLs. 2396 */ 2397 insn->type = INSN_JUMP_UNCONDITIONAL; 2398 2399 dest_off = arch_jump_destination(insn); 2400 insn->jump_dest = find_insn(file, insn->sec, dest_off); 2401 if (!insn->jump_dest) { 2402 ERROR_INSN(insn, "can't find call dest at %s+0x%lx", 2403 insn->sec->name, dest_off); 2404 return -1; 2405 } 2406 2407 return 0; 2408} 2409 2410static int __annotate_late(struct objtool_file *file, int type, struct instruction *insn) 2411{ 2412 struct symbol *sym; 2413 2414 switch (type) { 2415 case ANNOTYPE_NOENDBR: 2416 /* early */ 2417 break; 2418 2419 case ANNOTYPE_RETPOLINE_SAFE: 2420 if (insn->type != INSN_JUMP_DYNAMIC && 2421 insn->type != INSN_CALL_DYNAMIC && 2422 insn->type != INSN_RETURN && 2423 insn->type != INSN_NOP) { 2424 ERROR_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); 2425 return -1; 2426 } 2427 2428 insn->retpoline_safe = true; 2429 break; 2430 2431 case ANNOTYPE_INSTR_BEGIN: 2432 insn->instr++; 2433 break; 2434 2435 case ANNOTYPE_INSTR_END: 2436 insn->instr--; 2437 break; 2438 2439 case ANNOTYPE_UNRET_BEGIN: 2440 insn->unret = 1; 2441 break; 2442 2443 case ANNOTYPE_IGNORE_ALTS: 2444 /* early */ 2445 break; 2446 2447 case ANNOTYPE_INTRA_FUNCTION_CALL: 2448 /* ifc */ 2449 break; 2450 2451 case ANNOTYPE_REACHABLE: 2452 insn->dead_end = false; 2453 break; 2454 2455 case ANNOTYPE_NOCFI: 2456 sym = insn->sym; 2457 if (!sym) { 2458 ERROR_INSN(insn, "dodgy NOCFI annotation"); 2459 return -1; 2460 } 2461 insn->sym->nocfi = 1; 2462 break; 2463 2464 default: 2465 ERROR_INSN(insn, "Unknown annotation type: %d", type); 2466 return -1; 2467 } 2468 2469 return 0; 2470} 2471 2472/* 2473 * Return true if name matches an instrumentation function, where calls to that 2474 * function from noinstr code can safely be removed, but compilers won't do so. 2475 */ 2476static bool is_profiling_func(const char *name) 2477{ 2478 /* 2479 * Many compilers cannot disable KCOV with a function attribute. 2480 */ 2481 if (!strncmp(name, "__sanitizer_cov_", 16)) 2482 return true; 2483 2484 return false; 2485} 2486 2487static int classify_symbols(struct objtool_file *file) 2488{ 2489 struct symbol *func; 2490 size_t len; 2491 2492 for_each_sym(file->elf, func) { 2493 if (is_notype_sym(func) && strstarts(func->name, ".L")) 2494 func->local_label = true; 2495 2496 if (!is_global_sym(func)) 2497 continue; 2498 2499 if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, 2500 strlen(STATIC_CALL_TRAMP_PREFIX_STR))) 2501 func->static_call_tramp = true; 2502 2503 if (arch_is_retpoline(func)) 2504 func->retpoline_thunk = true; 2505 2506 if (arch_is_rethunk(func)) 2507 func->return_thunk = true; 2508 2509 if (arch_is_embedded_insn(func)) 2510 func->embedded_insn = true; 2511 2512 if (arch_ftrace_match(func->name)) 2513 func->fentry = true; 2514 2515 if (is_profiling_func(func->name)) 2516 func->profiling_func = true; 2517 2518 len = strlen(func->name); 2519 if (len > sym_name_max_len) 2520 sym_name_max_len = len; 2521 } 2522 2523 return 0; 2524} 2525 2526static void mark_rodata(struct objtool_file *file) 2527{ 2528 struct section *sec; 2529 bool found = false; 2530 2531 /* 2532 * Search for the following rodata sections, each of which can 2533 * potentially contain jump tables: 2534 * 2535 * - .rodata: can contain GCC switch tables 2536 * - .rodata.<func>: same, if -fdata-sections is being used 2537 * - .data.rel.ro.c_jump_table: contains C annotated jump tables 2538 * 2539 * .rodata.str1.* sections are ignored; they don't contain jump tables. 2540 */ 2541 for_each_sec(file->elf, sec) { 2542 if ((!strncmp(sec->name, ".rodata", 7) && 2543 !strstr(sec->name, ".str1.")) || 2544 !strncmp(sec->name, ".data.rel.ro", 12)) { 2545 sec->rodata = true; 2546 found = true; 2547 } 2548 } 2549 2550 file->rodata = found; 2551} 2552 2553static void mark_holes(struct objtool_file *file) 2554{ 2555 struct instruction *insn; 2556 bool in_hole = false; 2557 2558 if (!opts.link) 2559 return; 2560 2561 /* 2562 * Whole archive runs might encounter dead code from weak symbols. 2563 * This is where the linker will have dropped the weak symbol in 2564 * favour of a regular symbol, but leaves the code in place. 2565 */ 2566 for_each_insn(file, insn) { 2567 if (insn->sym || !find_symbol_hole_containing(insn->sec, insn->offset)) { 2568 in_hole = false; 2569 continue; 2570 } 2571 2572 /* Skip function padding and pfx code */ 2573 if (!in_hole && insn->type == INSN_NOP) 2574 continue; 2575 2576 in_hole = true; 2577 insn->hole = 1; 2578 2579 /* 2580 * If this hole jumps to a .cold function, mark it ignore. 2581 */ 2582 if (insn->jump_dest) { 2583 struct symbol *dest_func = insn_func(insn->jump_dest); 2584 2585 if (dest_func && dest_func->cold) 2586 dest_func->ignore = true; 2587 } 2588 } 2589} 2590 2591static bool validate_branch_enabled(void) 2592{ 2593 return opts.stackval || 2594 opts.orc || 2595 opts.uaccess || 2596 opts.checksum; 2597} 2598 2599static int decode_sections(struct objtool_file *file) 2600{ 2601 file->klp = is_livepatch_module(file); 2602 2603 mark_rodata(file); 2604 2605 if (init_pv_ops(file)) 2606 return -1; 2607 2608 /* 2609 * Must be before add_{jump_call}_destination. 2610 */ 2611 if (classify_symbols(file)) 2612 return -1; 2613 2614 if (decode_instructions(file)) 2615 return -1; 2616 2617 if (add_ignores(file)) 2618 return -1; 2619 2620 add_uaccess_safe(file); 2621 2622 if (read_annotate(file, __annotate_early)) 2623 return -1; 2624 2625 /* 2626 * Must be before add_jump_destinations(), which depends on 'func' 2627 * being set for alternatives, to enable proper sibling call detection. 2628 */ 2629 if (validate_branch_enabled() || opts.noinstr || opts.hack_jump_label || opts.disas) { 2630 if (add_special_section_alts(file)) 2631 return -1; 2632 } 2633 2634 if (add_jump_destinations(file)) 2635 return -1; 2636 2637 /* 2638 * Must be before add_call_destination(); it changes INSN_CALL to 2639 * INSN_JUMP. 2640 */ 2641 if (read_annotate(file, __annotate_ifc)) 2642 return -1; 2643 2644 if (add_call_destinations(file)) 2645 return -1; 2646 2647 if (add_jump_table_alts(file)) 2648 return -1; 2649 2650 if (read_unwind_hints(file)) 2651 return -1; 2652 2653 /* Must be after add_jump_destinations() */ 2654 mark_holes(file); 2655 2656 /* 2657 * Must be after add_call_destinations() such that it can override 2658 * dead_end_function() marks. 2659 */ 2660 if (read_annotate(file, __annotate_late)) 2661 return -1; 2662 2663 return 0; 2664} 2665 2666static bool is_special_call(struct instruction *insn) 2667{ 2668 if (insn->type == INSN_CALL) { 2669 struct symbol *dest = insn_call_dest(insn); 2670 2671 if (!dest) 2672 return false; 2673 2674 if (dest->fentry || dest->embedded_insn) 2675 return true; 2676 } 2677 2678 return false; 2679} 2680 2681static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state) 2682{ 2683 struct cfi_state *cfi = &state->cfi; 2684 int i; 2685 2686 if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap) 2687 return true; 2688 2689 if (cfi->cfa.offset != initial_func_cfi.cfa.offset) 2690 return true; 2691 2692 if (cfi->stack_size != initial_func_cfi.cfa.offset) 2693 return true; 2694 2695 for (i = 0; i < CFI_NUM_REGS; i++) { 2696 if (cfi->regs[i].base != initial_func_cfi.regs[i].base || 2697 cfi->regs[i].offset != initial_func_cfi.regs[i].offset) 2698 return true; 2699 } 2700 2701 return false; 2702} 2703 2704static bool check_reg_frame_pos(const struct cfi_reg *reg, 2705 int expected_offset) 2706{ 2707 return reg->base == CFI_CFA && 2708 reg->offset == expected_offset; 2709} 2710 2711static bool has_valid_stack_frame(struct insn_state *state) 2712{ 2713 struct cfi_state *cfi = &state->cfi; 2714 2715 if (cfi->cfa.base == CFI_BP && 2716 check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) && 2717 check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8)) 2718 return true; 2719 2720 if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP) 2721 return true; 2722 2723 return false; 2724} 2725 2726static int update_cfi_state_regs(struct instruction *insn, 2727 struct cfi_state *cfi, 2728 struct stack_op *op) 2729{ 2730 struct cfi_reg *cfa = &cfi->cfa; 2731 2732 if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT) 2733 return 0; 2734 2735 /* push */ 2736 if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF) 2737 cfa->offset += 8; 2738 2739 /* pop */ 2740 if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF) 2741 cfa->offset -= 8; 2742 2743 /* add immediate to sp */ 2744 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD && 2745 op->dest.reg == CFI_SP && op->src.reg == CFI_SP) 2746 cfa->offset -= op->src.offset; 2747 2748 return 0; 2749} 2750 2751static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset) 2752{ 2753 if (arch_callee_saved_reg(reg) && 2754 cfi->regs[reg].base == CFI_UNDEFINED) { 2755 cfi->regs[reg].base = base; 2756 cfi->regs[reg].offset = offset; 2757 } 2758} 2759 2760static void restore_reg(struct cfi_state *cfi, unsigned char reg) 2761{ 2762 cfi->regs[reg].base = initial_func_cfi.regs[reg].base; 2763 cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset; 2764} 2765 2766/* 2767 * A note about DRAP stack alignment: 2768 * 2769 * GCC has the concept of a DRAP register, which is used to help keep track of 2770 * the stack pointer when aligning the stack. r10 or r13 is used as the DRAP 2771 * register. The typical DRAP pattern is: 2772 * 2773 * 4c 8d 54 24 08 lea 0x8(%rsp),%r10 2774 * 48 83 e4 c0 and $0xffffffffffffffc0,%rsp 2775 * 41 ff 72 f8 pushq -0x8(%r10) 2776 * 55 push %rbp 2777 * 48 89 e5 mov %rsp,%rbp 2778 * (more pushes) 2779 * 41 52 push %r10 2780 * ... 2781 * 41 5a pop %r10 2782 * (more pops) 2783 * 5d pop %rbp 2784 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2785 * c3 retq 2786 * 2787 * There are some variations in the epilogues, like: 2788 * 2789 * 5b pop %rbx 2790 * 41 5a pop %r10 2791 * 41 5c pop %r12 2792 * 41 5d pop %r13 2793 * 41 5e pop %r14 2794 * c9 leaveq 2795 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2796 * c3 retq 2797 * 2798 * and: 2799 * 2800 * 4c 8b 55 e8 mov -0x18(%rbp),%r10 2801 * 48 8b 5d e0 mov -0x20(%rbp),%rbx 2802 * 4c 8b 65 f0 mov -0x10(%rbp),%r12 2803 * 4c 8b 6d f8 mov -0x8(%rbp),%r13 2804 * c9 leaveq 2805 * 49 8d 62 f8 lea -0x8(%r10),%rsp 2806 * c3 retq 2807 * 2808 * Sometimes r13 is used as the DRAP register, in which case it's saved and 2809 * restored beforehand: 2810 * 2811 * 41 55 push %r13 2812 * 4c 8d 6c 24 10 lea 0x10(%rsp),%r13 2813 * 48 83 e4 f0 and $0xfffffffffffffff0,%rsp 2814 * ... 2815 * 49 8d 65 f0 lea -0x10(%r13),%rsp 2816 * 41 5d pop %r13 2817 * c3 retq 2818 */ 2819static int update_cfi_state(struct instruction *insn, 2820 struct instruction *next_insn, 2821 struct cfi_state *cfi, struct stack_op *op) 2822{ 2823 struct cfi_reg *cfa = &cfi->cfa; 2824 struct cfi_reg *regs = cfi->regs; 2825 2826 /* ignore UNWIND_HINT_UNDEFINED regions */ 2827 if (cfi->force_undefined) 2828 return 0; 2829 2830 /* stack operations don't make sense with an undefined CFA */ 2831 if (cfa->base == CFI_UNDEFINED) { 2832 if (insn_func(insn)) { 2833 WARN_INSN(insn, "undefined stack state"); 2834 return 1; 2835 } 2836 return 0; 2837 } 2838 2839 if (cfi->type == UNWIND_HINT_TYPE_REGS || 2840 cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL) 2841 return update_cfi_state_regs(insn, cfi, op); 2842 2843 switch (op->dest.type) { 2844 2845 case OP_DEST_REG: 2846 switch (op->src.type) { 2847 2848 case OP_SRC_REG: 2849 if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP && 2850 cfa->base == CFI_SP && 2851 check_reg_frame_pos(®s[CFI_BP], -cfa->offset)) { 2852 2853 /* mov %rsp, %rbp */ 2854 cfa->base = op->dest.reg; 2855 cfi->bp_scratch = false; 2856 } 2857 2858 else if (op->src.reg == CFI_SP && 2859 op->dest.reg == CFI_BP && cfi->drap) { 2860 2861 /* drap: mov %rsp, %rbp */ 2862 regs[CFI_BP].base = CFI_BP; 2863 regs[CFI_BP].offset = -cfi->stack_size; 2864 cfi->bp_scratch = false; 2865 } 2866 2867 else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 2868 2869 /* 2870 * mov %rsp, %reg 2871 * 2872 * This is needed for the rare case where GCC 2873 * does: 2874 * 2875 * mov %rsp, %rax 2876 * ... 2877 * mov %rax, %rsp 2878 */ 2879 cfi->vals[op->dest.reg].base = CFI_CFA; 2880 cfi->vals[op->dest.reg].offset = -cfi->stack_size; 2881 } 2882 2883 else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP && 2884 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) { 2885 2886 /* 2887 * mov %rbp, %rsp 2888 * 2889 * Restore the original stack pointer (Clang). 2890 */ 2891 cfi->stack_size = -cfi->regs[CFI_BP].offset; 2892 } 2893 2894 else if (op->dest.reg == cfa->base) { 2895 2896 /* mov %reg, %rsp */ 2897 if (cfa->base == CFI_SP && 2898 cfi->vals[op->src.reg].base == CFI_CFA) { 2899 2900 /* 2901 * This is needed for the rare case 2902 * where GCC does something dumb like: 2903 * 2904 * lea 0x8(%rsp), %rcx 2905 * ... 2906 * mov %rcx, %rsp 2907 */ 2908 cfa->offset = -cfi->vals[op->src.reg].offset; 2909 cfi->stack_size = cfa->offset; 2910 2911 } else if (cfa->base == CFI_SP && 2912 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2913 cfi->vals[op->src.reg].offset == cfa->offset) { 2914 2915 /* 2916 * Stack swizzle: 2917 * 2918 * 1: mov %rsp, (%[tos]) 2919 * 2: mov %[tos], %rsp 2920 * ... 2921 * 3: pop %rsp 2922 * 2923 * Where: 2924 * 2925 * 1 - places a pointer to the previous 2926 * stack at the Top-of-Stack of the 2927 * new stack. 2928 * 2929 * 2 - switches to the new stack. 2930 * 2931 * 3 - pops the Top-of-Stack to restore 2932 * the original stack. 2933 * 2934 * Note: we set base to SP_INDIRECT 2935 * here and preserve offset. Therefore 2936 * when the unwinder reaches ToS it 2937 * will dereference SP and then add the 2938 * offset to find the next frame, IOW: 2939 * (%rsp) + offset. 2940 */ 2941 cfa->base = CFI_SP_INDIRECT; 2942 2943 } else { 2944 cfa->base = CFI_UNDEFINED; 2945 cfa->offset = 0; 2946 } 2947 } 2948 2949 else if (op->dest.reg == CFI_SP && 2950 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT && 2951 cfi->vals[op->src.reg].offset == cfa->offset) { 2952 2953 /* 2954 * The same stack swizzle case 2) as above. But 2955 * because we can't change cfa->base, case 3) 2956 * will become a regular POP. Pretend we're a 2957 * PUSH so things don't go unbalanced. 2958 */ 2959 cfi->stack_size += 8; 2960 } 2961 2962 else if (cfi->vals[op->src.reg].base == CFI_CFA) { 2963 /* 2964 * Clang RSP musical chairs: 2965 * 2966 * mov %rsp, %rdx [handled above] 2967 * ... 2968 * mov %rdx, %rbx [handled here] 2969 * ... 2970 * mov %rbx, %rsp [handled above] 2971 */ 2972 cfi->vals[op->dest.reg].base = CFI_CFA; 2973 cfi->vals[op->dest.reg].offset = cfi->vals[op->src.reg].offset; 2974 } 2975 2976 2977 break; 2978 2979 case OP_SRC_ADD: 2980 if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) { 2981 2982 /* add imm, %rsp */ 2983 cfi->stack_size -= op->src.offset; 2984 if (cfa->base == CFI_SP) 2985 cfa->offset -= op->src.offset; 2986 break; 2987 } 2988 2989 if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && 2990 insn->sym->frame_pointer) { 2991 /* addi.d fp,sp,imm on LoongArch */ 2992 if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { 2993 cfa->base = CFI_BP; 2994 cfa->offset = 0; 2995 } 2996 break; 2997 } 2998 2999 if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { 3000 /* addi.d sp,fp,imm on LoongArch */ 3001 if (cfa->base == CFI_BP && cfa->offset == 0) { 3002 if (insn->sym->frame_pointer) { 3003 cfa->base = CFI_SP; 3004 cfa->offset = -op->src.offset; 3005 } 3006 } else { 3007 /* lea disp(%rbp), %rsp */ 3008 cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); 3009 } 3010 break; 3011 } 3012 3013 if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { 3014 3015 /* drap: lea disp(%rsp), %drap */ 3016 cfi->drap_reg = op->dest.reg; 3017 3018 /* 3019 * lea disp(%rsp), %reg 3020 * 3021 * This is needed for the rare case where GCC 3022 * does something dumb like: 3023 * 3024 * lea 0x8(%rsp), %rcx 3025 * ... 3026 * mov %rcx, %rsp 3027 */ 3028 cfi->vals[op->dest.reg].base = CFI_CFA; 3029 cfi->vals[op->dest.reg].offset = \ 3030 -cfi->stack_size + op->src.offset; 3031 3032 break; 3033 } 3034 3035 if (cfi->drap && op->dest.reg == CFI_SP && 3036 op->src.reg == cfi->drap_reg) { 3037 3038 /* drap: lea disp(%drap), %rsp */ 3039 cfa->base = CFI_SP; 3040 cfa->offset = cfi->stack_size = -op->src.offset; 3041 cfi->drap_reg = CFI_UNDEFINED; 3042 cfi->drap = false; 3043 break; 3044 } 3045 3046 if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { 3047 WARN_INSN(insn, "unsupported stack register modification"); 3048 return -1; 3049 } 3050 3051 break; 3052 3053 case OP_SRC_AND: 3054 if (op->dest.reg != CFI_SP || 3055 (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || 3056 (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { 3057 WARN_INSN(insn, "unsupported stack pointer realignment"); 3058 return -1; 3059 } 3060 3061 if (cfi->drap_reg != CFI_UNDEFINED) { 3062 /* drap: and imm, %rsp */ 3063 cfa->base = cfi->drap_reg; 3064 cfa->offset = cfi->stack_size = 0; 3065 cfi->drap = true; 3066 } 3067 3068 /* 3069 * Older versions of GCC (4.8ish) realign the stack 3070 * without DRAP, with a frame pointer. 3071 */ 3072 3073 break; 3074 3075 case OP_SRC_POP: 3076 case OP_SRC_POPF: 3077 if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) { 3078 3079 /* pop %rsp; # restore from a stack swizzle */ 3080 cfa->base = CFI_SP; 3081 break; 3082 } 3083 3084 if (!cfi->drap && op->dest.reg == cfa->base) { 3085 3086 /* pop %rbp */ 3087 cfa->base = CFI_SP; 3088 } 3089 3090 if (cfi->drap && cfa->base == CFI_BP_INDIRECT && 3091 op->dest.reg == cfi->drap_reg && 3092 cfi->drap_offset == -cfi->stack_size) { 3093 3094 /* drap: pop %drap */ 3095 cfa->base = cfi->drap_reg; 3096 cfa->offset = 0; 3097 cfi->drap_offset = -1; 3098 3099 } else if (cfi->stack_size == -regs[op->dest.reg].offset) { 3100 3101 /* pop %reg */ 3102 restore_reg(cfi, op->dest.reg); 3103 } 3104 3105 cfi->stack_size -= 8; 3106 if (cfa->base == CFI_SP) 3107 cfa->offset -= 8; 3108 3109 break; 3110 3111 case OP_SRC_REG_INDIRECT: 3112 if (!cfi->drap && op->dest.reg == cfa->base && 3113 op->dest.reg == CFI_BP) { 3114 3115 /* mov disp(%rsp), %rbp */ 3116 cfa->base = CFI_SP; 3117 cfa->offset = cfi->stack_size; 3118 } 3119 3120 if (cfi->drap && op->src.reg == CFI_BP && 3121 op->src.offset == cfi->drap_offset) { 3122 3123 /* drap: mov disp(%rbp), %drap */ 3124 cfa->base = cfi->drap_reg; 3125 cfa->offset = 0; 3126 cfi->drap_offset = -1; 3127 } 3128 3129 if (cfi->drap && op->src.reg == CFI_BP && 3130 op->src.offset == regs[op->dest.reg].offset) { 3131 3132 /* drap: mov disp(%rbp), %reg */ 3133 restore_reg(cfi, op->dest.reg); 3134 3135 } else if (op->src.reg == cfa->base && 3136 op->src.offset == regs[op->dest.reg].offset + cfa->offset) { 3137 3138 /* mov disp(%rbp), %reg */ 3139 /* mov disp(%rsp), %reg */ 3140 restore_reg(cfi, op->dest.reg); 3141 3142 } else if (op->src.reg == CFI_SP && 3143 op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) { 3144 3145 /* mov disp(%rsp), %reg */ 3146 restore_reg(cfi, op->dest.reg); 3147 } 3148 3149 break; 3150 3151 default: 3152 WARN_INSN(insn, "unknown stack-related instruction"); 3153 return -1; 3154 } 3155 3156 break; 3157 3158 case OP_DEST_PUSH: 3159 case OP_DEST_PUSHF: 3160 cfi->stack_size += 8; 3161 if (cfa->base == CFI_SP) 3162 cfa->offset += 8; 3163 3164 if (op->src.type != OP_SRC_REG) 3165 break; 3166 3167 if (cfi->drap) { 3168 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3169 3170 /* drap: push %drap */ 3171 cfa->base = CFI_BP_INDIRECT; 3172 cfa->offset = -cfi->stack_size; 3173 3174 /* save drap so we know when to restore it */ 3175 cfi->drap_offset = -cfi->stack_size; 3176 3177 } else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) { 3178 3179 /* drap: push %rbp */ 3180 cfi->stack_size = 0; 3181 3182 } else { 3183 3184 /* drap: push %reg */ 3185 save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size); 3186 } 3187 3188 } else { 3189 3190 /* push %reg */ 3191 save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size); 3192 } 3193 3194 /* detect when asm code uses rbp as a scratch register */ 3195 if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP && 3196 cfa->base != CFI_BP) 3197 cfi->bp_scratch = true; 3198 break; 3199 3200 case OP_DEST_REG_INDIRECT: 3201 3202 if (cfi->drap) { 3203 if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) { 3204 3205 /* drap: mov %drap, disp(%rbp) */ 3206 cfa->base = CFI_BP_INDIRECT; 3207 cfa->offset = op->dest.offset; 3208 3209 /* save drap offset so we know when to restore it */ 3210 cfi->drap_offset = op->dest.offset; 3211 } else { 3212 3213 /* drap: mov reg, disp(%rbp) */ 3214 save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset); 3215 } 3216 3217 } else if (op->dest.reg == cfa->base) { 3218 3219 /* mov reg, disp(%rbp) */ 3220 /* mov reg, disp(%rsp) */ 3221 save_reg(cfi, op->src.reg, CFI_CFA, 3222 op->dest.offset - cfi->cfa.offset); 3223 3224 } else if (op->dest.reg == CFI_SP) { 3225 3226 /* mov reg, disp(%rsp) */ 3227 save_reg(cfi, op->src.reg, CFI_CFA, 3228 op->dest.offset - cfi->stack_size); 3229 3230 } else if (op->src.reg == CFI_SP && op->dest.offset == 0) { 3231 3232 /* mov %rsp, (%reg); # setup a stack swizzle. */ 3233 cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT; 3234 cfi->vals[op->dest.reg].offset = cfa->offset; 3235 } 3236 3237 break; 3238 3239 case OP_DEST_MEM: 3240 if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { 3241 WARN_INSN(insn, "unknown stack-related memory operation"); 3242 return -1; 3243 } 3244 3245 /* pop mem */ 3246 cfi->stack_size -= 8; 3247 if (cfa->base == CFI_SP) 3248 cfa->offset -= 8; 3249 3250 break; 3251 3252 default: 3253 WARN_INSN(insn, "unknown stack-related instruction"); 3254 return -1; 3255 } 3256 3257 return 0; 3258} 3259 3260/* 3261 * The stack layouts of alternatives instructions can sometimes diverge when 3262 * they have stack modifications. That's fine as long as the potential stack 3263 * layouts don't conflict at any given potential instruction boundary. 3264 * 3265 * Flatten the CFIs of the different alternative code streams (both original 3266 * and replacement) into a single shared CFI array which can be used to detect 3267 * conflicts and nicely feed a linear array of ORC entries to the unwinder. 3268 */ 3269static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn) 3270{ 3271 struct cfi_state **alt_cfi; 3272 int group_off; 3273 3274 if (!insn->alt_group) 3275 return 0; 3276 3277 if (!insn->cfi) { 3278 WARN("CFI missing"); 3279 return -1; 3280 } 3281 3282 alt_cfi = insn->alt_group->cfi; 3283 group_off = insn->offset - insn->alt_group->first_insn->offset; 3284 3285 if (!alt_cfi[group_off]) { 3286 alt_cfi[group_off] = insn->cfi; 3287 } else { 3288 if (cficmp(alt_cfi[group_off], insn->cfi)) { 3289 struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; 3290 struct instruction *orig = orig_group->first_insn; 3291 WARN_INSN(orig, "stack layout conflict in alternatives: %s", 3292 offstr(insn->sec, insn->offset)); 3293 return -1; 3294 } 3295 } 3296 3297 return 0; 3298} 3299 3300static int noinline handle_insn_ops(struct instruction *insn, 3301 struct instruction *next_insn, 3302 struct insn_state *state) 3303{ 3304 struct insn_state prev_state __maybe_unused = *state; 3305 struct stack_op *op; 3306 int ret = 0; 3307 3308 for (op = insn->stack_ops; op; op = op->next) { 3309 3310 ret = update_cfi_state(insn, next_insn, &state->cfi, op); 3311 if (ret) 3312 goto done; 3313 3314 if (!opts.uaccess || !insn->alt_group) 3315 continue; 3316 3317 if (op->dest.type == OP_DEST_PUSHF) { 3318 if (!state->uaccess_stack) { 3319 state->uaccess_stack = 1; 3320 } else if (state->uaccess_stack >> 31) { 3321 WARN_INSN(insn, "PUSHF stack exhausted"); 3322 ret = 1; 3323 goto done; 3324 } 3325 state->uaccess_stack <<= 1; 3326 state->uaccess_stack |= state->uaccess; 3327 } 3328 3329 if (op->src.type == OP_SRC_POPF) { 3330 if (state->uaccess_stack) { 3331 state->uaccess = state->uaccess_stack & 1; 3332 state->uaccess_stack >>= 1; 3333 if (state->uaccess_stack == 1) 3334 state->uaccess_stack = 0; 3335 } 3336 } 3337 } 3338 3339done: 3340 TRACE_INSN_STATE(insn, &prev_state, state); 3341 3342 return ret; 3343} 3344 3345static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) 3346{ 3347 struct cfi_state *cfi1 = insn->cfi; 3348 int i; 3349 3350 if (!cfi1) { 3351 WARN("CFI missing"); 3352 return false; 3353 } 3354 3355 if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { 3356 3357 WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", 3358 cfi1->cfa.base, cfi1->cfa.offset, 3359 cfi2->cfa.base, cfi2->cfa.offset); 3360 return false; 3361 3362 } 3363 3364 if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) { 3365 for (i = 0; i < CFI_NUM_REGS; i++) { 3366 3367 if (!memcmp(&cfi1->regs[i], &cfi2->regs[i], sizeof(struct cfi_reg))) 3368 continue; 3369 3370 WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", 3371 i, cfi1->regs[i].base, cfi1->regs[i].offset, 3372 i, cfi2->regs[i].base, cfi2->regs[i].offset); 3373 } 3374 return false; 3375 } 3376 3377 if (cfi1->type != cfi2->type) { 3378 3379 WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", 3380 cfi1->type, cfi2->type); 3381 return false; 3382 } 3383 3384 if (cfi1->drap != cfi2->drap || 3385 (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || 3386 (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { 3387 3388 WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", 3389 cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, 3390 cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); 3391 return false; 3392 } 3393 3394 return true; 3395} 3396 3397static inline bool func_uaccess_safe(struct symbol *func) 3398{ 3399 if (func) 3400 return func->uaccess_safe; 3401 3402 return false; 3403} 3404 3405static inline const char *call_dest_name(struct instruction *insn) 3406{ 3407 static char pvname[19]; 3408 struct reloc *reloc; 3409 int idx; 3410 3411 if (insn_call_dest(insn)) 3412 return insn_call_dest(insn)->name; 3413 3414 reloc = insn_reloc(NULL, insn); 3415 if (reloc && !strcmp(reloc->sym->name, "pv_ops")) { 3416 idx = (reloc_addend(reloc) / sizeof(void *)); 3417 snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx); 3418 return pvname; 3419 } 3420 3421 return "{dynamic}"; 3422} 3423 3424static bool pv_call_dest(struct objtool_file *file, struct instruction *insn) 3425{ 3426 struct symbol *target; 3427 struct reloc *reloc; 3428 int idx; 3429 3430 reloc = insn_reloc(file, insn); 3431 if (!reloc || strcmp(reloc->sym->name, "pv_ops")) 3432 return false; 3433 3434 idx = arch_insn_adjusted_addend(insn, reloc) / sizeof(void *); 3435 3436 if (file->pv_ops[idx].clean) 3437 return true; 3438 3439 file->pv_ops[idx].clean = true; 3440 3441 list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) { 3442 if (!target->sec->noinstr) { 3443 WARN("pv_ops[%d]: %s", idx, target->name); 3444 file->pv_ops[idx].clean = false; 3445 } 3446 } 3447 3448 return file->pv_ops[idx].clean; 3449} 3450 3451static inline bool noinstr_call_dest(struct objtool_file *file, 3452 struct instruction *insn, 3453 struct symbol *func) 3454{ 3455 /* 3456 * We can't deal with indirect function calls at present; 3457 * assume they're instrumented. 3458 */ 3459 if (!func) { 3460 if (file->pv_ops) 3461 return pv_call_dest(file, insn); 3462 3463 return false; 3464 } 3465 3466 /* 3467 * If the symbol is from a noinstr section; we good. 3468 */ 3469 if (func->sec->noinstr) 3470 return true; 3471 3472 /* 3473 * If the symbol is a static_call trampoline, we can't tell. 3474 */ 3475 if (func->static_call_tramp) 3476 return true; 3477 3478 /* 3479 * The __ubsan_handle_*() calls are like WARN(), they only happen when 3480 * something 'BAD' happened. At the risk of taking the machine down, 3481 * let them proceed to get the message out. 3482 */ 3483 if (!strncmp(func->name, "__ubsan_handle_", 15)) 3484 return true; 3485 3486 return false; 3487} 3488 3489static int validate_call(struct objtool_file *file, 3490 struct instruction *insn, 3491 struct insn_state *state) 3492{ 3493 if (state->noinstr && state->instr <= 0 && 3494 !noinstr_call_dest(file, insn, insn_call_dest(insn))) { 3495 WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); 3496 return 1; 3497 } 3498 3499 if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { 3500 WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); 3501 return 1; 3502 } 3503 3504 if (state->df) { 3505 WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); 3506 return 1; 3507 } 3508 3509 return 0; 3510} 3511 3512static int validate_sibling_call(struct objtool_file *file, 3513 struct instruction *insn, 3514 struct insn_state *state) 3515{ 3516 if (insn_func(insn) && has_modified_stack_frame(insn, state)) { 3517 WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); 3518 return 1; 3519 } 3520 3521 return validate_call(file, insn, state); 3522} 3523 3524static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) 3525{ 3526 if (state->noinstr && state->instr > 0) { 3527 WARN_INSN(insn, "return with instrumentation enabled"); 3528 return 1; 3529 } 3530 3531 if (state->uaccess && !func_uaccess_safe(func)) { 3532 WARN_INSN(insn, "return with UACCESS enabled"); 3533 return 1; 3534 } 3535 3536 if (!state->uaccess && func_uaccess_safe(func)) { 3537 WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); 3538 return 1; 3539 } 3540 3541 if (state->df) { 3542 WARN_INSN(insn, "return with DF set"); 3543 return 1; 3544 } 3545 3546 if (func && has_modified_stack_frame(insn, state)) { 3547 WARN_INSN(insn, "return with modified stack frame"); 3548 return 1; 3549 } 3550 3551 if (state->cfi.bp_scratch) { 3552 WARN_INSN(insn, "BP used as a scratch register"); 3553 return 1; 3554 } 3555 3556 return 0; 3557} 3558 3559static struct instruction *next_insn_to_validate(struct objtool_file *file, 3560 struct instruction *insn) 3561{ 3562 struct alt_group *alt_group = insn->alt_group; 3563 3564 /* 3565 * Simulate the fact that alternatives are patched in-place. When the 3566 * end of a replacement alt_group is reached, redirect objtool flow to 3567 * the end of the original alt_group. 3568 * 3569 * insn->alts->insn -> alt_group->first_insn 3570 * ... 3571 * alt_group->last_insn 3572 * [alt_group->nop] -> next(orig_group->last_insn) 3573 */ 3574 if (alt_group) { 3575 if (alt_group->nop) { 3576 /* ->nop implies ->orig_group */ 3577 if (insn == alt_group->last_insn) 3578 return alt_group->nop; 3579 if (insn == alt_group->nop) 3580 goto next_orig; 3581 } 3582 if (insn == alt_group->last_insn && alt_group->orig_group) 3583 goto next_orig; 3584 } 3585 3586 return next_insn_same_sec(file, insn); 3587 3588next_orig: 3589 return next_insn_same_sec(file, alt_group->orig_group->last_insn); 3590} 3591 3592static bool skip_alt_group(struct instruction *insn) 3593{ 3594 struct instruction *alt_insn = insn->alts ? insn->alts->insn : NULL; 3595 3596 if (!insn->alt_group) 3597 return false; 3598 3599 /* ANNOTATE_IGNORE_ALTERNATIVE */ 3600 if (insn->alt_group->ignore) { 3601 TRACE_ALT(insn, "alt group ignored"); 3602 return true; 3603 } 3604 3605 /* 3606 * For NOP patched with CLAC/STAC, only follow the latter to avoid 3607 * impossible code paths combining patched CLAC with unpatched STAC 3608 * or vice versa. 3609 * 3610 * ANNOTATE_IGNORE_ALTERNATIVE could have been used here, but Linus 3611 * requested not to do that to avoid hurting .s file readability 3612 * around CLAC/STAC alternative sites. 3613 */ 3614 3615 if (!alt_insn) 3616 return false; 3617 3618 /* Don't override ASM_{CLAC,STAC}_UNSAFE */ 3619 if (alt_insn->alt_group && alt_insn->alt_group->ignore) 3620 return false; 3621 3622 return alt_insn->type == INSN_CLAC || alt_insn->type == INSN_STAC; 3623} 3624 3625static int checksum_debug_init(struct objtool_file *file) 3626{ 3627 char *dup, *s; 3628 3629 if (!opts.debug_checksum) 3630 return 0; 3631 3632 dup = strdup(opts.debug_checksum); 3633 if (!dup) { 3634 ERROR_GLIBC("strdup"); 3635 return -1; 3636 } 3637 3638 s = dup; 3639 while (*s) { 3640 struct symbol *func; 3641 char *comma; 3642 3643 comma = strchr(s, ','); 3644 if (comma) 3645 *comma = '\0'; 3646 3647 func = find_symbol_by_name(file->elf, s); 3648 if (!func || !is_func_sym(func)) 3649 WARN("--debug-checksum: can't find '%s'", s); 3650 else 3651 func->debug_checksum = 1; 3652 3653 if (!comma) 3654 break; 3655 3656 s = comma + 1; 3657 } 3658 3659 free(dup); 3660 return 0; 3661} 3662 3663static void checksum_update_insn(struct objtool_file *file, struct symbol *func, 3664 struct instruction *insn) 3665{ 3666 struct reloc *reloc = insn_reloc(file, insn); 3667 unsigned long offset; 3668 struct symbol *sym; 3669 3670 if (insn->fake) 3671 return; 3672 3673 checksum_update(func, insn, insn->sec->data->d_buf + insn->offset, insn->len); 3674 3675 if (!reloc) { 3676 struct symbol *call_dest = insn_call_dest(insn); 3677 3678 if (call_dest) 3679 checksum_update(func, insn, call_dest->demangled_name, 3680 strlen(call_dest->demangled_name)); 3681 return; 3682 } 3683 3684 sym = reloc->sym; 3685 offset = arch_insn_adjusted_addend(insn, reloc); 3686 3687 if (is_string_sec(sym->sec)) { 3688 char *str; 3689 3690 str = sym->sec->data->d_buf + sym->offset + offset; 3691 checksum_update(func, insn, str, strlen(str)); 3692 return; 3693 } 3694 3695 if (is_sec_sym(sym)) { 3696 sym = find_symbol_containing(reloc->sym->sec, offset); 3697 if (!sym) 3698 return; 3699 3700 offset -= sym->offset; 3701 } 3702 3703 checksum_update(func, insn, sym->demangled_name, strlen(sym->demangled_name)); 3704 checksum_update(func, insn, &offset, sizeof(offset)); 3705} 3706 3707static int validate_branch(struct objtool_file *file, struct symbol *func, 3708 struct instruction *insn, struct insn_state state); 3709static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3710 struct instruction *insn, struct insn_state *state); 3711 3712static int validate_insn(struct objtool_file *file, struct symbol *func, 3713 struct instruction *insn, struct insn_state *statep, 3714 struct instruction *prev_insn, struct instruction *next_insn, 3715 bool *dead_end) 3716{ 3717 char *alt_name __maybe_unused = NULL; 3718 struct alternative *alt; 3719 u8 visited; 3720 int ret; 3721 3722 /* 3723 * Any returns before the end of this function are effectively dead 3724 * ends, i.e. validate_branch() has reached the end of the branch. 3725 */ 3726 *dead_end = true; 3727 3728 visited = VISITED_BRANCH << statep->uaccess; 3729 if (insn->visited & VISITED_BRANCH_MASK) { 3730 if (!insn->hint && !insn_cfi_match(insn, &statep->cfi)) 3731 return 1; 3732 3733 if (insn->visited & visited) { 3734 TRACE_INSN(insn, "already visited"); 3735 return 0; 3736 } 3737 } else { 3738 nr_insns_visited++; 3739 } 3740 3741 if (statep->noinstr) 3742 statep->instr += insn->instr; 3743 3744 if (insn->hint) { 3745 if (insn->restore) { 3746 struct instruction *save_insn, *i; 3747 3748 i = insn; 3749 save_insn = NULL; 3750 3751 sym_for_each_insn_continue_reverse(file, func, i) { 3752 if (i->save) { 3753 save_insn = i; 3754 break; 3755 } 3756 } 3757 3758 if (!save_insn) { 3759 WARN_INSN(insn, "no corresponding CFI save for CFI restore"); 3760 return 1; 3761 } 3762 3763 if (!save_insn->visited) { 3764 /* 3765 * If the restore hint insn is at the 3766 * beginning of a basic block and was 3767 * branched to from elsewhere, and the 3768 * save insn hasn't been visited yet, 3769 * defer following this branch for now. 3770 * It will be seen later via the 3771 * straight-line path. 3772 */ 3773 if (!prev_insn) { 3774 TRACE_INSN(insn, "defer restore"); 3775 return 0; 3776 } 3777 3778 WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); 3779 return 1; 3780 } 3781 3782 insn->cfi = save_insn->cfi; 3783 nr_cfi_reused++; 3784 } 3785 3786 statep->cfi = *insn->cfi; 3787 } else { 3788 /* XXX track if we actually changed statep->cfi */ 3789 3790 if (prev_insn && !cficmp(prev_insn->cfi, &statep->cfi)) { 3791 insn->cfi = prev_insn->cfi; 3792 nr_cfi_reused++; 3793 } else { 3794 insn->cfi = cfi_hash_find_or_add(&statep->cfi); 3795 } 3796 } 3797 3798 insn->visited |= visited; 3799 3800 if (propagate_alt_cfi(file, insn)) 3801 return 1; 3802 3803 if (insn->alts) { 3804 for (alt = insn->alts; alt; alt = alt->next) { 3805 TRACE_ALT_BEGIN(insn, alt, alt_name); 3806 ret = validate_branch(file, func, alt->insn, *statep); 3807 TRACE_ALT_END(insn, alt, alt_name); 3808 if (ret) { 3809 BT_INSN(insn, "(alt)"); 3810 return ret; 3811 } 3812 } 3813 TRACE_ALT_INFO_NOADDR(insn, "/ ", "DEFAULT"); 3814 } 3815 3816 if (skip_alt_group(insn)) 3817 return 0; 3818 3819 if (handle_insn_ops(insn, next_insn, statep)) 3820 return 1; 3821 3822 switch (insn->type) { 3823 3824 case INSN_RETURN: 3825 TRACE_INSN(insn, "return"); 3826 return validate_return(func, insn, statep); 3827 3828 case INSN_CALL: 3829 case INSN_CALL_DYNAMIC: 3830 if (insn->type == INSN_CALL) 3831 TRACE_INSN(insn, "call"); 3832 else 3833 TRACE_INSN(insn, "indirect call"); 3834 3835 ret = validate_call(file, insn, statep); 3836 if (ret) 3837 return ret; 3838 3839 if (opts.stackval && func && !is_special_call(insn) && 3840 !has_valid_stack_frame(statep)) { 3841 WARN_INSN(insn, "call without frame pointer save/setup"); 3842 return 1; 3843 } 3844 3845 break; 3846 3847 case INSN_JUMP_CONDITIONAL: 3848 case INSN_JUMP_UNCONDITIONAL: 3849 if (is_sibling_call(insn)) { 3850 TRACE_INSN(insn, "sibling call"); 3851 ret = validate_sibling_call(file, insn, statep); 3852 if (ret) 3853 return ret; 3854 3855 } else if (insn->jump_dest) { 3856 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3857 TRACE_INSN(insn, "unconditional jump"); 3858 else 3859 TRACE_INSN(insn, "jump taken"); 3860 3861 ret = validate_branch(file, func, insn->jump_dest, *statep); 3862 if (ret) { 3863 BT_INSN(insn, "(branch)"); 3864 return ret; 3865 } 3866 } 3867 3868 if (insn->type == INSN_JUMP_UNCONDITIONAL) 3869 return 0; 3870 3871 TRACE_INSN(insn, "jump not taken"); 3872 break; 3873 3874 case INSN_JUMP_DYNAMIC: 3875 case INSN_JUMP_DYNAMIC_CONDITIONAL: 3876 TRACE_INSN(insn, "indirect jump"); 3877 if (is_sibling_call(insn)) { 3878 ret = validate_sibling_call(file, insn, statep); 3879 if (ret) 3880 return ret; 3881 } 3882 3883 if (insn->type == INSN_JUMP_DYNAMIC) 3884 return 0; 3885 3886 break; 3887 3888 case INSN_SYSCALL: 3889 TRACE_INSN(insn, "syscall"); 3890 if (func && (!next_insn || !next_insn->hint)) { 3891 WARN_INSN(insn, "unsupported instruction in callable function"); 3892 return 1; 3893 } 3894 3895 break; 3896 3897 case INSN_SYSRET: 3898 TRACE_INSN(insn, "sysret"); 3899 if (func && (!next_insn || !next_insn->hint)) { 3900 WARN_INSN(insn, "unsupported instruction in callable function"); 3901 return 1; 3902 } 3903 3904 return 0; 3905 3906 case INSN_STAC: 3907 TRACE_INSN(insn, "stac"); 3908 if (!opts.uaccess) 3909 break; 3910 3911 if (statep->uaccess) { 3912 WARN_INSN(insn, "recursive UACCESS enable"); 3913 return 1; 3914 } 3915 3916 statep->uaccess = true; 3917 break; 3918 3919 case INSN_CLAC: 3920 TRACE_INSN(insn, "clac"); 3921 if (!opts.uaccess) 3922 break; 3923 3924 if (!statep->uaccess && func) { 3925 WARN_INSN(insn, "redundant UACCESS disable"); 3926 return 1; 3927 } 3928 3929 if (func_uaccess_safe(func) && !statep->uaccess_stack) { 3930 WARN_INSN(insn, "UACCESS-safe disables UACCESS"); 3931 return 1; 3932 } 3933 3934 statep->uaccess = false; 3935 break; 3936 3937 case INSN_STD: 3938 TRACE_INSN(insn, "std"); 3939 if (statep->df) { 3940 WARN_INSN(insn, "recursive STD"); 3941 return 1; 3942 } 3943 3944 statep->df = true; 3945 break; 3946 3947 case INSN_CLD: 3948 TRACE_INSN(insn, "cld"); 3949 if (!statep->df && func) { 3950 WARN_INSN(insn, "redundant CLD"); 3951 return 1; 3952 } 3953 3954 statep->df = false; 3955 break; 3956 3957 default: 3958 break; 3959 } 3960 3961 if (insn->dead_end) 3962 TRACE_INSN(insn, "dead end"); 3963 3964 *dead_end = insn->dead_end; 3965 return 0; 3966} 3967 3968/* 3969 * Follow the branch starting at the given instruction, and recursively follow 3970 * any other branches (jumps). Meanwhile, track the frame pointer state at 3971 * each instruction and validate all the rules described in 3972 * tools/objtool/Documentation/objtool.txt. 3973 */ 3974static int do_validate_branch(struct objtool_file *file, struct symbol *func, 3975 struct instruction *insn, struct insn_state *state) 3976{ 3977 struct instruction *next_insn, *prev_insn = NULL; 3978 bool dead_end; 3979 int ret; 3980 3981 if (func && func->ignore) 3982 return 0; 3983 3984 do { 3985 insn->trace = 0; 3986 next_insn = next_insn_to_validate(file, insn); 3987 3988 if (opts.checksum && func && insn->sec) 3989 checksum_update_insn(file, func, insn); 3990 3991 if (func && insn_func(insn) && func != insn_func(insn)->pfunc) { 3992 /* Ignore KCFI type preambles, which always fall through */ 3993 if (is_prefix_func(func)) 3994 return 0; 3995 3996 if (file->ignore_unreachables) 3997 return 0; 3998 3999 WARN("%s() falls through to next function %s()", 4000 func->name, insn_func(insn)->name); 4001 func->warned = 1; 4002 4003 return 1; 4004 } 4005 4006 ret = validate_insn(file, func, insn, state, prev_insn, next_insn, 4007 &dead_end); 4008 4009 if (!insn->trace) { 4010 if (ret) 4011 TRACE_INSN(insn, "warning (%d)", ret); 4012 else 4013 TRACE_INSN(insn, NULL); 4014 } 4015 4016 if (!dead_end && !next_insn) { 4017 if (state->cfi.cfa.base == CFI_UNDEFINED) 4018 return 0; 4019 if (file->ignore_unreachables) 4020 return 0; 4021 4022 WARN("%s%sunexpected end of section %s", 4023 func ? func->name : "", func ? "(): " : "", 4024 insn->sec->name); 4025 return 1; 4026 } 4027 4028 prev_insn = insn; 4029 insn = next_insn; 4030 4031 } while (!dead_end); 4032 4033 return ret; 4034} 4035 4036static int validate_branch(struct objtool_file *file, struct symbol *func, 4037 struct instruction *insn, struct insn_state state) 4038{ 4039 int ret; 4040 4041 trace_depth_inc(); 4042 ret = do_validate_branch(file, func, insn, &state); 4043 trace_depth_dec(); 4044 4045 return ret; 4046} 4047 4048static int validate_unwind_hint(struct objtool_file *file, 4049 struct instruction *insn, 4050 struct insn_state *state) 4051{ 4052 if (insn->hint && !insn->visited) { 4053 struct symbol *func = insn_func(insn); 4054 int ret; 4055 4056 if (opts.checksum) 4057 checksum_init(func); 4058 4059 ret = validate_branch(file, func, insn, *state); 4060 if (ret) 4061 BT_INSN(insn, "<=== (hint)"); 4062 return ret; 4063 } 4064 4065 return 0; 4066} 4067 4068static int validate_unwind_hints(struct objtool_file *file, struct section *sec) 4069{ 4070 struct instruction *insn; 4071 struct insn_state state; 4072 int warnings = 0; 4073 4074 if (!file->hints) 4075 return 0; 4076 4077 init_insn_state(file, &state, sec); 4078 4079 if (sec) { 4080 sec_for_each_insn(file, sec, insn) 4081 warnings += validate_unwind_hint(file, insn, &state); 4082 } else { 4083 for_each_insn(file, insn) 4084 warnings += validate_unwind_hint(file, insn, &state); 4085 } 4086 4087 return warnings; 4088} 4089 4090/* 4091 * Validate rethunk entry constraint: must untrain RET before the first RET. 4092 * 4093 * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes 4094 * before an actual RET instruction. 4095 */ 4096static int validate_unret(struct objtool_file *file, struct instruction *insn) 4097{ 4098 struct instruction *next, *dest; 4099 int ret; 4100 4101 for (;;) { 4102 next = next_insn_to_validate(file, insn); 4103 4104 if (insn->visited & VISITED_UNRET) 4105 return 0; 4106 4107 insn->visited |= VISITED_UNRET; 4108 4109 if (insn->alts) { 4110 struct alternative *alt; 4111 for (alt = insn->alts; alt; alt = alt->next) { 4112 ret = validate_unret(file, alt->insn); 4113 if (ret) { 4114 BT_INSN(insn, "(alt)"); 4115 return ret; 4116 } 4117 } 4118 } 4119 4120 switch (insn->type) { 4121 4122 case INSN_CALL_DYNAMIC: 4123 case INSN_JUMP_DYNAMIC: 4124 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4125 WARN_INSN(insn, "early indirect call"); 4126 return 1; 4127 4128 case INSN_JUMP_UNCONDITIONAL: 4129 case INSN_JUMP_CONDITIONAL: 4130 if (!is_sibling_call(insn)) { 4131 if (!insn->jump_dest) { 4132 WARN_INSN(insn, "unresolved jump target after linking?!?"); 4133 return 1; 4134 } 4135 ret = validate_unret(file, insn->jump_dest); 4136 if (ret) { 4137 BT_INSN(insn, "(branch%s)", 4138 insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); 4139 return ret; 4140 } 4141 4142 if (insn->type == INSN_JUMP_UNCONDITIONAL) 4143 return 0; 4144 4145 break; 4146 } 4147 4148 /* fallthrough */ 4149 case INSN_CALL: 4150 dest = find_insn(file, insn_call_dest(insn)->sec, 4151 insn_call_dest(insn)->offset); 4152 if (!dest) { 4153 WARN("Unresolved function after linking!?: %s", 4154 insn_call_dest(insn)->name); 4155 return 1; 4156 } 4157 4158 ret = validate_unret(file, dest); 4159 if (ret) { 4160 BT_INSN(insn, "(call)"); 4161 return ret; 4162 } 4163 /* 4164 * If a call returns without error, it must have seen UNTRAIN_RET. 4165 * Therefore any non-error return is a success. 4166 */ 4167 return 0; 4168 4169 case INSN_RETURN: 4170 WARN_INSN(insn, "RET before UNTRAIN"); 4171 return 1; 4172 4173 case INSN_SYSCALL: 4174 break; 4175 4176 case INSN_SYSRET: 4177 return 0; 4178 4179 case INSN_NOP: 4180 if (insn->retpoline_safe) 4181 return 0; 4182 break; 4183 4184 default: 4185 break; 4186 } 4187 4188 if (insn->dead_end) 4189 return 0; 4190 4191 if (!next) { 4192 WARN_INSN(insn, "teh end!"); 4193 return 1; 4194 } 4195 insn = next; 4196 } 4197 4198 return 0; 4199} 4200 4201/* 4202 * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter 4203 * VALIDATE_UNRET_END before RET. 4204 */ 4205static int validate_unrets(struct objtool_file *file) 4206{ 4207 struct instruction *insn; 4208 int warnings = 0; 4209 4210 for_each_insn(file, insn) { 4211 if (!insn->unret) 4212 continue; 4213 4214 warnings += validate_unret(file, insn); 4215 } 4216 4217 return warnings; 4218} 4219 4220static int validate_retpoline(struct objtool_file *file) 4221{ 4222 struct instruction *insn; 4223 int warnings = 0; 4224 4225 for_each_insn(file, insn) { 4226 if (insn->type != INSN_JUMP_DYNAMIC && 4227 insn->type != INSN_CALL_DYNAMIC && 4228 insn->type != INSN_RETURN) 4229 continue; 4230 4231 if (insn->retpoline_safe) 4232 continue; 4233 4234 if (insn->sec->init) 4235 continue; 4236 4237 if (insn->type == INSN_RETURN) { 4238 if (opts.rethunk) { 4239 WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build"); 4240 warnings++; 4241 } 4242 continue; 4243 } 4244 4245 WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build", 4246 insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); 4247 warnings++; 4248 } 4249 4250 if (!opts.cfi) 4251 return warnings; 4252 4253 /* 4254 * kCFI call sites look like: 4255 * 4256 * movl $(-0x12345678), %r10d 4257 * addl -4(%r11), %r10d 4258 * jz 1f 4259 * ud2 4260 * 1: cs call __x86_indirect_thunk_r11 4261 * 4262 * Verify all indirect calls are kCFI adorned by checking for the 4263 * UD2. Notably, doing __nocfi calls to regular (cfi) functions is 4264 * broken. 4265 */ 4266 list_for_each_entry(insn, &file->retpoline_call_list, call_node) { 4267 struct symbol *sym = insn->sym; 4268 4269 if (sym && (sym->type == STT_NOTYPE || 4270 sym->type == STT_FUNC) && !sym->nocfi) { 4271 struct instruction *prev = 4272 prev_insn_same_sym(file, insn); 4273 4274 if (!prev || prev->type != INSN_BUG) { 4275 WARN_INSN(insn, "no-cfi indirect call!"); 4276 warnings++; 4277 } 4278 } 4279 } 4280 4281 return warnings; 4282} 4283 4284static bool is_kasan_insn(struct instruction *insn) 4285{ 4286 return (insn->type == INSN_CALL && 4287 !strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return")); 4288} 4289 4290static bool is_ubsan_insn(struct instruction *insn) 4291{ 4292 return (insn->type == INSN_CALL && 4293 !strcmp(insn_call_dest(insn)->name, 4294 "__ubsan_handle_builtin_unreachable")); 4295} 4296 4297static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn) 4298{ 4299 struct symbol *func = insn_func(insn); 4300 struct instruction *prev_insn; 4301 int i; 4302 4303 if (insn->type == INSN_NOP || insn->type == INSN_TRAP || 4304 insn->hole || (func && func->ignore)) 4305 return true; 4306 4307 /* 4308 * Ignore alternative replacement instructions. This can happen 4309 * when a whitelisted function uses one of the ALTERNATIVE macros. 4310 */ 4311 if (!strcmp(insn->sec->name, ".altinstr_replacement") || 4312 !strcmp(insn->sec->name, ".altinstr_aux")) 4313 return true; 4314 4315 if (!func) 4316 return false; 4317 4318 if (func->static_call_tramp) 4319 return true; 4320 4321 /* 4322 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees 4323 * __builtin_unreachable(). The BUG() macro has an unreachable() after 4324 * the UD2, which causes GCC's undefined trap logic to emit another UD2 4325 * (or occasionally a JMP to UD2). 4326 * 4327 * It may also insert a UD2 after calling a __noreturn function. 4328 */ 4329 prev_insn = prev_insn_same_sec(file, insn); 4330 if (prev_insn && prev_insn->dead_end && 4331 (insn->type == INSN_BUG || 4332 (insn->type == INSN_JUMP_UNCONDITIONAL && 4333 insn->jump_dest && insn->jump_dest->type == INSN_BUG))) 4334 return true; 4335 4336 /* 4337 * Check if this (or a subsequent) instruction is related to 4338 * CONFIG_UBSAN or CONFIG_KASAN. 4339 * 4340 * End the search at 5 instructions to avoid going into the weeds. 4341 */ 4342 for (i = 0; i < 5; i++) { 4343 4344 if (is_kasan_insn(insn) || is_ubsan_insn(insn)) 4345 return true; 4346 4347 if (insn->type == INSN_JUMP_UNCONDITIONAL) { 4348 if (insn->jump_dest && 4349 insn_func(insn->jump_dest) == func) { 4350 insn = insn->jump_dest; 4351 continue; 4352 } 4353 4354 break; 4355 } 4356 4357 if (insn->offset + insn->len >= func->offset + func->len) 4358 break; 4359 4360 insn = next_insn_same_sec(file, insn); 4361 } 4362 4363 return false; 4364} 4365 4366/* 4367 * For FineIBT or kCFI, a certain number of bytes preceding the function may be 4368 * NOPs. Those NOPs may be rewritten at runtime and executed, so give them a 4369 * proper function name: __pfx_<func>. 4370 * 4371 * The NOPs may not exist for the following cases: 4372 * 4373 * - compiler cloned functions (*.cold, *.part0, etc) 4374 * - asm functions created with inline asm or without SYM_FUNC_START() 4375 * 4376 * Also, the function may already have a prefix from a previous objtool run 4377 * (livepatch extracted functions, or manually running objtool multiple times). 4378 * 4379 * So return 0 if the NOPs are missing or the function already has a prefix 4380 * symbol. 4381 */ 4382static int create_prefix_symbol(struct objtool_file *file, struct symbol *func) 4383{ 4384 struct instruction *insn, *prev; 4385 char name[SYM_NAME_LEN]; 4386 struct cfi_state *cfi; 4387 4388 if (!is_func_sym(func) || is_prefix_func(func) || 4389 func->cold || func->static_call_tramp) 4390 return 0; 4391 4392 if ((strlen(func->name) + sizeof("__pfx_") > SYM_NAME_LEN)) { 4393 WARN("%s: symbol name too long, can't create __pfx_ symbol", 4394 func->name); 4395 return 0; 4396 } 4397 4398 if (snprintf_check(name, SYM_NAME_LEN, "__pfx_%s", func->name)) 4399 return -1; 4400 4401 if (file->klp) { 4402 struct symbol *pfx; 4403 4404 pfx = find_symbol_by_offset(func->sec, func->offset - opts.prefix); 4405 if (pfx && is_prefix_func(pfx) && !strcmp(pfx->name, name)) 4406 return 0; 4407 } 4408 4409 insn = find_insn(file, func->sec, func->offset); 4410 if (!insn) { 4411 WARN("%s: can't find starting instruction", func->name); 4412 return -1; 4413 } 4414 4415 for (prev = prev_insn_same_sec(file, insn); 4416 prev; 4417 prev = prev_insn_same_sec(file, prev)) { 4418 u64 offset; 4419 4420 if (prev->type != INSN_NOP) 4421 return 0; 4422 4423 offset = func->offset - prev->offset; 4424 4425 if (offset > opts.prefix) 4426 return 0; 4427 4428 if (offset < opts.prefix) 4429 continue; 4430 4431 if (!elf_create_symbol(file->elf, name, func->sec, 4432 GELF_ST_BIND(func->sym.st_info), 4433 GELF_ST_TYPE(func->sym.st_info), 4434 prev->offset, opts.prefix)) 4435 return -1; 4436 4437 break; 4438 } 4439 4440 if (!prev) 4441 return 0; 4442 4443 if (!insn->cfi) { 4444 /* 4445 * This can happen if stack validation isn't enabled or the 4446 * function is annotated with STACK_FRAME_NON_STANDARD. 4447 */ 4448 return 0; 4449 } 4450 4451 /* Propagate insn->cfi to the prefix code */ 4452 cfi = cfi_hash_find_or_add(insn->cfi); 4453 for (; prev != insn; prev = next_insn_same_sec(file, prev)) 4454 prev->cfi = cfi; 4455 4456 return 0; 4457} 4458 4459static int create_prefix_symbols(struct objtool_file *file) 4460{ 4461 struct section *sec; 4462 struct symbol *func; 4463 4464 for_each_sec(file->elf, sec) { 4465 if (!is_text_sec(sec)) 4466 continue; 4467 4468 sec_for_each_sym(sec, func) { 4469 if (create_prefix_symbol(file, func)) 4470 return -1; 4471 } 4472 } 4473 4474 return 0; 4475} 4476 4477static int validate_symbol(struct objtool_file *file, struct section *sec, 4478 struct symbol *sym, struct insn_state *state) 4479{ 4480 struct instruction *insn; 4481 struct symbol *func; 4482 int ret; 4483 4484 if (!sym->len) { 4485 WARN("%s() is missing an ELF size annotation", sym->name); 4486 return 1; 4487 } 4488 4489 if (sym->pfunc != sym || sym->alias != sym) 4490 return 0; 4491 4492 insn = find_insn(file, sec, sym->offset); 4493 if (!insn || insn->visited) 4494 return 0; 4495 4496 if (opts.uaccess) 4497 state->uaccess = sym->uaccess_safe; 4498 4499 func = insn_func(insn); 4500 4501 if (opts.checksum) 4502 checksum_init(func); 4503 4504 if (opts.trace && !fnmatch(opts.trace, sym->name, 0)) { 4505 trace_enable(); 4506 TRACE("%s: validation begin\n", sym->name); 4507 } 4508 4509 ret = validate_branch(file, func, insn, *state); 4510 if (ret) 4511 BT_INSN(insn, "<=== (sym)"); 4512 4513 TRACE("%s: validation %s\n\n", sym->name, ret ? "failed" : "end"); 4514 trace_disable(); 4515 4516 if (opts.checksum) 4517 checksum_finish(func); 4518 4519 return ret; 4520} 4521 4522static int validate_section(struct objtool_file *file, struct section *sec) 4523{ 4524 struct insn_state state; 4525 struct symbol *func; 4526 int warnings = 0; 4527 4528 sec_for_each_sym(sec, func) { 4529 if (!is_func_sym(func)) 4530 continue; 4531 4532 init_insn_state(file, &state, sec); 4533 set_func_state(&state.cfi); 4534 4535 warnings += validate_symbol(file, sec, func, &state); 4536 } 4537 4538 return warnings; 4539} 4540 4541static int validate_noinstr_sections(struct objtool_file *file) 4542{ 4543 struct section *sec; 4544 int warnings = 0; 4545 4546 sec = find_section_by_name(file->elf, ".noinstr.text"); 4547 if (sec) { 4548 warnings += validate_section(file, sec); 4549 warnings += validate_unwind_hints(file, sec); 4550 } 4551 4552 sec = find_section_by_name(file->elf, ".entry.text"); 4553 if (sec) { 4554 warnings += validate_section(file, sec); 4555 warnings += validate_unwind_hints(file, sec); 4556 } 4557 4558 sec = find_section_by_name(file->elf, ".cpuidle.text"); 4559 if (sec) { 4560 warnings += validate_section(file, sec); 4561 warnings += validate_unwind_hints(file, sec); 4562 } 4563 4564 return warnings; 4565} 4566 4567static int validate_functions(struct objtool_file *file) 4568{ 4569 struct section *sec; 4570 int warnings = 0; 4571 4572 for_each_sec(file->elf, sec) { 4573 if (!is_text_sec(sec)) 4574 continue; 4575 4576 warnings += validate_section(file, sec); 4577 } 4578 4579 return warnings; 4580} 4581 4582static void mark_endbr_used(struct instruction *insn) 4583{ 4584 if (!list_empty(&insn->call_node)) 4585 list_del_init(&insn->call_node); 4586} 4587 4588static bool noendbr_range(struct objtool_file *file, struct instruction *insn) 4589{ 4590 struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1); 4591 struct instruction *first; 4592 4593 if (!sym) 4594 return false; 4595 4596 first = find_insn(file, sym->sec, sym->offset); 4597 if (!first) 4598 return false; 4599 4600 if (first->type != INSN_ENDBR && !first->noendbr) 4601 return false; 4602 4603 return insn->offset == sym->offset + sym->len; 4604} 4605 4606static int __validate_ibt_insn(struct objtool_file *file, struct instruction *insn, 4607 struct instruction *dest) 4608{ 4609 if (dest->type == INSN_ENDBR) { 4610 mark_endbr_used(dest); 4611 return 0; 4612 } 4613 4614 if (insn_func(dest) && insn_func(insn) && 4615 insn_func(dest)->pfunc == insn_func(insn)->pfunc) { 4616 /* 4617 * Anything from->to self is either _THIS_IP_ or 4618 * IRET-to-self. 4619 * 4620 * There is no sane way to annotate _THIS_IP_ since the 4621 * compiler treats the relocation as a constant and is 4622 * happy to fold in offsets, skewing any annotation we 4623 * do, leading to vast amounts of false-positives. 4624 * 4625 * There's also compiler generated _THIS_IP_ through 4626 * KCOV and such which we have no hope of annotating. 4627 * 4628 * As such, blanket accept self-references without 4629 * issue. 4630 */ 4631 return 0; 4632 } 4633 4634 /* 4635 * Accept anything ANNOTATE_NOENDBR. 4636 */ 4637 if (dest->noendbr) 4638 return 0; 4639 4640 /* 4641 * Accept if this is the instruction after a symbol 4642 * that is (no)endbr -- typical code-range usage. 4643 */ 4644 if (noendbr_range(file, dest)) 4645 return 0; 4646 4647 WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4648 return 1; 4649} 4650 4651static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) 4652{ 4653 struct instruction *dest; 4654 struct reloc *reloc; 4655 unsigned long off; 4656 int warnings = 0; 4657 4658 /* 4659 * Looking for function pointer load relocations. Ignore 4660 * direct/indirect branches: 4661 */ 4662 switch (insn->type) { 4663 4664 case INSN_CALL: 4665 case INSN_CALL_DYNAMIC: 4666 case INSN_JUMP_CONDITIONAL: 4667 case INSN_JUMP_UNCONDITIONAL: 4668 case INSN_JUMP_DYNAMIC: 4669 case INSN_JUMP_DYNAMIC_CONDITIONAL: 4670 case INSN_RETURN: 4671 case INSN_NOP: 4672 return 0; 4673 4674 case INSN_LEA_RIP: 4675 if (!insn_reloc(file, insn)) { 4676 /* local function pointer reference without reloc */ 4677 4678 off = arch_jump_destination(insn); 4679 4680 dest = find_insn(file, insn->sec, off); 4681 if (!dest) { 4682 WARN_INSN(insn, "corrupt function pointer reference"); 4683 return 1; 4684 } 4685 4686 return __validate_ibt_insn(file, insn, dest); 4687 } 4688 break; 4689 4690 default: 4691 break; 4692 } 4693 4694 for (reloc = insn_reloc(file, insn); 4695 reloc; 4696 reloc = find_reloc_by_dest_range(file->elf, insn->sec, 4697 reloc_offset(reloc) + 1, 4698 (insn->offset + insn->len) - (reloc_offset(reloc) + 1))) { 4699 4700 off = reloc->sym->offset + arch_insn_adjusted_addend(insn, reloc); 4701 4702 dest = find_insn(file, reloc->sym->sec, off); 4703 if (!dest) 4704 continue; 4705 4706 warnings += __validate_ibt_insn(file, insn, dest); 4707 } 4708 4709 return warnings; 4710} 4711 4712static int validate_ibt_data_reloc(struct objtool_file *file, 4713 struct reloc *reloc) 4714{ 4715 struct instruction *dest; 4716 4717 dest = find_insn(file, reloc->sym->sec, 4718 reloc->sym->offset + reloc_addend(reloc)); 4719 if (!dest) 4720 return 0; 4721 4722 if (dest->type == INSN_ENDBR) { 4723 mark_endbr_used(dest); 4724 return 0; 4725 } 4726 4727 if (dest->noendbr) 4728 return 0; 4729 4730 WARN_FUNC(reloc->sec->base, reloc_offset(reloc), 4731 "data relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); 4732 4733 return 1; 4734} 4735 4736/* 4737 * Validate IBT rules and remove used ENDBR instructions from the seal list. 4738 * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with 4739 * NOPs) later, in create_ibt_endbr_seal_sections(). 4740 */ 4741static int validate_ibt(struct objtool_file *file) 4742{ 4743 struct section *sec; 4744 struct reloc *reloc; 4745 struct instruction *insn; 4746 int warnings = 0; 4747 4748 for_each_insn(file, insn) 4749 warnings += validate_ibt_insn(file, insn); 4750 4751 for_each_sec(file->elf, sec) { 4752 4753 /* Already done by validate_ibt_insn() */ 4754 if (is_text_sec(sec)) 4755 continue; 4756 4757 if (!sec->rsec) 4758 continue; 4759 4760 /* 4761 * These sections can reference text addresses, but not with 4762 * the intent to indirect branch to them. 4763 */ 4764 if ((!strncmp(sec->name, ".discard", 8) && 4765 strcmp(sec->name, ".discard.ibt_endbr_noseal")) || 4766 !strncmp(sec->name, ".debug", 6) || 4767 !strcmp(sec->name, ".altinstructions") || 4768 !strcmp(sec->name, ".ibt_endbr_seal") || 4769 !strcmp(sec->name, ".kcfi_traps") || 4770 !strcmp(sec->name, ".orc_unwind_ip") || 4771 !strcmp(sec->name, ".retpoline_sites") || 4772 !strcmp(sec->name, ".smp_locks") || 4773 !strcmp(sec->name, ".static_call_sites") || 4774 !strcmp(sec->name, "_error_injection_whitelist") || 4775 !strcmp(sec->name, "_kprobe_blacklist") || 4776 !strcmp(sec->name, "__bug_table") || 4777 !strcmp(sec->name, "__ex_table") || 4778 !strcmp(sec->name, "__jump_table") || 4779 !strcmp(sec->name, ".init.klp_funcs") || 4780 !strcmp(sec->name, "__mcount_loc") || 4781 !strcmp(sec->name, ".llvm.call-graph-profile") || 4782 !strcmp(sec->name, ".llvm_bb_addr_map") || 4783 !strcmp(sec->name, "__tracepoints") || 4784 !strcmp(sec->name, ".return_sites") || 4785 !strcmp(sec->name, ".call_sites") || 4786 !strcmp(sec->name, "__patchable_function_entries")) 4787 continue; 4788 4789 for_each_reloc(sec->rsec, reloc) 4790 warnings += validate_ibt_data_reloc(file, reloc); 4791 } 4792 4793 return warnings; 4794} 4795 4796static int validate_sls(struct objtool_file *file) 4797{ 4798 struct instruction *insn, *next_insn; 4799 int warnings = 0; 4800 4801 for_each_insn(file, insn) { 4802 next_insn = next_insn_same_sec(file, insn); 4803 4804 if (insn->retpoline_safe) 4805 continue; 4806 4807 switch (insn->type) { 4808 case INSN_RETURN: 4809 if (!next_insn || next_insn->type != INSN_TRAP) { 4810 WARN_INSN(insn, "missing int3 after ret"); 4811 warnings++; 4812 } 4813 4814 break; 4815 case INSN_JUMP_DYNAMIC: 4816 if (!next_insn || next_insn->type != INSN_TRAP) { 4817 WARN_INSN(insn, "missing int3 after indirect jump"); 4818 warnings++; 4819 } 4820 break; 4821 default: 4822 break; 4823 } 4824 } 4825 4826 return warnings; 4827} 4828 4829static int validate_reachable_instructions(struct objtool_file *file) 4830{ 4831 struct instruction *insn, *prev_insn; 4832 struct symbol *call_dest; 4833 int warnings = 0; 4834 4835 if (file->ignore_unreachables) 4836 return 0; 4837 4838 for_each_insn(file, insn) { 4839 if (insn->visited || ignore_unreachable_insn(file, insn)) 4840 continue; 4841 4842 prev_insn = prev_insn_same_sec(file, insn); 4843 if (prev_insn && prev_insn->dead_end) { 4844 call_dest = insn_call_dest(prev_insn); 4845 if (call_dest) { 4846 WARN_INSN(insn, "%s() missing __noreturn in .c/.h or NORETURN() in noreturns.h", 4847 call_dest->name); 4848 warnings++; 4849 continue; 4850 } 4851 } 4852 4853 WARN_INSN(insn, "unreachable instruction"); 4854 warnings++; 4855 } 4856 4857 return warnings; 4858} 4859 4860__weak bool arch_absolute_reloc(struct elf *elf, struct reloc *reloc) 4861{ 4862 unsigned int type = reloc_type(reloc); 4863 size_t sz = elf_addr_size(elf); 4864 4865 return (sz == 8) ? (type == R_ABS64) : (type == R_ABS32); 4866} 4867 4868static int check_abs_references(struct objtool_file *file) 4869{ 4870 struct section *sec; 4871 struct reloc *reloc; 4872 int ret = 0; 4873 4874 for_each_sec(file->elf, sec) { 4875 /* absolute references in non-loadable sections are fine */ 4876 if (!(sec->sh.sh_flags & SHF_ALLOC)) 4877 continue; 4878 4879 /* section must have an associated .rela section */ 4880 if (!sec->rsec) 4881 continue; 4882 4883 /* 4884 * Special case for compiler generated metadata that is not 4885 * consumed until after boot. 4886 */ 4887 if (!strcmp(sec->name, "__patchable_function_entries")) 4888 continue; 4889 4890 for_each_reloc(sec->rsec, reloc) { 4891 if (arch_absolute_reloc(file->elf, reloc)) { 4892 WARN("section %s has absolute relocation at offset 0x%llx", 4893 sec->name, (unsigned long long)reloc_offset(reloc)); 4894 ret++; 4895 } 4896 } 4897 } 4898 return ret; 4899} 4900 4901struct insn_chunk { 4902 void *addr; 4903 struct insn_chunk *next; 4904}; 4905 4906/* 4907 * Reduce peak RSS usage by freeing insns memory before writing the ELF file, 4908 * which can trigger more allocations for .debug_* sections whose data hasn't 4909 * been read yet. 4910 */ 4911static void free_insns(struct objtool_file *file) 4912{ 4913 struct instruction *insn; 4914 struct insn_chunk *chunks = NULL, *chunk; 4915 4916 for_each_insn(file, insn) { 4917 if (!insn->idx) { 4918 chunk = malloc(sizeof(*chunk)); 4919 chunk->addr = insn; 4920 chunk->next = chunks; 4921 chunks = chunk; 4922 } 4923 } 4924 4925 for (chunk = chunks; chunk; chunk = chunk->next) 4926 free(chunk->addr); 4927} 4928 4929const char *objtool_disas_insn(struct instruction *insn) 4930{ 4931 struct disas_context *dctx = objtool_disas_ctx; 4932 4933 if (!dctx) 4934 return ""; 4935 4936 disas_insn(dctx, insn); 4937 return disas_result(dctx); 4938} 4939 4940int check(struct objtool_file *file) 4941{ 4942 struct disas_context *disas_ctx = NULL; 4943 int ret = 0, warnings = 0; 4944 4945 /* 4946 * Create a disassembly context if we might disassemble any 4947 * instruction or function. 4948 */ 4949 if (opts.verbose || opts.backtrace || opts.trace || opts.disas) { 4950 disas_ctx = disas_context_create(file); 4951 if (!disas_ctx) { 4952 opts.disas = false; 4953 opts.trace = false; 4954 } 4955 objtool_disas_ctx = disas_ctx; 4956 } 4957 4958 arch_initial_func_cfi_state(&initial_func_cfi); 4959 init_cfi_state(&init_cfi); 4960 init_cfi_state(&func_cfi); 4961 set_func_state(&func_cfi); 4962 init_cfi_state(&force_undefined_cfi); 4963 force_undefined_cfi.force_undefined = true; 4964 4965 if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3))) { 4966 ret = -1; 4967 goto out; 4968 } 4969 4970 cfi_hash_add(&init_cfi); 4971 cfi_hash_add(&func_cfi); 4972 4973 ret = checksum_debug_init(file); 4974 if (ret) 4975 goto out; 4976 4977 ret = decode_sections(file); 4978 if (ret) 4979 goto out; 4980 4981 if (!nr_insns) 4982 goto out; 4983 4984 if (opts.retpoline) 4985 warnings += validate_retpoline(file); 4986 4987 if (validate_branch_enabled()) { 4988 int w = 0; 4989 4990 w += validate_functions(file); 4991 w += validate_unwind_hints(file, NULL); 4992 if (!w) 4993 w += validate_reachable_instructions(file); 4994 4995 warnings += w; 4996 4997 } else if (opts.noinstr) { 4998 warnings += validate_noinstr_sections(file); 4999 } 5000 5001 if (opts.unret) { 5002 /* 5003 * Must be after validate_branch() and friends, it plays 5004 * further games with insn->visited. 5005 */ 5006 warnings += validate_unrets(file); 5007 } 5008 5009 if (opts.ibt) 5010 warnings += validate_ibt(file); 5011 5012 if (opts.sls) 5013 warnings += validate_sls(file); 5014 5015 if (opts.static_call) { 5016 ret = create_static_call_sections(file); 5017 if (ret) 5018 goto out; 5019 } 5020 5021 if (opts.retpoline) { 5022 ret = create_retpoline_sites_sections(file); 5023 if (ret) 5024 goto out; 5025 } 5026 5027 if (opts.cfi) { 5028 ret = create_cfi_sections(file); 5029 if (ret) 5030 goto out; 5031 } 5032 5033 if (opts.rethunk) { 5034 ret = create_return_sites_sections(file); 5035 if (ret) 5036 goto out; 5037 5038 if (opts.hack_skylake) { 5039 ret = create_direct_call_sections(file); 5040 if (ret) 5041 goto out; 5042 } 5043 } 5044 5045 if (opts.mcount) { 5046 ret = create_mcount_loc_sections(file); 5047 if (ret) 5048 goto out; 5049 } 5050 5051 if (opts.prefix) { 5052 ret = create_prefix_symbols(file); 5053 if (ret) 5054 goto out; 5055 } 5056 5057 if (opts.ibt) { 5058 ret = create_ibt_endbr_seal_sections(file); 5059 if (ret) 5060 goto out; 5061 } 5062 5063 if (opts.noabs) 5064 warnings += check_abs_references(file); 5065 5066 if (opts.checksum) { 5067 ret = create_sym_checksum_section(file); 5068 if (ret) 5069 goto out; 5070 } 5071 5072 if (opts.orc && nr_insns) { 5073 ret = orc_create(file); 5074 if (ret) 5075 goto out; 5076 } 5077 5078 if (opts.stats) { 5079 printf("nr_insns_visited: %ld\n", nr_insns_visited); 5080 printf("nr_cfi: %ld\n", nr_cfi); 5081 printf("nr_cfi_reused: %ld\n", nr_cfi_reused); 5082 printf("nr_cfi_cache: %ld\n", nr_cfi_cache); 5083 } 5084 5085out: 5086 if (ret || warnings) { 5087 if (opts.werror && warnings) 5088 ret = 1; 5089 5090 if (opts.verbose) { 5091 if (opts.werror && warnings) 5092 WARN("%d warning(s) upgraded to errors", warnings); 5093 disas_warned_funcs(disas_ctx); 5094 } 5095 } 5096 5097 if (opts.disas) 5098 disas_funcs(disas_ctx); 5099 5100 if (disas_ctx) { 5101 disas_context_destroy(disas_ctx); 5102 objtool_disas_ctx = NULL; 5103 } 5104 5105 free_insns(file); 5106 5107 if (!ret && !warnings) 5108 return 0; 5109 5110 if (opts.backup && make_backup()) 5111 return 1; 5112 5113 return ret; 5114}
~/
Read-only file preview.