sorttable.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * sorttable.c: Sort the kernel's table
   4 *
   5 * Added ORC unwind tables sort support and other updates:
   6 * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by:
   7 * Shile Zhang <shile.zhang@linux.alibaba.com>
   8 *
   9 * Copyright 2011 - 2012 Cavium, Inc.
  10 *
  11 * Based on code taken from recortmcount.c which is:
  12 *
  13 * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>.  All rights reserved.
  14 *
  15 * Restructured to fit Linux format, as well as other updates:
  16 * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
  17 */
  18
  19/*
  20 * Strategy: alter the vmlinux file in-place.
  21 */
  22
  23#include <sys/types.h>
  24#include <sys/stat.h>
  25#include <getopt.h>
  26#include <fcntl.h>
  27#include <stdio.h>
  28#include <stdlib.h>
  29#include <stdbool.h>
  30#include <string.h>
  31#include <unistd.h>
  32#include <errno.h>
  33#include <pthread.h>
  34
  35#include "elf-parse.h"
  36
  37#ifndef EM_ARCOMPACT
  38#define EM_ARCOMPACT	93
  39#endif
  40
  41#ifndef EM_XTENSA
  42#define EM_XTENSA	94
  43#endif
  44
  45#ifndef EM_AARCH64
  46#define EM_AARCH64	183
  47#endif
  48
  49#ifndef EM_MICROBLAZE
  50#define EM_MICROBLAZE	189
  51#endif
  52
  53#ifndef EM_ARCV2
  54#define EM_ARCV2	195
  55#endif
  56
  57#ifndef EM_RISCV
  58#define EM_RISCV	243
  59#endif
  60
  61#ifndef EM_LOONGARCH
  62#define EM_LOONGARCH	258
  63#endif
  64
  65typedef void (*table_sort_t)(char *, int);
  66
  67/*
  68 * Move reserved section indices SHN_LORESERVE..SHN_HIRESERVE out of
  69 * the way to -256..-1, to avoid conflicting with real section
  70 * indices.
  71 */
  72#define SPECIAL(i) ((i) - (SHN_HIRESERVE + 1))
  73
  74static inline int is_shndx_special(unsigned int i)
  75{
  76	return i != SHN_XINDEX && i >= SHN_LORESERVE && i <= SHN_HIRESERVE;
  77}
  78
  79/* Accessor for sym->st_shndx, hides ugliness of "64k sections" */
  80static inline unsigned int get_secindex(unsigned int shndx,
  81					unsigned int sym_offs,
  82					const Elf32_Word *symtab_shndx_start)
  83{
  84	if (is_shndx_special(shndx))
  85		return SPECIAL(shndx);
  86	if (shndx != SHN_XINDEX)
  87		return shndx;
  88	return elf_parser.r(&symtab_shndx_start[sym_offs]);
  89}
  90
  91static int compare_extable_32(const void *a, const void *b)
  92{
  93	Elf32_Addr av = elf_parser.r(a);
  94	Elf32_Addr bv = elf_parser.r(b);
  95
  96	if (av < bv)
  97		return -1;
  98	return av > bv;
  99}
 100
 101static int compare_extable_64(const void *a, const void *b)
 102{
 103	Elf64_Addr av = elf_parser.r8(a);
 104	Elf64_Addr bv = elf_parser.r8(b);
 105
 106	if (av < bv)
 107		return -1;
 108	return av > bv;
 109}
 110
 111static int (*compare_extable)(const void *a, const void *b);
 112
 113static inline void *get_index(void *start, int entsize, int index)
 114{
 115	return start + (entsize * index);
 116}
 117
 118static int extable_ent_size;
 119static int long_size;
 120
 121#define ERRSTR_MAXSZ	256
 122
 123#ifdef UNWINDER_ORC_ENABLED
 124/* ORC unwinder only support X86_64 */
 125#include <asm/orc_types.h>
 126
 127static char g_err[ERRSTR_MAXSZ];
 128static int *g_orc_ip_table;
 129static struct orc_entry *g_orc_table;
 130
 131static pthread_t orc_sort_thread;
 132
 133static inline unsigned long orc_ip(const int *ip)
 134{
 135	return (unsigned long)ip + *ip;
 136}
 137
 138static int orc_sort_cmp(const void *_a, const void *_b)
 139{
 140	struct orc_entry *orc_a, *orc_b;
 141	const int *a = g_orc_ip_table + *(int *)_a;
 142	const int *b = g_orc_ip_table + *(int *)_b;
 143	unsigned long a_val = orc_ip(a);
 144	unsigned long b_val = orc_ip(b);
 145
 146	if (a_val > b_val)
 147		return 1;
 148	if (a_val < b_val)
 149		return -1;
 150
 151	/*
 152	 * The "weak" section terminator entries need to always be on the left
 153	 * to ensure the lookup code skips them in favor of real entries.
 154	 * These terminator entries exist to handle any gaps created by
 155	 * whitelisted .o files which didn't get objtool generation.
 156	 */
 157	orc_a = g_orc_table + (a - g_orc_ip_table);
 158	orc_b = g_orc_table + (b - g_orc_ip_table);
 159	if (orc_a->type == ORC_TYPE_UNDEFINED && orc_b->type == ORC_TYPE_UNDEFINED)
 160		return 0;
 161	return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
 162}
 163
 164static void *sort_orctable(void *arg)
 165{
 166	int i;
 167	int *idxs = NULL;
 168	int *tmp_orc_ip_table = NULL;
 169	struct orc_entry *tmp_orc_table = NULL;
 170	unsigned int *orc_ip_size = (unsigned int *)arg;
 171	unsigned int num_entries = *orc_ip_size / sizeof(int);
 172	unsigned int orc_size = num_entries * sizeof(struct orc_entry);
 173
 174	idxs = (int *)malloc(*orc_ip_size);
 175	if (!idxs) {
 176		snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s",
 177			 strerror(errno));
 178		pthread_exit(g_err);
 179	}
 180
 181	tmp_orc_ip_table = (int *)malloc(*orc_ip_size);
 182	if (!tmp_orc_ip_table) {
 183		snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s",
 184			 strerror(errno));
 185		pthread_exit(g_err);
 186	}
 187
 188	tmp_orc_table = (struct orc_entry *)malloc(orc_size);
 189	if (!tmp_orc_table) {
 190		snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s",
 191			 strerror(errno));
 192		pthread_exit(g_err);
 193	}
 194
 195	/* initialize indices array, convert ip_table to absolute address */
 196	for (i = 0; i < num_entries; i++) {
 197		idxs[i] = i;
 198		tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int);
 199	}
 200	memcpy(tmp_orc_table, g_orc_table, orc_size);
 201
 202	qsort(idxs, num_entries, sizeof(int), orc_sort_cmp);
 203
 204	for (i = 0; i < num_entries; i++) {
 205		if (idxs[i] == i)
 206			continue;
 207
 208		/* convert back to relative address */
 209		g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int);
 210		g_orc_table[i] = tmp_orc_table[idxs[i]];
 211	}
 212
 213	free(idxs);
 214	free(tmp_orc_ip_table);
 215	free(tmp_orc_table);
 216	pthread_exit(NULL);
 217}
 218#endif
 219
 220#ifdef MCOUNT_SORT_ENABLED
 221
 222static int compare_values_64(const void *a, const void *b)
 223{
 224	uint64_t av = *(uint64_t *)a;
 225	uint64_t bv = *(uint64_t *)b;
 226
 227	if (av < bv)
 228		return -1;
 229	return av > bv;
 230}
 231
 232static int compare_values_32(const void *a, const void *b)
 233{
 234	uint32_t av = *(uint32_t *)a;
 235	uint32_t bv = *(uint32_t *)b;
 236
 237	if (av < bv)
 238		return -1;
 239	return av > bv;
 240}
 241
 242static int (*compare_values)(const void *a, const void *b);
 243
 244/* Only used for sorting mcount table */
 245static void rela_write_addend(Elf_Rela *rela, uint64_t val)
 246{
 247	elf_parser.rela_write_addend(rela, val);
 248}
 249
 250struct func_info {
 251	uint64_t	addr;
 252	uint64_t	size;
 253};
 254
 255/* List of functions created by: nm -S vmlinux */
 256static struct func_info *function_list;
 257static int function_list_size;
 258
 259/* Allocate functions in 1k blocks */
 260#define FUNC_BLK_SIZE	1024
 261#define FUNC_BLK_MASK	(FUNC_BLK_SIZE - 1)
 262
 263static int add_field(uint64_t addr, uint64_t size)
 264{
 265	struct func_info *fi;
 266	int fsize = function_list_size;
 267
 268	if (!(fsize & FUNC_BLK_MASK)) {
 269		fsize += FUNC_BLK_SIZE;
 270		fi = realloc(function_list, fsize * sizeof(struct func_info));
 271		if (!fi)
 272			return -1;
 273		function_list = fi;
 274	}
 275	fi = &function_list[function_list_size++];
 276	fi->addr = addr;
 277	fi->size = size;
 278	return 0;
 279}
 280
 281/* Used for when mcount/fentry is before the function entry */
 282static int before_func;
 283
 284/* Only return match if the address lies inside the function size */
 285static int cmp_func_addr(const void *K, const void *A)
 286{
 287	uint64_t key = *(const uint64_t *)K;
 288	const struct func_info *a = A;
 289
 290	if (key + before_func < a->addr)
 291		return -1;
 292	return key >= a->addr + a->size;
 293}
 294
 295/* Find the function in function list that is bounded by the function size */
 296static int find_func(uint64_t key)
 297{
 298	return bsearch(&key, function_list, function_list_size,
 299		       sizeof(struct func_info), cmp_func_addr) != NULL;
 300}
 301
 302static int cmp_funcs(const void *A, const void *B)
 303{
 304	const struct func_info *a = A;
 305	const struct func_info *b = B;
 306
 307	if (a->addr < b->addr)
 308		return -1;
 309	return a->addr > b->addr;
 310}
 311
 312static int parse_symbols(const char *fname)
 313{
 314	FILE *fp;
 315	char addr_str[20]; /* Only need 17, but round up to next int size */
 316	char size_str[20];
 317	char type;
 318
 319	fp = fopen(fname, "r");
 320	if (!fp) {
 321		perror(fname);
 322		return -1;
 323	}
 324
 325	while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) {
 326		uint64_t addr;
 327		uint64_t size;
 328
 329		/* Only care about functions */
 330		if (type != 't' && type != 'T' && type != 'W')
 331			continue;
 332
 333		addr = strtoull(addr_str, NULL, 16);
 334		size = strtoull(size_str, NULL, 16);
 335		if (add_field(addr, size) < 0)
 336			return -1;
 337	}
 338	fclose(fp);
 339
 340	qsort(function_list, function_list_size, sizeof(struct func_info), cmp_funcs);
 341
 342	return 0;
 343}
 344
 345static pthread_t mcount_sort_thread;
 346static bool sort_reloc;
 347
 348static long rela_type;
 349
 350static char m_err[ERRSTR_MAXSZ];
 351
 352struct elf_mcount_loc {
 353	Elf_Ehdr *ehdr;
 354	Elf_Shdr *init_data_sec;
 355	uint64_t start_mcount_loc;
 356	uint64_t stop_mcount_loc;
 357};
 358
 359/* Fill the array with the content of the relocs */
 360static int fill_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc)
 361{
 362	Elf_Shdr *shdr_start;
 363	Elf_Rela *rel;
 364	unsigned int shnum;
 365	unsigned int count = 0;
 366	int shentsize;
 367	void *array_end = ptr + size;
 368
 369	shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
 370	shentsize = ehdr_shentsize(ehdr);
 371
 372	shnum = ehdr_shnum(ehdr);
 373	if (shnum == SHN_UNDEF)
 374		shnum = shdr_size(shdr_start);
 375
 376	for (int i = 0; i < shnum; i++) {
 377		Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
 378		void *end;
 379
 380		if (shdr_type(shdr) != SHT_RELA)
 381			continue;
 382
 383		rel = (void *)ehdr + shdr_offset(shdr);
 384		end = (void *)rel + shdr_size(shdr);
 385
 386		for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
 387			uint64_t offset = rela_offset(rel);
 388
 389			if (offset >= start_loc && offset < start_loc + size) {
 390				if (ptr + long_size > array_end) {
 391					snprintf(m_err, ERRSTR_MAXSZ,
 392						 "Too many relocations");
 393					return -1;
 394				}
 395
 396				/* Make sure this has the correct type */
 397				if (rela_info(rel) != rela_type) {
 398					snprintf(m_err, ERRSTR_MAXSZ,
 399						"rela has type %lx but expected %lx\n",
 400						(long)rela_info(rel), rela_type);
 401					return -1;
 402				}
 403
 404				if (long_size == 4)
 405					*(uint32_t *)ptr = rela_addend(rel);
 406				else
 407					*(uint64_t *)ptr = rela_addend(rel);
 408				ptr += long_size;
 409				count++;
 410			}
 411		}
 412	}
 413	return count;
 414}
 415
 416/* Put the sorted vals back into the relocation elements */
 417static void replace_relocs(void *ptr, uint64_t size, Elf_Ehdr *ehdr, uint64_t start_loc)
 418{
 419	Elf_Shdr *shdr_start;
 420	Elf_Rela *rel;
 421	unsigned int shnum;
 422	int shentsize;
 423
 424	shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
 425	shentsize = ehdr_shentsize(ehdr);
 426
 427	shnum = ehdr_shnum(ehdr);
 428	if (shnum == SHN_UNDEF)
 429		shnum = shdr_size(shdr_start);
 430
 431	for (int i = 0; i < shnum; i++) {
 432		Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
 433		void *end;
 434
 435		if (shdr_type(shdr) != SHT_RELA)
 436			continue;
 437
 438		rel = (void *)ehdr + shdr_offset(shdr);
 439		end = (void *)rel + shdr_size(shdr);
 440
 441		for (; (void *)rel < end; rel = (void *)rel + shdr_entsize(shdr)) {
 442			uint64_t offset = rela_offset(rel);
 443
 444			if (offset >= start_loc && offset < start_loc + size) {
 445				if (long_size == 4)
 446					rela_write_addend(rel, *(uint32_t *)ptr);
 447				else
 448					rela_write_addend(rel, *(uint64_t *)ptr);
 449				ptr += long_size;
 450			}
 451		}
 452	}
 453}
 454
 455static int fill_addrs(void *ptr, uint64_t size, void *addrs)
 456{
 457	void *end = ptr + size;
 458	int count = 0;
 459
 460	for (; ptr < end; ptr += long_size, addrs += long_size, count++) {
 461		if (long_size == 4)
 462			*(uint32_t *)ptr = elf_parser.r(addrs);
 463		else
 464			*(uint64_t *)ptr = elf_parser.r8(addrs);
 465	}
 466	return count;
 467}
 468
 469static void replace_addrs(void *ptr, uint64_t size, void *addrs)
 470{
 471	void *end = ptr + size;
 472
 473	for (; ptr < end; ptr += long_size, addrs += long_size) {
 474		if (long_size == 4)
 475			elf_parser.w(*(uint32_t *)ptr, addrs);
 476		else
 477			elf_parser.w8(*(uint64_t *)ptr, addrs);
 478	}
 479}
 480
 481/* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */
 482static void *sort_mcount_loc(void *arg)
 483{
 484	struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg;
 485	uint64_t offset = emloc->start_mcount_loc - shdr_addr(emloc->init_data_sec)
 486					+ shdr_offset(emloc->init_data_sec);
 487	uint64_t size = emloc->stop_mcount_loc - emloc->start_mcount_loc;
 488	unsigned char *start_loc = (void *)emloc->ehdr + offset;
 489	Elf_Ehdr *ehdr = emloc->ehdr;
 490	void *e_msg = NULL;
 491	void *vals;
 492	int count;
 493
 494	vals = malloc(long_size * size);
 495	if (!vals) {
 496		snprintf(m_err, ERRSTR_MAXSZ, "Failed to allocate sort array");
 497		pthread_exit(m_err);
 498	}
 499
 500	if (sort_reloc) {
 501		count = fill_relocs(vals, size, ehdr, emloc->start_mcount_loc);
 502		/* gcc may use relocs to save the addresses, but clang does not. */
 503		if (!count) {
 504			count = fill_addrs(vals, size, start_loc);
 505			sort_reloc = 0;
 506		}
 507	} else
 508		count = fill_addrs(vals, size, start_loc);
 509
 510	if (count < 0) {
 511		e_msg = m_err;
 512		goto out;
 513	}
 514
 515	if (count != size / long_size) {
 516		snprintf(m_err, ERRSTR_MAXSZ, "Expected %u mcount elements but found %u\n",
 517			(int)(size / long_size), count);
 518		e_msg = m_err;
 519		goto out;
 520	}
 521
 522	/* zero out any locations not found by function list */
 523	if (function_list_size) {
 524		for (void *ptr = vals; ptr < vals + size; ptr += long_size) {
 525			uint64_t key;
 526
 527			key = long_size == 4 ? *(uint32_t *)ptr : *(uint64_t *)ptr;
 528			if (!find_func(key)) {
 529				if (long_size == 4)
 530					*(uint32_t *)ptr = 0;
 531				else
 532					*(uint64_t *)ptr = 0;
 533			}
 534		}
 535	}
 536
 537	compare_values = long_size == 4 ? compare_values_32 : compare_values_64;
 538
 539	qsort(vals, count, long_size, compare_values);
 540
 541	if (sort_reloc)
 542		replace_relocs(vals, size, ehdr, emloc->start_mcount_loc);
 543	else
 544		replace_addrs(vals, size, start_loc);
 545
 546out:
 547	free(vals);
 548
 549	pthread_exit(e_msg);
 550}
 551
 552/* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */
 553static void get_mcount_loc(struct elf_mcount_loc *emloc, Elf_Shdr *symtab_sec,
 554			   const char *strtab)
 555{
 556	Elf_Sym *sym, *end_sym;
 557	int symentsize = shdr_entsize(symtab_sec);
 558	int found = 0;
 559
 560	sym = (void *)emloc->ehdr + shdr_offset(symtab_sec);
 561	end_sym = (void *)sym + shdr_size(symtab_sec);
 562
 563	while (sym < end_sym) {
 564		if (!strcmp(strtab + sym_name(sym), "__start_mcount_loc")) {
 565			emloc->start_mcount_loc = sym_value(sym);
 566			if (++found == 2)
 567				break;
 568		} else if (!strcmp(strtab + sym_name(sym), "__stop_mcount_loc")) {
 569			emloc->stop_mcount_loc = sym_value(sym);
 570			if (++found == 2)
 571				break;
 572		}
 573		sym = (void *)sym + symentsize;
 574	}
 575
 576	if (!emloc->start_mcount_loc) {
 577		fprintf(stderr, "get start_mcount_loc error!");
 578		return;
 579	}
 580
 581	if (!emloc->stop_mcount_loc) {
 582		fprintf(stderr, "get stop_mcount_loc error!");
 583		return;
 584	}
 585}
 586#else /* MCOUNT_SORT_ENABLED */
 587static inline int parse_symbols(const char *fname) { return 0; }
 588#endif
 589
 590static int do_sort(Elf_Ehdr *ehdr,
 591		   char const *const fname,
 592		   table_sort_t custom_sort)
 593{
 594	int rc = -1;
 595	Elf_Shdr *shdr_start;
 596	Elf_Shdr *strtab_sec = NULL;
 597	Elf_Shdr *symtab_sec = NULL;
 598	Elf_Shdr *extab_sec = NULL;
 599	Elf_Shdr *string_sec;
 600	Elf_Sym *sym;
 601	const Elf_Sym *symtab;
 602	Elf32_Word *symtab_shndx = NULL;
 603	Elf_Sym *sort_needed_sym = NULL;
 604	Elf_Shdr *sort_needed_sec;
 605	uint32_t *sort_needed_loc;
 606	void *sym_start;
 607	void *sym_end;
 608	const char *secstrings;
 609	const char *strtab;
 610	char *extab_image;
 611	int sort_need_index;
 612	int symentsize;
 613	int shentsize;
 614	int idx;
 615	int i;
 616	unsigned int shnum;
 617	unsigned int shstrndx;
 618#ifdef MCOUNT_SORT_ENABLED
 619	struct elf_mcount_loc mstruct = {0};
 620#endif
 621#ifdef UNWINDER_ORC_ENABLED
 622	unsigned int orc_ip_size = 0;
 623	unsigned int orc_size = 0;
 624	unsigned int orc_num_entries = 0;
 625#endif
 626
 627	shdr_start = (Elf_Shdr *)((char *)ehdr + ehdr_shoff(ehdr));
 628	shentsize = ehdr_shentsize(ehdr);
 629
 630	shstrndx = ehdr_shstrndx(ehdr);
 631	if (shstrndx == SHN_XINDEX)
 632		shstrndx = shdr_link(shdr_start);
 633	string_sec = get_index(shdr_start, shentsize, shstrndx);
 634	secstrings = (const char *)ehdr + shdr_offset(string_sec);
 635
 636	shnum = ehdr_shnum(ehdr);
 637	if (shnum == SHN_UNDEF)
 638		shnum = shdr_size(shdr_start);
 639
 640	for (i = 0; i < shnum; i++) {
 641		Elf_Shdr *shdr = get_index(shdr_start, shentsize, i);
 642
 643		idx = shdr_name(shdr);
 644		if (!strcmp(secstrings + idx, "__ex_table"))
 645			extab_sec = shdr;
 646		if (!strcmp(secstrings + idx, ".symtab"))
 647			symtab_sec = shdr;
 648		if (!strcmp(secstrings + idx, ".strtab"))
 649			strtab_sec = shdr;
 650
 651		if (shdr_type(shdr) == SHT_SYMTAB_SHNDX)
 652			symtab_shndx = (Elf32_Word *)((const char *)ehdr +
 653						      shdr_offset(shdr));
 654
 655#ifdef MCOUNT_SORT_ENABLED
 656		/* locate the .init.data section in vmlinux */
 657		if (!strcmp(secstrings + idx, ".init.data"))
 658			mstruct.init_data_sec = shdr;
 659#endif
 660
 661#ifdef UNWINDER_ORC_ENABLED
 662		/* locate the ORC unwind tables */
 663		if (!strcmp(secstrings + idx, ".orc_unwind_ip")) {
 664			orc_ip_size = shdr_size(shdr);
 665			g_orc_ip_table = (int *)((void *)ehdr +
 666						   shdr_offset(shdr));
 667		}
 668		if (!strcmp(secstrings + idx, ".orc_unwind")) {
 669			orc_size = shdr_size(shdr);
 670			g_orc_table = (struct orc_entry *)((void *)ehdr +
 671							     shdr_offset(shdr));
 672		}
 673#endif
 674	} /* for loop */
 675
 676#ifdef UNWINDER_ORC_ENABLED
 677	if (!g_orc_ip_table || !g_orc_table) {
 678		fprintf(stderr,
 679			"incomplete ORC unwind tables in file: %s\n", fname);
 680		goto out;
 681	}
 682
 683	orc_num_entries = orc_ip_size / sizeof(int);
 684	if (orc_ip_size % sizeof(int) != 0 ||
 685	    orc_size % sizeof(struct orc_entry) != 0 ||
 686	    orc_num_entries != orc_size / sizeof(struct orc_entry)) {
 687		fprintf(stderr,
 688			"inconsistent ORC unwind table entries in file: %s\n",
 689			fname);
 690		goto out;
 691	}
 692
 693	/* create thread to sort ORC unwind tables concurrently */
 694	if (pthread_create(&orc_sort_thread, NULL,
 695			   sort_orctable, &orc_ip_size)) {
 696		fprintf(stderr,
 697			"pthread_create orc_sort_thread failed '%s': %s\n",
 698			strerror(errno), fname);
 699		goto out;
 700	}
 701#endif
 702	if (!extab_sec) {
 703		fprintf(stderr,	"no __ex_table in file: %s\n", fname);
 704		goto out;
 705	}
 706
 707	if (!symtab_sec) {
 708		fprintf(stderr,	"no .symtab in file: %s\n", fname);
 709		goto out;
 710	}
 711
 712	if (!strtab_sec) {
 713		fprintf(stderr,	"no .strtab in file: %s\n", fname);
 714		goto out;
 715	}
 716
 717	extab_image = (void *)ehdr + shdr_offset(extab_sec);
 718	strtab = (const char *)ehdr + shdr_offset(strtab_sec);
 719	symtab = (const Elf_Sym *)((const char *)ehdr + shdr_offset(symtab_sec));
 720
 721#ifdef MCOUNT_SORT_ENABLED
 722	mstruct.ehdr = ehdr;
 723	get_mcount_loc(&mstruct, symtab_sec, strtab);
 724
 725	if (!mstruct.init_data_sec || !mstruct.start_mcount_loc || !mstruct.stop_mcount_loc) {
 726		fprintf(stderr,
 727			"incomplete mcount's sort in file: %s\n",
 728			fname);
 729		goto out;
 730	}
 731
 732	/* create thread to sort mcount_loc concurrently */
 733	if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) {
 734		fprintf(stderr,
 735			"pthread_create mcount_sort_thread failed '%s': %s\n",
 736			strerror(errno), fname);
 737		goto out;
 738	}
 739#endif
 740
 741	if (custom_sort) {
 742		custom_sort(extab_image, shdr_size(extab_sec));
 743	} else {
 744		int num_entries = shdr_size(extab_sec) / extable_ent_size;
 745		qsort(extab_image, num_entries,
 746		      extable_ent_size, compare_extable);
 747	}
 748
 749	/* find the flag main_extable_sort_needed */
 750	sym_start = (void *)ehdr + shdr_offset(symtab_sec);
 751	sym_end = sym_start + shdr_size(symtab_sec);
 752	symentsize = shdr_entsize(symtab_sec);
 753
 754	for (sym = sym_start; (void *)sym + symentsize < sym_end;
 755	     sym = (void *)sym + symentsize) {
 756		if (sym_type(sym) != STT_OBJECT)
 757			continue;
 758		if (!strcmp(strtab + sym_name(sym),
 759			    "main_extable_sort_needed")) {
 760			sort_needed_sym = sym;
 761			break;
 762		}
 763	}
 764
 765	if (!sort_needed_sym) {
 766		fprintf(stderr,
 767			"no main_extable_sort_needed symbol in file: %s\n",
 768			fname);
 769		goto out;
 770	}
 771
 772	sort_need_index = get_secindex(sym_shndx(sym),
 773				       ((void *)sort_needed_sym - (void *)symtab) / symentsize,
 774				       symtab_shndx);
 775	sort_needed_sec = get_index(shdr_start, shentsize, sort_need_index);
 776	sort_needed_loc = (void *)ehdr +
 777		shdr_offset(sort_needed_sec) +
 778		sym_value(sort_needed_sym) - shdr_addr(sort_needed_sec);
 779
 780	/* extable has been sorted, clear the flag */
 781	elf_parser.w(0, sort_needed_loc);
 782	rc = 0;
 783
 784out:
 785#ifdef UNWINDER_ORC_ENABLED
 786	if (orc_sort_thread) {
 787		void *retval = NULL;
 788		/* wait for ORC tables sort done */
 789		rc = pthread_join(orc_sort_thread, &retval);
 790		if (rc) {
 791			fprintf(stderr,
 792				"pthread_join failed '%s': %s\n",
 793				strerror(errno), fname);
 794		} else if (retval) {
 795			rc = -1;
 796			fprintf(stderr,
 797				"failed to sort ORC tables '%s': %s\n",
 798				(char *)retval, fname);
 799		}
 800	}
 801#endif
 802
 803#ifdef MCOUNT_SORT_ENABLED
 804	if (mcount_sort_thread) {
 805		void *retval = NULL;
 806		/* wait for mcount sort done */
 807		rc = pthread_join(mcount_sort_thread, &retval);
 808		if (rc) {
 809			fprintf(stderr,
 810				"pthread_join failed '%s': %s\n",
 811				strerror(errno), fname);
 812		} else if (retval) {
 813			rc = -1;
 814			fprintf(stderr,
 815				"failed to sort mcount '%s': %s\n",
 816				(char *)retval, fname);
 817		}
 818	}
 819#endif
 820	return rc;
 821}
 822
 823static int compare_relative_table(const void *a, const void *b)
 824{
 825	int32_t av = (int32_t)elf_parser.r(a);
 826	int32_t bv = (int32_t)elf_parser.r(b);
 827
 828	if (av < bv)
 829		return -1;
 830	if (av > bv)
 831		return 1;
 832	return 0;
 833}
 834
 835static void sort_relative_table(char *extab_image, int image_size)
 836{
 837	int i = 0;
 838
 839	/*
 840	 * Do the same thing the runtime sort does, first normalize to
 841	 * being relative to the start of the section.
 842	 */
 843	while (i < image_size) {
 844		uint32_t *loc = (uint32_t *)(extab_image + i);
 845		elf_parser.w(elf_parser.r(loc) + i, loc);
 846		i += 4;
 847	}
 848
 849	qsort(extab_image, image_size / 8, 8, compare_relative_table);
 850
 851	/* Now denormalize. */
 852	i = 0;
 853	while (i < image_size) {
 854		uint32_t *loc = (uint32_t *)(extab_image + i);
 855		elf_parser.w(elf_parser.r(loc) - i, loc);
 856		i += 4;
 857	}
 858}
 859
 860static void sort_relative_table_with_data(char *extab_image, int image_size)
 861{
 862	int i = 0;
 863
 864	while (i < image_size) {
 865		uint32_t *loc = (uint32_t *)(extab_image + i);
 866
 867		elf_parser.w(elf_parser.r(loc) + i, loc);
 868		elf_parser.w(elf_parser.r(loc + 1) + i + 4, loc + 1);
 869		/* Don't touch the fixup type or data */
 870
 871		i += sizeof(uint32_t) * 3;
 872	}
 873
 874	qsort(extab_image, image_size / 12, 12, compare_relative_table);
 875
 876	i = 0;
 877	while (i < image_size) {
 878		uint32_t *loc = (uint32_t *)(extab_image + i);
 879
 880		elf_parser.w(elf_parser.r(loc) - i, loc);
 881		elf_parser.w(elf_parser.r(loc + 1) - (i + 4), loc + 1);
 882		/* Don't touch the fixup type or data */
 883
 884		i += sizeof(uint32_t) * 3;
 885	}
 886}
 887
 888static int do_file(char const *const fname, void *addr)
 889{
 890	Elf_Ehdr *ehdr = addr;
 891	table_sort_t custom_sort = NULL;
 892
 893	switch (elf_map_machine(ehdr)) {
 894	case EM_AARCH64:
 895#ifdef MCOUNT_SORT_ENABLED
 896		sort_reloc = true;
 897		rela_type = 0x403;
 898		/* arm64 uses patchable function entry placing before function */
 899		before_func = 8;
 900#endif
 901		/* fallthrough */
 902	case EM_386:
 903	case EM_LOONGARCH:
 904	case EM_RISCV:
 905	case EM_S390:
 906	case EM_X86_64:
 907		custom_sort = sort_relative_table_with_data;
 908		break;
 909	case EM_PARISC:
 910	case EM_PPC:
 911	case EM_PPC64:
 912		custom_sort = sort_relative_table;
 913		break;
 914	case EM_ARCOMPACT:
 915	case EM_ARCV2:
 916	case EM_ARM:
 917	case EM_MICROBLAZE:
 918	case EM_MIPS:
 919	case EM_XTENSA:
 920		break;
 921	default:
 922		fprintf(stderr, "unrecognized e_machine %d %s\n",
 923			elf_parser.r2(&ehdr->e32.e_machine), fname);
 924		return -1;
 925	}
 926
 927	switch (elf_map_long_size(addr)) {
 928	case 4:
 929		compare_extable	= compare_extable_32,
 930		long_size		= 4;
 931		extable_ent_size	= 8;
 932
 933		if (elf_parser.r2(&ehdr->e32.e_ehsize) != sizeof(Elf32_Ehdr) ||
 934		    elf_parser.r2(&ehdr->e32.e_shentsize) != sizeof(Elf32_Shdr)) {
 935			fprintf(stderr,
 936				"unrecognized ET_EXEC/ET_DYN file: %s\n", fname);
 937			return -1;
 938		}
 939
 940		break;
 941	case 8:
 942		compare_extable	= compare_extable_64,
 943		long_size		= 8;
 944		extable_ent_size	= 16;
 945
 946		if (elf_parser.r2(&ehdr->e64.e_ehsize) != sizeof(Elf64_Ehdr) ||
 947		    elf_parser.r2(&ehdr->e64.e_shentsize) != sizeof(Elf64_Shdr)) {
 948			fprintf(stderr,
 949				"unrecognized ET_EXEC/ET_DYN file: %s\n",
 950				fname);
 951			return -1;
 952		}
 953
 954		break;
 955	default:
 956		fprintf(stderr, "unrecognized ELF class %d %s\n",
 957			ehdr->e32.e_ident[EI_CLASS], fname);
 958		return -1;
 959	}
 960
 961	return do_sort(ehdr, fname, custom_sort);
 962}
 963
 964int main(int argc, char *argv[])
 965{
 966	int i, n_error = 0;  /* gcc-4.3.0 false positive complaint */
 967	size_t size = 0;
 968	void *addr = NULL;
 969	int c;
 970
 971	while ((c = getopt(argc, argv, "s:")) >= 0) {
 972		switch (c) {
 973		case 's':
 974			if (parse_symbols(optarg) < 0) {
 975				fprintf(stderr, "Could not parse %s\n", optarg);
 976				return -1;
 977			}
 978			break;
 979		default:
 980			fprintf(stderr, "usage: sorttable [-s nm-file] vmlinux...\n");
 981			return 0;
 982		}
 983	}
 984
 985	if ((argc - optind) < 1) {
 986		fprintf(stderr, "usage: sorttable vmlinux...\n");
 987		return 0;
 988	}
 989
 990	/* Process each file in turn, allowing deep failure. */
 991	for (i = optind; i < argc; i++) {
 992		addr = elf_map(argv[i], &size, (1 << ET_EXEC) | (1 << ET_DYN));
 993		if (!addr) {
 994			++n_error;
 995			continue;
 996		}
 997
 998		if (do_file(argv[i], addr))
 999			++n_error;
1000
1001		elf_unmap(addr, size);
1002	}
1003
1004	return !!n_error;
1005}