linker.c

   1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2/*
   3 * BPF static linker
   4 *
   5 * Copyright (c) 2021 Facebook
   6 */
   7#ifndef _GNU_SOURCE
   8#define _GNU_SOURCE
   9#endif
  10
  11#include <stdbool.h>
  12#include <stddef.h>
  13#include <stdio.h>
  14#include <stdlib.h>
  15#include <string.h>
  16#include <unistd.h>
  17#include <errno.h>
  18#include <linux/err.h>
  19#include <linux/btf.h>
  20#include <elf.h>
  21#include <libelf.h>
  22#include <fcntl.h>
  23#include <sys/mman.h>
  24#include "libbpf.h"
  25#include "btf.h"
  26#include "libbpf_internal.h"
  27#include "strset.h"
  28
  29#define BTF_EXTERN_SEC ".extern"
  30
  31struct src_sec {
  32	const char *sec_name;
  33	/* positional (not necessarily ELF) index in an array of sections */
  34	int id;
  35	/* positional (not necessarily ELF) index of a matching section in a final object file */
  36	int dst_id;
  37	/* section data offset in a matching output section */
  38	int dst_off;
  39	/* whether section is omitted from the final ELF file */
  40	bool skipped;
  41	/* whether section is an ephemeral section, not mapped to an ELF section */
  42	bool ephemeral;
  43
  44	/* ELF info */
  45	size_t sec_idx;
  46	Elf_Scn *scn;
  47	Elf64_Shdr *shdr;
  48	Elf_Data *data;
  49
  50	/* corresponding BTF DATASEC type ID */
  51	int sec_type_id;
  52};
  53
  54struct src_obj {
  55	const char *filename;
  56	int fd;
  57	Elf *elf;
  58	/* Section header strings section index */
  59	size_t shstrs_sec_idx;
  60	/* SYMTAB section index */
  61	size_t symtab_sec_idx;
  62
  63	struct btf *btf;
  64	struct btf_ext *btf_ext;
  65
  66	/* List of sections (including ephemeral). Slot zero is unused. */
  67	struct src_sec *secs;
  68	int sec_cnt;
  69
  70	/* mapping of symbol indices from src to dst ELF */
  71	int *sym_map;
  72	/* mapping from the src BTF type IDs to dst ones */
  73	int *btf_type_map;
  74};
  75
  76/* single .BTF.ext data section */
  77struct btf_ext_sec_data {
  78	size_t rec_cnt;
  79	__u32 rec_sz;
  80	void *recs;
  81};
  82
  83struct glob_sym {
  84	/* ELF symbol index */
  85	int sym_idx;
  86	/* associated section id for .ksyms, .kconfig, etc, but not .extern */
  87	int sec_id;
  88	/* extern name offset in STRTAB */
  89	int name_off;
  90	/* optional associated BTF type ID */
  91	int btf_id;
  92	/* BTF type ID to which VAR/FUNC type is pointing to; used for
  93	 * rewriting types when extern VAR/FUNC is resolved to a concrete
  94	 * definition
  95	 */
  96	int underlying_btf_id;
  97	/* sec_var index in the corresponding dst_sec, if exists */
  98	int var_idx;
  99
 100	/* extern or resolved/global symbol */
 101	bool is_extern;
 102	/* weak or strong symbol, never goes back from strong to weak */
 103	bool is_weak;
 104};
 105
 106struct dst_sec {
 107	char *sec_name;
 108	/* positional (not necessarily ELF) index in an array of sections */
 109	int id;
 110
 111	bool ephemeral;
 112
 113	/* ELF info */
 114	size_t sec_idx;
 115	Elf_Scn *scn;
 116	Elf64_Shdr *shdr;
 117	Elf_Data *data;
 118
 119	/* final output section size */
 120	int sec_sz;
 121	/* final output contents of the section */
 122	void *raw_data;
 123
 124	/* corresponding STT_SECTION symbol index in SYMTAB */
 125	int sec_sym_idx;
 126
 127	/* section's DATASEC variable info, emitted on BTF finalization */
 128	bool has_btf;
 129	int sec_var_cnt;
 130	struct btf_var_secinfo *sec_vars;
 131
 132	/* section's .BTF.ext data */
 133	struct btf_ext_sec_data func_info;
 134	struct btf_ext_sec_data line_info;
 135	struct btf_ext_sec_data core_relo_info;
 136};
 137
 138struct bpf_linker {
 139	char *filename;
 140	int fd;
 141	Elf *elf;
 142	Elf64_Ehdr *elf_hdr;
 143	bool swapped_endian;
 144
 145	/* Output sections metadata */
 146	struct dst_sec *secs;
 147	int sec_cnt;
 148
 149	struct strset *strtab_strs; /* STRTAB unique strings */
 150	size_t strtab_sec_idx; /* STRTAB section index */
 151	size_t symtab_sec_idx; /* SYMTAB section index */
 152
 153	struct btf *btf;
 154	struct btf_ext *btf_ext;
 155
 156	/* global (including extern) ELF symbols */
 157	int glob_sym_cnt;
 158	struct glob_sym *glob_syms;
 159
 160	bool fd_is_owned;
 161};
 162
 163#define pr_warn_elf(fmt, ...)									\
 164	libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 165
 166static int init_output_elf(struct bpf_linker *linker);
 167
 168static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
 169			       const char *filename);
 170
 171static int linker_load_obj_file(struct bpf_linker *linker,
 172				struct src_obj *obj);
 173static int linker_sanity_check_elf(struct src_obj *obj);
 174static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 175static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 176static int linker_sanity_check_btf(struct src_obj *obj);
 177static int linker_sanity_check_btf_ext(struct src_obj *obj);
 178static int linker_fixup_btf(struct src_obj *obj);
 179static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 180static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 181static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
 182				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
 183static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 184static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 185static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 186
 187static int finalize_btf(struct bpf_linker *linker);
 188static int finalize_btf_ext(struct bpf_linker *linker);
 189
 190void bpf_linker__free(struct bpf_linker *linker)
 191{
 192	int i;
 193
 194	if (!linker)
 195		return;
 196
 197	free(linker->filename);
 198
 199	if (linker->elf)
 200		elf_end(linker->elf);
 201
 202	if (linker->fd >= 0 && linker->fd_is_owned)
 203		close(linker->fd);
 204
 205	strset__free(linker->strtab_strs);
 206
 207	btf__free(linker->btf);
 208	btf_ext__free(linker->btf_ext);
 209
 210	for (i = 1; i < linker->sec_cnt; i++) {
 211		struct dst_sec *sec = &linker->secs[i];
 212
 213		free(sec->sec_name);
 214		free(sec->raw_data);
 215		free(sec->sec_vars);
 216
 217		free(sec->func_info.recs);
 218		free(sec->line_info.recs);
 219		free(sec->core_relo_info.recs);
 220	}
 221	free(linker->secs);
 222
 223	free(linker->glob_syms);
 224	free(linker);
 225}
 226
 227struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 228{
 229	struct bpf_linker *linker;
 230	int err;
 231
 232	if (!OPTS_VALID(opts, bpf_linker_opts))
 233		return errno = EINVAL, NULL;
 234
 235	if (elf_version(EV_CURRENT) == EV_NONE) {
 236		pr_warn_elf("libelf initialization failed");
 237		return errno = EINVAL, NULL;
 238	}
 239
 240	linker = calloc(1, sizeof(*linker));
 241	if (!linker)
 242		return errno = ENOMEM, NULL;
 243
 244	linker->filename = strdup(filename);
 245	if (!linker->filename) {
 246		err = -ENOMEM;
 247		goto err_out;
 248	}
 249
 250	linker->fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
 251	if (linker->fd < 0) {
 252		err = -errno;
 253		pr_warn("failed to create '%s': %d\n", filename, err);
 254		goto err_out;
 255	}
 256	linker->fd_is_owned = true;
 257
 258	err = init_output_elf(linker);
 259	if (err)
 260		goto err_out;
 261
 262	return linker;
 263
 264err_out:
 265	bpf_linker__free(linker);
 266	return errno = -err, NULL;
 267}
 268
 269struct bpf_linker *bpf_linker__new_fd(int fd, struct bpf_linker_opts *opts)
 270{
 271	struct bpf_linker *linker;
 272	char filename[32];
 273	int err;
 274
 275	if (fd < 0)
 276		return errno = EINVAL, NULL;
 277
 278	if (!OPTS_VALID(opts, bpf_linker_opts))
 279		return errno = EINVAL, NULL;
 280
 281	if (elf_version(EV_CURRENT) == EV_NONE) {
 282		pr_warn_elf("libelf initialization failed");
 283		return errno = EINVAL, NULL;
 284	}
 285
 286	linker = calloc(1, sizeof(*linker));
 287	if (!linker)
 288		return errno = ENOMEM, NULL;
 289
 290	snprintf(filename, sizeof(filename), "fd:%d", fd);
 291	linker->filename = strdup(filename);
 292	if (!linker->filename) {
 293		err = -ENOMEM;
 294		goto err_out;
 295	}
 296
 297	linker->fd = fd;
 298	linker->fd_is_owned = false;
 299
 300	err = init_output_elf(linker);
 301	if (err)
 302		goto err_out;
 303
 304	return linker;
 305
 306err_out:
 307	bpf_linker__free(linker);
 308	return errno = -err, NULL;
 309}
 310
 311static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 312{
 313	struct dst_sec *secs = linker->secs, *sec;
 314	size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 315
 316	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 317	if (!secs)
 318		return NULL;
 319
 320	/* zero out newly allocated memory */
 321	memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 322
 323	linker->secs = secs;
 324	linker->sec_cnt = new_cnt;
 325
 326	sec = &linker->secs[new_cnt - 1];
 327	sec->id = new_cnt - 1;
 328	sec->sec_name = strdup(sec_name);
 329	if (!sec->sec_name)
 330		return NULL;
 331
 332	return sec;
 333}
 334
 335static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 336{
 337	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 338	Elf64_Sym *syms, *sym;
 339	size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 340
 341	syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 342	if (!syms)
 343		return NULL;
 344
 345	sym = &syms[sym_cnt];
 346	memset(sym, 0, sizeof(*sym));
 347
 348	symtab->raw_data = syms;
 349	symtab->sec_sz += sizeof(*sym);
 350	symtab->shdr->sh_size += sizeof(*sym);
 351	symtab->data->d_size += sizeof(*sym);
 352
 353	if (sym_idx)
 354		*sym_idx = sym_cnt;
 355
 356	return sym;
 357}
 358
 359static int init_output_elf(struct bpf_linker *linker)
 360{
 361	int err, str_off;
 362	Elf64_Sym *init_sym;
 363	struct dst_sec *sec;
 364
 365	linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 366	if (!linker->elf) {
 367		pr_warn_elf("failed to create ELF object");
 368		return -EINVAL;
 369	}
 370
 371	/* ELF header */
 372	linker->elf_hdr = elf64_newehdr(linker->elf);
 373	if (!linker->elf_hdr) {
 374		pr_warn_elf("failed to create ELF header");
 375		return -EINVAL;
 376	}
 377
 378	linker->elf_hdr->e_machine = EM_BPF;
 379	linker->elf_hdr->e_type = ET_REL;
 380	/* Set unknown ELF endianness, assign later from input files */
 381	linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
 382
 383	/* STRTAB */
 384	/* initialize strset with an empty string to conform to ELF */
 385	linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 386	if (libbpf_get_error(linker->strtab_strs))
 387		return libbpf_get_error(linker->strtab_strs);
 388
 389	sec = add_dst_sec(linker, ".strtab");
 390	if (!sec)
 391		return -ENOMEM;
 392
 393	sec->scn = elf_newscn(linker->elf);
 394	if (!sec->scn) {
 395		pr_warn_elf("failed to create STRTAB section");
 396		return -EINVAL;
 397	}
 398
 399	sec->shdr = elf64_getshdr(sec->scn);
 400	if (!sec->shdr)
 401		return -EINVAL;
 402
 403	sec->data = elf_newdata(sec->scn);
 404	if (!sec->data) {
 405		pr_warn_elf("failed to create STRTAB data");
 406		return -EINVAL;
 407	}
 408
 409	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 410	if (str_off < 0)
 411		return str_off;
 412
 413	sec->sec_idx = elf_ndxscn(sec->scn);
 414	linker->elf_hdr->e_shstrndx = sec->sec_idx;
 415	linker->strtab_sec_idx = sec->sec_idx;
 416
 417	sec->shdr->sh_name = str_off;
 418	sec->shdr->sh_type = SHT_STRTAB;
 419	sec->shdr->sh_flags = SHF_STRINGS;
 420	sec->shdr->sh_offset = 0;
 421	sec->shdr->sh_link = 0;
 422	sec->shdr->sh_info = 0;
 423	sec->shdr->sh_addralign = 1;
 424	sec->shdr->sh_size = sec->sec_sz = 0;
 425	sec->shdr->sh_entsize = 0;
 426
 427	/* SYMTAB */
 428	sec = add_dst_sec(linker, ".symtab");
 429	if (!sec)
 430		return -ENOMEM;
 431
 432	sec->scn = elf_newscn(linker->elf);
 433	if (!sec->scn) {
 434		pr_warn_elf("failed to create SYMTAB section");
 435		return -EINVAL;
 436	}
 437
 438	sec->shdr = elf64_getshdr(sec->scn);
 439	if (!sec->shdr)
 440		return -EINVAL;
 441
 442	sec->data = elf_newdata(sec->scn);
 443	if (!sec->data) {
 444		pr_warn_elf("failed to create SYMTAB data");
 445		return -EINVAL;
 446	}
 447	/* Ensure libelf translates byte-order of symbol records */
 448	sec->data->d_type = ELF_T_SYM;
 449
 450	str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 451	if (str_off < 0)
 452		return str_off;
 453
 454	sec->sec_idx = elf_ndxscn(sec->scn);
 455	linker->symtab_sec_idx = sec->sec_idx;
 456
 457	sec->shdr->sh_name = str_off;
 458	sec->shdr->sh_type = SHT_SYMTAB;
 459	sec->shdr->sh_flags = 0;
 460	sec->shdr->sh_offset = 0;
 461	sec->shdr->sh_link = linker->strtab_sec_idx;
 462	/* sh_info should be one greater than the index of the last local
 463	 * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 464	 */
 465	sec->shdr->sh_info = 0;
 466	sec->shdr->sh_addralign = 8;
 467	sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 468
 469	/* .BTF */
 470	linker->btf = btf__new_empty();
 471	err = libbpf_get_error(linker->btf);
 472	if (err)
 473		return err;
 474
 475	/* add the special all-zero symbol */
 476	init_sym = add_new_sym(linker, NULL);
 477	if (!init_sym)
 478		return -EINVAL;
 479
 480	init_sym->st_name = 0;
 481	init_sym->st_info = 0;
 482	init_sym->st_other = 0;
 483	init_sym->st_shndx = SHN_UNDEF;
 484	init_sym->st_value = 0;
 485	init_sym->st_size = 0;
 486
 487	return 0;
 488}
 489
 490static int bpf_linker_add_file(struct bpf_linker *linker, int fd,
 491			       const char *filename)
 492{
 493	struct src_obj obj = {};
 494	int err = 0;
 495
 496	obj.filename = filename;
 497	obj.fd = fd;
 498
 499	err = err ?: linker_load_obj_file(linker, &obj);
 500	err = err ?: linker_append_sec_data(linker, &obj);
 501	err = err ?: linker_append_elf_syms(linker, &obj);
 502	err = err ?: linker_append_elf_relos(linker, &obj);
 503	err = err ?: linker_append_btf(linker, &obj);
 504	err = err ?: linker_append_btf_ext(linker, &obj);
 505
 506	/* free up src_obj resources */
 507	free(obj.btf_type_map);
 508	btf__free(obj.btf);
 509	btf_ext__free(obj.btf_ext);
 510	free(obj.secs);
 511	free(obj.sym_map);
 512	if (obj.elf)
 513		elf_end(obj.elf);
 514
 515	return err;
 516}
 517
 518int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
 519			 const struct bpf_linker_file_opts *opts)
 520{
 521	int fd, err;
 522
 523	if (!OPTS_VALID(opts, bpf_linker_file_opts))
 524		return libbpf_err(-EINVAL);
 525
 526	if (!linker->elf)
 527		return libbpf_err(-EINVAL);
 528
 529	fd = open(filename, O_RDONLY | O_CLOEXEC);
 530	if (fd < 0) {
 531		err = -errno;
 532		pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
 533		return libbpf_err(err);
 534	}
 535
 536	err = bpf_linker_add_file(linker, fd, filename);
 537	close(fd);
 538	return libbpf_err(err);
 539}
 540
 541int bpf_linker__add_fd(struct bpf_linker *linker, int fd,
 542		       const struct bpf_linker_file_opts *opts)
 543{
 544	char filename[32];
 545	int err;
 546
 547	if (!OPTS_VALID(opts, bpf_linker_file_opts))
 548		return libbpf_err(-EINVAL);
 549
 550	if (!linker->elf)
 551		return libbpf_err(-EINVAL);
 552
 553	if (fd < 0)
 554		return libbpf_err(-EINVAL);
 555
 556	snprintf(filename, sizeof(filename), "fd:%d", fd);
 557	err = bpf_linker_add_file(linker, fd, filename);
 558	return libbpf_err(err);
 559}
 560
 561int bpf_linker__add_buf(struct bpf_linker *linker, void *buf, size_t buf_sz,
 562			const struct bpf_linker_file_opts *opts)
 563{
 564	char filename[32];
 565	int fd, written, ret;
 566
 567	if (!OPTS_VALID(opts, bpf_linker_file_opts))
 568		return libbpf_err(-EINVAL);
 569
 570	if (!linker->elf)
 571		return libbpf_err(-EINVAL);
 572
 573	snprintf(filename, sizeof(filename), "mem:%p+%zu", buf, buf_sz);
 574
 575	fd = sys_memfd_create(filename, 0);
 576	if (fd < 0) {
 577		ret = -errno;
 578		pr_warn("failed to create memfd '%s': %s\n", filename, errstr(ret));
 579		return libbpf_err(ret);
 580	}
 581
 582	written = 0;
 583	while (written < buf_sz) {
 584		ret = write(fd, buf + written, buf_sz - written);
 585		if (ret < 0) {
 586			ret = -errno;
 587			pr_warn("failed to write '%s': %s\n", filename, errstr(ret));
 588			goto err_out;
 589		}
 590		written += ret;
 591	}
 592
 593	ret = bpf_linker_add_file(linker, fd, filename);
 594err_out:
 595	close(fd);
 596	return libbpf_err(ret);
 597}
 598
 599static bool is_dwarf_sec_name(const char *name)
 600{
 601	/* approximation, but the actual list is too long */
 602	return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 603}
 604
 605static bool is_ignored_sec(struct src_sec *sec)
 606{
 607	Elf64_Shdr *shdr = sec->shdr;
 608	const char *name = sec->sec_name;
 609
 610	/* no special handling of .strtab */
 611	if (shdr->sh_type == SHT_STRTAB)
 612		return true;
 613
 614	/* ignore .llvm_addrsig section as well */
 615	if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 616		return true;
 617
 618	/* no subprograms will lead to an empty .text section, ignore it */
 619	if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 620	    strcmp(sec->sec_name, ".text") == 0)
 621		return true;
 622
 623	/* DWARF sections */
 624	if (is_dwarf_sec_name(sec->sec_name))
 625		return true;
 626
 627	if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 628		name += sizeof(".rel") - 1;
 629		/* DWARF section relocations */
 630		if (is_dwarf_sec_name(name))
 631			return true;
 632
 633		/* .BTF and .BTF.ext don't need relocations */
 634		if (strcmp(name, BTF_ELF_SEC) == 0 ||
 635		    strcmp(name, BTF_EXT_ELF_SEC) == 0)
 636			return true;
 637	}
 638
 639	return false;
 640}
 641
 642static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 643{
 644	struct src_sec *secs = obj->secs, *sec;
 645	size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 646
 647	secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 648	if (!secs)
 649		return NULL;
 650
 651	/* zero out newly allocated memory */
 652	memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 653
 654	obj->secs = secs;
 655	obj->sec_cnt = new_cnt;
 656
 657	sec = &obj->secs[new_cnt - 1];
 658	sec->id = new_cnt - 1;
 659	sec->sec_name = sec_name;
 660
 661	return sec;
 662}
 663
 664static int linker_load_obj_file(struct bpf_linker *linker,
 665				struct src_obj *obj)
 666{
 667	int err = 0;
 668	Elf_Scn *scn;
 669	Elf_Data *data;
 670	Elf64_Ehdr *ehdr;
 671	Elf64_Shdr *shdr;
 672	struct src_sec *sec;
 673	unsigned char obj_byteorder;
 674	unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
 675#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 676	const unsigned char host_byteorder = ELFDATA2LSB;
 677#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 678	const unsigned char host_byteorder = ELFDATA2MSB;
 679#else
 680#error "Unknown __BYTE_ORDER__"
 681#endif
 682
 683	pr_debug("linker: adding object file '%s'...\n", obj->filename);
 684
 685	obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 686	if (!obj->elf) {
 687		pr_warn_elf("failed to parse ELF file '%s'", obj->filename);
 688		return -EINVAL;
 689	}
 690
 691	/* Sanity check ELF file high-level properties */
 692	ehdr = elf64_getehdr(obj->elf);
 693	if (!ehdr) {
 694		pr_warn_elf("failed to get ELF header for %s", obj->filename);
 695		return -EINVAL;
 696	}
 697
 698	/* Linker output endianness set by first input object */
 699	obj_byteorder = ehdr->e_ident[EI_DATA];
 700	if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
 701		err = -EOPNOTSUPP;
 702		pr_warn("unknown byte order of ELF file %s\n", obj->filename);
 703		return err;
 704	}
 705	if (link_byteorder == ELFDATANONE) {
 706		linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
 707		linker->swapped_endian = obj_byteorder != host_byteorder;
 708		pr_debug("linker: set %s-endian output byte order\n",
 709			 obj_byteorder == ELFDATA2MSB ? "big" : "little");
 710	} else if (link_byteorder != obj_byteorder) {
 711		err = -EOPNOTSUPP;
 712		pr_warn("byte order mismatch with ELF file %s\n", obj->filename);
 713		return err;
 714	}
 715
 716	if (ehdr->e_type != ET_REL
 717	    || ehdr->e_machine != EM_BPF
 718	    || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 719		err = -EOPNOTSUPP;
 720		pr_warn_elf("unsupported kind of ELF file %s", obj->filename);
 721		return err;
 722	}
 723
 724	if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 725		pr_warn_elf("failed to get SHSTRTAB section index for %s", obj->filename);
 726		return -EINVAL;
 727	}
 728
 729	scn = NULL;
 730	while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 731		size_t sec_idx = elf_ndxscn(scn);
 732		const char *sec_name;
 733
 734		shdr = elf64_getshdr(scn);
 735		if (!shdr) {
 736			pr_warn_elf("failed to get section #%zu header for %s",
 737				    sec_idx, obj->filename);
 738			return -EINVAL;
 739		}
 740
 741		sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 742		if (!sec_name) {
 743			pr_warn_elf("failed to get section #%zu name for %s",
 744				    sec_idx, obj->filename);
 745			return -EINVAL;
 746		}
 747
 748		data = elf_getdata(scn, 0);
 749		if (!data) {
 750			pr_warn_elf("failed to get section #%zu (%s) data from %s",
 751				    sec_idx, sec_name, obj->filename);
 752			return -EINVAL;
 753		}
 754
 755		sec = add_src_sec(obj, sec_name);
 756		if (!sec)
 757			return -ENOMEM;
 758
 759		sec->scn = scn;
 760		sec->shdr = shdr;
 761		sec->data = data;
 762		sec->sec_idx = elf_ndxscn(scn);
 763
 764		if (is_ignored_sec(sec)) {
 765			sec->skipped = true;
 766			continue;
 767		}
 768
 769		switch (shdr->sh_type) {
 770		case SHT_SYMTAB:
 771			if (obj->symtab_sec_idx) {
 772				err = -EOPNOTSUPP;
 773				pr_warn("multiple SYMTAB sections found, not supported\n");
 774				return err;
 775			}
 776			obj->symtab_sec_idx = sec_idx;
 777			break;
 778		case SHT_STRTAB:
 779			/* we'll construct our own string table */
 780			break;
 781		case SHT_PROGBITS:
 782			if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 783				obj->btf = btf__new(data->d_buf, shdr->sh_size);
 784				err = libbpf_get_error(obj->btf);
 785				if (err) {
 786					pr_warn("failed to parse .BTF from %s: %s\n",
 787						obj->filename, errstr(err));
 788					return err;
 789				}
 790				sec->skipped = true;
 791				continue;
 792			}
 793			if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 794				obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 795				err = libbpf_get_error(obj->btf_ext);
 796				if (err) {
 797					pr_warn("failed to parse .BTF.ext from '%s': %s\n",
 798						obj->filename, errstr(err));
 799					return err;
 800				}
 801				sec->skipped = true;
 802				continue;
 803			}
 804
 805			/* data & code */
 806			break;
 807		case SHT_NOBITS:
 808			/* BSS */
 809			break;
 810		case SHT_REL:
 811			/* relocations */
 812			break;
 813		default:
 814			pr_warn("unrecognized section #%zu (%s) in %s\n",
 815				sec_idx, sec_name, obj->filename);
 816			err = -EINVAL;
 817			return err;
 818		}
 819	}
 820
 821	err = err ?: linker_sanity_check_elf(obj);
 822	err = err ?: linker_sanity_check_btf(obj);
 823	err = err ?: linker_sanity_check_btf_ext(obj);
 824	err = err ?: linker_fixup_btf(obj);
 825
 826	return err;
 827}
 828
 829static int linker_sanity_check_elf(struct src_obj *obj)
 830{
 831	struct src_sec *sec;
 832	int i, err;
 833
 834	if (!obj->symtab_sec_idx) {
 835		pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 836		return -EINVAL;
 837	}
 838	if (!obj->shstrs_sec_idx) {
 839		pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 840		return -EINVAL;
 841	}
 842
 843	for (i = 1; i < obj->sec_cnt; i++) {
 844		sec = &obj->secs[i];
 845
 846		if (sec->sec_name[0] == '\0') {
 847			pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 848			return -EINVAL;
 849		}
 850
 851		if (is_dwarf_sec_name(sec->sec_name))
 852			continue;
 853
 854		if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) {
 855			pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n",
 856				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 857				obj->filename);
 858			return -EINVAL;
 859		}
 860		if (sec->shdr->sh_addralign != sec->data->d_align) {
 861			pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n",
 862				sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign,
 863				(long long unsigned)sec->data->d_align, obj->filename);
 864			return -EINVAL;
 865		}
 866
 867		if (sec->shdr->sh_size != sec->data->d_size) {
 868			pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n",
 869				sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 870				(long long unsigned)sec->data->d_size, obj->filename);
 871			return -EINVAL;
 872		}
 873
 874		switch (sec->shdr->sh_type) {
 875		case SHT_SYMTAB:
 876			err = linker_sanity_check_elf_symtab(obj, sec);
 877			if (err)
 878				return err;
 879			break;
 880		case SHT_STRTAB:
 881			break;
 882		case SHT_PROGBITS:
 883			if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 884				if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) {
 885					pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n",
 886						sec->sec_idx, (long long unsigned)sec->shdr->sh_size,
 887						obj->filename);
 888					return -EINVAL;
 889				}
 890			}
 891			break;
 892		case SHT_NOBITS:
 893			break;
 894		case SHT_REL:
 895			err = linker_sanity_check_elf_relos(obj, sec);
 896			if (err)
 897				return err;
 898			break;
 899		case SHT_LLVM_ADDRSIG:
 900			break;
 901		default:
 902			pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 903				sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 904			return -EINVAL;
 905		}
 906	}
 907
 908	return 0;
 909}
 910
 911static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 912{
 913	struct src_sec *link_sec;
 914	Elf64_Sym *sym;
 915	int i, n;
 916
 917	if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 918		return -EINVAL;
 919	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 920		return -EINVAL;
 921
 922	if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 923		pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 924			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 925		return -EINVAL;
 926	}
 927	link_sec = &obj->secs[sec->shdr->sh_link];
 928	if (link_sec->shdr->sh_type != SHT_STRTAB) {
 929		pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 930			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 931		return -EINVAL;
 932	}
 933
 934	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 935	sym = sec->data->d_buf;
 936	for (i = 0; i < n; i++, sym++) {
 937		int sym_type = ELF64_ST_TYPE(sym->st_info);
 938		int sym_bind = ELF64_ST_BIND(sym->st_info);
 939		int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 940
 941		if (i == 0) {
 942			if (sym->st_name != 0 || sym->st_info != 0
 943			    || sym->st_other != 0 || sym->st_shndx != 0
 944			    || sym->st_value != 0 || sym->st_size != 0) {
 945				pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 946				return -EINVAL;
 947			}
 948			continue;
 949		}
 950		if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 951			pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 952				i, sec->sec_idx, sym_bind);
 953			return -EINVAL;
 954		}
 955		if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 956			pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 957				i, sec->sec_idx, sym_vis);
 958			return -EINVAL;
 959		}
 960		if (sym->st_shndx == 0) {
 961			if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 962			    || sym->st_value != 0 || sym->st_size != 0) {
 963				pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 964					i, obj->filename);
 965
 966				return -EINVAL;
 967			}
 968			continue;
 969		}
 970		if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 971			pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 972				i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 973			return -EINVAL;
 974		}
 975		if (sym_type == STT_SECTION) {
 976			if (sym->st_value != 0)
 977				return -EINVAL;
 978			continue;
 979		}
 980	}
 981
 982	return 0;
 983}
 984
 985static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 986{
 987	struct src_sec *link_sec, *sym_sec;
 988	Elf64_Rel *relo;
 989	int i, n;
 990
 991	if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 992		return -EINVAL;
 993	if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 994		return -EINVAL;
 995
 996	/* SHT_REL's sh_link should point to SYMTAB */
 997	if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 998		pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 999			sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
1000		return -EINVAL;
1001	}
1002
1003	/* SHT_REL's sh_info points to relocated section */
1004	if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
1005		pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
1006			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1007		return -EINVAL;
1008	}
1009	link_sec = &obj->secs[sec->shdr->sh_info];
1010
1011	/* .rel<secname> -> <secname> pattern is followed */
1012	if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
1013	    || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
1014		pr_warn("ELF relo section #%zu name has invalid name in %s\n",
1015			sec->sec_idx, obj->filename);
1016		return -EINVAL;
1017	}
1018
1019	/* don't further validate relocations for ignored sections */
1020	if (link_sec->skipped)
1021		return 0;
1022
1023	/* relocatable section is data or instructions */
1024	if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
1025		pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
1026			sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
1027		return -EINVAL;
1028	}
1029
1030	/* check sanity of each relocation */
1031	n = sec->shdr->sh_size / sec->shdr->sh_entsize;
1032	relo = sec->data->d_buf;
1033	sym_sec = &obj->secs[obj->symtab_sec_idx];
1034	for (i = 0; i < n; i++, relo++) {
1035		size_t sym_idx = ELF64_R_SYM(relo->r_info);
1036		size_t sym_type = ELF64_R_TYPE(relo->r_info);
1037
1038		if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
1039		    sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
1040			pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
1041				i, sec->sec_idx, sym_type, obj->filename);
1042			return -EINVAL;
1043		}
1044
1045		if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
1046			pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
1047				i, sec->sec_idx, sym_idx, obj->filename);
1048			return -EINVAL;
1049		}
1050
1051		if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
1052			if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
1053				pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
1054					i, sec->sec_idx, sym_idx, obj->filename);
1055				return -EINVAL;
1056			}
1057		}
1058	}
1059
1060	return 0;
1061}
1062
1063static int check_btf_type_id(__u32 *type_id, void *ctx)
1064{
1065	struct btf *btf = ctx;
1066
1067	if (*type_id >= btf__type_cnt(btf))
1068		return -EINVAL;
1069
1070	return 0;
1071}
1072
1073static int check_btf_str_off(__u32 *str_off, void *ctx)
1074{
1075	struct btf *btf = ctx;
1076	const char *s;
1077
1078	s = btf__str_by_offset(btf, *str_off);
1079
1080	if (!s)
1081		return -EINVAL;
1082
1083	return 0;
1084}
1085
1086static int linker_sanity_check_btf(struct src_obj *obj)
1087{
1088	struct btf_type *t;
1089	int i, n, err;
1090
1091	if (!obj->btf)
1092		return 0;
1093
1094	n = btf__type_cnt(obj->btf);
1095	for (i = 1; i < n; i++) {
1096		struct btf_field_iter it;
1097		__u32 *type_id, *str_off;
1098
1099		t = btf_type_by_id(obj->btf, i);
1100
1101		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
1102		if (err)
1103			return err;
1104		while ((type_id = btf_field_iter_next(&it))) {
1105			if (*type_id >= n)
1106				return -EINVAL;
1107		}
1108
1109		err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1110		if (err)
1111			return err;
1112		while ((str_off = btf_field_iter_next(&it))) {
1113			if (!btf__str_by_offset(obj->btf, *str_off))
1114				return -EINVAL;
1115		}
1116	}
1117
1118	return 0;
1119}
1120
1121static int linker_sanity_check_btf_ext(struct src_obj *obj)
1122{
1123	int err = 0;
1124
1125	if (!obj->btf_ext)
1126		return 0;
1127
1128	/* can't use .BTF.ext without .BTF */
1129	if (!obj->btf)
1130		return -EINVAL;
1131
1132	err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
1133	err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
1134	if (err)
1135		return err;
1136
1137	return 0;
1138}
1139
1140static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
1141{
1142	Elf_Scn *scn;
1143	Elf_Data *data;
1144	Elf64_Shdr *shdr;
1145	int name_off;
1146
1147	dst_sec->sec_sz = 0;
1148	dst_sec->sec_idx = 0;
1149	dst_sec->ephemeral = src_sec->ephemeral;
1150
1151	/* ephemeral sections are just thin section shells lacking most parts */
1152	if (src_sec->ephemeral)
1153		return 0;
1154
1155	scn = elf_newscn(linker->elf);
1156	if (!scn)
1157		return -ENOMEM;
1158	data = elf_newdata(scn);
1159	if (!data)
1160		return -ENOMEM;
1161	shdr = elf64_getshdr(scn);
1162	if (!shdr)
1163		return -ENOMEM;
1164
1165	dst_sec->scn = scn;
1166	dst_sec->shdr = shdr;
1167	dst_sec->data = data;
1168	dst_sec->sec_idx = elf_ndxscn(scn);
1169
1170	name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1171	if (name_off < 0)
1172		return name_off;
1173
1174	shdr->sh_name = name_off;
1175	shdr->sh_type = src_sec->shdr->sh_type;
1176	shdr->sh_flags = src_sec->shdr->sh_flags;
1177	shdr->sh_size = 0;
1178	/* sh_link and sh_info have different meaning for different types of
1179	 * sections, so we leave it up to the caller code to fill them in, if
1180	 * necessary
1181	 */
1182	shdr->sh_link = 0;
1183	shdr->sh_info = 0;
1184	shdr->sh_addralign = src_sec->shdr->sh_addralign;
1185	shdr->sh_entsize = src_sec->shdr->sh_entsize;
1186
1187	data->d_type = src_sec->data->d_type;
1188	data->d_size = 0;
1189	data->d_buf = NULL;
1190	data->d_align = src_sec->data->d_align;
1191	data->d_off = 0;
1192
1193	return 0;
1194}
1195
1196static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1197{
1198	struct dst_sec *sec;
1199	int i;
1200
1201	for (i = 1; i < linker->sec_cnt; i++) {
1202		sec = &linker->secs[i];
1203
1204		if (strcmp(sec->sec_name, sec_name) == 0)
1205			return sec;
1206	}
1207
1208	return NULL;
1209}
1210
1211static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1212{
1213	if (dst->ephemeral || src->ephemeral)
1214		return true;
1215
1216	if (dst->shdr->sh_type != src->shdr->sh_type) {
1217		pr_warn("sec %s types mismatch\n", dst->sec_name);
1218		return false;
1219	}
1220	if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1221		pr_warn("sec %s flags mismatch\n", dst->sec_name);
1222		return false;
1223	}
1224	if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1225		pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1226		return false;
1227	}
1228
1229	return true;
1230}
1231
1232static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1233{
1234	if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1235		return false;
1236	if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1237		return false;
1238	return true;
1239}
1240
1241static bool is_exec_sec(struct dst_sec *sec)
1242{
1243	if (!sec || sec->ephemeral)
1244		return false;
1245	return (sec->shdr->sh_type == SHT_PROGBITS) &&
1246	       (sec->shdr->sh_flags & SHF_EXECINSTR);
1247}
1248
1249static void exec_sec_bswap(void *raw_data, int size)
1250{
1251	const int insn_cnt = size / sizeof(struct bpf_insn);
1252	struct bpf_insn *insn = raw_data;
1253	int i;
1254
1255	for (i = 0; i < insn_cnt; i++, insn++)
1256		bpf_insn_bswap(insn);
1257}
1258
1259static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1260{
1261	void *tmp;
1262	size_t dst_align, src_align;
1263	size_t dst_align_sz, dst_final_sz;
1264	int err;
1265
1266	/* Ephemeral source section doesn't contribute anything to ELF
1267	 * section data.
1268	 */
1269	if (src->ephemeral)
1270		return 0;
1271
1272	/* Some sections (like .maps) can contain both externs (and thus be
1273	 * ephemeral) and non-externs (map definitions). So it's possible that
1274	 * it has to be "upgraded" from ephemeral to non-ephemeral when the
1275	 * first non-ephemeral entity appears. In such case, we add ELF
1276	 * section, data, etc.
1277	 */
1278	if (dst->ephemeral) {
1279		err = init_sec(linker, dst, src);
1280		if (err)
1281			return err;
1282	}
1283
1284	dst_align = dst->shdr->sh_addralign;
1285	src_align = src->shdr->sh_addralign;
1286	if (dst_align == 0)
1287		dst_align = 1;
1288	if (dst_align < src_align)
1289		dst_align = src_align;
1290
1291	dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1292
1293	/* no need to re-align final size */
1294	dst_final_sz = dst_align_sz + src->shdr->sh_size;
1295
1296	if (src->shdr->sh_type != SHT_NOBITS) {
1297		tmp = realloc(dst->raw_data, dst_final_sz);
1298		/* If dst_align_sz == 0, realloc() behaves in a special way:
1299		 * 1. When dst->raw_data is NULL it returns:
1300		 *    "either NULL or a pointer suitable to be passed to free()" [1].
1301		 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL,
1302		 *    thus invalidating any "pointer suitable to be passed to free()" obtained
1303		 *    at step (1).
1304		 *
1305		 * The dst_align_sz > 0 check avoids error exit after (2), otherwise
1306		 * dst->raw_data would be freed again in bpf_linker__free().
1307		 *
1308		 * [1] man 3 realloc
1309		 */
1310		if (!tmp && dst_align_sz > 0)
1311			return -ENOMEM;
1312		dst->raw_data = tmp;
1313
1314		/* pad dst section, if it's alignment forced size increase */
1315		memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1316		/* now copy src data at a properly aligned offset */
1317		memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1318
1319		/* convert added bpf insns to native byte-order */
1320		if (linker->swapped_endian && is_exec_sec(dst))
1321			exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1322	}
1323
1324	dst->sec_sz = dst_final_sz;
1325	dst->shdr->sh_size = dst_final_sz;
1326	dst->data->d_size = dst_final_sz;
1327
1328	dst->shdr->sh_addralign = dst_align;
1329	dst->data->d_align = dst_align;
1330
1331	src->dst_off = dst_align_sz;
1332
1333	return 0;
1334}
1335
1336static bool is_data_sec(struct src_sec *sec)
1337{
1338	if (!sec || sec->skipped)
1339		return false;
1340	/* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1341	if (sec->ephemeral)
1342		return true;
1343	return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1344}
1345
1346static bool is_relo_sec(struct src_sec *sec)
1347{
1348	if (!sec || sec->skipped || sec->ephemeral)
1349		return false;
1350	return sec->shdr->sh_type == SHT_REL;
1351}
1352
1353static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1354{
1355	int i, err;
1356
1357	for (i = 1; i < obj->sec_cnt; i++) {
1358		struct src_sec *src_sec;
1359		struct dst_sec *dst_sec;
1360
1361		src_sec = &obj->secs[i];
1362		if (!is_data_sec(src_sec))
1363			continue;
1364
1365		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1366		if (!dst_sec) {
1367			dst_sec = add_dst_sec(linker, src_sec->sec_name);
1368			if (!dst_sec)
1369				return -ENOMEM;
1370			err = init_sec(linker, dst_sec, src_sec);
1371			if (err) {
1372				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1373				return err;
1374			}
1375		} else {
1376			if (!secs_match(dst_sec, src_sec)) {
1377				pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1378				return -EINVAL;
1379			}
1380
1381			/* "license" and "version" sections are deduped */
1382			if (strcmp(src_sec->sec_name, "license") == 0
1383			    || strcmp(src_sec->sec_name, "version") == 0) {
1384				if (!sec_content_is_same(dst_sec, src_sec)) {
1385					pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1386					return -EINVAL;
1387				}
1388				src_sec->skipped = true;
1389				src_sec->dst_id = dst_sec->id;
1390				continue;
1391			}
1392		}
1393
1394		/* record mapped section index */
1395		src_sec->dst_id = dst_sec->id;
1396
1397		err = extend_sec(linker, dst_sec, src_sec);
1398		if (err)
1399			return err;
1400	}
1401
1402	return 0;
1403}
1404
1405static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1406{
1407	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1408	Elf64_Sym *sym = symtab->data->d_buf;
1409	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1410	int str_sec_idx = symtab->shdr->sh_link;
1411	const char *sym_name;
1412
1413	obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1414	if (!obj->sym_map)
1415		return -ENOMEM;
1416
1417	for (i = 0; i < n; i++, sym++) {
1418		/* We already validated all-zero symbol #0 and we already
1419		 * appended it preventively to the final SYMTAB, so skip it.
1420		 */
1421		if (i == 0)
1422			continue;
1423
1424		sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1425		if (!sym_name) {
1426			pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1427			return -EINVAL;
1428		}
1429
1430		err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1431		if (err)
1432			return err;
1433	}
1434
1435	return 0;
1436}
1437
1438static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1439{
1440	struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1441	Elf64_Sym *syms = symtab->raw_data;
1442
1443	return &syms[sym_idx];
1444}
1445
1446static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1447{
1448	struct glob_sym *glob_sym;
1449	const char *name;
1450	int i;
1451
1452	for (i = 0; i < linker->glob_sym_cnt; i++) {
1453		glob_sym = &linker->glob_syms[i];
1454		name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1455
1456		if (strcmp(name, sym_name) == 0)
1457			return glob_sym;
1458	}
1459
1460	return NULL;
1461}
1462
1463static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1464{
1465	struct glob_sym *syms, *sym;
1466
1467	syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1468				   sizeof(*linker->glob_syms));
1469	if (!syms)
1470		return NULL;
1471
1472	sym = &syms[linker->glob_sym_cnt];
1473	memset(sym, 0, sizeof(*sym));
1474	sym->var_idx = -1;
1475
1476	linker->glob_syms = syms;
1477	linker->glob_sym_cnt++;
1478
1479	return sym;
1480}
1481
1482static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1483				 const struct btf *btf1, __u32 id1,
1484				 const struct btf *btf2, __u32 id2)
1485{
1486	const struct btf_type *t1, *t2;
1487	bool is_static1, is_static2;
1488	const char *n1, *n2;
1489	int i, n;
1490
1491recur:
1492	n1 = n2 = NULL;
1493	t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1494	t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1495
1496	/* check if only one side is FWD, otherwise handle with common logic */
1497	if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1498		n1 = btf__str_by_offset(btf1, t1->name_off);
1499		n2 = btf__str_by_offset(btf2, t2->name_off);
1500		if (strcmp(n1, n2) != 0) {
1501			pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1502				sym_name, n1, n2);
1503			return false;
1504		}
1505		/* validate if FWD kind matches concrete kind */
1506		if (btf_is_fwd(t1)) {
1507			if (btf_kflag(t1) && btf_is_union(t2))
1508				return true;
1509			if (!btf_kflag(t1) && btf_is_struct(t2))
1510				return true;
1511			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1512				sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1513		} else {
1514			if (btf_kflag(t2) && btf_is_union(t1))
1515				return true;
1516			if (!btf_kflag(t2) && btf_is_struct(t1))
1517				return true;
1518			pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1519				sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1520		}
1521		return false;
1522	}
1523
1524	if (btf_kind(t1) != btf_kind(t2)) {
1525		pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1526			sym_name, btf_kind_str(t1), btf_kind_str(t2));
1527		return false;
1528	}
1529
1530	switch (btf_kind(t1)) {
1531	case BTF_KIND_STRUCT:
1532	case BTF_KIND_UNION:
1533	case BTF_KIND_ENUM:
1534	case BTF_KIND_ENUM64:
1535	case BTF_KIND_FWD:
1536	case BTF_KIND_FUNC:
1537	case BTF_KIND_VAR:
1538		n1 = btf__str_by_offset(btf1, t1->name_off);
1539		n2 = btf__str_by_offset(btf2, t2->name_off);
1540		if (strcmp(n1, n2) != 0) {
1541			pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1542				sym_name, btf_kind_str(t1), n1, n2);
1543			return false;
1544		}
1545		break;
1546	default:
1547		break;
1548	}
1549
1550	switch (btf_kind(t1)) {
1551	case BTF_KIND_UNKN: /* void */
1552	case BTF_KIND_FWD:
1553		return true;
1554	case BTF_KIND_INT:
1555	case BTF_KIND_FLOAT:
1556	case BTF_KIND_ENUM:
1557	case BTF_KIND_ENUM64:
1558		/* ignore encoding for int and enum values for enum */
1559		if (t1->size != t2->size) {
1560			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1561				sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1562			return false;
1563		}
1564		return true;
1565	case BTF_KIND_PTR:
1566		/* just validate overall shape of the referenced type, so no
1567		 * contents comparison for struct/union, and allowed fwd vs
1568		 * struct/union
1569		 */
1570		exact = false;
1571		id1 = t1->type;
1572		id2 = t2->type;
1573		goto recur;
1574	case BTF_KIND_ARRAY:
1575		/* ignore index type and array size */
1576		id1 = btf_array(t1)->type;
1577		id2 = btf_array(t2)->type;
1578		goto recur;
1579	case BTF_KIND_FUNC:
1580		/* extern and global linkages are compatible */
1581		is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1582		is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1583		if (is_static1 != is_static2) {
1584			pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1585			return false;
1586		}
1587
1588		id1 = t1->type;
1589		id2 = t2->type;
1590		goto recur;
1591	case BTF_KIND_VAR:
1592		/* extern and global linkages are compatible */
1593		is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1594		is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1595		if (is_static1 != is_static2) {
1596			pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1597			return false;
1598		}
1599
1600		id1 = t1->type;
1601		id2 = t2->type;
1602		goto recur;
1603	case BTF_KIND_STRUCT:
1604	case BTF_KIND_UNION: {
1605		const struct btf_member *m1, *m2;
1606
1607		if (!exact)
1608			return true;
1609
1610		if (btf_vlen(t1) != btf_vlen(t2)) {
1611			pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1612				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1613			return false;
1614		}
1615
1616		n = btf_vlen(t1);
1617		m1 = btf_members(t1);
1618		m2 = btf_members(t2);
1619		for (i = 0; i < n; i++, m1++, m2++) {
1620			n1 = btf__str_by_offset(btf1, m1->name_off);
1621			n2 = btf__str_by_offset(btf2, m2->name_off);
1622			if (strcmp(n1, n2) != 0) {
1623				pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1624					sym_name, i, n1, n2);
1625				return false;
1626			}
1627			if (m1->offset != m2->offset) {
1628				pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1629					sym_name, i, n1);
1630				return false;
1631			}
1632			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1633				return false;
1634		}
1635
1636		return true;
1637	}
1638	case BTF_KIND_FUNC_PROTO: {
1639		const struct btf_param *m1, *m2;
1640
1641		if (btf_vlen(t1) != btf_vlen(t2)) {
1642			pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1643				sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1644			return false;
1645		}
1646
1647		n = btf_vlen(t1);
1648		m1 = btf_params(t1);
1649		m2 = btf_params(t2);
1650		for (i = 0; i < n; i++, m1++, m2++) {
1651			/* ignore func arg names */
1652			if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1653				return false;
1654		}
1655
1656		/* now check return type as well */
1657		id1 = t1->type;
1658		id2 = t2->type;
1659		goto recur;
1660	}
1661
1662	/* skip_mods_and_typedefs() make this impossible */
1663	case BTF_KIND_TYPEDEF:
1664	case BTF_KIND_VOLATILE:
1665	case BTF_KIND_CONST:
1666	case BTF_KIND_RESTRICT:
1667	/* DATASECs are never compared with each other */
1668	case BTF_KIND_DATASEC:
1669	default:
1670		pr_warn("global '%s': unsupported BTF kind %s\n",
1671			sym_name, btf_kind_str(t1));
1672		return false;
1673	}
1674}
1675
1676static bool map_defs_match(const char *sym_name,
1677			   const struct btf *main_btf,
1678			   const struct btf_map_def *main_def,
1679			   const struct btf_map_def *main_inner_def,
1680			   const struct btf *extra_btf,
1681			   const struct btf_map_def *extra_def,
1682			   const struct btf_map_def *extra_inner_def)
1683{
1684	const char *reason;
1685
1686	if (main_def->map_type != extra_def->map_type) {
1687		reason = "type";
1688		goto mismatch;
1689	}
1690
1691	/* check key type/size match */
1692	if (main_def->key_size != extra_def->key_size) {
1693		reason = "key_size";
1694		goto mismatch;
1695	}
1696	if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1697		reason = "key type";
1698		goto mismatch;
1699	}
1700	if ((main_def->parts & MAP_DEF_KEY_TYPE)
1701	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1702				      main_btf, main_def->key_type_id,
1703				      extra_btf, extra_def->key_type_id)) {
1704		reason = "key type";
1705		goto mismatch;
1706	}
1707
1708	/* validate value type/size match */
1709	if (main_def->value_size != extra_def->value_size) {
1710		reason = "value_size";
1711		goto mismatch;
1712	}
1713	if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1714		reason = "value type";
1715		goto mismatch;
1716	}
1717	if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1718	     && !glob_sym_btf_matches(sym_name, true /*exact*/,
1719				      main_btf, main_def->value_type_id,
1720				      extra_btf, extra_def->value_type_id)) {
1721		reason = "key type";
1722		goto mismatch;
1723	}
1724
1725	if (main_def->max_entries != extra_def->max_entries) {
1726		reason = "max_entries";
1727		goto mismatch;
1728	}
1729	if (main_def->map_flags != extra_def->map_flags) {
1730		reason = "map_flags";
1731		goto mismatch;
1732	}
1733	if (main_def->numa_node != extra_def->numa_node) {
1734		reason = "numa_node";
1735		goto mismatch;
1736	}
1737	if (main_def->pinning != extra_def->pinning) {
1738		reason = "pinning";
1739		goto mismatch;
1740	}
1741
1742	if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1743		reason = "inner map";
1744		goto mismatch;
1745	}
1746
1747	if (main_def->parts & MAP_DEF_INNER_MAP) {
1748		char inner_map_name[128];
1749
1750		snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1751
1752		return map_defs_match(inner_map_name,
1753				      main_btf, main_inner_def, NULL,
1754				      extra_btf, extra_inner_def, NULL);
1755	}
1756
1757	return true;
1758
1759mismatch:
1760	pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1761	return false;
1762}
1763
1764static bool glob_map_defs_match(const char *sym_name,
1765				struct bpf_linker *linker, struct glob_sym *glob_sym,
1766				struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1767{
1768	struct btf_map_def dst_def = {}, dst_inner_def = {};
1769	struct btf_map_def src_def = {}, src_inner_def = {};
1770	const struct btf_type *t;
1771	int err;
1772
1773	t = btf__type_by_id(obj->btf, btf_id);
1774	if (!btf_is_var(t)) {
1775		pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1776		return false;
1777	}
1778	t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1779
1780	err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def);
1781	if (err) {
1782		pr_warn("global '%s': invalid map definition\n", sym_name);
1783		return false;
1784	}
1785
1786	/* re-parse existing map definition */
1787	t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1788	t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1789	err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def);
1790	if (err) {
1791		/* this should not happen, because we already validated it */
1792		pr_warn("global '%s': invalid dst map definition\n", sym_name);
1793		return false;
1794	}
1795
1796	/* Currently extern map definition has to be complete and match
1797	 * concrete map definition exactly. This restriction might be lifted
1798	 * in the future.
1799	 */
1800	return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1801			      obj->btf, &src_def, &src_inner_def);
1802}
1803
1804static bool glob_syms_match(const char *sym_name,
1805			    struct bpf_linker *linker, struct glob_sym *glob_sym,
1806			    struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1807{
1808	const struct btf_type *src_t;
1809
1810	/* if we are dealing with externs, BTF types describing both global
1811	 * and extern VARs/FUNCs should be completely present in all files
1812	 */
1813	if (!glob_sym->btf_id || !btf_id) {
1814		pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1815		return false;
1816	}
1817
1818	src_t = btf__type_by_id(obj->btf, btf_id);
1819	if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1820		pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1821			btf_kind_str(src_t), sym_name);
1822		return false;
1823	}
1824
1825	/* deal with .maps definitions specially */
1826	if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1827		return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1828
1829	if (!glob_sym_btf_matches(sym_name, true /*exact*/,
1830				  linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1831		return false;
1832
1833	return true;
1834}
1835
1836static bool btf_is_non_static(const struct btf_type *t)
1837{
1838	return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1839	       || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1840}
1841
1842static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1843			     int *out_btf_sec_id, int *out_btf_id)
1844{
1845	int i, j, n, m, btf_id = 0;
1846	const struct btf_type *t;
1847	const struct btf_var_secinfo *vi;
1848	const char *name;
1849
1850	if (!obj->btf) {
1851		pr_warn("failed to find BTF info for object '%s'\n", obj->filename);
1852		return -EINVAL;
1853	}
1854
1855	n = btf__type_cnt(obj->btf);
1856	for (i = 1; i < n; i++) {
1857		t = btf__type_by_id(obj->btf, i);
1858
1859		/* some global and extern FUNCs and VARs might not be associated with any
1860		 * DATASEC, so try to detect them in the same pass
1861		 */
1862		if (btf_is_non_static(t)) {
1863			name = btf__str_by_offset(obj->btf, t->name_off);
1864			if (strcmp(name, sym_name) != 0)
1865				continue;
1866
1867			/* remember and still try to find DATASEC */
1868			btf_id = i;
1869			continue;
1870		}
1871
1872		if (!btf_is_datasec(t))
1873			continue;
1874
1875		vi = btf_var_secinfos(t);
1876		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1877			t = btf__type_by_id(obj->btf, vi->type);
1878			name = btf__str_by_offset(obj->btf, t->name_off);
1879
1880			if (strcmp(name, sym_name) != 0)
1881				continue;
1882			if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1883				continue;
1884			if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1885				continue;
1886
1887			if (btf_id && btf_id != vi->type) {
1888				pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1889					sym_name, btf_id, vi->type);
1890				return -EINVAL;
1891			}
1892
1893			*out_btf_sec_id = i;
1894			*out_btf_id = vi->type;
1895
1896			return 0;
1897		}
1898	}
1899
1900	/* free-floating extern or global FUNC */
1901	if (btf_id) {
1902		*out_btf_sec_id = 0;
1903		*out_btf_id = btf_id;
1904		return 0;
1905	}
1906
1907	pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1908	return -ENOENT;
1909}
1910
1911static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1912{
1913	struct src_sec *sec;
1914	int i;
1915
1916	for (i = 1; i < obj->sec_cnt; i++) {
1917		sec = &obj->secs[i];
1918
1919		if (strcmp(sec->sec_name, sec_name) == 0)
1920			return sec;
1921	}
1922
1923	return NULL;
1924}
1925
1926static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1927				    struct btf *src_btf, int src_id)
1928{
1929	struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1930	struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1931	struct btf_param *src_p, *dst_p;
1932	const char *s;
1933	int i, n, off;
1934
1935	/* We already made sure that source and destination types (FUNC or
1936	 * VAR) match in terms of types and argument names.
1937	 */
1938	if (btf_is_var(dst_t)) {
1939		btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1940		return 0;
1941	}
1942
1943	dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1944
1945	/* now onto FUNC_PROTO types */
1946	src_t = btf_type_by_id(src_btf, src_t->type);
1947	dst_t = btf_type_by_id(dst_btf, dst_t->type);
1948
1949	/* Fill in all the argument names, which for extern FUNCs are missing.
1950	 * We'll end up with two copies of FUNCs/VARs for externs, but that
1951	 * will be taken care of by BTF dedup at the very end.
1952	 * It might be that BTF types for extern in one file has less/more BTF
1953	 * information (e.g., FWD instead of full STRUCT/UNION information),
1954	 * but that should be (in most cases, subject to BTF dedup rules)
1955	 * handled and resolved by BTF dedup algorithm as well, so we won't
1956	 * worry about it. Our only job is to make sure that argument names
1957	 * are populated on both sides, otherwise BTF dedup will pedantically
1958	 * consider them different.
1959	 */
1960	src_p = btf_params(src_t);
1961	dst_p = btf_params(dst_t);
1962	for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1963		if (!src_p->name_off)
1964			continue;
1965
1966		/* src_btf has more complete info, so add name to dst_btf */
1967		s = btf__str_by_offset(src_btf, src_p->name_off);
1968		off = btf__add_str(dst_btf, s);
1969		if (off < 0)
1970			return off;
1971		dst_p->name_off = off;
1972	}
1973	return 0;
1974}
1975
1976static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1977{
1978	sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1979}
1980
1981static void sym_update_type(Elf64_Sym *sym, int sym_type)
1982{
1983	sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1984}
1985
1986static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1987{
1988	/* libelf doesn't provide setters for ST_VISIBILITY,
1989	 * but it is stored in the lower 2 bits of st_other
1990	 */
1991	sym->st_other &= ~0x03;
1992	sym->st_other |= sym_vis;
1993}
1994
1995static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1996				 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1997{
1998	struct src_sec *src_sec = NULL;
1999	struct dst_sec *dst_sec = NULL;
2000	struct glob_sym *glob_sym = NULL;
2001	int name_off, sym_type, sym_bind, sym_vis, err;
2002	int btf_sec_id = 0, btf_id = 0;
2003	size_t dst_sym_idx;
2004	Elf64_Sym *dst_sym;
2005	bool sym_is_extern;
2006
2007	sym_type = ELF64_ST_TYPE(sym->st_info);
2008	sym_bind = ELF64_ST_BIND(sym->st_info);
2009	sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
2010	sym_is_extern = sym->st_shndx == SHN_UNDEF;
2011
2012	if (sym_is_extern) {
2013		if (!obj->btf) {
2014			pr_warn("externs without BTF info are not supported\n");
2015			return -ENOTSUP;
2016		}
2017	} else if (sym->st_shndx < SHN_LORESERVE) {
2018		src_sec = &obj->secs[sym->st_shndx];
2019		if (src_sec->skipped)
2020			return 0;
2021		dst_sec = &linker->secs[src_sec->dst_id];
2022
2023		/* allow only one STT_SECTION symbol per section */
2024		if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
2025			obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
2026			return 0;
2027		}
2028
2029		if (strcmp(src_sec->sec_name, JUMPTABLES_SEC) == 0)
2030			goto add_sym;
2031	}
2032
2033	if (sym_bind == STB_LOCAL)
2034		goto add_sym;
2035
2036	/* find matching BTF info */
2037	err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
2038	if (err)
2039		return err;
2040
2041	if (sym_is_extern && btf_sec_id) {
2042		const char *sec_name = NULL;
2043		const struct btf_type *t;
2044
2045		t = btf__type_by_id(obj->btf, btf_sec_id);
2046		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2047
2048		/* Clang puts unannotated extern vars into
2049		 * '.extern' BTF DATASEC. Treat them the same
2050		 * as unannotated extern funcs (which are
2051		 * currently not put into any DATASECs).
2052		 * Those don't have associated src_sec/dst_sec.
2053		 */
2054		if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
2055			src_sec = find_src_sec_by_name(obj, sec_name);
2056			if (!src_sec) {
2057				pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
2058				return -ENOENT;
2059			}
2060			dst_sec = &linker->secs[src_sec->dst_id];
2061		}
2062	}
2063
2064	glob_sym = find_glob_sym(linker, sym_name);
2065	if (glob_sym) {
2066		/* Preventively resolve to existing symbol. This is
2067		 * needed for further relocation symbol remapping in
2068		 * the next step of linking.
2069		 */
2070		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2071
2072		/* If both symbols are non-externs, at least one of
2073		 * them has to be STB_WEAK, otherwise they are in
2074		 * a conflict with each other.
2075		 */
2076		if (!sym_is_extern && !glob_sym->is_extern
2077		    && !glob_sym->is_weak && sym_bind != STB_WEAK) {
2078			pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
2079				src_sym_idx, sym_name, obj->filename);
2080			return -EINVAL;
2081		}
2082
2083		if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
2084			return -EINVAL;
2085
2086		dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
2087
2088		/* If new symbol is strong, then force dst_sym to be strong as
2089		 * well; this way a mix of weak and non-weak extern
2090		 * definitions will end up being strong.
2091		 */
2092		if (sym_bind == STB_GLOBAL) {
2093			/* We still need to preserve type (NOTYPE or
2094			 * OBJECT/FUNC, depending on whether the symbol is
2095			 * extern or not)
2096			 */
2097			sym_update_bind(dst_sym, STB_GLOBAL);
2098			glob_sym->is_weak = false;
2099		}
2100
2101		/* Non-default visibility is "contaminating", with stricter
2102		 * visibility overwriting more permissive ones, even if more
2103		 * permissive visibility comes from just an extern definition.
2104		 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and
2105		 * ensured by ELF symbol sanity checks above.
2106		 */
2107		if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
2108			sym_update_visibility(dst_sym, sym_vis);
2109
2110		/* If the new symbol is extern, then regardless if
2111		 * existing symbol is extern or resolved global, just
2112		 * keep the existing one untouched.
2113		 */
2114		if (sym_is_extern)
2115			return 0;
2116
2117		/* If existing symbol is a strong resolved symbol, bail out,
2118		 * because we lost resolution battle have nothing to
2119		 * contribute. We already checked above that there is no
2120		 * strong-strong conflict. We also already tightened binding
2121		 * and visibility, so nothing else to contribute at that point.
2122		 */
2123		if (!glob_sym->is_extern && sym_bind == STB_WEAK)
2124			return 0;
2125
2126		/* At this point, new symbol is strong non-extern,
2127		 * so overwrite glob_sym with new symbol information.
2128		 * Preserve binding and visibility.
2129		 */
2130		sym_update_type(dst_sym, sym_type);
2131		dst_sym->st_shndx = dst_sec->sec_idx;
2132		dst_sym->st_value = src_sec->dst_off + sym->st_value;
2133		dst_sym->st_size = sym->st_size;
2134
2135		/* see comment below about dst_sec->id vs dst_sec->sec_idx */
2136		glob_sym->sec_id = dst_sec->id;
2137		glob_sym->is_extern = false;
2138
2139		if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
2140					     obj->btf, btf_id))
2141			return -EINVAL;
2142
2143		/* request updating VAR's/FUNC's underlying BTF type when appending BTF type */
2144		glob_sym->underlying_btf_id = 0;
2145
2146		obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
2147		return 0;
2148	}
2149
2150add_sym:
2151	name_off = strset__add_str(linker->strtab_strs, sym_name);
2152	if (name_off < 0)
2153		return name_off;
2154
2155	dst_sym = add_new_sym(linker, &dst_sym_idx);
2156	if (!dst_sym)
2157		return -ENOMEM;
2158
2159	dst_sym->st_name = name_off;
2160	dst_sym->st_info = sym->st_info;
2161	dst_sym->st_other = sym->st_other;
2162	dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
2163	dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
2164	dst_sym->st_size = sym->st_size;
2165
2166	obj->sym_map[src_sym_idx] = dst_sym_idx;
2167
2168	if (sym_type == STT_SECTION && dst_sec) {
2169		dst_sec->sec_sym_idx = dst_sym_idx;
2170		dst_sym->st_value = 0;
2171	}
2172
2173	if (sym_bind != STB_LOCAL) {
2174		glob_sym = add_glob_sym(linker);
2175		if (!glob_sym)
2176			return -ENOMEM;
2177
2178		glob_sym->sym_idx = dst_sym_idx;
2179		/* we use dst_sec->id (and not dst_sec->sec_idx), because
2180		 * ephemeral sections (.kconfig, .ksyms, etc) don't have
2181		 * sec_idx (as they don't have corresponding ELF section), but
2182		 * still have id. .extern doesn't have even ephemeral section
2183		 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0.
2184		 */
2185		glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
2186		glob_sym->name_off = name_off;
2187		/* we will fill btf_id in during BTF merging step */
2188		glob_sym->btf_id = 0;
2189		glob_sym->is_extern = sym_is_extern;
2190		glob_sym->is_weak = sym_bind == STB_WEAK;
2191	}
2192
2193	return 0;
2194}
2195
2196static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
2197{
2198	struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
2199	int i, err;
2200
2201	for (i = 1; i < obj->sec_cnt; i++) {
2202		struct src_sec *src_sec, *src_linked_sec;
2203		struct dst_sec *dst_sec, *dst_linked_sec;
2204		Elf64_Rel *src_rel, *dst_rel;
2205		int j, n;
2206
2207		src_sec = &obj->secs[i];
2208		if (!is_relo_sec(src_sec))
2209			continue;
2210
2211		/* shdr->sh_info points to relocatable section */
2212		src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2213		if (src_linked_sec->skipped)
2214			continue;
2215
2216		dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2217		if (!dst_sec) {
2218			dst_sec = add_dst_sec(linker, src_sec->sec_name);
2219			if (!dst_sec)
2220				return -ENOMEM;
2221			err = init_sec(linker, dst_sec, src_sec);
2222			if (err) {
2223				pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2224				return err;
2225			}
2226		} else if (!secs_match(dst_sec, src_sec)) {
2227			pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2228			return -EINVAL;
2229		}
2230
2231		/* shdr->sh_link points to SYMTAB */
2232		dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2233
2234		/* shdr->sh_info points to relocated section */
2235		dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2236		dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2237
2238		src_sec->dst_id = dst_sec->id;
2239		err = extend_sec(linker, dst_sec, src_sec);
2240		if (err)
2241			return err;
2242
2243		src_rel = src_sec->data->d_buf;
2244		dst_rel = dst_sec->raw_data + src_sec->dst_off;
2245		n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2246		for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2247			size_t src_sym_idx, dst_sym_idx, sym_type;
2248			Elf64_Sym *src_sym;
2249
2250			src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2251			src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2252
2253			dst_sym_idx = obj->sym_map[src_sym_idx];
2254			dst_rel->r_offset += src_linked_sec->dst_off;
2255			sym_type = ELF64_R_TYPE(src_rel->r_info);
2256			dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2257
2258			if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2259				struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2260				struct bpf_insn *insn;
2261
2262				if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2263					/* calls to the very first static function inside
2264					 * .text section at offset 0 will
2265					 * reference section symbol, not the
2266					 * function symbol. Fix that up,
2267					 * otherwise it won't be possible to
2268					 * relocate calls to two different
2269					 * static functions with the same name
2270					 * (rom two different object files)
2271					 */
2272					insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2273					if (insn->code == (BPF_JMP | BPF_CALL))
2274						insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2275					else
2276						insn->imm += sec->dst_off;
2277				} else {
2278					pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2279					return -EINVAL;
2280				}
2281			}
2282
2283		}
2284	}
2285
2286	return 0;
2287}
2288
2289static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2290				   int sym_type, const char *sym_name)
2291{
2292	struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2293	Elf64_Sym *sym = symtab->data->d_buf;
2294	int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2295	int str_sec_idx = symtab->shdr->sh_link;
2296	const char *name;
2297
2298	for (i = 0; i < n; i++, sym++) {
2299		if (sym->st_shndx != sec_idx)
2300			continue;
2301		if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2302			continue;
2303
2304		name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2305		if (!name)
2306			return NULL;
2307
2308		if (strcmp(sym_name, name) != 0)
2309			continue;
2310
2311		return sym;
2312	}
2313
2314	return NULL;
2315}
2316
2317static int linker_fixup_btf(struct src_obj *obj)
2318{
2319	const char *sec_name;
2320	struct src_sec *sec;
2321	int i, j, n, m;
2322
2323	if (!obj->btf)
2324		return 0;
2325
2326	n = btf__type_cnt(obj->btf);
2327	for (i = 1; i < n; i++) {
2328		struct btf_var_secinfo *vi;
2329		struct btf_type *t;
2330
2331		t = btf_type_by_id(obj->btf, i);
2332		if (btf_kind(t) != BTF_KIND_DATASEC)
2333			continue;
2334
2335		sec_name = btf__str_by_offset(obj->btf, t->name_off);
2336		sec = find_src_sec_by_name(obj, sec_name);
2337		if (sec) {
2338			/* record actual section size, unless ephemeral */
2339			if (sec->shdr)
2340				t->size = sec->shdr->sh_size;
2341		} else {
2342			/* BTF can have some sections that are not represented
2343			 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used
2344			 * for special extern variables.
2345			 *
2346			 * For all but one such special (ephemeral)
2347			 * sections, we pre-create "section shells" to be able
2348			 * to keep track of extra per-section metadata later
2349			 * (e.g., those BTF extern variables).
2350			 *
2351			 * .extern is even more special, though, because it
2352			 * contains extern variables that need to be resolved
2353			 * by static linker, not libbpf and kernel. When such
2354			 * externs are resolved, we are going to remove them
2355			 * from .extern BTF section and might end up not
2356			 * needing it at all. Each resolved extern should have
2357			 * matching non-extern VAR/FUNC in other sections.
2358			 *
2359			 * We do support leaving some of the externs
2360			 * unresolved, though, to support cases of building
2361			 * libraries, which will later be linked against final
2362			 * BPF applications. So if at finalization we still
2363			 * see unresolved externs, we'll create .extern
2364			 * section on our own.
2365			 */
2366			if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2367				continue;
2368
2369			sec = add_src_sec(obj, sec_name);
2370			if (!sec)
2371				return -ENOMEM;
2372
2373			sec->ephemeral = true;
2374			sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
2375		}
2376
2377		/* remember ELF section and its BTF type ID match */
2378		sec->sec_type_id = i;
2379
2380		/* fix up variable offsets */
2381		vi = btf_var_secinfos(t);
2382		for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2383			const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2384			const char *var_name;
2385			int var_linkage;
2386			Elf64_Sym *sym;
2387
2388			/* could be a variable or function */
2389			if (!btf_is_var(vt))
2390				continue;
2391
2392			var_name = btf__str_by_offset(obj->btf, vt->name_off);
2393			var_linkage = btf_var(vt)->linkage;
2394
2395			/* no need to patch up static or extern vars */
2396			if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2397				continue;
2398
2399			sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2400			if (!sym) {
2401				pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2402				return -ENOENT;
2403			}
2404
2405			vi->offset = sym->st_value;
2406		}
2407	}
2408
2409	return 0;
2410}
2411
2412static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2413{
2414	const struct btf_type *t;
2415	int i, j, n, start_id, id, err;
2416	const char *name;
2417
2418	if (!obj->btf)
2419		return 0;
2420
2421	start_id = btf__type_cnt(linker->btf);
2422	n = btf__type_cnt(obj->btf);
2423
2424	obj->btf_type_map = calloc(n + 1, sizeof(int));
2425	if (!obj->btf_type_map)
2426		return -ENOMEM;
2427
2428	for (i = 1; i < n; i++) {
2429		struct glob_sym *glob_sym = NULL;
2430
2431		t = btf__type_by_id(obj->btf, i);
2432
2433		/* DATASECs are handled specially below */
2434		if (btf_kind(t) == BTF_KIND_DATASEC)
2435			continue;
2436
2437		if (btf_is_non_static(t)) {
2438			/* there should be glob_sym already */
2439			name = btf__str_by_offset(obj->btf, t->name_off);
2440			glob_sym = find_glob_sym(linker, name);
2441
2442			/* VARs without corresponding glob_sym are those that
2443			 * belong to skipped/deduplicated sections (i.e.,
2444			 * license and version), so just skip them
2445			 */
2446			if (!glob_sym)
2447				continue;
2448
2449			/* linker_append_elf_sym() might have requested
2450			 * updating underlying type ID, if extern was resolved
2451			 * to strong symbol or weak got upgraded to non-weak
2452			 */
2453			if (glob_sym->underlying_btf_id == 0)
2454				glob_sym->underlying_btf_id = -t->type;
2455
2456			/* globals from previous object files that match our
2457			 * VAR/FUNC already have a corresponding associated
2458			 * BTF type, so just make sure to use it
2459			 */
2460			if (glob_sym->btf_id) {
2461				/* reuse existing BTF type for global var/func */
2462				obj->btf_type_map[i] = glob_sym->btf_id;
2463				continue;
2464			}
2465		}
2466
2467		id = btf__add_type(linker->btf, obj->btf, t);
2468		if (id < 0) {
2469			pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2470			return id;
2471		}
2472
2473		obj->btf_type_map[i] = id;
2474
2475		/* record just appended BTF type for var/func */
2476		if (glob_sym) {
2477			glob_sym->btf_id = id;
2478			glob_sym->underlying_btf_id = -t->type;
2479		}
2480	}
2481
2482	/* remap all the types except DATASECs */
2483	n = btf__type_cnt(linker->btf);
2484	for (i = start_id; i < n; i++) {
2485		struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2486		struct btf_field_iter it;
2487		__u32 *type_id;
2488
2489		err = btf_field_iter_init(&it, dst_t, BTF_FIELD_ITER_IDS);
2490		if (err)
2491			return err;
2492
2493		while ((type_id = btf_field_iter_next(&it))) {
2494			int new_id = obj->btf_type_map[*type_id];
2495
2496			/* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */
2497			if (new_id == 0 && *type_id != 0) {
2498				pr_warn("failed to find new ID mapping for original BTF type ID %u\n",
2499					*type_id);
2500				return -EINVAL;
2501			}
2502
2503			*type_id = obj->btf_type_map[*type_id];
2504		}
2505	}
2506
2507	/* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's
2508	 * actual type), if necessary
2509	 */
2510	for (i = 0; i < linker->glob_sym_cnt; i++) {
2511		struct glob_sym *glob_sym = &linker->glob_syms[i];
2512		struct btf_type *glob_t;
2513
2514		if (glob_sym->underlying_btf_id >= 0)
2515			continue;
2516
2517		glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2518
2519		glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2520		glob_t->type = glob_sym->underlying_btf_id;
2521	}
2522
2523	/* append DATASEC info */
2524	for (i = 1; i < obj->sec_cnt; i++) {
2525		struct src_sec *src_sec;
2526		struct dst_sec *dst_sec;
2527		const struct btf_var_secinfo *src_var;
2528		struct btf_var_secinfo *dst_var;
2529
2530		src_sec = &obj->secs[i];
2531		if (!src_sec->sec_type_id || src_sec->skipped)
2532			continue;
2533		dst_sec = &linker->secs[src_sec->dst_id];
2534
2535		/* Mark section as having BTF regardless of the presence of
2536		 * variables. In some cases compiler might generate empty BTF
2537		 * with no variables information. E.g., when promoting local
2538		 * array/structure variable initial values and BPF object
2539		 * file otherwise has no read-only static variables in
2540		 * .rodata. We need to preserve such empty BTF and just set
2541		 * correct section size.
2542		 */
2543		dst_sec->has_btf = true;
2544
2545		t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2546		src_var = btf_var_secinfos(t);
2547		n = btf_vlen(t);
2548		for (j = 0; j < n; j++, src_var++) {
2549			void *sec_vars = dst_sec->sec_vars;
2550			int new_id = obj->btf_type_map[src_var->type];
2551			struct glob_sym *glob_sym = NULL;
2552
2553			t = btf_type_by_id(linker->btf, new_id);
2554			if (btf_is_non_static(t)) {
2555				name = btf__str_by_offset(linker->btf, t->name_off);
2556				glob_sym = find_glob_sym(linker, name);
2557				if (glob_sym->sec_id != dst_sec->id) {
2558					pr_warn("global '%s': section mismatch %d vs %d\n",
2559						name, glob_sym->sec_id, dst_sec->id);
2560					return -EINVAL;
2561				}
2562			}
2563
2564			/* If there is already a member (VAR or FUNC) mapped
2565			 * to the same type, don't add a duplicate entry.
2566			 * This will happen when multiple object files define
2567			 * the same extern VARs/FUNCs.
2568			 */
2569			if (glob_sym && glob_sym->var_idx >= 0) {
2570				__s64 sz;
2571
2572				/* FUNCs don't have size, nothing to update */
2573				if (btf_is_func(t))
2574					continue;
2575
2576				dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2577				/* Because underlying BTF type might have
2578				 * changed, so might its size have changed, so
2579				 * re-calculate and update it in sec_var.
2580				 */
2581				sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2582				if (sz < 0) {
2583					pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2584						name, (int)sz);
2585					return -EINVAL;
2586				}
2587				dst_var->size = sz;
2588				continue;
2589			}
2590
2591			sec_vars = libbpf_reallocarray(sec_vars,
2592						       dst_sec->sec_var_cnt + 1,
2593						       sizeof(*dst_sec->sec_vars));
2594			if (!sec_vars)
2595				return -ENOMEM;
2596
2597			dst_sec->sec_vars = sec_vars;
2598			dst_sec->sec_var_cnt++;
2599
2600			dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2601			dst_var->type = obj->btf_type_map[src_var->type];
2602			dst_var->size = src_var->size;
2603			dst_var->offset = src_sec->dst_off + src_var->offset;
2604
2605			if (glob_sym)
2606				glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2607		}
2608	}
2609
2610	return 0;
2611}
2612
2613static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2614{
2615	void *tmp;
2616
2617	tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2618	if (!tmp)
2619		return NULL;
2620	ext_data->recs = tmp;
2621
2622	tmp += ext_data->rec_cnt * ext_data->rec_sz;
2623	memcpy(tmp, src_rec, ext_data->rec_sz);
2624
2625	ext_data->rec_cnt++;
2626
2627	return tmp;
2628}
2629
2630static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2631{
2632	const struct btf_ext_info_sec *ext_sec;
2633	const char *sec_name, *s;
2634	struct src_sec *src_sec;
2635	struct dst_sec *dst_sec;
2636	int rec_sz, str_off, i;
2637
2638	if (!obj->btf_ext)
2639		return 0;
2640
2641	rec_sz = obj->btf_ext->func_info.rec_size;
2642	for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2643		struct bpf_func_info_min *src_rec, *dst_rec;
2644
2645		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2646		src_sec = find_src_sec_by_name(obj, sec_name);
2647		if (!src_sec) {
2648			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2649			return -EINVAL;
2650		}
2651		dst_sec = &linker->secs[src_sec->dst_id];
2652
2653		if (dst_sec->func_info.rec_sz == 0)
2654			dst_sec->func_info.rec_sz = rec_sz;
2655		if (dst_sec->func_info.rec_sz != rec_sz) {
2656			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2657			return -EINVAL;
2658		}
2659
2660		for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2661			dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2662			if (!dst_rec)
2663				return -ENOMEM;
2664
2665			dst_rec->insn_off += src_sec->dst_off;
2666			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2667		}
2668	}
2669
2670	rec_sz = obj->btf_ext->line_info.rec_size;
2671	for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2672		struct bpf_line_info_min *src_rec, *dst_rec;
2673
2674		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2675		src_sec = find_src_sec_by_name(obj, sec_name);
2676		if (!src_sec) {
2677			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2678			return -EINVAL;
2679		}
2680		dst_sec = &linker->secs[src_sec->dst_id];
2681
2682		if (dst_sec->line_info.rec_sz == 0)
2683			dst_sec->line_info.rec_sz = rec_sz;
2684		if (dst_sec->line_info.rec_sz != rec_sz) {
2685			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2686			return -EINVAL;
2687		}
2688
2689		for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2690			dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2691			if (!dst_rec)
2692				return -ENOMEM;
2693
2694			dst_rec->insn_off += src_sec->dst_off;
2695
2696			s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2697			str_off = btf__add_str(linker->btf, s);
2698			if (str_off < 0)
2699				return -ENOMEM;
2700			dst_rec->file_name_off = str_off;
2701
2702			s = btf__str_by_offset(obj->btf, src_rec->line_off);
2703			str_off = btf__add_str(linker->btf, s);
2704			if (str_off < 0)
2705				return -ENOMEM;
2706			dst_rec->line_off = str_off;
2707
2708			/* dst_rec->line_col is fine */
2709		}
2710	}
2711
2712	rec_sz = obj->btf_ext->core_relo_info.rec_size;
2713	for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2714		struct bpf_core_relo *src_rec, *dst_rec;
2715
2716		sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2717		src_sec = find_src_sec_by_name(obj, sec_name);
2718		if (!src_sec) {
2719			pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2720			return -EINVAL;
2721		}
2722		dst_sec = &linker->secs[src_sec->dst_id];
2723
2724		if (dst_sec->core_relo_info.rec_sz == 0)
2725			dst_sec->core_relo_info.rec_sz = rec_sz;
2726		if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2727			pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2728			return -EINVAL;
2729		}
2730
2731		for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2732			dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2733			if (!dst_rec)
2734				return -ENOMEM;
2735
2736			dst_rec->insn_off += src_sec->dst_off;
2737			dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2738
2739			s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2740			str_off = btf__add_str(linker->btf, s);
2741			if (str_off < 0)
2742				return -ENOMEM;
2743			dst_rec->access_str_off = str_off;
2744
2745			/* dst_rec->kind is fine */
2746		}
2747	}
2748
2749	return 0;
2750}
2751
2752int bpf_linker__finalize(struct bpf_linker *linker)
2753{
2754	struct dst_sec *sec;
2755	size_t strs_sz;
2756	const void *strs;
2757	int err, i;
2758
2759	if (!linker->elf)
2760		return libbpf_err(-EINVAL);
2761
2762	err = finalize_btf(linker);
2763	if (err)
2764		return libbpf_err(err);
2765
2766	/* Finalize strings */
2767	strs_sz = strset__data_size(linker->strtab_strs);
2768	strs = strset__data(linker->strtab_strs);
2769
2770	sec = &linker->secs[linker->strtab_sec_idx];
2771	sec->data->d_align = 1;
2772	sec->data->d_off = 0LL;
2773	sec->data->d_buf = (void *)strs;
2774	sec->data->d_type = ELF_T_BYTE;
2775	sec->data->d_size = strs_sz;
2776	sec->shdr->sh_size = strs_sz;
2777
2778	for (i = 1; i < linker->sec_cnt; i++) {
2779		sec = &linker->secs[i];
2780
2781		/* STRTAB is handled specially above */
2782		if (sec->sec_idx == linker->strtab_sec_idx)
2783			continue;
2784
2785		/* special ephemeral sections (.ksyms, .kconfig, etc) */
2786		if (!sec->scn)
2787			continue;
2788
2789		/* restore sections with bpf insns to target byte-order */
2790		if (linker->swapped_endian && is_exec_sec(sec))
2791			exec_sec_bswap(sec->raw_data, sec->sec_sz);
2792
2793		sec->data->d_buf = sec->raw_data;
2794	}
2795
2796	/* Finalize ELF layout */
2797	if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2798		err = -EINVAL;
2799		pr_warn_elf("failed to finalize ELF layout");
2800		return libbpf_err(err);
2801	}
2802
2803	/* Write out final ELF contents */
2804	if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2805		err = -EINVAL;
2806		pr_warn_elf("failed to write ELF contents");
2807		return libbpf_err(err);
2808	}
2809
2810	elf_end(linker->elf);
2811	linker->elf = NULL;
2812
2813	if (linker->fd_is_owned)
2814		close(linker->fd);
2815	linker->fd = -1;
2816
2817	return 0;
2818}
2819
2820static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2821			     size_t align, const void *raw_data, size_t raw_sz)
2822{
2823	Elf_Scn *scn;
2824	Elf_Data *data;
2825	Elf64_Shdr *shdr;
2826	int name_off;
2827
2828	name_off = strset__add_str(linker->strtab_strs, sec_name);
2829	if (name_off < 0)
2830		return name_off;
2831
2832	scn = elf_newscn(linker->elf);
2833	if (!scn)
2834		return -ENOMEM;
2835	data = elf_newdata(scn);
2836	if (!data)
2837		return -ENOMEM;
2838	shdr = elf64_getshdr(scn);
2839	if (!shdr)
2840		return -EINVAL;
2841
2842	shdr->sh_name = name_off;
2843	shdr->sh_type = SHT_PROGBITS;
2844	shdr->sh_flags = 0;
2845	shdr->sh_size = raw_sz;
2846	shdr->sh_link = 0;
2847	shdr->sh_info = 0;
2848	shdr->sh_addralign = align;
2849	shdr->sh_entsize = 0;
2850
2851	data->d_type = ELF_T_BYTE;
2852	data->d_size = raw_sz;
2853	data->d_buf = (void *)raw_data;
2854	data->d_align = align;
2855	data->d_off = 0;
2856
2857	return 0;
2858}
2859
2860static int finalize_btf(struct bpf_linker *linker)
2861{
2862	enum btf_endianness link_endianness;
2863	LIBBPF_OPTS(btf_dedup_opts, opts);
2864	struct btf *btf = linker->btf;
2865	const void *raw_data;
2866	int i, j, id, err;
2867	__u32 raw_sz;
2868
2869	/* bail out if no BTF data was produced */
2870	if (btf__type_cnt(linker->btf) == 1)
2871		return 0;
2872
2873	for (i = 1; i < linker->sec_cnt; i++) {
2874		struct dst_sec *sec = &linker->secs[i];
2875
2876		if (!sec->has_btf)
2877			continue;
2878
2879		id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2880		if (id < 0) {
2881			pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2882				sec->sec_name, id);
2883			return id;
2884		}
2885
2886		for (j = 0; j < sec->sec_var_cnt; j++) {
2887			struct btf_var_secinfo *vi = &sec->sec_vars[j];
2888
2889			if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2890				return -EINVAL;
2891		}
2892	}
2893
2894	err = finalize_btf_ext(linker);
2895	if (err) {
2896		pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2897		return err;
2898	}
2899
2900	opts.btf_ext = linker->btf_ext;
2901	err = btf__dedup(linker->btf, &opts);
2902	if (err) {
2903		pr_warn("BTF dedup failed: %s\n", errstr(err));
2904		return err;
2905	}
2906
2907	/* Set .BTF and .BTF.ext output byte order */
2908	link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2909			  BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2910	btf__set_endianness(linker->btf, link_endianness);
2911	if (linker->btf_ext)
2912		btf_ext__set_endianness(linker->btf_ext, link_endianness);
2913
2914	/* Emit .BTF section */
2915	raw_data = btf__raw_data(linker->btf, &raw_sz);
2916	if (!raw_data)
2917		return -ENOMEM;
2918
2919	err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2920	if (err) {
2921		pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2922		return err;
2923	}
2924
2925	/* Emit .BTF.ext section */
2926	if (linker->btf_ext) {
2927		raw_data = btf_ext__raw_data(linker->btf_ext, &raw_sz);
2928		if (!raw_data)
2929			return -ENOMEM;
2930
2931		err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2932		if (err) {
2933			pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2934			return err;
2935		}
2936	}
2937
2938	return 0;
2939}
2940
2941static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2942			     const char *sec_name, struct btf_ext_sec_data *sec_data)
2943{
2944	struct btf_ext_info_sec *sec_info;
2945	void *cur = output;
2946	int str_off;
2947	size_t sz;
2948
2949	if (!sec_data->rec_cnt)
2950		return 0;
2951
2952	str_off = btf__add_str(linker->btf, sec_name);
2953	if (str_off < 0)
2954		return -ENOMEM;
2955
2956	sec_info = cur;
2957	sec_info->sec_name_off = str_off;
2958	sec_info->num_info = sec_data->rec_cnt;
2959	cur += sizeof(struct btf_ext_info_sec);
2960
2961	sz = sec_data->rec_cnt * sec_data->rec_sz;
2962	memcpy(cur, sec_data->recs, sz);
2963	cur += sz;
2964
2965	return cur - output;
2966}
2967
2968static int finalize_btf_ext(struct bpf_linker *linker)
2969{
2970	size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2971	size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2972	struct btf_ext_header *hdr;
2973	void *data, *cur;
2974	int i, err, sz;
2975
2976	/* validate that all sections have the same .BTF.ext record sizes
2977	 * and calculate total data size for each type of data (func info,
2978	 * line info, core relos)
2979	 */
2980	for (i = 1; i < linker->sec_cnt; i++) {
2981		struct dst_sec *sec = &linker->secs[i];
2982
2983		if (sec->func_info.rec_cnt) {
2984			if (func_rec_sz == 0)
2985				func_rec_sz = sec->func_info.rec_sz;
2986			if (func_rec_sz != sec->func_info.rec_sz) {
2987				pr_warn("mismatch in func_info record size %zu != %u\n",
2988					func_rec_sz, sec->func_info.rec_sz);
2989				return -EINVAL;
2990			}
2991
2992			funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2993		}
2994		if (sec->line_info.rec_cnt) {
2995			if (line_rec_sz == 0)
2996				line_rec_sz = sec->line_info.rec_sz;
2997			if (line_rec_sz != sec->line_info.rec_sz) {
2998				pr_warn("mismatch in line_info record size %zu != %u\n",
2999					line_rec_sz, sec->line_info.rec_sz);
3000				return -EINVAL;
3001			}
3002
3003			lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
3004		}
3005		if (sec->core_relo_info.rec_cnt) {
3006			if (core_relo_rec_sz == 0)
3007				core_relo_rec_sz = sec->core_relo_info.rec_sz;
3008			if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
3009				pr_warn("mismatch in core_relo_info record size %zu != %u\n",
3010					core_relo_rec_sz, sec->core_relo_info.rec_sz);
3011				return -EINVAL;
3012			}
3013
3014			core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
3015		}
3016	}
3017
3018	if (!funcs_sz && !lines_sz && !core_relos_sz)
3019		return 0;
3020
3021	total_sz += sizeof(struct btf_ext_header);
3022	if (funcs_sz) {
3023		funcs_sz += sizeof(__u32); /* record size prefix */
3024		total_sz += funcs_sz;
3025	}
3026	if (lines_sz) {
3027		lines_sz += sizeof(__u32); /* record size prefix */
3028		total_sz += lines_sz;
3029	}
3030	if (core_relos_sz) {
3031		core_relos_sz += sizeof(__u32); /* record size prefix */
3032		total_sz += core_relos_sz;
3033	}
3034
3035	cur = data = calloc(1, total_sz);
3036	if (!data)
3037		return -ENOMEM;
3038
3039	hdr = cur;
3040	hdr->magic = BTF_MAGIC;
3041	hdr->version = BTF_VERSION;
3042	hdr->flags = 0;
3043	hdr->hdr_len = sizeof(struct btf_ext_header);
3044	cur += sizeof(struct btf_ext_header);
3045
3046	/* All offsets are in bytes relative to the end of this header */
3047	hdr->func_info_off = 0;
3048	hdr->func_info_len = funcs_sz;
3049	hdr->line_info_off = funcs_sz;
3050	hdr->line_info_len = lines_sz;
3051	hdr->core_relo_off = funcs_sz + lines_sz;
3052	hdr->core_relo_len = core_relos_sz;
3053
3054	if (funcs_sz) {
3055		*(__u32 *)cur = func_rec_sz;
3056		cur += sizeof(__u32);
3057
3058		for (i = 1; i < linker->sec_cnt; i++) {
3059			struct dst_sec *sec = &linker->secs[i];
3060
3061			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
3062			if (sz < 0) {
3063				err = sz;
3064				goto out;
3065			}
3066
3067			cur += sz;
3068		}
3069	}
3070
3071	if (lines_sz) {
3072		*(__u32 *)cur = line_rec_sz;
3073		cur += sizeof(__u32);
3074
3075		for (i = 1; i < linker->sec_cnt; i++) {
3076			struct dst_sec *sec = &linker->secs[i];
3077
3078			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
3079			if (sz < 0) {
3080				err = sz;
3081				goto out;
3082			}
3083
3084			cur += sz;
3085		}
3086	}
3087
3088	if (core_relos_sz) {
3089		*(__u32 *)cur = core_relo_rec_sz;
3090		cur += sizeof(__u32);
3091
3092		for (i = 1; i < linker->sec_cnt; i++) {
3093			struct dst_sec *sec = &linker->secs[i];
3094
3095			sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
3096			if (sz < 0) {
3097				err = sz;
3098				goto out;
3099			}
3100
3101			cur += sz;
3102		}
3103	}
3104
3105	linker->btf_ext = btf_ext__new(data, total_sz);
3106	err = libbpf_get_error(linker->btf_ext);
3107	if (err) {
3108		linker->btf_ext = NULL;
3109		pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
3110		goto out;
3111	}
3112
3113out:
3114	free(data);
3115	return err;
3116}