relo_core.c

   1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2/* Copyright (c) 2019 Facebook */
   3
   4#ifdef __KERNEL__
   5#include <linux/bpf.h>
   6#include <linux/btf.h>
   7#include <linux/string.h>
   8#include <linux/bpf_verifier.h>
   9#include "relo_core.h"
  10
  11static const char *btf_kind_str(const struct btf_type *t)
  12{
  13	return btf_type_str(t);
  14}
  15
  16static bool is_ldimm64_insn(struct bpf_insn *insn)
  17{
  18	return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
  19}
  20
  21static const struct btf_type *
  22skip_mods_and_typedefs(const struct btf *btf, u32 id, u32 *res_id)
  23{
  24	return btf_type_skip_modifiers(btf, id, res_id);
  25}
  26
  27static const char *btf__name_by_offset(const struct btf *btf, u32 offset)
  28{
  29	return btf_name_by_offset(btf, offset);
  30}
  31
  32static s64 btf__resolve_size(const struct btf *btf, u32 type_id)
  33{
  34	const struct btf_type *t;
  35	int size;
  36
  37	t = btf_type_by_id(btf, type_id);
  38	t = btf_resolve_size(btf, t, &size);
  39	if (IS_ERR(t))
  40		return PTR_ERR(t);
  41	return size;
  42}
  43
  44enum libbpf_print_level {
  45	LIBBPF_WARN,
  46	LIBBPF_INFO,
  47	LIBBPF_DEBUG,
  48};
  49
  50#undef pr_warn
  51#undef pr_info
  52#undef pr_debug
  53#define pr_warn(fmt, log, ...)	bpf_log((void *)log, fmt, "", ##__VA_ARGS__)
  54#define pr_info(fmt, log, ...)	bpf_log((void *)log, fmt, "", ##__VA_ARGS__)
  55#define pr_debug(fmt, log, ...)	bpf_log((void *)log, fmt, "", ##__VA_ARGS__)
  56#define libbpf_print(level, fmt, ...)	bpf_log((void *)prog_name, fmt, ##__VA_ARGS__)
  57#else
  58#include <stdio.h>
  59#include <string.h>
  60#include <errno.h>
  61#include <ctype.h>
  62#include <linux/err.h>
  63
  64#include "libbpf.h"
  65#include "bpf.h"
  66#include "btf.h"
  67#include "libbpf_internal.h"
  68#endif
  69
  70static bool is_flex_arr(const struct btf *btf,
  71			const struct bpf_core_accessor *acc,
  72			const struct btf_array *arr)
  73{
  74	const struct btf_type *t;
  75
  76	/* not a flexible array, if not inside a struct or has non-zero size */
  77	if (!acc->name || arr->nelems > 0)
  78		return false;
  79
  80	/* has to be the last member of enclosing struct */
  81	t = btf_type_by_id(btf, acc->type_id);
  82	return acc->idx == btf_vlen(t) - 1;
  83}
  84
  85static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
  86{
  87	switch (kind) {
  88	case BPF_CORE_FIELD_BYTE_OFFSET: return "byte_off";
  89	case BPF_CORE_FIELD_BYTE_SIZE: return "byte_sz";
  90	case BPF_CORE_FIELD_EXISTS: return "field_exists";
  91	case BPF_CORE_FIELD_SIGNED: return "signed";
  92	case BPF_CORE_FIELD_LSHIFT_U64: return "lshift_u64";
  93	case BPF_CORE_FIELD_RSHIFT_U64: return "rshift_u64";
  94	case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id";
  95	case BPF_CORE_TYPE_ID_TARGET: return "target_type_id";
  96	case BPF_CORE_TYPE_EXISTS: return "type_exists";
  97	case BPF_CORE_TYPE_MATCHES: return "type_matches";
  98	case BPF_CORE_TYPE_SIZE: return "type_size";
  99	case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists";
 100	case BPF_CORE_ENUMVAL_VALUE: return "enumval_value";
 101	default: return "unknown";
 102	}
 103}
 104
 105static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
 106{
 107	switch (kind) {
 108	case BPF_CORE_FIELD_BYTE_OFFSET:
 109	case BPF_CORE_FIELD_BYTE_SIZE:
 110	case BPF_CORE_FIELD_EXISTS:
 111	case BPF_CORE_FIELD_SIGNED:
 112	case BPF_CORE_FIELD_LSHIFT_U64:
 113	case BPF_CORE_FIELD_RSHIFT_U64:
 114		return true;
 115	default:
 116		return false;
 117	}
 118}
 119
 120static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
 121{
 122	switch (kind) {
 123	case BPF_CORE_TYPE_ID_LOCAL:
 124	case BPF_CORE_TYPE_ID_TARGET:
 125	case BPF_CORE_TYPE_EXISTS:
 126	case BPF_CORE_TYPE_MATCHES:
 127	case BPF_CORE_TYPE_SIZE:
 128		return true;
 129	default:
 130		return false;
 131	}
 132}
 133
 134static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
 135{
 136	switch (kind) {
 137	case BPF_CORE_ENUMVAL_EXISTS:
 138	case BPF_CORE_ENUMVAL_VALUE:
 139		return true;
 140	default:
 141		return false;
 142	}
 143}
 144
 145int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
 146				const struct btf *targ_btf, __u32 targ_id, int level)
 147{
 148	const struct btf_type *local_type, *targ_type;
 149	int depth = 32; /* max recursion depth */
 150
 151	/* caller made sure that names match (ignoring flavor suffix) */
 152	local_type = btf_type_by_id(local_btf, local_id);
 153	targ_type = btf_type_by_id(targ_btf, targ_id);
 154	if (!btf_kind_core_compat(local_type, targ_type))
 155		return 0;
 156
 157recur:
 158	depth--;
 159	if (depth < 0)
 160		return -EINVAL;
 161
 162	local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
 163	targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
 164	if (!local_type || !targ_type)
 165		return -EINVAL;
 166
 167	if (!btf_kind_core_compat(local_type, targ_type))
 168		return 0;
 169
 170	switch (btf_kind(local_type)) {
 171	case BTF_KIND_UNKN:
 172	case BTF_KIND_STRUCT:
 173	case BTF_KIND_UNION:
 174	case BTF_KIND_ENUM:
 175	case BTF_KIND_FWD:
 176	case BTF_KIND_ENUM64:
 177		return 1;
 178	case BTF_KIND_INT:
 179		/* just reject deprecated bitfield-like integers; all other
 180		 * integers are by default compatible between each other
 181		 */
 182		return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
 183	case BTF_KIND_PTR:
 184		local_id = local_type->type;
 185		targ_id = targ_type->type;
 186		goto recur;
 187	case BTF_KIND_ARRAY:
 188		local_id = btf_array(local_type)->type;
 189		targ_id = btf_array(targ_type)->type;
 190		goto recur;
 191	case BTF_KIND_FUNC_PROTO: {
 192		struct btf_param *local_p = btf_params(local_type);
 193		struct btf_param *targ_p = btf_params(targ_type);
 194		__u16 local_vlen = btf_vlen(local_type);
 195		__u16 targ_vlen = btf_vlen(targ_type);
 196		int i, err;
 197
 198		if (local_vlen != targ_vlen)
 199			return 0;
 200
 201		for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
 202			if (level <= 0)
 203				return -EINVAL;
 204
 205			skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
 206			skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
 207			err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id,
 208							  level - 1);
 209			if (err <= 0)
 210				return err;
 211		}
 212
 213		/* tail recurse for return type check */
 214		skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
 215		skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
 216		goto recur;
 217	}
 218	default:
 219		pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
 220			btf_kind_str(local_type), local_id, targ_id);
 221		return 0;
 222	}
 223}
 224
 225/*
 226 * Turn bpf_core_relo into a low- and high-level spec representation,
 227 * validating correctness along the way, as well as calculating resulting
 228 * field bit offset, specified by accessor string. Low-level spec captures
 229 * every single level of nestedness, including traversing anonymous
 230 * struct/union members. High-level one only captures semantically meaningful
 231 * "turning points": named fields and array indicies.
 232 * E.g., for this case:
 233 *
 234 *   struct sample {
 235 *       int __unimportant;
 236 *       struct {
 237 *           int __1;
 238 *           int __2;
 239 *           int a[7];
 240 *       };
 241 *   };
 242 *
 243 *   struct sample *s = ...;
 244 *
 245 *   int x = &s->a[3]; // access string = '0:1:2:3'
 246 *
 247 * Low-level spec has 1:1 mapping with each element of access string (it's
 248 * just a parsed access string representation): [0, 1, 2, 3].
 249 *
 250 * High-level spec will capture only 3 points:
 251 *   - initial zero-index access by pointer (&s->... is the same as &s[0]...);
 252 *   - field 'a' access (corresponds to '2' in low-level spec);
 253 *   - array element #3 access (corresponds to '3' in low-level spec).
 254 *
 255 * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE,
 256 * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
 257 * spec and raw_spec are kept empty.
 258 *
 259 * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
 260 * string to specify enumerator's value index that need to be relocated.
 261 */
 262int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
 263			const struct bpf_core_relo *relo,
 264			struct bpf_core_spec *spec)
 265{
 266	int access_idx, parsed_len, i;
 267	struct bpf_core_accessor *acc;
 268	const struct btf_type *t;
 269	const char *name, *spec_str;
 270	__u32 id, name_off;
 271	__s64 sz;
 272
 273	spec_str = btf__name_by_offset(btf, relo->access_str_off);
 274	if (str_is_empty(spec_str) || *spec_str == ':')
 275		return -EINVAL;
 276
 277	memset(spec, 0, sizeof(*spec));
 278	spec->btf = btf;
 279	spec->root_type_id = relo->type_id;
 280	spec->relo_kind = relo->kind;
 281
 282	/* type-based relocations don't have a field access string */
 283	if (core_relo_is_type_based(relo->kind)) {
 284		if (strcmp(spec_str, "0"))
 285			return -EINVAL;
 286		return 0;
 287	}
 288
 289	/* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
 290	while (*spec_str) {
 291		if (*spec_str == ':')
 292			++spec_str;
 293		if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
 294			return -EINVAL;
 295		if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
 296			return -E2BIG;
 297		spec_str += parsed_len;
 298		spec->raw_spec[spec->raw_len++] = access_idx;
 299	}
 300
 301	if (spec->raw_len == 0)
 302		return -EINVAL;
 303
 304	t = skip_mods_and_typedefs(btf, relo->type_id, &id);
 305	if (!t)
 306		return -EINVAL;
 307
 308	access_idx = spec->raw_spec[0];
 309	acc = &spec->spec[0];
 310	acc->type_id = id;
 311	acc->idx = access_idx;
 312	spec->len++;
 313
 314	if (core_relo_is_enumval_based(relo->kind)) {
 315		if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
 316			return -EINVAL;
 317
 318		/* record enumerator name in a first accessor */
 319		name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off
 320					  : btf_enum64(t)[access_idx].name_off;
 321		acc->name = btf__name_by_offset(btf, name_off);
 322		return 0;
 323	}
 324
 325	if (!core_relo_is_field_based(relo->kind))
 326		return -EINVAL;
 327
 328	sz = btf__resolve_size(btf, id);
 329	if (sz < 0)
 330		return sz;
 331	spec->bit_offset = access_idx * sz * 8;
 332
 333	for (i = 1; i < spec->raw_len; i++) {
 334		t = skip_mods_and_typedefs(btf, id, &id);
 335		if (!t)
 336			return -EINVAL;
 337
 338		access_idx = spec->raw_spec[i];
 339		acc = &spec->spec[spec->len];
 340
 341		if (btf_is_composite(t)) {
 342			const struct btf_member *m;
 343			__u32 bit_offset;
 344
 345			if (access_idx >= btf_vlen(t))
 346				return -EINVAL;
 347
 348			bit_offset = btf_member_bit_offset(t, access_idx);
 349			spec->bit_offset += bit_offset;
 350
 351			m = btf_members(t) + access_idx;
 352			if (m->name_off) {
 353				name = btf__name_by_offset(btf, m->name_off);
 354				if (str_is_empty(name))
 355					return -EINVAL;
 356
 357				acc->type_id = id;
 358				acc->idx = access_idx;
 359				acc->name = name;
 360				spec->len++;
 361			}
 362
 363			id = m->type;
 364		} else if (btf_is_array(t)) {
 365			const struct btf_array *a = btf_array(t);
 366			bool flex;
 367
 368			t = skip_mods_and_typedefs(btf, a->type, &id);
 369			if (!t)
 370				return -EINVAL;
 371
 372			flex = is_flex_arr(btf, acc - 1, a);
 373			if (!flex && access_idx >= a->nelems)
 374				return -EINVAL;
 375
 376			spec->spec[spec->len].type_id = id;
 377			spec->spec[spec->len].idx = access_idx;
 378			spec->len++;
 379
 380			sz = btf__resolve_size(btf, id);
 381			if (sz < 0)
 382				return sz;
 383			spec->bit_offset += access_idx * sz * 8;
 384		} else {
 385			pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
 386				prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t));
 387			return -EINVAL;
 388		}
 389	}
 390
 391	return 0;
 392}
 393
 394/* Check two types for compatibility for the purpose of field access
 395 * relocation. const/volatile/restrict and typedefs are skipped to ensure we
 396 * are relocating semantically compatible entities:
 397 *   - any two STRUCTs/UNIONs are compatible and can be mixed;
 398 *   - any two FWDs are compatible, if their names match (modulo flavor suffix);
 399 *   - any two PTRs are always compatible;
 400 *   - for ENUMs, names should be the same (ignoring flavor suffix) or at
 401 *     least one of enums should be anonymous;
 402 *   - for ENUMs, check sizes, names are ignored;
 403 *   - for INT, size and signedness are ignored;
 404 *   - any two FLOATs are always compatible;
 405 *   - for ARRAY, dimensionality is ignored, element types are checked for
 406 *     compatibility recursively;
 407 *   - everything else shouldn't be ever a target of relocation.
 408 * These rules are not set in stone and probably will be adjusted as we get
 409 * more experience with using BPF CO-RE relocations.
 410 */
 411static int bpf_core_fields_are_compat(const struct btf *local_btf,
 412				      __u32 local_id,
 413				      const struct btf *targ_btf,
 414				      __u32 targ_id)
 415{
 416	const struct btf_type *local_type, *targ_type;
 417
 418recur:
 419	local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
 420	targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
 421	if (!local_type || !targ_type)
 422		return -EINVAL;
 423
 424	if (btf_is_composite(local_type) && btf_is_composite(targ_type))
 425		return 1;
 426	if (!btf_kind_core_compat(local_type, targ_type))
 427		return 0;
 428
 429	switch (btf_kind(local_type)) {
 430	case BTF_KIND_PTR:
 431	case BTF_KIND_FLOAT:
 432		return 1;
 433	case BTF_KIND_FWD:
 434	case BTF_KIND_ENUM64:
 435	case BTF_KIND_ENUM: {
 436		const char *local_name, *targ_name;
 437		size_t local_len, targ_len;
 438
 439		local_name = btf__name_by_offset(local_btf,
 440						 local_type->name_off);
 441		targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
 442		local_len = bpf_core_essential_name_len(local_name);
 443		targ_len = bpf_core_essential_name_len(targ_name);
 444		/* one of them is anonymous or both w/ same flavor-less names */
 445		return local_len == 0 || targ_len == 0 ||
 446		       (local_len == targ_len &&
 447			strncmp(local_name, targ_name, local_len) == 0);
 448	}
 449	case BTF_KIND_INT:
 450		/* just reject deprecated bitfield-like integers; all other
 451		 * integers are by default compatible between each other
 452		 */
 453		return btf_int_offset(local_type) == 0 &&
 454		       btf_int_offset(targ_type) == 0;
 455	case BTF_KIND_ARRAY:
 456		local_id = btf_array(local_type)->type;
 457		targ_id = btf_array(targ_type)->type;
 458		goto recur;
 459	default:
 460		return 0;
 461	}
 462}
 463
 464/*
 465 * Given single high-level named field accessor in local type, find
 466 * corresponding high-level accessor for a target type. Along the way,
 467 * maintain low-level spec for target as well. Also keep updating target
 468 * bit offset.
 469 *
 470 * Searching is performed through recursive exhaustive enumeration of all
 471 * fields of a struct/union. If there are any anonymous (embedded)
 472 * structs/unions, they are recursively searched as well. If field with
 473 * desired name is found, check compatibility between local and target types,
 474 * before returning result.
 475 *
 476 * 1 is returned, if field is found.
 477 * 0 is returned if no compatible field is found.
 478 * <0 is returned on error.
 479 */
 480static int bpf_core_match_member(const struct btf *local_btf,
 481				 const struct bpf_core_accessor *local_acc,
 482				 const struct btf *targ_btf,
 483				 __u32 targ_id,
 484				 struct bpf_core_spec *spec,
 485				 __u32 *next_targ_id)
 486{
 487	const struct btf_type *local_type, *targ_type;
 488	const struct btf_member *local_member, *m;
 489	const char *local_name, *targ_name;
 490	__u32 local_id;
 491	int i, n, found;
 492
 493	targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
 494	if (!targ_type)
 495		return -EINVAL;
 496	if (!btf_is_composite(targ_type))
 497		return 0;
 498
 499	local_id = local_acc->type_id;
 500	local_type = btf_type_by_id(local_btf, local_id);
 501	local_member = btf_members(local_type) + local_acc->idx;
 502	local_name = btf__name_by_offset(local_btf, local_member->name_off);
 503
 504	n = btf_vlen(targ_type);
 505	m = btf_members(targ_type);
 506	for (i = 0; i < n; i++, m++) {
 507		__u32 bit_offset;
 508
 509		bit_offset = btf_member_bit_offset(targ_type, i);
 510
 511		/* too deep struct/union/array nesting */
 512		if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
 513			return -E2BIG;
 514
 515		/* speculate this member will be the good one */
 516		spec->bit_offset += bit_offset;
 517		spec->raw_spec[spec->raw_len++] = i;
 518
 519		targ_name = btf__name_by_offset(targ_btf, m->name_off);
 520		if (str_is_empty(targ_name)) {
 521			/* embedded struct/union, we need to go deeper */
 522			found = bpf_core_match_member(local_btf, local_acc,
 523						      targ_btf, m->type,
 524						      spec, next_targ_id);
 525			if (found) /* either found or error */
 526				return found;
 527		} else if (strcmp(local_name, targ_name) == 0) {
 528			/* matching named field */
 529			struct bpf_core_accessor *targ_acc;
 530
 531			targ_acc = &spec->spec[spec->len++];
 532			targ_acc->type_id = targ_id;
 533			targ_acc->idx = i;
 534			targ_acc->name = targ_name;
 535
 536			*next_targ_id = m->type;
 537			found = bpf_core_fields_are_compat(local_btf,
 538							   local_member->type,
 539							   targ_btf, m->type);
 540			if (!found)
 541				spec->len--; /* pop accessor */
 542			return found;
 543		}
 544		/* member turned out not to be what we looked for */
 545		spec->bit_offset -= bit_offset;
 546		spec->raw_len--;
 547	}
 548
 549	return 0;
 550}
 551
 552/*
 553 * Try to match local spec to a target type and, if successful, produce full
 554 * target spec (high-level, low-level + bit offset).
 555 */
 556static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
 557			       const struct btf *targ_btf, __u32 targ_id,
 558			       struct bpf_core_spec *targ_spec)
 559{
 560	const struct btf_type *targ_type;
 561	const struct bpf_core_accessor *local_acc;
 562	struct bpf_core_accessor *targ_acc;
 563	int i, sz, matched;
 564	__u32 name_off;
 565
 566	memset(targ_spec, 0, sizeof(*targ_spec));
 567	targ_spec->btf = targ_btf;
 568	targ_spec->root_type_id = targ_id;
 569	targ_spec->relo_kind = local_spec->relo_kind;
 570
 571	if (core_relo_is_type_based(local_spec->relo_kind)) {
 572		if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES)
 573			return bpf_core_types_match(local_spec->btf,
 574						    local_spec->root_type_id,
 575						    targ_btf, targ_id);
 576		else
 577			return bpf_core_types_are_compat(local_spec->btf,
 578							 local_spec->root_type_id,
 579							 targ_btf, targ_id);
 580	}
 581
 582	local_acc = &local_spec->spec[0];
 583	targ_acc = &targ_spec->spec[0];
 584
 585	if (core_relo_is_enumval_based(local_spec->relo_kind)) {
 586		size_t local_essent_len, targ_essent_len;
 587		const char *targ_name;
 588
 589		/* has to resolve to an enum */
 590		targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
 591		if (!btf_is_any_enum(targ_type))
 592			return 0;
 593
 594		local_essent_len = bpf_core_essential_name_len(local_acc->name);
 595
 596		for (i = 0; i < btf_vlen(targ_type); i++) {
 597			if (btf_is_enum(targ_type))
 598				name_off = btf_enum(targ_type)[i].name_off;
 599			else
 600				name_off = btf_enum64(targ_type)[i].name_off;
 601
 602			targ_name = btf__name_by_offset(targ_spec->btf, name_off);
 603			targ_essent_len = bpf_core_essential_name_len(targ_name);
 604			if (targ_essent_len != local_essent_len)
 605				continue;
 606			if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
 607				targ_acc->type_id = targ_id;
 608				targ_acc->idx = i;
 609				targ_acc->name = targ_name;
 610				targ_spec->len++;
 611				targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
 612				targ_spec->raw_len++;
 613				return 1;
 614			}
 615		}
 616		return 0;
 617	}
 618
 619	if (!core_relo_is_field_based(local_spec->relo_kind))
 620		return -EINVAL;
 621
 622	for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
 623		targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
 624						   &targ_id);
 625		if (!targ_type)
 626			return -EINVAL;
 627
 628		if (local_acc->name) {
 629			matched = bpf_core_match_member(local_spec->btf,
 630							local_acc,
 631							targ_btf, targ_id,
 632							targ_spec, &targ_id);
 633			if (matched <= 0)
 634				return matched;
 635		} else {
 636			/* for i=0, targ_id is already treated as array element
 637			 * type (because it's the original struct), for others
 638			 * we should find array element type first
 639			 */
 640			if (i > 0) {
 641				const struct btf_array *a;
 642				bool flex;
 643
 644				if (!btf_is_array(targ_type))
 645					return 0;
 646
 647				a = btf_array(targ_type);
 648				flex = is_flex_arr(targ_btf, targ_acc - 1, a);
 649				if (!flex && local_acc->idx >= a->nelems)
 650					return 0;
 651				if (!skip_mods_and_typedefs(targ_btf, a->type,
 652							    &targ_id))
 653					return -EINVAL;
 654			}
 655
 656			/* too deep struct/union/array nesting */
 657			if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
 658				return -E2BIG;
 659
 660			targ_acc->type_id = targ_id;
 661			targ_acc->idx = local_acc->idx;
 662			targ_acc->name = NULL;
 663			targ_spec->len++;
 664			targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
 665			targ_spec->raw_len++;
 666
 667			sz = btf__resolve_size(targ_btf, targ_id);
 668			if (sz < 0)
 669				return sz;
 670			targ_spec->bit_offset += local_acc->idx * sz * 8;
 671		}
 672	}
 673
 674	return 1;
 675}
 676
 677static int bpf_core_calc_field_relo(const char *prog_name,
 678				    const struct bpf_core_relo *relo,
 679				    const struct bpf_core_spec *spec,
 680				    __u64 *val, __u32 *field_sz, __u32 *type_id,
 681				    bool *validate)
 682{
 683	const struct bpf_core_accessor *acc;
 684	const struct btf_type *t;
 685	__u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id, elem_id;
 686	const struct btf_member *m;
 687	const struct btf_type *mt;
 688	bool bitfield;
 689	__s64 sz;
 690
 691	*field_sz = 0;
 692
 693	if (relo->kind == BPF_CORE_FIELD_EXISTS) {
 694		*val = spec ? 1 : 0;
 695		return 0;
 696	}
 697
 698	if (!spec)
 699		return -EUCLEAN; /* request instruction poisoning */
 700
 701	acc = &spec->spec[spec->len - 1];
 702	t = btf_type_by_id(spec->btf, acc->type_id);
 703
 704	/* a[n] accessor needs special handling */
 705	if (!acc->name) {
 706		if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) {
 707			*val = spec->bit_offset / 8;
 708			/* remember field size for load/store mem size;
 709			 * note, for arrays we care about individual element
 710			 * sizes, not the overall array size
 711			 */
 712			t = skip_mods_and_typedefs(spec->btf, acc->type_id, &elem_id);
 713			while (btf_is_array(t))
 714				t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
 715			sz = btf__resolve_size(spec->btf, elem_id);
 716			if (sz < 0)
 717				return -EINVAL;
 718			*field_sz = sz;
 719			*type_id = acc->type_id;
 720		} else if (relo->kind == BPF_CORE_FIELD_BYTE_SIZE) {
 721			sz = btf__resolve_size(spec->btf, acc->type_id);
 722			if (sz < 0)
 723				return -EINVAL;
 724			*val = sz;
 725		} else {
 726			pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
 727				prog_name, relo->kind, relo->insn_off / 8);
 728			return -EINVAL;
 729		}
 730		if (validate)
 731			*validate = true;
 732		return 0;
 733	}
 734
 735	m = btf_members(t) + acc->idx;
 736	mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
 737	bit_off = spec->bit_offset;
 738	bit_sz = btf_member_bitfield_size(t, acc->idx);
 739
 740	bitfield = bit_sz > 0;
 741	if (bitfield) {
 742		byte_sz = mt->size;
 743		byte_off = bit_off / 8 / byte_sz * byte_sz;
 744		/* figure out smallest int size necessary for bitfield load */
 745		while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
 746			if (byte_sz >= 8) {
 747				/* bitfield can't be read with 64-bit read */
 748				pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
 749					prog_name, relo->kind, relo->insn_off / 8);
 750				return -E2BIG;
 751			}
 752			byte_sz *= 2;
 753			byte_off = bit_off / 8 / byte_sz * byte_sz;
 754		}
 755	} else {
 756		sz = btf__resolve_size(spec->btf, field_type_id);
 757		if (sz < 0)
 758			return -EINVAL;
 759		byte_sz = sz;
 760		byte_off = spec->bit_offset / 8;
 761		bit_sz = byte_sz * 8;
 762	}
 763
 764	/* for bitfields, all the relocatable aspects are ambiguous and we
 765	 * might disagree with compiler, so turn off validation of expected
 766	 * value, except for signedness
 767	 */
 768	if (validate)
 769		*validate = !bitfield;
 770
 771	switch (relo->kind) {
 772	case BPF_CORE_FIELD_BYTE_OFFSET:
 773		*val = byte_off;
 774		if (!bitfield) {
 775			/* remember field size for load/store mem size;
 776			 * note, for arrays we care about individual element
 777			 * sizes, not the overall array size
 778			 */
 779			t = skip_mods_and_typedefs(spec->btf, field_type_id, &elem_id);
 780			while (btf_is_array(t))
 781				t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
 782			sz = btf__resolve_size(spec->btf, elem_id);
 783			if (sz < 0)
 784				return -EINVAL;
 785			*field_sz = sz;
 786			*type_id = field_type_id;
 787		}
 788		break;
 789	case BPF_CORE_FIELD_BYTE_SIZE:
 790		*val = byte_sz;
 791		break;
 792	case BPF_CORE_FIELD_SIGNED:
 793		*val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) ||
 794		       (btf_is_int(mt) && (btf_int_encoding(mt) & BTF_INT_SIGNED));
 795		if (validate)
 796			*validate = true; /* signedness is never ambiguous */
 797		break;
 798	case BPF_CORE_FIELD_LSHIFT_U64:
 799#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 800		*val = 64 - (bit_off + bit_sz - byte_off  * 8);
 801#else
 802		*val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
 803#endif
 804		break;
 805	case BPF_CORE_FIELD_RSHIFT_U64:
 806		*val = 64 - bit_sz;
 807		if (validate)
 808			*validate = true; /* right shift is never ambiguous */
 809		break;
 810	case BPF_CORE_FIELD_EXISTS:
 811	default:
 812		return -EOPNOTSUPP;
 813	}
 814
 815	return 0;
 816}
 817
 818static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
 819				   const struct bpf_core_spec *spec,
 820				   __u64 *val, bool *validate)
 821{
 822	__s64 sz;
 823
 824	/* by default, always check expected value in bpf_insn */
 825	if (validate)
 826		*validate = true;
 827
 828	/* type-based relos return zero when target type is not found */
 829	if (!spec) {
 830		*val = 0;
 831		return 0;
 832	}
 833
 834	switch (relo->kind) {
 835	case BPF_CORE_TYPE_ID_TARGET:
 836		*val = spec->root_type_id;
 837		/* type ID, embedded in bpf_insn, might change during linking,
 838		 * so enforcing it is pointless
 839		 */
 840		if (validate)
 841			*validate = false;
 842		break;
 843	case BPF_CORE_TYPE_EXISTS:
 844	case BPF_CORE_TYPE_MATCHES:
 845		*val = 1;
 846		break;
 847	case BPF_CORE_TYPE_SIZE:
 848		sz = btf__resolve_size(spec->btf, spec->root_type_id);
 849		if (sz < 0)
 850			return -EINVAL;
 851		*val = sz;
 852		break;
 853	case BPF_CORE_TYPE_ID_LOCAL:
 854	/* BPF_CORE_TYPE_ID_LOCAL is handled specially and shouldn't get here */
 855	default:
 856		return -EOPNOTSUPP;
 857	}
 858
 859	return 0;
 860}
 861
 862static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
 863				      const struct bpf_core_spec *spec,
 864				      __u64 *val)
 865{
 866	const struct btf_type *t;
 867
 868	switch (relo->kind) {
 869	case BPF_CORE_ENUMVAL_EXISTS:
 870		*val = spec ? 1 : 0;
 871		break;
 872	case BPF_CORE_ENUMVAL_VALUE:
 873		if (!spec)
 874			return -EUCLEAN; /* request instruction poisoning */
 875		t = btf_type_by_id(spec->btf, spec->spec[0].type_id);
 876		if (btf_is_enum(t))
 877			*val = btf_enum(t)[spec->spec[0].idx].val;
 878		else
 879			*val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx);
 880		break;
 881	default:
 882		return -EOPNOTSUPP;
 883	}
 884
 885	return 0;
 886}
 887
 888/* Calculate original and target relocation values, given local and target
 889 * specs and relocation kind. These values are calculated for each candidate.
 890 * If there are multiple candidates, resulting values should all be consistent
 891 * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.
 892 * If instruction has to be poisoned, *poison will be set to true.
 893 */
 894static int bpf_core_calc_relo(const char *prog_name,
 895			      const struct bpf_core_relo *relo,
 896			      int relo_idx,
 897			      const struct bpf_core_spec *local_spec,
 898			      const struct bpf_core_spec *targ_spec,
 899			      struct bpf_core_relo_res *res)
 900{
 901	int err = -EOPNOTSUPP;
 902
 903	res->orig_val = 0;
 904	res->new_val = 0;
 905	res->poison = false;
 906	res->validate = true;
 907	res->fail_memsz_adjust = false;
 908	res->orig_sz = res->new_sz = 0;
 909	res->orig_type_id = res->new_type_id = 0;
 910
 911	if (core_relo_is_field_based(relo->kind)) {
 912		err = bpf_core_calc_field_relo(prog_name, relo, local_spec,
 913					       &res->orig_val, &res->orig_sz,
 914					       &res->orig_type_id, &res->validate);
 915		err = err ?: bpf_core_calc_field_relo(prog_name, relo, targ_spec,
 916						      &res->new_val, &res->new_sz,
 917						      &res->new_type_id, NULL);
 918		if (err)
 919			goto done;
 920		/* Validate if it's safe to adjust load/store memory size.
 921		 * Adjustments are performed only if original and new memory
 922		 * sizes differ.
 923		 */
 924		res->fail_memsz_adjust = false;
 925		if (res->orig_sz != res->new_sz) {
 926			const struct btf_type *orig_t, *new_t;
 927
 928			orig_t = btf_type_by_id(local_spec->btf, res->orig_type_id);
 929			new_t = btf_type_by_id(targ_spec->btf, res->new_type_id);
 930
 931			/* There are two use cases in which it's safe to
 932			 * adjust load/store's mem size:
 933			 *   - reading a 32-bit kernel pointer, while on BPF
 934			 *   size pointers are always 64-bit; in this case
 935			 *   it's safe to "downsize" instruction size due to
 936			 *   pointer being treated as unsigned integer with
 937			 *   zero-extended upper 32-bits;
 938			 *   - reading unsigned integers, again due to
 939			 *   zero-extension is preserving the value correctly.
 940			 *
 941			 * In all other cases it's incorrect to attempt to
 942			 * load/store field because read value will be
 943			 * incorrect, so we poison relocated instruction.
 944			 */
 945			if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
 946				goto done;
 947			if (btf_is_int(orig_t) && btf_is_int(new_t) &&
 948			    btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
 949			    btf_int_encoding(new_t) != BTF_INT_SIGNED)
 950				goto done;
 951
 952			/* mark as invalid mem size adjustment, but this will
 953			 * only be checked for LDX/STX/ST insns
 954			 */
 955			res->fail_memsz_adjust = true;
 956		}
 957	} else if (core_relo_is_type_based(relo->kind)) {
 958		err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val, &res->validate);
 959		err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val, NULL);
 960	} else if (core_relo_is_enumval_based(relo->kind)) {
 961		err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
 962		err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
 963	}
 964
 965done:
 966	if (err == -EUCLEAN) {
 967		/* EUCLEAN is used to signal instruction poisoning request */
 968		res->poison = true;
 969		err = 0;
 970	} else if (err == -EOPNOTSUPP) {
 971		/* EOPNOTSUPP means unknown/unsupported relocation */
 972		pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
 973			prog_name, relo_idx, core_relo_kind_str(relo->kind),
 974			relo->kind, relo->insn_off / 8);
 975	}
 976
 977	return err;
 978}
 979
 980/*
 981 * Turn instruction for which CO_RE relocation failed into invalid one with
 982 * distinct signature.
 983 */
 984static void bpf_core_poison_insn(const char *prog_name, int relo_idx,
 985				 int insn_idx, struct bpf_insn *insn)
 986{
 987	pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
 988		 prog_name, relo_idx, insn_idx);
 989	insn->code = BPF_JMP | BPF_CALL;
 990	insn->dst_reg = 0;
 991	insn->src_reg = 0;
 992	insn->off = 0;
 993	/* if this instruction is reachable (not a dead code),
 994	 * verifier will complain with the following message:
 995	 * invalid func unknown#195896080
 996	 */
 997	insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
 998}
 999
1000static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
1001{
1002	switch (BPF_SIZE(insn->code)) {
1003	case BPF_DW: return 8;
1004	case BPF_W: return 4;
1005	case BPF_H: return 2;
1006	case BPF_B: return 1;
1007	default: return -1;
1008	}
1009}
1010
1011static int insn_bytes_to_bpf_size(__u32 sz)
1012{
1013	switch (sz) {
1014	case 8: return BPF_DW;
1015	case 4: return BPF_W;
1016	case 2: return BPF_H;
1017	case 1: return BPF_B;
1018	default: return -1;
1019	}
1020}
1021
1022/*
1023 * Patch relocatable BPF instruction.
1024 *
1025 * Patched value is determined by relocation kind and target specification.
1026 * For existence relocations target spec will be NULL if field/type is not found.
1027 * Expected insn->imm value is determined using relocation kind and local
1028 * spec, and is checked before patching instruction. If actual insn->imm value
1029 * is wrong, bail out with error.
1030 *
1031 * Currently supported classes of BPF instruction are:
1032 * 1. rX = <imm> (assignment with immediate operand);
1033 * 2. rX += <imm> (arithmetic operations with immediate operand);
1034 * 3. rX = <imm64> (load with 64-bit immediate value);
1035 * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};
1036 * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
1037 * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
1038 */
1039int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
1040			int insn_idx, const struct bpf_core_relo *relo,
1041			int relo_idx, const struct bpf_core_relo_res *res)
1042{
1043	__u64 orig_val, new_val;
1044	__u8 class;
1045
1046	class = BPF_CLASS(insn->code);
1047
1048	if (res->poison) {
1049poison:
1050		/* poison second part of ldimm64 to avoid confusing error from
1051		 * verifier about "unknown opcode 00"
1052		 */
1053		if (is_ldimm64_insn(insn))
1054			bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1);
1055		bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn);
1056		return 0;
1057	}
1058
1059	orig_val = res->orig_val;
1060	new_val = res->new_val;
1061
1062	switch (class) {
1063	case BPF_ALU:
1064	case BPF_ALU64:
1065		if (BPF_SRC(insn->code) != BPF_K)
1066			return -EINVAL;
1067		if (res->validate && insn->imm != orig_val) {
1068			pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n",
1069				prog_name, relo_idx,
1070				insn_idx, insn->imm, (unsigned long long)orig_val,
1071				(unsigned long long)new_val);
1072			return -EINVAL;
1073		}
1074		orig_val = insn->imm;
1075		insn->imm = new_val;
1076		pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n",
1077			 prog_name, relo_idx, insn_idx,
1078			 (unsigned long long)orig_val, (unsigned long long)new_val);
1079		break;
1080	case BPF_LDX:
1081	case BPF_ST:
1082	case BPF_STX:
1083		if (res->validate && insn->off != orig_val) {
1084			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n",
1085				prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val,
1086				(unsigned long long)new_val);
1087			return -EINVAL;
1088		}
1089		if (new_val > SHRT_MAX) {
1090			pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n",
1091				prog_name, relo_idx, insn_idx, (unsigned long long)new_val);
1092			return -ERANGE;
1093		}
1094		if (res->fail_memsz_adjust) {
1095			pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
1096				"Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
1097				prog_name, relo_idx, insn_idx);
1098			goto poison;
1099		}
1100
1101		orig_val = insn->off;
1102		insn->off = new_val;
1103		pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n",
1104			 prog_name, relo_idx, insn_idx, (unsigned long long)orig_val,
1105			 (unsigned long long)new_val);
1106
1107		if (res->new_sz != res->orig_sz) {
1108			int insn_bytes_sz, insn_bpf_sz;
1109
1110			insn_bytes_sz = insn_bpf_size_to_bytes(insn);
1111			if (insn_bytes_sz != res->orig_sz) {
1112				pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
1113					prog_name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
1114				return -EINVAL;
1115			}
1116
1117			insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
1118			if (insn_bpf_sz < 0) {
1119				pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
1120					prog_name, relo_idx, insn_idx, res->new_sz);
1121				return -EINVAL;
1122			}
1123
1124			insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
1125			pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
1126				 prog_name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
1127		}
1128		break;
1129	case BPF_LD: {
1130		__u64 imm;
1131
1132		if (!is_ldimm64_insn(insn) ||
1133		    insn[0].src_reg != 0 || insn[0].off != 0 ||
1134		    insn[1].code != 0 || insn[1].dst_reg != 0 ||
1135		    insn[1].src_reg != 0 || insn[1].off != 0) {
1136			pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
1137				prog_name, relo_idx, insn_idx);
1138			return -EINVAL;
1139		}
1140
1141		imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32);
1142		if (res->validate && imm != orig_val) {
1143			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n",
1144				prog_name, relo_idx,
1145				insn_idx, (unsigned long long)imm,
1146				(unsigned long long)orig_val, (unsigned long long)new_val);
1147			return -EINVAL;
1148		}
1149
1150		insn[0].imm = new_val;
1151		insn[1].imm = new_val >> 32;
1152		pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n",
1153			 prog_name, relo_idx, insn_idx,
1154			 (unsigned long long)imm, (unsigned long long)new_val);
1155		break;
1156	}
1157	default:
1158		pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
1159			prog_name, relo_idx, insn_idx, insn->code,
1160			insn->src_reg, insn->dst_reg, insn->off, insn->imm);
1161		return -EINVAL;
1162	}
1163
1164	return 0;
1165}
1166
1167/* Output spec definition in the format:
1168 * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
1169 * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
1170 */
1171int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec)
1172{
1173	const struct btf_type *t;
1174	const char *s;
1175	__u32 type_id;
1176	int i, len = 0;
1177
1178#define append_buf(fmt, args...)				\
1179	({							\
1180		int r;						\
1181		r = snprintf(buf, buf_sz, fmt, ##args);		\
1182		len += r;					\
1183		if (r >= buf_sz)				\
1184			r = buf_sz;				\
1185		buf += r;					\
1186		buf_sz -= r;					\
1187	})
1188
1189	type_id = spec->root_type_id;
1190	t = btf_type_by_id(spec->btf, type_id);
1191	s = btf__name_by_offset(spec->btf, t->name_off);
1192
1193	append_buf("<%s> [%u] %s %s",
1194		   core_relo_kind_str(spec->relo_kind),
1195		   type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
1196
1197	if (core_relo_is_type_based(spec->relo_kind))
1198		return len;
1199
1200	if (core_relo_is_enumval_based(spec->relo_kind)) {
1201		t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
1202		if (btf_is_enum(t)) {
1203			const struct btf_enum *e;
1204			const char *fmt_str;
1205
1206			e = btf_enum(t) + spec->raw_spec[0];
1207			s = btf__name_by_offset(spec->btf, e->name_off);
1208			fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u";
1209			append_buf(fmt_str, s, e->val);
1210		} else {
1211			const struct btf_enum64 *e;
1212			const char *fmt_str;
1213
1214			e = btf_enum64(t) + spec->raw_spec[0];
1215			s = btf__name_by_offset(spec->btf, e->name_off);
1216			fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu";
1217			append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e));
1218		}
1219		return len;
1220	}
1221
1222	if (core_relo_is_field_based(spec->relo_kind)) {
1223		for (i = 0; i < spec->len; i++) {
1224			if (spec->spec[i].name)
1225				append_buf(".%s", spec->spec[i].name);
1226			else if (i > 0 || spec->spec[i].idx > 0)
1227				append_buf("[%u]", spec->spec[i].idx);
1228		}
1229
1230		append_buf(" (");
1231		for (i = 0; i < spec->raw_len; i++)
1232			append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
1233
1234		if (spec->bit_offset % 8)
1235			append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8);
1236		else
1237			append_buf(" @ offset %u)", spec->bit_offset / 8);
1238		return len;
1239	}
1240
1241	return len;
1242#undef append_buf
1243}
1244
1245/*
1246 * Calculate CO-RE relocation target result.
1247 *
1248 * The outline and important points of the algorithm:
1249 * 1. For given local type, find corresponding candidate target types.
1250 *    Candidate type is a type with the same "essential" name, ignoring
1251 *    everything after last triple underscore (___). E.g., `sample`,
1252 *    `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
1253 *    for each other. Names with triple underscore are referred to as
1254 *    "flavors" and are useful, among other things, to allow to
1255 *    specify/support incompatible variations of the same kernel struct, which
1256 *    might differ between different kernel versions and/or build
1257 *    configurations.
1258 *
1259 *    N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
1260 *    converter, when deduplicated BTF of a kernel still contains more than
1261 *    one different types with the same name. In that case, ___2, ___3, etc
1262 *    are appended starting from second name conflict. But start flavors are
1263 *    also useful to be defined "locally", in BPF program, to extract same
1264 *    data from incompatible changes between different kernel
1265 *    versions/configurations. For instance, to handle field renames between
1266 *    kernel versions, one can use two flavors of the struct name with the
1267 *    same common name and use conditional relocations to extract that field,
1268 *    depending on target kernel version.
1269 * 2. For each candidate type, try to match local specification to this
1270 *    candidate target type. Matching involves finding corresponding
1271 *    high-level spec accessors, meaning that all named fields should match,
1272 *    as well as all array accesses should be within the actual bounds. Also,
1273 *    types should be compatible (see bpf_core_fields_are_compat for details).
1274 * 3. It is supported and expected that there might be multiple flavors
1275 *    matching the spec. As long as all the specs resolve to the same set of
1276 *    offsets across all candidates, there is no error. If there is any
1277 *    ambiguity, CO-RE relocation will fail. This is necessary to accommodate
1278 *    imperfection of BTF deduplication, which can cause slight duplication of
1279 *    the same BTF type, if some directly or indirectly referenced (by
1280 *    pointer) type gets resolved to different actual types in different
1281 *    object files. If such a situation occurs, deduplicated BTF will end up
1282 *    with two (or more) structurally identical types, which differ only in
1283 *    types they refer to through pointer. This should be OK in most cases and
1284 *    is not an error.
1285 * 4. Candidate types search is performed by linearly scanning through all
1286 *    types in target BTF. It is anticipated that this is overall more
1287 *    efficient memory-wise and not significantly worse (if not better)
1288 *    CPU-wise compared to prebuilding a map from all local type names to
1289 *    a list of candidate type names. It's also sped up by caching resolved
1290 *    list of matching candidates per each local "root" type ID, that has at
1291 *    least one bpf_core_relo associated with it. This list is shared
1292 *    between multiple relocations for the same type ID and is updated as some
1293 *    of the candidates are pruned due to structural incompatibility.
1294 */
1295int bpf_core_calc_relo_insn(const char *prog_name,
1296			    const struct bpf_core_relo *relo,
1297			    int relo_idx,
1298			    const struct btf *local_btf,
1299			    struct bpf_core_cand_list *cands,
1300			    struct bpf_core_spec *specs_scratch,
1301			    struct bpf_core_relo_res *targ_res)
1302{
1303	struct bpf_core_spec *local_spec = &specs_scratch[0];
1304	struct bpf_core_spec *cand_spec = &specs_scratch[1];
1305	struct bpf_core_spec *targ_spec = &specs_scratch[2];
1306	struct bpf_core_relo_res cand_res;
1307	const struct btf_type *local_type;
1308	const char *local_name;
1309	__u32 local_id;
1310	char spec_buf[256];
1311	int i, j, err;
1312
1313	local_id = relo->type_id;
1314	local_type = btf_type_by_id(local_btf, local_id);
1315	local_name = btf__name_by_offset(local_btf, local_type->name_off);
1316	if (!local_name)
1317		return -EINVAL;
1318
1319	err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec);
1320	if (err) {
1321		const char *spec_str;
1322
1323		spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
1324		pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
1325			prog_name, relo_idx, local_id, btf_kind_str(local_type),
1326			str_is_empty(local_name) ? "<anon>" : local_name,
1327			spec_str ?: "<?>", err);
1328		return -EINVAL;
1329	}
1330
1331	bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec);
1332	pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf);
1333
1334	/* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
1335	if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {
1336		/* bpf_insn's imm value could get out of sync during linking */
1337		memset(targ_res, 0, sizeof(*targ_res));
1338		targ_res->validate = false;
1339		targ_res->poison = false;
1340		targ_res->orig_val = local_spec->root_type_id;
1341		targ_res->new_val = local_spec->root_type_id;
1342		return 0;
1343	}
1344
1345	/* libbpf doesn't support candidate search for anonymous types */
1346	if (str_is_empty(local_name)) {
1347		pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
1348			prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
1349		return -EOPNOTSUPP;
1350	}
1351
1352	for (i = 0, j = 0; i < cands->len; i++) {
1353		err = bpf_core_spec_match(local_spec, cands->cands[i].btf,
1354					  cands->cands[i].id, cand_spec);
1355		if (err < 0) {
1356			bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
1357			pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n",
1358				prog_name, relo_idx, i, spec_buf, err);
1359			return err;
1360		}
1361
1362		bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
1363		pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name,
1364			 relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf);
1365
1366		if (err == 0)
1367			continue;
1368
1369		err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res);
1370		if (err)
1371			return err;
1372
1373		if (j == 0) {
1374			*targ_res = cand_res;
1375			*targ_spec = *cand_spec;
1376		} else if (cand_spec->bit_offset != targ_spec->bit_offset) {
1377			/* if there are many field relo candidates, they
1378			 * should all resolve to the same bit offset
1379			 */
1380			pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
1381				prog_name, relo_idx, cand_spec->bit_offset,
1382				targ_spec->bit_offset);
1383			return -EINVAL;
1384		} else if (cand_res.poison != targ_res->poison ||
1385			   cand_res.new_val != targ_res->new_val) {
1386			/* all candidates should result in the same relocation
1387			 * decision and value, otherwise it's dangerous to
1388			 * proceed due to ambiguity
1389			 */
1390			pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n",
1391				prog_name, relo_idx,
1392				cand_res.poison ? "failure" : "success",
1393				(unsigned long long)cand_res.new_val,
1394				targ_res->poison ? "failure" : "success",
1395				(unsigned long long)targ_res->new_val);
1396			return -EINVAL;
1397		}
1398
1399		cands->cands[j++] = cands->cands[i];
1400	}
1401
1402	/*
1403	 * For BPF_CORE_FIELD_EXISTS relo or when used BPF program has field
1404	 * existence checks or kernel version/config checks, it's expected
1405	 * that we might not find any candidates. In this case, if field
1406	 * wasn't found in any candidate, the list of candidates shouldn't
1407	 * change at all, we'll just handle relocating appropriately,
1408	 * depending on relo's kind.
1409	 */
1410	if (j > 0)
1411		cands->len = j;
1412
1413	/*
1414	 * If no candidates were found, it might be both a programmer error,
1415	 * as well as expected case, depending whether instruction w/
1416	 * relocation is guarded in some way that makes it unreachable (dead
1417	 * code) if relocation can't be resolved. This is handled in
1418	 * bpf_core_patch_insn() uniformly by replacing that instruction with
1419	 * BPF helper call insn (using invalid helper ID). If that instruction
1420	 * is indeed unreachable, then it will be ignored and eliminated by
1421	 * verifier. If it was an error, then verifier will complain and point
1422	 * to a specific instruction number in its log.
1423	 */
1424	if (j == 0) {
1425		pr_debug("prog '%s': relo #%d: no matching targets found\n",
1426			 prog_name, relo_idx);
1427
1428		/* calculate single target relo result explicitly */
1429		err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, targ_res);
1430		if (err)
1431			return err;
1432	}
1433
1434	return 0;
1435}
1436
1437static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off,
1438				 const struct btf *targ_btf, size_t targ_name_off)
1439{
1440	const char *local_n, *targ_n;
1441	size_t local_len, targ_len;
1442
1443	local_n = btf__name_by_offset(local_btf, local_name_off);
1444	targ_n = btf__name_by_offset(targ_btf, targ_name_off);
1445
1446	if (str_is_empty(targ_n))
1447		return str_is_empty(local_n);
1448
1449	targ_len = bpf_core_essential_name_len(targ_n);
1450	local_len = bpf_core_essential_name_len(local_n);
1451
1452	return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0;
1453}
1454
1455static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t,
1456				const struct btf *targ_btf, const struct btf_type *targ_t)
1457{
1458	__u16 local_vlen = btf_vlen(local_t);
1459	__u16 targ_vlen = btf_vlen(targ_t);
1460	int i, j;
1461
1462	if (local_t->size != targ_t->size)
1463		return 0;
1464
1465	if (local_vlen > targ_vlen)
1466		return 0;
1467
1468	/* iterate over the local enum's variants and make sure each has
1469	 * a symbolic name correspondent in the target
1470	 */
1471	for (i = 0; i < local_vlen; i++) {
1472		bool matched = false;
1473		__u32 local_n_off, targ_n_off;
1474
1475		local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off :
1476						     btf_enum64(local_t)[i].name_off;
1477
1478		for (j = 0; j < targ_vlen; j++) {
1479			targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off :
1480							   btf_enum64(targ_t)[j].name_off;
1481
1482			if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) {
1483				matched = true;
1484				break;
1485			}
1486		}
1487
1488		if (!matched)
1489			return 0;
1490	}
1491	return 1;
1492}
1493
1494static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t,
1495				     const struct btf *targ_btf, const struct btf_type *targ_t,
1496				     bool behind_ptr, int level)
1497{
1498	const struct btf_member *local_m = btf_members(local_t);
1499	__u16 local_vlen = btf_vlen(local_t);
1500	__u16 targ_vlen = btf_vlen(targ_t);
1501	int i, j, err;
1502
1503	if (local_vlen > targ_vlen)
1504		return 0;
1505
1506	/* check that all local members have a match in the target */
1507	for (i = 0; i < local_vlen; i++, local_m++) {
1508		const struct btf_member *targ_m = btf_members(targ_t);
1509		bool matched = false;
1510
1511		for (j = 0; j < targ_vlen; j++, targ_m++) {
1512			if (!bpf_core_names_match(local_btf, local_m->name_off,
1513						  targ_btf, targ_m->name_off))
1514				continue;
1515
1516			err = __bpf_core_types_match(local_btf, local_m->type, targ_btf,
1517						     targ_m->type, behind_ptr, level - 1);
1518			if (err < 0)
1519				return err;
1520			if (err > 0) {
1521				matched = true;
1522				break;
1523			}
1524		}
1525
1526		if (!matched)
1527			return 0;
1528	}
1529	return 1;
1530}
1531
1532/* Check that two types "match". This function assumes that root types were
1533 * already checked for name match.
1534 *
1535 * The matching relation is defined as follows:
1536 * - modifiers and typedefs are stripped (and, hence, effectively ignored)
1537 * - generally speaking types need to be of same kind (struct vs. struct, union
1538 *   vs. union, etc.)
1539 *   - exceptions are struct/union behind a pointer which could also match a
1540 *     forward declaration of a struct or union, respectively, and enum vs.
1541 *     enum64 (see below)
1542 * Then, depending on type:
1543 * - integers:
1544 *   - match if size and signedness match
1545 * - arrays & pointers:
1546 *   - target types are recursively matched
1547 * - structs & unions:
1548 *   - local members need to exist in target with the same name
1549 *   - for each member we recursively check match unless it is already behind a
1550 *     pointer, in which case we only check matching names and compatible kind
1551 * - enums:
1552 *   - local variants have to have a match in target by symbolic name (but not
1553 *     numeric value)
1554 *   - size has to match (but enum may match enum64 and vice versa)
1555 * - function pointers:
1556 *   - number and position of arguments in local type has to match target
1557 *   - for each argument and the return value we recursively check match
1558 */
1559int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf,
1560			   __u32 targ_id, bool behind_ptr, int level)
1561{
1562	const struct btf_type *local_t, *targ_t;
1563	int depth = 32; /* max recursion depth */
1564	__u16 local_k, targ_k;
1565
1566	if (level <= 0)
1567		return -EINVAL;
1568
1569recur:
1570	depth--;
1571	if (depth < 0)
1572		return -EINVAL;
1573
1574	local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id);
1575	targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
1576	if (!local_t || !targ_t)
1577		return -EINVAL;
1578
1579	/* While the name check happens after typedefs are skipped, root-level
1580	 * typedefs would still be name-matched as that's the contract with
1581	 * callers.
1582	 */
1583	if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off))
1584		return 0;
1585
1586	local_k = btf_kind(local_t);
1587	targ_k = btf_kind(targ_t);
1588
1589	switch (local_k) {
1590	case BTF_KIND_UNKN:
1591		return local_k == targ_k;
1592	case BTF_KIND_FWD: {
1593		bool local_f = BTF_INFO_KFLAG(local_t->info);
1594
1595		if (behind_ptr) {
1596			if (local_k == targ_k)
1597				return local_f == BTF_INFO_KFLAG(targ_t->info);
1598
1599			/* for forward declarations kflag dictates whether the
1600			 * target is a struct (0) or union (1)
1601			 */
1602			return (targ_k == BTF_KIND_STRUCT && !local_f) ||
1603			       (targ_k == BTF_KIND_UNION && local_f);
1604		} else {
1605			if (local_k != targ_k)
1606				return 0;
1607
1608			/* match if the forward declaration is for the same kind */
1609			return local_f == BTF_INFO_KFLAG(targ_t->info);
1610		}
1611	}
1612	case BTF_KIND_ENUM:
1613	case BTF_KIND_ENUM64:
1614		if (!btf_is_any_enum(targ_t))
1615			return 0;
1616
1617		return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t);
1618	case BTF_KIND_STRUCT:
1619	case BTF_KIND_UNION:
1620		if (behind_ptr) {
1621			bool targ_f = BTF_INFO_KFLAG(targ_t->info);
1622
1623			if (local_k == targ_k)
1624				return 1;
1625
1626			if (targ_k != BTF_KIND_FWD)
1627				return 0;
1628
1629			return (local_k == BTF_KIND_UNION) == targ_f;
1630		} else {
1631			if (local_k != targ_k)
1632				return 0;
1633
1634			return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t,
1635							 behind_ptr, level);
1636		}
1637	case BTF_KIND_INT: {
1638		__u8 local_sgn;
1639		__u8 targ_sgn;
1640
1641		if (local_k != targ_k)
1642			return 0;
1643
1644		local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED;
1645		targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED;
1646
1647		return local_t->size == targ_t->size && local_sgn == targ_sgn;
1648	}
1649	case BTF_KIND_PTR:
1650		if (local_k != targ_k)
1651			return 0;
1652
1653		behind_ptr = true;
1654
1655		local_id = local_t->type;
1656		targ_id = targ_t->type;
1657		goto recur;
1658	case BTF_KIND_ARRAY: {
1659		const struct btf_array *local_array = btf_array(local_t);
1660		const struct btf_array *targ_array = btf_array(targ_t);
1661
1662		if (local_k != targ_k)
1663			return 0;
1664
1665		if (local_array->nelems != targ_array->nelems)
1666			return 0;
1667
1668		local_id = local_array->type;
1669		targ_id = targ_array->type;
1670		goto recur;
1671	}
1672	case BTF_KIND_FUNC_PROTO: {
1673		struct btf_param *local_p = btf_params(local_t);
1674		struct btf_param *targ_p = btf_params(targ_t);
1675		__u16 local_vlen = btf_vlen(local_t);
1676		__u16 targ_vlen = btf_vlen(targ_t);
1677		int i, err;
1678
1679		if (local_k != targ_k)
1680			return 0;
1681
1682		if (local_vlen != targ_vlen)
1683			return 0;
1684
1685		for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
1686			err = __bpf_core_types_match(local_btf, local_p->type, targ_btf,
1687						     targ_p->type, behind_ptr, level - 1);
1688			if (err <= 0)
1689				return err;
1690		}
1691
1692		/* tail recurse for return type check */
1693		local_id = local_t->type;
1694		targ_id = targ_t->type;
1695		goto recur;
1696	}
1697	default:
1698		pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
1699			btf_kind_str(local_t), local_id, targ_id);
1700		return 0;
1701	}
1702}