bpf.c

   1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3/*
   4 * common eBPF ELF operations.
   5 *
   6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8 * Copyright (C) 2015 Huawei Inc.
   9 *
  10 * This program is free software; you can redistribute it and/or
  11 * modify it under the terms of the GNU Lesser General Public
  12 * License as published by the Free Software Foundation;
  13 * version 2.1 of the License (not later!)
  14 *
  15 * This program is distributed in the hope that it will be useful,
  16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 * GNU Lesser General Public License for more details.
  19 *
  20 * You should have received a copy of the GNU Lesser General Public
  21 * License along with this program; if not,  see <http://www.gnu.org/licenses>
  22 */
  23
  24#include <stdlib.h>
  25#include <string.h>
  26#include <memory.h>
  27#include <unistd.h>
  28#include <asm/unistd.h>
  29#include <errno.h>
  30#include <linux/bpf.h>
  31#include <linux/filter.h>
  32#include <linux/kernel.h>
  33#include <limits.h>
  34#include <sys/resource.h>
  35#include "bpf.h"
  36#include "libbpf.h"
  37#include "libbpf_internal.h"
  38
  39/*
  40 * When building perf, unistd.h is overridden. __NR_bpf is
  41 * required to be defined explicitly.
  42 */
  43#ifndef __NR_bpf
  44# if defined(__i386__)
  45#  define __NR_bpf 357
  46# elif defined(__x86_64__)
  47#  define __NR_bpf 321
  48# elif defined(__aarch64__)
  49#  define __NR_bpf 280
  50# elif defined(__sparc__)
  51#  define __NR_bpf 349
  52# elif defined(__s390__)
  53#  define __NR_bpf 351
  54# elif defined(__arc__)
  55#  define __NR_bpf 280
  56# elif defined(__mips__) && defined(_ABIO32)
  57#  define __NR_bpf 4355
  58# elif defined(__mips__) && defined(_ABIN32)
  59#  define __NR_bpf 6319
  60# elif defined(__mips__) && defined(_ABI64)
  61#  define __NR_bpf 5315
  62# else
  63#  error __NR_bpf not defined. libbpf does not support your arch.
  64# endif
  65#endif
  66
  67static inline __u64 ptr_to_u64(const void *ptr)
  68{
  69	return (__u64) (unsigned long) ptr;
  70}
  71
  72static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
  73			  unsigned int size)
  74{
  75	return syscall(__NR_bpf, cmd, attr, size);
  76}
  77
  78static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr,
  79			     unsigned int size)
  80{
  81	int fd;
  82
  83	fd = sys_bpf(cmd, attr, size);
  84	return ensure_good_fd(fd);
  85}
  86
  87int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts)
  88{
  89	int fd;
  90
  91	do {
  92		fd = sys_bpf_fd(BPF_PROG_LOAD, attr, size);
  93	} while (fd < 0 && errno == EAGAIN && --attempts > 0);
  94
  95	return fd;
  96}
  97
  98/* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to
  99 * memcg-based memory accounting for BPF maps and progs. This was done in [0].
 100 * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in
 101 * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF.
 102 *
 103 *   [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/
 104 *   [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper")
 105 */
 106int probe_memcg_account(int token_fd)
 107{
 108	const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd);
 109	struct bpf_insn insns[] = {
 110		BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns),
 111		BPF_EXIT_INSN(),
 112	};
 113	size_t insn_cnt = ARRAY_SIZE(insns);
 114	union bpf_attr attr;
 115	int prog_fd;
 116
 117	/* attempt loading freplace trying to use custom BTF */
 118	memset(&attr, 0, attr_sz);
 119	attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
 120	attr.insns = ptr_to_u64(insns);
 121	attr.insn_cnt = insn_cnt;
 122	attr.license = ptr_to_u64("GPL");
 123	attr.prog_token_fd = token_fd;
 124	if (token_fd)
 125		attr.prog_flags |= BPF_F_TOKEN_FD;
 126
 127	prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz);
 128	if (prog_fd >= 0) {
 129		close(prog_fd);
 130		return 1;
 131	}
 132	return 0;
 133}
 134
 135static bool memlock_bumped;
 136static rlim_t memlock_rlim = RLIM_INFINITY;
 137
 138int libbpf_set_memlock_rlim(size_t memlock_bytes)
 139{
 140	if (memlock_bumped)
 141		return libbpf_err(-EBUSY);
 142
 143	memlock_rlim = memlock_bytes;
 144	return 0;
 145}
 146
 147int bump_rlimit_memlock(void)
 148{
 149	struct rlimit rlim;
 150
 151	/* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */
 152	if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT))
 153		return 0;
 154
 155	memlock_bumped = true;
 156
 157	/* zero memlock_rlim disables auto-bumping RLIMIT_MEMLOCK */
 158	if (memlock_rlim == 0)
 159		return 0;
 160
 161	rlim.rlim_cur = rlim.rlim_max = memlock_rlim;
 162	if (setrlimit(RLIMIT_MEMLOCK, &rlim))
 163		return -errno;
 164
 165	return 0;
 166}
 167
 168int bpf_map_create(enum bpf_map_type map_type,
 169		   const char *map_name,
 170		   __u32 key_size,
 171		   __u32 value_size,
 172		   __u32 max_entries,
 173		   const struct bpf_map_create_opts *opts)
 174{
 175	const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size);
 176	union bpf_attr attr;
 177	int fd;
 178
 179	bump_rlimit_memlock();
 180
 181	memset(&attr, 0, attr_sz);
 182
 183	if (!OPTS_VALID(opts, bpf_map_create_opts))
 184		return libbpf_err(-EINVAL);
 185
 186	attr.map_type = map_type;
 187	if (map_name && feat_supported(NULL, FEAT_PROG_NAME))
 188		libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
 189	attr.key_size = key_size;
 190	attr.value_size = value_size;
 191	attr.max_entries = max_entries;
 192
 193	attr.btf_fd = OPTS_GET(opts, btf_fd, 0);
 194	attr.btf_key_type_id = OPTS_GET(opts, btf_key_type_id, 0);
 195	attr.btf_value_type_id = OPTS_GET(opts, btf_value_type_id, 0);
 196	attr.btf_vmlinux_value_type_id = OPTS_GET(opts, btf_vmlinux_value_type_id, 0);
 197	attr.value_type_btf_obj_fd = OPTS_GET(opts, value_type_btf_obj_fd, 0);
 198
 199	attr.inner_map_fd = OPTS_GET(opts, inner_map_fd, 0);
 200	attr.map_flags = OPTS_GET(opts, map_flags, 0);
 201	attr.map_extra = OPTS_GET(opts, map_extra, 0);
 202	attr.numa_node = OPTS_GET(opts, numa_node, 0);
 203	attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0);
 204
 205	attr.map_token_fd = OPTS_GET(opts, token_fd, 0);
 206	attr.excl_prog_hash = ptr_to_u64(OPTS_GET(opts, excl_prog_hash, NULL));
 207	attr.excl_prog_hash_size = OPTS_GET(opts, excl_prog_hash_size, 0);
 208
 209	fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz);
 210	return libbpf_err_errno(fd);
 211}
 212
 213static void *
 214alloc_zero_tailing_info(const void *orecord, __u32 cnt,
 215			__u32 actual_rec_size, __u32 expected_rec_size)
 216{
 217	__u64 info_len = (__u64)actual_rec_size * cnt;
 218	void *info, *nrecord;
 219	int i;
 220
 221	info = malloc(info_len);
 222	if (!info)
 223		return NULL;
 224
 225	/* zero out bytes kernel does not understand */
 226	nrecord = info;
 227	for (i = 0; i < cnt; i++) {
 228		memcpy(nrecord, orecord, expected_rec_size);
 229		memset(nrecord + expected_rec_size, 0,
 230		       actual_rec_size - expected_rec_size);
 231		orecord += actual_rec_size;
 232		nrecord += actual_rec_size;
 233	}
 234
 235	return info;
 236}
 237
 238int bpf_prog_load(enum bpf_prog_type prog_type,
 239		  const char *prog_name, const char *license,
 240		  const struct bpf_insn *insns, size_t insn_cnt,
 241		  struct bpf_prog_load_opts *opts)
 242{
 243	const size_t attr_sz = offsetofend(union bpf_attr, keyring_id);
 244	void *finfo = NULL, *linfo = NULL;
 245	const char *func_info, *line_info;
 246	__u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd;
 247	__u32 func_info_rec_size, line_info_rec_size;
 248	int fd, attempts;
 249	union bpf_attr attr;
 250	char *log_buf;
 251
 252	bump_rlimit_memlock();
 253
 254	if (!OPTS_VALID(opts, bpf_prog_load_opts))
 255		return libbpf_err(-EINVAL);
 256
 257	attempts = OPTS_GET(opts, attempts, 0);
 258	if (attempts < 0)
 259		return libbpf_err(-EINVAL);
 260	if (attempts == 0)
 261		attempts = PROG_LOAD_ATTEMPTS;
 262
 263	memset(&attr, 0, attr_sz);
 264
 265	attr.prog_type = prog_type;
 266	attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0);
 267
 268	attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0);
 269	attr.prog_flags = OPTS_GET(opts, prog_flags, 0);
 270	attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0);
 271	attr.kern_version = OPTS_GET(opts, kern_version, 0);
 272	attr.prog_token_fd = OPTS_GET(opts, token_fd, 0);
 273
 274	if (prog_name && feat_supported(NULL, FEAT_PROG_NAME))
 275		libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
 276	attr.license = ptr_to_u64(license);
 277
 278	if (insn_cnt > UINT_MAX)
 279		return libbpf_err(-E2BIG);
 280
 281	attr.insns = ptr_to_u64(insns);
 282	attr.insn_cnt = (__u32)insn_cnt;
 283
 284	attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
 285	attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0);
 286
 287	if (attach_prog_fd && attach_btf_obj_fd)
 288		return libbpf_err(-EINVAL);
 289
 290	attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0);
 291	if (attach_prog_fd)
 292		attr.attach_prog_fd = attach_prog_fd;
 293	else
 294		attr.attach_btf_obj_fd = attach_btf_obj_fd;
 295
 296	log_buf = OPTS_GET(opts, log_buf, NULL);
 297	log_size = OPTS_GET(opts, log_size, 0);
 298	log_level = OPTS_GET(opts, log_level, 0);
 299
 300	if (!!log_buf != !!log_size)
 301		return libbpf_err(-EINVAL);
 302
 303	func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0);
 304	func_info = OPTS_GET(opts, func_info, NULL);
 305	attr.func_info_rec_size = func_info_rec_size;
 306	attr.func_info = ptr_to_u64(func_info);
 307	attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0);
 308
 309	line_info_rec_size = OPTS_GET(opts, line_info_rec_size, 0);
 310	line_info = OPTS_GET(opts, line_info, NULL);
 311	attr.line_info_rec_size = line_info_rec_size;
 312	attr.line_info = ptr_to_u64(line_info);
 313	attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0);
 314
 315	attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL));
 316	attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0);
 317
 318	if (log_level) {
 319		attr.log_buf = ptr_to_u64(log_buf);
 320		attr.log_size = log_size;
 321		attr.log_level = log_level;
 322	}
 323
 324	fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
 325	OPTS_SET(opts, log_true_size, attr.log_true_size);
 326	if (fd >= 0)
 327		return fd;
 328
 329	/* After bpf_prog_load, the kernel may modify certain attributes
 330	 * to give user space a hint how to deal with loading failure.
 331	 * Check to see whether we can make some changes and load again.
 332	 */
 333	while (errno == E2BIG && (!finfo || !linfo)) {
 334		if (!finfo && attr.func_info_cnt &&
 335		    attr.func_info_rec_size < func_info_rec_size) {
 336			/* try with corrected func info records */
 337			finfo = alloc_zero_tailing_info(func_info,
 338							attr.func_info_cnt,
 339							func_info_rec_size,
 340							attr.func_info_rec_size);
 341			if (!finfo) {
 342				errno = E2BIG;
 343				goto done;
 344			}
 345
 346			attr.func_info = ptr_to_u64(finfo);
 347			attr.func_info_rec_size = func_info_rec_size;
 348		} else if (!linfo && attr.line_info_cnt &&
 349			   attr.line_info_rec_size < line_info_rec_size) {
 350			linfo = alloc_zero_tailing_info(line_info,
 351							attr.line_info_cnt,
 352							line_info_rec_size,
 353							attr.line_info_rec_size);
 354			if (!linfo) {
 355				errno = E2BIG;
 356				goto done;
 357			}
 358
 359			attr.line_info = ptr_to_u64(linfo);
 360			attr.line_info_rec_size = line_info_rec_size;
 361		} else {
 362			break;
 363		}
 364
 365		fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
 366		OPTS_SET(opts, log_true_size, attr.log_true_size);
 367		if (fd >= 0)
 368			goto done;
 369	}
 370
 371	if (log_level == 0 && log_buf) {
 372		/* log_level == 0 with non-NULL log_buf requires retrying on error
 373		 * with log_level == 1 and log_buf/log_buf_size set, to get details of
 374		 * failure
 375		 */
 376		attr.log_buf = ptr_to_u64(log_buf);
 377		attr.log_size = log_size;
 378		attr.log_level = 1;
 379
 380		fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
 381		OPTS_SET(opts, log_true_size, attr.log_true_size);
 382	}
 383done:
 384	/* free() doesn't affect errno, so we don't need to restore it */
 385	free(finfo);
 386	free(linfo);
 387	return libbpf_err_errno(fd);
 388}
 389
 390int bpf_map_update_elem(int fd, const void *key, const void *value,
 391			__u64 flags)
 392{
 393	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 394	union bpf_attr attr;
 395	int ret;
 396
 397	memset(&attr, 0, attr_sz);
 398	attr.map_fd = fd;
 399	attr.key = ptr_to_u64(key);
 400	attr.value = ptr_to_u64(value);
 401	attr.flags = flags;
 402
 403	ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
 404	return libbpf_err_errno(ret);
 405}
 406
 407int bpf_map_lookup_elem(int fd, const void *key, void *value)
 408{
 409	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 410	union bpf_attr attr;
 411	int ret;
 412
 413	memset(&attr, 0, attr_sz);
 414	attr.map_fd = fd;
 415	attr.key = ptr_to_u64(key);
 416	attr.value = ptr_to_u64(value);
 417
 418	ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz);
 419	return libbpf_err_errno(ret);
 420}
 421
 422int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags)
 423{
 424	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 425	union bpf_attr attr;
 426	int ret;
 427
 428	memset(&attr, 0, attr_sz);
 429	attr.map_fd = fd;
 430	attr.key = ptr_to_u64(key);
 431	attr.value = ptr_to_u64(value);
 432	attr.flags = flags;
 433
 434	ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz);
 435	return libbpf_err_errno(ret);
 436}
 437
 438int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value)
 439{
 440	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 441	union bpf_attr attr;
 442	int ret;
 443
 444	memset(&attr, 0, attr_sz);
 445	attr.map_fd = fd;
 446	attr.key = ptr_to_u64(key);
 447	attr.value = ptr_to_u64(value);
 448
 449	ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz);
 450	return libbpf_err_errno(ret);
 451}
 452
 453int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags)
 454{
 455	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 456	union bpf_attr attr;
 457	int ret;
 458
 459	memset(&attr, 0, attr_sz);
 460	attr.map_fd = fd;
 461	attr.key = ptr_to_u64(key);
 462	attr.value = ptr_to_u64(value);
 463	attr.flags = flags;
 464
 465	ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz);
 466	return libbpf_err_errno(ret);
 467}
 468
 469int bpf_map_delete_elem(int fd, const void *key)
 470{
 471	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 472	union bpf_attr attr;
 473	int ret;
 474
 475	memset(&attr, 0, attr_sz);
 476	attr.map_fd = fd;
 477	attr.key = ptr_to_u64(key);
 478
 479	ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
 480	return libbpf_err_errno(ret);
 481}
 482
 483int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags)
 484{
 485	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 486	union bpf_attr attr;
 487	int ret;
 488
 489	memset(&attr, 0, attr_sz);
 490	attr.map_fd = fd;
 491	attr.key = ptr_to_u64(key);
 492	attr.flags = flags;
 493
 494	ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
 495	return libbpf_err_errno(ret);
 496}
 497
 498int bpf_map_get_next_key(int fd, const void *key, void *next_key)
 499{
 500	const size_t attr_sz = offsetofend(union bpf_attr, next_key);
 501	union bpf_attr attr;
 502	int ret;
 503
 504	memset(&attr, 0, attr_sz);
 505	attr.map_fd = fd;
 506	attr.key = ptr_to_u64(key);
 507	attr.next_key = ptr_to_u64(next_key);
 508
 509	ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, attr_sz);
 510	return libbpf_err_errno(ret);
 511}
 512
 513int bpf_map_freeze(int fd)
 514{
 515	const size_t attr_sz = offsetofend(union bpf_attr, map_fd);
 516	union bpf_attr attr;
 517	int ret;
 518
 519	memset(&attr, 0, attr_sz);
 520	attr.map_fd = fd;
 521
 522	ret = sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz);
 523	return libbpf_err_errno(ret);
 524}
 525
 526static int bpf_map_batch_common(int cmd, int fd, void  *in_batch,
 527				void *out_batch, void *keys, void *values,
 528				__u32 *count,
 529				const struct bpf_map_batch_opts *opts)
 530{
 531	const size_t attr_sz = offsetofend(union bpf_attr, batch);
 532	union bpf_attr attr;
 533	int ret;
 534
 535	if (!OPTS_VALID(opts, bpf_map_batch_opts))
 536		return libbpf_err(-EINVAL);
 537
 538	memset(&attr, 0, attr_sz);
 539	attr.batch.map_fd = fd;
 540	attr.batch.in_batch = ptr_to_u64(in_batch);
 541	attr.batch.out_batch = ptr_to_u64(out_batch);
 542	attr.batch.keys = ptr_to_u64(keys);
 543	attr.batch.values = ptr_to_u64(values);
 544	attr.batch.count = *count;
 545	attr.batch.elem_flags  = OPTS_GET(opts, elem_flags, 0);
 546	attr.batch.flags = OPTS_GET(opts, flags, 0);
 547
 548	ret = sys_bpf(cmd, &attr, attr_sz);
 549	*count = attr.batch.count;
 550
 551	return libbpf_err_errno(ret);
 552}
 553
 554int bpf_map_delete_batch(int fd, const void *keys, __u32 *count,
 555			 const struct bpf_map_batch_opts *opts)
 556{
 557	return bpf_map_batch_common(BPF_MAP_DELETE_BATCH, fd, NULL,
 558				    NULL, (void *)keys, NULL, count, opts);
 559}
 560
 561int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, void *keys,
 562			 void *values, __u32 *count,
 563			 const struct bpf_map_batch_opts *opts)
 564{
 565	return bpf_map_batch_common(BPF_MAP_LOOKUP_BATCH, fd, in_batch,
 566				    out_batch, keys, values, count, opts);
 567}
 568
 569int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, void *out_batch,
 570				    void *keys, void *values, __u32 *count,
 571				    const struct bpf_map_batch_opts *opts)
 572{
 573	return bpf_map_batch_common(BPF_MAP_LOOKUP_AND_DELETE_BATCH,
 574				    fd, in_batch, out_batch, keys, values,
 575				    count, opts);
 576}
 577
 578int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *count,
 579			 const struct bpf_map_batch_opts *opts)
 580{
 581	return bpf_map_batch_common(BPF_MAP_UPDATE_BATCH, fd, NULL, NULL,
 582				    (void *)keys, (void *)values, count, opts);
 583}
 584
 585int bpf_obj_pin_opts(int fd, const char *pathname, const struct bpf_obj_pin_opts *opts)
 586{
 587	const size_t attr_sz = offsetofend(union bpf_attr, path_fd);
 588	union bpf_attr attr;
 589	int ret;
 590
 591	if (!OPTS_VALID(opts, bpf_obj_pin_opts))
 592		return libbpf_err(-EINVAL);
 593
 594	memset(&attr, 0, attr_sz);
 595	attr.path_fd = OPTS_GET(opts, path_fd, 0);
 596	attr.pathname = ptr_to_u64((void *)pathname);
 597	attr.file_flags = OPTS_GET(opts, file_flags, 0);
 598	attr.bpf_fd = fd;
 599
 600	ret = sys_bpf(BPF_OBJ_PIN, &attr, attr_sz);
 601	return libbpf_err_errno(ret);
 602}
 603
 604int bpf_obj_pin(int fd, const char *pathname)
 605{
 606	return bpf_obj_pin_opts(fd, pathname, NULL);
 607}
 608
 609int bpf_obj_get(const char *pathname)
 610{
 611	return bpf_obj_get_opts(pathname, NULL);
 612}
 613
 614int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts)
 615{
 616	const size_t attr_sz = offsetofend(union bpf_attr, path_fd);
 617	union bpf_attr attr;
 618	int fd;
 619
 620	if (!OPTS_VALID(opts, bpf_obj_get_opts))
 621		return libbpf_err(-EINVAL);
 622
 623	memset(&attr, 0, attr_sz);
 624	attr.path_fd = OPTS_GET(opts, path_fd, 0);
 625	attr.pathname = ptr_to_u64((void *)pathname);
 626	attr.file_flags = OPTS_GET(opts, file_flags, 0);
 627
 628	fd = sys_bpf_fd(BPF_OBJ_GET, &attr, attr_sz);
 629	return libbpf_err_errno(fd);
 630}
 631
 632int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
 633		    unsigned int flags)
 634{
 635	DECLARE_LIBBPF_OPTS(bpf_prog_attach_opts, opts,
 636		.flags = flags,
 637	);
 638
 639	return bpf_prog_attach_opts(prog_fd, target_fd, type, &opts);
 640}
 641
 642int bpf_prog_attach_opts(int prog_fd, int target, enum bpf_attach_type type,
 643			 const struct bpf_prog_attach_opts *opts)
 644{
 645	const size_t attr_sz = offsetofend(union bpf_attr, expected_revision);
 646	__u32 relative_id, flags;
 647	int ret, relative_fd;
 648	union bpf_attr attr;
 649
 650	if (!OPTS_VALID(opts, bpf_prog_attach_opts))
 651		return libbpf_err(-EINVAL);
 652
 653	relative_id = OPTS_GET(opts, relative_id, 0);
 654	relative_fd = OPTS_GET(opts, relative_fd, 0);
 655	flags = OPTS_GET(opts, flags, 0);
 656
 657	/* validate we don't have unexpected combinations of non-zero fields */
 658	if (relative_fd && relative_id)
 659		return libbpf_err(-EINVAL);
 660
 661	memset(&attr, 0, attr_sz);
 662	attr.target_fd		= target;
 663	attr.attach_bpf_fd	= prog_fd;
 664	attr.attach_type	= type;
 665	attr.replace_bpf_fd	= OPTS_GET(opts, replace_fd, 0);
 666	attr.expected_revision	= OPTS_GET(opts, expected_revision, 0);
 667
 668	if (relative_id) {
 669		attr.attach_flags = flags | BPF_F_ID;
 670		attr.relative_id  = relative_id;
 671	} else {
 672		attr.attach_flags = flags;
 673		attr.relative_fd  = relative_fd;
 674	}
 675
 676	ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz);
 677	return libbpf_err_errno(ret);
 678}
 679
 680int bpf_prog_detach_opts(int prog_fd, int target, enum bpf_attach_type type,
 681			 const struct bpf_prog_detach_opts *opts)
 682{
 683	const size_t attr_sz = offsetofend(union bpf_attr, expected_revision);
 684	__u32 relative_id, flags;
 685	int ret, relative_fd;
 686	union bpf_attr attr;
 687
 688	if (!OPTS_VALID(opts, bpf_prog_detach_opts))
 689		return libbpf_err(-EINVAL);
 690
 691	relative_id = OPTS_GET(opts, relative_id, 0);
 692	relative_fd = OPTS_GET(opts, relative_fd, 0);
 693	flags = OPTS_GET(opts, flags, 0);
 694
 695	/* validate we don't have unexpected combinations of non-zero fields */
 696	if (relative_fd && relative_id)
 697		return libbpf_err(-EINVAL);
 698
 699	memset(&attr, 0, attr_sz);
 700	attr.target_fd		= target;
 701	attr.attach_bpf_fd	= prog_fd;
 702	attr.attach_type	= type;
 703	attr.expected_revision	= OPTS_GET(opts, expected_revision, 0);
 704
 705	if (relative_id) {
 706		attr.attach_flags = flags | BPF_F_ID;
 707		attr.relative_id  = relative_id;
 708	} else {
 709		attr.attach_flags = flags;
 710		attr.relative_fd  = relative_fd;
 711	}
 712
 713	ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz);
 714	return libbpf_err_errno(ret);
 715}
 716
 717int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
 718{
 719	return bpf_prog_detach_opts(0, target_fd, type, NULL);
 720}
 721
 722int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
 723{
 724	return bpf_prog_detach_opts(prog_fd, target_fd, type, NULL);
 725}
 726
 727int bpf_link_create(int prog_fd, int target_fd,
 728		    enum bpf_attach_type attach_type,
 729		    const struct bpf_link_create_opts *opts)
 730{
 731	const size_t attr_sz = offsetofend(union bpf_attr, link_create);
 732	__u32 target_btf_id, iter_info_len, relative_id;
 733	int fd, err, relative_fd;
 734	union bpf_attr attr;
 735
 736	if (!OPTS_VALID(opts, bpf_link_create_opts))
 737		return libbpf_err(-EINVAL);
 738
 739	iter_info_len = OPTS_GET(opts, iter_info_len, 0);
 740	target_btf_id = OPTS_GET(opts, target_btf_id, 0);
 741
 742	/* validate we don't have unexpected combinations of non-zero fields */
 743	if (iter_info_len || target_btf_id) {
 744		if (iter_info_len && target_btf_id)
 745			return libbpf_err(-EINVAL);
 746		if (!OPTS_ZEROED(opts, target_btf_id))
 747			return libbpf_err(-EINVAL);
 748	}
 749
 750	memset(&attr, 0, attr_sz);
 751	attr.link_create.prog_fd = prog_fd;
 752	attr.link_create.target_fd = target_fd;
 753	attr.link_create.attach_type = attach_type;
 754	attr.link_create.flags = OPTS_GET(opts, flags, 0);
 755
 756	if (target_btf_id) {
 757		attr.link_create.target_btf_id = target_btf_id;
 758		goto proceed;
 759	}
 760
 761	switch (attach_type) {
 762	case BPF_TRACE_ITER:
 763		attr.link_create.iter_info = ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0));
 764		attr.link_create.iter_info_len = iter_info_len;
 765		break;
 766	case BPF_PERF_EVENT:
 767		attr.link_create.perf_event.bpf_cookie = OPTS_GET(opts, perf_event.bpf_cookie, 0);
 768		if (!OPTS_ZEROED(opts, perf_event))
 769			return libbpf_err(-EINVAL);
 770		break;
 771	case BPF_TRACE_KPROBE_MULTI:
 772	case BPF_TRACE_KPROBE_SESSION:
 773		attr.link_create.kprobe_multi.flags = OPTS_GET(opts, kprobe_multi.flags, 0);
 774		attr.link_create.kprobe_multi.cnt = OPTS_GET(opts, kprobe_multi.cnt, 0);
 775		attr.link_create.kprobe_multi.syms = ptr_to_u64(OPTS_GET(opts, kprobe_multi.syms, 0));
 776		attr.link_create.kprobe_multi.addrs = ptr_to_u64(OPTS_GET(opts, kprobe_multi.addrs, 0));
 777		attr.link_create.kprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, kprobe_multi.cookies, 0));
 778		if (!OPTS_ZEROED(opts, kprobe_multi))
 779			return libbpf_err(-EINVAL);
 780		break;
 781	case BPF_TRACE_UPROBE_MULTI:
 782	case BPF_TRACE_UPROBE_SESSION:
 783		attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
 784		attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
 785		attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));
 786		attr.link_create.uprobe_multi.offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.offsets, 0));
 787		attr.link_create.uprobe_multi.ref_ctr_offsets = ptr_to_u64(OPTS_GET(opts, uprobe_multi.ref_ctr_offsets, 0));
 788		attr.link_create.uprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, uprobe_multi.cookies, 0));
 789		attr.link_create.uprobe_multi.pid = OPTS_GET(opts, uprobe_multi.pid, 0);
 790		if (!OPTS_ZEROED(opts, uprobe_multi))
 791			return libbpf_err(-EINVAL);
 792		break;
 793	case BPF_TRACE_RAW_TP:
 794	case BPF_TRACE_FENTRY:
 795	case BPF_TRACE_FEXIT:
 796	case BPF_MODIFY_RETURN:
 797	case BPF_LSM_MAC:
 798		attr.link_create.tracing.cookie = OPTS_GET(opts, tracing.cookie, 0);
 799		if (!OPTS_ZEROED(opts, tracing))
 800			return libbpf_err(-EINVAL);
 801		break;
 802	case BPF_NETFILTER:
 803		attr.link_create.netfilter.pf = OPTS_GET(opts, netfilter.pf, 0);
 804		attr.link_create.netfilter.hooknum = OPTS_GET(opts, netfilter.hooknum, 0);
 805		attr.link_create.netfilter.priority = OPTS_GET(opts, netfilter.priority, 0);
 806		attr.link_create.netfilter.flags = OPTS_GET(opts, netfilter.flags, 0);
 807		if (!OPTS_ZEROED(opts, netfilter))
 808			return libbpf_err(-EINVAL);
 809		break;
 810	case BPF_TCX_INGRESS:
 811	case BPF_TCX_EGRESS:
 812		relative_fd = OPTS_GET(opts, tcx.relative_fd, 0);
 813		relative_id = OPTS_GET(opts, tcx.relative_id, 0);
 814		if (relative_fd && relative_id)
 815			return libbpf_err(-EINVAL);
 816		if (relative_id) {
 817			attr.link_create.tcx.relative_id = relative_id;
 818			attr.link_create.flags |= BPF_F_ID;
 819		} else {
 820			attr.link_create.tcx.relative_fd = relative_fd;
 821		}
 822		attr.link_create.tcx.expected_revision = OPTS_GET(opts, tcx.expected_revision, 0);
 823		if (!OPTS_ZEROED(opts, tcx))
 824			return libbpf_err(-EINVAL);
 825		break;
 826	case BPF_NETKIT_PRIMARY:
 827	case BPF_NETKIT_PEER:
 828		relative_fd = OPTS_GET(opts, netkit.relative_fd, 0);
 829		relative_id = OPTS_GET(opts, netkit.relative_id, 0);
 830		if (relative_fd && relative_id)
 831			return libbpf_err(-EINVAL);
 832		if (relative_id) {
 833			attr.link_create.netkit.relative_id = relative_id;
 834			attr.link_create.flags |= BPF_F_ID;
 835		} else {
 836			attr.link_create.netkit.relative_fd = relative_fd;
 837		}
 838		attr.link_create.netkit.expected_revision = OPTS_GET(opts, netkit.expected_revision, 0);
 839		if (!OPTS_ZEROED(opts, netkit))
 840			return libbpf_err(-EINVAL);
 841		break;
 842	case BPF_CGROUP_INET_INGRESS:
 843	case BPF_CGROUP_INET_EGRESS:
 844	case BPF_CGROUP_INET_SOCK_CREATE:
 845	case BPF_CGROUP_INET_SOCK_RELEASE:
 846	case BPF_CGROUP_INET4_BIND:
 847	case BPF_CGROUP_INET6_BIND:
 848	case BPF_CGROUP_INET4_POST_BIND:
 849	case BPF_CGROUP_INET6_POST_BIND:
 850	case BPF_CGROUP_INET4_CONNECT:
 851	case BPF_CGROUP_INET6_CONNECT:
 852	case BPF_CGROUP_UNIX_CONNECT:
 853	case BPF_CGROUP_INET4_GETPEERNAME:
 854	case BPF_CGROUP_INET6_GETPEERNAME:
 855	case BPF_CGROUP_UNIX_GETPEERNAME:
 856	case BPF_CGROUP_INET4_GETSOCKNAME:
 857	case BPF_CGROUP_INET6_GETSOCKNAME:
 858	case BPF_CGROUP_UNIX_GETSOCKNAME:
 859	case BPF_CGROUP_UDP4_SENDMSG:
 860	case BPF_CGROUP_UDP6_SENDMSG:
 861	case BPF_CGROUP_UNIX_SENDMSG:
 862	case BPF_CGROUP_UDP4_RECVMSG:
 863	case BPF_CGROUP_UDP6_RECVMSG:
 864	case BPF_CGROUP_UNIX_RECVMSG:
 865	case BPF_CGROUP_SOCK_OPS:
 866	case BPF_CGROUP_DEVICE:
 867	case BPF_CGROUP_SYSCTL:
 868	case BPF_CGROUP_GETSOCKOPT:
 869	case BPF_CGROUP_SETSOCKOPT:
 870	case BPF_LSM_CGROUP:
 871		relative_fd = OPTS_GET(opts, cgroup.relative_fd, 0);
 872		relative_id = OPTS_GET(opts, cgroup.relative_id, 0);
 873		if (relative_fd && relative_id)
 874			return libbpf_err(-EINVAL);
 875		if (relative_id) {
 876			attr.link_create.cgroup.relative_id = relative_id;
 877			attr.link_create.flags |= BPF_F_ID;
 878		} else {
 879			attr.link_create.cgroup.relative_fd = relative_fd;
 880		}
 881		attr.link_create.cgroup.expected_revision =
 882			OPTS_GET(opts, cgroup.expected_revision, 0);
 883		if (!OPTS_ZEROED(opts, cgroup))
 884			return libbpf_err(-EINVAL);
 885		break;
 886	default:
 887		if (!OPTS_ZEROED(opts, flags))
 888			return libbpf_err(-EINVAL);
 889		break;
 890	}
 891proceed:
 892	fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, attr_sz);
 893	if (fd >= 0)
 894		return fd;
 895	/* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry
 896	 * and other similar programs
 897	 */
 898	err = -errno;
 899	if (err != -EINVAL)
 900		return libbpf_err(err);
 901
 902	/* if user used features not supported by
 903	 * BPF_RAW_TRACEPOINT_OPEN command, then just give up immediately
 904	 */
 905	if (attr.link_create.target_fd || attr.link_create.target_btf_id)
 906		return libbpf_err(err);
 907	if (!OPTS_ZEROED(opts, sz))
 908		return libbpf_err(err);
 909
 910	/* otherwise, for few select kinds of programs that can be
 911	 * attached using BPF_RAW_TRACEPOINT_OPEN command, try that as
 912	 * a fallback for older kernels
 913	 */
 914	switch (attach_type) {
 915	case BPF_TRACE_RAW_TP:
 916	case BPF_LSM_MAC:
 917	case BPF_TRACE_FENTRY:
 918	case BPF_TRACE_FEXIT:
 919	case BPF_MODIFY_RETURN:
 920		return bpf_raw_tracepoint_open(NULL, prog_fd);
 921	default:
 922		return libbpf_err(err);
 923	}
 924}
 925
 926int bpf_link_detach(int link_fd)
 927{
 928	const size_t attr_sz = offsetofend(union bpf_attr, link_detach);
 929	union bpf_attr attr;
 930	int ret;
 931
 932	memset(&attr, 0, attr_sz);
 933	attr.link_detach.link_fd = link_fd;
 934
 935	ret = sys_bpf(BPF_LINK_DETACH, &attr, attr_sz);
 936	return libbpf_err_errno(ret);
 937}
 938
 939int bpf_link_update(int link_fd, int new_prog_fd,
 940		    const struct bpf_link_update_opts *opts)
 941{
 942	const size_t attr_sz = offsetofend(union bpf_attr, link_update);
 943	union bpf_attr attr;
 944	int ret;
 945
 946	if (!OPTS_VALID(opts, bpf_link_update_opts))
 947		return libbpf_err(-EINVAL);
 948
 949	if (OPTS_GET(opts, old_prog_fd, 0) && OPTS_GET(opts, old_map_fd, 0))
 950		return libbpf_err(-EINVAL);
 951
 952	memset(&attr, 0, attr_sz);
 953	attr.link_update.link_fd = link_fd;
 954	attr.link_update.new_prog_fd = new_prog_fd;
 955	attr.link_update.flags = OPTS_GET(opts, flags, 0);
 956	if (OPTS_GET(opts, old_prog_fd, 0))
 957		attr.link_update.old_prog_fd = OPTS_GET(opts, old_prog_fd, 0);
 958	else if (OPTS_GET(opts, old_map_fd, 0))
 959		attr.link_update.old_map_fd = OPTS_GET(opts, old_map_fd, 0);
 960
 961	ret = sys_bpf(BPF_LINK_UPDATE, &attr, attr_sz);
 962	return libbpf_err_errno(ret);
 963}
 964
 965int bpf_iter_create(int link_fd)
 966{
 967	const size_t attr_sz = offsetofend(union bpf_attr, iter_create);
 968	union bpf_attr attr;
 969	int fd;
 970
 971	memset(&attr, 0, attr_sz);
 972	attr.iter_create.link_fd = link_fd;
 973
 974	fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, attr_sz);
 975	return libbpf_err_errno(fd);
 976}
 977
 978int bpf_prog_query_opts(int target, enum bpf_attach_type type,
 979			struct bpf_prog_query_opts *opts)
 980{
 981	const size_t attr_sz = offsetofend(union bpf_attr, query);
 982	union bpf_attr attr;
 983	int ret;
 984
 985	if (!OPTS_VALID(opts, bpf_prog_query_opts))
 986		return libbpf_err(-EINVAL);
 987
 988	memset(&attr, 0, attr_sz);
 989	attr.query.target_fd		= target;
 990	attr.query.attach_type		= type;
 991	attr.query.query_flags		= OPTS_GET(opts, query_flags, 0);
 992	attr.query.count		= OPTS_GET(opts, count, 0);
 993	attr.query.prog_ids		= ptr_to_u64(OPTS_GET(opts, prog_ids, NULL));
 994	attr.query.link_ids		= ptr_to_u64(OPTS_GET(opts, link_ids, NULL));
 995	attr.query.prog_attach_flags	= ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL));
 996	attr.query.link_attach_flags	= ptr_to_u64(OPTS_GET(opts, link_attach_flags, NULL));
 997
 998	ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz);
 999
1000	OPTS_SET(opts, attach_flags, attr.query.attach_flags);
1001	OPTS_SET(opts, revision, attr.query.revision);
1002	OPTS_SET(opts, count, attr.query.count);
1003
1004	return libbpf_err_errno(ret);
1005}
1006
1007int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
1008		   __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
1009{
1010	LIBBPF_OPTS(bpf_prog_query_opts, opts);
1011	int ret;
1012
1013	opts.query_flags = query_flags;
1014	opts.prog_ids = prog_ids;
1015	opts.prog_cnt = *prog_cnt;
1016
1017	ret = bpf_prog_query_opts(target_fd, type, &opts);
1018
1019	if (attach_flags)
1020		*attach_flags = opts.attach_flags;
1021	*prog_cnt = opts.prog_cnt;
1022
1023	return libbpf_err_errno(ret);
1024}
1025
1026int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
1027{
1028	const size_t attr_sz = offsetofend(union bpf_attr, test);
1029	union bpf_attr attr;
1030	int ret;
1031
1032	if (!OPTS_VALID(opts, bpf_test_run_opts))
1033		return libbpf_err(-EINVAL);
1034
1035	memset(&attr, 0, attr_sz);
1036	attr.test.prog_fd = prog_fd;
1037	attr.test.batch_size = OPTS_GET(opts, batch_size, 0);
1038	attr.test.cpu = OPTS_GET(opts, cpu, 0);
1039	attr.test.flags = OPTS_GET(opts, flags, 0);
1040	attr.test.repeat = OPTS_GET(opts, repeat, 0);
1041	attr.test.duration = OPTS_GET(opts, duration, 0);
1042	attr.test.ctx_size_in = OPTS_GET(opts, ctx_size_in, 0);
1043	attr.test.ctx_size_out = OPTS_GET(opts, ctx_size_out, 0);
1044	attr.test.data_size_in = OPTS_GET(opts, data_size_in, 0);
1045	attr.test.data_size_out = OPTS_GET(opts, data_size_out, 0);
1046	attr.test.ctx_in = ptr_to_u64(OPTS_GET(opts, ctx_in, NULL));
1047	attr.test.ctx_out = ptr_to_u64(OPTS_GET(opts, ctx_out, NULL));
1048	attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL));
1049	attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL));
1050
1051	ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, attr_sz);
1052
1053	OPTS_SET(opts, data_size_out, attr.test.data_size_out);
1054	OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out);
1055	OPTS_SET(opts, duration, attr.test.duration);
1056	OPTS_SET(opts, retval, attr.test.retval);
1057
1058	return libbpf_err_errno(ret);
1059}
1060
1061static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd)
1062{
1063	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1064	union bpf_attr attr;
1065	int err;
1066
1067	memset(&attr, 0, attr_sz);
1068	attr.start_id = start_id;
1069
1070	err = sys_bpf(cmd, &attr, attr_sz);
1071	if (!err)
1072		*next_id = attr.next_id;
1073
1074	return libbpf_err_errno(err);
1075}
1076
1077int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id)
1078{
1079	return bpf_obj_get_next_id(start_id, next_id, BPF_PROG_GET_NEXT_ID);
1080}
1081
1082int bpf_map_get_next_id(__u32 start_id, __u32 *next_id)
1083{
1084	return bpf_obj_get_next_id(start_id, next_id, BPF_MAP_GET_NEXT_ID);
1085}
1086
1087int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id)
1088{
1089	return bpf_obj_get_next_id(start_id, next_id, BPF_BTF_GET_NEXT_ID);
1090}
1091
1092int bpf_link_get_next_id(__u32 start_id, __u32 *next_id)
1093{
1094	return bpf_obj_get_next_id(start_id, next_id, BPF_LINK_GET_NEXT_ID);
1095}
1096
1097int bpf_prog_get_fd_by_id_opts(__u32 id,
1098			       const struct bpf_get_fd_by_id_opts *opts)
1099{
1100	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1101	union bpf_attr attr;
1102	int fd;
1103
1104	if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1105		return libbpf_err(-EINVAL);
1106
1107	memset(&attr, 0, attr_sz);
1108	attr.prog_id = id;
1109	attr.open_flags = OPTS_GET(opts, open_flags, 0);
1110
1111	fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, attr_sz);
1112	return libbpf_err_errno(fd);
1113}
1114
1115int bpf_prog_get_fd_by_id(__u32 id)
1116{
1117	return bpf_prog_get_fd_by_id_opts(id, NULL);
1118}
1119
1120int bpf_map_get_fd_by_id_opts(__u32 id,
1121			      const struct bpf_get_fd_by_id_opts *opts)
1122{
1123	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1124	union bpf_attr attr;
1125	int fd;
1126
1127	if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1128		return libbpf_err(-EINVAL);
1129
1130	memset(&attr, 0, attr_sz);
1131	attr.map_id = id;
1132	attr.open_flags = OPTS_GET(opts, open_flags, 0);
1133
1134	fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
1135	return libbpf_err_errno(fd);
1136}
1137
1138int bpf_map_get_fd_by_id(__u32 id)
1139{
1140	return bpf_map_get_fd_by_id_opts(id, NULL);
1141}
1142
1143int bpf_btf_get_fd_by_id_opts(__u32 id,
1144			      const struct bpf_get_fd_by_id_opts *opts)
1145{
1146	const size_t attr_sz = offsetofend(union bpf_attr, fd_by_id_token_fd);
1147	union bpf_attr attr;
1148	int fd;
1149
1150	if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1151		return libbpf_err(-EINVAL);
1152
1153	memset(&attr, 0, attr_sz);
1154	attr.btf_id = id;
1155	attr.open_flags = OPTS_GET(opts, open_flags, 0);
1156	attr.fd_by_id_token_fd = OPTS_GET(opts, token_fd, 0);
1157
1158	fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz);
1159	return libbpf_err_errno(fd);
1160}
1161
1162int bpf_btf_get_fd_by_id(__u32 id)
1163{
1164	return bpf_btf_get_fd_by_id_opts(id, NULL);
1165}
1166
1167int bpf_link_get_fd_by_id_opts(__u32 id,
1168			       const struct bpf_get_fd_by_id_opts *opts)
1169{
1170	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
1171	union bpf_attr attr;
1172	int fd;
1173
1174	if (!OPTS_VALID(opts, bpf_get_fd_by_id_opts))
1175		return libbpf_err(-EINVAL);
1176
1177	memset(&attr, 0, attr_sz);
1178	attr.link_id = id;
1179	attr.open_flags = OPTS_GET(opts, open_flags, 0);
1180
1181	fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, attr_sz);
1182	return libbpf_err_errno(fd);
1183}
1184
1185int bpf_link_get_fd_by_id(__u32 id)
1186{
1187	return bpf_link_get_fd_by_id_opts(id, NULL);
1188}
1189
1190int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len)
1191{
1192	const size_t attr_sz = offsetofend(union bpf_attr, info);
1193	union bpf_attr attr;
1194	int err;
1195
1196	memset(&attr, 0, attr_sz);
1197	attr.info.bpf_fd = bpf_fd;
1198	attr.info.info_len = *info_len;
1199	attr.info.info = ptr_to_u64(info);
1200
1201	err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz);
1202	if (!err)
1203		*info_len = attr.info.info_len;
1204	return libbpf_err_errno(err);
1205}
1206
1207int bpf_prog_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, __u32 *info_len)
1208{
1209	return bpf_obj_get_info_by_fd(prog_fd, info, info_len);
1210}
1211
1212int bpf_map_get_info_by_fd(int map_fd, struct bpf_map_info *info, __u32 *info_len)
1213{
1214	return bpf_obj_get_info_by_fd(map_fd, info, info_len);
1215}
1216
1217int bpf_btf_get_info_by_fd(int btf_fd, struct bpf_btf_info *info, __u32 *info_len)
1218{
1219	return bpf_obj_get_info_by_fd(btf_fd, info, info_len);
1220}
1221
1222int bpf_link_get_info_by_fd(int link_fd, struct bpf_link_info *info, __u32 *info_len)
1223{
1224	return bpf_obj_get_info_by_fd(link_fd, info, info_len);
1225}
1226
1227int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts)
1228{
1229	const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint);
1230	union bpf_attr attr;
1231	int fd;
1232
1233	if (!OPTS_VALID(opts, bpf_raw_tp_opts))
1234		return libbpf_err(-EINVAL);
1235
1236	memset(&attr, 0, attr_sz);
1237	attr.raw_tracepoint.prog_fd = prog_fd;
1238	attr.raw_tracepoint.name = ptr_to_u64(OPTS_GET(opts, tp_name, NULL));
1239	attr.raw_tracepoint.cookie = OPTS_GET(opts, cookie, 0);
1240
1241	fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
1242	return libbpf_err_errno(fd);
1243}
1244
1245int bpf_raw_tracepoint_open(const char *name, int prog_fd)
1246{
1247	LIBBPF_OPTS(bpf_raw_tp_opts, opts, .tp_name = name);
1248
1249	return bpf_raw_tracepoint_open_opts(prog_fd, &opts);
1250}
1251
1252int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts)
1253{
1254	const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd);
1255	union bpf_attr attr;
1256	char *log_buf;
1257	size_t log_size;
1258	__u32 log_level;
1259	int fd;
1260
1261	bump_rlimit_memlock();
1262
1263	memset(&attr, 0, attr_sz);
1264
1265	if (!OPTS_VALID(opts, bpf_btf_load_opts))
1266		return libbpf_err(-EINVAL);
1267
1268	log_buf = OPTS_GET(opts, log_buf, NULL);
1269	log_size = OPTS_GET(opts, log_size, 0);
1270	log_level = OPTS_GET(opts, log_level, 0);
1271
1272	if (log_size > UINT_MAX)
1273		return libbpf_err(-EINVAL);
1274	if (log_size && !log_buf)
1275		return libbpf_err(-EINVAL);
1276
1277	attr.btf = ptr_to_u64(btf_data);
1278	attr.btf_size = btf_size;
1279
1280	attr.btf_flags = OPTS_GET(opts, btf_flags, 0);
1281	attr.btf_token_fd = OPTS_GET(opts, token_fd, 0);
1282
1283	/* log_level == 0 and log_buf != NULL means "try loading without
1284	 * log_buf, but retry with log_buf and log_level=1 on error", which is
1285	 * consistent across low-level and high-level BTF and program loading
1286	 * APIs within libbpf and provides a sensible behavior in practice
1287	 */
1288	if (log_level) {
1289		attr.btf_log_buf = ptr_to_u64(log_buf);
1290		attr.btf_log_size = (__u32)log_size;
1291		attr.btf_log_level = log_level;
1292	}
1293
1294	fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz);
1295	if (fd < 0 && log_buf && log_level == 0) {
1296		attr.btf_log_buf = ptr_to_u64(log_buf);
1297		attr.btf_log_size = (__u32)log_size;
1298		attr.btf_log_level = 1;
1299		fd = sys_bpf_fd(BPF_BTF_LOAD, &attr, attr_sz);
1300	}
1301
1302	OPTS_SET(opts, log_true_size, attr.btf_log_true_size);
1303	return libbpf_err_errno(fd);
1304}
1305
1306int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len,
1307		      __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset,
1308		      __u64 *probe_addr)
1309{
1310	const size_t attr_sz = offsetofend(union bpf_attr, task_fd_query);
1311	union bpf_attr attr;
1312	int err;
1313
1314	memset(&attr, 0, attr_sz);
1315	attr.task_fd_query.pid = pid;
1316	attr.task_fd_query.fd = fd;
1317	attr.task_fd_query.flags = flags;
1318	attr.task_fd_query.buf = ptr_to_u64(buf);
1319	attr.task_fd_query.buf_len = *buf_len;
1320
1321	err = sys_bpf(BPF_TASK_FD_QUERY, &attr, attr_sz);
1322
1323	*buf_len = attr.task_fd_query.buf_len;
1324	*prog_id = attr.task_fd_query.prog_id;
1325	*fd_type = attr.task_fd_query.fd_type;
1326	*probe_offset = attr.task_fd_query.probe_offset;
1327	*probe_addr = attr.task_fd_query.probe_addr;
1328
1329	return libbpf_err_errno(err);
1330}
1331
1332int bpf_enable_stats(enum bpf_stats_type type)
1333{
1334	const size_t attr_sz = offsetofend(union bpf_attr, enable_stats);
1335	union bpf_attr attr;
1336	int fd;
1337
1338	memset(&attr, 0, attr_sz);
1339	attr.enable_stats.type = type;
1340
1341	fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, attr_sz);
1342	return libbpf_err_errno(fd);
1343}
1344
1345int bpf_prog_bind_map(int prog_fd, int map_fd,
1346		      const struct bpf_prog_bind_opts *opts)
1347{
1348	const size_t attr_sz = offsetofend(union bpf_attr, prog_bind_map);
1349	union bpf_attr attr;
1350	int ret;
1351
1352	if (!OPTS_VALID(opts, bpf_prog_bind_opts))
1353		return libbpf_err(-EINVAL);
1354
1355	memset(&attr, 0, attr_sz);
1356	attr.prog_bind_map.prog_fd = prog_fd;
1357	attr.prog_bind_map.map_fd = map_fd;
1358	attr.prog_bind_map.flags = OPTS_GET(opts, flags, 0);
1359
1360	ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz);
1361	return libbpf_err_errno(ret);
1362}
1363
1364int bpf_token_create(int bpffs_fd, struct bpf_token_create_opts *opts)
1365{
1366	const size_t attr_sz = offsetofend(union bpf_attr, token_create);
1367	union bpf_attr attr;
1368	int fd;
1369
1370	if (!OPTS_VALID(opts, bpf_token_create_opts))
1371		return libbpf_err(-EINVAL);
1372
1373	memset(&attr, 0, attr_sz);
1374	attr.token_create.bpffs_fd = bpffs_fd;
1375	attr.token_create.flags = OPTS_GET(opts, flags, 0);
1376
1377	fd = sys_bpf_fd(BPF_TOKEN_CREATE, &attr, attr_sz);
1378	return libbpf_err_errno(fd);
1379}
1380
1381int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len,
1382			 struct bpf_prog_stream_read_opts *opts)
1383{
1384	const size_t attr_sz = offsetofend(union bpf_attr, prog_stream_read);
1385	union bpf_attr attr;
1386	int err;
1387
1388	if (!OPTS_VALID(opts, bpf_prog_stream_read_opts))
1389		return libbpf_err(-EINVAL);
1390
1391	memset(&attr, 0, attr_sz);
1392	attr.prog_stream_read.stream_buf = ptr_to_u64(buf);
1393	attr.prog_stream_read.stream_buf_len = buf_len;
1394	attr.prog_stream_read.stream_id = stream_id;
1395	attr.prog_stream_read.prog_fd = prog_fd;
1396
1397	err = sys_bpf(BPF_PROG_STREAM_READ_BY_FD, &attr, attr_sz);
1398	return libbpf_err_errno(err);
1399}