bpf_iter.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2020 Facebook */
   3#include <test_progs.h>
   4#include <unistd.h>
   5#include <sys/syscall.h>
   6#include <task_local_storage_helpers.h>
   7#include "bpf_iter_ipv6_route.skel.h"
   8#include "bpf_iter_netlink.skel.h"
   9#include "bpf_iter_bpf_map.skel.h"
  10#include "bpf_iter_tasks.skel.h"
  11#include "bpf_iter_task_stack.skel.h"
  12#include "bpf_iter_task_file.skel.h"
  13#include "bpf_iter_task_vmas.skel.h"
  14#include "bpf_iter_task_btf.skel.h"
  15#include "bpf_iter_tcp4.skel.h"
  16#include "bpf_iter_tcp6.skel.h"
  17#include "bpf_iter_udp4.skel.h"
  18#include "bpf_iter_udp6.skel.h"
  19#include "bpf_iter_unix.skel.h"
  20#include "bpf_iter_vma_offset.skel.h"
  21#include "bpf_iter_test_kern1.skel.h"
  22#include "bpf_iter_test_kern2.skel.h"
  23#include "bpf_iter_test_kern3.skel.h"
  24#include "bpf_iter_test_kern4.skel.h"
  25#include "bpf_iter_bpf_hash_map.skel.h"
  26#include "bpf_iter_bpf_percpu_hash_map.skel.h"
  27#include "bpf_iter_bpf_array_map.skel.h"
  28#include "bpf_iter_bpf_percpu_array_map.skel.h"
  29#include "bpf_iter_bpf_sk_storage_helpers.skel.h"
  30#include "bpf_iter_bpf_sk_storage_map.skel.h"
  31#include "bpf_iter_test_kern5.skel.h"
  32#include "bpf_iter_test_kern6.skel.h"
  33#include "bpf_iter_bpf_link.skel.h"
  34#include "bpf_iter_ksym.skel.h"
  35#include "bpf_iter_sockmap.skel.h"
  36
  37static void test_btf_id_or_null(void)
  38{
  39	struct bpf_iter_test_kern3 *skel;
  40
  41	skel = bpf_iter_test_kern3__open_and_load();
  42	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {
  43		bpf_iter_test_kern3__destroy(skel);
  44		return;
  45	}
  46}
  47
  48static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
  49{
  50	struct bpf_link *link;
  51	char buf[16] = {};
  52	int iter_fd, len;
  53
  54	link = bpf_program__attach_iter(prog, opts);
  55	if (!ASSERT_OK_PTR(link, "attach_iter"))
  56		return;
  57
  58	iter_fd = bpf_iter_create(bpf_link__fd(link));
  59	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
  60		goto free_link;
  61
  62	/* not check contents, but ensure read() ends without error */
  63	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
  64		;
  65	ASSERT_GE(len, 0, "read");
  66
  67	close(iter_fd);
  68
  69free_link:
  70	bpf_link__destroy(link);
  71}
  72
  73static void do_dummy_read(struct bpf_program *prog)
  74{
  75	do_dummy_read_opts(prog, NULL);
  76}
  77
  78static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
  79				struct bpf_map *map)
  80{
  81	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
  82	union bpf_iter_link_info linfo;
  83	struct bpf_link *link;
  84	char buf[16] = {};
  85	int iter_fd, len;
  86
  87	memset(&linfo, 0, sizeof(linfo));
  88	linfo.map.map_fd = bpf_map__fd(map);
  89	opts.link_info = &linfo;
  90	opts.link_info_len = sizeof(linfo);
  91	link = bpf_program__attach_iter(prog, &opts);
  92	if (!ASSERT_OK_PTR(link, "attach_map_iter"))
  93		return;
  94
  95	iter_fd = bpf_iter_create(bpf_link__fd(link));
  96	if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {
  97		bpf_link__destroy(link);
  98		return;
  99	}
 100
 101	/* Close link and map fd prematurely */
 102	bpf_link__destroy(link);
 103	bpf_object__destroy_skeleton(*skel);
 104	*skel = NULL;
 105
 106	/* Try to let map free work to run first if map is freed */
 107	usleep(100);
 108	/* Memory used by both sock map and sock local storage map are
 109	 * freed after two synchronize_rcu() calls, so wait for it
 110	 */
 111	kern_sync_rcu();
 112	kern_sync_rcu();
 113
 114	/* Read after both map fd and link fd are closed */
 115	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 116		;
 117	ASSERT_GE(len, 0, "read_iterator");
 118
 119	close(iter_fd);
 120}
 121
 122static int read_fd_into_buffer(int fd, char *buf, int size)
 123{
 124	int bufleft = size;
 125	int len;
 126
 127	do {
 128		len = read(fd, buf, bufleft);
 129		if (len > 0) {
 130			buf += len;
 131			bufleft -= len;
 132		}
 133	} while (len > 0);
 134
 135	return len < 0 ? len : size - bufleft;
 136}
 137
 138static void test_ipv6_route(void)
 139{
 140	struct bpf_iter_ipv6_route *skel;
 141
 142	skel = bpf_iter_ipv6_route__open_and_load();
 143	if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))
 144		return;
 145
 146	do_dummy_read(skel->progs.dump_ipv6_route);
 147
 148	bpf_iter_ipv6_route__destroy(skel);
 149}
 150
 151static void test_netlink(void)
 152{
 153	struct bpf_iter_netlink *skel;
 154
 155	skel = bpf_iter_netlink__open_and_load();
 156	if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))
 157		return;
 158
 159	do_dummy_read(skel->progs.dump_netlink);
 160
 161	bpf_iter_netlink__destroy(skel);
 162}
 163
 164static void test_bpf_map(void)
 165{
 166	struct bpf_iter_bpf_map *skel;
 167
 168	skel = bpf_iter_bpf_map__open_and_load();
 169	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))
 170		return;
 171
 172	do_dummy_read(skel->progs.dump_bpf_map);
 173
 174	bpf_iter_bpf_map__destroy(skel);
 175}
 176
 177static void check_bpf_link_info(const struct bpf_program *prog)
 178{
 179	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 180	union bpf_iter_link_info linfo;
 181	struct bpf_link_info info = {};
 182	struct bpf_link *link;
 183	__u32 info_len;
 184	int err;
 185
 186	memset(&linfo, 0, sizeof(linfo));
 187	linfo.task.tid = getpid();
 188	opts.link_info = &linfo;
 189	opts.link_info_len = sizeof(linfo);
 190
 191	link = bpf_program__attach_iter(prog, &opts);
 192	if (!ASSERT_OK_PTR(link, "attach_iter"))
 193		return;
 194
 195	info_len = sizeof(info);
 196	err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
 197	ASSERT_OK(err, "bpf_link_get_info_by_fd");
 198	ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
 199
 200	bpf_link__destroy(link);
 201}
 202
 203static pthread_mutex_t do_nothing_mutex;
 204
 205static void *do_nothing_wait(void *arg)
 206{
 207	pthread_mutex_lock(&do_nothing_mutex);
 208	pthread_mutex_unlock(&do_nothing_mutex);
 209
 210	pthread_exit(arg);
 211}
 212
 213static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
 214				     int *num_unknown, int *num_known)
 215{
 216	struct bpf_iter_tasks *skel;
 217	pthread_t thread_id;
 218	void *ret;
 219
 220	skel = bpf_iter_tasks__open_and_load();
 221	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
 222		return;
 223
 224	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 225
 226	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
 227		  "pthread_create");
 228
 229	skel->bss->tid = sys_gettid();
 230
 231	do_dummy_read_opts(skel->progs.dump_task, opts);
 232
 233	*num_unknown = skel->bss->num_unknown_tid;
 234	*num_known = skel->bss->num_known_tid;
 235
 236	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
 237	ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
 238		     "pthread_join");
 239
 240	bpf_iter_tasks__destroy(skel);
 241}
 242
 243static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
 244{
 245	int num_unknown_tid, num_known_tid;
 246
 247	test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
 248	ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
 249	ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
 250}
 251
 252static void *run_test_task_tid(void *arg)
 253{
 254	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 255	union bpf_iter_link_info linfo;
 256	int num_unknown_tid, num_known_tid;
 257
 258	ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id");
 259
 260	memset(&linfo, 0, sizeof(linfo));
 261	linfo.task.tid = sys_gettid();
 262	opts.link_info = &linfo;
 263	opts.link_info_len = sizeof(linfo);
 264	test_task_common(&opts, 0, 1);
 265
 266	linfo.task.tid = 0;
 267	linfo.task.pid = getpid();
 268	/* This includes the parent thread, this thread, watchdog timer thread
 269	 * and the do_nothing_wait thread
 270	 */
 271	test_task_common(&opts, 3, 1);
 272
 273	test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
 274	ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
 275	ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
 276
 277	return NULL;
 278}
 279
 280static void test_task_tid(void)
 281{
 282	pthread_t thread_id;
 283
 284	/* Create a new thread so pid and tid aren't the same */
 285	ASSERT_OK(pthread_create(&thread_id, NULL, &run_test_task_tid, NULL),
 286		  "pthread_create");
 287	ASSERT_FALSE(pthread_join(thread_id, NULL), "pthread_join");
 288}
 289
 290static void test_task_pid(void)
 291{
 292	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 293	union bpf_iter_link_info linfo;
 294
 295	memset(&linfo, 0, sizeof(linfo));
 296	linfo.task.pid = getpid();
 297	opts.link_info = &linfo;
 298	opts.link_info_len = sizeof(linfo);
 299
 300	test_task_common(&opts, 2, 1);
 301}
 302
 303static void test_task_pidfd(void)
 304{
 305	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 306	union bpf_iter_link_info linfo;
 307	int pidfd;
 308
 309	pidfd = sys_pidfd_open(getpid(), 0);
 310	if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))
 311		return;
 312
 313	memset(&linfo, 0, sizeof(linfo));
 314	linfo.task.pid_fd = pidfd;
 315	opts.link_info = &linfo;
 316	opts.link_info_len = sizeof(linfo);
 317
 318	test_task_common(&opts, 2, 1);
 319
 320	close(pidfd);
 321}
 322
 323static void test_task_sleepable(void)
 324{
 325	struct bpf_iter_tasks *skel;
 326	int pid, status, err, data_pipe[2], finish_pipe[2], c = 0;
 327	char *test_data = NULL;
 328	char *test_data_long = NULL;
 329	char *data[2];
 330
 331	if (!ASSERT_OK(pipe(data_pipe), "data_pipe") ||
 332	    !ASSERT_OK(pipe(finish_pipe), "finish_pipe"))
 333		return;
 334
 335	skel = bpf_iter_tasks__open_and_load();
 336	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
 337		return;
 338
 339	pid = fork();
 340	if (!ASSERT_GE(pid, 0, "fork"))
 341		return;
 342
 343	if (pid == 0) {
 344		/* child */
 345		close(data_pipe[0]);
 346		close(finish_pipe[1]);
 347
 348		test_data = malloc(sizeof(char) * 10);
 349		strncpy(test_data, "test_data", 10);
 350		test_data[9] = '\0';
 351
 352		test_data_long = malloc(sizeof(char) * 5000);
 353		for (int i = 0; i < 5000; ++i) {
 354			if (i % 2 == 0)
 355				test_data_long[i] = 'b';
 356			else
 357				test_data_long[i] = 'a';
 358		}
 359		test_data_long[4999] = '\0';
 360
 361		data[0] = test_data;
 362		data[1] = test_data_long;
 363
 364		write(data_pipe[1], &data, sizeof(data));
 365
 366		/* keep child alive until after the test */
 367		err = read(finish_pipe[0], &c, 1);
 368		if (err != 1)
 369			exit(-1);
 370
 371		close(data_pipe[1]);
 372		close(finish_pipe[0]);
 373		_exit(0);
 374	}
 375
 376	/* parent */
 377	close(data_pipe[1]);
 378	close(finish_pipe[0]);
 379
 380	err = read(data_pipe[0], &data, sizeof(data));
 381	ASSERT_EQ(err, sizeof(data), "read_check");
 382
 383	skel->bss->user_ptr = data[0];
 384	skel->bss->user_ptr_long = data[1];
 385	skel->bss->pid = pid;
 386
 387	do_dummy_read(skel->progs.dump_task_sleepable);
 388
 389	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
 390		  "num_expected_failure_copy_from_user_task");
 391	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
 392		  "num_success_copy_from_user_task");
 393	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0,
 394		  "num_expected_failure_copy_from_user_task_str");
 395	ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0,
 396		  "num_success_copy_from_user_task_str");
 397
 398	bpf_iter_tasks__destroy(skel);
 399
 400	write(finish_pipe[1], &c, 1);
 401	err = waitpid(pid, &status, 0);
 402	ASSERT_EQ(err, pid, "waitpid");
 403	ASSERT_EQ(status, 0, "zero_child_exit");
 404
 405	close(data_pipe[0]);
 406	close(finish_pipe[1]);
 407}
 408
 409static void test_task_stack(void)
 410{
 411	struct bpf_iter_task_stack *skel;
 412
 413	skel = bpf_iter_task_stack__open_and_load();
 414	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))
 415		return;
 416
 417	do_dummy_read(skel->progs.dump_task_stack);
 418	do_dummy_read(skel->progs.get_task_user_stacks);
 419
 420	ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
 421
 422	bpf_iter_task_stack__destroy(skel);
 423}
 424
 425static void test_task_file(void)
 426{
 427	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 428	struct bpf_iter_task_file *skel;
 429	union bpf_iter_link_info linfo;
 430	pthread_t thread_id;
 431	void *ret;
 432
 433	skel = bpf_iter_task_file__open_and_load();
 434	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))
 435		return;
 436
 437	skel->bss->tgid = getpid();
 438
 439	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 440
 441	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
 442		  "pthread_create");
 443
 444	memset(&linfo, 0, sizeof(linfo));
 445	linfo.task.tid = getpid();
 446	opts.link_info = &linfo;
 447	opts.link_info_len = sizeof(linfo);
 448
 449	do_dummy_read_opts(skel->progs.dump_task_file, &opts);
 450
 451	ASSERT_EQ(skel->bss->count, 0, "check_count");
 452	ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
 453
 454	skel->bss->last_tgid = 0;
 455	skel->bss->count = 0;
 456	skel->bss->unique_tgid_count = 0;
 457
 458	do_dummy_read(skel->progs.dump_task_file);
 459
 460	ASSERT_EQ(skel->bss->count, 0, "check_count");
 461	ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
 462
 463	check_bpf_link_info(skel->progs.dump_task_file);
 464
 465	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
 466	ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
 467	ASSERT_NULL(ret, "pthread_join");
 468
 469	bpf_iter_task_file__destroy(skel);
 470}
 471
 472#define TASKBUFSZ		32768
 473
 474static char taskbuf[TASKBUFSZ];
 475
 476static int do_btf_read(struct bpf_iter_task_btf *skel)
 477{
 478	struct bpf_program *prog = skel->progs.dump_task_struct;
 479	struct bpf_iter_task_btf__bss *bss = skel->bss;
 480	int iter_fd = -1, err;
 481	struct bpf_link *link;
 482	char *buf = taskbuf;
 483	int ret = 0;
 484
 485	link = bpf_program__attach_iter(prog, NULL);
 486	if (!ASSERT_OK_PTR(link, "attach_iter"))
 487		return ret;
 488
 489	iter_fd = bpf_iter_create(bpf_link__fd(link));
 490	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 491		goto free_link;
 492
 493	err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);
 494	if (bss->skip) {
 495		printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
 496		ret = 1;
 497		test__skip();
 498		goto free_link;
 499	}
 500
 501	if (!ASSERT_GE(err, 0, "read"))
 502		goto free_link;
 503
 504	ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
 505	      "check for btf representation of task_struct in iter data");
 506free_link:
 507	if (iter_fd > 0)
 508		close(iter_fd);
 509	bpf_link__destroy(link);
 510	return ret;
 511}
 512
 513static void test_task_btf(void)
 514{
 515	struct bpf_iter_task_btf__bss *bss;
 516	struct bpf_iter_task_btf *skel;
 517	int ret;
 518
 519	skel = bpf_iter_task_btf__open_and_load();
 520	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))
 521		return;
 522
 523	bss = skel->bss;
 524
 525	ret = do_btf_read(skel);
 526	if (ret)
 527		goto cleanup;
 528
 529	if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))
 530		goto cleanup;
 531
 532	ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");
 533
 534cleanup:
 535	bpf_iter_task_btf__destroy(skel);
 536}
 537
 538static void test_tcp4(void)
 539{
 540	struct bpf_iter_tcp4 *skel;
 541
 542	skel = bpf_iter_tcp4__open_and_load();
 543	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))
 544		return;
 545
 546	do_dummy_read(skel->progs.dump_tcp4);
 547
 548	bpf_iter_tcp4__destroy(skel);
 549}
 550
 551static void test_tcp6(void)
 552{
 553	struct bpf_iter_tcp6 *skel;
 554
 555	skel = bpf_iter_tcp6__open_and_load();
 556	if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))
 557		return;
 558
 559	do_dummy_read(skel->progs.dump_tcp6);
 560
 561	bpf_iter_tcp6__destroy(skel);
 562}
 563
 564static void test_udp4(void)
 565{
 566	struct bpf_iter_udp4 *skel;
 567
 568	skel = bpf_iter_udp4__open_and_load();
 569	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))
 570		return;
 571
 572	do_dummy_read(skel->progs.dump_udp4);
 573
 574	bpf_iter_udp4__destroy(skel);
 575}
 576
 577static void test_udp6(void)
 578{
 579	struct bpf_iter_udp6 *skel;
 580
 581	skel = bpf_iter_udp6__open_and_load();
 582	if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))
 583		return;
 584
 585	do_dummy_read(skel->progs.dump_udp6);
 586
 587	bpf_iter_udp6__destroy(skel);
 588}
 589
 590static void test_unix(void)
 591{
 592	struct bpf_iter_unix *skel;
 593
 594	skel = bpf_iter_unix__open_and_load();
 595	if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))
 596		return;
 597
 598	do_dummy_read(skel->progs.dump_unix);
 599
 600	bpf_iter_unix__destroy(skel);
 601}
 602
 603/* The expected string is less than 16 bytes */
 604static int do_read_with_fd(int iter_fd, const char *expected,
 605			   bool read_one_char)
 606{
 607	int len, read_buf_len, start;
 608	char buf[16] = {};
 609
 610	read_buf_len = read_one_char ? 1 : 16;
 611	start = 0;
 612	while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
 613		start += len;
 614		if (!ASSERT_LT(start, 16, "read"))
 615			return -1;
 616		read_buf_len = read_one_char ? 1 : 16 - start;
 617	}
 618	if (!ASSERT_GE(len, 0, "read"))
 619		return -1;
 620
 621	if (!ASSERT_STREQ(buf, expected, "read"))
 622		return -1;
 623
 624	return 0;
 625}
 626
 627static void test_anon_iter(bool read_one_char)
 628{
 629	struct bpf_iter_test_kern1 *skel;
 630	struct bpf_link *link;
 631	int iter_fd, err;
 632
 633	skel = bpf_iter_test_kern1__open_and_load();
 634	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))
 635		return;
 636
 637	err = bpf_iter_test_kern1__attach(skel);
 638	if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {
 639		goto out;
 640	}
 641
 642	link = skel->links.dump_task;
 643	iter_fd = bpf_iter_create(bpf_link__fd(link));
 644	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 645		goto out;
 646
 647	do_read_with_fd(iter_fd, "abcd", read_one_char);
 648	close(iter_fd);
 649
 650out:
 651	bpf_iter_test_kern1__destroy(skel);
 652}
 653
 654static int do_read(const char *path, const char *expected)
 655{
 656	int err, iter_fd;
 657
 658	iter_fd = open(path, O_RDONLY);
 659	if (!ASSERT_GE(iter_fd, 0, "open"))
 660		return -1;
 661
 662	err = do_read_with_fd(iter_fd, expected, false);
 663	close(iter_fd);
 664	return err;
 665}
 666
 667static void test_file_iter(void)
 668{
 669	const char *path = "/sys/fs/bpf/bpf_iter_test1";
 670	struct bpf_iter_test_kern1 *skel1;
 671	struct bpf_iter_test_kern2 *skel2;
 672	struct bpf_link *link;
 673	int err;
 674
 675	skel1 = bpf_iter_test_kern1__open_and_load();
 676	if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))
 677		return;
 678
 679	link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);
 680	if (!ASSERT_OK_PTR(link, "attach_iter"))
 681		goto out;
 682
 683	/* unlink this path if it exists. */
 684	unlink(path);
 685
 686	err = bpf_link__pin(link, path);
 687	if (!ASSERT_OK(err, "pin_iter"))
 688		goto free_link;
 689
 690	err = do_read(path, "abcd");
 691	if (err)
 692		goto unlink_path;
 693
 694	/* file based iterator seems working fine. Let us a link update
 695	 * of the underlying link and `cat` the iterator again, its content
 696	 * should change.
 697	 */
 698	skel2 = bpf_iter_test_kern2__open_and_load();
 699	if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))
 700		goto unlink_path;
 701
 702	err = bpf_link__update_program(link, skel2->progs.dump_task);
 703	if (!ASSERT_OK(err, "update_prog"))
 704		goto destroy_skel2;
 705
 706	do_read(path, "ABCD");
 707
 708destroy_skel2:
 709	bpf_iter_test_kern2__destroy(skel2);
 710unlink_path:
 711	unlink(path);
 712free_link:
 713	bpf_link__destroy(link);
 714out:
 715	bpf_iter_test_kern1__destroy(skel1);
 716}
 717
 718static void test_overflow(bool test_e2big_overflow, bool ret1)
 719{
 720	__u32 map_info_len, total_read_len, expected_read_len;
 721	int err, iter_fd, map1_fd, map2_fd, len;
 722	struct bpf_map_info map_info = {};
 723	struct bpf_iter_test_kern4 *skel;
 724	struct bpf_link *link;
 725	__u32 iter_size;
 726	char *buf;
 727
 728	skel = bpf_iter_test_kern4__open();
 729	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))
 730		return;
 731
 732	/* create two maps: bpf program will only do bpf_seq_write
 733	 * for these two maps. The goal is one map output almost
 734	 * fills seq_file buffer and then the other will trigger
 735	 * overflow and needs restart.
 736	 */
 737	map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
 738	if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
 739		goto out;
 740	map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
 741	if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
 742		goto free_map1;
 743
 744	/* bpf_seq_printf kernel buffer is 8 pages, so one map
 745	 * bpf_seq_write will mostly fill it, and the other map
 746	 * will partially fill and then trigger overflow and need
 747	 * bpf_seq_read restart.
 748	 */
 749	iter_size = sysconf(_SC_PAGE_SIZE) << 3;
 750
 751	if (test_e2big_overflow) {
 752		skel->rodata->print_len = (iter_size + 8) / 8;
 753		expected_read_len = 2 * (iter_size + 8);
 754	} else if (!ret1) {
 755		skel->rodata->print_len = (iter_size - 8) / 8;
 756		expected_read_len = 2 * (iter_size - 8);
 757	} else {
 758		skel->rodata->print_len = 1;
 759		expected_read_len = 2 * 8;
 760	}
 761	skel->rodata->ret1 = ret1;
 762
 763	if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),
 764		  "bpf_iter_test_kern4__load"))
 765		goto free_map2;
 766
 767	/* setup filtering map_id in bpf program */
 768	map_info_len = sizeof(map_info);
 769	err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
 770	if (!ASSERT_OK(err, "get_map_info"))
 771		goto free_map2;
 772	skel->bss->map1_id = map_info.id;
 773
 774	err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
 775	if (!ASSERT_OK(err, "get_map_info"))
 776		goto free_map2;
 777	skel->bss->map2_id = map_info.id;
 778
 779	link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);
 780	if (!ASSERT_OK_PTR(link, "attach_iter"))
 781		goto free_map2;
 782
 783	iter_fd = bpf_iter_create(bpf_link__fd(link));
 784	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 785		goto free_link;
 786
 787	buf = malloc(expected_read_len);
 788	if (!ASSERT_OK_PTR(buf, "malloc"))
 789		goto close_iter;
 790
 791	/* do read */
 792	total_read_len = 0;
 793	if (test_e2big_overflow) {
 794		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 795			total_read_len += len;
 796
 797		ASSERT_EQ(len, -1, "read");
 798		ASSERT_EQ(errno, E2BIG, "read");
 799		goto free_buf;
 800	} else if (!ret1) {
 801		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 802			total_read_len += len;
 803
 804		if (!ASSERT_GE(len, 0, "read"))
 805			goto free_buf;
 806	} else {
 807		do {
 808			len = read(iter_fd, buf, expected_read_len);
 809			if (len > 0)
 810				total_read_len += len;
 811		} while (len > 0 || len == -EAGAIN);
 812
 813		if (!ASSERT_GE(len, 0, "read"))
 814			goto free_buf;
 815	}
 816
 817	if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))
 818		goto free_buf;
 819
 820	if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))
 821		goto free_buf;
 822
 823	if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))
 824		goto free_buf;
 825
 826	ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");
 827
 828free_buf:
 829	free(buf);
 830close_iter:
 831	close(iter_fd);
 832free_link:
 833	bpf_link__destroy(link);
 834free_map2:
 835	close(map2_fd);
 836free_map1:
 837	close(map1_fd);
 838out:
 839	bpf_iter_test_kern4__destroy(skel);
 840}
 841
 842static void test_bpf_hash_map(void)
 843{
 844	__u32 expected_key_a = 0, expected_key_b = 0;
 845	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 846	struct bpf_iter_bpf_hash_map *skel;
 847	int err, i, len, map_fd, iter_fd;
 848	union bpf_iter_link_info linfo;
 849	__u64 val, expected_val = 0;
 850	struct bpf_link *link;
 851	struct key_t {
 852		int a;
 853		int b;
 854		int c;
 855	} key;
 856	char buf[64];
 857
 858	skel = bpf_iter_bpf_hash_map__open();
 859	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))
 860		return;
 861
 862	skel->bss->in_test_mode = true;
 863
 864	err = bpf_iter_bpf_hash_map__load(skel);
 865	if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))
 866		goto out;
 867
 868	/* iterator with hashmap2 and hashmap3 should fail */
 869	memset(&linfo, 0, sizeof(linfo));
 870	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);
 871	opts.link_info = &linfo;
 872	opts.link_info_len = sizeof(linfo);
 873	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 874	if (!ASSERT_ERR_PTR(link, "attach_iter"))
 875		goto out;
 876
 877	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);
 878	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 879	if (!ASSERT_ERR_PTR(link, "attach_iter"))
 880		goto out;
 881
 882	/* hashmap1 should be good, update map values here */
 883	map_fd = bpf_map__fd(skel->maps.hashmap1);
 884	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
 885		key.a = i + 1;
 886		key.b = i + 2;
 887		key.c = i + 3;
 888		val = i + 4;
 889		expected_key_a += key.a;
 890		expected_key_b += key.b;
 891		expected_val += val;
 892
 893		err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
 894		if (!ASSERT_OK(err, "map_update"))
 895			goto out;
 896	}
 897
 898	/* Sleepable program is prohibited for hash map iterator */
 899	linfo.map.map_fd = map_fd;
 900	link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);
 901	if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))
 902		goto out;
 903
 904	linfo.map.map_fd = map_fd;
 905	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
 906	if (!ASSERT_OK_PTR(link, "attach_iter"))
 907		goto out;
 908
 909	iter_fd = bpf_iter_create(bpf_link__fd(link));
 910	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 911		goto free_link;
 912
 913	/* do some tests */
 914	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 915		;
 916	if (!ASSERT_GE(len, 0, "read"))
 917		goto close_iter;
 918
 919	/* test results */
 920	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
 921		goto close_iter;
 922	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
 923		goto close_iter;
 924	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
 925		goto close_iter;
 926
 927close_iter:
 928	close(iter_fd);
 929free_link:
 930	bpf_link__destroy(link);
 931out:
 932	bpf_iter_bpf_hash_map__destroy(skel);
 933}
 934
 935static void test_bpf_percpu_hash_map(void)
 936{
 937	__u32 expected_key_a = 0, expected_key_b = 0;
 938	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 939	struct bpf_iter_bpf_percpu_hash_map *skel;
 940	int err, i, j, len, map_fd, iter_fd;
 941	union bpf_iter_link_info linfo;
 942	__u32 expected_val = 0;
 943	struct bpf_link *link;
 944	struct key_t {
 945		int a;
 946		int b;
 947		int c;
 948	} key;
 949	char buf[64];
 950	void *val;
 951
 952	skel = bpf_iter_bpf_percpu_hash_map__open();
 953	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))
 954		return;
 955
 956	skel->rodata->num_cpus = bpf_num_possible_cpus();
 957	val = malloc(8 * bpf_num_possible_cpus());
 958	if (!ASSERT_OK_PTR(val, "malloc"))
 959		goto out;
 960
 961	err = bpf_iter_bpf_percpu_hash_map__load(skel);
 962	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
 963		goto out;
 964
 965	/* update map values here */
 966	map_fd = bpf_map__fd(skel->maps.hashmap1);
 967	for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {
 968		key.a = i + 1;
 969		key.b = i + 2;
 970		key.c = i + 3;
 971		expected_key_a += key.a;
 972		expected_key_b += key.b;
 973
 974		for (j = 0; j < bpf_num_possible_cpus(); j++) {
 975			*(__u32 *)(val + j * 8) = i + j;
 976			expected_val += i + j;
 977		}
 978
 979		err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);
 980		if (!ASSERT_OK(err, "map_update"))
 981			goto out;
 982	}
 983
 984	memset(&linfo, 0, sizeof(linfo));
 985	linfo.map.map_fd = map_fd;
 986	opts.link_info = &linfo;
 987	opts.link_info_len = sizeof(linfo);
 988	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);
 989	if (!ASSERT_OK_PTR(link, "attach_iter"))
 990		goto out;
 991
 992	iter_fd = bpf_iter_create(bpf_link__fd(link));
 993	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 994		goto free_link;
 995
 996	/* do some tests */
 997	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 998		;
 999	if (!ASSERT_GE(len, 0, "read"))
1000		goto close_iter;
1001
1002	/* test results */
1003	if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))
1004		goto close_iter;
1005	if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))
1006		goto close_iter;
1007	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1008		goto close_iter;
1009
1010close_iter:
1011	close(iter_fd);
1012free_link:
1013	bpf_link__destroy(link);
1014out:
1015	bpf_iter_bpf_percpu_hash_map__destroy(skel);
1016	free(val);
1017}
1018
1019static void test_bpf_array_map(void)
1020{
1021	__u64 val, expected_val = 0, res_first_val, first_val = 0;
1022	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1023	__u32 key, expected_key = 0, res_first_key;
1024	int err, i, map_fd, hash_fd, iter_fd;
1025	struct bpf_iter_bpf_array_map *skel;
1026	union bpf_iter_link_info linfo;
1027	struct bpf_link *link;
1028	char buf[64] = {};
1029	int len, start;
1030
1031	skel = bpf_iter_bpf_array_map__open_and_load();
1032	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1033		return;
1034
1035	map_fd = bpf_map__fd(skel->maps.arraymap1);
1036	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1037		val = i + 4;
1038		expected_key += i;
1039		expected_val += val;
1040
1041		if (i == 0)
1042			first_val = val;
1043
1044		err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);
1045		if (!ASSERT_OK(err, "map_update"))
1046			goto out;
1047	}
1048
1049	memset(&linfo, 0, sizeof(linfo));
1050	linfo.map.map_fd = map_fd;
1051	opts.link_info = &linfo;
1052	opts.link_info_len = sizeof(linfo);
1053	link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);
1054	if (!ASSERT_OK_PTR(link, "attach_iter"))
1055		goto out;
1056
1057	iter_fd = bpf_iter_create(bpf_link__fd(link));
1058	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1059		goto free_link;
1060
1061	/* do some tests */
1062	start = 0;
1063	while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
1064		start += len;
1065	if (!ASSERT_GE(len, 0, "read"))
1066		goto close_iter;
1067
1068	/* test results */
1069	res_first_key = *(__u32 *)buf;
1070	res_first_val = *(__u64 *)(buf + sizeof(__u32));
1071	if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
1072			!ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
1073		goto close_iter;
1074
1075	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1076		goto close_iter;
1077	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1078		goto close_iter;
1079
1080	hash_fd = bpf_map__fd(skel->maps.hashmap1);
1081	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1082		err = bpf_map_lookup_elem(map_fd, &i, &val);
1083		if (!ASSERT_OK(err, "map_lookup arraymap1"))
1084			goto close_iter;
1085		if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))
1086			goto close_iter;
1087
1088		val = i + 4;
1089		err = bpf_map_lookup_elem(hash_fd, &val, &key);
1090		if (!ASSERT_OK(err, "map_lookup hashmap1"))
1091			goto close_iter;
1092		if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))
1093			goto close_iter;
1094	}
1095
1096close_iter:
1097	close(iter_fd);
1098free_link:
1099	bpf_link__destroy(link);
1100out:
1101	bpf_iter_bpf_array_map__destroy(skel);
1102}
1103
1104static void test_bpf_array_map_iter_fd(void)
1105{
1106	struct bpf_iter_bpf_array_map *skel;
1107
1108	skel = bpf_iter_bpf_array_map__open_and_load();
1109	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))
1110		return;
1111
1112	do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,
1113			    skel->maps.arraymap1);
1114
1115	bpf_iter_bpf_array_map__destroy(skel);
1116}
1117
1118static void test_bpf_percpu_array_map(void)
1119{
1120	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1121	struct bpf_iter_bpf_percpu_array_map *skel;
1122	__u32 expected_key = 0, expected_val = 0;
1123	union bpf_iter_link_info linfo;
1124	int err, i, j, map_fd, iter_fd;
1125	struct bpf_link *link;
1126	char buf[64];
1127	void *val;
1128	int len;
1129
1130	skel = bpf_iter_bpf_percpu_array_map__open();
1131	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))
1132		return;
1133
1134	skel->rodata->num_cpus = bpf_num_possible_cpus();
1135	val = malloc(8 * bpf_num_possible_cpus());
1136	if (!ASSERT_OK_PTR(val, "malloc"))
1137		goto out;
1138
1139	err = bpf_iter_bpf_percpu_array_map__load(skel);
1140	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
1141		goto out;
1142
1143	/* update map values here */
1144	map_fd = bpf_map__fd(skel->maps.arraymap1);
1145	for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {
1146		expected_key += i;
1147
1148		for (j = 0; j < bpf_num_possible_cpus(); j++) {
1149			*(__u32 *)(val + j * 8) = i + j;
1150			expected_val += i + j;
1151		}
1152
1153		err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);
1154		if (!ASSERT_OK(err, "map_update"))
1155			goto out;
1156	}
1157
1158	memset(&linfo, 0, sizeof(linfo));
1159	linfo.map.map_fd = map_fd;
1160	opts.link_info = &linfo;
1161	opts.link_info_len = sizeof(linfo);
1162	link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);
1163	if (!ASSERT_OK_PTR(link, "attach_iter"))
1164		goto out;
1165
1166	iter_fd = bpf_iter_create(bpf_link__fd(link));
1167	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1168		goto free_link;
1169
1170	/* do some tests */
1171	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1172		;
1173	if (!ASSERT_GE(len, 0, "read"))
1174		goto close_iter;
1175
1176	/* test results */
1177	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
1178		goto close_iter;
1179	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1180		goto close_iter;
1181
1182close_iter:
1183	close(iter_fd);
1184free_link:
1185	bpf_link__destroy(link);
1186out:
1187	bpf_iter_bpf_percpu_array_map__destroy(skel);
1188	free(val);
1189}
1190
1191/* An iterator program deletes all local storage in a map. */
1192static void test_bpf_sk_storage_delete(void)
1193{
1194	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1195	struct bpf_iter_bpf_sk_storage_helpers *skel;
1196	union bpf_iter_link_info linfo;
1197	int err, len, map_fd, iter_fd;
1198	struct bpf_link *link;
1199	int sock_fd = -1;
1200	__u32 val = 42;
1201	char buf[64];
1202
1203	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1204	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1205		return;
1206
1207	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1208
1209	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1210	if (!ASSERT_GE(sock_fd, 0, "socket"))
1211		goto out;
1212
1213	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1214	if (!ASSERT_OK(err, "map_update"))
1215		goto out;
1216
1217	memset(&linfo, 0, sizeof(linfo));
1218	linfo.map.map_fd = map_fd;
1219	opts.link_info = &linfo;
1220	opts.link_info_len = sizeof(linfo);
1221	link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,
1222					&opts);
1223	if (!ASSERT_OK_PTR(link, "attach_iter"))
1224		goto out;
1225
1226	iter_fd = bpf_iter_create(bpf_link__fd(link));
1227	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1228		goto free_link;
1229
1230	/* do some tests */
1231	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1232		;
1233	if (!ASSERT_GE(len, 0, "read"))
1234		goto close_iter;
1235
1236	/* test results */
1237	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1238
1239	 /* Note: The following assertions serve to ensure
1240	  * the value was deleted. It does so by asserting
1241	  * that bpf_map_lookup_elem has failed. This might
1242	  * seem counterintuitive at first.
1243	  */
1244	ASSERT_ERR(err, "bpf_map_lookup_elem");
1245	ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
1246
1247close_iter:
1248	close(iter_fd);
1249free_link:
1250	bpf_link__destroy(link);
1251out:
1252	if (sock_fd >= 0)
1253		close(sock_fd);
1254	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1255}
1256
1257/* This creates a socket and its local storage. It then runs a task_iter BPF
1258 * program that replaces the existing socket local storage with the tgid of the
1259 * only task owning a file descriptor to this socket, this process, prog_tests.
1260 * It then runs a tcp socket iterator that negates the value in the existing
1261 * socket local storage, the test verifies that the resulting value is -pid.
1262 */
1263static void test_bpf_sk_storage_get(void)
1264{
1265	struct bpf_iter_bpf_sk_storage_helpers *skel;
1266	int err, map_fd, val = -1;
1267	int sock_fd = -1;
1268
1269	skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();
1270	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))
1271		return;
1272
1273	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
1274	if (!ASSERT_GE(sock_fd, 0, "socket"))
1275		goto out;
1276
1277	err = listen(sock_fd, 1);
1278	if (!ASSERT_OK(err, "listen"))
1279		goto close_socket;
1280
1281	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1282
1283	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
1284	if (!ASSERT_OK(err, "bpf_map_update_elem"))
1285		goto close_socket;
1286
1287	do_dummy_read(skel->progs.fill_socket_owner);
1288
1289	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1290	if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
1291			!ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
1292		goto close_socket;
1293
1294	do_dummy_read(skel->progs.negate_socket_local_storage);
1295
1296	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
1297	ASSERT_OK(err, "bpf_map_lookup_elem");
1298	ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
1299
1300close_socket:
1301	close(sock_fd);
1302out:
1303	bpf_iter_bpf_sk_storage_helpers__destroy(skel);
1304}
1305
1306static void test_bpf_sk_storage_map_iter_fd(void)
1307{
1308	struct bpf_iter_bpf_sk_storage_map *skel;
1309
1310	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1311	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1312		return;
1313
1314	do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,
1315			    skel->maps.sk_stg_map);
1316
1317	bpf_iter_bpf_sk_storage_map__destroy(skel);
1318}
1319
1320static void test_bpf_sk_storage_map(void)
1321{
1322	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1323	int err, i, len, map_fd, iter_fd, num_sockets;
1324	struct bpf_iter_bpf_sk_storage_map *skel;
1325	union bpf_iter_link_info linfo;
1326	int sock_fd[3] = {-1, -1, -1};
1327	__u32 val, expected_val = 0;
1328	struct bpf_link *link;
1329	char buf[64];
1330
1331	skel = bpf_iter_bpf_sk_storage_map__open_and_load();
1332	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))
1333		return;
1334
1335	map_fd = bpf_map__fd(skel->maps.sk_stg_map);
1336	num_sockets = ARRAY_SIZE(sock_fd);
1337	for (i = 0; i < num_sockets; i++) {
1338		sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);
1339		if (!ASSERT_GE(sock_fd[i], 0, "socket"))
1340			goto out;
1341
1342		val = i + 1;
1343		expected_val += val;
1344
1345		err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,
1346					  BPF_NOEXIST);
1347		if (!ASSERT_OK(err, "map_update"))
1348			goto out;
1349	}
1350
1351	memset(&linfo, 0, sizeof(linfo));
1352	linfo.map.map_fd = map_fd;
1353	opts.link_info = &linfo;
1354	opts.link_info_len = sizeof(linfo);
1355	link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);
1356	err = libbpf_get_error(link);
1357	if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {
1358		if (!err)
1359			bpf_link__destroy(link);
1360		goto out;
1361	}
1362
1363	link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);
1364	if (!ASSERT_OK_PTR(link, "attach_iter"))
1365		goto out;
1366
1367	iter_fd = bpf_iter_create(bpf_link__fd(link));
1368	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1369		goto free_link;
1370
1371	skel->bss->to_add_val = time(NULL);
1372	/* do some tests */
1373	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1374		;
1375	if (!ASSERT_GE(len, 0, "read"))
1376		goto close_iter;
1377
1378	/* test results */
1379	if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))
1380		goto close_iter;
1381
1382	if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))
1383		goto close_iter;
1384
1385	for (i = 0; i < num_sockets; i++) {
1386		err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);
1387		if (!ASSERT_OK(err, "map_lookup") ||
1388		    !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))
1389			break;
1390	}
1391
1392close_iter:
1393	close(iter_fd);
1394free_link:
1395	bpf_link__destroy(link);
1396out:
1397	for (i = 0; i < num_sockets; i++) {
1398		if (sock_fd[i] >= 0)
1399			close(sock_fd[i]);
1400	}
1401	bpf_iter_bpf_sk_storage_map__destroy(skel);
1402}
1403
1404static void test_rdonly_buf_out_of_bound(void)
1405{
1406	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1407	struct bpf_iter_test_kern5 *skel;
1408	union bpf_iter_link_info linfo;
1409	struct bpf_link *link;
1410
1411	skel = bpf_iter_test_kern5__open_and_load();
1412	if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))
1413		return;
1414
1415	memset(&linfo, 0, sizeof(linfo));
1416	linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);
1417	opts.link_info = &linfo;
1418	opts.link_info_len = sizeof(linfo);
1419	link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);
1420	if (!ASSERT_ERR_PTR(link, "attach_iter"))
1421		bpf_link__destroy(link);
1422
1423	bpf_iter_test_kern5__destroy(skel);
1424}
1425
1426static void test_buf_neg_offset(void)
1427{
1428	struct bpf_iter_test_kern6 *skel;
1429
1430	skel = bpf_iter_test_kern6__open_and_load();
1431	if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))
1432		bpf_iter_test_kern6__destroy(skel);
1433}
1434
1435static void test_link_iter(void)
1436{
1437	struct bpf_iter_bpf_link *skel;
1438
1439	skel = bpf_iter_bpf_link__open_and_load();
1440	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))
1441		return;
1442
1443	do_dummy_read(skel->progs.dump_bpf_link);
1444
1445	bpf_iter_bpf_link__destroy(skel);
1446}
1447
1448static void test_ksym_iter(void)
1449{
1450	struct bpf_iter_ksym *skel;
1451
1452	skel = bpf_iter_ksym__open_and_load();
1453	if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))
1454		return;
1455
1456	do_dummy_read(skel->progs.dump_ksym);
1457
1458	bpf_iter_ksym__destroy(skel);
1459}
1460
1461#define CMP_BUFFER_SIZE 1024
1462static char task_vma_output[CMP_BUFFER_SIZE];
1463static char proc_maps_output[CMP_BUFFER_SIZE];
1464
1465/* remove \0 and \t from str, and only keep the first line */
1466static void str_strip_first_line(char *str)
1467{
1468	char *dst = str, *src = str;
1469
1470	do {
1471		if (*src == ' ' || *src == '\t')
1472			src++;
1473		else
1474			*(dst++) = *(src++);
1475
1476	} while (*src != '\0' && *src != '\n');
1477
1478	*dst = '\0';
1479}
1480
1481static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
1482{
1483	int err, iter_fd = -1, proc_maps_fd = -1;
1484	struct bpf_iter_task_vmas *skel;
1485	int len, read_size = 4;
1486	char maps_path[64];
1487
1488	skel = bpf_iter_task_vmas__open();
1489	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1490		return;
1491
1492	skel->bss->pid = getpid();
1493	skel->bss->one_task = opts ? 1 : 0;
1494
1495	err = bpf_iter_task_vmas__load(skel);
1496	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1497		goto out;
1498
1499	skel->links.proc_maps = bpf_program__attach_iter(
1500		skel->progs.proc_maps, opts);
1501
1502	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1503		skel->links.proc_maps = NULL;
1504		goto out;
1505	}
1506
1507	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1508	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1509		goto out;
1510
1511	/* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks
1512	 * to trigger seq_file corner cases.
1513	 */
1514	len = 0;
1515	while (len < CMP_BUFFER_SIZE) {
1516		err = read_fd_into_buffer(iter_fd, task_vma_output + len,
1517					  MIN(read_size, CMP_BUFFER_SIZE - len));
1518		if (!err)
1519			break;
1520		if (!ASSERT_GE(err, 0, "read_iter_fd"))
1521			goto out;
1522		len += err;
1523	}
1524	if (opts)
1525		ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
1526
1527	/* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
1528	snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
1529	proc_maps_fd = open(maps_path, O_RDONLY);
1530	if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))
1531		goto out;
1532	err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);
1533	if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))
1534		goto out;
1535
1536	/* strip and compare the first line of the two files */
1537	str_strip_first_line(task_vma_output);
1538	str_strip_first_line(proc_maps_output);
1539
1540	ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
1541
1542	check_bpf_link_info(skel->progs.proc_maps);
1543
1544out:
1545	close(proc_maps_fd);
1546	close(iter_fd);
1547	bpf_iter_task_vmas__destroy(skel);
1548}
1549
1550static void test_task_vma_dead_task(void)
1551{
1552	struct bpf_iter_task_vmas *skel;
1553	int wstatus, child_pid = -1;
1554	time_t start_tm, cur_tm;
1555	int err, iter_fd = -1;
1556	int wait_sec = 3;
1557
1558	skel = bpf_iter_task_vmas__open();
1559	if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))
1560		return;
1561
1562	skel->bss->pid = getpid();
1563
1564	err = bpf_iter_task_vmas__load(skel);
1565	if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))
1566		goto out;
1567
1568	skel->links.proc_maps = bpf_program__attach_iter(
1569		skel->progs.proc_maps, NULL);
1570
1571	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
1572		skel->links.proc_maps = NULL;
1573		goto out;
1574	}
1575
1576	start_tm = time(NULL);
1577	cur_tm = start_tm;
1578
1579	child_pid = fork();
1580	if (child_pid == 0) {
1581		/* Fork short-lived processes in the background. */
1582		while (cur_tm < start_tm + wait_sec) {
1583			system("echo > /dev/null");
1584			cur_tm = time(NULL);
1585		}
1586		exit(0);
1587	}
1588
1589	if (!ASSERT_GE(child_pid, 0, "fork_child"))
1590		goto out;
1591
1592	while (cur_tm < start_tm + wait_sec) {
1593		iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));
1594		if (!ASSERT_GE(iter_fd, 0, "create_iter"))
1595			goto out;
1596
1597		/* Drain all data from iter_fd. */
1598		while (cur_tm < start_tm + wait_sec) {
1599			err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);
1600			if (!ASSERT_GE(err, 0, "read_iter_fd"))
1601				goto out;
1602
1603			cur_tm = time(NULL);
1604
1605			if (err == 0)
1606				break;
1607		}
1608
1609		close(iter_fd);
1610		iter_fd = -1;
1611	}
1612
1613	check_bpf_link_info(skel->progs.proc_maps);
1614
1615out:
1616	waitpid(child_pid, &wstatus, 0);
1617	close(iter_fd);
1618	bpf_iter_task_vmas__destroy(skel);
1619}
1620
1621void test_bpf_sockmap_map_iter_fd(void)
1622{
1623	struct bpf_iter_sockmap *skel;
1624
1625	skel = bpf_iter_sockmap__open_and_load();
1626	if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
1627		return;
1628
1629	do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);
1630
1631	bpf_iter_sockmap__destroy(skel);
1632}
1633
1634static void test_task_vma(void)
1635{
1636	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1637	union bpf_iter_link_info linfo;
1638
1639	memset(&linfo, 0, sizeof(linfo));
1640	linfo.task.tid = getpid();
1641	opts.link_info = &linfo;
1642	opts.link_info_len = sizeof(linfo);
1643
1644	test_task_vma_common(&opts);
1645	test_task_vma_common(NULL);
1646}
1647
1648/* uprobe attach point */
1649static noinline int trigger_func(int arg)
1650{
1651	asm volatile ("");
1652	return arg + 1;
1653}
1654
1655static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
1656{
1657	struct bpf_iter_vma_offset *skel;
1658	char buf[16] = {};
1659	int iter_fd, len;
1660	int pgsz, shift;
1661
1662	skel = bpf_iter_vma_offset__open_and_load();
1663	if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
1664		return;
1665
1666	skel->bss->pid = getpid();
1667	skel->bss->address = (uintptr_t)trigger_func;
1668	for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
1669		;
1670	skel->bss->page_shift = shift;
1671
1672	skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
1673	if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))
1674		goto exit;
1675
1676	iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));
1677	if (!ASSERT_GT(iter_fd, 0, "create_iter"))
1678		goto exit;
1679
1680	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
1681		;
1682	buf[15] = 0;
1683	ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
1684
1685	ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
1686	if (one_proc)
1687		ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1688	else
1689		ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
1690
1691	close(iter_fd);
1692
1693exit:
1694	bpf_iter_vma_offset__destroy(skel);
1695}
1696
1697static void test_task_vma_offset(void)
1698{
1699	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
1700	union bpf_iter_link_info linfo;
1701
1702	memset(&linfo, 0, sizeof(linfo));
1703	linfo.task.pid = getpid();
1704	opts.link_info = &linfo;
1705	opts.link_info_len = sizeof(linfo);
1706
1707	test_task_vma_offset_common(&opts, true);
1708
1709	linfo.task.pid = 0;
1710	linfo.task.tid = getpid();
1711	test_task_vma_offset_common(&opts, true);
1712
1713	test_task_vma_offset_common(NULL, false);
1714}
1715
1716void test_bpf_iter(void)
1717{
1718	ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
1719
1720	if (test__start_subtest("btf_id_or_null"))
1721		test_btf_id_or_null();
1722	if (test__start_subtest("ipv6_route"))
1723		test_ipv6_route();
1724	if (test__start_subtest("netlink"))
1725		test_netlink();
1726	if (test__start_subtest("bpf_map"))
1727		test_bpf_map();
1728	if (test__start_subtest("task_tid"))
1729		test_task_tid();
1730	if (test__start_subtest("task_pid"))
1731		test_task_pid();
1732	if (test__start_subtest("task_pidfd"))
1733		test_task_pidfd();
1734	if (test__start_subtest("task_sleepable"))
1735		test_task_sleepable();
1736	if (test__start_subtest("task_stack"))
1737		test_task_stack();
1738	if (test__start_subtest("task_file"))
1739		test_task_file();
1740	if (test__start_subtest("task_vma"))
1741		test_task_vma();
1742	if (test__start_subtest("task_vma_dead_task"))
1743		test_task_vma_dead_task();
1744	if (test__start_subtest("task_btf"))
1745		test_task_btf();
1746	if (test__start_subtest("tcp4"))
1747		test_tcp4();
1748	if (test__start_subtest("tcp6"))
1749		test_tcp6();
1750	if (test__start_subtest("udp4"))
1751		test_udp4();
1752	if (test__start_subtest("udp6"))
1753		test_udp6();
1754	if (test__start_subtest("unix"))
1755		test_unix();
1756	if (test__start_subtest("anon"))
1757		test_anon_iter(false);
1758	if (test__start_subtest("anon-read-one-char"))
1759		test_anon_iter(true);
1760	if (test__start_subtest("file"))
1761		test_file_iter();
1762	if (test__start_subtest("overflow"))
1763		test_overflow(false, false);
1764	if (test__start_subtest("overflow-e2big"))
1765		test_overflow(true, false);
1766	if (test__start_subtest("prog-ret-1"))
1767		test_overflow(false, true);
1768	if (test__start_subtest("bpf_hash_map"))
1769		test_bpf_hash_map();
1770	if (test__start_subtest("bpf_percpu_hash_map"))
1771		test_bpf_percpu_hash_map();
1772	if (test__start_subtest("bpf_array_map"))
1773		test_bpf_array_map();
1774	if (test__start_subtest("bpf_array_map_iter_fd"))
1775		test_bpf_array_map_iter_fd();
1776	if (test__start_subtest("bpf_percpu_array_map"))
1777		test_bpf_percpu_array_map();
1778	if (test__start_subtest("bpf_sk_storage_map"))
1779		test_bpf_sk_storage_map();
1780	if (test__start_subtest("bpf_sk_storage_map_iter_fd"))
1781		test_bpf_sk_storage_map_iter_fd();
1782	if (test__start_subtest("bpf_sk_storage_delete"))
1783		test_bpf_sk_storage_delete();
1784	if (test__start_subtest("bpf_sk_storage_get"))
1785		test_bpf_sk_storage_get();
1786	if (test__start_subtest("rdonly-buf-out-of-bound"))
1787		test_rdonly_buf_out_of_bound();
1788	if (test__start_subtest("buf-neg-offset"))
1789		test_buf_neg_offset();
1790	if (test__start_subtest("link-iter"))
1791		test_link_iter();
1792	if (test__start_subtest("ksym"))
1793		test_ksym_iter();
1794	if (test__start_subtest("bpf_sockmap_map_iter_fd"))
1795		test_bpf_sockmap_map_iter_fd();
1796	if (test__start_subtest("vma_offset"))
1797		test_task_vma_offset();
1798}