select_reuseport.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2018 Facebook */
  3
  4#include <stdlib.h>
  5#include <unistd.h>
  6#include <stdbool.h>
  7#include <string.h>
  8#include <errno.h>
  9#include <assert.h>
 10#include <fcntl.h>
 11#include <linux/bpf.h>
 12#include <linux/err.h>
 13#include <linux/types.h>
 14#include <linux/if_ether.h>
 15#include <sys/types.h>
 16#include <sys/epoll.h>
 17#include <sys/socket.h>
 18#include <netinet/in.h>
 19#include <bpf/bpf.h>
 20#include <bpf/libbpf.h>
 21#include "bpf_util.h"
 22
 23#include "test_progs.h"
 24#include "test_select_reuseport_common.h"
 25
 26#define MAX_TEST_NAME 80
 27#define MIN_TCPHDR_LEN 20
 28#define UDPHDR_LEN 8
 29
 30#define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies"
 31#define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen"
 32#define REUSEPORT_ARRAY_SIZE 32
 33
 34static int result_map, tmp_index_ovr_map, linum_map, data_check_map;
 35static __u32 expected_results[NR_RESULTS];
 36static int sk_fds[REUSEPORT_ARRAY_SIZE];
 37static int reuseport_array = -1, outer_map = -1;
 38static enum bpf_map_type inner_map_type;
 39static int select_by_skb_data_prog;
 40static struct bpf_object *obj;
 41static __u32 index_zero;
 42static int epfd;
 43
 44static struct sockaddr_storage srv_sa;
 45
 46#define RET_IF(condition, tag, format...) ({				\
 47	if (CHECK_FAIL(condition)) {					\
 48		printf(tag " " format);					\
 49		return;							\
 50	}								\
 51})
 52
 53#define RET_ERR(condition, tag, format...) ({				\
 54	if (CHECK_FAIL(condition)) {					\
 55		printf(tag " " format);					\
 56		return -1;						\
 57	}								\
 58})
 59
 60static int create_maps(enum bpf_map_type inner_type)
 61{
 62	LIBBPF_OPTS(bpf_map_create_opts, opts);
 63
 64	inner_map_type = inner_type;
 65
 66	/* Creating reuseport_array */
 67	reuseport_array = bpf_map_create(inner_type, "reuseport_array",
 68					 sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL);
 69	RET_ERR(reuseport_array < 0, "creating reuseport_array",
 70		"reuseport_array:%d errno:%d\n", reuseport_array, errno);
 71
 72	/* Creating outer_map */
 73	opts.inner_map_fd = reuseport_array;
 74	outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map",
 75				   sizeof(__u32), sizeof(__u32), 1, &opts);
 76	RET_ERR(outer_map < 0, "creating outer_map",
 77		"outer_map:%d errno:%d\n", outer_map, errno);
 78
 79	return 0;
 80}
 81
 82static int prepare_bpf_obj(void)
 83{
 84	struct bpf_program *prog;
 85	struct bpf_map *map;
 86	int err;
 87
 88	obj = bpf_object__open("test_select_reuseport_kern.bpf.o");
 89	err = libbpf_get_error(obj);
 90	RET_ERR(err, "open test_select_reuseport_kern.bpf.o",
 91		"obj:%p PTR_ERR(obj):%d\n", obj, err);
 92
 93	map = bpf_object__find_map_by_name(obj, "outer_map");
 94	RET_ERR(!map, "find outer_map", "!map\n");
 95	err = bpf_map__reuse_fd(map, outer_map);
 96	RET_ERR(err, "reuse outer_map", "err:%d\n", err);
 97
 98	err = bpf_object__load(obj);
 99	RET_ERR(err, "load bpf_object", "err:%d\n", err);
100
101	prog = bpf_object__next_program(obj, NULL);
102	RET_ERR(!prog, "get first bpf_program", "!prog\n");
103	select_by_skb_data_prog = bpf_program__fd(prog);
104	RET_ERR(select_by_skb_data_prog < 0, "get prog fd",
105		"select_by_skb_data_prog:%d\n", select_by_skb_data_prog);
106
107	map = bpf_object__find_map_by_name(obj, "result_map");
108	RET_ERR(!map, "find result_map", "!map\n");
109	result_map = bpf_map__fd(map);
110	RET_ERR(result_map < 0, "get result_map fd",
111		"result_map:%d\n", result_map);
112
113	map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map");
114	RET_ERR(!map, "find tmp_index_ovr_map\n", "!map");
115	tmp_index_ovr_map = bpf_map__fd(map);
116	RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd",
117		"tmp_index_ovr_map:%d\n", tmp_index_ovr_map);
118
119	map = bpf_object__find_map_by_name(obj, "linum_map");
120	RET_ERR(!map, "find linum_map", "!map\n");
121	linum_map = bpf_map__fd(map);
122	RET_ERR(linum_map < 0, "get linum_map fd",
123		"linum_map:%d\n", linum_map);
124
125	map = bpf_object__find_map_by_name(obj, "data_check_map");
126	RET_ERR(!map, "find data_check_map", "!map\n");
127	data_check_map = bpf_map__fd(map);
128	RET_ERR(data_check_map < 0, "get data_check_map fd",
129		"data_check_map:%d\n", data_check_map);
130
131	return 0;
132}
133
134static void ss_init_loopback(struct sockaddr_storage *sa, sa_family_t family)
135{
136	memset(sa, 0, sizeof(*sa));
137	sa->ss_family = family;
138	if (sa->ss_family == AF_INET6)
139		((struct sockaddr_in6 *)sa)->sin6_addr = in6addr_loopback;
140	else
141		((struct sockaddr_in *)sa)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
142}
143
144static void ss_init_inany(struct sockaddr_storage *sa, sa_family_t family)
145{
146	memset(sa, 0, sizeof(*sa));
147	sa->ss_family = family;
148	if (sa->ss_family == AF_INET6)
149		((struct sockaddr_in6 *)sa)->sin6_addr = in6addr_any;
150	else
151		((struct sockaddr_in *)sa)->sin_addr.s_addr = INADDR_ANY;
152}
153
154static int read_int_sysctl(const char *sysctl)
155{
156	char buf[16];
157	int fd, ret;
158
159	fd = open(sysctl, 0);
160	RET_ERR(fd == -1, "open(sysctl)",
161		"sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
162
163	ret = read(fd, buf, sizeof(buf));
164	RET_ERR(ret <= 0, "read(sysctl)",
165		"sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno);
166
167	close(fd);
168	return atoi(buf);
169}
170
171static int write_int_sysctl(const char *sysctl, int v)
172{
173	int fd, ret, size;
174	char buf[16];
175
176	fd = open(sysctl, O_RDWR);
177	RET_ERR(fd == -1, "open(sysctl)",
178		"sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
179
180	size = snprintf(buf, sizeof(buf), "%d", v);
181	ret = write(fd, buf, size);
182	RET_ERR(ret != size, "write(sysctl)",
183		"sysctl:%s ret:%d size:%d errno:%d\n",
184		sysctl, ret, size, errno);
185
186	close(fd);
187	return 0;
188}
189
190static int enable_fastopen(void)
191{
192	int fo;
193
194	fo = read_int_sysctl(TCP_FO_SYSCTL);
195	if (fo < 0)
196		return -1;
197
198	return write_int_sysctl(TCP_FO_SYSCTL, fo | 7);
199}
200
201static int enable_syncookie(void)
202{
203	return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2);
204}
205
206static int disable_syncookie(void)
207{
208	return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0);
209}
210
211static long get_linum(void)
212{
213	__u32 linum;
214	int err;
215
216	err = bpf_map_lookup_elem(linum_map, &index_zero, &linum);
217	RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n",
218		err, errno);
219
220	return linum;
221}
222
223static void check_data(int type, sa_family_t family, const struct cmd *cmd,
224		       int cli_fd)
225{
226	struct data_check expected = {}, result;
227	struct sockaddr_storage cli_sa;
228	socklen_t addrlen;
229	int err;
230
231	addrlen = sizeof(cli_sa);
232	err = getsockname(cli_fd, (struct sockaddr *)&cli_sa,
233			  &addrlen);
234	RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n",
235	       err, errno);
236
237	err = bpf_map_lookup_elem(data_check_map, &index_zero, &result);
238	RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n",
239	       err, errno);
240
241	if (type == SOCK_STREAM) {
242		expected.len = MIN_TCPHDR_LEN;
243		expected.ip_protocol = IPPROTO_TCP;
244	} else {
245		expected.len = UDPHDR_LEN;
246		expected.ip_protocol = IPPROTO_UDP;
247	}
248
249	if (family == AF_INET6) {
250		struct sockaddr_in6 *srv_v6 = (struct sockaddr_in6 *)&srv_sa;
251		struct sockaddr_in6 *cli_v6 = (struct sockaddr_in6 *)&cli_sa;
252
253		expected.eth_protocol = htons(ETH_P_IPV6);
254		expected.bind_inany = !srv_v6->sin6_addr.s6_addr32[3] &&
255			!srv_v6->sin6_addr.s6_addr32[2] &&
256			!srv_v6->sin6_addr.s6_addr32[1] &&
257			!srv_v6->sin6_addr.s6_addr32[0];
258
259		memcpy(&expected.skb_addrs[0], cli_v6->sin6_addr.s6_addr32,
260		       sizeof(cli_v6->sin6_addr));
261		memcpy(&expected.skb_addrs[4], &in6addr_loopback,
262		       sizeof(in6addr_loopback));
263		expected.skb_ports[0] = cli_v6->sin6_port;
264		expected.skb_ports[1] = srv_v6->sin6_port;
265	} else {
266		struct sockaddr_in *srv_v4 = (struct sockaddr_in *)&srv_sa;
267		struct sockaddr_in *cli_v4 = (struct sockaddr_in *)&cli_sa;
268
269		expected.eth_protocol = htons(ETH_P_IP);
270		expected.bind_inany = !srv_v4->sin_addr.s_addr;
271
272		expected.skb_addrs[0] = cli_v4->sin_addr.s_addr;
273		expected.skb_addrs[1] = htonl(INADDR_LOOPBACK);
274		expected.skb_ports[0] = cli_v4->sin_port;
275		expected.skb_ports[1] = srv_v4->sin_port;
276	}
277
278	if (memcmp(&result, &expected, offsetof(struct data_check,
279						equal_check_end))) {
280		printf("unexpected data_check\n");
281		printf("  result: (0x%x, %u, %u)\n",
282		       result.eth_protocol, result.ip_protocol,
283		       result.bind_inany);
284		printf("expected: (0x%x, %u, %u)\n",
285		       expected.eth_protocol, expected.ip_protocol,
286		       expected.bind_inany);
287		RET_IF(1, "data_check result != expected",
288		       "bpf_prog_linum:%ld\n", get_linum());
289	}
290
291	RET_IF(!result.hash, "data_check result.hash empty",
292	       "result.hash:%u", result.hash);
293
294	expected.len += cmd ? sizeof(*cmd) : 0;
295	if (type == SOCK_STREAM)
296		RET_IF(expected.len > result.len, "expected.len > result.len",
297		       "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
298		       expected.len, result.len, get_linum());
299	else
300		RET_IF(expected.len != result.len, "expected.len != result.len",
301		       "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
302		       expected.len, result.len, get_linum());
303}
304
305static const char *result_to_str(enum result res)
306{
307	switch (res) {
308	case DROP_ERR_INNER_MAP:
309		return "DROP_ERR_INNER_MAP";
310	case DROP_ERR_SKB_DATA:
311		return "DROP_ERR_SKB_DATA";
312	case DROP_ERR_SK_SELECT_REUSEPORT:
313		return "DROP_ERR_SK_SELECT_REUSEPORT";
314	case DROP_MISC:
315		return "DROP_MISC";
316	case PASS:
317		return "PASS";
318	case PASS_ERR_SK_SELECT_REUSEPORT:
319		return "PASS_ERR_SK_SELECT_REUSEPORT";
320	default:
321		return "UNKNOWN";
322	}
323}
324
325static void check_results(void)
326{
327	__u32 results[NR_RESULTS];
328	__u32 i, broken = 0;
329	int err;
330
331	for (i = 0; i < NR_RESULTS; i++) {
332		err = bpf_map_lookup_elem(result_map, &i, &results[i]);
333		RET_IF(err < 0, "lookup_elem(result_map)",
334		       "i:%u err:%d errno:%d\n", i, err, errno);
335	}
336
337	for (i = 0; i < NR_RESULTS; i++) {
338		if (results[i] != expected_results[i]) {
339			broken = i;
340			break;
341		}
342	}
343
344	if (i == NR_RESULTS)
345		return;
346
347	printf("unexpected result\n");
348	printf(" result: [");
349	printf("%u", results[0]);
350	for (i = 1; i < NR_RESULTS; i++)
351		printf(", %u", results[i]);
352	printf("]\n");
353
354	printf("expected: [");
355	printf("%u", expected_results[0]);
356	for (i = 1; i < NR_RESULTS; i++)
357		printf(", %u", expected_results[i]);
358	printf("]\n");
359
360	printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken),
361	       get_linum());
362
363	CHECK_FAIL(true);
364}
365
366static int send_data(int type, sa_family_t family, void *data, size_t len,
367		     enum result expected)
368{
369	struct sockaddr_storage cli_sa;
370	int fd, err;
371
372	fd = socket(family, type, 0);
373	RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno);
374
375	ss_init_loopback(&cli_sa, family);
376	err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa));
377	RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno);
378	err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa,
379		     sizeof(srv_sa));
380	RET_ERR(err != len && expected >= PASS,
381		"sendto()", "family:%u err:%d errno:%d expected:%d\n",
382		family, err, errno, expected);
383
384	return fd;
385}
386
387static void do_test(int type, sa_family_t family, struct cmd *cmd,
388		    enum result expected)
389{
390	int nev, srv_fd, cli_fd;
391	struct epoll_event ev;
392	struct cmd rcv_cmd;
393	ssize_t nread;
394
395	cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0,
396			   expected);
397	if (cli_fd < 0)
398		return;
399	nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0);
400	RET_IF((nev <= 0 && expected >= PASS) ||
401	       (nev > 0 && expected < PASS),
402	       "nev <> expected",
403	       "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n",
404	       nev, expected, type, family,
405	       cmd ? cmd->reuseport_index : -1,
406	       cmd ? cmd->pass_on_failure : -1);
407	check_results();
408	check_data(type, family, cmd, cli_fd);
409
410	if (expected < PASS)
411		return;
412
413	RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT &&
414	       cmd->reuseport_index != ev.data.u32,
415	       "check cmd->reuseport_index",
416	       "cmd:(%u, %u) ev.data.u32:%u\n",
417	       cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32);
418
419	srv_fd = sk_fds[ev.data.u32];
420	if (type == SOCK_STREAM) {
421		int new_fd = accept(srv_fd, NULL, 0);
422
423		RET_IF(new_fd == -1, "accept(srv_fd)",
424		       "ev.data.u32:%u new_fd:%d errno:%d\n",
425		       ev.data.u32, new_fd, errno);
426
427		nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
428		RET_IF(nread != sizeof(rcv_cmd),
429		       "recv(new_fd)",
430		       "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
431		       ev.data.u32, nread, sizeof(rcv_cmd), errno);
432
433		close(new_fd);
434	} else {
435		nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
436		RET_IF(nread != sizeof(rcv_cmd),
437		       "recv(sk_fds)",
438		       "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
439		       ev.data.u32, nread, sizeof(rcv_cmd), errno);
440	}
441
442	close(cli_fd);
443}
444
445static void test_err_inner_map(int type, sa_family_t family)
446{
447	struct cmd cmd = {
448		.reuseport_index = 0,
449		.pass_on_failure = 0,
450	};
451
452	expected_results[DROP_ERR_INNER_MAP]++;
453	do_test(type, family, &cmd, DROP_ERR_INNER_MAP);
454}
455
456static void test_err_skb_data(int type, sa_family_t family)
457{
458	expected_results[DROP_ERR_SKB_DATA]++;
459	do_test(type, family, NULL, DROP_ERR_SKB_DATA);
460}
461
462static void test_err_sk_select_port(int type, sa_family_t family)
463{
464	struct cmd cmd = {
465		.reuseport_index = REUSEPORT_ARRAY_SIZE,
466		.pass_on_failure = 0,
467	};
468
469	expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++;
470	do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT);
471}
472
473static void test_pass(int type, sa_family_t family)
474{
475	struct cmd cmd;
476	int i;
477
478	cmd.pass_on_failure = 0;
479	for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
480		expected_results[PASS]++;
481		cmd.reuseport_index = i;
482		do_test(type, family, &cmd, PASS);
483	}
484}
485
486static void test_syncookie(int type, sa_family_t family)
487{
488	int err, tmp_index = 1;
489	struct cmd cmd = {
490		.reuseport_index = 0,
491		.pass_on_failure = 0,
492	};
493
494	/*
495	 * +1 for TCP-SYN and
496	 * +1 for the TCP-ACK (ack the syncookie)
497	 */
498	expected_results[PASS] += 2;
499	enable_syncookie();
500	/*
501	 * Simulate TCP-SYN and TCP-ACK are handled by two different sk:
502	 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the
503	 *          tmp_index_ovr_map
504	 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index
505	 *          is from the cmd.reuseport_index
506	 */
507	err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero,
508				  &tmp_index, BPF_ANY);
509	RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)",
510	       "err:%d errno:%d\n", err, errno);
511	do_test(type, family, &cmd, PASS);
512	err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero,
513				  &tmp_index);
514	RET_IF(err < 0 || tmp_index >= 0,
515	       "lookup_elem(tmp_index_ovr_map)",
516	       "err:%d errno:%d tmp_index:%d\n",
517	       err, errno, tmp_index);
518	disable_syncookie();
519}
520
521static void test_pass_on_err(int type, sa_family_t family)
522{
523	struct cmd cmd = {
524		.reuseport_index = REUSEPORT_ARRAY_SIZE,
525		.pass_on_failure = 1,
526	};
527
528	expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1;
529	do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT);
530}
531
532static void test_detach_bpf(int type, sa_family_t family)
533{
534#ifdef SO_DETACH_REUSEPORT_BPF
535	__u32 nr_run_before = 0, nr_run_after = 0, tmp, i;
536	struct epoll_event ev;
537	int cli_fd, err, nev;
538	struct cmd cmd = {};
539	int optvalue = 0;
540
541	err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
542			 &optvalue, sizeof(optvalue));
543	RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
544	       "err:%d errno:%d\n", err, errno);
545
546	err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
547			 &optvalue, sizeof(optvalue));
548	RET_IF(err == 0 || errno != ENOENT,
549	       "setsockopt(SO_DETACH_REUSEPORT_BPF)",
550	       "err:%d errno:%d\n", err, errno);
551
552	for (i = 0; i < NR_RESULTS; i++) {
553		err = bpf_map_lookup_elem(result_map, &i, &tmp);
554		RET_IF(err < 0, "lookup_elem(result_map)",
555		       "i:%u err:%d errno:%d\n", i, err, errno);
556		nr_run_before += tmp;
557	}
558
559	cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS);
560	if (cli_fd < 0)
561		return;
562	nev = epoll_wait(epfd, &ev, 1, 5);
563	RET_IF(nev <= 0, "nev <= 0",
564	       "nev:%d expected:1 type:%d family:%d data:(0, 0)\n",
565	       nev,  type, family);
566
567	for (i = 0; i < NR_RESULTS; i++) {
568		err = bpf_map_lookup_elem(result_map, &i, &tmp);
569		RET_IF(err < 0, "lookup_elem(result_map)",
570		       "i:%u err:%d errno:%d\n", i, err, errno);
571		nr_run_after += tmp;
572	}
573
574	RET_IF(nr_run_before != nr_run_after,
575	       "nr_run_before != nr_run_after",
576	       "nr_run_before:%u nr_run_after:%u\n",
577	       nr_run_before, nr_run_after);
578
579	close(cli_fd);
580#else
581	test__skip();
582#endif
583}
584
585static void prepare_sk_fds(int type, sa_family_t family, bool inany)
586{
587	const int first = REUSEPORT_ARRAY_SIZE - 1;
588	int i, err, optval = 1;
589	struct epoll_event ev;
590	socklen_t addrlen;
591
592	if (inany)
593		ss_init_inany(&srv_sa, family);
594	else
595		ss_init_loopback(&srv_sa, family);
596	addrlen = sizeof(srv_sa);
597
598	/*
599	 * The sk_fds[] is filled from the back such that the order
600	 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[].
601	 */
602	for (i = first; i >= 0; i--) {
603		sk_fds[i] = socket(family, type, 0);
604		RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n",
605		       i, sk_fds[i], errno);
606		err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT,
607				 &optval, sizeof(optval));
608		RET_IF(err == -1, "setsockopt(SO_REUSEPORT)",
609		       "sk_fds[%d] err:%d errno:%d\n",
610		       i, err, errno);
611
612		if (i == first) {
613			err = setsockopt(sk_fds[i], SOL_SOCKET,
614					 SO_ATTACH_REUSEPORT_EBPF,
615					 &select_by_skb_data_prog,
616					 sizeof(select_by_skb_data_prog));
617			RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)",
618			       "err:%d errno:%d\n", err, errno);
619		}
620
621		err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen);
622		RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n",
623		       i, err, errno);
624
625		if (type == SOCK_STREAM) {
626			err = listen(sk_fds[i], 10);
627			RET_IF(err < 0, "listen()",
628			       "sk_fds[%d] err:%d errno:%d\n",
629			       i, err, errno);
630		}
631
632		err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i],
633					  BPF_NOEXIST);
634		RET_IF(err < 0, "update_elem(reuseport_array)",
635		       "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
636
637		if (i == first) {
638			socklen_t addrlen = sizeof(srv_sa);
639
640			err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa,
641					  &addrlen);
642			RET_IF(err == -1, "getsockname()",
643			       "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
644		}
645	}
646
647	epfd = epoll_create(1);
648	RET_IF(epfd == -1, "epoll_create(1)",
649	       "epfd:%d errno:%d\n", epfd, errno);
650
651	ev.events = EPOLLIN;
652	for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
653		ev.data.u32 = i;
654		err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev);
655		RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i);
656	}
657}
658
659static void setup_per_test(int type, sa_family_t family, bool inany,
660			   bool no_inner_map)
661{
662	int ovr = -1, err;
663
664	prepare_sk_fds(type, family, inany);
665	err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr,
666				  BPF_ANY);
667	RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)",
668	       "err:%d errno:%d\n", err, errno);
669
670	/* Install reuseport_array to outer_map? */
671	if (no_inner_map)
672		return;
673
674	err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array,
675				  BPF_ANY);
676	RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)",
677	       "err:%d errno:%d\n", err, errno);
678}
679
680static void cleanup_per_test(bool no_inner_map)
681{
682	int i, err, zero = 0;
683
684	memset(expected_results, 0, sizeof(expected_results));
685
686	for (i = 0; i < NR_RESULTS; i++) {
687		err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY);
688		RET_IF(err, "reset elem in result_map",
689		       "i:%u err:%d errno:%d\n", i, err, errno);
690	}
691
692	err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY);
693	RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n",
694	       err, errno);
695
696	for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++)
697		close(sk_fds[i]);
698	close(epfd);
699
700	/* Delete reuseport_array from outer_map? */
701	if (no_inner_map)
702		return;
703
704	err = bpf_map_delete_elem(outer_map, &index_zero);
705	RET_IF(err < 0, "delete_elem(outer_map)",
706	       "err:%d errno:%d\n", err, errno);
707}
708
709static void cleanup(void)
710{
711	if (outer_map >= 0) {
712		close(outer_map);
713		outer_map = -1;
714	}
715
716	if (reuseport_array >= 0) {
717		close(reuseport_array);
718		reuseport_array = -1;
719	}
720
721	if (obj) {
722		bpf_object__close(obj);
723		obj = NULL;
724	}
725
726	memset(expected_results, 0, sizeof(expected_results));
727}
728
729static const char *maptype_str(enum bpf_map_type type)
730{
731	switch (type) {
732	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
733		return "reuseport_sockarray";
734	case BPF_MAP_TYPE_SOCKMAP:
735		return "sockmap";
736	case BPF_MAP_TYPE_SOCKHASH:
737		return "sockhash";
738	default:
739		return "unknown";
740	}
741}
742
743static const char *family_str(sa_family_t family)
744{
745	switch (family) {
746	case AF_INET:
747		return "IPv4";
748	case AF_INET6:
749		return "IPv6";
750	default:
751		return "unknown";
752	}
753}
754
755static const char *sotype_str(int sotype)
756{
757	switch (sotype) {
758	case SOCK_STREAM:
759		return "TCP";
760	case SOCK_DGRAM:
761		return "UDP";
762	default:
763		return "unknown";
764	}
765}
766
767#define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ }
768
769static void test_config(int sotype, sa_family_t family, bool inany)
770{
771	const struct test {
772		void (*fn)(int sotype, sa_family_t family);
773		const char *name;
774		bool no_inner_map;
775		int need_sotype;
776	} tests[] = {
777		TEST_INIT(test_err_inner_map,
778			  .no_inner_map = true),
779		TEST_INIT(test_err_skb_data),
780		TEST_INIT(test_err_sk_select_port),
781		TEST_INIT(test_pass),
782		TEST_INIT(test_syncookie,
783			  .need_sotype = SOCK_STREAM),
784		TEST_INIT(test_pass_on_err),
785		TEST_INIT(test_detach_bpf),
786	};
787	struct netns_obj *netns;
788	char s[MAX_TEST_NAME];
789	const struct test *t;
790
791	for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
792		if (t->need_sotype && t->need_sotype != sotype)
793			continue; /* test not compatible with socket type */
794
795		snprintf(s, sizeof(s), "%s %s/%s %s %s",
796			 maptype_str(inner_map_type),
797			 family_str(family), sotype_str(sotype),
798			 inany ? "INANY" : "LOOPBACK", t->name);
799
800		if (!test__start_subtest(s))
801			continue;
802
803		netns = netns_new("select_reuseport", true);
804		if (!ASSERT_OK_PTR(netns, "netns_new"))
805			continue;
806
807		if (CHECK_FAIL(enable_fastopen()))
808			goto out;
809		if (CHECK_FAIL(disable_syncookie()))
810			goto out;
811
812		setup_per_test(sotype, family, inany, t->no_inner_map);
813		t->fn(sotype, family);
814		cleanup_per_test(t->no_inner_map);
815
816out:
817		netns_free(netns);
818	}
819}
820
821#define BIND_INANY true
822
823static void test_all(void)
824{
825	const struct config {
826		int sotype;
827		sa_family_t family;
828		bool inany;
829	} configs[] = {
830		{ SOCK_STREAM, AF_INET },
831		{ SOCK_STREAM, AF_INET, BIND_INANY },
832		{ SOCK_STREAM, AF_INET6 },
833		{ SOCK_STREAM, AF_INET6, BIND_INANY },
834		{ SOCK_DGRAM, AF_INET },
835		{ SOCK_DGRAM, AF_INET6 },
836	};
837	const struct config *c;
838
839	for (c = configs; c < configs + ARRAY_SIZE(configs); c++)
840		test_config(c->sotype, c->family, c->inany);
841}
842
843void test_map_type(enum bpf_map_type mt)
844{
845	if (create_maps(mt))
846		goto out;
847	if (prepare_bpf_obj())
848		goto out;
849
850	test_all();
851out:
852	cleanup();
853}
854
855void serial_test_select_reuseport(void)
856{
857	test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY);
858	test_map_type(BPF_MAP_TYPE_SOCKMAP);
859	test_map_type(BPF_MAP_TYPE_SOCKHASH);
860}