xdp_adjust_tail.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <test_progs.h>
  3#include <network_helpers.h>
  4
  5static void test_xdp_adjust_tail_shrink(void)
  6{
  7	const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
  8	__u32 expect_sz;
  9	struct bpf_object *obj;
 10	int err, prog_fd;
 11	char buf[128];
 12	LIBBPF_OPTS(bpf_test_run_opts, topts,
 13		.data_in = &pkt_v4,
 14		.data_size_in = sizeof(pkt_v4),
 15		.data_out = buf,
 16		.data_size_out = sizeof(buf),
 17		.repeat = 1,
 18	);
 19
 20	err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 21	if (!ASSERT_OK(err, "test_xdp_adjust_tail_shrink"))
 22		return;
 23
 24	err = bpf_prog_test_run_opts(prog_fd, &topts);
 25	ASSERT_OK(err, "ipv4");
 26	ASSERT_EQ(topts.retval, XDP_DROP, "ipv4 retval");
 27
 28	expect_sz = sizeof(pkt_v6) - 20;  /* Test shrink with 20 bytes */
 29	topts.data_in = &pkt_v6;
 30	topts.data_size_in = sizeof(pkt_v6);
 31	topts.data_size_out = sizeof(buf);
 32	err = bpf_prog_test_run_opts(prog_fd, &topts);
 33	ASSERT_OK(err, "ipv6");
 34	ASSERT_EQ(topts.retval, XDP_TX, "ipv6 retval");
 35	ASSERT_EQ(topts.data_size_out, expect_sz, "ipv6 size");
 36
 37	bpf_object__close(obj);
 38}
 39
 40static void test_xdp_adjust_tail_grow(bool is_64k_pagesize)
 41{
 42	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
 43	struct bpf_object *obj;
 44	char buf[8192]; /* avoid segfault: large buf to hold grow results */
 45	__u32 expect_sz;
 46	int err, prog_fd;
 47	LIBBPF_OPTS(bpf_test_run_opts, topts,
 48		.data_in = &pkt_v4,
 49		.data_out = buf,
 50		.data_size_out = sizeof(buf),
 51		.repeat = 1,
 52	);
 53
 54	/* topts.data_size_in as a special signal to bpf prog */
 55	if (is_64k_pagesize)
 56		topts.data_size_in = sizeof(pkt_v4) - 1;
 57	else
 58		topts.data_size_in = sizeof(pkt_v4);
 59
 60	err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 61	if (!ASSERT_OK(err, "test_xdp_adjust_tail_grow"))
 62		return;
 63
 64	err = bpf_prog_test_run_opts(prog_fd, &topts);
 65	ASSERT_OK(err, "ipv4");
 66	ASSERT_EQ(topts.retval, XDP_DROP, "ipv4 retval");
 67
 68	expect_sz = sizeof(pkt_v6) + 40; /* Test grow with 40 bytes */
 69	topts.data_in = &pkt_v6;
 70	topts.data_size_in = sizeof(pkt_v6);
 71	topts.data_size_out = sizeof(buf);
 72	err = bpf_prog_test_run_opts(prog_fd, &topts);
 73	ASSERT_OK(err, "ipv6");
 74	ASSERT_EQ(topts.retval, XDP_TX, "ipv6 retval");
 75	ASSERT_EQ(topts.data_size_out, expect_sz, "ipv6 size");
 76
 77	bpf_object__close(obj);
 78}
 79
 80static void test_xdp_adjust_tail_grow2(void)
 81{
 82	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
 83	char buf[4096]; /* avoid segfault: large buf to hold grow results */
 84	struct bpf_object *obj;
 85	int err, cnt, i;
 86	int max_grow, prog_fd;
 87	/* SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) */
 88#if defined(__s390x__)
 89	int tailroom = 512;
 90#elif defined(__powerpc__)
 91	int tailroom = 384;
 92#else
 93	int tailroom = 320;
 94#endif
 95
 96	LIBBPF_OPTS(bpf_test_run_opts, tattr,
 97		.repeat		= 1,
 98		.data_in	= &buf,
 99		.data_out	= &buf,
100		.data_size_in	= 0, /* Per test */
101		.data_size_out	= 0, /* Per test */
102	);
103
104	err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
105	if (!ASSERT_OK(err, "test_xdp_adjust_tail_grow"))
106		return;
107
108	/* Test case-64 */
109	memset(buf, 1, sizeof(buf));
110	tattr.data_size_in  =  64; /* Determine test case via pkt size */
111	tattr.data_size_out = 128; /* Limit copy_size */
112	/* Kernel side alloc packet memory area that is zero init */
113	err = bpf_prog_test_run_opts(prog_fd, &tattr);
114
115	ASSERT_EQ(errno, ENOSPC, "case-64 errno"); /* Due limit copy_size in bpf_test_finish */
116	ASSERT_EQ(tattr.retval, XDP_TX, "case-64 retval");
117	ASSERT_EQ(tattr.data_size_out, 192, "case-64 data_size_out"); /* Expected grow size */
118
119	/* Extra checks for data contents */
120	ASSERT_EQ(buf[0], 1, "case-64-data buf[0]"); /*  0-63  memset to 1 */
121	ASSERT_EQ(buf[63], 1, "case-64-data buf[63]");
122	ASSERT_EQ(buf[64], 0, "case-64-data buf[64]"); /* 64-127 memset to 0 */
123	ASSERT_EQ(buf[127], 0, "case-64-data buf[127]");
124	ASSERT_EQ(buf[128], 1, "case-64-data buf[128]"); /* 128-191 memset to 1 */
125	ASSERT_EQ(buf[191], 1, "case-64-data buf[191]");
126
127	/* Test case-128 */
128	memset(buf, 2, sizeof(buf));
129	tattr.data_size_in  = 128; /* Determine test case via pkt size */
130	tattr.data_size_out = sizeof(buf);   /* Copy everything */
131	err = bpf_prog_test_run_opts(prog_fd, &tattr);
132
133	max_grow = 4096 - XDP_PACKET_HEADROOM -	tailroom; /* 3520 */
134	ASSERT_OK(err, "case-128");
135	ASSERT_EQ(tattr.retval, XDP_TX, "case-128 retval");
136	ASSERT_EQ(tattr.data_size_out, max_grow, "case-128 data_size_out"); /* Expect max grow */
137
138	/* Extra checks for data content: Count grow size, will contain zeros */
139	for (i = 0, cnt = 0; i < sizeof(buf); i++) {
140		if (buf[i] == 0)
141			cnt++;
142	}
143	ASSERT_EQ(cnt, max_grow - tattr.data_size_in, "case-128-data cnt"); /* Grow increase */
144	ASSERT_EQ(tattr.data_size_out, max_grow, "case-128-data data_size_out"); /* Total grow */
145
146	bpf_object__close(obj);
147}
148
149static void test_xdp_adjust_frags_tail_shrink(void)
150{
151	const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
152	__u32 exp_size;
153	struct bpf_program *prog;
154	struct bpf_object *obj;
155	int err, prog_fd;
156	__u8 *buf;
157	LIBBPF_OPTS(bpf_test_run_opts, topts);
158
159	/* For the individual test cases, the first byte in the packet
160	 * indicates which test will be run.
161	 */
162	obj = bpf_object__open(file);
163	if (libbpf_get_error(obj))
164		return;
165
166	prog = bpf_object__next_program(obj, NULL);
167	if (bpf_object__load(obj))
168		return;
169
170	prog_fd = bpf_program__fd(prog);
171
172	buf = malloc(9000);
173	if (!ASSERT_OK_PTR(buf, "alloc buf 9Kb"))
174		goto out;
175
176	memset(buf, 0, 9000);
177
178	/* Test case removing 10 bytes from last frag, NOT freeing it */
179	exp_size = 8990; /* 9000 - 10 */
180	topts.data_in = buf;
181	topts.data_out = buf;
182	topts.data_size_in = 9000;
183	topts.data_size_out = 9000;
184	err = bpf_prog_test_run_opts(prog_fd, &topts);
185
186	ASSERT_OK(err, "9Kb-10b");
187	ASSERT_EQ(topts.retval, XDP_TX, "9Kb-10b retval");
188	ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-10b size");
189
190	/* Test case removing one of two pages, assuming 4K pages */
191	buf[0] = 1;
192	exp_size = 4900; /* 9000 - 4100 */
193
194	topts.data_size_out = 9000; /* reset from previous invocation */
195	err = bpf_prog_test_run_opts(prog_fd, &topts);
196
197	ASSERT_OK(err, "9Kb-4Kb");
198	ASSERT_EQ(topts.retval, XDP_TX, "9Kb-4Kb retval");
199	ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-4Kb size");
200
201	/* Test case removing two pages resulting in a linear xdp_buff */
202	buf[0] = 2;
203	exp_size = 800; /* 9000 - 8200 */
204	topts.data_size_out = 9000; /* reset from previous invocation */
205	err = bpf_prog_test_run_opts(prog_fd, &topts);
206
207	ASSERT_OK(err, "9Kb-9Kb");
208	ASSERT_EQ(topts.retval, XDP_TX, "9Kb-9Kb retval");
209	ASSERT_EQ(topts.data_size_out, exp_size, "9Kb-9Kb size");
210
211	free(buf);
212out:
213	bpf_object__close(obj);
214}
215
216static void test_xdp_adjust_frags_tail_grow_4k(void)
217{
218	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
219	__u32 exp_size;
220	struct bpf_program *prog;
221	struct bpf_object *obj;
222	int err, i, prog_fd;
223	__u8 *buf;
224	LIBBPF_OPTS(bpf_test_run_opts, topts);
225
226	obj = bpf_object__open(file);
227	if (libbpf_get_error(obj))
228		return;
229
230	prog = bpf_object__next_program(obj, NULL);
231	if (bpf_object__load(obj))
232		goto out;
233
234	prog_fd = bpf_program__fd(prog);
235
236	buf = malloc(16384);
237	if (!ASSERT_OK_PTR(buf, "alloc buf 16Kb"))
238		goto out;
239
240	/* Test case add 10 bytes to last frag */
241	memset(buf, 1, 16384);
242	exp_size = 9000 + 10;
243
244	topts.data_in = buf;
245	topts.data_out = buf;
246	topts.data_size_in = 9000;
247	topts.data_size_out = 16384;
248	err = bpf_prog_test_run_opts(prog_fd, &topts);
249
250	ASSERT_OK(err, "9Kb+10b");
251	ASSERT_EQ(topts.retval, XDP_TX, "9Kb+10b retval");
252	ASSERT_EQ(topts.data_size_out, exp_size, "9Kb+10b size");
253
254	for (i = 0; i < 9000; i++) {
255		if (buf[i] != 1)
256			ASSERT_EQ(buf[i], 1, "9Kb+10b-old");
257	}
258
259	for (i = 9000; i < 9010; i++) {
260		if (buf[i] != 0)
261			ASSERT_EQ(buf[i], 0, "9Kb+10b-new");
262	}
263
264	for (i = 9010; i < 16384; i++) {
265		if (buf[i] != 1)
266			ASSERT_EQ(buf[i], 1, "9Kb+10b-untouched");
267	}
268
269	/* Test a too large grow */
270	memset(buf, 1, 16384);
271	exp_size = 9001;
272
273	topts.data_in = topts.data_out = buf;
274	topts.data_size_in = 9001;
275	topts.data_size_out = 16384;
276	err = bpf_prog_test_run_opts(prog_fd, &topts);
277
278	ASSERT_OK(err, "9Kb+10b");
279	ASSERT_EQ(topts.retval, XDP_DROP, "9Kb+10b retval");
280	ASSERT_EQ(topts.data_size_out, exp_size, "9Kb+10b size");
281
282	free(buf);
283out:
284	bpf_object__close(obj);
285}
286
287static void test_xdp_adjust_frags_tail_grow_64k(void)
288{
289	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
290	__u32 exp_size;
291	struct bpf_program *prog;
292	struct bpf_object *obj;
293	int err, i, prog_fd;
294	__u8 *buf;
295	LIBBPF_OPTS(bpf_test_run_opts, topts);
296
297	obj = bpf_object__open(file);
298	if (libbpf_get_error(obj))
299		return;
300
301	prog = bpf_object__next_program(obj, NULL);
302	if (bpf_object__load(obj))
303		goto out;
304
305	prog_fd = bpf_program__fd(prog);
306
307	buf = malloc(262144);
308	if (!ASSERT_OK_PTR(buf, "alloc buf 256Kb"))
309		goto out;
310
311	/* Test case add 10 bytes to last frag */
312	memset(buf, 1, 262144);
313	exp_size = 90000 + 10;
314
315	topts.data_in = buf;
316	topts.data_out = buf;
317	topts.data_size_in = 90000;
318	topts.data_size_out = 262144;
319	err = bpf_prog_test_run_opts(prog_fd, &topts);
320
321	ASSERT_OK(err, "90Kb+10b");
322	ASSERT_EQ(topts.retval, XDP_TX, "90Kb+10b retval");
323	ASSERT_EQ(topts.data_size_out, exp_size, "90Kb+10b size");
324
325	for (i = 0; i < 90000; i++) {
326		if (buf[i] != 1)
327			ASSERT_EQ(buf[i], 1, "90Kb+10b-old");
328	}
329
330	for (i = 90000; i < 90010; i++) {
331		if (buf[i] != 0)
332			ASSERT_EQ(buf[i], 0, "90Kb+10b-new");
333	}
334
335	for (i = 90010; i < 262144; i++) {
336		if (buf[i] != 1)
337			ASSERT_EQ(buf[i], 1, "90Kb+10b-untouched");
338	}
339
340	/* Test a too large grow */
341	memset(buf, 1, 262144);
342	exp_size = 90001;
343
344	topts.data_in = topts.data_out = buf;
345	topts.data_size_in = 90001;
346	topts.data_size_out = 262144;
347	err = bpf_prog_test_run_opts(prog_fd, &topts);
348
349	ASSERT_OK(err, "90Kb+10b");
350	ASSERT_EQ(topts.retval, XDP_DROP, "90Kb+10b retval");
351	ASSERT_EQ(topts.data_size_out, exp_size, "90Kb+10b size");
352
353	free(buf);
354out:
355	bpf_object__close(obj);
356}
357
358void test_xdp_adjust_tail(void)
359{
360	int page_size = getpagesize();
361
362	if (test__start_subtest("xdp_adjust_tail_shrink"))
363		test_xdp_adjust_tail_shrink();
364	if (test__start_subtest("xdp_adjust_tail_grow"))
365		test_xdp_adjust_tail_grow(page_size == 65536);
366	if (test__start_subtest("xdp_adjust_tail_grow2"))
367		test_xdp_adjust_tail_grow2();
368	if (test__start_subtest("xdp_adjust_frags_tail_shrink"))
369		test_xdp_adjust_frags_tail_shrink();
370	if (test__start_subtest("xdp_adjust_frags_tail_grow")) {
371		if (page_size == 65536)
372			test_xdp_adjust_frags_tail_grow_64k();
373		else
374			test_xdp_adjust_frags_tail_grow_4k();
375	}
376}