xskxceiver.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2020 Intel Corporation. */
  3
  4/*
  5 * Some functions in this program are taken from
  6 * Linux kernel samples/bpf/xdpsock* and modified
  7 * for use.
  8 *
  9 * See test_xsk.sh for detailed information on test topology
 10 * and prerequisite network setup.
 11 *
 12 * This test program contains two threads, each thread is single socket with
 13 * a unique UMEM. It validates in-order packet delivery and packet content
 14 * by sending packets to each other.
 15 *
 16 * Tests Information:
 17 * ------------------
 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
 19 * Virtual Ethernet interfaces.
 20 *
 21 * For each mode, the following tests are run:
 22 *    a. nopoll - soft-irq processing in run-to-completion mode
 23 *    b. poll - using poll() syscall
 24 *    c. Socket Teardown
 25 *       Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
 26 *       both sockets, then repeat multiple times. Only nopoll mode is used
 27 *    d. Bi-directional sockets
 28 *       Configure sockets as bi-directional tx/rx sockets, sets up fill and
 29 *       completion rings on each socket, tx/rx in both directions. Only nopoll
 30 *       mode is used
 31 *    e. Statistics
 32 *       Trigger some error conditions and ensure that the appropriate statistics
 33 *       are incremented. Within this test, the following statistics are tested:
 34 *       i.   rx dropped
 35 *            Increase the UMEM frame headroom to a value which results in
 36 *            insufficient space in the rx buffer for both the packet and the headroom.
 37 *       ii.  tx invalid
 38 *            Set the 'len' field of tx descriptors to an invalid value (umem frame
 39 *            size + 1).
 40 *       iii. rx ring full
 41 *            Reduce the size of the RX ring to a fraction of the fill ring size.
 42 *       iv.  fill queue empty
 43 *            Do not populate the fill queue and then try to receive pkts.
 44 *    f. bpf_link resource persistence
 45 *       Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
 46 *       then remove xsk sockets from queue 0 on both veth interfaces and
 47 *       finally run a traffic on queues ids 1
 48 *    g. unaligned mode
 49 *    h. tests for invalid and corner case Tx descriptors so that the correct ones
 50 *       are discarded and let through, respectively.
 51 *    i. 2K frame size tests
 52 *    j. If multi-buffer is supported, send 9k packets divided into 3 frames
 53 *    k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
 54 *       using unaligned mode
 55 *    l. If multi-buffer is supported, try various nasty combinations of descriptors to
 56 *       check if they pass the validation or not
 57 *
 58 * Flow:
 59 * -----
 60 * - Single process spawns two threads: Tx and Rx
 61 * - Each of these two threads attach to a veth interface
 62 * - Each thread creates one AF_XDP socket connected to a unique umem for each
 63 *   veth interface
 64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
 65 * - Rx thread verifies if all packets were received and delivered in-order,
 66 *   and have the right content
 67 *
 68 * Enable/disable packet dump mode:
 69 * --------------------------
 70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
 71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
 72 */
 73
 74#define _GNU_SOURCE
 75#include <assert.h>
 76#include <fcntl.h>
 77#include <getopt.h>
 78#include <linux/if_link.h>
 79#include <linux/if_ether.h>
 80#include <linux/mman.h>
 81#include <linux/netdev.h>
 82#include <linux/ethtool.h>
 83#include <arpa/inet.h>
 84#include <net/if.h>
 85#include <locale.h>
 86#include <stdio.h>
 87#include <stdlib.h>
 88#include <libgen.h>
 89#include <stddef.h>
 90#include <sys/mman.h>
 91#include <sys/types.h>
 92
 93#include "prog_tests/test_xsk.h"
 94#include "xsk_xdp_progs.skel.h"
 95#include "xsk.h"
 96#include "xskxceiver.h"
 97#include <bpf/bpf.h>
 98#include <linux/filter.h>
 99#include "kselftest.h"
100#include "xsk_xdp_common.h"
101
102#include <network_helpers.h>
103
104static bool opt_print_tests;
105static enum test_mode opt_mode = TEST_MODE_ALL;
106static u32 opt_run_test = RUN_ALL_TESTS;
107
108void test__fail(void) { /* for network_helpers.c */ }
109
110static void __exit_with_error(int error, const char *file, const char *func, int line)
111{
112	ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line,
113			      error, strerror(error));
114	ksft_exit_xfail();
115}
116
117#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
118
119static bool ifobj_zc_avail(struct ifobject *ifobject)
120{
121	size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
122	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
123	struct xsk_socket_info *xsk;
124	struct xsk_umem_info *umem;
125	bool zc_avail = false;
126	void *bufs;
127	int ret;
128
129	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
130	if (bufs == MAP_FAILED)
131		exit_with_error(errno);
132
133	umem = calloc(1, sizeof(struct xsk_umem_info));
134	if (!umem) {
135		munmap(bufs, umem_sz);
136		exit_with_error(ENOMEM);
137	}
138	umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
139	ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
140	if (ret)
141		exit_with_error(-ret);
142
143	xsk = calloc(1, sizeof(struct xsk_socket_info));
144	if (!xsk)
145		goto out;
146	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
147	ifobject->rx_on = true;
148	xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
149	ret = xsk_configure_socket(xsk, umem, ifobject, false);
150	if (!ret)
151		zc_avail = true;
152
153	xsk_socket__delete(xsk->xsk);
154	free(xsk);
155out:
156	munmap(umem->buffer, umem_sz);
157	xsk_umem__delete(umem->umem);
158	free(umem);
159	return zc_avail;
160}
161
162static struct option long_options[] = {
163	{"interface", required_argument, 0, 'i'},
164	{"busy-poll", no_argument, 0, 'b'},
165	{"verbose", no_argument, 0, 'v'},
166	{"mode", required_argument, 0, 'm'},
167	{"list", no_argument, 0, 'l'},
168	{"test", required_argument, 0, 't'},
169	{"help", no_argument, 0, 'h'},
170	{0, 0, 0, 0}
171};
172
173static void print_usage(char **argv)
174{
175	const char *str =
176		"  Usage: xskxceiver [OPTIONS]\n"
177		"  Options:\n"
178		"  -i, --interface      Use interface\n"
179		"  -v, --verbose        Verbose output\n"
180		"  -b, --busy-poll      Enable busy poll\n"
181		"  -m, --mode           Run only mode skb, drv, or zc\n"
182		"  -l, --list           List all available tests\n"
183		"  -t, --test           Run a specific test. Enter number from -l option.\n"
184		"  -h, --help           Display this help and exit\n";
185
186	ksft_print_msg(str, basename(argv[0]));
187	ksft_exit_xfail();
188}
189
190static bool validate_interface(struct ifobject *ifobj)
191{
192	if (!strcmp(ifobj->ifname, ""))
193		return false;
194	return true;
195}
196
197static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
198			       char **argv)
199{
200	struct ifobject *ifobj;
201	u32 interface_nb = 0;
202	int option_index, c;
203
204	opterr = 0;
205
206	for (;;) {
207		c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
208		if (c == -1)
209			break;
210
211		switch (c) {
212		case 'i':
213			if (interface_nb == 0)
214				ifobj = ifobj_tx;
215			else if (interface_nb == 1)
216				ifobj = ifobj_rx;
217			else
218				break;
219
220			memcpy(ifobj->ifname, optarg,
221			       min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
222
223			ifobj->ifindex = if_nametoindex(ifobj->ifname);
224			if (!ifobj->ifindex)
225				exit_with_error(errno);
226
227			interface_nb++;
228			break;
229		case 'v':
230			opt_verbose = true;
231			break;
232		case 'b':
233			ifobj_tx->busy_poll = true;
234			ifobj_rx->busy_poll = true;
235			break;
236		case 'm':
237			if (!strncmp("skb", optarg, strlen(optarg)))
238				opt_mode = TEST_MODE_SKB;
239			else if (!strncmp("drv", optarg, strlen(optarg)))
240				opt_mode = TEST_MODE_DRV;
241			else if (!strncmp("zc", optarg, strlen(optarg)))
242				opt_mode = TEST_MODE_ZC;
243			else
244				print_usage(argv);
245			break;
246		case 'l':
247			opt_print_tests = true;
248			break;
249		case 't':
250			errno = 0;
251			opt_run_test = strtol(optarg, NULL, 0);
252			if (errno)
253				print_usage(argv);
254			break;
255		case 'h':
256		default:
257			print_usage(argv);
258		}
259	}
260}
261
262static void xsk_unload_xdp_programs(struct ifobject *ifobj)
263{
264	xsk_xdp_progs__destroy(ifobj->xdp_progs);
265}
266
267static void run_pkt_test(struct test_spec *test)
268{
269	int ret;
270
271	ret = test->test_func(test);
272
273	switch (ret) {
274	case TEST_PASS:
275		ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
276				      test->name);
277		break;
278	case TEST_SKIP:
279		ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test),
280				      test->name);
281		break;
282	case TEST_FAILURE:
283		ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
284				      test->name);
285		break;
286	default:
287		ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n",
288				      mode_string(test), busy_poll_string(test), test->name, ret);
289	}
290
291	pkt_stream_restore_default(test);
292}
293
294static bool is_xdp_supported(int ifindex)
295{
296	int flags = XDP_FLAGS_DRV_MODE;
297
298	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
299	struct bpf_insn insns[2] = {
300		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
301		BPF_EXIT_INSN()
302	};
303	int prog_fd, insn_cnt = ARRAY_SIZE(insns);
304	int err;
305
306	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
307	if (prog_fd < 0)
308		return false;
309
310	err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
311	if (err) {
312		close(prog_fd);
313		return false;
314	}
315
316	bpf_xdp_detach(ifindex, flags, NULL);
317	close(prog_fd);
318
319	return true;
320}
321
322static void print_tests(void)
323{
324	u32 i;
325
326	printf("Tests:\n");
327	for (i = 0; i < ARRAY_SIZE(tests); i++)
328		printf("%u: %s\n", i, tests[i].name);
329	for (i = ARRAY_SIZE(tests); i < ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); i++)
330		printf("%u: %s\n", i, ci_skip_tests[i - ARRAY_SIZE(tests)].name);
331}
332
333int main(int argc, char **argv)
334{
335	const size_t total_tests = ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests);
336	struct pkt_stream *rx_pkt_stream_default;
337	struct pkt_stream *tx_pkt_stream_default;
338	struct ifobject *ifobj_tx, *ifobj_rx;
339	u32 i, j, failed_tests = 0, nb_tests;
340	int modes = TEST_MODE_SKB + 1;
341	struct test_spec test;
342	bool shared_netdev;
343	int ret;
344
345	/* Use libbpf 1.0 API mode */
346	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
347
348	ifobj_tx = ifobject_create();
349	if (!ifobj_tx)
350		exit_with_error(ENOMEM);
351	ifobj_rx = ifobject_create();
352	if (!ifobj_rx)
353		exit_with_error(ENOMEM);
354
355	setlocale(LC_ALL, "");
356
357	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
358
359	if (opt_print_tests) {
360		print_tests();
361		ksft_exit_xpass();
362	}
363	if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= total_tests) {
364		ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
365		ksft_exit_xfail();
366	}
367
368	shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
369	ifobj_tx->shared_umem = shared_netdev;
370	ifobj_rx->shared_umem = shared_netdev;
371
372	if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
373		print_usage(argv);
374
375	if (is_xdp_supported(ifobj_tx->ifindex)) {
376		modes++;
377		if (ifobj_zc_avail(ifobj_tx))
378			modes++;
379	}
380
381	ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
382	if (!ret) {
383		ifobj_tx->hw_ring_size_supp = true;
384		ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;
385		ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;
386	}
387
388	if (init_iface(ifobj_rx, worker_testapp_validate_rx) ||
389	    init_iface(ifobj_tx, worker_testapp_validate_tx)) {
390		ksft_print_msg("Error : can't initialize interfaces\n");
391		ksft_exit_xfail();
392	}
393
394	test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
395	tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
396	rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
397	if (!tx_pkt_stream_default || !rx_pkt_stream_default)
398		exit_with_error(ENOMEM);
399	test.tx_pkt_stream_default = tx_pkt_stream_default;
400	test.rx_pkt_stream_default = rx_pkt_stream_default;
401
402	if (opt_run_test == RUN_ALL_TESTS)
403		nb_tests = total_tests;
404	else
405		nb_tests = 1;
406	if (opt_mode == TEST_MODE_ALL) {
407		ksft_set_plan(modes * nb_tests);
408	} else {
409		if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
410			ksft_print_msg("Error: XDP_DRV mode not supported.\n");
411			ksft_exit_xfail();
412		}
413		if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
414			ksft_print_msg("Error: zero-copy mode not supported.\n");
415			ksft_exit_xfail();
416		}
417
418		ksft_set_plan(nb_tests);
419	}
420
421	for (i = 0; i < modes; i++) {
422		if (opt_mode != TEST_MODE_ALL && i != opt_mode)
423			continue;
424
425		for (j = 0; j < total_tests; j++) {
426			if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
427				continue;
428
429			if (j < ARRAY_SIZE(tests))
430				test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
431			else
432				test_init(&test, ifobj_tx, ifobj_rx, i,
433					  &ci_skip_tests[j - ARRAY_SIZE(tests)]);
434			run_pkt_test(&test);
435			usleep(USLEEP_MAX);
436
437			if (test.fail)
438				failed_tests++;
439		}
440	}
441
442	if (ifobj_tx->hw_ring_size_supp)
443		hw_ring_size_reset(ifobj_tx);
444
445	pkt_stream_delete(tx_pkt_stream_default);
446	pkt_stream_delete(rx_pkt_stream_default);
447	xsk_unload_xdp_programs(ifobj_tx);
448	xsk_unload_xdp_programs(ifobj_rx);
449	ifobject_delete(ifobj_tx);
450	ifobject_delete(ifobj_rx);
451
452	if (failed_tests)
453		ksft_exit_fail();
454	else
455		ksft_exit_pass();
456}