io_uring_zerocopy_tx.c

  1/* SPDX-License-Identifier: MIT */
  2/* based on linux-kernel/tools/testing/selftests/net/msg_zerocopy.c */
  3#include <assert.h>
  4#include <errno.h>
  5#include <error.h>
  6#include <fcntl.h>
  7#include <limits.h>
  8#include <stdbool.h>
  9#include <stdint.h>
 10#include <stdio.h>
 11#include <stdlib.h>
 12#include <string.h>
 13#include <unistd.h>
 14
 15#include <arpa/inet.h>
 16#include <linux/errqueue.h>
 17#include <linux/if_packet.h>
 18#include <linux/io_uring.h>
 19#include <linux/ipv6.h>
 20#include <linux/socket.h>
 21#include <linux/sockios.h>
 22#include <net/ethernet.h>
 23#include <net/if.h>
 24#include <netinet/in.h>
 25#include <netinet/ip.h>
 26#include <netinet/ip6.h>
 27#include <netinet/tcp.h>
 28#include <netinet/udp.h>
 29#include <sys/ioctl.h>
 30#include <sys/mman.h>
 31#include <sys/resource.h>
 32#include <sys/socket.h>
 33#include <sys/stat.h>
 34#include <sys/time.h>
 35#include <sys/types.h>
 36#include <sys/un.h>
 37#include <sys/wait.h>
 38
 39#include <io_uring/mini_liburing.h>
 40
 41#define NOTIF_TAG 0xfffffffULL
 42#define NONZC_TAG 0
 43#define ZC_TAG 1
 44
 45enum {
 46	MODE_NONZC	= 0,
 47	MODE_ZC		= 1,
 48	MODE_ZC_FIXED	= 2,
 49	MODE_MIXED	= 3,
 50};
 51
 52static bool cfg_cork		= false;
 53static int  cfg_mode		= MODE_ZC_FIXED;
 54static int  cfg_nr_reqs		= 8;
 55static int  cfg_family		= PF_UNSPEC;
 56static int  cfg_payload_len;
 57static int  cfg_port		= 8000;
 58static int  cfg_runtime_ms	= 4200;
 59
 60static socklen_t cfg_alen;
 61static struct sockaddr_storage cfg_dst_addr;
 62
 63static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));
 64
 65static unsigned long gettimeofday_ms(void)
 66{
 67	struct timeval tv;
 68
 69	gettimeofday(&tv, NULL);
 70	return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
 71}
 72
 73static void do_setsockopt(int fd, int level, int optname, int val)
 74{
 75	if (setsockopt(fd, level, optname, &val, sizeof(val)))
 76		error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
 77}
 78
 79static int do_setup_tx(int domain, int type, int protocol)
 80{
 81	int fd;
 82
 83	fd = socket(domain, type, protocol);
 84	if (fd == -1)
 85		error(1, errno, "socket t");
 86
 87	do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21);
 88
 89	if (connect(fd, (void *) &cfg_dst_addr, cfg_alen))
 90		error(1, errno, "connect");
 91	return fd;
 92}
 93
 94static void do_tx(int domain, int type, int protocol)
 95{
 96	struct io_uring_sqe *sqe;
 97	struct io_uring_cqe *cqe;
 98	unsigned long packets = 0, bytes = 0;
 99	struct io_uring ring;
100	struct iovec iov;
101	uint64_t tstop;
102	int i, fd, ret;
103	int compl_cqes = 0;
104
105	fd = do_setup_tx(domain, type, protocol);
106
107	ret = io_uring_queue_init(512, &ring, 0);
108	if (ret)
109		error(1, -ret, "io_uring: queue init");
110
111	iov.iov_base = payload;
112	iov.iov_len = cfg_payload_len;
113
114	ret = io_uring_register_buffers(&ring, &iov, 1);
115	if (ret)
116		error(1, -ret, "io_uring: buffer registration");
117
118	tstop = gettimeofday_ms() + cfg_runtime_ms;
119	do {
120		if (cfg_cork)
121			do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 1);
122
123		for (i = 0; i < cfg_nr_reqs; i++) {
124			unsigned zc_flags = 0;
125			unsigned buf_idx = 0;
126			unsigned mode = cfg_mode;
127			unsigned msg_flags = MSG_WAITALL;
128
129			if (cfg_mode == MODE_MIXED)
130				mode = rand() % 3;
131
132			sqe = io_uring_get_sqe(&ring);
133
134			if (mode == MODE_NONZC) {
135				io_uring_prep_send(sqe, fd, payload,
136						   cfg_payload_len, msg_flags);
137				sqe->user_data = NONZC_TAG;
138			} else {
139				io_uring_prep_sendzc(sqe, fd, payload,
140						     cfg_payload_len,
141						     msg_flags, zc_flags);
142				if (mode == MODE_ZC_FIXED) {
143					sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
144					sqe->buf_index = buf_idx;
145				}
146				sqe->user_data = ZC_TAG;
147			}
148		}
149
150		ret = io_uring_submit(&ring);
151		if (ret != cfg_nr_reqs)
152			error(1, -ret, "submit");
153
154		if (cfg_cork)
155			do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0);
156		for (i = 0; i < cfg_nr_reqs; i++) {
157			ret = io_uring_wait_cqe(&ring, &cqe);
158			if (ret)
159				error(1, -ret, "wait cqe");
160
161			if (cqe->user_data != NONZC_TAG &&
162			    cqe->user_data != ZC_TAG)
163				error(1, EINVAL, "invalid cqe->user_data");
164
165			if (cqe->flags & IORING_CQE_F_NOTIF) {
166				if (cqe->flags & IORING_CQE_F_MORE)
167					error(1, EINVAL, "invalid notif flags");
168				if (compl_cqes <= 0)
169					error(1, EINVAL, "notification mismatch");
170				compl_cqes--;
171				i--;
172				io_uring_cqe_seen(&ring);
173				continue;
174			}
175			if (cqe->flags & IORING_CQE_F_MORE) {
176				if (cqe->user_data != ZC_TAG)
177					error(1, -cqe->res, "unexpected F_MORE");
178				compl_cqes++;
179			}
180			if (cqe->res >= 0) {
181				packets++;
182				bytes += cqe->res;
183			} else if (cqe->res != -EAGAIN) {
184				error(1, -cqe->res, "send failed");
185			}
186			io_uring_cqe_seen(&ring);
187		}
188	} while (gettimeofday_ms() < tstop);
189
190	while (compl_cqes) {
191		ret = io_uring_wait_cqe(&ring, &cqe);
192		if (ret)
193			error(1, -ret, "wait cqe");
194		if (cqe->flags & IORING_CQE_F_MORE)
195			error(1, EINVAL, "invalid notif flags");
196		if (!(cqe->flags & IORING_CQE_F_NOTIF))
197			error(1, EINVAL, "missing notif flag");
198
199		io_uring_cqe_seen(&ring);
200		compl_cqes--;
201	}
202
203	fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
204			packets, bytes >> 20,
205			packets / (cfg_runtime_ms / 1000),
206			(bytes >> 20) / (cfg_runtime_ms / 1000));
207
208	if (close(fd))
209		error(1, errno, "close");
210}
211
212static void do_test(int domain, int type, int protocol)
213{
214	int i;
215
216	for (i = 0; i < IP_MAXPACKET; i++)
217		payload[i] = 'a' + (i % 26);
218	do_tx(domain, type, protocol);
219}
220
221static void usage(const char *filepath)
222{
223	error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] "
224		    "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath);
225}
226
227static void parse_opts(int argc, char **argv)
228{
229	const int max_payload_len = sizeof(payload) -
230				    sizeof(struct ipv6hdr) -
231				    sizeof(struct tcphdr) -
232				    40 /* max tcp options */;
233	struct sockaddr_in6 *addr6 = (void *) &cfg_dst_addr;
234	struct sockaddr_in *addr4 = (void *) &cfg_dst_addr;
235	char *daddr = NULL;
236	int c;
237
238	if (argc <= 1)
239		usage(argv[0]);
240	cfg_payload_len = max_payload_len;
241
242	while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) {
243		switch (c) {
244		case '4':
245			if (cfg_family != PF_UNSPEC)
246				error(1, 0, "Pass one of -4 or -6");
247			cfg_family = PF_INET;
248			cfg_alen = sizeof(struct sockaddr_in);
249			break;
250		case '6':
251			if (cfg_family != PF_UNSPEC)
252				error(1, 0, "Pass one of -4 or -6");
253			cfg_family = PF_INET6;
254			cfg_alen = sizeof(struct sockaddr_in6);
255			break;
256		case 'D':
257			daddr = optarg;
258			break;
259		case 'p':
260			cfg_port = strtoul(optarg, NULL, 0);
261			break;
262		case 's':
263			cfg_payload_len = strtoul(optarg, NULL, 0);
264			break;
265		case 't':
266			cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000;
267			break;
268		case 'n':
269			cfg_nr_reqs = strtoul(optarg, NULL, 0);
270			break;
271		case 'c':
272			cfg_cork = strtol(optarg, NULL, 0);
273			break;
274		case 'm':
275			cfg_mode = strtol(optarg, NULL, 0);
276			break;
277		}
278	}
279
280	switch (cfg_family) {
281	case PF_INET:
282		memset(addr4, 0, sizeof(*addr4));
283		addr4->sin_family = AF_INET;
284		addr4->sin_port = htons(cfg_port);
285		if (daddr &&
286		    inet_pton(AF_INET, daddr, &(addr4->sin_addr)) != 1)
287			error(1, 0, "ipv4 parse error: %s", daddr);
288		break;
289	case PF_INET6:
290		memset(addr6, 0, sizeof(*addr6));
291		addr6->sin6_family = AF_INET6;
292		addr6->sin6_port = htons(cfg_port);
293		if (daddr &&
294		    inet_pton(AF_INET6, daddr, &(addr6->sin6_addr)) != 1)
295			error(1, 0, "ipv6 parse error: %s", daddr);
296		break;
297	default:
298		error(1, 0, "illegal domain");
299	}
300
301	if (cfg_payload_len > max_payload_len)
302		error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
303	if (optind != argc - 1)
304		usage(argv[0]);
305}
306
307int main(int argc, char **argv)
308{
309	const char *cfg_test = argv[argc - 1];
310
311	parse_opts(argc, argv);
312
313	if (!strcmp(cfg_test, "tcp"))
314		do_test(cfg_family, SOCK_STREAM, 0);
315	else if (!strcmp(cfg_test, "udp"))
316		do_test(cfg_family, SOCK_DGRAM, 0);
317	else
318		error(1, 0, "unknown cfg_test %s", cfg_test);
319	return 0;
320}