ncdevmem.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP
   4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider.
   5 *
   6 * Usage:
   7 *
   8 *     On server:
   9 *     ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201
  10 *
  11 *     On client:
  12 *     echo -n "hello\nworld" | \
  13 *		ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
  14 *
  15 * Note this is compatible with regular netcat. i.e. the sender or receiver can
  16 * be replaced with regular netcat to test the RX or TX path in isolation.
  17 *
  18 * Test data validation (devmem TCP on RX only):
  19 *
  20 *     On server:
  21 *     ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7
  22 *
  23 *     On client:
  24 *     yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \
  25 *             head -c 1G | \
  26 *             nc <server IP> 5201 -p 5201
  27 *
  28 * Test data validation (devmem TCP on RX and TX, validation happens on RX):
  29 *
  30 *	On server:
  31 *	ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1
  32 *
  33 *	On client:
  34 *	yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \
  35 *		head -c 1M | \
  36 *		ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
  37 */
  38#define _GNU_SOURCE
  39#define __EXPORTED_HEADERS__
  40
  41#include <linux/uio.h>
  42#include <stdarg.h>
  43#include <stdio.h>
  44#include <stdlib.h>
  45#include <unistd.h>
  46#include <stdbool.h>
  47#include <string.h>
  48#include <errno.h>
  49#define __iovec_defined
  50#include <fcntl.h>
  51#include <malloc.h>
  52#include <error.h>
  53#include <poll.h>
  54
  55#include <arpa/inet.h>
  56#include <sys/socket.h>
  57#include <sys/mman.h>
  58#include <sys/ioctl.h>
  59#include <sys/syscall.h>
  60#include <sys/time.h>
  61
  62#include <linux/memfd.h>
  63#include <linux/dma-buf.h>
  64#include <linux/errqueue.h>
  65#include <linux/udmabuf.h>
  66#include <linux/types.h>
  67#include <linux/netlink.h>
  68#include <linux/genetlink.h>
  69#include <linux/netdev.h>
  70#include <linux/ethtool_netlink.h>
  71#include <time.h>
  72#include <net/if.h>
  73
  74#include "netdev-user.h"
  75#include "ethtool-user.h"
  76#include <ynl.h>
  77
  78#define PAGE_SHIFT 12
  79#define TEST_PREFIX "ncdevmem"
  80#define NUM_PAGES 16000
  81
  82#ifndef MSG_SOCK_DEVMEM
  83#define MSG_SOCK_DEVMEM 0x2000000
  84#endif
  85
  86#define MAX_IOV 1024
  87
  88static size_t max_chunk;
  89static char *server_ip;
  90static char *client_ip;
  91static char *port;
  92static size_t do_validation;
  93static int start_queue = -1;
  94static int num_queues = -1;
  95static char *ifname;
  96static unsigned int ifindex;
  97static unsigned int dmabuf_id;
  98static uint32_t tx_dmabuf_id;
  99static int waittime_ms = 500;
 100
 101/* System state loaded by current_config_load() */
 102#define MAX_FLOWS	8
 103static int ntuple_ids[MAX_FLOWS] = { -1, -1, -1, -1, -1, -1, -1, -1, };
 104
 105struct memory_buffer {
 106	int fd;
 107	size_t size;
 108
 109	int devfd;
 110	int memfd;
 111	char *buf_mem;
 112};
 113
 114struct memory_provider {
 115	struct memory_buffer *(*alloc)(size_t size);
 116	void (*free)(struct memory_buffer *ctx);
 117	void (*memcpy_to_device)(struct memory_buffer *dst, size_t off,
 118				 void *src, int n);
 119	void (*memcpy_from_device)(void *dst, struct memory_buffer *src,
 120				   size_t off, int n);
 121};
 122
 123static void pr_err(const char *fmt, ...)
 124{
 125	va_list args;
 126
 127	fprintf(stderr, "%s: ", TEST_PREFIX);
 128
 129	va_start(args, fmt);
 130	vfprintf(stderr, fmt, args);
 131	va_end(args);
 132
 133	if (errno != 0)
 134		fprintf(stderr, ": %s", strerror(errno));
 135	fprintf(stderr, "\n");
 136}
 137
 138static struct memory_buffer *udmabuf_alloc(size_t size)
 139{
 140	struct udmabuf_create create;
 141	struct memory_buffer *ctx;
 142	int ret;
 143
 144	ctx = malloc(sizeof(*ctx));
 145	if (!ctx)
 146		return NULL;
 147
 148	ctx->size = size;
 149
 150	ctx->devfd = open("/dev/udmabuf", O_RDWR);
 151	if (ctx->devfd < 0) {
 152		pr_err("[skip,no-udmabuf: Unable to access DMA buffer device file]");
 153		goto err_free_ctx;
 154	}
 155
 156	ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING);
 157	if (ctx->memfd < 0) {
 158		pr_err("[skip,no-memfd]");
 159		goto err_close_dev;
 160	}
 161
 162	ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK);
 163	if (ret < 0) {
 164		pr_err("[skip,fcntl-add-seals]");
 165		goto err_close_memfd;
 166	}
 167
 168	ret = ftruncate(ctx->memfd, size);
 169	if (ret == -1) {
 170		pr_err("[FAIL,memfd-truncate]");
 171		goto err_close_memfd;
 172	}
 173
 174	memset(&create, 0, sizeof(create));
 175
 176	create.memfd = ctx->memfd;
 177	create.offset = 0;
 178	create.size = size;
 179	ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create);
 180	if (ctx->fd < 0) {
 181		pr_err("[FAIL, create udmabuf]");
 182		goto err_close_fd;
 183	}
 184
 185	ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
 186			    ctx->fd, 0);
 187	if (ctx->buf_mem == MAP_FAILED) {
 188		pr_err("[FAIL, map udmabuf]");
 189		goto err_close_fd;
 190	}
 191
 192	return ctx;
 193
 194err_close_fd:
 195	close(ctx->fd);
 196err_close_memfd:
 197	close(ctx->memfd);
 198err_close_dev:
 199	close(ctx->devfd);
 200err_free_ctx:
 201	free(ctx);
 202	return NULL;
 203}
 204
 205static void udmabuf_free(struct memory_buffer *ctx)
 206{
 207	munmap(ctx->buf_mem, ctx->size);
 208	close(ctx->fd);
 209	close(ctx->memfd);
 210	close(ctx->devfd);
 211	free(ctx);
 212}
 213
 214static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off,
 215				     void *src, int n)
 216{
 217	struct dma_buf_sync sync = {};
 218
 219	sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE;
 220	ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
 221
 222	memcpy(dst->buf_mem + off, src, n);
 223
 224	sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE;
 225	ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
 226}
 227
 228static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src,
 229				       size_t off, int n)
 230{
 231	struct dma_buf_sync sync = {};
 232
 233	sync.flags = DMA_BUF_SYNC_START;
 234	ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
 235
 236	memcpy(dst, src->buf_mem + off, n);
 237
 238	sync.flags = DMA_BUF_SYNC_END;
 239	ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
 240}
 241
 242static struct memory_provider udmabuf_memory_provider = {
 243	.alloc = udmabuf_alloc,
 244	.free = udmabuf_free,
 245	.memcpy_to_device = udmabuf_memcpy_to_device,
 246	.memcpy_from_device = udmabuf_memcpy_from_device,
 247};
 248
 249static struct memory_provider *provider = &udmabuf_memory_provider;
 250
 251static void print_nonzero_bytes(void *ptr, size_t size)
 252{
 253	unsigned char *p = ptr;
 254	unsigned int i;
 255
 256	for (i = 0; i < size; i++)
 257		putchar(p[i]);
 258}
 259
 260int validate_buffer(void *line, size_t size)
 261{
 262	static unsigned char seed = 1;
 263	unsigned char *ptr = line;
 264	unsigned char expected;
 265	static int errors;
 266	size_t i;
 267
 268	for (i = 0; i < size; i++) {
 269		expected = seed ? seed : '\n';
 270		if (ptr[i] != expected) {
 271			fprintf(stderr,
 272				"Failed validation: expected=%u, actual=%u, index=%lu\n",
 273				expected, ptr[i], i);
 274			errors++;
 275			if (errors > 20) {
 276				pr_err("validation failed");
 277				return -1;
 278			}
 279		}
 280		seed++;
 281		if (seed == do_validation)
 282			seed = 0;
 283	}
 284
 285	fprintf(stdout, "Validated buffer\n");
 286	return 0;
 287}
 288
 289static int
 290__run_command(char *out, size_t outlen, const char *cmd, va_list args)
 291{
 292	char command[256];
 293	FILE *fp;
 294
 295	vsnprintf(command, sizeof(command), cmd, args);
 296
 297	fprintf(stderr, "Running: %s\n", command);
 298	fp = popen(command, "r");
 299	if (!fp)
 300		return -1;
 301	if (out) {
 302		size_t len;
 303
 304		if (!fgets(out, outlen, fp))
 305			return -1;
 306
 307		/* Remove trailing newline if present */
 308		len = strlen(out);
 309		if (len && out[len - 1] == '\n')
 310			out[len - 1] = '\0';
 311	}
 312	return pclose(fp);
 313}
 314
 315static int run_command(const char *cmd, ...)
 316{
 317	va_list args;
 318	int ret;
 319
 320	va_start(args, cmd);
 321	ret = __run_command(NULL, 0, cmd, args);
 322	va_end(args);
 323
 324	return ret;
 325}
 326
 327static int ethtool_add_flow(const char *format, ...)
 328{
 329	char local_output[256], cmd[256];
 330	const char *id_start;
 331	int flow_idx, ret;
 332	char *endptr;
 333	long flow_id;
 334	va_list args;
 335
 336	for (flow_idx = 0; flow_idx < MAX_FLOWS; flow_idx++)
 337		if (ntuple_ids[flow_idx] == -1)
 338			break;
 339	if (flow_idx == MAX_FLOWS) {
 340		fprintf(stderr, "Error: too many flows\n");
 341		return -1;
 342	}
 343
 344	snprintf(cmd, sizeof(cmd), "ethtool -N %s %s", ifname, format);
 345
 346	va_start(args, format);
 347	ret = __run_command(local_output, sizeof(local_output), cmd, args);
 348	va_end(args);
 349
 350	if (ret != 0)
 351		return ret;
 352
 353	/* Extract the ID from the output */
 354	id_start = strstr(local_output, "Added rule with ID ");
 355	if (!id_start)
 356		return -1;
 357	id_start += strlen("Added rule with ID ");
 358
 359	flow_id = strtol(id_start, &endptr, 10);
 360	if (endptr == id_start || flow_id < 0 || flow_id > INT_MAX)
 361		return -1;
 362
 363	fprintf(stderr, "Added flow rule with ID %ld\n", flow_id);
 364	ntuple_ids[flow_idx] = flow_id;
 365	return flow_id;
 366}
 367
 368static int rxq_num(int ifindex)
 369{
 370	struct ethtool_channels_get_req *req;
 371	struct ethtool_channels_get_rsp *rsp;
 372	struct ynl_error yerr;
 373	struct ynl_sock *ys;
 374	int num = -1;
 375
 376	ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
 377	if (!ys) {
 378		fprintf(stderr, "YNL: %s\n", yerr.msg);
 379		return -1;
 380	}
 381
 382	req = ethtool_channels_get_req_alloc();
 383	ethtool_channels_get_req_set_header_dev_index(req, ifindex);
 384	rsp = ethtool_channels_get(ys, req);
 385	if (rsp)
 386		num = rsp->rx_count + rsp->combined_count;
 387	ethtool_channels_get_req_free(req);
 388	ethtool_channels_get_rsp_free(rsp);
 389
 390	ynl_sock_destroy(ys);
 391
 392	return num;
 393}
 394
 395static void reset_flow_steering(void)
 396{
 397	int i;
 398
 399	for (i = 0; i < MAX_FLOWS; i++) {
 400		if (ntuple_ids[i] == -1)
 401			continue;
 402		run_command("ethtool -N %s delete %d",
 403			    ifname, ntuple_ids[i]);
 404		ntuple_ids[i] = -1;
 405	}
 406}
 407
 408static const char *tcp_data_split_str(int val)
 409{
 410	switch (val) {
 411	case 0:
 412		return "off";
 413	case 1:
 414		return "auto";
 415	case 2:
 416		return "on";
 417	default:
 418		return "?";
 419	}
 420}
 421
 422static struct ethtool_rings_get_rsp *get_ring_config(void)
 423{
 424	struct ethtool_rings_get_req *get_req;
 425	struct ethtool_rings_get_rsp *get_rsp;
 426	struct ynl_error yerr;
 427	struct ynl_sock *ys;
 428
 429	ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
 430	if (!ys) {
 431		fprintf(stderr, "YNL: %s\n", yerr.msg);
 432		return NULL;
 433	}
 434
 435	get_req = ethtool_rings_get_req_alloc();
 436	ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
 437	get_rsp = ethtool_rings_get(ys, get_req);
 438	ethtool_rings_get_req_free(get_req);
 439
 440	ynl_sock_destroy(ys);
 441
 442	return get_rsp;
 443}
 444
 445static void restore_ring_config(const struct ethtool_rings_get_rsp *config)
 446{
 447	struct ethtool_rings_get_req *get_req;
 448	struct ethtool_rings_get_rsp *get_rsp;
 449	struct ethtool_rings_set_req *req;
 450	struct ynl_error yerr;
 451	struct ynl_sock *ys;
 452	int ret;
 453
 454	if (!config)
 455		return;
 456
 457	ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
 458	if (!ys) {
 459		fprintf(stderr, "YNL: %s\n", yerr.msg);
 460		return;
 461	}
 462
 463	req = ethtool_rings_set_req_alloc();
 464	ethtool_rings_set_req_set_header_dev_index(req, ifindex);
 465	ethtool_rings_set_req_set_tcp_data_split(req,
 466						ETHTOOL_TCP_DATA_SPLIT_UNKNOWN);
 467	if (config->_present.hds_thresh)
 468		ethtool_rings_set_req_set_hds_thresh(req, config->hds_thresh);
 469
 470	ret = ethtool_rings_set(ys, req);
 471	if (ret < 0)
 472		fprintf(stderr, "YNL restoring HDS cfg: %s\n", ys->err.msg);
 473
 474	get_req = ethtool_rings_get_req_alloc();
 475	ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
 476	get_rsp = ethtool_rings_get(ys, get_req);
 477	ethtool_rings_get_req_free(get_req);
 478
 479	/* use explicit value if UKNOWN didn't give us the previous */
 480	if (get_rsp->tcp_data_split != config->tcp_data_split) {
 481		ethtool_rings_set_req_set_tcp_data_split(req,
 482							config->tcp_data_split);
 483		ret = ethtool_rings_set(ys, req);
 484		if (ret < 0)
 485			fprintf(stderr, "YNL restoring expl HDS cfg: %s\n",
 486				ys->err.msg);
 487	}
 488
 489	ethtool_rings_get_rsp_free(get_rsp);
 490	ethtool_rings_set_req_free(req);
 491
 492	ynl_sock_destroy(ys);
 493}
 494
 495static int
 496configure_headersplit(const struct ethtool_rings_get_rsp *old, bool on)
 497{
 498	struct ethtool_rings_get_req *get_req;
 499	struct ethtool_rings_get_rsp *get_rsp;
 500	struct ethtool_rings_set_req *req;
 501	struct ynl_error yerr;
 502	struct ynl_sock *ys;
 503	int ret;
 504
 505	ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
 506	if (!ys) {
 507		fprintf(stderr, "YNL: %s\n", yerr.msg);
 508		return -1;
 509	}
 510
 511	req = ethtool_rings_set_req_alloc();
 512	ethtool_rings_set_req_set_header_dev_index(req, ifindex);
 513	if (on) {
 514		ethtool_rings_set_req_set_tcp_data_split(req,
 515						ETHTOOL_TCP_DATA_SPLIT_ENABLED);
 516		if (old->_present.hds_thresh)
 517			ethtool_rings_set_req_set_hds_thresh(req, 0);
 518	} else {
 519		ethtool_rings_set_req_set_tcp_data_split(req,
 520						ETHTOOL_TCP_DATA_SPLIT_UNKNOWN);
 521	}
 522	ret = ethtool_rings_set(ys, req);
 523	if (ret < 0)
 524		fprintf(stderr, "YNL failed: %s\n", ys->err.msg);
 525	ethtool_rings_set_req_free(req);
 526
 527	if (ret == 0) {
 528		get_req = ethtool_rings_get_req_alloc();
 529		ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
 530		get_rsp = ethtool_rings_get(ys, get_req);
 531		ethtool_rings_get_req_free(get_req);
 532		if (get_rsp)
 533			fprintf(stderr, "TCP header split: %s\n",
 534				tcp_data_split_str(get_rsp->tcp_data_split));
 535		ethtool_rings_get_rsp_free(get_rsp);
 536	}
 537
 538	ynl_sock_destroy(ys);
 539
 540	return ret;
 541}
 542
 543static int configure_rss(void)
 544{
 545	return run_command("ethtool -X %s equal %d >&2", ifname, start_queue);
 546}
 547
 548static void reset_rss(void)
 549{
 550	run_command("ethtool -X %s default >&2", ifname, start_queue);
 551}
 552
 553static int check_changing_channels(unsigned int rx, unsigned int tx)
 554{
 555	struct ethtool_channels_get_req *gchan;
 556	struct ethtool_channels_set_req *schan;
 557	struct ethtool_channels_get_rsp *chan;
 558	struct ynl_error yerr;
 559	struct ynl_sock *ys;
 560	int ret;
 561
 562	fprintf(stderr, "setting channel count rx:%u tx:%u\n", rx, tx);
 563
 564	ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
 565	if (!ys) {
 566		fprintf(stderr, "YNL: %s\n", yerr.msg);
 567		return -1;
 568	}
 569
 570	gchan = ethtool_channels_get_req_alloc();
 571	if (!gchan) {
 572		ret = -1;
 573		goto exit_close_sock;
 574	}
 575
 576	ethtool_channels_get_req_set_header_dev_index(gchan, ifindex);
 577	chan = ethtool_channels_get(ys, gchan);
 578	ethtool_channels_get_req_free(gchan);
 579	if (!chan) {
 580		fprintf(stderr, "YNL get channels: %s\n", ys->err.msg);
 581		ret = -1;
 582		goto exit_close_sock;
 583	}
 584
 585	schan =	ethtool_channels_set_req_alloc();
 586	if (!schan) {
 587		ret = -1;
 588		goto exit_free_chan;
 589	}
 590
 591	ethtool_channels_set_req_set_header_dev_index(schan, ifindex);
 592
 593	if (chan->_present.combined_count) {
 594		if (chan->_present.rx_count || chan->_present.tx_count) {
 595			ethtool_channels_set_req_set_rx_count(schan, 0);
 596			ethtool_channels_set_req_set_tx_count(schan, 0);
 597		}
 598
 599		if (rx == tx) {
 600			ethtool_channels_set_req_set_combined_count(schan, rx);
 601		} else if (rx > tx) {
 602			ethtool_channels_set_req_set_combined_count(schan, tx);
 603			ethtool_channels_set_req_set_rx_count(schan, rx - tx);
 604		} else {
 605			ethtool_channels_set_req_set_combined_count(schan, rx);
 606			ethtool_channels_set_req_set_tx_count(schan, tx - rx);
 607		}
 608
 609	} else if (chan->_present.rx_count) {
 610		ethtool_channels_set_req_set_rx_count(schan, rx);
 611		ethtool_channels_set_req_set_tx_count(schan, tx);
 612	} else {
 613		fprintf(stderr, "Error: device has neither combined nor rx channels\n");
 614		ret = -1;
 615		goto exit_free_schan;
 616	}
 617
 618	ret = ethtool_channels_set(ys, schan);
 619	if (ret) {
 620		fprintf(stderr, "YNL set channels: %s\n", ys->err.msg);
 621	} else {
 622		/* We were expecting a failure, go back to previous settings */
 623		ethtool_channels_set_req_set_combined_count(schan,
 624							    chan->combined_count);
 625		ethtool_channels_set_req_set_rx_count(schan, chan->rx_count);
 626		ethtool_channels_set_req_set_tx_count(schan, chan->tx_count);
 627
 628		ret = ethtool_channels_set(ys, schan);
 629		if (ret)
 630			fprintf(stderr, "YNL un-setting channels: %s\n",
 631				ys->err.msg);
 632	}
 633
 634exit_free_schan:
 635	ethtool_channels_set_req_free(schan);
 636exit_free_chan:
 637	ethtool_channels_get_rsp_free(chan);
 638exit_close_sock:
 639	ynl_sock_destroy(ys);
 640
 641	return ret;
 642}
 643
 644static int configure_flow_steering(struct sockaddr_in6 *server_sin)
 645{
 646	const char *type = "tcp6";
 647	const char *server_addr;
 648	char buf[40];
 649	int flow_id;
 650
 651	inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf));
 652	server_addr = buf;
 653
 654	if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) {
 655		type = "tcp4";
 656		server_addr = strrchr(server_addr, ':') + 1;
 657	}
 658
 659	/* Try configure 5-tuple */
 660	flow_id = ethtool_add_flow("flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d",
 661				   type,
 662				   client_ip ? "src-ip" : "",
 663				   client_ip ?: "",
 664				   server_addr,
 665				   client_ip ? "src-port" : "",
 666				   client_ip ? port : "",
 667				   port, start_queue);
 668	if (flow_id < 0) {
 669		/* If that fails, try configure 3-tuple */
 670		flow_id = ethtool_add_flow("flow-type %s dst-ip %s dst-port %s queue %d",
 671					   type, server_addr, port, start_queue);
 672		if (flow_id < 0)
 673			/* If that fails, return error */
 674			return -1;
 675	}
 676
 677	return 0;
 678}
 679
 680static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
 681			 struct netdev_queue_id *queues,
 682			 unsigned int n_queue_index, struct ynl_sock **ys)
 683{
 684	struct netdev_bind_rx_req *req = NULL;
 685	struct netdev_bind_rx_rsp *rsp = NULL;
 686	struct ynl_error yerr;
 687
 688	*ys = ynl_sock_create(&ynl_netdev_family, &yerr);
 689	if (!*ys) {
 690		netdev_queue_id_free(queues);
 691		fprintf(stderr, "YNL: %s\n", yerr.msg);
 692		return -1;
 693	}
 694
 695	req = netdev_bind_rx_req_alloc();
 696	netdev_bind_rx_req_set_ifindex(req, ifindex);
 697	netdev_bind_rx_req_set_fd(req, dmabuf_fd);
 698	__netdev_bind_rx_req_set_queues(req, queues, n_queue_index);
 699
 700	rsp = netdev_bind_rx(*ys, req);
 701	if (!rsp) {
 702		perror("netdev_bind_rx");
 703		goto err_close;
 704	}
 705
 706	if (!rsp->_present.id) {
 707		perror("id not present");
 708		goto err_close;
 709	}
 710
 711	fprintf(stderr, "got dmabuf id=%d\n", rsp->id);
 712	dmabuf_id = rsp->id;
 713
 714	netdev_bind_rx_req_free(req);
 715	netdev_bind_rx_rsp_free(rsp);
 716
 717	return 0;
 718
 719err_close:
 720	fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
 721	netdev_bind_rx_req_free(req);
 722	ynl_sock_destroy(*ys);
 723	return -1;
 724}
 725
 726static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
 727			 struct ynl_sock **ys)
 728{
 729	struct netdev_bind_tx_req *req = NULL;
 730	struct netdev_bind_tx_rsp *rsp = NULL;
 731	struct ynl_error yerr;
 732
 733	*ys = ynl_sock_create(&ynl_netdev_family, &yerr);
 734	if (!*ys) {
 735		fprintf(stderr, "YNL: %s\n", yerr.msg);
 736		return -1;
 737	}
 738
 739	req = netdev_bind_tx_req_alloc();
 740	netdev_bind_tx_req_set_ifindex(req, ifindex);
 741	netdev_bind_tx_req_set_fd(req, dmabuf_fd);
 742
 743	rsp = netdev_bind_tx(*ys, req);
 744	if (!rsp) {
 745		perror("netdev_bind_tx");
 746		goto err_close;
 747	}
 748
 749	if (!rsp->_present.id) {
 750		perror("id not present");
 751		goto err_close;
 752	}
 753
 754	fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id);
 755	tx_dmabuf_id = rsp->id;
 756
 757	netdev_bind_tx_req_free(req);
 758	netdev_bind_tx_rsp_free(rsp);
 759
 760	return 0;
 761
 762err_close:
 763	fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
 764	netdev_bind_tx_req_free(req);
 765	ynl_sock_destroy(*ys);
 766	return -1;
 767}
 768
 769static int enable_reuseaddr(int fd)
 770{
 771	int opt = 1;
 772	int ret;
 773
 774	ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
 775	if (ret) {
 776		pr_err("SO_REUSEPORT failed");
 777		return -1;
 778	}
 779
 780	ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
 781	if (ret) {
 782		pr_err("SO_REUSEADDR failed");
 783		return -1;
 784	}
 785
 786	return 0;
 787}
 788
 789static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6)
 790{
 791	int ret;
 792
 793	sin6->sin6_family = AF_INET6;
 794	sin6->sin6_port = htons(port);
 795
 796	ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr);
 797	if (ret != 1) {
 798		/* fallback to plain IPv4 */
 799		ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]);
 800		if (ret != 1)
 801			return -1;
 802
 803		/* add ::ffff prefix */
 804		sin6->sin6_addr.s6_addr32[0] = 0;
 805		sin6->sin6_addr.s6_addr32[1] = 0;
 806		sin6->sin6_addr.s6_addr16[4] = 0;
 807		sin6->sin6_addr.s6_addr16[5] = 0xffff;
 808	}
 809
 810	return 0;
 811}
 812
 813static struct netdev_queue_id *create_queues(void)
 814{
 815	struct netdev_queue_id *queues;
 816	size_t i = 0;
 817
 818	queues = netdev_queue_id_alloc(num_queues);
 819	for (i = 0; i < num_queues; i++) {
 820		netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX);
 821		netdev_queue_id_set_id(&queues[i], start_queue + i);
 822	}
 823
 824	return queues;
 825}
 826
 827static int do_server(struct memory_buffer *mem)
 828{
 829	struct ethtool_rings_get_rsp *ring_config;
 830	char ctrl_data[sizeof(int) * 20000];
 831	size_t non_page_aligned_frags = 0;
 832	struct sockaddr_in6 client_addr;
 833	struct sockaddr_in6 server_sin;
 834	size_t page_aligned_frags = 0;
 835	size_t total_received = 0;
 836	socklen_t client_addr_len;
 837	bool is_devmem = false;
 838	char *tmp_mem = NULL;
 839	struct ynl_sock *ys;
 840	char iobuf[819200];
 841	int ret, err = -1;
 842	char buffer[256];
 843	int socket_fd;
 844	int client_fd;
 845
 846	ret = parse_address(server_ip, atoi(port), &server_sin);
 847	if (ret < 0) {
 848		pr_err("parse server address");
 849		return -1;
 850	}
 851
 852	ring_config = get_ring_config();
 853	if (!ring_config) {
 854		pr_err("Failed to get current ring configuration");
 855		return -1;
 856	}
 857
 858	if (configure_headersplit(ring_config, 1)) {
 859		pr_err("Failed to enable TCP header split");
 860		goto err_free_ring_config;
 861	}
 862
 863	/* Configure RSS to divert all traffic from our devmem queues */
 864	if (configure_rss()) {
 865		pr_err("Failed to configure rss");
 866		goto err_reset_headersplit;
 867	}
 868
 869	/* Flow steer our devmem flows to start_queue */
 870	if (configure_flow_steering(&server_sin)) {
 871		pr_err("Failed to configure flow steering");
 872		goto err_reset_rss;
 873	}
 874
 875	if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) {
 876		pr_err("Failed to bind");
 877		goto err_reset_flow_steering;
 878	}
 879
 880	tmp_mem = malloc(mem->size);
 881	if (!tmp_mem)
 882		goto err_unbind;
 883
 884	socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
 885	if (socket_fd < 0) {
 886		pr_err("Failed to create socket");
 887		goto err_free_tmp;
 888	}
 889
 890	if (enable_reuseaddr(socket_fd))
 891		goto err_close_socket;
 892
 893	fprintf(stderr, "binding to address %s:%d\n", server_ip,
 894		ntohs(server_sin.sin6_port));
 895
 896	ret = bind(socket_fd, &server_sin, sizeof(server_sin));
 897	if (ret) {
 898		pr_err("Failed to bind");
 899		goto err_close_socket;
 900	}
 901
 902	ret = listen(socket_fd, 1);
 903	if (ret) {
 904		pr_err("Failed to listen");
 905		goto err_close_socket;
 906	}
 907
 908	client_addr_len = sizeof(client_addr);
 909
 910	inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer,
 911		  sizeof(buffer));
 912	fprintf(stderr, "Waiting or connection on %s:%d\n", buffer,
 913		ntohs(server_sin.sin6_port));
 914	client_fd = accept(socket_fd, &client_addr, &client_addr_len);
 915	if (client_fd < 0) {
 916		pr_err("Failed to accept");
 917		goto err_close_socket;
 918	}
 919
 920	inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer,
 921		  sizeof(buffer));
 922	fprintf(stderr, "Got connection from %s:%d\n", buffer,
 923		ntohs(client_addr.sin6_port));
 924
 925	while (1) {
 926		struct iovec iov = { .iov_base = iobuf,
 927				     .iov_len = sizeof(iobuf) };
 928		struct dmabuf_cmsg *dmabuf_cmsg = NULL;
 929		struct cmsghdr *cm = NULL;
 930		struct msghdr msg = { 0 };
 931		struct dmabuf_token token;
 932		ssize_t ret;
 933
 934		is_devmem = false;
 935
 936		msg.msg_iov = &iov;
 937		msg.msg_iovlen = 1;
 938		msg.msg_control = ctrl_data;
 939		msg.msg_controllen = sizeof(ctrl_data);
 940		ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM);
 941		fprintf(stderr, "recvmsg ret=%ld\n", ret);
 942		if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
 943			continue;
 944		if (ret < 0) {
 945			perror("recvmsg");
 946			if (errno == EFAULT) {
 947				pr_err("received EFAULT, won't recover");
 948				goto err_close_client;
 949			}
 950			continue;
 951		}
 952		if (ret == 0) {
 953			errno = 0;
 954			pr_err("client exited");
 955			goto cleanup;
 956		}
 957
 958		for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
 959			if (cm->cmsg_level != SOL_SOCKET ||
 960			    (cm->cmsg_type != SCM_DEVMEM_DMABUF &&
 961			     cm->cmsg_type != SCM_DEVMEM_LINEAR)) {
 962				fprintf(stderr, "skipping non-devmem cmsg\n");
 963				continue;
 964			}
 965
 966			dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);
 967			is_devmem = true;
 968
 969			if (cm->cmsg_type == SCM_DEVMEM_LINEAR) {
 970				/* TODO: process data copied from skb's linear
 971				 * buffer.
 972				 */
 973				fprintf(stderr,
 974					"SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n",
 975					dmabuf_cmsg->frag_size);
 976
 977				continue;
 978			}
 979
 980			token.token_start = dmabuf_cmsg->frag_token;
 981			token.token_count = 1;
 982
 983			total_received += dmabuf_cmsg->frag_size;
 984			fprintf(stderr,
 985				"received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n",
 986				dmabuf_cmsg->frag_offset >> PAGE_SHIFT,
 987				dmabuf_cmsg->frag_offset % getpagesize(),
 988				dmabuf_cmsg->frag_offset,
 989				dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token,
 990				total_received, dmabuf_cmsg->dmabuf_id);
 991
 992			if (dmabuf_cmsg->dmabuf_id != dmabuf_id) {
 993				pr_err("received on wrong dmabuf_id: flow steering error");
 994				goto err_close_client;
 995			}
 996
 997			if (dmabuf_cmsg->frag_size % getpagesize())
 998				non_page_aligned_frags++;
 999			else
1000				page_aligned_frags++;
1001
1002			provider->memcpy_from_device(tmp_mem, mem,
1003						     dmabuf_cmsg->frag_offset,
1004						     dmabuf_cmsg->frag_size);
1005
1006			if (do_validation) {
1007				if (validate_buffer(tmp_mem,
1008						    dmabuf_cmsg->frag_size))
1009					goto err_close_client;
1010			} else {
1011				print_nonzero_bytes(tmp_mem,
1012						    dmabuf_cmsg->frag_size);
1013			}
1014
1015			ret = setsockopt(client_fd, SOL_SOCKET,
1016					 SO_DEVMEM_DONTNEED, &token,
1017					 sizeof(token));
1018			if (ret != 1) {
1019				pr_err("SO_DEVMEM_DONTNEED not enough tokens");
1020				goto err_close_client;
1021			}
1022		}
1023		if (!is_devmem) {
1024			pr_err("flow steering error");
1025			goto err_close_client;
1026		}
1027
1028		fprintf(stderr, "total_received=%lu\n", total_received);
1029	}
1030
1031	fprintf(stderr, "%s: ok\n", TEST_PREFIX);
1032
1033	fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n",
1034		page_aligned_frags, non_page_aligned_frags);
1035
1036cleanup:
1037	err = 0;
1038
1039err_close_client:
1040	close(client_fd);
1041err_close_socket:
1042	close(socket_fd);
1043err_free_tmp:
1044	free(tmp_mem);
1045err_unbind:
1046	ynl_sock_destroy(ys);
1047err_reset_flow_steering:
1048	reset_flow_steering();
1049err_reset_rss:
1050	reset_rss();
1051err_reset_headersplit:
1052	restore_ring_config(ring_config);
1053err_free_ring_config:
1054	ethtool_rings_get_rsp_free(ring_config);
1055	return err;
1056}
1057
1058int run_devmem_tests(void)
1059{
1060	struct ethtool_rings_get_rsp *ring_config;
1061	struct netdev_queue_id *queues;
1062	struct memory_buffer *mem;
1063	struct ynl_sock *ys;
1064	int err = -1;
1065
1066	mem = provider->alloc(getpagesize() * NUM_PAGES);
1067	if (!mem) {
1068		pr_err("Failed to allocate memory buffer");
1069		return -1;
1070	}
1071
1072	ring_config = get_ring_config();
1073	if (!ring_config) {
1074		pr_err("Failed to get current ring configuration");
1075		goto err_free_mem;
1076	}
1077
1078	/* Configure RSS to divert all traffic from our devmem queues */
1079	if (configure_rss()) {
1080		pr_err("rss error");
1081		goto err_free_ring_config;
1082	}
1083
1084	if (configure_headersplit(ring_config, 1)) {
1085		pr_err("Failed to configure header split");
1086		goto err_reset_rss;
1087	}
1088
1089	queues = netdev_queue_id_alloc(num_queues);
1090	if (!queues) {
1091		pr_err("Failed to allocate empty queues array");
1092		goto err_reset_headersplit;
1093	}
1094
1095	if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1096		pr_err("Binding empty queues array should have failed");
1097		goto err_unbind;
1098	}
1099
1100	if (configure_headersplit(ring_config, 0)) {
1101		pr_err("Failed to configure header split");
1102		goto err_reset_headersplit;
1103	}
1104
1105	queues = create_queues();
1106	if (!queues) {
1107		pr_err("Failed to create queues");
1108		goto err_reset_headersplit;
1109	}
1110
1111	if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1112		pr_err("Configure dmabuf with header split off should have failed");
1113		goto err_unbind;
1114	}
1115
1116	if (configure_headersplit(ring_config, 1)) {
1117		pr_err("Failed to configure header split");
1118		goto err_reset_headersplit;
1119	}
1120
1121	queues = create_queues();
1122	if (!queues) {
1123		pr_err("Failed to create queues");
1124		goto err_reset_headersplit;
1125	}
1126
1127	if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1128		pr_err("Failed to bind");
1129		goto err_reset_headersplit;
1130	}
1131
1132	/* Deactivating a bound queue should not be legal */
1133	if (!check_changing_channels(num_queues, num_queues)) {
1134		pr_err("Deactivating a bound queue should be illegal");
1135		goto err_unbind;
1136	}
1137
1138	err = 0;
1139	goto err_unbind;
1140
1141err_unbind:
1142	ynl_sock_destroy(ys);
1143err_reset_headersplit:
1144	restore_ring_config(ring_config);
1145err_reset_rss:
1146	reset_rss();
1147err_free_ring_config:
1148	ethtool_rings_get_rsp_free(ring_config);
1149err_free_mem:
1150	provider->free(mem);
1151	return err;
1152}
1153
1154static uint64_t gettimeofday_ms(void)
1155{
1156	struct timeval tv;
1157
1158	gettimeofday(&tv, NULL);
1159	return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL);
1160}
1161
1162static int do_poll(int fd)
1163{
1164	struct pollfd pfd;
1165	int ret;
1166
1167	pfd.revents = 0;
1168	pfd.fd = fd;
1169
1170	ret = poll(&pfd, 1, waittime_ms);
1171	if (ret == -1) {
1172		pr_err("poll");
1173		return -1;
1174	}
1175
1176	return ret && (pfd.revents & POLLERR);
1177}
1178
1179static int wait_compl(int fd)
1180{
1181	int64_t tstop = gettimeofday_ms() + waittime_ms;
1182	char control[CMSG_SPACE(100)] = {};
1183	struct sock_extended_err *serr;
1184	struct msghdr msg = {};
1185	struct cmsghdr *cm;
1186	__u32 hi, lo;
1187	int ret;
1188
1189	msg.msg_control = control;
1190	msg.msg_controllen = sizeof(control);
1191
1192	while (gettimeofday_ms() < tstop) {
1193		ret = do_poll(fd);
1194		if (ret < 0)
1195			return ret;
1196		if (!ret)
1197			continue;
1198
1199		ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
1200		if (ret < 0) {
1201			if (errno == EAGAIN)
1202				continue;
1203			pr_err("recvmsg(MSG_ERRQUEUE)");
1204			return -1;
1205		}
1206		if (msg.msg_flags & MSG_CTRUNC) {
1207			pr_err("MSG_CTRUNC");
1208			return -1;
1209		}
1210
1211		for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
1212			if (cm->cmsg_level != SOL_IP &&
1213			    cm->cmsg_level != SOL_IPV6)
1214				continue;
1215			if (cm->cmsg_level == SOL_IP &&
1216			    cm->cmsg_type != IP_RECVERR)
1217				continue;
1218			if (cm->cmsg_level == SOL_IPV6 &&
1219			    cm->cmsg_type != IPV6_RECVERR)
1220				continue;
1221
1222			serr = (void *)CMSG_DATA(cm);
1223			if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) {
1224				pr_err("wrong origin %u", serr->ee_origin);
1225				return -1;
1226			}
1227			if (serr->ee_errno != 0) {
1228				pr_err("wrong errno %d", serr->ee_errno);
1229				return -1;
1230			}
1231
1232			hi = serr->ee_data;
1233			lo = serr->ee_info;
1234
1235			fprintf(stderr, "tx complete [%d,%d]\n", lo, hi);
1236			return 0;
1237		}
1238	}
1239
1240	pr_err("did not receive tx completion");
1241	return -1;
1242}
1243
1244static int do_client(struct memory_buffer *mem)
1245{
1246	char ctrl_data[CMSG_SPACE(sizeof(__u32))];
1247	struct sockaddr_in6 server_sin;
1248	struct sockaddr_in6 client_sin;
1249	struct ynl_sock *ys = NULL;
1250	struct iovec iov[MAX_IOV];
1251	struct msghdr msg = {};
1252	ssize_t line_size = 0;
1253	struct cmsghdr *cmsg;
1254	char *line = NULL;
1255	int ret, err = -1;
1256	size_t len = 0;
1257	int socket_fd;
1258	__u32 ddmabuf;
1259	int opt = 1;
1260
1261	ret = parse_address(server_ip, atoi(port), &server_sin);
1262	if (ret < 0) {
1263		pr_err("parse server address");
1264		return -1;
1265	}
1266
1267	if (client_ip) {
1268		ret = parse_address(client_ip, atoi(port), &client_sin);
1269		if (ret < 0) {
1270			pr_err("parse client address");
1271			return ret;
1272		}
1273	}
1274
1275	socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
1276	if (socket_fd < 0) {
1277		pr_err("create socket");
1278		return -1;
1279	}
1280
1281	if (enable_reuseaddr(socket_fd))
1282		goto err_close_socket;
1283
1284	ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname,
1285			 strlen(ifname) + 1);
1286	if (ret) {
1287		pr_err("bindtodevice");
1288		goto err_close_socket;
1289	}
1290
1291	if (bind_tx_queue(ifindex, mem->fd, &ys)) {
1292		pr_err("Failed to bind");
1293		goto err_close_socket;
1294	}
1295
1296	if (client_ip) {
1297		ret = bind(socket_fd, &client_sin, sizeof(client_sin));
1298		if (ret) {
1299			pr_err("bind");
1300			goto err_unbind;
1301		}
1302	}
1303
1304	ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt));
1305	if (ret) {
1306		pr_err("set sock opt");
1307		goto err_unbind;
1308	}
1309
1310	fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip,
1311		ntohs(server_sin.sin6_port), ifname);
1312
1313	ret = connect(socket_fd, &server_sin, sizeof(server_sin));
1314	if (ret) {
1315		pr_err("connect");
1316		goto err_unbind;
1317	}
1318
1319	while (1) {
1320		free(line);
1321		line = NULL;
1322		line_size = getline(&line, &len, stdin);
1323
1324		if (line_size < 0)
1325			break;
1326
1327		if (max_chunk) {
1328			msg.msg_iovlen =
1329				(line_size + max_chunk - 1) / max_chunk;
1330			if (msg.msg_iovlen > MAX_IOV) {
1331				pr_err("can't partition %zd bytes into maximum of %d chunks",
1332				       line_size, MAX_IOV);
1333				goto err_free_line;
1334			}
1335
1336			for (int i = 0; i < msg.msg_iovlen; i++) {
1337				iov[i].iov_base = (void *)(i * max_chunk);
1338				iov[i].iov_len = max_chunk;
1339			}
1340
1341			iov[msg.msg_iovlen - 1].iov_len =
1342				line_size - (msg.msg_iovlen - 1) * max_chunk;
1343		} else {
1344			iov[0].iov_base = 0;
1345			iov[0].iov_len = line_size;
1346			msg.msg_iovlen = 1;
1347		}
1348
1349		msg.msg_iov = iov;
1350		provider->memcpy_to_device(mem, 0, line, line_size);
1351
1352		msg.msg_control = ctrl_data;
1353		msg.msg_controllen = sizeof(ctrl_data);
1354
1355		cmsg = CMSG_FIRSTHDR(&msg);
1356		cmsg->cmsg_level = SOL_SOCKET;
1357		cmsg->cmsg_type = SCM_DEVMEM_DMABUF;
1358		cmsg->cmsg_len = CMSG_LEN(sizeof(__u32));
1359
1360		ddmabuf = tx_dmabuf_id;
1361
1362		*((__u32 *)CMSG_DATA(cmsg)) = ddmabuf;
1363
1364		ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY);
1365		if (ret < 0) {
1366			pr_err("Failed sendmsg");
1367			goto err_free_line;
1368		}
1369
1370		fprintf(stderr, "sendmsg_ret=%d\n", ret);
1371
1372		if (ret != line_size) {
1373			pr_err("Did not send all bytes %d vs %zd", ret, line_size);
1374			goto err_free_line;
1375		}
1376
1377		if (wait_compl(socket_fd))
1378			goto err_free_line;
1379	}
1380
1381	fprintf(stderr, "%s: tx ok\n", TEST_PREFIX);
1382
1383	err = 0;
1384
1385err_free_line:
1386	free(line);
1387err_unbind:
1388	ynl_sock_destroy(ys);
1389err_close_socket:
1390	close(socket_fd);
1391	return err;
1392}
1393
1394int main(int argc, char *argv[])
1395{
1396	struct memory_buffer *mem;
1397	int is_server = 0, opt;
1398	int ret, err = 1;
1399
1400	while ((opt = getopt(argc, argv, "ls:c:p:v:q:t:f:z:")) != -1) {
1401		switch (opt) {
1402		case 'l':
1403			is_server = 1;
1404			break;
1405		case 's':
1406			server_ip = optarg;
1407			break;
1408		case 'c':
1409			client_ip = optarg;
1410			break;
1411		case 'p':
1412			port = optarg;
1413			break;
1414		case 'v':
1415			do_validation = atoll(optarg);
1416			break;
1417		case 'q':
1418			num_queues = atoi(optarg);
1419			break;
1420		case 't':
1421			start_queue = atoi(optarg);
1422			break;
1423		case 'f':
1424			ifname = optarg;
1425			break;
1426		case 'z':
1427			max_chunk = atoi(optarg);
1428			break;
1429		case '?':
1430			fprintf(stderr, "unknown option: %c\n", optopt);
1431			break;
1432		}
1433	}
1434
1435	if (!ifname) {
1436		pr_err("Missing -f argument");
1437		return 1;
1438	}
1439
1440	ifindex = if_nametoindex(ifname);
1441
1442	fprintf(stderr, "using ifindex=%u\n", ifindex);
1443
1444	if (!server_ip && !client_ip) {
1445		if (start_queue < 0 && num_queues < 0) {
1446			num_queues = rxq_num(ifindex);
1447			if (num_queues < 0) {
1448				pr_err("couldn't detect number of queues");
1449				return 1;
1450			}
1451			if (num_queues < 2) {
1452				pr_err("number of device queues is too low");
1453				return 1;
1454			}
1455			/* make sure can bind to multiple queues */
1456			start_queue = num_queues / 2;
1457			num_queues /= 2;
1458		}
1459
1460		if (start_queue < 0 || num_queues < 0) {
1461			pr_err("Both -t and -q are required");
1462			return 1;
1463		}
1464
1465		return run_devmem_tests();
1466	}
1467
1468	if (start_queue < 0 && num_queues < 0) {
1469		num_queues = rxq_num(ifindex);
1470		if (num_queues < 2) {
1471			pr_err("number of device queues is too low");
1472			return 1;
1473		}
1474
1475		num_queues = 1;
1476		start_queue = rxq_num(ifindex) - num_queues;
1477
1478		if (start_queue < 0) {
1479			pr_err("couldn't detect number of queues");
1480			return 1;
1481		}
1482
1483		fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues);
1484	}
1485
1486	for (; optind < argc; optind++)
1487		fprintf(stderr, "extra arguments: %s\n", argv[optind]);
1488
1489	if (start_queue < 0) {
1490		pr_err("Missing -t argument");
1491		return 1;
1492	}
1493
1494	if (num_queues < 0) {
1495		pr_err("Missing -q argument");
1496		return 1;
1497	}
1498
1499	if (!server_ip) {
1500		pr_err("Missing -s argument");
1501		return 1;
1502	}
1503
1504	if (!port) {
1505		pr_err("Missing -p argument");
1506		return 1;
1507	}
1508
1509	mem = provider->alloc(getpagesize() * NUM_PAGES);
1510	if (!mem) {
1511		pr_err("Failed to allocate memory buffer");
1512		return 1;
1513	}
1514
1515	ret = is_server ? do_server(mem) : do_client(mem);
1516	if (ret)
1517		goto err_free_mem;
1518
1519	err = 0;
1520
1521err_free_mem:
1522	provider->free(mem);
1523	return err;
1524}