test_xsk.c

   1// SPDX-License-Identifier: GPL-2.0
   2#include <bpf/bpf.h>
   3#include <errno.h>
   4#include <linux/bitmap.h>
   5#include <linux/if_link.h>
   6#include <linux/mman.h>
   7#include <linux/netdev.h>
   8#include <poll.h>
   9#include <pthread.h>
  10#include <signal.h>
  11#include <string.h>
  12#include <sys/mman.h>
  13#include <sys/socket.h>
  14#include <sys/time.h>
  15#include <unistd.h>
  16
  17#include "network_helpers.h"
  18#include "test_xsk.h"
  19#include "xsk_xdp_common.h"
  20#include "xsk_xdp_progs.skel.h"
  21
  22#define DEFAULT_BATCH_SIZE		64
  23#define MIN_PKT_SIZE			64
  24#define MAX_ETH_JUMBO_SIZE		9000
  25#define MAX_INTERFACES			2
  26#define MAX_TEARDOWN_ITER		10
  27#define MAX_TX_BUDGET_DEFAULT		32
  28#define PKT_DUMP_NB_TO_PRINT		16
  29/* Just to align the data in the packet */
  30#define PKT_HDR_SIZE			(sizeof(struct ethhdr) + 2)
  31#define POLL_TMOUT			1000
  32#define THREAD_TMOUT			3
  33#define UMEM_HEADROOM_TEST_SIZE		128
  34#define XSK_DESC__INVALID_OPTION	(0xffff)
  35#define XSK_UMEM__INVALID_FRAME_SIZE	(MAX_ETH_JUMBO_SIZE + 1)
  36#define XSK_UMEM__LARGE_FRAME_SIZE	(3 * 1024)
  37#define XSK_UMEM__MAX_FRAME_SIZE	(4 * 1024)
  38
  39static const u8 g_mac[ETH_ALEN] = {0x55, 0x44, 0x33, 0x22, 0x11, 0x00};
  40
  41bool opt_verbose;
  42pthread_barrier_t barr;
  43pthread_mutex_t pacing_mutex = PTHREAD_MUTEX_INITIALIZER;
  44
  45int pkts_in_flight;
  46
  47/* The payload is a word consisting of a packet sequence number in the upper
  48 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's
  49 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0.
  50 */
  51static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)
  52{
  53	u32 *ptr = (u32 *)dest, i;
  54
  55	start /= sizeof(*ptr);
  56	size /= sizeof(*ptr);
  57	for (i = 0; i < size; i++)
  58		ptr[i] = htonl(pkt_nb << 16 | (i + start));
  59}
  60
  61static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr)
  62{
  63	memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN);
  64	memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN);
  65	eth_hdr->h_proto = htons(ETH_P_LOOPBACK);
  66}
  67
  68static bool is_umem_valid(struct ifobject *ifobj)
  69{
  70	return !!ifobj->umem->umem;
  71}
  72
  73static u32 mode_to_xdp_flags(enum test_mode mode)
  74{
  75	return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
  76}
  77
  78static u64 umem_size(struct xsk_umem_info *umem)
  79{
  80	return umem->num_frames * umem->frame_size;
  81}
  82
  83int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,
  84			      u64 size)
  85{
  86	struct xsk_umem_config cfg = {
  87		.fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
  88		.comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
  89		.frame_size = umem->frame_size,
  90		.frame_headroom = umem->frame_headroom,
  91		.flags = XSK_UMEM__DEFAULT_FLAGS
  92	};
  93	int ret;
  94
  95	if (umem->fill_size)
  96		cfg.fill_size = umem->fill_size;
  97
  98	if (umem->comp_size)
  99		cfg.comp_size = umem->comp_size;
 100
 101	if (umem->unaligned_mode)
 102		cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
 103
 104	ret = xsk_umem__create(&umem->umem, buffer, size,
 105			       &umem->fq, &umem->cq, &cfg);
 106	if (ret)
 107		return ret;
 108
 109	umem->buffer = buffer;
 110	if (ifobj->shared_umem && ifobj->rx_on) {
 111		umem->base_addr = umem_size(umem);
 112		umem->next_buffer = umem_size(umem);
 113	}
 114
 115	return 0;
 116}
 117
 118static u64 umem_alloc_buffer(struct xsk_umem_info *umem)
 119{
 120	u64 addr;
 121
 122	addr = umem->next_buffer;
 123	umem->next_buffer += umem->frame_size;
 124	if (umem->next_buffer >= umem->base_addr + umem_size(umem))
 125		umem->next_buffer = umem->base_addr;
 126
 127	return addr;
 128}
 129
 130static void umem_reset_alloc(struct xsk_umem_info *umem)
 131{
 132	umem->next_buffer = 0;
 133}
 134
 135static int enable_busy_poll(struct xsk_socket_info *xsk)
 136{
 137	int sock_opt;
 138
 139	sock_opt = 1;
 140	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
 141		       (void *)&sock_opt, sizeof(sock_opt)) < 0)
 142		return -errno;
 143
 144	sock_opt = 20;
 145	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
 146		       (void *)&sock_opt, sizeof(sock_opt)) < 0)
 147		return -errno;
 148
 149	sock_opt = xsk->batch_size;
 150	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
 151		       (void *)&sock_opt, sizeof(sock_opt)) < 0)
 152		return -errno;
 153
 154	return 0;
 155}
 156
 157int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
 158				  struct ifobject *ifobject, bool shared)
 159{
 160	struct xsk_socket_config cfg = {};
 161	struct xsk_ring_cons *rxr;
 162	struct xsk_ring_prod *txr;
 163
 164	xsk->umem = umem;
 165	cfg.rx_size = xsk->rxqsize;
 166	cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
 167	cfg.bind_flags = ifobject->bind_flags;
 168	if (shared)
 169		cfg.bind_flags |= XDP_SHARED_UMEM;
 170	if (ifobject->mtu > MAX_ETH_PKT_SIZE)
 171		cfg.bind_flags |= XDP_USE_SG;
 172	if (umem->comp_size)
 173		cfg.tx_size = umem->comp_size;
 174	if (umem->fill_size)
 175		cfg.rx_size = umem->fill_size;
 176
 177	txr = ifobject->tx_on ? &xsk->tx : NULL;
 178	rxr = ifobject->rx_on ? &xsk->rx : NULL;
 179	return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);
 180}
 181
 182#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags"
 183static unsigned int get_max_skb_frags(void)
 184{
 185	unsigned int max_skb_frags = 0;
 186	FILE *file;
 187
 188	file = fopen(MAX_SKB_FRAGS_PATH, "r");
 189	if (!file) {
 190		ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH);
 191		return 0;
 192	}
 193
 194	if (fscanf(file, "%u", &max_skb_frags) != 1)
 195		ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH);
 196
 197	fclose(file);
 198	return max_skb_frags;
 199}
 200
 201static int set_ring_size(struct ifobject *ifobj)
 202{
 203	int ret;
 204	u32 ctr = 0;
 205
 206	while (ctr++ < SOCK_RECONF_CTR) {
 207		ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring);
 208		if (!ret)
 209			break;
 210
 211		/* Retry if it fails */
 212		if (ctr >= SOCK_RECONF_CTR || errno != EBUSY)
 213			return -errno;
 214
 215		usleep(USLEEP_MAX);
 216	}
 217
 218	return ret;
 219}
 220
 221int hw_ring_size_reset(struct ifobject *ifobj)
 222{
 223	ifobj->ring.tx_pending = ifobj->set_ring.default_tx;
 224	ifobj->ring.rx_pending = ifobj->set_ring.default_rx;
 225	return set_ring_size(ifobj);
 226}
 227
 228static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
 229			     struct ifobject *ifobj_rx)
 230{
 231	u32 i, j;
 232
 233	for (i = 0; i < MAX_INTERFACES; i++) {
 234		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
 235
 236		ifobj->xsk = &ifobj->xsk_arr[0];
 237		ifobj->use_poll = false;
 238		ifobj->use_fill_ring = true;
 239		ifobj->release_rx = true;
 240		ifobj->validation_func = NULL;
 241		ifobj->use_metadata = false;
 242
 243		if (i == 0) {
 244			ifobj->rx_on = false;
 245			ifobj->tx_on = true;
 246		} else {
 247			ifobj->rx_on = true;
 248			ifobj->tx_on = false;
 249		}
 250
 251		memset(ifobj->umem, 0, sizeof(*ifobj->umem));
 252		ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
 253		ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
 254
 255		for (j = 0; j < MAX_SOCKETS; j++) {
 256			memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
 257			ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
 258			ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE;
 259			if (i == 0)
 260				ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default;
 261			else
 262				ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default;
 263
 264			memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN);
 265			memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN);
 266			ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0);
 267			ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1);
 268		}
 269	}
 270
 271	if (ifobj_tx->hw_ring_size_supp)
 272		hw_ring_size_reset(ifobj_tx);
 273
 274	test->ifobj_tx = ifobj_tx;
 275	test->ifobj_rx = ifobj_rx;
 276	test->current_step = 0;
 277	test->total_steps = 1;
 278	test->nb_sockets = 1;
 279	test->fail = false;
 280	test->set_ring = false;
 281	test->adjust_tail = false;
 282	test->adjust_tail_support = false;
 283	test->mtu = MAX_ETH_PKT_SIZE;
 284	test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;
 285	test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;
 286	test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;
 287	test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;
 288}
 289
 290void test_init(struct test_spec *test, struct ifobject *ifobj_tx,
 291			   struct ifobject *ifobj_rx, enum test_mode mode,
 292			   const struct test_spec *test_to_run)
 293{
 294	struct pkt_stream *tx_pkt_stream;
 295	struct pkt_stream *rx_pkt_stream;
 296	u32 i;
 297
 298	tx_pkt_stream = test->tx_pkt_stream_default;
 299	rx_pkt_stream = test->rx_pkt_stream_default;
 300	memset(test, 0, sizeof(*test));
 301	test->tx_pkt_stream_default = tx_pkt_stream;
 302	test->rx_pkt_stream_default = rx_pkt_stream;
 303
 304	for (i = 0; i < MAX_INTERFACES; i++) {
 305		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
 306
 307		ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
 308		if (mode == TEST_MODE_ZC)
 309			ifobj->bind_flags |= XDP_ZEROCOPY;
 310		else
 311			ifobj->bind_flags |= XDP_COPY;
 312	}
 313
 314	memcpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE);
 315	test->test_func = test_to_run->test_func;
 316	test->mode = mode;
 317	__test_spec_init(test, ifobj_tx, ifobj_rx);
 318}
 319
 320static void test_spec_reset(struct test_spec *test)
 321{
 322	__test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
 323}
 324
 325static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,
 326				   struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,
 327				   struct bpf_map *xskmap_tx)
 328{
 329	test->xdp_prog_rx = xdp_prog_rx;
 330	test->xdp_prog_tx = xdp_prog_tx;
 331	test->xskmap_rx = xskmap_rx;
 332	test->xskmap_tx = xskmap_tx;
 333}
 334
 335static int test_spec_set_mtu(struct test_spec *test, int mtu)
 336{
 337	int err;
 338
 339	if (test->ifobj_rx->mtu != mtu) {
 340		err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu);
 341		if (err)
 342			return err;
 343		test->ifobj_rx->mtu = mtu;
 344	}
 345	if (test->ifobj_tx->mtu != mtu) {
 346		err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);
 347		if (err)
 348			return err;
 349		test->ifobj_tx->mtu = mtu;
 350	}
 351
 352	return 0;
 353}
 354
 355void pkt_stream_reset(struct pkt_stream *pkt_stream)
 356{
 357	if (pkt_stream) {
 358		pkt_stream->current_pkt_nb = 0;
 359		pkt_stream->nb_rx_pkts = 0;
 360	}
 361}
 362
 363static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)
 364{
 365	if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts)
 366		return NULL;
 367
 368	return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
 369}
 370
 371static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
 372{
 373	while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
 374		(*pkts_sent)++;
 375		if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid)
 376			return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
 377		pkt_stream->current_pkt_nb++;
 378	}
 379	return NULL;
 380}
 381
 382void pkt_stream_delete(struct pkt_stream *pkt_stream)
 383{
 384	free(pkt_stream->pkts);
 385	free(pkt_stream);
 386}
 387
 388void pkt_stream_restore_default(struct test_spec *test)
 389{
 390	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;
 391	struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream;
 392
 393	if (tx_pkt_stream != test->tx_pkt_stream_default) {
 394		pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);
 395		test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;
 396	}
 397
 398	if (rx_pkt_stream != test->rx_pkt_stream_default) {
 399		pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);
 400		test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default;
 401	}
 402}
 403
 404static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
 405{
 406	struct pkt_stream *pkt_stream;
 407
 408	pkt_stream = calloc(1, sizeof(*pkt_stream));
 409	if (!pkt_stream)
 410		return NULL;
 411
 412	pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
 413	if (!pkt_stream->pkts) {
 414		free(pkt_stream);
 415		return NULL;
 416	}
 417
 418	pkt_stream->nb_pkts = nb_pkts;
 419	return pkt_stream;
 420}
 421
 422static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt)
 423{
 424	u32 nb_frags = 1, next_frag;
 425
 426	if (!pkt)
 427		return 1;
 428
 429	if (!pkt_stream->verbatim) {
 430		if (!pkt->valid || !pkt->len)
 431			return 1;
 432		return ceil_u32(pkt->len, frame_size);
 433	}
 434
 435	/* Search for the end of the packet in verbatim mode */
 436	if (!pkt_continues(pkt->options) || !pkt->valid)
 437		return nb_frags;
 438
 439	next_frag = pkt_stream->current_pkt_nb;
 440	pkt++;
 441	while (next_frag++ < pkt_stream->nb_pkts) {
 442		nb_frags++;
 443		if (!pkt_continues(pkt->options) || !pkt->valid)
 444			break;
 445		pkt++;
 446	}
 447	return nb_frags;
 448}
 449
 450static bool set_pkt_valid(int offset, u32 len)
 451{
 452	return len <= MAX_ETH_JUMBO_SIZE;
 453}
 454
 455static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
 456{
 457	pkt->offset = offset;
 458	pkt->len = len;
 459	pkt->valid = set_pkt_valid(offset, len);
 460}
 461
 462static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
 463{
 464	bool prev_pkt_valid = pkt->valid;
 465
 466	pkt_set(pkt_stream, pkt, offset, len);
 467	pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid;
 468}
 469
 470static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
 471{
 472	return ceil_u32(len, umem->frame_size) * umem->frame_size;
 473}
 474
 475static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off)
 476{
 477	struct pkt_stream *pkt_stream;
 478	u32 i;
 479
 480	pkt_stream = __pkt_stream_alloc(nb_pkts);
 481	if (!pkt_stream)
 482		return NULL;
 483
 484	pkt_stream->nb_pkts = nb_pkts;
 485	pkt_stream->max_pkt_len = pkt_len;
 486	for (i = 0; i < nb_pkts; i++) {
 487		struct pkt *pkt = &pkt_stream->pkts[i];
 488
 489		pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len);
 490		pkt->pkt_nb = nb_start + i * nb_off;
 491	}
 492
 493	return pkt_stream;
 494}
 495
 496struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)
 497{
 498	return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1);
 499}
 500
 501static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream)
 502{
 503	return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
 504}
 505
 506static int pkt_stream_replace_ifobject(struct ifobject *ifobj, u32 nb_pkts, u32 pkt_len)
 507{
 508	ifobj->xsk->pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
 509
 510	if (!ifobj->xsk->pkt_stream)
 511		return -ENOMEM;
 512
 513	return 0;
 514}
 515
 516static int pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
 517{
 518	int ret;
 519
 520	ret = pkt_stream_replace_ifobject(test->ifobj_tx, nb_pkts, pkt_len);
 521	if (ret)
 522		return ret;
 523
 524	return pkt_stream_replace_ifobject(test->ifobj_rx, nb_pkts, pkt_len);
 525}
 526
 527static int __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
 528				      int offset)
 529{
 530	struct pkt_stream *pkt_stream;
 531	u32 i;
 532
 533	pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream);
 534	if (!pkt_stream)
 535		return -ENOMEM;
 536
 537	for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2)
 538		pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len);
 539
 540	ifobj->xsk->pkt_stream = pkt_stream;
 541
 542	return 0;
 543}
 544
 545static int pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
 546{
 547	int ret = __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
 548
 549	if (ret)
 550		return ret;
 551
 552	return __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
 553}
 554
 555static int pkt_stream_receive_half(struct test_spec *test)
 556{
 557	struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;
 558	u32 i;
 559
 560	if (test->ifobj_rx->xsk->pkt_stream != test->rx_pkt_stream_default)
 561		/* Packet stream has already been replaced so we have to release this one.
 562		 * The newly created one will be freed by the restore_default() at the
 563		 * end of the test
 564		 */
 565		pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);
 566
 567	test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts,
 568							      pkt_stream->pkts[0].len);
 569	if (!test->ifobj_rx->xsk->pkt_stream)
 570		return -ENOMEM;
 571
 572	pkt_stream = test->ifobj_rx->xsk->pkt_stream;
 573	for (i = 1; i < pkt_stream->nb_pkts; i += 2)
 574		pkt_stream->pkts[i].valid = false;
 575
 576	pkt_stream->nb_valid_entries /= 2;
 577
 578	return 0;
 579}
 580
 581static int pkt_stream_even_odd_sequence(struct test_spec *test)
 582{
 583	struct pkt_stream *pkt_stream;
 584	u32 i;
 585
 586	for (i = 0; i < test->nb_sockets; i++) {
 587		pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;
 588		pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
 589						   pkt_stream->pkts[0].len, i, 2);
 590		if (!pkt_stream)
 591			return -ENOMEM;
 592		test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;
 593
 594		pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream;
 595		pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
 596						   pkt_stream->pkts[0].len, i, 2);
 597		if (!pkt_stream)
 598			return -ENOMEM;
 599		test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream;
 600	}
 601
 602	return 0;
 603}
 604
 605static void release_even_odd_sequence(struct test_spec *test)
 606{
 607	struct pkt_stream *later_free_tx = test->ifobj_tx->xsk->pkt_stream;
 608	struct pkt_stream *later_free_rx = test->ifobj_rx->xsk->pkt_stream;
 609	int i;
 610
 611	for (i = 0; i < test->nb_sockets; i++) {
 612		/* later_free_{rx/tx} will be freed by restore_default() */
 613		if (test->ifobj_tx->xsk_arr[i].pkt_stream != later_free_tx)
 614			pkt_stream_delete(test->ifobj_tx->xsk_arr[i].pkt_stream);
 615		if (test->ifobj_rx->xsk_arr[i].pkt_stream != later_free_rx)
 616			pkt_stream_delete(test->ifobj_rx->xsk_arr[i].pkt_stream);
 617	}
 618
 619}
 620
 621static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)
 622{
 623	if (!pkt->valid)
 624		return pkt->offset;
 625	return pkt->offset + umem_alloc_buffer(umem);
 626}
 627
 628static void pkt_stream_cancel(struct pkt_stream *pkt_stream)
 629{
 630	pkt_stream->current_pkt_nb--;
 631}
 632
 633static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len,
 634			 u32 pkt_nb, u32 bytes_written)
 635{
 636	void *data = xsk_umem__get_data(umem->buffer, addr);
 637
 638	if (len < MIN_PKT_SIZE)
 639		return;
 640
 641	if (!bytes_written) {
 642		gen_eth_hdr(xsk, data);
 643
 644		len -= PKT_HDR_SIZE;
 645		data += PKT_HDR_SIZE;
 646	} else {
 647		bytes_written -= PKT_HDR_SIZE;
 648	}
 649
 650	write_payload(data, pkt_nb, bytes_written, len);
 651}
 652
 653static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,
 654						       u32 nb_frames, bool verbatim)
 655{
 656	u32 i, len = 0, pkt_nb = 0, payload = 0;
 657	struct pkt_stream *pkt_stream;
 658
 659	pkt_stream = __pkt_stream_alloc(nb_frames);
 660	if (!pkt_stream)
 661		return NULL;
 662
 663	for (i = 0; i < nb_frames; i++) {
 664		struct pkt *pkt = &pkt_stream->pkts[pkt_nb];
 665		struct pkt *frame = &frames[i];
 666
 667		pkt->offset = frame->offset;
 668		if (verbatim) {
 669			*pkt = *frame;
 670			pkt->pkt_nb = payload;
 671			if (!frame->valid || !pkt_continues(frame->options))
 672				payload++;
 673		} else {
 674			if (frame->valid)
 675				len += frame->len;
 676			if (frame->valid && pkt_continues(frame->options))
 677				continue;
 678
 679			pkt->pkt_nb = pkt_nb;
 680			pkt->len = len;
 681			pkt->valid = frame->valid;
 682			pkt->options = 0;
 683
 684			len = 0;
 685		}
 686
 687		print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n",
 688			      pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb);
 689
 690		if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)
 691			pkt_stream->max_pkt_len = pkt->len;
 692
 693		if (pkt->valid)
 694			pkt_stream->nb_valid_entries++;
 695
 696		pkt_nb++;
 697	}
 698
 699	pkt_stream->nb_pkts = pkt_nb;
 700	pkt_stream->verbatim = verbatim;
 701	return pkt_stream;
 702}
 703
 704static int pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
 705{
 706	struct pkt_stream *pkt_stream;
 707
 708	pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);
 709	if (!pkt_stream)
 710		return -ENOMEM;
 711	test->ifobj_tx->xsk->pkt_stream = pkt_stream;
 712
 713	pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);
 714	if (!pkt_stream)
 715		return -ENOMEM;
 716	test->ifobj_rx->xsk->pkt_stream = pkt_stream;
 717
 718	return 0;
 719}
 720
 721static void pkt_print_data(u32 *data, u32 cnt)
 722{
 723	u32 i;
 724
 725	for (i = 0; i < cnt; i++) {
 726		u32 seqnum, pkt_nb;
 727
 728		seqnum = ntohl(*data) & 0xffff;
 729		pkt_nb = ntohl(*data) >> 16;
 730		ksft_print_msg("%u:%u ", pkt_nb, seqnum);
 731		data++;
 732	}
 733}
 734
 735static void pkt_dump(void *pkt, u32 len, bool eth_header)
 736{
 737	struct ethhdr *ethhdr = pkt;
 738	u32 i, *data;
 739
 740	if (eth_header) {
 741		/*extract L2 frame */
 742		ksft_print_msg("DEBUG>> L2: dst mac: ");
 743		for (i = 0; i < ETH_ALEN; i++)
 744			ksft_print_msg("%02X", ethhdr->h_dest[i]);
 745
 746		ksft_print_msg("\nDEBUG>> L2: src mac: ");
 747		for (i = 0; i < ETH_ALEN; i++)
 748			ksft_print_msg("%02X", ethhdr->h_source[i]);
 749
 750		data = pkt + PKT_HDR_SIZE;
 751	} else {
 752		data = pkt;
 753	}
 754
 755	/*extract L5 frame */
 756	ksft_print_msg("\nDEBUG>> L5: seqnum: ");
 757	pkt_print_data(data, PKT_DUMP_NB_TO_PRINT);
 758	ksft_print_msg("....");
 759	if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) {
 760		ksft_print_msg("\n.... ");
 761		pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT,
 762			       PKT_DUMP_NB_TO_PRINT);
 763	}
 764	ksft_print_msg("\n---------------------------------------\n");
 765}
 766
 767static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr)
 768{
 769	u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
 770	u32 offset = addr % umem->frame_size, expected_offset;
 771	int pkt_offset = pkt->valid ? pkt->offset : 0;
 772
 773	if (!umem->unaligned_mode)
 774		pkt_offset = 0;
 775
 776	expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
 777
 778	if (offset == expected_offset)
 779		return true;
 780
 781	ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
 782	return false;
 783}
 784
 785static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)
 786{
 787	void *data = xsk_umem__get_data(buffer, addr);
 788	struct xdp_info *meta = data - sizeof(struct xdp_info);
 789
 790	if (meta->count != pkt->pkt_nb) {
 791		ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n",
 792			       __func__, pkt->pkt_nb,
 793			       (unsigned long long)meta->count);
 794		return false;
 795	}
 796
 797	return true;
 798}
 799
 800static int is_adjust_tail_supported(struct xsk_xdp_progs *skel_rx, bool *supported)
 801{
 802	struct bpf_map *data_map;
 803	int adjust_value = 0;
 804	int key = 0;
 805	int ret;
 806
 807	data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
 808	if (!data_map || !bpf_map__is_internal(data_map)) {
 809		ksft_print_msg("Error: could not find bss section of XDP program\n");
 810		return -EINVAL;
 811	}
 812
 813	ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &key, &adjust_value);
 814	if (ret) {
 815		ksft_print_msg("Error: bpf_map_lookup_elem failed with error %d\n", ret);
 816		return ret;
 817	}
 818
 819	/* Set the 'adjust_value' variable to -EOPNOTSUPP in the XDP program if the adjust_tail
 820	 * helper is not supported. Skip the adjust_tail test case in this scenario.
 821	 */
 822	*supported = adjust_value != -EOPNOTSUPP;
 823
 824	return 0;
 825}
 826
 827static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb,
 828			  u32 bytes_processed)
 829{
 830	u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum;
 831	void *data = xsk_umem__get_data(umem->buffer, addr);
 832
 833	addr -= umem->base_addr;
 834
 835	if (addr >= umem->num_frames * umem->frame_size ||
 836	    addr + len > umem->num_frames * umem->frame_size) {
 837		ksft_print_msg("Frag invalid addr: %llx len: %u\n",
 838			       (unsigned long long)addr, len);
 839		return false;
 840	}
 841	if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) {
 842		ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n",
 843			       (unsigned long long)addr, len);
 844		return false;
 845	}
 846
 847	pkt_data = data;
 848	if (!bytes_processed) {
 849		pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data);
 850		len -= PKT_HDR_SIZE;
 851	} else {
 852		bytes_processed -= PKT_HDR_SIZE;
 853	}
 854
 855	expected_seqnum = bytes_processed / sizeof(*pkt_data);
 856	seqnum = ntohl(*pkt_data) & 0xffff;
 857	pkt_nb = ntohl(*pkt_data) >> 16;
 858
 859	if (expected_pkt_nb != pkt_nb) {
 860		ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n",
 861			       __func__, expected_pkt_nb, pkt_nb);
 862		goto error;
 863	}
 864	if (expected_seqnum != seqnum) {
 865		ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n",
 866			       __func__, expected_seqnum, seqnum);
 867		goto error;
 868	}
 869
 870	words_to_end = len / sizeof(*pkt_data) - 1;
 871	pkt_data += words_to_end;
 872	seqnum = ntohl(*pkt_data) & 0xffff;
 873	expected_seqnum += words_to_end;
 874	if (expected_seqnum != seqnum) {
 875		ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n",
 876			       __func__, expected_seqnum, seqnum);
 877		goto error;
 878	}
 879
 880	return true;
 881
 882error:
 883	pkt_dump(data, len, !bytes_processed);
 884	return false;
 885}
 886
 887static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
 888{
 889	if (pkt->len != len) {
 890		ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n",
 891			       __func__, pkt->len, len);
 892		pkt_dump(xsk_umem__get_data(buffer, addr), len, true);
 893		return false;
 894	}
 895
 896	return true;
 897}
 898
 899static u32 load_value(u32 *counter)
 900{
 901	return __atomic_load_n(counter, __ATOMIC_ACQUIRE);
 902}
 903
 904static bool kick_tx_with_check(struct xsk_socket_info *xsk, int *ret)
 905{
 906	u32 max_budget = MAX_TX_BUDGET_DEFAULT;
 907	u32 cons, ready_to_send;
 908	int delta;
 909
 910	cons = load_value(xsk->tx.consumer);
 911	ready_to_send = load_value(xsk->tx.producer) - cons;
 912	*ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
 913
 914	delta = load_value(xsk->tx.consumer) - cons;
 915	/* By default, xsk should consume exact @max_budget descs at one
 916	 * send in this case where hitting the max budget limit in while
 917	 * loop is triggered in __xsk_generic_xmit(). Please make sure that
 918	 * the number of descs to be sent is larger than @max_budget, or
 919	 * else the tx.consumer will be updated in xskq_cons_peek_desc()
 920	 * in time which hides the issue we try to verify.
 921	 */
 922	if (ready_to_send > max_budget && delta != max_budget)
 923		return false;
 924
 925	return true;
 926}
 927
 928int kick_tx(struct xsk_socket_info *xsk)
 929{
 930	int ret;
 931
 932	if (xsk->check_consumer) {
 933		if (!kick_tx_with_check(xsk, &ret))
 934			return TEST_FAILURE;
 935	} else {
 936		ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
 937	}
 938	if (ret >= 0)
 939		return TEST_PASS;
 940	if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
 941		usleep(100);
 942		return TEST_PASS;
 943	}
 944	return TEST_FAILURE;
 945}
 946
 947int kick_rx(struct xsk_socket_info *xsk)
 948{
 949	int ret;
 950
 951	ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
 952	if (ret < 0)
 953		return TEST_FAILURE;
 954
 955	return TEST_PASS;
 956}
 957
 958static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
 959{
 960	unsigned int rcvd;
 961	u32 idx;
 962	int ret;
 963
 964	if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
 965		ret = kick_tx(xsk);
 966		if (ret)
 967			return TEST_FAILURE;
 968	}
 969
 970	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
 971	if (rcvd) {
 972		if (rcvd > xsk->outstanding_tx) {
 973			u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
 974
 975			ksft_print_msg("[%s] Too many packets completed\n", __func__);
 976			ksft_print_msg("Last completion address: %llx\n",
 977				       (unsigned long long)addr);
 978			return TEST_FAILURE;
 979		}
 980
 981		xsk_ring_cons__release(&xsk->umem->cq, rcvd);
 982		xsk->outstanding_tx -= rcvd;
 983	}
 984
 985	return TEST_PASS;
 986}
 987
 988static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk)
 989{
 990	u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
 991	u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;
 992	struct pkt_stream *pkt_stream = xsk->pkt_stream;
 993	struct ifobject *ifobj = test->ifobj_rx;
 994	struct xsk_umem_info *umem = xsk->umem;
 995	struct pollfd fds = { };
 996	struct pkt *pkt;
 997	u64 first_addr = 0;
 998	int ret;
 999
1000	fds.fd = xsk_socket__fd(xsk->xsk);
1001	fds.events = POLLIN;
1002
1003	ret = kick_rx(xsk);
1004	if (ret)
1005		return TEST_FAILURE;
1006
1007	if (ifobj->use_poll) {
1008		ret = poll(&fds, 1, POLL_TMOUT);
1009		if (ret < 0)
1010			return TEST_FAILURE;
1011
1012		if (!ret) {
1013			if (!is_umem_valid(test->ifobj_tx))
1014				return TEST_PASS;
1015
1016			ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
1017			return TEST_CONTINUE;
1018		}
1019
1020		if (!(fds.revents & POLLIN))
1021			return TEST_CONTINUE;
1022	}
1023
1024	rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx);
1025	if (!rcvd)
1026		return TEST_CONTINUE;
1027
1028	if (ifobj->use_fill_ring) {
1029		ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1030		while (ret != rcvd) {
1031			if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
1032				ret = poll(&fds, 1, POLL_TMOUT);
1033				if (ret < 0)
1034					return TEST_FAILURE;
1035			}
1036			ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1037		}
1038	}
1039
1040	while (frags_processed < rcvd) {
1041		const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
1042		u64 addr = desc->addr, orig;
1043
1044		orig = xsk_umem__extract_addr(addr);
1045		addr = xsk_umem__add_offset_to_addr(addr);
1046
1047		if (!nb_frags) {
1048			pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
1049			if (!pkt) {
1050				ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",
1051					       __func__, addr, desc->len);
1052				return TEST_FAILURE;
1053			}
1054		}
1055
1056		print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",
1057			      addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);
1058
1059		if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
1060		    !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata &&
1061		    !is_metadata_correct(pkt, umem->buffer, addr)))
1062			return TEST_FAILURE;
1063
1064		if (!nb_frags++)
1065			first_addr = addr;
1066		frags_processed++;
1067		pkt_len += desc->len;
1068		if (ifobj->use_fill_ring)
1069			*xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
1070
1071		if (pkt_continues(desc->options))
1072			continue;
1073
1074		/* The complete packet has been received */
1075		if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
1076		    !is_offset_correct(umem, pkt, addr))
1077			return TEST_FAILURE;
1078
1079		pkt_stream->nb_rx_pkts++;
1080		nb_frags = 0;
1081		pkt_len = 0;
1082	}
1083
1084	if (nb_frags) {
1085		/* In the middle of a packet. Start over from beginning of packet. */
1086		idx_rx -= nb_frags;
1087		xsk_ring_cons__cancel(&xsk->rx, nb_frags);
1088		if (ifobj->use_fill_ring) {
1089			idx_fq -= nb_frags;
1090			xsk_ring_prod__cancel(&umem->fq, nb_frags);
1091		}
1092		frags_processed -= nb_frags;
1093		pkt_stream_cancel(pkt_stream);
1094		pkts_sent--;
1095	}
1096
1097	if (ifobj->use_fill_ring)
1098		xsk_ring_prod__submit(&umem->fq, frags_processed);
1099	if (ifobj->release_rx)
1100		xsk_ring_cons__release(&xsk->rx, frags_processed);
1101
1102	pthread_mutex_lock(&pacing_mutex);
1103	pkts_in_flight -= pkts_sent;
1104	pthread_mutex_unlock(&pacing_mutex);
1105	pkts_sent = 0;
1106
1107	return TEST_CONTINUE;
1108}
1109
1110bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num,
1111			  unsigned long *bitmap)
1112{
1113	struct pkt_stream *pkt_stream = xsk->pkt_stream;
1114
1115	if (!pkt_stream) {
1116		__set_bit(sock_num, bitmap);
1117		return false;
1118	}
1119
1120	if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) {
1121		__set_bit(sock_num, bitmap);
1122		if (bitmap_full(bitmap, test->nb_sockets))
1123			return true;
1124	}
1125
1126	return false;
1127}
1128
1129static int receive_pkts(struct test_spec *test)
1130{
1131	struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1132	DECLARE_BITMAP(bitmap, test->nb_sockets);
1133	struct xsk_socket_info *xsk;
1134	u32 sock_num = 0;
1135	int res, ret;
1136
1137	bitmap_zero(bitmap, test->nb_sockets);
1138
1139	ret = gettimeofday(&tv_now, NULL);
1140	if (ret)
1141		return TEST_FAILURE;
1142
1143	timeradd(&tv_now, &tv_timeout, &tv_end);
1144
1145	while (1) {
1146		xsk = &test->ifobj_rx->xsk_arr[sock_num];
1147
1148		if ((all_packets_received(test, xsk, sock_num, bitmap)))
1149			break;
1150
1151		res = __receive_pkts(test, xsk);
1152		if (!(res == TEST_PASS || res == TEST_CONTINUE))
1153			return res;
1154
1155		ret = gettimeofday(&tv_now, NULL);
1156		if (ret)
1157			return TEST_FAILURE;
1158
1159		if (timercmp(&tv_now, &tv_end, >)) {
1160			ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
1161			return TEST_FAILURE;
1162		}
1163		sock_num = (sock_num + 1) % test->nb_sockets;
1164	}
1165
1166	return TEST_PASS;
1167}
1168
1169static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout)
1170{
1171	u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
1172	struct pkt_stream *pkt_stream = xsk->pkt_stream;
1173	struct xsk_umem_info *umem = ifobject->umem;
1174	bool use_poll = ifobject->use_poll;
1175	struct pollfd fds = { };
1176	int ret;
1177
1178	buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);
1179	/* pkts_in_flight might be negative if many invalid packets are sent */
1180	if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) /
1181	    buffer_len)) {
1182		ret = kick_tx(xsk);
1183		if (ret)
1184			return TEST_FAILURE;
1185		return TEST_CONTINUE;
1186	}
1187
1188	fds.fd = xsk_socket__fd(xsk->xsk);
1189	fds.events = POLLOUT;
1190
1191	while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) {
1192		if (use_poll) {
1193			ret = poll(&fds, 1, POLL_TMOUT);
1194			if (timeout) {
1195				if (ret < 0) {
1196					ksft_print_msg("ERROR: [%s] Poll error %d\n",
1197						       __func__, errno);
1198					return TEST_FAILURE;
1199				}
1200				if (ret == 0)
1201					return TEST_PASS;
1202				break;
1203			}
1204			if (ret <= 0) {
1205				ksft_print_msg("ERROR: [%s] Poll error %d\n",
1206					       __func__, errno);
1207				return TEST_FAILURE;
1208			}
1209		}
1210
1211		complete_pkts(xsk, xsk->batch_size);
1212	}
1213
1214	for (i = 0; i < xsk->batch_size; i++) {
1215		struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1216		u32 nb_frags_left, nb_frags, bytes_written = 0;
1217
1218		if (!pkt)
1219			break;
1220
1221		nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt);
1222		if (nb_frags > xsk->batch_size - i) {
1223			pkt_stream_cancel(pkt_stream);
1224			xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i);
1225			break;
1226		}
1227		nb_frags_left = nb_frags;
1228
1229		while (nb_frags_left--) {
1230			struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1231
1232			tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);
1233			if (pkt_stream->verbatim) {
1234				tx_desc->len = pkt->len;
1235				tx_desc->options = pkt->options;
1236			} else if (nb_frags_left) {
1237				tx_desc->len = umem->frame_size;
1238				tx_desc->options = XDP_PKT_CONTD;
1239			} else {
1240				tx_desc->len = pkt->len - bytes_written;
1241				tx_desc->options = 0;
1242			}
1243			if (pkt->valid)
1244				pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
1245					     bytes_written);
1246			bytes_written += tx_desc->len;
1247
1248			print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n",
1249				      tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb);
1250
1251			if (nb_frags_left) {
1252				i++;
1253				if (pkt_stream->verbatim)
1254					pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1255			}
1256		}
1257
1258		if (pkt && pkt->valid) {
1259			valid_pkts++;
1260			valid_frags += nb_frags;
1261		}
1262	}
1263
1264	pthread_mutex_lock(&pacing_mutex);
1265	pkts_in_flight += valid_pkts;
1266	pthread_mutex_unlock(&pacing_mutex);
1267
1268	xsk_ring_prod__submit(&xsk->tx, i);
1269	xsk->outstanding_tx += valid_frags;
1270
1271	if (use_poll) {
1272		ret = poll(&fds, 1, POLL_TMOUT);
1273		if (ret <= 0) {
1274			if (ret == 0 && timeout)
1275				return TEST_PASS;
1276
1277			ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1278			return TEST_FAILURE;
1279		}
1280	}
1281
1282	if (!timeout) {
1283		if (complete_pkts(xsk, i))
1284			return TEST_FAILURE;
1285
1286		usleep(10);
1287		return TEST_PASS;
1288	}
1289
1290	return TEST_CONTINUE;
1291}
1292
1293static int wait_for_tx_completion(struct xsk_socket_info *xsk)
1294{
1295	struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1296	int ret;
1297
1298	ret = gettimeofday(&tv_now, NULL);
1299	if (ret)
1300		return TEST_FAILURE;
1301	timeradd(&tv_now, &tv_timeout, &tv_end);
1302
1303	while (xsk->outstanding_tx) {
1304		ret = gettimeofday(&tv_now, NULL);
1305		if (ret)
1306			return TEST_FAILURE;
1307		if (timercmp(&tv_now, &tv_end, >)) {
1308			ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__);
1309			return TEST_FAILURE;
1310		}
1311
1312		complete_pkts(xsk, xsk->batch_size);
1313	}
1314
1315	return TEST_PASS;
1316}
1317
1318bool all_packets_sent(struct test_spec *test, unsigned long *bitmap)
1319{
1320	return bitmap_full(bitmap, test->nb_sockets);
1321}
1322
1323static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1324{
1325	bool timeout = !is_umem_valid(test->ifobj_rx);
1326	DECLARE_BITMAP(bitmap, test->nb_sockets);
1327	u32 i, ret;
1328
1329	bitmap_zero(bitmap, test->nb_sockets);
1330
1331	while (!(all_packets_sent(test, bitmap))) {
1332		for (i = 0; i < test->nb_sockets; i++) {
1333			struct pkt_stream *pkt_stream;
1334
1335			pkt_stream = ifobject->xsk_arr[i].pkt_stream;
1336			if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) {
1337				__set_bit(i, bitmap);
1338				continue;
1339			}
1340			ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout);
1341			if (ret == TEST_CONTINUE && !test->fail)
1342				continue;
1343
1344			if ((ret || test->fail) && !timeout)
1345				return TEST_FAILURE;
1346
1347			if (ret == TEST_PASS && timeout)
1348				return ret;
1349
1350			ret = wait_for_tx_completion(&ifobject->xsk_arr[i]);
1351			if (ret)
1352				return TEST_FAILURE;
1353		}
1354	}
1355
1356	return TEST_PASS;
1357}
1358
1359static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1360{
1361	int fd = xsk_socket__fd(xsk), err;
1362	socklen_t optlen, expected_len;
1363
1364	optlen = sizeof(*stats);
1365	err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1366	if (err) {
1367		ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1368			       __func__, -err, strerror(-err));
1369		return TEST_FAILURE;
1370	}
1371
1372	expected_len = sizeof(struct xdp_statistics);
1373	if (optlen != expected_len) {
1374		ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1375			       __func__, expected_len, optlen);
1376		return TEST_FAILURE;
1377	}
1378
1379	return TEST_PASS;
1380}
1381
1382static int validate_rx_dropped(struct ifobject *ifobject)
1383{
1384	struct xsk_socket *xsk = ifobject->xsk->xsk;
1385	struct xdp_statistics stats;
1386	int err;
1387
1388	err = kick_rx(ifobject->xsk);
1389	if (err)
1390		return TEST_FAILURE;
1391
1392	err = get_xsk_stats(xsk, &stats);
1393	if (err)
1394		return TEST_FAILURE;
1395
1396	/* The receiver calls getsockopt after receiving the last (valid)
1397	 * packet which is not the final packet sent in this test (valid and
1398	 * invalid packets are sent in alternating fashion with the final
1399	 * packet being invalid). Since the last packet may or may not have
1400	 * been dropped already, both outcomes must be allowed.
1401	 */
1402	if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 ||
1403	    stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1)
1404		return TEST_PASS;
1405
1406	return TEST_FAILURE;
1407}
1408
1409static int validate_rx_full(struct ifobject *ifobject)
1410{
1411	struct xsk_socket *xsk = ifobject->xsk->xsk;
1412	struct xdp_statistics stats;
1413	int err;
1414
1415	usleep(1000);
1416	err = kick_rx(ifobject->xsk);
1417	if (err)
1418		return TEST_FAILURE;
1419
1420	err = get_xsk_stats(xsk, &stats);
1421	if (err)
1422		return TEST_FAILURE;
1423
1424	if (stats.rx_ring_full)
1425		return TEST_PASS;
1426
1427	return TEST_FAILURE;
1428}
1429
1430static int validate_fill_empty(struct ifobject *ifobject)
1431{
1432	struct xsk_socket *xsk = ifobject->xsk->xsk;
1433	struct xdp_statistics stats;
1434	int err;
1435
1436	usleep(1000);
1437	err = kick_rx(ifobject->xsk);
1438	if (err)
1439		return TEST_FAILURE;
1440
1441	err = get_xsk_stats(xsk, &stats);
1442	if (err)
1443		return TEST_FAILURE;
1444
1445	if (stats.rx_fill_ring_empty_descs)
1446		return TEST_PASS;
1447
1448	return TEST_FAILURE;
1449}
1450
1451static int validate_tx_invalid_descs(struct ifobject *ifobject)
1452{
1453	struct xsk_socket *xsk = ifobject->xsk->xsk;
1454	int fd = xsk_socket__fd(xsk);
1455	struct xdp_statistics stats;
1456	socklen_t optlen;
1457	int err;
1458
1459	optlen = sizeof(stats);
1460	err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1461	if (err) {
1462		ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1463			       __func__, -err, strerror(-err));
1464		return TEST_FAILURE;
1465	}
1466
1467	if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) {
1468		ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n",
1469			       __func__,
1470			       (unsigned long long)stats.tx_invalid_descs,
1471			       ifobject->xsk->pkt_stream->nb_pkts);
1472		return TEST_FAILURE;
1473	}
1474
1475	return TEST_PASS;
1476}
1477
1478static int xsk_configure(struct test_spec *test, struct ifobject *ifobject,
1479			  struct xsk_umem_info *umem, bool tx)
1480{
1481	int i, ret;
1482
1483	for (i = 0; i < test->nb_sockets; i++) {
1484		bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1485		u32 ctr = 0;
1486
1487		while (ctr++ < SOCK_RECONF_CTR) {
1488			ret = xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1489						     ifobject, shared);
1490			if (!ret)
1491				break;
1492
1493			/* Retry if it fails as xsk_socket__create() is asynchronous */
1494			if (ctr >= SOCK_RECONF_CTR)
1495				return ret;
1496			usleep(USLEEP_MAX);
1497		}
1498		if (ifobject->busy_poll) {
1499			ret = enable_busy_poll(&ifobject->xsk_arr[i]);
1500			if (ret)
1501				return ret;
1502		}
1503	}
1504
1505	return 0;
1506}
1507
1508static int thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1509{
1510	int ret = xsk_configure(test, ifobject, test->ifobj_rx->umem, true);
1511
1512	if (ret)
1513		return ret;
1514	ifobject->xsk = &ifobject->xsk_arr[0];
1515	ifobject->xskmap = test->ifobj_rx->xskmap;
1516	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1517	ifobject->umem->base_addr = 0;
1518
1519	return 0;
1520}
1521
1522static int xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream,
1523				   bool fill_up)
1524{
1525	u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM;
1526	u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts;
1527	int ret;
1528
1529	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1530		buffers_to_fill = umem->num_frames;
1531	else
1532		buffers_to_fill = umem->fill_size;
1533
1534	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1535	if (ret != buffers_to_fill)
1536		return -ENOSPC;
1537
1538	while (filled < buffers_to_fill) {
1539		struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts);
1540		u64 addr;
1541		u32 i;
1542
1543		for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) {
1544			if (!pkt) {
1545				if (!fill_up)
1546					break;
1547				addr = filled * umem->frame_size + umem->base_addr;
1548			} else if (pkt->offset >= 0) {
1549				addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem);
1550			} else {
1551				addr = pkt->offset + umem_alloc_buffer(umem);
1552			}
1553
1554			*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1555			if (++filled >= buffers_to_fill)
1556				break;
1557		}
1558	}
1559	xsk_ring_prod__submit(&umem->fq, filled);
1560	xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled);
1561
1562	pkt_stream_reset(pkt_stream);
1563	umem_reset_alloc(umem);
1564
1565	return 0;
1566}
1567
1568static int thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1569{
1570	LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1571	int mmap_flags;
1572	u64 umem_sz;
1573	void *bufs;
1574	int ret;
1575	u32 i;
1576
1577	umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1578	mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1579
1580	if (ifobject->umem->unaligned_mode)
1581		mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;
1582
1583	if (ifobject->shared_umem)
1584		umem_sz *= 2;
1585
1586	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1587	if (bufs == MAP_FAILED)
1588		return -errno;
1589
1590	ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);
1591	if (ret)
1592		return ret;
1593
1594	ret = xsk_configure(test, ifobject, ifobject->umem, false);
1595	if (ret)
1596		return ret;
1597
1598	ifobject->xsk = &ifobject->xsk_arr[0];
1599
1600	if (!ifobject->rx_on)
1601		return 0;
1602
1603	ret = xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream,
1604				     ifobject->use_fill_ring);
1605	if (ret)
1606		return ret;
1607
1608	for (i = 0; i < test->nb_sockets; i++) {
1609		ifobject->xsk = &ifobject->xsk_arr[i];
1610		ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i);
1611		if (ret)
1612			return ret;
1613	}
1614
1615	return 0;
1616}
1617
1618void *worker_testapp_validate_tx(void *arg)
1619{
1620	struct test_spec *test = (struct test_spec *)arg;
1621	struct ifobject *ifobject = test->ifobj_tx;
1622	int err;
1623
1624	if (test->current_step == 1) {
1625		if (!ifobject->shared_umem) {
1626			if (thread_common_ops(test, ifobject)) {
1627				test->fail = true;
1628				pthread_exit(NULL);
1629			}
1630		} else {
1631			if (thread_common_ops_tx(test, ifobject)) {
1632				test->fail = true;
1633				pthread_exit(NULL);
1634			}
1635		}
1636	}
1637
1638	err = send_pkts(test, ifobject);
1639
1640	if (!err && ifobject->validation_func)
1641		err = ifobject->validation_func(ifobject);
1642	if (err)
1643		test->fail = true;
1644
1645	pthread_exit(NULL);
1646}
1647
1648void *worker_testapp_validate_rx(void *arg)
1649{
1650	struct test_spec *test = (struct test_spec *)arg;
1651	struct ifobject *ifobject = test->ifobj_rx;
1652	int err;
1653
1654	if (test->current_step == 1) {
1655		err = thread_common_ops(test, ifobject);
1656	} else {
1657		xsk_clear_xskmap(ifobject->xskmap);
1658		err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0);
1659		if (err)
1660			ksft_print_msg("Error: Failed to update xskmap, error %s\n",
1661				       strerror(-err));
1662	}
1663
1664	pthread_barrier_wait(&barr);
1665
1666	/* We leave only now in case of error to avoid getting stuck in the barrier */
1667	if (err) {
1668		test->fail = true;
1669		pthread_exit(NULL);
1670	}
1671
1672	err = receive_pkts(test);
1673
1674	if (!err && ifobject->validation_func)
1675		err = ifobject->validation_func(ifobject);
1676
1677	if (err) {
1678		if (!test->adjust_tail) {
1679			test->fail = true;
1680		} else {
1681			bool supported;
1682
1683			if (is_adjust_tail_supported(ifobject->xdp_progs, &supported))
1684				test->fail = true;
1685			else if (!supported)
1686				test->adjust_tail_support = false;
1687			else
1688				test->fail = true;
1689		}
1690	}
1691
1692	pthread_exit(NULL);
1693}
1694
1695static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1696{
1697	u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1698
1699	if (ifobj->shared_umem)
1700		umem_sz *= 2;
1701
1702	umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
1703	xsk_umem__delete(ifobj->umem->umem);
1704	munmap(ifobj->umem->buffer, umem_sz);
1705}
1706
1707static void handler(int signum)
1708{
1709	pthread_exit(NULL);
1710}
1711
1712static bool xdp_prog_changed_rx(struct test_spec *test)
1713{
1714	struct ifobject *ifobj = test->ifobj_rx;
1715
1716	return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;
1717}
1718
1719static bool xdp_prog_changed_tx(struct test_spec *test)
1720{
1721	struct ifobject *ifobj = test->ifobj_tx;
1722
1723	return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode;
1724}
1725
1726static int xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,
1727			     struct bpf_map *xskmap, enum test_mode mode)
1728{
1729	int err;
1730
1731	xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));
1732	err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));
1733	if (err) {
1734		ksft_print_msg("Error attaching XDP program\n");
1735		return err;
1736	}
1737
1738	if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))
1739		if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {
1740			ksft_print_msg("ERROR: XDP prog not in DRV mode\n");
1741			return -EINVAL;
1742		}
1743
1744	ifobj->xdp_prog = xdp_prog;
1745	ifobj->xskmap = xskmap;
1746	ifobj->mode = mode;
1747
1748	return 0;
1749}
1750
1751static int xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,
1752				 struct ifobject *ifobj_tx)
1753{
1754	int err = 0;
1755
1756	if (xdp_prog_changed_rx(test)) {
1757		err = xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);
1758		if (err)
1759			return err;
1760	}
1761
1762	if (!ifobj_tx || ifobj_tx->shared_umem)
1763		return 0;
1764
1765	if (xdp_prog_changed_tx(test))
1766		err = xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);
1767
1768	return err;
1769}
1770
1771static void clean_sockets(struct test_spec *test, struct ifobject *ifobj)
1772{
1773	u32 i;
1774
1775	if (!ifobj || !test)
1776		return;
1777
1778	for (i = 0; i < test->nb_sockets; i++)
1779		xsk_socket__delete(ifobj->xsk_arr[i].xsk);
1780}
1781
1782static void clean_umem(struct test_spec *test, struct ifobject *ifobj1, struct ifobject *ifobj2)
1783{
1784	if (!ifobj1)
1785		return;
1786
1787	testapp_clean_xsk_umem(ifobj1);
1788	if (ifobj2 && !ifobj2->shared_umem)
1789		testapp_clean_xsk_umem(ifobj2);
1790}
1791
1792static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,
1793				      struct ifobject *ifobj2)
1794{
1795	pthread_t t0, t1;
1796	int err;
1797
1798	if (test->mtu > MAX_ETH_PKT_SIZE) {
1799		if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp ||
1800						   (ifobj2 && !ifobj2->multi_buff_zc_supp))) {
1801			ksft_print_msg("Multi buffer for zero-copy not supported.\n");
1802			return TEST_SKIP;
1803		}
1804		if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp ||
1805						   (ifobj2 && !ifobj2->multi_buff_supp))) {
1806			ksft_print_msg("Multi buffer not supported.\n");
1807			return TEST_SKIP;
1808		}
1809	}
1810	err = test_spec_set_mtu(test, test->mtu);
1811	if (err) {
1812		ksft_print_msg("Error, could not set mtu.\n");
1813		return TEST_FAILURE;
1814	}
1815
1816	if (ifobj2) {
1817		if (pthread_barrier_init(&barr, NULL, 2))
1818			return TEST_FAILURE;
1819		pkt_stream_reset(ifobj2->xsk->pkt_stream);
1820	}
1821
1822	test->current_step++;
1823	pkt_stream_reset(ifobj1->xsk->pkt_stream);
1824	pkts_in_flight = 0;
1825
1826	signal(SIGUSR1, handler);
1827	/*Spawn RX thread */
1828	pthread_create(&t0, NULL, ifobj1->func_ptr, test);
1829
1830	if (ifobj2) {
1831		pthread_barrier_wait(&barr);
1832		if (pthread_barrier_destroy(&barr)) {
1833			pthread_kill(t0, SIGUSR1);
1834			clean_sockets(test, ifobj1);
1835			clean_umem(test, ifobj1, NULL);
1836			return TEST_FAILURE;
1837		}
1838
1839		/*Spawn TX thread */
1840		pthread_create(&t1, NULL, ifobj2->func_ptr, test);
1841
1842		pthread_join(t1, NULL);
1843	}
1844
1845	if (!ifobj2)
1846		pthread_kill(t0, SIGUSR1);
1847	else
1848		pthread_join(t0, NULL);
1849
1850	if (test->total_steps == test->current_step || test->fail) {
1851		clean_sockets(test, ifobj1);
1852		clean_sockets(test, ifobj2);
1853		clean_umem(test, ifobj1, ifobj2);
1854	}
1855
1856	if (test->fail)
1857		return TEST_FAILURE;
1858
1859	return TEST_PASS;
1860}
1861
1862static int testapp_validate_traffic(struct test_spec *test)
1863{
1864	struct ifobject *ifobj_rx = test->ifobj_rx;
1865	struct ifobject *ifobj_tx = test->ifobj_tx;
1866
1867	if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) ||
1868	    (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {
1869		ksft_print_msg("No huge pages present.\n");
1870		return TEST_SKIP;
1871	}
1872
1873	if (test->set_ring) {
1874		if (ifobj_tx->hw_ring_size_supp) {
1875			if (set_ring_size(ifobj_tx)) {
1876				ksft_print_msg("Failed to change HW ring size.\n");
1877				return TEST_FAILURE;
1878			}
1879		} else {
1880			ksft_print_msg("Changing HW ring size not supported.\n");
1881			return TEST_SKIP;
1882		}
1883	}
1884
1885	if (xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx))
1886		return TEST_FAILURE;
1887	return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);
1888}
1889
1890static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)
1891{
1892	return __testapp_validate_traffic(test, ifobj, NULL);
1893}
1894
1895int testapp_teardown(struct test_spec *test)
1896{
1897	int i;
1898
1899	for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1900		if (testapp_validate_traffic(test))
1901			return TEST_FAILURE;
1902		test_spec_reset(test);
1903	}
1904
1905	return TEST_PASS;
1906}
1907
1908static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1909{
1910	thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1911	struct ifobject *tmp_ifobj = (*ifobj1);
1912
1913	(*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1914	(*ifobj2)->func_ptr = tmp_func_ptr;
1915
1916	*ifobj1 = *ifobj2;
1917	*ifobj2 = tmp_ifobj;
1918}
1919
1920int testapp_bidirectional(struct test_spec *test)
1921{
1922	int res;
1923
1924	test->ifobj_tx->rx_on = true;
1925	test->ifobj_rx->tx_on = true;
1926	test->total_steps = 2;
1927	if (testapp_validate_traffic(test))
1928		return TEST_FAILURE;
1929
1930	print_verbose("Switching Tx/Rx direction\n");
1931	swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1932	res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);
1933
1934	swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1935	return res;
1936}
1937
1938static int swap_xsk_resources(struct test_spec *test)
1939{
1940	int ret;
1941
1942	test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;
1943	test->ifobj_rx->xsk_arr[0].pkt_stream = NULL;
1944	test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;
1945	test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default;
1946	test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];
1947	test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1];
1948
1949	ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0);
1950	if (ret)
1951		return TEST_FAILURE;
1952
1953	return TEST_PASS;
1954}
1955
1956int testapp_xdp_prog_cleanup(struct test_spec *test)
1957{
1958	test->total_steps = 2;
1959	test->nb_sockets = 2;
1960	if (testapp_validate_traffic(test))
1961		return TEST_FAILURE;
1962
1963	if (swap_xsk_resources(test)) {
1964		clean_sockets(test, test->ifobj_rx);
1965		clean_sockets(test, test->ifobj_tx);
1966		clean_umem(test, test->ifobj_rx, test->ifobj_tx);
1967		return TEST_FAILURE;
1968	}
1969
1970	return testapp_validate_traffic(test);
1971}
1972
1973int testapp_headroom(struct test_spec *test)
1974{
1975	test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1976	return testapp_validate_traffic(test);
1977}
1978
1979int testapp_stats_rx_dropped(struct test_spec *test)
1980{
1981	if (test->mode == TEST_MODE_ZC) {
1982		ksft_print_msg("Can not run RX_DROPPED test for ZC mode\n");
1983		return TEST_SKIP;
1984	}
1985
1986	if (pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0))
1987		return TEST_FAILURE;
1988	test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1989		XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1990	if (pkt_stream_receive_half(test))
1991		return TEST_FAILURE;
1992	test->ifobj_rx->validation_func = validate_rx_dropped;
1993	return testapp_validate_traffic(test);
1994}
1995
1996int testapp_stats_tx_invalid_descs(struct test_spec *test)
1997{
1998	if (pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0))
1999		return TEST_FAILURE;
2000	test->ifobj_tx->validation_func = validate_tx_invalid_descs;
2001	return testapp_validate_traffic(test);
2002}
2003
2004int testapp_stats_rx_full(struct test_spec *test)
2005{
2006	if (pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE))
2007		return TEST_FAILURE;
2008	test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
2009
2010	test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
2011	test->ifobj_rx->release_rx = false;
2012	test->ifobj_rx->validation_func = validate_rx_full;
2013	return testapp_validate_traffic(test);
2014}
2015
2016int testapp_stats_fill_empty(struct test_spec *test)
2017{
2018	if (pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE))
2019		return TEST_FAILURE;
2020	test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
2021
2022	test->ifobj_rx->use_fill_ring = false;
2023	test->ifobj_rx->validation_func = validate_fill_empty;
2024	return testapp_validate_traffic(test);
2025}
2026
2027int testapp_send_receive_unaligned(struct test_spec *test)
2028{
2029	test->ifobj_tx->umem->unaligned_mode = true;
2030	test->ifobj_rx->umem->unaligned_mode = true;
2031	/* Let half of the packets straddle a 4K buffer boundary */
2032	if (pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2))
2033		return TEST_FAILURE;
2034
2035	return testapp_validate_traffic(test);
2036}
2037
2038int testapp_send_receive_unaligned_mb(struct test_spec *test)
2039{
2040	test->mtu = MAX_ETH_JUMBO_SIZE;
2041	test->ifobj_tx->umem->unaligned_mode = true;
2042	test->ifobj_rx->umem->unaligned_mode = true;
2043	if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE))
2044		return TEST_FAILURE;
2045	return testapp_validate_traffic(test);
2046}
2047
2048int testapp_single_pkt(struct test_spec *test)
2049{
2050	struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}};
2051
2052	if (pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)))
2053		return TEST_FAILURE;
2054	return testapp_validate_traffic(test);
2055}
2056
2057int testapp_send_receive_mb(struct test_spec *test)
2058{
2059	test->mtu = MAX_ETH_JUMBO_SIZE;
2060	if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE))
2061		return TEST_FAILURE;
2062
2063	return testapp_validate_traffic(test);
2064}
2065
2066int testapp_invalid_desc_mb(struct test_spec *test)
2067{
2068	struct xsk_umem_info *umem = test->ifobj_tx->umem;
2069	u64 umem_size = umem->num_frames * umem->frame_size;
2070	struct pkt pkts[] = {
2071		/* Valid packet for synch to start with */
2072		{0, MIN_PKT_SIZE, 0, true, 0},
2073		/* Zero frame len is not legal */
2074		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2075		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2076		{0, 0, 0, false, 0},
2077		/* Invalid address in the second frame */
2078		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2079		{umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2080		/* Invalid len in the middle */
2081		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2082		{0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2083		/* Invalid options in the middle */
2084		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2085		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION},
2086		/* Transmit 2 frags, receive 3 */
2087		{0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD},
2088		{0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0},
2089		/* Middle frame crosses chunk boundary with small length */
2090		{0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2091		{-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0},
2092		/* Valid packet for synch so that something is received */
2093		{0, MIN_PKT_SIZE, 0, true, 0}};
2094
2095	if (umem->unaligned_mode) {
2096		/* Crossing a chunk boundary allowed */
2097		pkts[12].valid = true;
2098		pkts[13].valid = true;
2099	}
2100
2101	test->mtu = MAX_ETH_JUMBO_SIZE;
2102	if (pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)))
2103		return TEST_FAILURE;
2104	return testapp_validate_traffic(test);
2105}
2106
2107int testapp_invalid_desc(struct test_spec *test)
2108{
2109	struct xsk_umem_info *umem = test->ifobj_tx->umem;
2110	u64 umem_size = umem->num_frames * umem->frame_size;
2111	struct pkt pkts[] = {
2112		/* Zero packet address allowed */
2113		{0, MIN_PKT_SIZE, 0, true},
2114		/* Allowed packet */
2115		{0, MIN_PKT_SIZE, 0, true},
2116		/* Straddling the start of umem */
2117		{-2, MIN_PKT_SIZE, 0, false},
2118		/* Packet too large */
2119		{0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
2120		/* Up to end of umem allowed */
2121		{umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true},
2122		/* After umem ends */
2123		{umem_size, MIN_PKT_SIZE, 0, false},
2124		/* Straddle the end of umem */
2125		{umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2126		/* Straddle a 4K boundary */
2127		{0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2128		/* Straddle a 2K boundary */
2129		{0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true},
2130		/* Valid packet for synch so that something is received */
2131		{0, MIN_PKT_SIZE, 0, true}};
2132
2133	if (umem->unaligned_mode) {
2134		/* Crossing a page boundary allowed */
2135		pkts[7].valid = true;
2136	}
2137	if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
2138		/* Crossing a 2K frame size boundary not allowed */
2139		pkts[8].valid = false;
2140	}
2141
2142	if (test->ifobj_tx->shared_umem) {
2143		pkts[4].offset += umem_size;
2144		pkts[5].offset += umem_size;
2145		pkts[6].offset += umem_size;
2146	}
2147
2148	if (pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)))
2149		return TEST_FAILURE;
2150	return testapp_validate_traffic(test);
2151}
2152
2153int testapp_xdp_drop(struct test_spec *test)
2154{
2155	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2156	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2157
2158	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,
2159			       skel_rx->maps.xsk, skel_tx->maps.xsk);
2160
2161	if (pkt_stream_receive_half(test))
2162		return TEST_FAILURE;
2163	return testapp_validate_traffic(test);
2164}
2165
2166int testapp_xdp_metadata_copy(struct test_spec *test)
2167{
2168	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2169	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2170
2171	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,
2172			       skel_tx->progs.xsk_xdp_populate_metadata,
2173			       skel_rx->maps.xsk, skel_tx->maps.xsk);
2174	test->ifobj_rx->use_metadata = true;
2175
2176	skel_rx->bss->count = 0;
2177
2178	return testapp_validate_traffic(test);
2179}
2180
2181int testapp_xdp_shared_umem(struct test_spec *test)
2182{
2183	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2184	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2185	int ret;
2186
2187	test->total_steps = 1;
2188	test->nb_sockets = 2;
2189
2190	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem,
2191			       skel_tx->progs.xsk_xdp_shared_umem,
2192			       skel_rx->maps.xsk, skel_tx->maps.xsk);
2193
2194	if (pkt_stream_even_odd_sequence(test))
2195		return TEST_FAILURE;
2196
2197	ret = testapp_validate_traffic(test);
2198
2199	release_even_odd_sequence(test);
2200
2201	return ret;
2202}
2203
2204int testapp_poll_txq_tmout(struct test_spec *test)
2205{
2206	test->ifobj_tx->use_poll = true;
2207	/* create invalid frame by set umem frame_size and pkt length equal to 2048 */
2208	test->ifobj_tx->umem->frame_size = 2048;
2209	if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048))
2210		return TEST_FAILURE;
2211	return testapp_validate_traffic_single_thread(test, test->ifobj_tx);
2212}
2213
2214int testapp_poll_rxq_tmout(struct test_spec *test)
2215{
2216	test->ifobj_rx->use_poll = true;
2217	return testapp_validate_traffic_single_thread(test, test->ifobj_rx);
2218}
2219
2220int testapp_too_many_frags(struct test_spec *test)
2221{
2222	struct pkt *pkts;
2223	u32 max_frags, i;
2224	int ret = TEST_FAILURE;
2225
2226	if (test->mode == TEST_MODE_ZC) {
2227		max_frags = test->ifobj_tx->xdp_zc_max_segs;
2228	} else {
2229		max_frags = get_max_skb_frags();
2230		if (!max_frags) {
2231			ksft_print_msg("Can't get MAX_SKB_FRAGS from system, using default (17)\n");
2232			max_frags = 17;
2233		}
2234		max_frags += 1;
2235	}
2236
2237	pkts = calloc(2 * max_frags + 2, sizeof(struct pkt));
2238	if (!pkts)
2239		return TEST_FAILURE;
2240
2241	test->mtu = MAX_ETH_JUMBO_SIZE;
2242
2243	/* Valid packet for synch */
2244	pkts[0].len = MIN_PKT_SIZE;
2245	pkts[0].valid = true;
2246
2247	/* One valid packet with the max amount of frags */
2248	for (i = 1; i < max_frags + 1; i++) {
2249		pkts[i].len = MIN_PKT_SIZE;
2250		pkts[i].options = XDP_PKT_CONTD;
2251		pkts[i].valid = true;
2252	}
2253	pkts[max_frags].options = 0;
2254
2255	/* An invalid packet with the max amount of frags but signals packet
2256	 * continues on the last frag
2257	 */
2258	for (i = max_frags + 1; i < 2 * max_frags + 1; i++) {
2259		pkts[i].len = MIN_PKT_SIZE;
2260		pkts[i].options = XDP_PKT_CONTD;
2261		pkts[i].valid = false;
2262	}
2263
2264	/* Valid packet for synch */
2265	pkts[2 * max_frags + 1].len = MIN_PKT_SIZE;
2266	pkts[2 * max_frags + 1].valid = true;
2267
2268	if (pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2)) {
2269		free(pkts);
2270		return TEST_FAILURE;
2271	}
2272
2273	ret = testapp_validate_traffic(test);
2274	free(pkts);
2275	return ret;
2276}
2277
2278static int xsk_load_xdp_programs(struct ifobject *ifobj)
2279{
2280	ifobj->xdp_progs = xsk_xdp_progs__open_and_load();
2281	if (libbpf_get_error(ifobj->xdp_progs))
2282		return libbpf_get_error(ifobj->xdp_progs);
2283
2284	return 0;
2285}
2286
2287/* Simple test */
2288static bool hugepages_present(void)
2289{
2290	size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
2291	void *bufs;
2292
2293	bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
2294		    MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB);
2295	if (bufs == MAP_FAILED)
2296		return false;
2297
2298	mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
2299	munmap(bufs, mmap_sz);
2300	return true;
2301}
2302
2303int init_iface(struct ifobject *ifobj, thread_func_t func_ptr)
2304{
2305	LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
2306	int err;
2307
2308	ifobj->func_ptr = func_ptr;
2309
2310	err = xsk_load_xdp_programs(ifobj);
2311	if (err) {
2312		ksft_print_msg("Error loading XDP program\n");
2313		return err;
2314	}
2315
2316	if (hugepages_present())
2317		ifobj->unaligned_supp = true;
2318
2319	err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts);
2320	if (err) {
2321		ksft_print_msg("Error querying XDP capabilities\n");
2322		return err;
2323	}
2324	if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG)
2325		ifobj->multi_buff_supp = true;
2326	if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
2327		if (query_opts.xdp_zc_max_segs > 1) {
2328			ifobj->multi_buff_zc_supp = true;
2329			ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs;
2330		} else {
2331			ifobj->xdp_zc_max_segs = 0;
2332		}
2333	}
2334
2335	return 0;
2336}
2337
2338int testapp_send_receive(struct test_spec *test)
2339{
2340	return testapp_validate_traffic(test);
2341}
2342
2343int testapp_send_receive_2k_frame(struct test_spec *test)
2344{
2345	test->ifobj_tx->umem->frame_size = 2048;
2346	test->ifobj_rx->umem->frame_size = 2048;
2347	if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE))
2348		return TEST_FAILURE;
2349	return testapp_validate_traffic(test);
2350}
2351
2352int testapp_poll_rx(struct test_spec *test)
2353{
2354	test->ifobj_rx->use_poll = true;
2355	return testapp_validate_traffic(test);
2356}
2357
2358int testapp_poll_tx(struct test_spec *test)
2359{
2360	test->ifobj_tx->use_poll = true;
2361	return testapp_validate_traffic(test);
2362}
2363
2364int testapp_aligned_inv_desc(struct test_spec *test)
2365{
2366	return testapp_invalid_desc(test);
2367}
2368
2369int testapp_aligned_inv_desc_2k_frame(struct test_spec *test)
2370{
2371	test->ifobj_tx->umem->frame_size = 2048;
2372	test->ifobj_rx->umem->frame_size = 2048;
2373	return testapp_invalid_desc(test);
2374}
2375
2376int testapp_unaligned_inv_desc(struct test_spec *test)
2377{
2378	test->ifobj_tx->umem->unaligned_mode = true;
2379	test->ifobj_rx->umem->unaligned_mode = true;
2380	return testapp_invalid_desc(test);
2381}
2382
2383int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test)
2384{
2385	u64 page_size, umem_size;
2386
2387	/* Odd frame size so the UMEM doesn't end near a page boundary. */
2388	test->ifobj_tx->umem->frame_size = 4001;
2389	test->ifobj_rx->umem->frame_size = 4001;
2390	test->ifobj_tx->umem->unaligned_mode = true;
2391	test->ifobj_rx->umem->unaligned_mode = true;
2392	/* This test exists to test descriptors that staddle the end of
2393	 * the UMEM but not a page.
2394	 */
2395	page_size = sysconf(_SC_PAGESIZE);
2396	umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;
2397	assert(umem_size % page_size > MIN_PKT_SIZE);
2398	assert(umem_size % page_size < page_size - MIN_PKT_SIZE);
2399
2400	return testapp_invalid_desc(test);
2401}
2402
2403int testapp_aligned_inv_desc_mb(struct test_spec *test)
2404{
2405	return testapp_invalid_desc_mb(test);
2406}
2407
2408int testapp_unaligned_inv_desc_mb(struct test_spec *test)
2409{
2410	test->ifobj_tx->umem->unaligned_mode = true;
2411	test->ifobj_rx->umem->unaligned_mode = true;
2412	return testapp_invalid_desc_mb(test);
2413}
2414
2415int testapp_xdp_metadata(struct test_spec *test)
2416{
2417	return testapp_xdp_metadata_copy(test);
2418}
2419
2420int testapp_xdp_metadata_mb(struct test_spec *test)
2421{
2422	test->mtu = MAX_ETH_JUMBO_SIZE;
2423	return testapp_xdp_metadata_copy(test);
2424}
2425
2426int testapp_hw_sw_min_ring_size(struct test_spec *test)
2427{
2428	int ret;
2429
2430	test->set_ring = true;
2431	test->total_steps = 2;
2432	test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE;
2433	test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2;
2434	test->ifobj_tx->xsk->batch_size = 1;
2435	test->ifobj_rx->xsk->batch_size = 1;
2436	ret = testapp_validate_traffic(test);
2437	if (ret)
2438		return ret;
2439
2440	/* Set batch size to hw_ring_size - 1 */
2441	test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;
2442	test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;
2443	return testapp_validate_traffic(test);
2444}
2445
2446int testapp_hw_sw_max_ring_size(struct test_spec *test)
2447{
2448	u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4;
2449	int ret;
2450
2451	test->set_ring = true;
2452	test->total_steps = 2;
2453	test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending;
2454	test->ifobj_tx->ring.rx_pending  = test->ifobj_tx->ring.rx_max_pending;
2455	test->ifobj_rx->umem->num_frames = max_descs;
2456	test->ifobj_rx->umem->fill_size = max_descs;
2457	test->ifobj_rx->umem->comp_size = max_descs;
2458	test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
2459	test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
2460
2461	ret = testapp_validate_traffic(test);
2462	if (ret)
2463		return ret;
2464
2465	/* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when
2466	 * updating the Rx HW tail register.
2467	 */
2468	test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;
2469	test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;
2470	if (pkt_stream_replace(test, max_descs, MIN_PKT_SIZE)) {
2471		clean_sockets(test, test->ifobj_tx);
2472		clean_sockets(test, test->ifobj_rx);
2473		clean_umem(test, test->ifobj_rx, test->ifobj_tx);
2474		return TEST_FAILURE;
2475	}
2476
2477	return testapp_validate_traffic(test);
2478}
2479
2480static int testapp_xdp_adjust_tail(struct test_spec *test, int adjust_value)
2481{
2482	struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2483	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2484
2485	test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_adjust_tail,
2486			       skel_tx->progs.xsk_xdp_adjust_tail,
2487			       skel_rx->maps.xsk, skel_tx->maps.xsk);
2488
2489	skel_rx->bss->adjust_value = adjust_value;
2490
2491	return testapp_validate_traffic(test);
2492}
2493
2494static int testapp_adjust_tail(struct test_spec *test, u32 value, u32 pkt_len)
2495{
2496	int ret;
2497
2498	test->adjust_tail_support = true;
2499	test->adjust_tail = true;
2500	test->total_steps = 1;
2501
2502	ret = pkt_stream_replace_ifobject(test->ifobj_tx, DEFAULT_BATCH_SIZE, pkt_len);
2503	if (ret)
2504		return TEST_FAILURE;
2505
2506	ret = pkt_stream_replace_ifobject(test->ifobj_rx, DEFAULT_BATCH_SIZE, pkt_len + value);
2507	if (ret)
2508		return TEST_FAILURE;
2509
2510	ret = testapp_xdp_adjust_tail(test, value);
2511	if (ret)
2512		return ret;
2513
2514	if (!test->adjust_tail_support) {
2515		ksft_print_msg("%s %sResize pkt with bpf_xdp_adjust_tail() not supported\n",
2516				      mode_string(test), busy_poll_string(test));
2517		return TEST_SKIP;
2518	}
2519
2520	return 0;
2521}
2522
2523int testapp_adjust_tail_shrink(struct test_spec *test)
2524{
2525	/* Shrink by 4 bytes for testing purpose */
2526	return testapp_adjust_tail(test, -4, MIN_PKT_SIZE * 2);
2527}
2528
2529int testapp_adjust_tail_shrink_mb(struct test_spec *test)
2530{
2531	test->mtu = MAX_ETH_JUMBO_SIZE;
2532	/* Shrink by the frag size */
2533	return testapp_adjust_tail(test, -XSK_UMEM__MAX_FRAME_SIZE, XSK_UMEM__LARGE_FRAME_SIZE * 2);
2534}
2535
2536int testapp_adjust_tail_grow(struct test_spec *test)
2537{
2538	/* Grow by 4 bytes for testing purpose */
2539	return testapp_adjust_tail(test, 4, MIN_PKT_SIZE * 2);
2540}
2541
2542int testapp_adjust_tail_grow_mb(struct test_spec *test)
2543{
2544	test->mtu = MAX_ETH_JUMBO_SIZE;
2545	/* Grow by (frag_size - last_frag_Size) - 1 to stay inside the last fragment */
2546	return testapp_adjust_tail(test, (XSK_UMEM__MAX_FRAME_SIZE / 2) - 1,
2547				   XSK_UMEM__LARGE_FRAME_SIZE * 2);
2548}
2549
2550int testapp_tx_queue_consumer(struct test_spec *test)
2551{
2552	int nr_packets;
2553
2554	if (test->mode == TEST_MODE_ZC) {
2555		ksft_print_msg("Can not run TX_QUEUE_CONSUMER test for ZC mode\n");
2556		return TEST_SKIP;
2557	}
2558
2559	nr_packets = MAX_TX_BUDGET_DEFAULT + 1;
2560	if (pkt_stream_replace(test, nr_packets, MIN_PKT_SIZE))
2561		return TEST_FAILURE;
2562	test->ifobj_tx->xsk->batch_size = nr_packets;
2563	test->ifobj_tx->xsk->check_consumer = true;
2564
2565	return testapp_validate_traffic(test);
2566}
2567
2568struct ifobject *ifobject_create(void)
2569{
2570	struct ifobject *ifobj;
2571
2572	ifobj = calloc(1, sizeof(struct ifobject));
2573	if (!ifobj)
2574		return NULL;
2575
2576	ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
2577	if (!ifobj->xsk_arr)
2578		goto out_xsk_arr;
2579
2580	ifobj->umem = calloc(1, sizeof(*ifobj->umem));
2581	if (!ifobj->umem)
2582		goto out_umem;
2583
2584	return ifobj;
2585
2586out_umem:
2587	free(ifobj->xsk_arr);
2588out_xsk_arr:
2589	free(ifobj);
2590	return NULL;
2591}
2592
2593void ifobject_delete(struct ifobject *ifobj)
2594{
2595	free(ifobj->umem);
2596	free(ifobj->xsk_arr);
2597	free(ifobj);
2598}