toeplitz.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Toeplitz test
  3 *
  4 * 1. Read packets and their rx_hash using PF_PACKET/TPACKET_V3
  5 * 2. Compute the rx_hash in software based on the packet contents
  6 * 3. Compare the two
  7 *
  8 * Optionally, either '-C $rx_irq_cpu_list' or '-r $rps_bitmap' may be given.
  9 *
 10 * If '-C $rx_irq_cpu_list' is given, also
 11 *
 12 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU
 13 * 5. Compute the rxqueue that RSS would select based on this rx_hash
 14 * 6. Using the $rx_irq_cpu_list map, identify the arriving cpu based on rxq irq
 15 * 7. Compare the cpus from 4 and 6
 16 *
 17 * Else if '-r $rps_bitmap' is given, also
 18 *
 19 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU
 20 * 5. Compute the cpu that RPS should select based on rx_hash and $rps_bitmap
 21 * 6. Compare the cpus from 4 and 5
 22 */
 23
 24#define _GNU_SOURCE
 25
 26#include <arpa/inet.h>
 27#include <errno.h>
 28#include <error.h>
 29#include <fcntl.h>
 30#include <getopt.h>
 31#include <linux/filter.h>
 32#include <linux/if_ether.h>
 33#include <linux/if_packet.h>
 34#include <net/if.h>
 35#include <netdb.h>
 36#include <netinet/ip.h>
 37#include <netinet/ip6.h>
 38#include <netinet/tcp.h>
 39#include <netinet/udp.h>
 40#include <poll.h>
 41#include <stdbool.h>
 42#include <stddef.h>
 43#include <stdint.h>
 44#include <stdio.h>
 45#include <stdlib.h>
 46#include <string.h>
 47#include <sys/mman.h>
 48#include <sys/socket.h>
 49#include <sys/stat.h>
 50#include <sys/sysinfo.h>
 51#include <sys/time.h>
 52#include <sys/types.h>
 53#include <unistd.h>
 54
 55#include <ynl.h>
 56#include "ethtool-user.h"
 57
 58#include "kselftest.h"
 59#include "../../../net/lib/ksft.h"
 60
 61#define TOEPLITZ_KEY_MIN_LEN	40
 62#define TOEPLITZ_KEY_MAX_LEN	60
 63
 64#define TOEPLITZ_STR_LEN(K)	(((K) * 3) - 1)	/* hex encoded: AA:BB:CC:...:ZZ */
 65#define TOEPLITZ_STR_MIN_LEN	TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN)
 66#define TOEPLITZ_STR_MAX_LEN	TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN)
 67
 68#define FOUR_TUPLE_MAX_LEN	((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2))
 69
 70#define RSS_MAX_CPUS (1 << 16)	/* real constraint is PACKET_FANOUT_MAX */
 71#define RSS_MAX_INDIR	(1 << 16)
 72
 73#define RPS_MAX_CPUS 16UL	/* must be a power of 2 */
 74
 75/* configuration options (cmdline arguments) */
 76static uint16_t cfg_dport =	8000;
 77static int cfg_family =		AF_INET6;
 78static char *cfg_ifname =	"eth0";
 79static int cfg_num_queues;
 80static int cfg_num_rps_cpus;
 81static bool cfg_sink;
 82static int cfg_type =		SOCK_STREAM;
 83static int cfg_timeout_msec =	1000;
 84static bool cfg_verbose;
 85
 86/* global vars */
 87static int num_cpus;
 88static int ring_block_nr;
 89static int ring_block_sz;
 90
 91/* stats */
 92static int frames_received;
 93static int frames_nohash;
 94static int frames_error;
 95
 96#define log_verbose(args...)	do { if (cfg_verbose) fprintf(stderr, args); } while (0)
 97
 98/* tpacket ring */
 99struct ring_state {
100	int fd;
101	char *mmap;
102	int idx;
103	int cpu;
104};
105
106static unsigned int rx_irq_cpus[RSS_MAX_CPUS];	/* map from rxq to cpu */
107static int rps_silo_to_cpu[RPS_MAX_CPUS];
108static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN];
109static unsigned int rss_indir_tbl[RSS_MAX_INDIR];
110static unsigned int rss_indir_tbl_size;
111static struct ring_state rings[RSS_MAX_CPUS];
112
113static inline uint32_t toeplitz(const unsigned char *four_tuple,
114				const unsigned char *key)
115{
116	int i, bit, ret = 0;
117	uint32_t key32;
118
119	key32 = ntohl(*((uint32_t *)key));
120	key += 4;
121
122	for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) {
123		for (bit = 7; bit >= 0; bit--) {
124			if (four_tuple[i] & (1 << bit))
125				ret ^= key32;
126
127			key32 <<= 1;
128			key32 |= !!(key[0] & (1 << bit));
129		}
130		key++;
131	}
132
133	return ret;
134}
135
136/* Compare computed cpu with arrival cpu from packet_fanout_cpu */
137static void verify_rss(uint32_t rx_hash, int cpu)
138{
139	int queue;
140
141	if (rss_indir_tbl_size)
142		queue = rss_indir_tbl[rx_hash % rss_indir_tbl_size];
143	else
144		queue = rx_hash % cfg_num_queues;
145
146	log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]);
147	if (rx_irq_cpus[queue] != cpu) {
148		log_verbose(". error: rss cpu mismatch (%d)", cpu);
149		frames_error++;
150	}
151}
152
153static void verify_rps(uint64_t rx_hash, int cpu)
154{
155	int silo = (rx_hash * cfg_num_rps_cpus) >> 32;
156
157	log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]);
158	if (rps_silo_to_cpu[silo] != cpu) {
159		log_verbose(". error: rps cpu mismatch (%d)", cpu);
160		frames_error++;
161	}
162}
163
164static void log_rxhash(int cpu, uint32_t rx_hash,
165		       const char *addrs, int addr_len)
166{
167	char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN];
168	uint16_t *ports;
169
170	if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) ||
171	    !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr)))
172		error(1, 0, "address parse error");
173
174	ports = (void *)addrs + (addr_len * 2);
175	log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]",
176		    cpu, rx_hash, saddr, daddr,
177		    ntohs(ports[0]), ntohs(ports[1]));
178}
179
180/* Compare computed rxhash with rxhash received from tpacket_v3 */
181static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu)
182{
183	unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0};
184	uint32_t rx_hash_sw;
185	const char *addrs;
186	int addr_len;
187
188	if (cfg_family == AF_INET) {
189		addr_len = sizeof(struct in_addr);
190		addrs = pkt + offsetof(struct iphdr, saddr);
191	} else {
192		addr_len = sizeof(struct in6_addr);
193		addrs = pkt + offsetof(struct ip6_hdr, ip6_src);
194	}
195
196	memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2));
197	rx_hash_sw = toeplitz(four_tuple, toeplitz_key);
198
199	if (cfg_verbose)
200		log_rxhash(cpu, rx_hash, addrs, addr_len);
201
202	if (rx_hash != rx_hash_sw) {
203		log_verbose(" != expected 0x%x\n", rx_hash_sw);
204		frames_error++;
205		return;
206	}
207
208	log_verbose(" OK");
209	if (cfg_num_queues)
210		verify_rss(rx_hash, cpu);
211	else if (cfg_num_rps_cpus)
212		verify_rps(rx_hash, cpu);
213	log_verbose("\n");
214}
215
216static char *recv_frame(const struct ring_state *ring, char *frame)
217{
218	struct tpacket3_hdr *hdr = (void *)frame;
219
220	if (hdr->hv1.tp_rxhash)
221		verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash,
222			      ring->cpu);
223	else
224		frames_nohash++;
225
226	return frame + hdr->tp_next_offset;
227}
228
229/* A single TPACKET_V3 block can hold multiple frames */
230static bool recv_block(struct ring_state *ring)
231{
232	struct tpacket_block_desc *block;
233	char *frame;
234	int i;
235
236	block = (void *)(ring->mmap + ring->idx * ring_block_sz);
237	if (!(block->hdr.bh1.block_status & TP_STATUS_USER))
238		return false;
239
240	frame = (char *)block;
241	frame += block->hdr.bh1.offset_to_first_pkt;
242
243	for (i = 0; i < block->hdr.bh1.num_pkts; i++) {
244		frame = recv_frame(ring, frame);
245		frames_received++;
246	}
247
248	block->hdr.bh1.block_status = TP_STATUS_KERNEL;
249	ring->idx = (ring->idx + 1) % ring_block_nr;
250
251	return true;
252}
253
254/* simple test: sleep once unconditionally and then process all rings */
255static void process_rings(void)
256{
257	int i;
258
259	usleep(1000 * cfg_timeout_msec);
260
261	for (i = 0; i < num_cpus; i++)
262		do {} while (recv_block(&rings[i]));
263
264	fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n",
265		frames_received - frames_nohash - frames_error,
266		frames_nohash, frames_error);
267}
268
269static char *setup_ring(int fd)
270{
271	struct tpacket_req3 req3 = {0};
272	void *ring;
273
274	req3.tp_retire_blk_tov = cfg_timeout_msec / 8;
275	req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH;
276
277	req3.tp_frame_size = 2048;
278	req3.tp_frame_nr = 1 << 10;
279	req3.tp_block_nr = 16;
280
281	req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr;
282	req3.tp_block_size /= req3.tp_block_nr;
283
284	if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3)))
285		error(1, errno, "setsockopt PACKET_RX_RING");
286
287	ring_block_sz = req3.tp_block_size;
288	ring_block_nr = req3.tp_block_nr;
289
290	ring = mmap(0, req3.tp_block_size * req3.tp_block_nr,
291		    PROT_READ | PROT_WRITE,
292		    MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0);
293	if (ring == MAP_FAILED)
294		error(1, 0, "mmap failed");
295
296	return ring;
297}
298
299static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport)
300{
301	struct sock_filter filter[] = {
302		BPF_STMT(BPF_LD  + BPF_B   + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE),
303		BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4),
304		BPF_STMT(BPF_LD  + BPF_B   + BPF_ABS, off_proto),
305		BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2),
306		BPF_STMT(BPF_LD  + BPF_H   + BPF_ABS, off_dport),
307		BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0),
308		BPF_STMT(BPF_RET + BPF_K, 0),
309		BPF_STMT(BPF_RET + BPF_K, 0xFFFF),
310	};
311	struct sock_fprog prog = {};
312
313	prog.filter = filter;
314	prog.len = ARRAY_SIZE(filter);
315	if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
316		error(1, errno, "setsockopt filter");
317}
318
319/* filter on transport protocol and destination port */
320static void set_filter(int fd)
321{
322	const int off_dport = offsetof(struct tcphdr, dest);	/* same for udp */
323	uint8_t proto;
324
325	proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP;
326	if (cfg_family == AF_INET)
327		__set_filter(fd, offsetof(struct iphdr, protocol), proto,
328			     sizeof(struct iphdr) + off_dport);
329	else
330		__set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto,
331			     sizeof(struct ip6_hdr) + off_dport);
332}
333
334/* drop everything: used temporarily during setup */
335static void set_filter_null(int fd)
336{
337	struct sock_filter filter[] = {
338		BPF_STMT(BPF_RET + BPF_K, 0),
339	};
340	struct sock_fprog prog = {};
341
342	prog.filter = filter;
343	prog.len = ARRAY_SIZE(filter);
344	if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
345		error(1, errno, "setsockopt filter");
346}
347
348static int create_ring(char **ring)
349{
350	struct fanout_args args = {
351		.id = 1,
352		.type_flags = PACKET_FANOUT_CPU,
353		.max_num_members = RSS_MAX_CPUS
354	};
355	struct sockaddr_ll ll = { 0 };
356	int fd, val;
357
358	fd = socket(PF_PACKET, SOCK_DGRAM, 0);
359	if (fd == -1)
360		error(1, errno, "socket creation failed");
361
362	val = TPACKET_V3;
363	if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val)))
364		error(1, errno, "setsockopt PACKET_VERSION");
365	*ring = setup_ring(fd);
366
367	/* block packets until all rings are added to the fanout group:
368	 * else packets can arrive during setup and get misclassified
369	 */
370	set_filter_null(fd);
371
372	ll.sll_family = AF_PACKET;
373	ll.sll_ifindex = if_nametoindex(cfg_ifname);
374	ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) :
375						  htons(ETH_P_IPV6);
376	if (bind(fd, (void *)&ll, sizeof(ll)))
377		error(1, errno, "bind");
378
379	/* must come after bind: verifies all programs in group match */
380	if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) {
381		/* on failure, retry using old API if that is sufficient:
382		 * it has a hard limit of 256 sockets, so only try if
383		 * (a) only testing rxhash, not RSS or (b) <= 256 cpus.
384		 * in this API, the third argument is left implicit.
385		 */
386		if (cfg_num_queues || num_cpus > 256 ||
387		    setsockopt(fd, SOL_PACKET, PACKET_FANOUT,
388			       &args, sizeof(uint32_t)))
389			error(1, errno, "setsockopt PACKET_FANOUT cpu");
390	}
391
392	return fd;
393}
394
395/* setup inet(6) socket to blackhole the test traffic, if arg '-s' */
396static int setup_sink(void)
397{
398	int fd, val;
399
400	fd = socket(cfg_family, cfg_type, 0);
401	if (fd == -1)
402		error(1, errno, "socket %d.%d", cfg_family, cfg_type);
403
404	val = 1 << 20;
405	if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val)))
406		error(1, errno, "setsockopt rcvbuf");
407
408	return fd;
409}
410
411static void setup_rings(void)
412{
413	int i;
414
415	for (i = 0; i < num_cpus; i++) {
416		rings[i].cpu = i;
417		rings[i].fd = create_ring(&rings[i].mmap);
418	}
419
420	/* accept packets once all rings in the fanout group are up */
421	for (i = 0; i < num_cpus; i++)
422		set_filter(rings[i].fd);
423}
424
425static void cleanup_rings(void)
426{
427	int i;
428
429	for (i = 0; i < num_cpus; i++) {
430		if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz))
431			error(1, errno, "munmap");
432		if (close(rings[i].fd))
433			error(1, errno, "close");
434	}
435}
436
437static void parse_cpulist(const char *arg)
438{
439	do {
440		rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10);
441
442		arg = strchr(arg, ',');
443		if (!arg)
444			break;
445		arg++;			// skip ','
446	} while (1);
447}
448
449static void show_cpulist(void)
450{
451	int i;
452
453	for (i = 0; i < cfg_num_queues; i++)
454		fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]);
455}
456
457static void show_silos(void)
458{
459	int i;
460
461	for (i = 0; i < cfg_num_rps_cpus; i++)
462		fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]);
463}
464
465static void parse_toeplitz_key(const char *str, int slen, unsigned char *key)
466{
467	int i, ret, off;
468
469	if (slen < TOEPLITZ_STR_MIN_LEN ||
470	    slen > TOEPLITZ_STR_MAX_LEN + 1)
471		error(1, 0, "invalid toeplitz key");
472
473	for (i = 0, off = 0; off < slen; i++, off += 3) {
474		ret = sscanf(str + off, "%hhx", &key[i]);
475		if (ret != 1)
476			error(1, 0, "key parse error at %d off %d len %d",
477			      i, off, slen);
478	}
479}
480
481static void parse_rps_bitmap(const char *arg)
482{
483	unsigned long bitmap;
484	int i;
485
486	bitmap = strtoul(arg, NULL, 0);
487
488	if (bitmap & ~((1UL << RPS_MAX_CPUS) - 1))
489		error(1, 0, "rps bitmap 0x%lx out of bounds, max cpu %lu",
490		      bitmap, RPS_MAX_CPUS - 1);
491
492	for (i = 0; i < RPS_MAX_CPUS; i++)
493		if (bitmap & 1UL << i)
494			rps_silo_to_cpu[cfg_num_rps_cpus++] = i;
495}
496
497static void read_rss_dev_info_ynl(void)
498{
499	struct ethtool_rss_get_req *req;
500	struct ethtool_rss_get_rsp *rsp;
501	struct ynl_sock *ys;
502
503	ys = ynl_sock_create(&ynl_ethtool_family, NULL);
504	if (!ys)
505		error(1, errno, "ynl_sock_create failed");
506
507	req = ethtool_rss_get_req_alloc();
508	if (!req)
509		error(1, errno, "ethtool_rss_get_req_alloc failed");
510
511	ethtool_rss_get_req_set_header_dev_name(req, cfg_ifname);
512
513	rsp = ethtool_rss_get(ys, req);
514	if (!rsp)
515		error(1, ys->err.code, "YNL: %s", ys->err.msg);
516
517	if (!rsp->_len.hkey)
518		error(1, 0, "RSS key not available for %s", cfg_ifname);
519
520	if (rsp->_len.hkey < TOEPLITZ_KEY_MIN_LEN ||
521	    rsp->_len.hkey > TOEPLITZ_KEY_MAX_LEN)
522		error(1, 0, "RSS key length %u out of bounds [%u, %u]",
523		      rsp->_len.hkey, TOEPLITZ_KEY_MIN_LEN,
524		      TOEPLITZ_KEY_MAX_LEN);
525
526	memcpy(toeplitz_key, rsp->hkey, rsp->_len.hkey);
527
528	if (rsp->_count.indir > RSS_MAX_INDIR)
529		error(1, 0, "RSS indirection table too large (%u > %u)",
530		      rsp->_count.indir, RSS_MAX_INDIR);
531
532	/* If indir table not available we'll fallback to simple modulo math */
533	if (rsp->_count.indir) {
534		memcpy(rss_indir_tbl, rsp->indir,
535		       rsp->_count.indir * sizeof(rss_indir_tbl[0]));
536		rss_indir_tbl_size = rsp->_count.indir;
537
538		log_verbose("RSS indirection table size: %u\n",
539			    rss_indir_tbl_size);
540	}
541
542	ethtool_rss_get_rsp_free(rsp);
543	ethtool_rss_get_req_free(req);
544	ynl_sock_destroy(ys);
545}
546
547static void parse_opts(int argc, char **argv)
548{
549	static struct option long_options[] = {
550	    {"dport",	required_argument, 0, 'd'},
551	    {"cpus",	required_argument, 0, 'C'},
552	    {"key",	required_argument, 0, 'k'},
553	    {"iface",	required_argument, 0, 'i'},
554	    {"ipv4",	no_argument, 0, '4'},
555	    {"ipv6",	no_argument, 0, '6'},
556	    {"sink",	no_argument, 0, 's'},
557	    {"tcp",	no_argument, 0, 't'},
558	    {"timeout",	required_argument, 0, 'T'},
559	    {"udp",	no_argument, 0, 'u'},
560	    {"verbose",	no_argument, 0, 'v'},
561	    {"rps",	required_argument, 0, 'r'},
562	    {0, 0, 0, 0}
563	};
564	bool have_toeplitz = false;
565	int index, c;
566
567	while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) {
568		switch (c) {
569		case '4':
570			cfg_family = AF_INET;
571			break;
572		case '6':
573			cfg_family = AF_INET6;
574			break;
575		case 'C':
576			parse_cpulist(optarg);
577			break;
578		case 'd':
579			cfg_dport = strtol(optarg, NULL, 0);
580			break;
581		case 'i':
582			cfg_ifname = optarg;
583			break;
584		case 'k':
585			parse_toeplitz_key(optarg, strlen(optarg),
586					   toeplitz_key);
587			have_toeplitz = true;
588			break;
589		case 'r':
590			parse_rps_bitmap(optarg);
591			break;
592		case 's':
593			cfg_sink = true;
594			break;
595		case 't':
596			cfg_type = SOCK_STREAM;
597			break;
598		case 'T':
599			cfg_timeout_msec = strtol(optarg, NULL, 0);
600			break;
601		case 'u':
602			cfg_type = SOCK_DGRAM;
603			break;
604		case 'v':
605			cfg_verbose = true;
606			break;
607
608		default:
609			error(1, 0, "unknown option %c", optopt);
610			break;
611		}
612	}
613
614	if (!have_toeplitz)
615		read_rss_dev_info_ynl();
616
617	num_cpus = get_nprocs();
618	if (num_cpus > RSS_MAX_CPUS)
619		error(1, 0, "increase RSS_MAX_CPUS");
620
621	if (cfg_num_queues && cfg_num_rps_cpus)
622		error(1, 0,
623		      "Can't supply both RSS cpus ('-C') and RPS map ('-r')");
624	if (cfg_verbose) {
625		show_cpulist();
626		show_silos();
627	}
628}
629
630int main(int argc, char **argv)
631{
632	const int min_tests = 10;
633	int fd_sink = -1;
634
635	parse_opts(argc, argv);
636
637	if (cfg_sink)
638		fd_sink = setup_sink();
639
640	setup_rings();
641
642	/* Signal to test framework that we're ready to receive */
643	ksft_ready();
644
645	process_rings();
646	cleanup_rings();
647
648	if (cfg_sink && close(fd_sink))
649		error(1, errno, "close sink");
650
651	if (frames_received - frames_nohash < min_tests)
652		error(1, 0, "too few frames for verification");
653
654	return frames_error;
655}