fou_core.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2#include <linux/module.h>
   3#include <linux/errno.h>
   4#include <linux/socket.h>
   5#include <linux/skbuff.h>
   6#include <linux/ip.h>
   7#include <linux/icmp.h>
   8#include <linux/udp.h>
   9#include <linux/types.h>
  10#include <linux/kernel.h>
  11#include <net/genetlink.h>
  12#include <net/gro.h>
  13#include <net/gue.h>
  14#include <net/fou.h>
  15#include <net/ip.h>
  16#include <net/protocol.h>
  17#include <net/udp.h>
  18#include <net/udp_tunnel.h>
  19#include <uapi/linux/fou.h>
  20#include <uapi/linux/genetlink.h>
  21
  22#include "fou_nl.h"
  23
  24struct fou {
  25	struct socket *sock;
  26	u8 protocol;
  27	u8 flags;
  28	__be16 port;
  29	u8 family;
  30	u16 type;
  31	struct list_head list;
  32	struct rcu_head rcu;
  33};
  34
  35#define FOU_F_REMCSUM_NOPARTIAL BIT(0)
  36
  37struct fou_cfg {
  38	u16 type;
  39	u8 protocol;
  40	u8 flags;
  41	struct udp_port_cfg udp_config;
  42};
  43
  44static unsigned int fou_net_id;
  45
  46struct fou_net {
  47	struct list_head fou_list;
  48	struct mutex fou_lock;
  49};
  50
  51static inline struct fou *fou_from_sock(struct sock *sk)
  52{
  53	return rcu_dereference_sk_user_data(sk);
  54}
  55
  56static int fou_recv_pull(struct sk_buff *skb, struct fou *fou, size_t len)
  57{
  58	/* Remove 'len' bytes from the packet (UDP header and
  59	 * FOU header if present).
  60	 */
  61	if (fou->family == AF_INET)
  62		ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
  63	else
  64		ipv6_hdr(skb)->payload_len =
  65		    htons(ntohs(ipv6_hdr(skb)->payload_len) - len);
  66
  67	__skb_pull(skb, len);
  68	skb_postpull_rcsum(skb, udp_hdr(skb), len);
  69	skb_reset_transport_header(skb);
  70	return iptunnel_pull_offloads(skb);
  71}
  72
  73static int fou_udp_recv(struct sock *sk, struct sk_buff *skb)
  74{
  75	struct fou *fou = fou_from_sock(sk);
  76
  77	if (!fou)
  78		return 1;
  79
  80	if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
  81		goto drop;
  82
  83	return -fou->protocol;
  84
  85drop:
  86	kfree_skb(skb);
  87	return 0;
  88}
  89
  90static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr,
  91				  void *data, size_t hdrlen, u8 ipproto,
  92				  bool nopartial)
  93{
  94	__be16 *pd = data;
  95	size_t start = ntohs(pd[0]);
  96	size_t offset = ntohs(pd[1]);
  97	size_t plen = sizeof(struct udphdr) + hdrlen +
  98	    max_t(size_t, offset + sizeof(u16), start);
  99
 100	if (skb->remcsum_offload)
 101		return guehdr;
 102
 103	if (!pskb_may_pull(skb, plen))
 104		return NULL;
 105	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
 106
 107	skb_remcsum_process(skb, (void *)guehdr + hdrlen,
 108			    start, offset, nopartial);
 109
 110	return guehdr;
 111}
 112
 113static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr)
 114{
 115	/* No support yet */
 116	kfree_skb(skb);
 117	return 0;
 118}
 119
 120static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
 121{
 122	struct fou *fou = fou_from_sock(sk);
 123	size_t len, optlen, hdrlen;
 124	struct guehdr *guehdr;
 125	void *data;
 126	u16 doffset = 0;
 127	u8 proto_ctype;
 128
 129	if (!fou)
 130		return 1;
 131
 132	len = sizeof(struct udphdr) + sizeof(struct guehdr);
 133	if (!pskb_may_pull(skb, len))
 134		goto drop;
 135
 136	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
 137
 138	switch (guehdr->version) {
 139	case 0: /* Full GUE header present */
 140		break;
 141
 142	case 1: {
 143		/* Direct encapsulation of IPv4 or IPv6 */
 144
 145		int prot;
 146
 147		switch (((struct iphdr *)guehdr)->version) {
 148		case 4:
 149			prot = IPPROTO_IPIP;
 150			break;
 151		case 6:
 152			prot = IPPROTO_IPV6;
 153			break;
 154		default:
 155			goto drop;
 156		}
 157
 158		if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
 159			goto drop;
 160
 161		return -prot;
 162	}
 163
 164	default: /* Undefined version */
 165		goto drop;
 166	}
 167
 168	optlen = guehdr->hlen << 2;
 169	len += optlen;
 170
 171	if (!pskb_may_pull(skb, len))
 172		goto drop;
 173
 174	/* guehdr may change after pull */
 175	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
 176
 177	if (validate_gue_flags(guehdr, optlen))
 178		goto drop;
 179
 180	hdrlen = sizeof(struct guehdr) + optlen;
 181
 182	if (fou->family == AF_INET)
 183		ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
 184	else
 185		ipv6_hdr(skb)->payload_len =
 186		    htons(ntohs(ipv6_hdr(skb)->payload_len) - len);
 187
 188	/* Pull csum through the guehdr now . This can be used if
 189	 * there is a remote checksum offload.
 190	 */
 191	skb_postpull_rcsum(skb, udp_hdr(skb), len);
 192
 193	data = &guehdr[1];
 194
 195	if (guehdr->flags & GUE_FLAG_PRIV) {
 196		__be32 flags = *(__be32 *)(data + doffset);
 197
 198		doffset += GUE_LEN_PRIV;
 199
 200		if (flags & GUE_PFLAG_REMCSUM) {
 201			guehdr = gue_remcsum(skb, guehdr, data + doffset,
 202					     hdrlen, guehdr->proto_ctype,
 203					     !!(fou->flags &
 204						FOU_F_REMCSUM_NOPARTIAL));
 205			if (!guehdr)
 206				goto drop;
 207
 208			data = &guehdr[1];
 209
 210			doffset += GUE_PLEN_REMCSUM;
 211		}
 212	}
 213
 214	if (unlikely(guehdr->control))
 215		return gue_control_message(skb, guehdr);
 216
 217	proto_ctype = guehdr->proto_ctype;
 218	if (unlikely(!proto_ctype))
 219		goto drop;
 220
 221	__skb_pull(skb, sizeof(struct udphdr) + hdrlen);
 222	skb_reset_transport_header(skb);
 223
 224	if (iptunnel_pull_offloads(skb))
 225		goto drop;
 226
 227	return -proto_ctype;
 228
 229drop:
 230	kfree_skb(skb);
 231	return 0;
 232}
 233
 234static const struct net_offload *fou_gro_ops(const struct sock *sk,
 235					     int proto)
 236{
 237	const struct net_offload __rcu **offloads;
 238
 239	/* FOU doesn't allow IPv4 on IPv6 sockets. */
 240	offloads = sk->sk_family == AF_INET6 ? inet6_offloads : inet_offloads;
 241	return rcu_dereference(offloads[proto]);
 242}
 243
 244static struct sk_buff *fou_gro_receive(struct sock *sk,
 245				       struct list_head *head,
 246				       struct sk_buff *skb)
 247{
 248	struct fou *fou = fou_from_sock(sk);
 249	const struct net_offload *ops;
 250	struct sk_buff *pp = NULL;
 251
 252	if (!fou)
 253		goto out;
 254
 255	/* We can clear the encap_mark for FOU as we are essentially doing
 256	 * one of two possible things.  We are either adding an L4 tunnel
 257	 * header to the outer L3 tunnel header, or we are simply
 258	 * treating the GRE tunnel header as though it is a UDP protocol
 259	 * specific header such as VXLAN or GENEVE.
 260	 */
 261	NAPI_GRO_CB(skb)->encap_mark = 0;
 262
 263	/* Flag this frame as already having an outer encap header */
 264	NAPI_GRO_CB(skb)->is_fou = 1;
 265
 266	ops = fou_gro_ops(sk, fou->protocol);
 267	if (!ops || !ops->callbacks.gro_receive)
 268		goto out;
 269
 270	pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
 271
 272out:
 273	return pp;
 274}
 275
 276static int fou_gro_complete(struct sock *sk, struct sk_buff *skb,
 277			    int nhoff)
 278{
 279	struct fou *fou = fou_from_sock(sk);
 280	const struct net_offload *ops;
 281	int err;
 282
 283	if (!fou) {
 284		err = -ENOENT;
 285		goto out;
 286	}
 287
 288	ops = fou_gro_ops(sk, fou->protocol);
 289	if (WARN_ON(!ops || !ops->callbacks.gro_complete)) {
 290		err = -ENOSYS;
 291		goto out;
 292	}
 293
 294	err = ops->callbacks.gro_complete(skb, nhoff);
 295
 296	skb_set_inner_mac_header(skb, nhoff);
 297
 298out:
 299	return err;
 300}
 301
 302static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off,
 303				      struct guehdr *guehdr, void *data,
 304				      size_t hdrlen, struct gro_remcsum *grc,
 305				      bool nopartial)
 306{
 307	__be16 *pd = data;
 308	size_t start = ntohs(pd[0]);
 309	size_t offset = ntohs(pd[1]);
 310
 311	if (skb->remcsum_offload)
 312		return guehdr;
 313
 314	if (!NAPI_GRO_CB(skb)->csum_valid)
 315		return NULL;
 316
 317	guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen,
 318					 start, offset, grc, nopartial);
 319
 320	skb->remcsum_offload = 1;
 321
 322	return guehdr;
 323}
 324
 325static struct sk_buff *gue_gro_receive(struct sock *sk,
 326				       struct list_head *head,
 327				       struct sk_buff *skb)
 328{
 329	const struct net_offload *ops;
 330	struct sk_buff *pp = NULL;
 331	struct sk_buff *p;
 332	struct guehdr *guehdr;
 333	size_t len, optlen, hdrlen, off;
 334	void *data;
 335	u16 doffset = 0;
 336	int flush = 1;
 337	struct fou *fou = fou_from_sock(sk);
 338	struct gro_remcsum grc;
 339	u8 proto;
 340
 341	skb_gro_remcsum_init(&grc);
 342
 343	if (!fou)
 344		goto out;
 345
 346	off = skb_gro_offset(skb);
 347	len = off + sizeof(*guehdr);
 348
 349	guehdr = skb_gro_header(skb, len, off);
 350	if (unlikely(!guehdr))
 351		goto out;
 352
 353	switch (guehdr->version) {
 354	case 0:
 355		break;
 356	case 1:
 357		switch (((struct iphdr *)guehdr)->version) {
 358		case 4:
 359			proto = IPPROTO_IPIP;
 360			break;
 361		case 6:
 362			proto = IPPROTO_IPV6;
 363			break;
 364		default:
 365			goto out;
 366		}
 367		goto next_proto;
 368	default:
 369		goto out;
 370	}
 371
 372	optlen = guehdr->hlen << 2;
 373	len += optlen;
 374
 375	if (!skb_gro_may_pull(skb, len)) {
 376		guehdr = skb_gro_header_slow(skb, len, off);
 377		if (unlikely(!guehdr))
 378			goto out;
 379	}
 380
 381	if (unlikely(guehdr->control) || guehdr->version != 0 ||
 382	    validate_gue_flags(guehdr, optlen))
 383		goto out;
 384
 385	hdrlen = sizeof(*guehdr) + optlen;
 386
 387	/* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr,
 388	 * this is needed if there is a remote checkcsum offload.
 389	 */
 390	skb_gro_postpull_rcsum(skb, guehdr, hdrlen);
 391
 392	data = &guehdr[1];
 393
 394	if (guehdr->flags & GUE_FLAG_PRIV) {
 395		__be32 flags = *(__be32 *)(data + doffset);
 396
 397		doffset += GUE_LEN_PRIV;
 398
 399		if (flags & GUE_PFLAG_REMCSUM) {
 400			guehdr = gue_gro_remcsum(skb, off, guehdr,
 401						 data + doffset, hdrlen, &grc,
 402						 !!(fou->flags &
 403						    FOU_F_REMCSUM_NOPARTIAL));
 404
 405			if (!guehdr)
 406				goto out;
 407
 408			data = &guehdr[1];
 409
 410			doffset += GUE_PLEN_REMCSUM;
 411		}
 412	}
 413
 414	skb_gro_pull(skb, hdrlen);
 415
 416	list_for_each_entry(p, head, list) {
 417		const struct guehdr *guehdr2;
 418
 419		if (!NAPI_GRO_CB(p)->same_flow)
 420			continue;
 421
 422		guehdr2 = (struct guehdr *)(p->data + off);
 423
 424		/* Compare base GUE header to be equal (covers
 425		 * hlen, version, proto_ctype, and flags.
 426		 */
 427		if (guehdr->word != guehdr2->word) {
 428			NAPI_GRO_CB(p)->same_flow = 0;
 429			continue;
 430		}
 431
 432		/* Compare optional fields are the same. */
 433		if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1],
 434					   guehdr->hlen << 2)) {
 435			NAPI_GRO_CB(p)->same_flow = 0;
 436			continue;
 437		}
 438	}
 439
 440	proto = guehdr->proto_ctype;
 441
 442next_proto:
 443
 444	/* We can clear the encap_mark for GUE as we are essentially doing
 445	 * one of two possible things.  We are either adding an L4 tunnel
 446	 * header to the outer L3 tunnel header, or we are simply
 447	 * treating the GRE tunnel header as though it is a UDP protocol
 448	 * specific header such as VXLAN or GENEVE.
 449	 */
 450	NAPI_GRO_CB(skb)->encap_mark = 0;
 451
 452	/* Flag this frame as already having an outer encap header */
 453	NAPI_GRO_CB(skb)->is_fou = 1;
 454
 455	ops = fou_gro_ops(sk, proto);
 456	if (!ops || !ops->callbacks.gro_receive)
 457		goto out;
 458
 459	pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
 460	flush = 0;
 461
 462out:
 463	skb_gro_flush_final_remcsum(skb, pp, flush, &grc);
 464
 465	return pp;
 466}
 467
 468static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
 469{
 470	struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
 471	const struct net_offload *ops;
 472	unsigned int guehlen = 0;
 473	u8 proto;
 474	int err = -ENOENT;
 475
 476	switch (guehdr->version) {
 477	case 0:
 478		proto = guehdr->proto_ctype;
 479		guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
 480		break;
 481	case 1:
 482		switch (((struct iphdr *)guehdr)->version) {
 483		case 4:
 484			proto = IPPROTO_IPIP;
 485			break;
 486		case 6:
 487			proto = IPPROTO_IPV6;
 488			break;
 489		default:
 490			return err;
 491		}
 492		break;
 493	default:
 494		return err;
 495	}
 496
 497	ops = fou_gro_ops(sk, proto);
 498	if (WARN_ON(!ops || !ops->callbacks.gro_complete))
 499		goto out;
 500
 501	err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
 502
 503	skb_set_inner_mac_header(skb, nhoff + guehlen);
 504
 505out:
 506	return err;
 507}
 508
 509static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg)
 510{
 511	struct sock *sk = fou->sock->sk;
 512	struct udp_port_cfg *udp_cfg = &cfg->udp_config;
 513
 514	if (fou->family != udp_cfg->family ||
 515	    fou->port != udp_cfg->local_udp_port ||
 516	    sk->sk_dport != udp_cfg->peer_udp_port ||
 517	    sk->sk_bound_dev_if != udp_cfg->bind_ifindex)
 518		return false;
 519
 520	if (fou->family == AF_INET) {
 521		if (sk->sk_rcv_saddr != udp_cfg->local_ip.s_addr ||
 522		    sk->sk_daddr != udp_cfg->peer_ip.s_addr)
 523			return false;
 524		else
 525			return true;
 526#if IS_ENABLED(CONFIG_IPV6)
 527	} else {
 528		if (ipv6_addr_cmp(&sk->sk_v6_rcv_saddr, &udp_cfg->local_ip6) ||
 529		    ipv6_addr_cmp(&sk->sk_v6_daddr, &udp_cfg->peer_ip6))
 530			return false;
 531		else
 532			return true;
 533#endif
 534	}
 535
 536	return false;
 537}
 538
 539static int fou_add_to_port_list(struct net *net, struct fou *fou,
 540				struct fou_cfg *cfg)
 541{
 542	struct fou_net *fn = net_generic(net, fou_net_id);
 543	struct fou *fout;
 544
 545	mutex_lock(&fn->fou_lock);
 546	list_for_each_entry(fout, &fn->fou_list, list) {
 547		if (fou_cfg_cmp(fout, cfg)) {
 548			mutex_unlock(&fn->fou_lock);
 549			return -EALREADY;
 550		}
 551	}
 552
 553	list_add(&fou->list, &fn->fou_list);
 554	mutex_unlock(&fn->fou_lock);
 555
 556	return 0;
 557}
 558
 559static void fou_release(struct fou *fou)
 560{
 561	struct socket *sock = fou->sock;
 562
 563	list_del(&fou->list);
 564	udp_tunnel_sock_release(sock);
 565
 566	kfree_rcu(fou, rcu);
 567}
 568
 569static int fou_create(struct net *net, struct fou_cfg *cfg,
 570		      struct socket **sockp)
 571{
 572	struct socket *sock = NULL;
 573	struct fou *fou = NULL;
 574	struct sock *sk;
 575	struct udp_tunnel_sock_cfg tunnel_cfg;
 576	int err;
 577
 578	/* Open UDP socket */
 579	err = udp_sock_create(net, &cfg->udp_config, &sock);
 580	if (err < 0)
 581		goto error;
 582
 583	/* Allocate FOU port structure */
 584	fou = kzalloc(sizeof(*fou), GFP_KERNEL);
 585	if (!fou) {
 586		err = -ENOMEM;
 587		goto error;
 588	}
 589
 590	sk = sock->sk;
 591
 592	fou->port = cfg->udp_config.local_udp_port;
 593	fou->family = cfg->udp_config.family;
 594	fou->flags = cfg->flags;
 595	fou->type = cfg->type;
 596	fou->sock = sock;
 597
 598	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
 599	tunnel_cfg.encap_type = 1;
 600	tunnel_cfg.sk_user_data = fou;
 601	tunnel_cfg.encap_destroy = NULL;
 602
 603	/* Initial for fou type */
 604	switch (cfg->type) {
 605	case FOU_ENCAP_DIRECT:
 606		tunnel_cfg.encap_rcv = fou_udp_recv;
 607		tunnel_cfg.gro_receive = fou_gro_receive;
 608		tunnel_cfg.gro_complete = fou_gro_complete;
 609		fou->protocol = cfg->protocol;
 610		break;
 611	case FOU_ENCAP_GUE:
 612		tunnel_cfg.encap_rcv = gue_udp_recv;
 613		tunnel_cfg.gro_receive = gue_gro_receive;
 614		tunnel_cfg.gro_complete = gue_gro_complete;
 615		break;
 616	default:
 617		err = -EINVAL;
 618		goto error;
 619	}
 620
 621	setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
 622
 623	sk->sk_allocation = GFP_ATOMIC;
 624
 625	err = fou_add_to_port_list(net, fou, cfg);
 626	if (err)
 627		goto error;
 628
 629	if (sockp)
 630		*sockp = sock;
 631
 632	return 0;
 633
 634error:
 635	kfree(fou);
 636	if (sock)
 637		udp_tunnel_sock_release(sock);
 638
 639	return err;
 640}
 641
 642static int fou_destroy(struct net *net, struct fou_cfg *cfg)
 643{
 644	struct fou_net *fn = net_generic(net, fou_net_id);
 645	int err = -EINVAL;
 646	struct fou *fou;
 647
 648	mutex_lock(&fn->fou_lock);
 649	list_for_each_entry(fou, &fn->fou_list, list) {
 650		if (fou_cfg_cmp(fou, cfg)) {
 651			fou_release(fou);
 652			err = 0;
 653			break;
 654		}
 655	}
 656	mutex_unlock(&fn->fou_lock);
 657
 658	return err;
 659}
 660
 661static struct genl_family fou_nl_family;
 662
 663static int parse_nl_config(struct genl_info *info,
 664			   struct fou_cfg *cfg)
 665{
 666	bool has_local = false, has_peer = false;
 667	struct nlattr *attr;
 668	int ifindex;
 669	__be16 port;
 670
 671	memset(cfg, 0, sizeof(*cfg));
 672
 673	cfg->udp_config.family = AF_INET;
 674
 675	if (info->attrs[FOU_ATTR_AF]) {
 676		u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]);
 677
 678		switch (family) {
 679		case AF_INET:
 680			break;
 681		case AF_INET6:
 682			cfg->udp_config.ipv6_v6only = 1;
 683			break;
 684		default:
 685			return -EAFNOSUPPORT;
 686		}
 687
 688		cfg->udp_config.family = family;
 689	}
 690
 691	if (info->attrs[FOU_ATTR_PORT]) {
 692		port = nla_get_be16(info->attrs[FOU_ATTR_PORT]);
 693		cfg->udp_config.local_udp_port = port;
 694	}
 695
 696	if (info->attrs[FOU_ATTR_IPPROTO])
 697		cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]);
 698
 699	if (info->attrs[FOU_ATTR_TYPE])
 700		cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]);
 701
 702	if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL])
 703		cfg->flags |= FOU_F_REMCSUM_NOPARTIAL;
 704
 705	if (cfg->udp_config.family == AF_INET) {
 706		if (info->attrs[FOU_ATTR_LOCAL_V4]) {
 707			attr = info->attrs[FOU_ATTR_LOCAL_V4];
 708			cfg->udp_config.local_ip.s_addr = nla_get_in_addr(attr);
 709			has_local = true;
 710		}
 711
 712		if (info->attrs[FOU_ATTR_PEER_V4]) {
 713			attr = info->attrs[FOU_ATTR_PEER_V4];
 714			cfg->udp_config.peer_ip.s_addr = nla_get_in_addr(attr);
 715			has_peer = true;
 716		}
 717#if IS_ENABLED(CONFIG_IPV6)
 718	} else {
 719		if (info->attrs[FOU_ATTR_LOCAL_V6]) {
 720			attr = info->attrs[FOU_ATTR_LOCAL_V6];
 721			cfg->udp_config.local_ip6 = nla_get_in6_addr(attr);
 722			has_local = true;
 723		}
 724
 725		if (info->attrs[FOU_ATTR_PEER_V6]) {
 726			attr = info->attrs[FOU_ATTR_PEER_V6];
 727			cfg->udp_config.peer_ip6 = nla_get_in6_addr(attr);
 728			has_peer = true;
 729		}
 730#endif
 731	}
 732
 733	if (has_peer) {
 734		if (info->attrs[FOU_ATTR_PEER_PORT]) {
 735			port = nla_get_be16(info->attrs[FOU_ATTR_PEER_PORT]);
 736			cfg->udp_config.peer_udp_port = port;
 737		} else {
 738			return -EINVAL;
 739		}
 740	}
 741
 742	if (info->attrs[FOU_ATTR_IFINDEX]) {
 743		if (!has_local)
 744			return -EINVAL;
 745
 746		ifindex = nla_get_s32(info->attrs[FOU_ATTR_IFINDEX]);
 747
 748		cfg->udp_config.bind_ifindex = ifindex;
 749	}
 750
 751	return 0;
 752}
 753
 754int fou_nl_add_doit(struct sk_buff *skb, struct genl_info *info)
 755{
 756	struct net *net = genl_info_net(info);
 757	struct fou_cfg cfg;
 758	int err;
 759
 760	err = parse_nl_config(info, &cfg);
 761	if (err)
 762		return err;
 763
 764	return fou_create(net, &cfg, NULL);
 765}
 766
 767int fou_nl_del_doit(struct sk_buff *skb, struct genl_info *info)
 768{
 769	struct net *net = genl_info_net(info);
 770	struct fou_cfg cfg;
 771	int err;
 772
 773	err = parse_nl_config(info, &cfg);
 774	if (err)
 775		return err;
 776
 777	return fou_destroy(net, &cfg);
 778}
 779
 780static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
 781{
 782	struct sock *sk = fou->sock->sk;
 783
 784	if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
 785	    nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
 786	    nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) ||
 787	    nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
 788	    nla_put_u8(msg, FOU_ATTR_TYPE, fou->type) ||
 789	    nla_put_s32(msg, FOU_ATTR_IFINDEX, sk->sk_bound_dev_if))
 790		return -1;
 791
 792	if (fou->flags & FOU_F_REMCSUM_NOPARTIAL)
 793		if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
 794			return -1;
 795
 796	if (fou->sock->sk->sk_family == AF_INET) {
 797		if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr))
 798			return -1;
 799
 800		if (nla_put_in_addr(msg, FOU_ATTR_PEER_V4, sk->sk_daddr))
 801			return -1;
 802#if IS_ENABLED(CONFIG_IPV6)
 803	} else {
 804		if (nla_put_in6_addr(msg, FOU_ATTR_LOCAL_V6,
 805				     &sk->sk_v6_rcv_saddr))
 806			return -1;
 807
 808		if (nla_put_in6_addr(msg, FOU_ATTR_PEER_V6, &sk->sk_v6_daddr))
 809			return -1;
 810#endif
 811	}
 812
 813	return 0;
 814}
 815
 816static int fou_dump_info(struct fou *fou, u32 portid, u32 seq,
 817			 u32 flags, struct sk_buff *skb, u8 cmd)
 818{
 819	void *hdr;
 820
 821	hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd);
 822	if (!hdr)
 823		return -ENOMEM;
 824
 825	if (fou_fill_info(fou, skb) < 0)
 826		goto nla_put_failure;
 827
 828	genlmsg_end(skb, hdr);
 829	return 0;
 830
 831nla_put_failure:
 832	genlmsg_cancel(skb, hdr);
 833	return -EMSGSIZE;
 834}
 835
 836int fou_nl_get_doit(struct sk_buff *skb, struct genl_info *info)
 837{
 838	struct net *net = genl_info_net(info);
 839	struct fou_net *fn = net_generic(net, fou_net_id);
 840	struct sk_buff *msg;
 841	struct fou_cfg cfg;
 842	struct fou *fout;
 843	__be16 port;
 844	u8 family;
 845	int ret;
 846
 847	ret = parse_nl_config(info, &cfg);
 848	if (ret)
 849		return ret;
 850	port = cfg.udp_config.local_udp_port;
 851	if (port == 0)
 852		return -EINVAL;
 853
 854	family = cfg.udp_config.family;
 855	if (family != AF_INET && family != AF_INET6)
 856		return -EINVAL;
 857
 858	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 859	if (!msg)
 860		return -ENOMEM;
 861
 862	ret = -ESRCH;
 863	mutex_lock(&fn->fou_lock);
 864	list_for_each_entry(fout, &fn->fou_list, list) {
 865		if (fou_cfg_cmp(fout, &cfg)) {
 866			ret = fou_dump_info(fout, info->snd_portid,
 867					    info->snd_seq, 0, msg,
 868					    info->genlhdr->cmd);
 869			break;
 870		}
 871	}
 872	mutex_unlock(&fn->fou_lock);
 873	if (ret < 0)
 874		goto out_free;
 875
 876	return genlmsg_reply(msg, info);
 877
 878out_free:
 879	nlmsg_free(msg);
 880	return ret;
 881}
 882
 883int fou_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 884{
 885	struct net *net = sock_net(skb->sk);
 886	struct fou_net *fn = net_generic(net, fou_net_id);
 887	struct fou *fout;
 888	int idx = 0, ret;
 889
 890	mutex_lock(&fn->fou_lock);
 891	list_for_each_entry(fout, &fn->fou_list, list) {
 892		if (idx++ < cb->args[0])
 893			continue;
 894		ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid,
 895				    cb->nlh->nlmsg_seq, NLM_F_MULTI,
 896				    skb, FOU_CMD_GET);
 897		if (ret)
 898			break;
 899	}
 900	mutex_unlock(&fn->fou_lock);
 901
 902	cb->args[0] = idx;
 903	return skb->len;
 904}
 905
 906static struct genl_family fou_nl_family __ro_after_init = {
 907	.hdrsize	= 0,
 908	.name		= FOU_GENL_NAME,
 909	.version	= FOU_GENL_VERSION,
 910	.maxattr	= FOU_ATTR_MAX,
 911	.policy		= fou_nl_policy,
 912	.netnsok	= true,
 913	.module		= THIS_MODULE,
 914	.small_ops	= fou_nl_ops,
 915	.n_small_ops	= ARRAY_SIZE(fou_nl_ops),
 916	.resv_start_op	= FOU_CMD_GET + 1,
 917};
 918
 919size_t fou_encap_hlen(struct ip_tunnel_encap *e)
 920{
 921	return sizeof(struct udphdr);
 922}
 923EXPORT_SYMBOL(fou_encap_hlen);
 924
 925size_t gue_encap_hlen(struct ip_tunnel_encap *e)
 926{
 927	size_t len;
 928	bool need_priv = false;
 929
 930	len = sizeof(struct udphdr) + sizeof(struct guehdr);
 931
 932	if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) {
 933		len += GUE_PLEN_REMCSUM;
 934		need_priv = true;
 935	}
 936
 937	len += need_priv ? GUE_LEN_PRIV : 0;
 938
 939	return len;
 940}
 941EXPORT_SYMBOL(gue_encap_hlen);
 942
 943int __fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
 944		       u8 *protocol, __be16 *sport, int type)
 945{
 946	int err;
 947
 948	err = iptunnel_handle_offloads(skb, type);
 949	if (err)
 950		return err;
 951
 952	*sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
 953						skb, 0, 0, false);
 954
 955	return 0;
 956}
 957EXPORT_SYMBOL(__fou_build_header);
 958
 959int __gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
 960		       u8 *protocol, __be16 *sport, int type)
 961{
 962	struct guehdr *guehdr;
 963	size_t hdrlen, optlen = 0;
 964	void *data;
 965	bool need_priv = false;
 966	int err;
 967
 968	if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) &&
 969	    skb->ip_summed == CHECKSUM_PARTIAL) {
 970		optlen += GUE_PLEN_REMCSUM;
 971		type |= SKB_GSO_TUNNEL_REMCSUM;
 972		need_priv = true;
 973	}
 974
 975	optlen += need_priv ? GUE_LEN_PRIV : 0;
 976
 977	err = iptunnel_handle_offloads(skb, type);
 978	if (err)
 979		return err;
 980
 981	/* Get source port (based on flow hash) before skb_push */
 982	*sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
 983						skb, 0, 0, false);
 984
 985	hdrlen = sizeof(struct guehdr) + optlen;
 986
 987	skb_push(skb, hdrlen);
 988
 989	guehdr = (struct guehdr *)skb->data;
 990
 991	guehdr->control = 0;
 992	guehdr->version = 0;
 993	guehdr->hlen = optlen >> 2;
 994	guehdr->flags = 0;
 995	guehdr->proto_ctype = *protocol;
 996
 997	data = &guehdr[1];
 998
 999	if (need_priv) {
1000		__be32 *flags = data;
1001
1002		guehdr->flags |= GUE_FLAG_PRIV;
1003		*flags = 0;
1004		data += GUE_LEN_PRIV;
1005
1006		if (type & SKB_GSO_TUNNEL_REMCSUM) {
1007			u16 csum_start = skb_checksum_start_offset(skb);
1008			__be16 *pd = data;
1009
1010			if (csum_start < hdrlen)
1011				return -EINVAL;
1012
1013			csum_start -= hdrlen;
1014			pd[0] = htons(csum_start);
1015			pd[1] = htons(csum_start + skb->csum_offset);
1016
1017			if (!skb_is_gso(skb)) {
1018				skb->ip_summed = CHECKSUM_NONE;
1019				skb->encapsulation = 0;
1020			}
1021
1022			*flags |= GUE_PFLAG_REMCSUM;
1023			data += GUE_PLEN_REMCSUM;
1024		}
1025
1026	}
1027
1028	return 0;
1029}
1030EXPORT_SYMBOL(__gue_build_header);
1031
1032#ifdef CONFIG_NET_FOU_IP_TUNNELS
1033
1034static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e,
1035			  struct flowi4 *fl4, u8 *protocol, __be16 sport)
1036{
1037	struct udphdr *uh;
1038
1039	skb_push(skb, sizeof(struct udphdr));
1040	skb_reset_transport_header(skb);
1041
1042	uh = udp_hdr(skb);
1043
1044	uh->dest = e->dport;
1045	uh->source = sport;
1046	uh->len = htons(skb->len);
1047	udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb,
1048		     fl4->saddr, fl4->daddr, skb->len);
1049
1050	*protocol = IPPROTO_UDP;
1051}
1052
1053static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
1054			    u8 *protocol, struct flowi4 *fl4)
1055{
1056	int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
1057						       SKB_GSO_UDP_TUNNEL;
1058	__be16 sport;
1059	int err;
1060
1061	err = __fou_build_header(skb, e, protocol, &sport, type);
1062	if (err)
1063		return err;
1064
1065	fou_build_udp(skb, e, fl4, protocol, sport);
1066
1067	return 0;
1068}
1069
1070static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
1071			    u8 *protocol, struct flowi4 *fl4)
1072{
1073	int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
1074						       SKB_GSO_UDP_TUNNEL;
1075	__be16 sport;
1076	int err;
1077
1078	err = __gue_build_header(skb, e, protocol, &sport, type);
1079	if (err)
1080		return err;
1081
1082	fou_build_udp(skb, e, fl4, protocol, sport);
1083
1084	return 0;
1085}
1086
1087static int gue_err_proto_handler(int proto, struct sk_buff *skb, u32 info)
1088{
1089	const struct net_protocol *ipprot = rcu_dereference(inet_protos[proto]);
1090
1091	if (ipprot && ipprot->err_handler) {
1092		if (!ipprot->err_handler(skb, info))
1093			return 0;
1094	}
1095
1096	return -ENOENT;
1097}
1098
1099static int gue_err(struct sk_buff *skb, u32 info)
1100{
1101	int transport_offset = skb_transport_offset(skb);
1102	struct guehdr *guehdr;
1103	size_t len, optlen;
1104	int ret;
1105
1106	len = sizeof(struct udphdr) + sizeof(struct guehdr);
1107	if (!pskb_may_pull(skb, transport_offset + len))
1108		return -EINVAL;
1109
1110	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
1111
1112	switch (guehdr->version) {
1113	case 0: /* Full GUE header present */
1114		break;
1115	case 1: {
1116		/* Direct encapsulation of IPv4 or IPv6 */
1117		skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
1118
1119		switch (((struct iphdr *)guehdr)->version) {
1120		case 4:
1121			ret = gue_err_proto_handler(IPPROTO_IPIP, skb, info);
1122			goto out;
1123#if IS_ENABLED(CONFIG_IPV6)
1124		case 6:
1125			ret = gue_err_proto_handler(IPPROTO_IPV6, skb, info);
1126			goto out;
1127#endif
1128		default:
1129			ret = -EOPNOTSUPP;
1130			goto out;
1131		}
1132	}
1133	default: /* Undefined version */
1134		return -EOPNOTSUPP;
1135	}
1136
1137	if (guehdr->control)
1138		return -ENOENT;
1139
1140	optlen = guehdr->hlen << 2;
1141
1142	if (!pskb_may_pull(skb, transport_offset + len + optlen))
1143		return -EINVAL;
1144
1145	guehdr = (struct guehdr *)&udp_hdr(skb)[1];
1146	if (validate_gue_flags(guehdr, optlen))
1147		return -EINVAL;
1148
1149	/* Handling exceptions for direct UDP encapsulation in GUE would lead to
1150	 * recursion. Besides, this kind of encapsulation can't even be
1151	 * configured currently. Discard this.
1152	 */
1153	if (guehdr->proto_ctype == IPPROTO_UDP ||
1154	    guehdr->proto_ctype == IPPROTO_UDPLITE)
1155		return -EOPNOTSUPP;
1156
1157	skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
1158	ret = gue_err_proto_handler(guehdr->proto_ctype, skb, info);
1159
1160out:
1161	skb_set_transport_header(skb, transport_offset);
1162	return ret;
1163}
1164
1165
1166static const struct ip_tunnel_encap_ops fou_iptun_ops = {
1167	.encap_hlen = fou_encap_hlen,
1168	.build_header = fou_build_header,
1169	.err_handler = gue_err,
1170};
1171
1172static const struct ip_tunnel_encap_ops gue_iptun_ops = {
1173	.encap_hlen = gue_encap_hlen,
1174	.build_header = gue_build_header,
1175	.err_handler = gue_err,
1176};
1177
1178static int ip_tunnel_encap_add_fou_ops(void)
1179{
1180	int ret;
1181
1182	ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1183	if (ret < 0) {
1184		pr_err("can't add fou ops\n");
1185		return ret;
1186	}
1187
1188	ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
1189	if (ret < 0) {
1190		pr_err("can't add gue ops\n");
1191		ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1192		return ret;
1193	}
1194
1195	return 0;
1196}
1197
1198static void ip_tunnel_encap_del_fou_ops(void)
1199{
1200	ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1201	ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
1202}
1203
1204#else
1205
1206static int ip_tunnel_encap_add_fou_ops(void)
1207{
1208	return 0;
1209}
1210
1211static void ip_tunnel_encap_del_fou_ops(void)
1212{
1213}
1214
1215#endif
1216
1217static __net_init int fou_init_net(struct net *net)
1218{
1219	struct fou_net *fn = net_generic(net, fou_net_id);
1220
1221	INIT_LIST_HEAD(&fn->fou_list);
1222	mutex_init(&fn->fou_lock);
1223	return 0;
1224}
1225
1226static __net_exit void fou_exit_net(struct net *net)
1227{
1228	struct fou_net *fn = net_generic(net, fou_net_id);
1229	struct fou *fou, *next;
1230
1231	/* Close all the FOU sockets */
1232	mutex_lock(&fn->fou_lock);
1233	list_for_each_entry_safe(fou, next, &fn->fou_list, list)
1234		fou_release(fou);
1235	mutex_unlock(&fn->fou_lock);
1236}
1237
1238static struct pernet_operations fou_net_ops = {
1239	.init = fou_init_net,
1240	.exit = fou_exit_net,
1241	.id   = &fou_net_id,
1242	.size = sizeof(struct fou_net),
1243};
1244
1245static int __init fou_init(void)
1246{
1247	int ret;
1248
1249	ret = register_pernet_device(&fou_net_ops);
1250	if (ret)
1251		goto exit;
1252
1253	ret = genl_register_family(&fou_nl_family);
1254	if (ret < 0)
1255		goto unregister;
1256
1257	ret = register_fou_bpf();
1258	if (ret < 0)
1259		goto kfunc_failed;
1260
1261	ret = ip_tunnel_encap_add_fou_ops();
1262	if (ret == 0)
1263		return 0;
1264
1265kfunc_failed:
1266	genl_unregister_family(&fou_nl_family);
1267unregister:
1268	unregister_pernet_device(&fou_net_ops);
1269exit:
1270	return ret;
1271}
1272
1273static void __exit fou_fini(void)
1274{
1275	ip_tunnel_encap_del_fou_ops();
1276	genl_unregister_family(&fou_nl_family);
1277	unregister_pernet_device(&fou_net_ops);
1278}
1279
1280module_init(fou_init);
1281module_exit(fou_fini);
1282MODULE_AUTHOR("Tom Herbert <therbert@google.com>");
1283MODULE_LICENSE("GPL");
1284MODULE_DESCRIPTION("Foo over UDP");