geneve.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * GENEVE: Generic Network Virtualization Encapsulation
   4 *
   5 * Copyright (c) 2015 Red Hat, Inc.
   6 */
   7
   8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10#include <linux/ethtool.h>
  11#include <linux/kernel.h>
  12#include <linux/module.h>
  13#include <linux/etherdevice.h>
  14#include <linux/hash.h>
  15#include <net/ipv6_stubs.h>
  16#include <net/dst_metadata.h>
  17#include <net/gro_cells.h>
  18#include <net/rtnetlink.h>
  19#include <net/geneve.h>
  20#include <net/gro.h>
  21#include <net/netdev_lock.h>
  22#include <net/protocol.h>
  23
  24#define GENEVE_NETDEV_VER	"0.6"
  25
  26#define GENEVE_N_VID		(1u << 24)
  27#define GENEVE_VID_MASK		(GENEVE_N_VID - 1)
  28
  29#define VNI_HASH_BITS		10
  30#define VNI_HASH_SIZE		(1<<VNI_HASH_BITS)
  31
  32static bool log_ecn_error = true;
  33module_param(log_ecn_error, bool, 0644);
  34MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");
  35
  36#define GENEVE_VER 0
  37#define GENEVE_BASE_HLEN (sizeof(struct udphdr) + sizeof(struct genevehdr))
  38#define GENEVE_IPV4_HLEN (ETH_HLEN + sizeof(struct iphdr) + GENEVE_BASE_HLEN)
  39#define GENEVE_IPV6_HLEN (ETH_HLEN + sizeof(struct ipv6hdr) + GENEVE_BASE_HLEN)
  40
  41/* per-network namespace private data for this module */
  42struct geneve_net {
  43	struct list_head	geneve_list;
  44	/* sock_list is protected by rtnl lock */
  45	struct list_head	sock_list;
  46};
  47
  48static unsigned int geneve_net_id;
  49
  50struct geneve_dev_node {
  51	struct hlist_node hlist;
  52	struct geneve_dev *geneve;
  53};
  54
  55struct geneve_config {
  56	bool			collect_md;
  57	bool			use_udp6_rx_checksums;
  58	bool			ttl_inherit;
  59	enum ifla_geneve_df	df;
  60	bool			inner_proto_inherit;
  61	u16			port_min;
  62	u16			port_max;
  63
  64	/* Must be last --ends in a flexible-array member. */
  65	struct ip_tunnel_info	info;
  66};
  67
  68/* Pseudo network device */
  69struct geneve_dev {
  70	struct geneve_dev_node hlist4;	/* vni hash table for IPv4 socket */
  71#if IS_ENABLED(CONFIG_IPV6)
  72	struct geneve_dev_node hlist6;	/* vni hash table for IPv6 socket */
  73#endif
  74	struct net	   *net;	/* netns for packet i/o */
  75	struct net_device  *dev;	/* netdev for geneve tunnel */
  76	struct geneve_sock __rcu *sock4;	/* IPv4 socket used for geneve tunnel */
  77#if IS_ENABLED(CONFIG_IPV6)
  78	struct geneve_sock __rcu *sock6;	/* IPv6 socket used for geneve tunnel */
  79#endif
  80	struct list_head   next;	/* geneve's per namespace list */
  81	struct gro_cells   gro_cells;
  82	struct geneve_config cfg;
  83};
  84
  85struct geneve_sock {
  86	bool			collect_md;
  87	struct list_head	list;
  88	struct socket		*sock;
  89	struct rcu_head		rcu;
  90	int			refcnt;
  91	struct hlist_head	vni_list[VNI_HASH_SIZE];
  92};
  93
  94static inline __u32 geneve_net_vni_hash(u8 vni[3])
  95{
  96	__u32 vnid;
  97
  98	vnid = (vni[0] << 16) | (vni[1] << 8) | vni[2];
  99	return hash_32(vnid, VNI_HASH_BITS);
 100}
 101
 102static __be64 vni_to_tunnel_id(const __u8 *vni)
 103{
 104#ifdef __BIG_ENDIAN
 105	return (vni[0] << 16) | (vni[1] << 8) | vni[2];
 106#else
 107	return (__force __be64)(((__force u64)vni[0] << 40) |
 108				((__force u64)vni[1] << 48) |
 109				((__force u64)vni[2] << 56));
 110#endif
 111}
 112
 113/* Convert 64 bit tunnel ID to 24 bit VNI. */
 114static void tunnel_id_to_vni(__be64 tun_id, __u8 *vni)
 115{
 116#ifdef __BIG_ENDIAN
 117	vni[0] = (__force __u8)(tun_id >> 16);
 118	vni[1] = (__force __u8)(tun_id >> 8);
 119	vni[2] = (__force __u8)tun_id;
 120#else
 121	vni[0] = (__force __u8)((__force u64)tun_id >> 40);
 122	vni[1] = (__force __u8)((__force u64)tun_id >> 48);
 123	vni[2] = (__force __u8)((__force u64)tun_id >> 56);
 124#endif
 125}
 126
 127static bool eq_tun_id_and_vni(u8 *tun_id, u8 *vni)
 128{
 129	return !memcmp(vni, &tun_id[5], 3);
 130}
 131
 132static sa_family_t geneve_get_sk_family(struct geneve_sock *gs)
 133{
 134	return gs->sock->sk->sk_family;
 135}
 136
 137static struct geneve_dev *geneve_lookup(struct geneve_sock *gs,
 138					__be32 addr, u8 vni[])
 139{
 140	struct hlist_head *vni_list_head;
 141	struct geneve_dev_node *node;
 142	__u32 hash;
 143
 144	/* Find the device for this VNI */
 145	hash = geneve_net_vni_hash(vni);
 146	vni_list_head = &gs->vni_list[hash];
 147	hlist_for_each_entry_rcu(node, vni_list_head, hlist) {
 148		if (eq_tun_id_and_vni((u8 *)&node->geneve->cfg.info.key.tun_id, vni) &&
 149		    addr == node->geneve->cfg.info.key.u.ipv4.dst)
 150			return node->geneve;
 151	}
 152	return NULL;
 153}
 154
 155#if IS_ENABLED(CONFIG_IPV6)
 156static struct geneve_dev *geneve6_lookup(struct geneve_sock *gs,
 157					 struct in6_addr addr6, u8 vni[])
 158{
 159	struct hlist_head *vni_list_head;
 160	struct geneve_dev_node *node;
 161	__u32 hash;
 162
 163	/* Find the device for this VNI */
 164	hash = geneve_net_vni_hash(vni);
 165	vni_list_head = &gs->vni_list[hash];
 166	hlist_for_each_entry_rcu(node, vni_list_head, hlist) {
 167		if (eq_tun_id_and_vni((u8 *)&node->geneve->cfg.info.key.tun_id, vni) &&
 168		    ipv6_addr_equal(&addr6, &node->geneve->cfg.info.key.u.ipv6.dst))
 169			return node->geneve;
 170	}
 171	return NULL;
 172}
 173#endif
 174
 175static inline struct genevehdr *geneve_hdr(const struct sk_buff *skb)
 176{
 177	return (struct genevehdr *)(udp_hdr(skb) + 1);
 178}
 179
 180static struct geneve_dev *geneve_lookup_skb(struct geneve_sock *gs,
 181					    struct sk_buff *skb)
 182{
 183	static u8 zero_vni[3];
 184	u8 *vni;
 185
 186	if (geneve_get_sk_family(gs) == AF_INET) {
 187		struct iphdr *iph;
 188		__be32 addr;
 189
 190		iph = ip_hdr(skb); /* outer IP header... */
 191
 192		if (gs->collect_md) {
 193			vni = zero_vni;
 194			addr = 0;
 195		} else {
 196			vni = geneve_hdr(skb)->vni;
 197			addr = iph->saddr;
 198		}
 199
 200		return geneve_lookup(gs, addr, vni);
 201#if IS_ENABLED(CONFIG_IPV6)
 202	} else if (geneve_get_sk_family(gs) == AF_INET6) {
 203		static struct in6_addr zero_addr6;
 204		struct ipv6hdr *ip6h;
 205		struct in6_addr addr6;
 206
 207		ip6h = ipv6_hdr(skb); /* outer IPv6 header... */
 208
 209		if (gs->collect_md) {
 210			vni = zero_vni;
 211			addr6 = zero_addr6;
 212		} else {
 213			vni = geneve_hdr(skb)->vni;
 214			addr6 = ip6h->saddr;
 215		}
 216
 217		return geneve6_lookup(gs, addr6, vni);
 218#endif
 219	}
 220	return NULL;
 221}
 222
 223/* geneve receive/decap routine */
 224static void geneve_rx(struct geneve_dev *geneve, struct geneve_sock *gs,
 225		      struct sk_buff *skb)
 226{
 227	struct genevehdr *gnvh = geneve_hdr(skb);
 228	struct metadata_dst *tun_dst = NULL;
 229	unsigned int len;
 230	int nh, err = 0;
 231	void *oiph;
 232
 233	if (ip_tunnel_collect_metadata() || gs->collect_md) {
 234		IP_TUNNEL_DECLARE_FLAGS(flags) = { };
 235
 236		__set_bit(IP_TUNNEL_KEY_BIT, flags);
 237		__assign_bit(IP_TUNNEL_OAM_BIT, flags, gnvh->oam);
 238		__assign_bit(IP_TUNNEL_CRIT_OPT_BIT, flags, gnvh->critical);
 239
 240		tun_dst = udp_tun_rx_dst(skb, geneve_get_sk_family(gs), flags,
 241					 vni_to_tunnel_id(gnvh->vni),
 242					 gnvh->opt_len * 4);
 243		if (!tun_dst) {
 244			dev_dstats_rx_dropped(geneve->dev);
 245			goto drop;
 246		}
 247		/* Update tunnel dst according to Geneve options. */
 248		ip_tunnel_flags_zero(flags);
 249		__set_bit(IP_TUNNEL_GENEVE_OPT_BIT, flags);
 250		ip_tunnel_info_opts_set(&tun_dst->u.tun_info,
 251					gnvh->options, gnvh->opt_len * 4,
 252					flags);
 253	} else {
 254		/* Drop packets w/ critical options,
 255		 * since we don't support any...
 256		 */
 257		if (gnvh->critical) {
 258			DEV_STATS_INC(geneve->dev, rx_frame_errors);
 259			DEV_STATS_INC(geneve->dev, rx_errors);
 260			goto drop;
 261		}
 262	}
 263
 264	if (tun_dst)
 265		skb_dst_set(skb, &tun_dst->dst);
 266
 267	if (gnvh->proto_type == htons(ETH_P_TEB)) {
 268		skb_reset_mac_header(skb);
 269		skb->protocol = eth_type_trans(skb, geneve->dev);
 270		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
 271
 272		/* Ignore packet loops (and multicast echo) */
 273		if (ether_addr_equal(eth_hdr(skb)->h_source,
 274				     geneve->dev->dev_addr)) {
 275			DEV_STATS_INC(geneve->dev, rx_errors);
 276			goto drop;
 277		}
 278	} else {
 279		skb_reset_mac_header(skb);
 280		skb->dev = geneve->dev;
 281		skb->pkt_type = PACKET_HOST;
 282	}
 283
 284	/* Save offset of outer header relative to skb->head,
 285	 * because we are going to reset the network header to the inner header
 286	 * and might change skb->head.
 287	 */
 288	nh = skb_network_header(skb) - skb->head;
 289
 290	skb_reset_network_header(skb);
 291
 292	if (!pskb_inet_may_pull(skb)) {
 293		DEV_STATS_INC(geneve->dev, rx_length_errors);
 294		DEV_STATS_INC(geneve->dev, rx_errors);
 295		goto drop;
 296	}
 297
 298	/* Get the outer header. */
 299	oiph = skb->head + nh;
 300
 301	if (geneve_get_sk_family(gs) == AF_INET)
 302		err = IP_ECN_decapsulate(oiph, skb);
 303#if IS_ENABLED(CONFIG_IPV6)
 304	else
 305		err = IP6_ECN_decapsulate(oiph, skb);
 306#endif
 307
 308	if (unlikely(err)) {
 309		if (log_ecn_error) {
 310			if (geneve_get_sk_family(gs) == AF_INET)
 311				net_info_ratelimited("non-ECT from %pI4 "
 312						     "with TOS=%#x\n",
 313						     &((struct iphdr *)oiph)->saddr,
 314						     ((struct iphdr *)oiph)->tos);
 315#if IS_ENABLED(CONFIG_IPV6)
 316			else
 317				net_info_ratelimited("non-ECT from %pI6\n",
 318						     &((struct ipv6hdr *)oiph)->saddr);
 319#endif
 320		}
 321		if (err > 1) {
 322			DEV_STATS_INC(geneve->dev, rx_frame_errors);
 323			DEV_STATS_INC(geneve->dev, rx_errors);
 324			goto drop;
 325		}
 326	}
 327
 328	len = skb->len;
 329	err = gro_cells_receive(&geneve->gro_cells, skb);
 330	if (likely(err == NET_RX_SUCCESS))
 331		dev_dstats_rx_add(geneve->dev, len);
 332
 333	return;
 334drop:
 335	/* Consume bad packet */
 336	kfree_skb(skb);
 337}
 338
 339/* Setup stats when device is created */
 340static int geneve_init(struct net_device *dev)
 341{
 342	struct geneve_dev *geneve = netdev_priv(dev);
 343	int err;
 344
 345	err = gro_cells_init(&geneve->gro_cells, dev);
 346	if (err)
 347		return err;
 348
 349	err = dst_cache_init(&geneve->cfg.info.dst_cache, GFP_KERNEL);
 350	if (err) {
 351		gro_cells_destroy(&geneve->gro_cells);
 352		return err;
 353	}
 354	netdev_lockdep_set_classes(dev);
 355	return 0;
 356}
 357
 358static void geneve_uninit(struct net_device *dev)
 359{
 360	struct geneve_dev *geneve = netdev_priv(dev);
 361
 362	dst_cache_destroy(&geneve->cfg.info.dst_cache);
 363	gro_cells_destroy(&geneve->gro_cells);
 364}
 365
 366/* Callback from net/ipv4/udp.c to receive packets */
 367static int geneve_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 368{
 369	struct genevehdr *geneveh;
 370	struct geneve_dev *geneve;
 371	struct geneve_sock *gs;
 372	__be16 inner_proto;
 373	int opts_len;
 374
 375	/* Need UDP and Geneve header to be present */
 376	if (unlikely(!pskb_may_pull(skb, GENEVE_BASE_HLEN)))
 377		goto drop;
 378
 379	/* Return packets with reserved bits set */
 380	geneveh = geneve_hdr(skb);
 381	if (unlikely(geneveh->ver != GENEVE_VER))
 382		goto drop;
 383
 384	gs = rcu_dereference_sk_user_data(sk);
 385	if (!gs)
 386		goto drop;
 387
 388	geneve = geneve_lookup_skb(gs, skb);
 389	if (!geneve)
 390		goto drop;
 391
 392	inner_proto = geneveh->proto_type;
 393
 394	if (unlikely((!geneve->cfg.inner_proto_inherit &&
 395		      inner_proto != htons(ETH_P_TEB)))) {
 396		dev_dstats_rx_dropped(geneve->dev);
 397		goto drop;
 398	}
 399
 400	opts_len = geneveh->opt_len * 4;
 401	if (iptunnel_pull_header(skb, GENEVE_BASE_HLEN + opts_len, inner_proto,
 402				 !net_eq(geneve->net, dev_net(geneve->dev)))) {
 403		dev_dstats_rx_dropped(geneve->dev);
 404		goto drop;
 405	}
 406
 407	geneve_rx(geneve, gs, skb);
 408	return 0;
 409
 410drop:
 411	/* Consume bad packet */
 412	kfree_skb(skb);
 413	return 0;
 414}
 415
 416/* Callback from net/ipv{4,6}/udp.c to check that we have a tunnel for errors */
 417static int geneve_udp_encap_err_lookup(struct sock *sk, struct sk_buff *skb)
 418{
 419	struct genevehdr *geneveh;
 420	struct geneve_sock *gs;
 421	u8 zero_vni[3] = { 0 };
 422	u8 *vni = zero_vni;
 423
 424	if (!pskb_may_pull(skb, skb_transport_offset(skb) + GENEVE_BASE_HLEN))
 425		return -EINVAL;
 426
 427	geneveh = geneve_hdr(skb);
 428	if (geneveh->ver != GENEVE_VER)
 429		return -EINVAL;
 430
 431	if (geneveh->proto_type != htons(ETH_P_TEB))
 432		return -EINVAL;
 433
 434	gs = rcu_dereference_sk_user_data(sk);
 435	if (!gs)
 436		return -ENOENT;
 437
 438	if (geneve_get_sk_family(gs) == AF_INET) {
 439		struct iphdr *iph = ip_hdr(skb);
 440		__be32 addr4 = 0;
 441
 442		if (!gs->collect_md) {
 443			vni = geneve_hdr(skb)->vni;
 444			addr4 = iph->daddr;
 445		}
 446
 447		return geneve_lookup(gs, addr4, vni) ? 0 : -ENOENT;
 448	}
 449
 450#if IS_ENABLED(CONFIG_IPV6)
 451	if (geneve_get_sk_family(gs) == AF_INET6) {
 452		struct ipv6hdr *ip6h = ipv6_hdr(skb);
 453		struct in6_addr addr6;
 454
 455		memset(&addr6, 0, sizeof(struct in6_addr));
 456
 457		if (!gs->collect_md) {
 458			vni = geneve_hdr(skb)->vni;
 459			addr6 = ip6h->daddr;
 460		}
 461
 462		return geneve6_lookup(gs, addr6, vni) ? 0 : -ENOENT;
 463	}
 464#endif
 465
 466	return -EPFNOSUPPORT;
 467}
 468
 469static struct socket *geneve_create_sock(struct net *net, bool ipv6,
 470					 __be16 port, bool ipv6_rx_csum)
 471{
 472	struct socket *sock;
 473	struct udp_port_cfg udp_conf;
 474	int err;
 475
 476	memset(&udp_conf, 0, sizeof(udp_conf));
 477
 478	if (ipv6) {
 479		udp_conf.family = AF_INET6;
 480		udp_conf.ipv6_v6only = 1;
 481		udp_conf.use_udp6_rx_checksums = ipv6_rx_csum;
 482	} else {
 483		udp_conf.family = AF_INET;
 484		udp_conf.local_ip.s_addr = htonl(INADDR_ANY);
 485	}
 486
 487	udp_conf.local_udp_port = port;
 488
 489	/* Open UDP socket */
 490	err = udp_sock_create(net, &udp_conf, &sock);
 491	if (err < 0)
 492		return ERR_PTR(err);
 493
 494	udp_allow_gso(sock->sk);
 495	return sock;
 496}
 497
 498static int geneve_hlen(struct genevehdr *gh)
 499{
 500	return sizeof(*gh) + gh->opt_len * 4;
 501}
 502
 503static struct sk_buff *geneve_gro_receive(struct sock *sk,
 504					  struct list_head *head,
 505					  struct sk_buff *skb)
 506{
 507	struct sk_buff *pp = NULL;
 508	struct sk_buff *p;
 509	struct genevehdr *gh, *gh2;
 510	unsigned int hlen, gh_len, off_gnv;
 511	const struct packet_offload *ptype;
 512	__be16 type;
 513	int flush = 1;
 514
 515	off_gnv = skb_gro_offset(skb);
 516	hlen = off_gnv + sizeof(*gh);
 517	gh = skb_gro_header(skb, hlen, off_gnv);
 518	if (unlikely(!gh))
 519		goto out;
 520
 521	if (gh->ver != GENEVE_VER || gh->oam)
 522		goto out;
 523	gh_len = geneve_hlen(gh);
 524
 525	hlen = off_gnv + gh_len;
 526	if (!skb_gro_may_pull(skb, hlen)) {
 527		gh = skb_gro_header_slow(skb, hlen, off_gnv);
 528		if (unlikely(!gh))
 529			goto out;
 530	}
 531
 532	list_for_each_entry(p, head, list) {
 533		if (!NAPI_GRO_CB(p)->same_flow)
 534			continue;
 535
 536		gh2 = (struct genevehdr *)(p->data + off_gnv);
 537		if (gh->opt_len != gh2->opt_len ||
 538		    memcmp(gh, gh2, gh_len)) {
 539			NAPI_GRO_CB(p)->same_flow = 0;
 540			continue;
 541		}
 542	}
 543
 544	skb_gro_pull(skb, gh_len);
 545	skb_gro_postpull_rcsum(skb, gh, gh_len);
 546	type = gh->proto_type;
 547	if (likely(type == htons(ETH_P_TEB)))
 548		return call_gro_receive(eth_gro_receive, head, skb);
 549
 550	ptype = gro_find_receive_by_type(type);
 551	if (!ptype)
 552		goto out;
 553
 554	pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
 555	flush = 0;
 556
 557out:
 558	skb_gro_flush_final(skb, pp, flush);
 559
 560	return pp;
 561}
 562
 563static int geneve_gro_complete(struct sock *sk, struct sk_buff *skb,
 564			       int nhoff)
 565{
 566	struct genevehdr *gh;
 567	struct packet_offload *ptype;
 568	__be16 type;
 569	int gh_len;
 570	int err = -ENOSYS;
 571
 572	gh = (struct genevehdr *)(skb->data + nhoff);
 573	gh_len = geneve_hlen(gh);
 574	type = gh->proto_type;
 575
 576	/* since skb->encapsulation is set, eth_gro_complete() sets the inner mac header */
 577	if (likely(type == htons(ETH_P_TEB)))
 578		return eth_gro_complete(skb, nhoff + gh_len);
 579
 580	ptype = gro_find_complete_by_type(type);
 581	if (ptype)
 582		err = ptype->callbacks.gro_complete(skb, nhoff + gh_len);
 583
 584	skb_set_inner_mac_header(skb, nhoff + gh_len);
 585
 586	return err;
 587}
 588
 589/* Create new listen socket if needed */
 590static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
 591						bool ipv6, bool ipv6_rx_csum)
 592{
 593	struct geneve_net *gn = net_generic(net, geneve_net_id);
 594	struct geneve_sock *gs;
 595	struct socket *sock;
 596	struct udp_tunnel_sock_cfg tunnel_cfg;
 597	int h;
 598
 599	gs = kzalloc(sizeof(*gs), GFP_KERNEL);
 600	if (!gs)
 601		return ERR_PTR(-ENOMEM);
 602
 603	sock = geneve_create_sock(net, ipv6, port, ipv6_rx_csum);
 604	if (IS_ERR(sock)) {
 605		kfree(gs);
 606		return ERR_CAST(sock);
 607	}
 608
 609	gs->sock = sock;
 610	gs->refcnt = 1;
 611	for (h = 0; h < VNI_HASH_SIZE; ++h)
 612		INIT_HLIST_HEAD(&gs->vni_list[h]);
 613
 614	/* Initialize the geneve udp offloads structure */
 615	udp_tunnel_notify_add_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
 616
 617	/* Mark socket as an encapsulation socket */
 618	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
 619	tunnel_cfg.sk_user_data = gs;
 620	tunnel_cfg.encap_type = 1;
 621	tunnel_cfg.gro_receive = geneve_gro_receive;
 622	tunnel_cfg.gro_complete = geneve_gro_complete;
 623	tunnel_cfg.encap_rcv = geneve_udp_encap_recv;
 624	tunnel_cfg.encap_err_lookup = geneve_udp_encap_err_lookup;
 625	tunnel_cfg.encap_destroy = NULL;
 626	setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
 627	list_add(&gs->list, &gn->sock_list);
 628	return gs;
 629}
 630
 631static void __geneve_sock_release(struct geneve_sock *gs)
 632{
 633	if (!gs || --gs->refcnt)
 634		return;
 635
 636	list_del(&gs->list);
 637	udp_tunnel_notify_del_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
 638	udp_tunnel_sock_release(gs->sock);
 639	kfree_rcu(gs, rcu);
 640}
 641
 642static void geneve_sock_release(struct geneve_dev *geneve)
 643{
 644	struct geneve_sock *gs4 = rtnl_dereference(geneve->sock4);
 645#if IS_ENABLED(CONFIG_IPV6)
 646	struct geneve_sock *gs6 = rtnl_dereference(geneve->sock6);
 647
 648	rcu_assign_pointer(geneve->sock6, NULL);
 649#endif
 650
 651	rcu_assign_pointer(geneve->sock4, NULL);
 652	synchronize_net();
 653
 654	__geneve_sock_release(gs4);
 655#if IS_ENABLED(CONFIG_IPV6)
 656	__geneve_sock_release(gs6);
 657#endif
 658}
 659
 660static struct geneve_sock *geneve_find_sock(struct geneve_net *gn,
 661					    sa_family_t family,
 662					    __be16 dst_port)
 663{
 664	struct geneve_sock *gs;
 665
 666	list_for_each_entry(gs, &gn->sock_list, list) {
 667		if (inet_sk(gs->sock->sk)->inet_sport == dst_port &&
 668		    geneve_get_sk_family(gs) == family) {
 669			return gs;
 670		}
 671	}
 672	return NULL;
 673}
 674
 675static int geneve_sock_add(struct geneve_dev *geneve, bool ipv6)
 676{
 677	struct net *net = geneve->net;
 678	struct geneve_net *gn = net_generic(net, geneve_net_id);
 679	struct geneve_dev_node *node;
 680	struct geneve_sock *gs;
 681	__u8 vni[3];
 682	__u32 hash;
 683
 684	gs = geneve_find_sock(gn, ipv6 ? AF_INET6 : AF_INET, geneve->cfg.info.key.tp_dst);
 685	if (gs) {
 686		gs->refcnt++;
 687		goto out;
 688	}
 689
 690	gs = geneve_socket_create(net, geneve->cfg.info.key.tp_dst, ipv6,
 691				  geneve->cfg.use_udp6_rx_checksums);
 692	if (IS_ERR(gs))
 693		return PTR_ERR(gs);
 694
 695out:
 696	gs->collect_md = geneve->cfg.collect_md;
 697#if IS_ENABLED(CONFIG_IPV6)
 698	if (ipv6) {
 699		rcu_assign_pointer(geneve->sock6, gs);
 700		node = &geneve->hlist6;
 701	} else
 702#endif
 703	{
 704		rcu_assign_pointer(geneve->sock4, gs);
 705		node = &geneve->hlist4;
 706	}
 707	node->geneve = geneve;
 708
 709	tunnel_id_to_vni(geneve->cfg.info.key.tun_id, vni);
 710	hash = geneve_net_vni_hash(vni);
 711	hlist_add_head_rcu(&node->hlist, &gs->vni_list[hash]);
 712	return 0;
 713}
 714
 715static int geneve_open(struct net_device *dev)
 716{
 717	struct geneve_dev *geneve = netdev_priv(dev);
 718	bool metadata = geneve->cfg.collect_md;
 719	bool ipv4, ipv6;
 720	int ret = 0;
 721
 722	ipv6 = geneve->cfg.info.mode & IP_TUNNEL_INFO_IPV6 || metadata;
 723	ipv4 = !ipv6 || metadata;
 724#if IS_ENABLED(CONFIG_IPV6)
 725	if (ipv6) {
 726		ret = geneve_sock_add(geneve, true);
 727		if (ret < 0 && ret != -EAFNOSUPPORT)
 728			ipv4 = false;
 729	}
 730#endif
 731	if (ipv4)
 732		ret = geneve_sock_add(geneve, false);
 733	if (ret < 0)
 734		geneve_sock_release(geneve);
 735
 736	return ret;
 737}
 738
 739static int geneve_stop(struct net_device *dev)
 740{
 741	struct geneve_dev *geneve = netdev_priv(dev);
 742
 743	hlist_del_init_rcu(&geneve->hlist4.hlist);
 744#if IS_ENABLED(CONFIG_IPV6)
 745	hlist_del_init_rcu(&geneve->hlist6.hlist);
 746#endif
 747	geneve_sock_release(geneve);
 748	return 0;
 749}
 750
 751static void geneve_build_header(struct genevehdr *geneveh,
 752				const struct ip_tunnel_info *info,
 753				__be16 inner_proto)
 754{
 755	geneveh->ver = GENEVE_VER;
 756	geneveh->opt_len = info->options_len / 4;
 757	geneveh->oam = test_bit(IP_TUNNEL_OAM_BIT, info->key.tun_flags);
 758	geneveh->critical = test_bit(IP_TUNNEL_CRIT_OPT_BIT,
 759				     info->key.tun_flags);
 760	geneveh->rsvd1 = 0;
 761	tunnel_id_to_vni(info->key.tun_id, geneveh->vni);
 762	geneveh->proto_type = inner_proto;
 763	geneveh->rsvd2 = 0;
 764
 765	if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, info->key.tun_flags))
 766		ip_tunnel_info_opts_get(geneveh->options, info);
 767}
 768
 769static int geneve_build_skb(struct dst_entry *dst, struct sk_buff *skb,
 770			    const struct ip_tunnel_info *info,
 771			    bool xnet, int ip_hdr_len,
 772			    bool inner_proto_inherit)
 773{
 774	bool udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
 775	struct genevehdr *gnvh;
 776	__be16 inner_proto;
 777	int min_headroom;
 778	int err;
 779
 780	skb_reset_mac_header(skb);
 781	skb_scrub_packet(skb, xnet);
 782
 783	min_headroom = LL_RESERVED_SPACE(dst->dev) + dst->header_len +
 784		       GENEVE_BASE_HLEN + info->options_len + ip_hdr_len;
 785	err = skb_cow_head(skb, min_headroom);
 786	if (unlikely(err))
 787		goto free_dst;
 788
 789	err = udp_tunnel_handle_offloads(skb, udp_sum);
 790	if (err)
 791		goto free_dst;
 792
 793	gnvh = __skb_push(skb, sizeof(*gnvh) + info->options_len);
 794	inner_proto = inner_proto_inherit ? skb->protocol : htons(ETH_P_TEB);
 795	geneve_build_header(gnvh, info, inner_proto);
 796	skb_set_inner_protocol(skb, inner_proto);
 797	return 0;
 798
 799free_dst:
 800	dst_release(dst);
 801	return err;
 802}
 803
 804static u8 geneve_get_dsfield(struct sk_buff *skb, struct net_device *dev,
 805			     const struct ip_tunnel_info *info,
 806			     bool *use_cache)
 807{
 808	struct geneve_dev *geneve = netdev_priv(dev);
 809	u8 dsfield;
 810
 811	dsfield = info->key.tos;
 812	if (dsfield == 1 && !geneve->cfg.collect_md) {
 813		dsfield = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
 814		*use_cache = false;
 815	}
 816
 817	return dsfield;
 818}
 819
 820static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 821			   struct geneve_dev *geneve,
 822			   const struct ip_tunnel_info *info)
 823{
 824	bool inner_proto_inherit = geneve->cfg.inner_proto_inherit;
 825	bool xnet = !net_eq(geneve->net, dev_net(geneve->dev));
 826	struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
 827	const struct ip_tunnel_key *key = &info->key;
 828	struct rtable *rt;
 829	bool use_cache;
 830	__u8 tos, ttl;
 831	__be16 df = 0;
 832	__be32 saddr;
 833	__be16 sport;
 834	int err;
 835
 836	if (skb_vlan_inet_prepare(skb, inner_proto_inherit))
 837		return -EINVAL;
 838
 839	if (!gs4)
 840		return -EIO;
 841
 842	use_cache = ip_tunnel_dst_cache_usable(skb, info);
 843	tos = geneve_get_dsfield(skb, dev, info, &use_cache);
 844	sport = udp_flow_src_port(geneve->net, skb,
 845				  geneve->cfg.port_min,
 846				  geneve->cfg.port_max, true);
 847
 848	rt = udp_tunnel_dst_lookup(skb, dev, geneve->net, 0, &saddr,
 849				   &info->key,
 850				   sport, geneve->cfg.info.key.tp_dst, tos,
 851				   use_cache ?
 852				   (struct dst_cache *)&info->dst_cache : NULL);
 853	if (IS_ERR(rt))
 854		return PTR_ERR(rt);
 855
 856	err = skb_tunnel_check_pmtu(skb, &rt->dst,
 857				    GENEVE_IPV4_HLEN + info->options_len,
 858				    netif_is_any_bridge_port(dev));
 859	if (err < 0) {
 860		dst_release(&rt->dst);
 861		return err;
 862	} else if (err) {
 863		struct ip_tunnel_info *info;
 864
 865		info = skb_tunnel_info(skb);
 866		if (info) {
 867			struct ip_tunnel_info *unclone;
 868
 869			unclone = skb_tunnel_info_unclone(skb);
 870			if (unlikely(!unclone)) {
 871				dst_release(&rt->dst);
 872				return -ENOMEM;
 873			}
 874
 875			unclone->key.u.ipv4.dst = saddr;
 876			unclone->key.u.ipv4.src = info->key.u.ipv4.dst;
 877		}
 878
 879		if (!pskb_may_pull(skb, ETH_HLEN)) {
 880			dst_release(&rt->dst);
 881			return -EINVAL;
 882		}
 883
 884		skb->protocol = eth_type_trans(skb, geneve->dev);
 885		__netif_rx(skb);
 886		dst_release(&rt->dst);
 887		return -EMSGSIZE;
 888	}
 889
 890	tos = ip_tunnel_ecn_encap(tos, ip_hdr(skb), skb);
 891	if (geneve->cfg.collect_md) {
 892		ttl = key->ttl;
 893
 894		df = test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, key->tun_flags) ?
 895		     htons(IP_DF) : 0;
 896	} else {
 897		if (geneve->cfg.ttl_inherit)
 898			ttl = ip_tunnel_get_ttl(ip_hdr(skb), skb);
 899		else
 900			ttl = key->ttl;
 901		ttl = ttl ? : ip4_dst_hoplimit(&rt->dst);
 902
 903		if (geneve->cfg.df == GENEVE_DF_SET) {
 904			df = htons(IP_DF);
 905		} else if (geneve->cfg.df == GENEVE_DF_INHERIT) {
 906			struct ethhdr *eth = skb_eth_hdr(skb);
 907
 908			if (ntohs(eth->h_proto) == ETH_P_IPV6) {
 909				df = htons(IP_DF);
 910			} else if (ntohs(eth->h_proto) == ETH_P_IP) {
 911				struct iphdr *iph = ip_hdr(skb);
 912
 913				if (iph->frag_off & htons(IP_DF))
 914					df = htons(IP_DF);
 915			}
 916		}
 917	}
 918
 919	err = geneve_build_skb(&rt->dst, skb, info, xnet, sizeof(struct iphdr),
 920			       inner_proto_inherit);
 921	if (unlikely(err))
 922		return err;
 923
 924	udp_tunnel_xmit_skb(rt, gs4->sock->sk, skb, saddr, info->key.u.ipv4.dst,
 925			    tos, ttl, df, sport, geneve->cfg.info.key.tp_dst,
 926			    !net_eq(geneve->net, dev_net(geneve->dev)),
 927			    !test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags),
 928			    0);
 929	return 0;
 930}
 931
 932#if IS_ENABLED(CONFIG_IPV6)
 933static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 934			    struct geneve_dev *geneve,
 935			    const struct ip_tunnel_info *info)
 936{
 937	bool inner_proto_inherit = geneve->cfg.inner_proto_inherit;
 938	bool xnet = !net_eq(geneve->net, dev_net(geneve->dev));
 939	struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
 940	const struct ip_tunnel_key *key = &info->key;
 941	struct dst_entry *dst = NULL;
 942	struct in6_addr saddr;
 943	bool use_cache;
 944	__u8 prio, ttl;
 945	__be16 sport;
 946	int err;
 947
 948	if (skb_vlan_inet_prepare(skb, inner_proto_inherit))
 949		return -EINVAL;
 950
 951	if (!gs6)
 952		return -EIO;
 953
 954	use_cache = ip_tunnel_dst_cache_usable(skb, info);
 955	prio = geneve_get_dsfield(skb, dev, info, &use_cache);
 956	sport = udp_flow_src_port(geneve->net, skb,
 957				  geneve->cfg.port_min,
 958				  geneve->cfg.port_max, true);
 959
 960	dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sock, 0,
 961				     &saddr, key, sport,
 962				     geneve->cfg.info.key.tp_dst, prio,
 963				     use_cache ?
 964				     (struct dst_cache *)&info->dst_cache : NULL);
 965	if (IS_ERR(dst))
 966		return PTR_ERR(dst);
 967
 968	err = skb_tunnel_check_pmtu(skb, dst,
 969				    GENEVE_IPV6_HLEN + info->options_len,
 970				    netif_is_any_bridge_port(dev));
 971	if (err < 0) {
 972		dst_release(dst);
 973		return err;
 974	} else if (err) {
 975		struct ip_tunnel_info *info = skb_tunnel_info(skb);
 976
 977		if (info) {
 978			struct ip_tunnel_info *unclone;
 979
 980			unclone = skb_tunnel_info_unclone(skb);
 981			if (unlikely(!unclone)) {
 982				dst_release(dst);
 983				return -ENOMEM;
 984			}
 985
 986			unclone->key.u.ipv6.dst = saddr;
 987			unclone->key.u.ipv6.src = info->key.u.ipv6.dst;
 988		}
 989
 990		if (!pskb_may_pull(skb, ETH_HLEN)) {
 991			dst_release(dst);
 992			return -EINVAL;
 993		}
 994
 995		skb->protocol = eth_type_trans(skb, geneve->dev);
 996		__netif_rx(skb);
 997		dst_release(dst);
 998		return -EMSGSIZE;
 999	}
1000
1001	prio = ip_tunnel_ecn_encap(prio, ip_hdr(skb), skb);
1002	if (geneve->cfg.collect_md) {
1003		ttl = key->ttl;
1004	} else {
1005		if (geneve->cfg.ttl_inherit)
1006			ttl = ip_tunnel_get_ttl(ip_hdr(skb), skb);
1007		else
1008			ttl = key->ttl;
1009		ttl = ttl ? : ip6_dst_hoplimit(dst);
1010	}
1011	err = geneve_build_skb(dst, skb, info, xnet, sizeof(struct ipv6hdr),
1012			       inner_proto_inherit);
1013	if (unlikely(err))
1014		return err;
1015
1016	udp_tunnel6_xmit_skb(dst, gs6->sock->sk, skb, dev,
1017			     &saddr, &key->u.ipv6.dst, prio, ttl,
1018			     info->key.label, sport, geneve->cfg.info.key.tp_dst,
1019			     !test_bit(IP_TUNNEL_CSUM_BIT,
1020				       info->key.tun_flags),
1021			     0);
1022	return 0;
1023}
1024#endif
1025
1026static netdev_tx_t geneve_xmit(struct sk_buff *skb, struct net_device *dev)
1027{
1028	struct geneve_dev *geneve = netdev_priv(dev);
1029	struct ip_tunnel_info *info = NULL;
1030	int err;
1031
1032	if (geneve->cfg.collect_md) {
1033		info = skb_tunnel_info(skb);
1034		if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) {
1035			netdev_dbg(dev, "no tunnel metadata\n");
1036			dev_kfree_skb(skb);
1037			dev_dstats_tx_dropped(dev);
1038			return NETDEV_TX_OK;
1039		}
1040	} else {
1041		info = &geneve->cfg.info;
1042	}
1043
1044	rcu_read_lock();
1045#if IS_ENABLED(CONFIG_IPV6)
1046	if (info->mode & IP_TUNNEL_INFO_IPV6)
1047		err = geneve6_xmit_skb(skb, dev, geneve, info);
1048	else
1049#endif
1050		err = geneve_xmit_skb(skb, dev, geneve, info);
1051	rcu_read_unlock();
1052
1053	if (likely(!err))
1054		return NETDEV_TX_OK;
1055
1056	if (err != -EMSGSIZE)
1057		dev_kfree_skb(skb);
1058
1059	if (err == -ELOOP)
1060		DEV_STATS_INC(dev, collisions);
1061	else if (err == -ENETUNREACH)
1062		DEV_STATS_INC(dev, tx_carrier_errors);
1063
1064	DEV_STATS_INC(dev, tx_errors);
1065	return NETDEV_TX_OK;
1066}
1067
1068static int geneve_change_mtu(struct net_device *dev, int new_mtu)
1069{
1070	if (new_mtu > dev->max_mtu)
1071		new_mtu = dev->max_mtu;
1072	else if (new_mtu < dev->min_mtu)
1073		new_mtu = dev->min_mtu;
1074
1075	WRITE_ONCE(dev->mtu, new_mtu);
1076	return 0;
1077}
1078
1079static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
1080{
1081	struct ip_tunnel_info *info = skb_tunnel_info(skb);
1082	struct geneve_dev *geneve = netdev_priv(dev);
1083	__be16 sport;
1084
1085	if (ip_tunnel_info_af(info) == AF_INET) {
1086		struct rtable *rt;
1087		struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
1088		bool use_cache;
1089		__be32 saddr;
1090		u8 tos;
1091
1092		if (!gs4)
1093			return -EIO;
1094
1095		use_cache = ip_tunnel_dst_cache_usable(skb, info);
1096		tos = geneve_get_dsfield(skb, dev, info, &use_cache);
1097		sport = udp_flow_src_port(geneve->net, skb,
1098					  geneve->cfg.port_min,
1099					  geneve->cfg.port_max, true);
1100
1101		rt = udp_tunnel_dst_lookup(skb, dev, geneve->net, 0, &saddr,
1102					   &info->key,
1103					   sport, geneve->cfg.info.key.tp_dst,
1104					   tos,
1105					   use_cache ? &info->dst_cache : NULL);
1106		if (IS_ERR(rt))
1107			return PTR_ERR(rt);
1108
1109		ip_rt_put(rt);
1110		info->key.u.ipv4.src = saddr;
1111#if IS_ENABLED(CONFIG_IPV6)
1112	} else if (ip_tunnel_info_af(info) == AF_INET6) {
1113		struct dst_entry *dst;
1114		struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
1115		struct in6_addr saddr;
1116		bool use_cache;
1117		u8 prio;
1118
1119		if (!gs6)
1120			return -EIO;
1121
1122		use_cache = ip_tunnel_dst_cache_usable(skb, info);
1123		prio = geneve_get_dsfield(skb, dev, info, &use_cache);
1124		sport = udp_flow_src_port(geneve->net, skb,
1125					  geneve->cfg.port_min,
1126					  geneve->cfg.port_max, true);
1127
1128		dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sock, 0,
1129					     &saddr, &info->key, sport,
1130					     geneve->cfg.info.key.tp_dst, prio,
1131					     use_cache ? &info->dst_cache : NULL);
1132		if (IS_ERR(dst))
1133			return PTR_ERR(dst);
1134
1135		dst_release(dst);
1136		info->key.u.ipv6.src = saddr;
1137#endif
1138	} else {
1139		return -EINVAL;
1140	}
1141
1142	info->key.tp_src = sport;
1143	info->key.tp_dst = geneve->cfg.info.key.tp_dst;
1144	return 0;
1145}
1146
1147static const struct net_device_ops geneve_netdev_ops = {
1148	.ndo_init		= geneve_init,
1149	.ndo_uninit		= geneve_uninit,
1150	.ndo_open		= geneve_open,
1151	.ndo_stop		= geneve_stop,
1152	.ndo_start_xmit		= geneve_xmit,
1153	.ndo_change_mtu		= geneve_change_mtu,
1154	.ndo_validate_addr	= eth_validate_addr,
1155	.ndo_set_mac_address	= eth_mac_addr,
1156	.ndo_fill_metadata_dst	= geneve_fill_metadata_dst,
1157};
1158
1159static void geneve_get_drvinfo(struct net_device *dev,
1160			       struct ethtool_drvinfo *drvinfo)
1161{
1162	strscpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version));
1163	strscpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver));
1164}
1165
1166static const struct ethtool_ops geneve_ethtool_ops = {
1167	.get_drvinfo	= geneve_get_drvinfo,
1168	.get_link	= ethtool_op_get_link,
1169};
1170
1171/* Info for udev, that this is a virtual tunnel endpoint */
1172static const struct device_type geneve_type = {
1173	.name = "geneve",
1174};
1175
1176/* Calls the ndo_udp_tunnel_add of the caller in order to
1177 * supply the listening GENEVE udp ports. Callers are expected
1178 * to implement the ndo_udp_tunnel_add.
1179 */
1180static void geneve_offload_rx_ports(struct net_device *dev, bool push)
1181{
1182	struct net *net = dev_net(dev);
1183	struct geneve_net *gn = net_generic(net, geneve_net_id);
1184	struct geneve_sock *gs;
1185
1186	ASSERT_RTNL();
1187
1188	list_for_each_entry(gs, &gn->sock_list, list) {
1189		if (push) {
1190			udp_tunnel_push_rx_port(dev, gs->sock,
1191						UDP_TUNNEL_TYPE_GENEVE);
1192		} else {
1193			udp_tunnel_drop_rx_port(dev, gs->sock,
1194						UDP_TUNNEL_TYPE_GENEVE);
1195		}
1196	}
1197}
1198
1199/* Initialize the device structure. */
1200static void geneve_setup(struct net_device *dev)
1201{
1202	ether_setup(dev);
1203
1204	dev->netdev_ops = &geneve_netdev_ops;
1205	dev->ethtool_ops = &geneve_ethtool_ops;
1206	dev->needs_free_netdev = true;
1207
1208	SET_NETDEV_DEVTYPE(dev, &geneve_type);
1209
1210	dev->features    |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
1211	dev->features    |= NETIF_F_RXCSUM;
1212	dev->features    |= NETIF_F_GSO_SOFTWARE;
1213
1214	dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
1215	dev->hw_features |= NETIF_F_RXCSUM;
1216	dev->hw_features |= NETIF_F_GSO_SOFTWARE;
1217
1218	dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
1219	/* MTU range: 68 - (something less than 65535) */
1220	dev->min_mtu = ETH_MIN_MTU;
1221	/* The max_mtu calculation does not take account of GENEVE
1222	 * options, to avoid excluding potentially valid
1223	 * configurations. This will be further reduced by IPvX hdr size.
1224	 */
1225	dev->max_mtu = IP_MAX_MTU - GENEVE_BASE_HLEN - dev->hard_header_len;
1226
1227	netif_keep_dst(dev);
1228	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1229	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE;
1230	dev->lltx = true;
1231	eth_hw_addr_random(dev);
1232}
1233
1234static const struct nla_policy geneve_policy[IFLA_GENEVE_MAX + 1] = {
1235	[IFLA_GENEVE_UNSPEC]		= { .strict_start_type = IFLA_GENEVE_INNER_PROTO_INHERIT },
1236	[IFLA_GENEVE_ID]		= { .type = NLA_U32 },
1237	[IFLA_GENEVE_REMOTE]		= { .len = sizeof_field(struct iphdr, daddr) },
1238	[IFLA_GENEVE_REMOTE6]		= { .len = sizeof(struct in6_addr) },
1239	[IFLA_GENEVE_TTL]		= { .type = NLA_U8 },
1240	[IFLA_GENEVE_TOS]		= { .type = NLA_U8 },
1241	[IFLA_GENEVE_LABEL]		= { .type = NLA_U32 },
1242	[IFLA_GENEVE_PORT]		= { .type = NLA_U16 },
1243	[IFLA_GENEVE_COLLECT_METADATA]	= { .type = NLA_FLAG },
1244	[IFLA_GENEVE_UDP_CSUM]		= { .type = NLA_U8 },
1245	[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]	= { .type = NLA_U8 },
1246	[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]	= { .type = NLA_U8 },
1247	[IFLA_GENEVE_TTL_INHERIT]	= { .type = NLA_U8 },
1248	[IFLA_GENEVE_DF]		= { .type = NLA_U8 },
1249	[IFLA_GENEVE_INNER_PROTO_INHERIT]	= { .type = NLA_FLAG },
1250	[IFLA_GENEVE_PORT_RANGE]	= NLA_POLICY_EXACT_LEN(sizeof(struct ifla_geneve_port_range)),
1251};
1252
1253static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
1254			   struct netlink_ext_ack *extack)
1255{
1256	if (tb[IFLA_ADDRESS]) {
1257		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
1258			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
1259					    "Provided link layer address is not Ethernet");
1260			return -EINVAL;
1261		}
1262
1263		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
1264			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
1265					    "Provided Ethernet address is not unicast");
1266			return -EADDRNOTAVAIL;
1267		}
1268	}
1269
1270	if (!data) {
1271		NL_SET_ERR_MSG(extack,
1272			       "Not enough attributes provided to perform the operation");
1273		return -EINVAL;
1274	}
1275
1276	if (data[IFLA_GENEVE_ID]) {
1277		__u32 vni =  nla_get_u32(data[IFLA_GENEVE_ID]);
1278
1279		if (vni >= GENEVE_N_VID) {
1280			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_ID],
1281					    "Geneve ID must be lower than 16777216");
1282			return -ERANGE;
1283		}
1284	}
1285
1286	if (data[IFLA_GENEVE_DF]) {
1287		enum ifla_geneve_df df = nla_get_u8(data[IFLA_GENEVE_DF]);
1288
1289		if (df < 0 || df > GENEVE_DF_MAX) {
1290			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_DF],
1291					    "Invalid DF attribute");
1292			return -EINVAL;
1293		}
1294	}
1295
1296	if (data[IFLA_GENEVE_PORT_RANGE]) {
1297		const struct ifla_geneve_port_range *p;
1298
1299		p = nla_data(data[IFLA_GENEVE_PORT_RANGE]);
1300		if (ntohs(p->high) < ntohs(p->low)) {
1301			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_PORT_RANGE],
1302					    "Invalid source port range");
1303			return -EINVAL;
1304		}
1305	}
1306
1307	return 0;
1308}
1309
1310static struct geneve_dev *geneve_find_dev(struct geneve_net *gn,
1311					  const struct ip_tunnel_info *info,
1312					  bool *tun_on_same_port,
1313					  bool *tun_collect_md)
1314{
1315	struct geneve_dev *geneve, *t = NULL;
1316
1317	*tun_on_same_port = false;
1318	*tun_collect_md = false;
1319	list_for_each_entry(geneve, &gn->geneve_list, next) {
1320		if (info->key.tp_dst == geneve->cfg.info.key.tp_dst) {
1321			*tun_collect_md = geneve->cfg.collect_md;
1322			*tun_on_same_port = true;
1323		}
1324		if (info->key.tun_id == geneve->cfg.info.key.tun_id &&
1325		    info->key.tp_dst == geneve->cfg.info.key.tp_dst &&
1326		    !memcmp(&info->key.u, &geneve->cfg.info.key.u, sizeof(info->key.u)))
1327			t = geneve;
1328	}
1329	return t;
1330}
1331
1332static bool is_tnl_info_zero(const struct ip_tunnel_info *info)
1333{
1334	return !(info->key.tun_id || info->key.tos ||
1335		 !ip_tunnel_flags_empty(info->key.tun_flags) ||
1336		 info->key.ttl || info->key.label || info->key.tp_src ||
1337		 memchr_inv(&info->key.u, 0, sizeof(info->key.u)));
1338}
1339
1340static bool geneve_dst_addr_equal(struct ip_tunnel_info *a,
1341				  struct ip_tunnel_info *b)
1342{
1343	if (ip_tunnel_info_af(a) == AF_INET)
1344		return a->key.u.ipv4.dst == b->key.u.ipv4.dst;
1345	else
1346		return ipv6_addr_equal(&a->key.u.ipv6.dst, &b->key.u.ipv6.dst);
1347}
1348
1349static int geneve_configure(struct net *net, struct net_device *dev,
1350			    struct netlink_ext_ack *extack,
1351			    const struct geneve_config *cfg)
1352{
1353	struct geneve_net *gn = net_generic(net, geneve_net_id);
1354	struct geneve_dev *t, *geneve = netdev_priv(dev);
1355	const struct ip_tunnel_info *info = &cfg->info;
1356	bool tun_collect_md, tun_on_same_port;
1357	int err, encap_len;
1358
1359	if (cfg->collect_md && !is_tnl_info_zero(info)) {
1360		NL_SET_ERR_MSG(extack,
1361			       "Device is externally controlled, so attributes (VNI, Port, and so on) must not be specified");
1362		return -EINVAL;
1363	}
1364
1365	geneve->net = net;
1366	geneve->dev = dev;
1367
1368	t = geneve_find_dev(gn, info, &tun_on_same_port, &tun_collect_md);
1369	if (t)
1370		return -EBUSY;
1371
1372	/* make enough headroom for basic scenario */
1373	encap_len = GENEVE_BASE_HLEN + ETH_HLEN;
1374	if (!cfg->collect_md && ip_tunnel_info_af(info) == AF_INET) {
1375		encap_len += sizeof(struct iphdr);
1376		dev->max_mtu -= sizeof(struct iphdr);
1377	} else {
1378		encap_len += sizeof(struct ipv6hdr);
1379		dev->max_mtu -= sizeof(struct ipv6hdr);
1380	}
1381	dev->needed_headroom = encap_len + ETH_HLEN;
1382
1383	if (cfg->collect_md) {
1384		if (tun_on_same_port) {
1385			NL_SET_ERR_MSG(extack,
1386				       "There can be only one externally controlled device on a destination port");
1387			return -EPERM;
1388		}
1389	} else {
1390		if (tun_collect_md) {
1391			NL_SET_ERR_MSG(extack,
1392				       "There already exists an externally controlled device on this destination port");
1393			return -EPERM;
1394		}
1395	}
1396
1397	dst_cache_reset(&geneve->cfg.info.dst_cache);
1398	memcpy(&geneve->cfg, cfg, sizeof(*cfg));
1399
1400	if (geneve->cfg.inner_proto_inherit) {
1401		dev->header_ops = NULL;
1402		dev->type = ARPHRD_NONE;
1403		dev->hard_header_len = 0;
1404		dev->addr_len = 0;
1405		dev->flags = IFF_POINTOPOINT | IFF_NOARP;
1406	}
1407
1408	err = register_netdevice(dev);
1409	if (err)
1410		return err;
1411
1412	list_add(&geneve->next, &gn->geneve_list);
1413	return 0;
1414}
1415
1416static void init_tnl_info(struct ip_tunnel_info *info, __u16 dst_port)
1417{
1418	memset(info, 0, sizeof(*info));
1419	info->key.tp_dst = htons(dst_port);
1420}
1421
1422static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
1423			  struct netlink_ext_ack *extack,
1424			  struct geneve_config *cfg, bool changelink)
1425{
1426	struct ip_tunnel_info *info = &cfg->info;
1427	int attrtype;
1428
1429	if (data[IFLA_GENEVE_REMOTE] && data[IFLA_GENEVE_REMOTE6]) {
1430		NL_SET_ERR_MSG(extack,
1431			       "Cannot specify both IPv4 and IPv6 Remote addresses");
1432		return -EINVAL;
1433	}
1434
1435	if (data[IFLA_GENEVE_REMOTE]) {
1436		if (changelink && (ip_tunnel_info_af(info) == AF_INET6)) {
1437			attrtype = IFLA_GENEVE_REMOTE;
1438			goto change_notsup;
1439		}
1440
1441		info->key.u.ipv4.dst =
1442			nla_get_in_addr(data[IFLA_GENEVE_REMOTE]);
1443
1444		if (ipv4_is_multicast(info->key.u.ipv4.dst)) {
1445			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE],
1446					    "Remote IPv4 address cannot be Multicast");
1447			return -EINVAL;
1448		}
1449	}
1450
1451	if (data[IFLA_GENEVE_REMOTE6]) {
1452#if IS_ENABLED(CONFIG_IPV6)
1453		if (changelink && (ip_tunnel_info_af(info) == AF_INET)) {
1454			attrtype = IFLA_GENEVE_REMOTE6;
1455			goto change_notsup;
1456		}
1457
1458		info->mode = IP_TUNNEL_INFO_IPV6;
1459		info->key.u.ipv6.dst =
1460			nla_get_in6_addr(data[IFLA_GENEVE_REMOTE6]);
1461
1462		if (ipv6_addr_type(&info->key.u.ipv6.dst) &
1463		    IPV6_ADDR_LINKLOCAL) {
1464			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
1465					    "Remote IPv6 address cannot be link-local");
1466			return -EINVAL;
1467		}
1468		if (ipv6_addr_is_multicast(&info->key.u.ipv6.dst)) {
1469			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
1470					    "Remote IPv6 address cannot be Multicast");
1471			return -EINVAL;
1472		}
1473		__set_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
1474		cfg->use_udp6_rx_checksums = true;
1475#else
1476		NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
1477				    "IPv6 support not enabled in the kernel");
1478		return -EPFNOSUPPORT;
1479#endif
1480	}
1481
1482	if (data[IFLA_GENEVE_ID]) {
1483		__u32 vni;
1484		__u8 tvni[3];
1485		__be64 tunid;
1486
1487		vni = nla_get_u32(data[IFLA_GENEVE_ID]);
1488		tvni[0] = (vni & 0x00ff0000) >> 16;
1489		tvni[1] = (vni & 0x0000ff00) >> 8;
1490		tvni[2] =  vni & 0x000000ff;
1491
1492		tunid = vni_to_tunnel_id(tvni);
1493		if (changelink && (tunid != info->key.tun_id)) {
1494			attrtype = IFLA_GENEVE_ID;
1495			goto change_notsup;
1496		}
1497		info->key.tun_id = tunid;
1498	}
1499
1500	if (data[IFLA_GENEVE_TTL_INHERIT]) {
1501		if (nla_get_u8(data[IFLA_GENEVE_TTL_INHERIT]))
1502			cfg->ttl_inherit = true;
1503		else
1504			cfg->ttl_inherit = false;
1505	} else if (data[IFLA_GENEVE_TTL]) {
1506		info->key.ttl = nla_get_u8(data[IFLA_GENEVE_TTL]);
1507		cfg->ttl_inherit = false;
1508	}
1509
1510	if (data[IFLA_GENEVE_TOS])
1511		info->key.tos = nla_get_u8(data[IFLA_GENEVE_TOS]);
1512
1513	if (data[IFLA_GENEVE_DF])
1514		cfg->df = nla_get_u8(data[IFLA_GENEVE_DF]);
1515
1516	if (data[IFLA_GENEVE_LABEL]) {
1517		info->key.label = nla_get_be32(data[IFLA_GENEVE_LABEL]) &
1518				  IPV6_FLOWLABEL_MASK;
1519		if (info->key.label && (!(info->mode & IP_TUNNEL_INFO_IPV6))) {
1520			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_LABEL],
1521					    "Label attribute only applies for IPv6 Geneve devices");
1522			return -EINVAL;
1523		}
1524	}
1525
1526	if (data[IFLA_GENEVE_PORT]) {
1527		if (changelink) {
1528			attrtype = IFLA_GENEVE_PORT;
1529			goto change_notsup;
1530		}
1531		info->key.tp_dst = nla_get_be16(data[IFLA_GENEVE_PORT]);
1532	}
1533
1534	if (data[IFLA_GENEVE_PORT_RANGE]) {
1535		const struct ifla_geneve_port_range *p;
1536
1537		if (changelink) {
1538			attrtype = IFLA_GENEVE_PORT_RANGE;
1539			goto change_notsup;
1540		}
1541		p = nla_data(data[IFLA_GENEVE_PORT_RANGE]);
1542		cfg->port_min = ntohs(p->low);
1543		cfg->port_max = ntohs(p->high);
1544	}
1545
1546	if (data[IFLA_GENEVE_COLLECT_METADATA]) {
1547		if (changelink) {
1548			attrtype = IFLA_GENEVE_COLLECT_METADATA;
1549			goto change_notsup;
1550		}
1551		cfg->collect_md = true;
1552	}
1553
1554	if (data[IFLA_GENEVE_UDP_CSUM]) {
1555		if (changelink) {
1556			attrtype = IFLA_GENEVE_UDP_CSUM;
1557			goto change_notsup;
1558		}
1559		if (nla_get_u8(data[IFLA_GENEVE_UDP_CSUM]))
1560			__set_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
1561	}
1562
1563	if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]) {
1564#if IS_ENABLED(CONFIG_IPV6)
1565		if (changelink) {
1566			attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_TX;
1567			goto change_notsup;
1568		}
1569		if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]))
1570			__clear_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
1571#else
1572		NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX],
1573				    "IPv6 support not enabled in the kernel");
1574		return -EPFNOSUPPORT;
1575#endif
1576	}
1577
1578	if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]) {
1579#if IS_ENABLED(CONFIG_IPV6)
1580		if (changelink) {
1581			attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_RX;
1582			goto change_notsup;
1583		}
1584		if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]))
1585			cfg->use_udp6_rx_checksums = false;
1586#else
1587		NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX],
1588				    "IPv6 support not enabled in the kernel");
1589		return -EPFNOSUPPORT;
1590#endif
1591	}
1592
1593	if (data[IFLA_GENEVE_INNER_PROTO_INHERIT]) {
1594		if (changelink) {
1595			attrtype = IFLA_GENEVE_INNER_PROTO_INHERIT;
1596			goto change_notsup;
1597		}
1598		cfg->inner_proto_inherit = true;
1599	}
1600
1601	return 0;
1602change_notsup:
1603	NL_SET_ERR_MSG_ATTR(extack, data[attrtype],
1604			    "Changing VNI, Port, endpoint IP address family, external, inner_proto_inherit, and UDP checksum attributes are not supported");
1605	return -EOPNOTSUPP;
1606}
1607
1608static void geneve_link_config(struct net_device *dev,
1609			       struct ip_tunnel_info *info, struct nlattr *tb[])
1610{
1611	struct geneve_dev *geneve = netdev_priv(dev);
1612	int ldev_mtu = 0;
1613
1614	if (tb[IFLA_MTU]) {
1615		geneve_change_mtu(dev, nla_get_u32(tb[IFLA_MTU]));
1616		return;
1617	}
1618
1619	switch (ip_tunnel_info_af(info)) {
1620	case AF_INET: {
1621		struct flowi4 fl4 = { .daddr = info->key.u.ipv4.dst };
1622		struct rtable *rt = ip_route_output_key(geneve->net, &fl4);
1623
1624		if (!IS_ERR(rt) && rt->dst.dev) {
1625			ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV4_HLEN;
1626			ip_rt_put(rt);
1627		}
1628		break;
1629	}
1630#if IS_ENABLED(CONFIG_IPV6)
1631	case AF_INET6: {
1632		struct rt6_info *rt;
1633
1634		if (!__in6_dev_get(dev))
1635			break;
1636
1637		rt = rt6_lookup(geneve->net, &info->key.u.ipv6.dst, NULL, 0,
1638				NULL, 0);
1639
1640		if (rt && rt->dst.dev)
1641			ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV6_HLEN;
1642		ip6_rt_put(rt);
1643		break;
1644	}
1645#endif
1646	}
1647
1648	if (ldev_mtu <= 0)
1649		return;
1650
1651	geneve_change_mtu(dev, ldev_mtu - info->options_len);
1652}
1653
1654static int geneve_newlink(struct net_device *dev,
1655			  struct rtnl_newlink_params *params,
1656			  struct netlink_ext_ack *extack)
1657{
1658	struct net *link_net = rtnl_newlink_link_net(params);
1659	struct nlattr **data = params->data;
1660	struct nlattr **tb = params->tb;
1661	struct geneve_config cfg = {
1662		.df = GENEVE_DF_UNSET,
1663		.use_udp6_rx_checksums = false,
1664		.ttl_inherit = false,
1665		.collect_md = false,
1666		.port_min = 1,
1667		.port_max = USHRT_MAX,
1668	};
1669	int err;
1670
1671	init_tnl_info(&cfg.info, GENEVE_UDP_PORT);
1672	err = geneve_nl2info(tb, data, extack, &cfg, false);
1673	if (err)
1674		return err;
1675
1676	err = geneve_configure(link_net, dev, extack, &cfg);
1677	if (err)
1678		return err;
1679
1680	geneve_link_config(dev, &cfg.info, tb);
1681
1682	return 0;
1683}
1684
1685/* Quiesces the geneve device data path for both TX and RX.
1686 *
1687 * On transmit geneve checks for non-NULL geneve_sock before it proceeds.
1688 * So, if we set that socket to NULL under RCU and wait for synchronize_net()
1689 * to complete for the existing set of in-flight packets to be transmitted,
1690 * then we would have quiesced the transmit data path. All the future packets
1691 * will get dropped until we unquiesce the data path.
1692 *
1693 * On receive geneve dereference the geneve_sock stashed in the socket. So,
1694 * if we set that to NULL under RCU and wait for synchronize_net() to
1695 * complete, then we would have quiesced the receive data path.
1696 */
1697static void geneve_quiesce(struct geneve_dev *geneve, struct geneve_sock **gs4,
1698			   struct geneve_sock **gs6)
1699{
1700	*gs4 = rtnl_dereference(geneve->sock4);
1701	rcu_assign_pointer(geneve->sock4, NULL);
1702	if (*gs4)
1703		rcu_assign_sk_user_data((*gs4)->sock->sk, NULL);
1704#if IS_ENABLED(CONFIG_IPV6)
1705	*gs6 = rtnl_dereference(geneve->sock6);
1706	rcu_assign_pointer(geneve->sock6, NULL);
1707	if (*gs6)
1708		rcu_assign_sk_user_data((*gs6)->sock->sk, NULL);
1709#else
1710	*gs6 = NULL;
1711#endif
1712	synchronize_net();
1713}
1714
1715/* Resumes the geneve device data path for both TX and RX. */
1716static void geneve_unquiesce(struct geneve_dev *geneve, struct geneve_sock *gs4,
1717			     struct geneve_sock __maybe_unused *gs6)
1718{
1719	rcu_assign_pointer(geneve->sock4, gs4);
1720	if (gs4)
1721		rcu_assign_sk_user_data(gs4->sock->sk, gs4);
1722#if IS_ENABLED(CONFIG_IPV6)
1723	rcu_assign_pointer(geneve->sock6, gs6);
1724	if (gs6)
1725		rcu_assign_sk_user_data(gs6->sock->sk, gs6);
1726#endif
1727	synchronize_net();
1728}
1729
1730static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
1731			     struct nlattr *data[],
1732			     struct netlink_ext_ack *extack)
1733{
1734	struct geneve_dev *geneve = netdev_priv(dev);
1735	struct geneve_sock *gs4, *gs6;
1736	struct geneve_config cfg;
1737	int err;
1738
1739	/* If the geneve device is configured for metadata (or externally
1740	 * controlled, for example, OVS), then nothing can be changed.
1741	 */
1742	if (geneve->cfg.collect_md)
1743		return -EOPNOTSUPP;
1744
1745	/* Start with the existing info. */
1746	memcpy(&cfg, &geneve->cfg, sizeof(cfg));
1747	err = geneve_nl2info(tb, data, extack, &cfg, true);
1748	if (err)
1749		return err;
1750
1751	if (!geneve_dst_addr_equal(&geneve->cfg.info, &cfg.info)) {
1752		dst_cache_reset(&cfg.info.dst_cache);
1753		geneve_link_config(dev, &cfg.info, tb);
1754	}
1755
1756	geneve_quiesce(geneve, &gs4, &gs6);
1757	memcpy(&geneve->cfg, &cfg, sizeof(cfg));
1758	geneve_unquiesce(geneve, gs4, gs6);
1759
1760	return 0;
1761}
1762
1763static void geneve_dellink(struct net_device *dev, struct list_head *head)
1764{
1765	struct geneve_dev *geneve = netdev_priv(dev);
1766
1767	list_del(&geneve->next);
1768	unregister_netdevice_queue(dev, head);
1769}
1770
1771static size_t geneve_get_size(const struct net_device *dev)
1772{
1773	return nla_total_size(sizeof(__u32)) +	/* IFLA_GENEVE_ID */
1774		nla_total_size(sizeof(struct in6_addr)) + /* IFLA_GENEVE_REMOTE{6} */
1775		nla_total_size(sizeof(__u8)) +  /* IFLA_GENEVE_TTL */
1776		nla_total_size(sizeof(__u8)) +  /* IFLA_GENEVE_TOS */
1777		nla_total_size(sizeof(__u8)) +	/* IFLA_GENEVE_DF */
1778		nla_total_size(sizeof(__be32)) +  /* IFLA_GENEVE_LABEL */
1779		nla_total_size(sizeof(__be16)) +  /* IFLA_GENEVE_PORT */
1780		nla_total_size(0) +	 /* IFLA_GENEVE_COLLECT_METADATA */
1781		nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_CSUM */
1782		nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_TX */
1783		nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_RX */
1784		nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TTL_INHERIT */
1785		nla_total_size(0) +	 /* IFLA_GENEVE_INNER_PROTO_INHERIT */
1786		nla_total_size(sizeof(struct ifla_geneve_port_range)) + /* IFLA_GENEVE_PORT_RANGE */
1787		0;
1788}
1789
1790static int geneve_fill_info(struct sk_buff *skb, const struct net_device *dev)
1791{
1792	struct geneve_dev *geneve = netdev_priv(dev);
1793	struct ip_tunnel_info *info = &geneve->cfg.info;
1794	bool ttl_inherit = geneve->cfg.ttl_inherit;
1795	bool metadata = geneve->cfg.collect_md;
1796	struct ifla_geneve_port_range ports = {
1797		.low	= htons(geneve->cfg.port_min),
1798		.high	= htons(geneve->cfg.port_max),
1799	};
1800	__u8 tmp_vni[3];
1801	__u32 vni;
1802
1803	tunnel_id_to_vni(info->key.tun_id, tmp_vni);
1804	vni = (tmp_vni[0] << 16) | (tmp_vni[1] << 8) | tmp_vni[2];
1805	if (nla_put_u32(skb, IFLA_GENEVE_ID, vni))
1806		goto nla_put_failure;
1807
1808	if (!metadata && ip_tunnel_info_af(info) == AF_INET) {
1809		if (nla_put_in_addr(skb, IFLA_GENEVE_REMOTE,
1810				    info->key.u.ipv4.dst))
1811			goto nla_put_failure;
1812		if (nla_put_u8(skb, IFLA_GENEVE_UDP_CSUM,
1813			       test_bit(IP_TUNNEL_CSUM_BIT,
1814					info->key.tun_flags)))
1815			goto nla_put_failure;
1816
1817#if IS_ENABLED(CONFIG_IPV6)
1818	} else if (!metadata) {
1819		if (nla_put_in6_addr(skb, IFLA_GENEVE_REMOTE6,
1820				     &info->key.u.ipv6.dst))
1821			goto nla_put_failure;
1822		if (nla_put_u8(skb, IFLA_GENEVE_UDP_ZERO_CSUM6_TX,
1823			       !test_bit(IP_TUNNEL_CSUM_BIT,
1824					 info->key.tun_flags)))
1825			goto nla_put_failure;
1826#endif
1827	}
1828
1829	if (nla_put_u8(skb, IFLA_GENEVE_TTL, info->key.ttl) ||
1830	    nla_put_u8(skb, IFLA_GENEVE_TOS, info->key.tos) ||
1831	    nla_put_be32(skb, IFLA_GENEVE_LABEL, info->key.label))
1832		goto nla_put_failure;
1833
1834	if (nla_put_u8(skb, IFLA_GENEVE_DF, geneve->cfg.df))
1835		goto nla_put_failure;
1836
1837	if (nla_put_be16(skb, IFLA_GENEVE_PORT, info->key.tp_dst))
1838		goto nla_put_failure;
1839
1840	if (metadata && nla_put_flag(skb, IFLA_GENEVE_COLLECT_METADATA))
1841		goto nla_put_failure;
1842
1843#if IS_ENABLED(CONFIG_IPV6)
1844	if (nla_put_u8(skb, IFLA_GENEVE_UDP_ZERO_CSUM6_RX,
1845		       !geneve->cfg.use_udp6_rx_checksums))
1846		goto nla_put_failure;
1847#endif
1848
1849	if (nla_put_u8(skb, IFLA_GENEVE_TTL_INHERIT, ttl_inherit))
1850		goto nla_put_failure;
1851
1852	if (geneve->cfg.inner_proto_inherit &&
1853	    nla_put_flag(skb, IFLA_GENEVE_INNER_PROTO_INHERIT))
1854		goto nla_put_failure;
1855
1856	if (nla_put(skb, IFLA_GENEVE_PORT_RANGE, sizeof(ports), &ports))
1857		goto nla_put_failure;
1858
1859	return 0;
1860
1861nla_put_failure:
1862	return -EMSGSIZE;
1863}
1864
1865static struct rtnl_link_ops geneve_link_ops __read_mostly = {
1866	.kind		= "geneve",
1867	.maxtype	= IFLA_GENEVE_MAX,
1868	.policy		= geneve_policy,
1869	.priv_size	= sizeof(struct geneve_dev),
1870	.setup		= geneve_setup,
1871	.validate	= geneve_validate,
1872	.newlink	= geneve_newlink,
1873	.changelink	= geneve_changelink,
1874	.dellink	= geneve_dellink,
1875	.get_size	= geneve_get_size,
1876	.fill_info	= geneve_fill_info,
1877};
1878
1879struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
1880					u8 name_assign_type, u16 dst_port)
1881{
1882	struct nlattr *tb[IFLA_MAX + 1];
1883	struct net_device *dev;
1884	LIST_HEAD(list_kill);
1885	int err;
1886	struct geneve_config cfg = {
1887		.df = GENEVE_DF_UNSET,
1888		.use_udp6_rx_checksums = true,
1889		.ttl_inherit = false,
1890		.collect_md = true,
1891		.port_min = 1,
1892		.port_max = USHRT_MAX,
1893	};
1894
1895	memset(tb, 0, sizeof(tb));
1896	dev = rtnl_create_link(net, name, name_assign_type,
1897			       &geneve_link_ops, tb, NULL);
1898	if (IS_ERR(dev))
1899		return dev;
1900
1901	init_tnl_info(&cfg.info, dst_port);
1902	err = geneve_configure(net, dev, NULL, &cfg);
1903	if (err) {
1904		free_netdev(dev);
1905		return ERR_PTR(err);
1906	}
1907
1908	/* openvswitch users expect packet sizes to be unrestricted,
1909	 * so set the largest MTU we can.
1910	 */
1911	err = geneve_change_mtu(dev, IP_MAX_MTU);
1912	if (err)
1913		goto err;
1914
1915	err = rtnl_configure_link(dev, NULL, 0, NULL);
1916	if (err < 0)
1917		goto err;
1918
1919	return dev;
1920err:
1921	geneve_dellink(dev, &list_kill);
1922	unregister_netdevice_many(&list_kill);
1923	return ERR_PTR(err);
1924}
1925EXPORT_SYMBOL_GPL(geneve_dev_create_fb);
1926
1927static int geneve_netdevice_event(struct notifier_block *unused,
1928				  unsigned long event, void *ptr)
1929{
1930	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1931
1932	if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
1933		geneve_offload_rx_ports(dev, true);
1934	else if (event == NETDEV_UDP_TUNNEL_DROP_INFO)
1935		geneve_offload_rx_ports(dev, false);
1936
1937	return NOTIFY_DONE;
1938}
1939
1940static struct notifier_block geneve_notifier_block __read_mostly = {
1941	.notifier_call = geneve_netdevice_event,
1942};
1943
1944static __net_init int geneve_init_net(struct net *net)
1945{
1946	struct geneve_net *gn = net_generic(net, geneve_net_id);
1947
1948	INIT_LIST_HEAD(&gn->geneve_list);
1949	INIT_LIST_HEAD(&gn->sock_list);
1950	return 0;
1951}
1952
1953static void __net_exit geneve_exit_rtnl_net(struct net *net,
1954					    struct list_head *dev_to_kill)
1955{
1956	struct geneve_net *gn = net_generic(net, geneve_net_id);
1957	struct geneve_dev *geneve, *next;
1958
1959	list_for_each_entry_safe(geneve, next, &gn->geneve_list, next)
1960		geneve_dellink(geneve->dev, dev_to_kill);
1961}
1962
1963static void __net_exit geneve_exit_net(struct net *net)
1964{
1965	const struct geneve_net *gn = net_generic(net, geneve_net_id);
1966
1967	WARN_ON_ONCE(!list_empty(&gn->sock_list));
1968}
1969
1970static struct pernet_operations geneve_net_ops = {
1971	.init = geneve_init_net,
1972	.exit_rtnl = geneve_exit_rtnl_net,
1973	.exit = geneve_exit_net,
1974	.id   = &geneve_net_id,
1975	.size = sizeof(struct geneve_net),
1976};
1977
1978static int __init geneve_init_module(void)
1979{
1980	int rc;
1981
1982	rc = register_pernet_subsys(&geneve_net_ops);
1983	if (rc)
1984		goto out1;
1985
1986	rc = register_netdevice_notifier(&geneve_notifier_block);
1987	if (rc)
1988		goto out2;
1989
1990	rc = rtnl_link_register(&geneve_link_ops);
1991	if (rc)
1992		goto out3;
1993
1994	return 0;
1995out3:
1996	unregister_netdevice_notifier(&geneve_notifier_block);
1997out2:
1998	unregister_pernet_subsys(&geneve_net_ops);
1999out1:
2000	return rc;
2001}
2002late_initcall(geneve_init_module);
2003
2004static void __exit geneve_cleanup_module(void)
2005{
2006	rtnl_link_unregister(&geneve_link_ops);
2007	unregister_netdevice_notifier(&geneve_notifier_block);
2008	unregister_pernet_subsys(&geneve_net_ops);
2009}
2010module_exit(geneve_cleanup_module);
2011
2012MODULE_LICENSE("GPL");
2013MODULE_VERSION(GENEVE_NETDEV_VER);
2014MODULE_AUTHOR("John W. Linville <linville@tuxdriver.com>");
2015MODULE_DESCRIPTION("Interface driver for GENEVE encapsulated traffic");
2016MODULE_ALIAS_RTNL_LINK("geneve");