xfrm_interface_core.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *	XFRM virtual interface
   4 *
   5 *	Copyright (C) 2018 secunet Security Networks AG
   6 *
   7 *	Author:
   8 *	Steffen Klassert <steffen.klassert@secunet.com>
   9 */
  10
  11#include <linux/module.h>
  12#include <linux/capability.h>
  13#include <linux/errno.h>
  14#include <linux/types.h>
  15#include <linux/sockios.h>
  16#include <linux/icmp.h>
  17#include <linux/if.h>
  18#include <linux/in.h>
  19#include <linux/ip.h>
  20#include <linux/net.h>
  21#include <linux/in6.h>
  22#include <linux/netdevice.h>
  23#include <linux/if_link.h>
  24#include <linux/if_arp.h>
  25#include <linux/icmpv6.h>
  26#include <linux/init.h>
  27#include <linux/route.h>
  28#include <linux/rtnetlink.h>
  29#include <linux/netfilter_ipv6.h>
  30#include <linux/slab.h>
  31#include <linux/hash.h>
  32
  33#include <linux/uaccess.h>
  34#include <linux/atomic.h>
  35
  36#include <net/gso.h>
  37#include <net/icmp.h>
  38#include <net/ip.h>
  39#include <net/ipv6.h>
  40#include <net/ip6_route.h>
  41#include <net/ip_tunnels.h>
  42#include <net/addrconf.h>
  43#include <net/xfrm.h>
  44#include <net/net_namespace.h>
  45#include <net/dst_metadata.h>
  46#include <net/netns/generic.h>
  47#include <linux/etherdevice.h>
  48
  49static int xfrmi_dev_init(struct net_device *dev);
  50static void xfrmi_dev_setup(struct net_device *dev);
  51static struct rtnl_link_ops xfrmi_link_ops __read_mostly;
  52static unsigned int xfrmi_net_id __read_mostly;
  53static const struct net_device_ops xfrmi_netdev_ops;
  54
  55#define XFRMI_HASH_BITS	8
  56#define XFRMI_HASH_SIZE	BIT(XFRMI_HASH_BITS)
  57
  58struct xfrmi_net {
  59	/* lists for storing interfaces in use */
  60	struct xfrm_if __rcu *xfrmi[XFRMI_HASH_SIZE];
  61	struct xfrm_if __rcu *collect_md_xfrmi;
  62};
  63
  64static const struct nla_policy xfrm_lwt_policy[LWT_XFRM_MAX + 1] = {
  65	[LWT_XFRM_IF_ID]	= NLA_POLICY_MIN(NLA_U32, 1),
  66	[LWT_XFRM_LINK]		= NLA_POLICY_MIN(NLA_U32, 1),
  67};
  68
  69static void xfrmi_destroy_state(struct lwtunnel_state *lwt)
  70{
  71}
  72
  73static int xfrmi_build_state(struct net *net, struct nlattr *nla,
  74			     unsigned int family, const void *cfg,
  75			     struct lwtunnel_state **ts,
  76			     struct netlink_ext_ack *extack)
  77{
  78	struct nlattr *tb[LWT_XFRM_MAX + 1];
  79	struct lwtunnel_state *new_state;
  80	struct xfrm_md_info *info;
  81	int ret;
  82
  83	ret = nla_parse_nested(tb, LWT_XFRM_MAX, nla, xfrm_lwt_policy, extack);
  84	if (ret < 0)
  85		return ret;
  86
  87	if (!tb[LWT_XFRM_IF_ID]) {
  88		NL_SET_ERR_MSG(extack, "if_id must be set");
  89		return -EINVAL;
  90	}
  91
  92	new_state = lwtunnel_state_alloc(sizeof(*info));
  93	if (!new_state) {
  94		NL_SET_ERR_MSG(extack, "failed to create encap info");
  95		return -ENOMEM;
  96	}
  97
  98	new_state->type = LWTUNNEL_ENCAP_XFRM;
  99
 100	info = lwt_xfrm_info(new_state);
 101
 102	info->if_id = nla_get_u32(tb[LWT_XFRM_IF_ID]);
 103
 104	if (tb[LWT_XFRM_LINK])
 105		info->link = nla_get_u32(tb[LWT_XFRM_LINK]);
 106
 107	*ts = new_state;
 108	return 0;
 109}
 110
 111static int xfrmi_fill_encap_info(struct sk_buff *skb,
 112				 struct lwtunnel_state *lwt)
 113{
 114	struct xfrm_md_info *info = lwt_xfrm_info(lwt);
 115
 116	if (nla_put_u32(skb, LWT_XFRM_IF_ID, info->if_id) ||
 117	    (info->link && nla_put_u32(skb, LWT_XFRM_LINK, info->link)))
 118		return -EMSGSIZE;
 119
 120	return 0;
 121}
 122
 123static int xfrmi_encap_nlsize(struct lwtunnel_state *lwtstate)
 124{
 125	return nla_total_size(sizeof(u32)) + /* LWT_XFRM_IF_ID */
 126		nla_total_size(sizeof(u32)); /* LWT_XFRM_LINK */
 127}
 128
 129static int xfrmi_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b)
 130{
 131	struct xfrm_md_info *a_info = lwt_xfrm_info(a);
 132	struct xfrm_md_info *b_info = lwt_xfrm_info(b);
 133
 134	return memcmp(a_info, b_info, sizeof(*a_info));
 135}
 136
 137static const struct lwtunnel_encap_ops xfrmi_encap_ops = {
 138	.build_state	= xfrmi_build_state,
 139	.destroy_state	= xfrmi_destroy_state,
 140	.fill_encap	= xfrmi_fill_encap_info,
 141	.get_encap_size = xfrmi_encap_nlsize,
 142	.cmp_encap	= xfrmi_encap_cmp,
 143	.owner		= THIS_MODULE,
 144};
 145
 146#define for_each_xfrmi_rcu(start, xi) \
 147	for (xi = rcu_dereference(start); xi; xi = rcu_dereference(xi->next))
 148
 149static u32 xfrmi_hash(u32 if_id)
 150{
 151	return hash_32(if_id, XFRMI_HASH_BITS);
 152}
 153
 154static struct xfrm_if *xfrmi_lookup(struct net *net, struct xfrm_state *x)
 155{
 156	struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 157	struct xfrm_if *xi;
 158
 159	for_each_xfrmi_rcu(xfrmn->xfrmi[xfrmi_hash(x->if_id)], xi) {
 160		if (x->if_id == xi->p.if_id &&
 161		    (xi->dev->flags & IFF_UP))
 162			return xi;
 163	}
 164
 165	xi = rcu_dereference(xfrmn->collect_md_xfrmi);
 166	if (xi && (xi->dev->flags & IFF_UP))
 167		return xi;
 168
 169	return NULL;
 170}
 171
 172static bool xfrmi_decode_session(struct sk_buff *skb,
 173				 unsigned short family,
 174				 struct xfrm_if_decode_session_result *res)
 175{
 176	struct net_device *dev;
 177	struct xfrm_if *xi;
 178	int ifindex = 0;
 179
 180	if (!secpath_exists(skb) || !skb->dev)
 181		return false;
 182
 183	switch (family) {
 184	case AF_INET6:
 185		ifindex = inet6_sdif(skb);
 186		break;
 187	case AF_INET:
 188		ifindex = inet_sdif(skb);
 189		break;
 190	}
 191
 192	if (ifindex) {
 193		struct net *net = xs_net(xfrm_input_state(skb));
 194
 195		dev = dev_get_by_index_rcu(net, ifindex);
 196	} else {
 197		dev = skb->dev;
 198	}
 199
 200	if (!dev || !(dev->flags & IFF_UP))
 201		return false;
 202	if (dev->netdev_ops != &xfrmi_netdev_ops)
 203		return false;
 204
 205	xi = netdev_priv(dev);
 206	res->net = xi->net;
 207
 208	if (xi->p.collect_md)
 209		res->if_id = xfrm_input_state(skb)->if_id;
 210	else
 211		res->if_id = xi->p.if_id;
 212	return true;
 213}
 214
 215static void xfrmi_link(struct xfrmi_net *xfrmn, struct xfrm_if *xi)
 216{
 217	struct xfrm_if __rcu **xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)];
 218
 219	rcu_assign_pointer(xi->next , rtnl_dereference(*xip));
 220	rcu_assign_pointer(*xip, xi);
 221}
 222
 223static void xfrmi_unlink(struct xfrmi_net *xfrmn, struct xfrm_if *xi)
 224{
 225	struct xfrm_if __rcu **xip;
 226	struct xfrm_if *iter;
 227
 228	for (xip = &xfrmn->xfrmi[xfrmi_hash(xi->p.if_id)];
 229	     (iter = rtnl_dereference(*xip)) != NULL;
 230	     xip = &iter->next) {
 231		if (xi == iter) {
 232			rcu_assign_pointer(*xip, xi->next);
 233			break;
 234		}
 235	}
 236}
 237
 238static void xfrmi_dev_free(struct net_device *dev)
 239{
 240	struct xfrm_if *xi = netdev_priv(dev);
 241
 242	gro_cells_destroy(&xi->gro_cells);
 243}
 244
 245static int xfrmi_create(struct net *net, struct net_device *dev)
 246{
 247	struct xfrm_if *xi = netdev_priv(dev);
 248	struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 249	int err;
 250
 251	dev->rtnl_link_ops = &xfrmi_link_ops;
 252	err = register_netdevice(dev);
 253	if (err < 0)
 254		goto out;
 255
 256	if (xi->p.collect_md)
 257		rcu_assign_pointer(xfrmn->collect_md_xfrmi, xi);
 258	else
 259		xfrmi_link(xfrmn, xi);
 260
 261	return 0;
 262
 263out:
 264	return err;
 265}
 266
 267static struct xfrm_if *xfrmi_locate(struct net *net, struct xfrm_if_parms *p)
 268{
 269	struct xfrm_if __rcu **xip;
 270	struct xfrm_if *xi;
 271	struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 272
 273	for (xip = &xfrmn->xfrmi[xfrmi_hash(p->if_id)];
 274	     (xi = rtnl_dereference(*xip)) != NULL;
 275	     xip = &xi->next)
 276		if (xi->p.if_id == p->if_id)
 277			return xi;
 278
 279	return NULL;
 280}
 281
 282static void xfrmi_dev_uninit(struct net_device *dev)
 283{
 284	struct xfrm_if *xi = netdev_priv(dev);
 285	struct xfrmi_net *xfrmn = net_generic(xi->net, xfrmi_net_id);
 286
 287	if (xi->p.collect_md)
 288		RCU_INIT_POINTER(xfrmn->collect_md_xfrmi, NULL);
 289	else
 290		xfrmi_unlink(xfrmn, xi);
 291}
 292
 293static void xfrmi_scrub_packet(struct sk_buff *skb, bool xnet)
 294{
 295	skb_clear_tstamp(skb);
 296	skb->pkt_type = PACKET_HOST;
 297	skb->skb_iif = 0;
 298	skb->ignore_df = 0;
 299	skb_dst_drop(skb);
 300	nf_reset_ct(skb);
 301	nf_reset_trace(skb);
 302
 303	if (!xnet)
 304		return;
 305
 306	ipvs_reset(skb);
 307	secpath_reset(skb);
 308	skb_orphan(skb);
 309	skb->mark = 0;
 310}
 311
 312static int xfrmi_input(struct sk_buff *skb, int nexthdr, __be32 spi,
 313		       int encap_type, unsigned short family)
 314{
 315	struct sec_path *sp;
 316
 317	sp = skb_sec_path(skb);
 318	if (sp && (sp->len || sp->olen) &&
 319	    !xfrm_policy_check(NULL, XFRM_POLICY_IN, skb, family))
 320		goto discard;
 321
 322	XFRM_SPI_SKB_CB(skb)->family = family;
 323	if (family == AF_INET) {
 324		XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);
 325		XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
 326	} else {
 327		XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
 328		XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
 329	}
 330
 331	return xfrm_input(skb, nexthdr, spi, encap_type);
 332discard:
 333	kfree_skb(skb);
 334	return 0;
 335}
 336
 337static int xfrmi4_rcv(struct sk_buff *skb)
 338{
 339	return xfrmi_input(skb, ip_hdr(skb)->protocol, 0, 0, AF_INET);
 340}
 341
 342static int xfrmi6_rcv(struct sk_buff *skb)
 343{
 344	return xfrmi_input(skb, skb_network_header(skb)[IP6CB(skb)->nhoff],
 345			   0, 0, AF_INET6);
 346}
 347
 348static int xfrmi4_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
 349{
 350	return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET);
 351}
 352
 353static int xfrmi6_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
 354{
 355	return xfrmi_input(skb, nexthdr, spi, encap_type, AF_INET6);
 356}
 357
 358static int xfrmi_rcv_cb(struct sk_buff *skb, int err)
 359{
 360	const struct xfrm_mode *inner_mode;
 361	struct net_device *dev;
 362	struct xfrm_state *x;
 363	struct xfrm_if *xi;
 364	bool xnet;
 365	int link;
 366
 367	if (err && !secpath_exists(skb))
 368		return 0;
 369
 370	x = xfrm_input_state(skb);
 371
 372	xi = xfrmi_lookup(xs_net(x), x);
 373	if (!xi)
 374		return 1;
 375
 376	link = skb->dev->ifindex;
 377	dev = xi->dev;
 378	skb->dev = dev;
 379
 380	if (err) {
 381		DEV_STATS_INC(dev, rx_errors);
 382		DEV_STATS_INC(dev, rx_dropped);
 383
 384		return 0;
 385	}
 386
 387	xnet = !net_eq(xi->net, dev_net(skb->dev));
 388
 389	if (xnet) {
 390		inner_mode = &x->inner_mode;
 391
 392		if (x->sel.family == AF_UNSPEC) {
 393			inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
 394			if (inner_mode == NULL) {
 395				XFRM_INC_STATS(dev_net(skb->dev),
 396					       LINUX_MIB_XFRMINSTATEMODEERROR);
 397				return -EINVAL;
 398			}
 399		}
 400
 401		if (!xfrm_policy_check(NULL, XFRM_POLICY_IN, skb,
 402				       inner_mode->family))
 403			return -EPERM;
 404	}
 405
 406	xfrmi_scrub_packet(skb, xnet);
 407	if (xi->p.collect_md) {
 408		struct metadata_dst *md_dst;
 409
 410		md_dst = metadata_dst_alloc(0, METADATA_XFRM, GFP_ATOMIC);
 411		if (!md_dst)
 412			return -ENOMEM;
 413
 414		md_dst->u.xfrm_info.if_id = x->if_id;
 415		md_dst->u.xfrm_info.link = link;
 416		skb_dst_set(skb, (struct dst_entry *)md_dst);
 417	}
 418	dev_sw_netstats_rx_add(dev, skb->len);
 419
 420	return 0;
 421}
 422
 423static int
 424xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl)
 425{
 426	struct xfrm_if *xi = netdev_priv(dev);
 427	struct dst_entry *dst = skb_dst(skb);
 428	unsigned int length = skb->len;
 429	struct net_device *tdev;
 430	struct xfrm_state *x;
 431	int err = -1;
 432	u32 if_id;
 433	int mtu;
 434
 435	if (xi->p.collect_md) {
 436		struct xfrm_md_info *md_info = skb_xfrm_md_info(skb);
 437
 438		if (unlikely(!md_info))
 439			return -EINVAL;
 440
 441		if_id = md_info->if_id;
 442		fl->flowi_oif = md_info->link;
 443		if (md_info->dst_orig) {
 444			struct dst_entry *tmp_dst = dst;
 445
 446			dst = md_info->dst_orig;
 447			skb_dst_set(skb, dst);
 448			md_info->dst_orig = NULL;
 449			dst_release(tmp_dst);
 450		}
 451	} else {
 452		if_id = xi->p.if_id;
 453	}
 454
 455	dst_hold(dst);
 456	dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, if_id);
 457	if (IS_ERR(dst)) {
 458		err = PTR_ERR(dst);
 459		dst = NULL;
 460		goto tx_err_link_failure;
 461	}
 462
 463	x = dst->xfrm;
 464	if (!x)
 465		goto tx_err_link_failure;
 466
 467	if (x->if_id != if_id)
 468		goto tx_err_link_failure;
 469
 470	tdev = dst->dev;
 471
 472	if (tdev == dev) {
 473		DEV_STATS_INC(dev, collisions);
 474		net_warn_ratelimited("%s: Local routing loop detected!\n",
 475				     dev->name);
 476		goto tx_err_dst_release;
 477	}
 478
 479	mtu = dst_mtu(dst);
 480	if ((!skb_is_gso(skb) && skb->len > mtu) ||
 481	    (skb_is_gso(skb) && !skb_gso_validate_network_len(skb, mtu))) {
 482		skb_dst_update_pmtu_no_confirm(skb, mtu);
 483
 484		if (skb->protocol == htons(ETH_P_IPV6)) {
 485			if (mtu < IPV6_MIN_MTU)
 486				mtu = IPV6_MIN_MTU;
 487
 488			if (skb->len > 1280)
 489				icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 490			else
 491				goto xmit;
 492		} else {
 493			if (!(ip_hdr(skb)->frag_off & htons(IP_DF)))
 494				goto xmit;
 495			icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
 496				      htonl(mtu));
 497		}
 498
 499		dst_release(dst);
 500		return -EMSGSIZE;
 501	}
 502
 503xmit:
 504	xfrmi_scrub_packet(skb, !net_eq(xi->net, dev_net(dev)));
 505	skb_dst_set(skb, dst);
 506	skb->dev = tdev;
 507
 508	err = dst_output(xi->net, skb_to_full_sk(skb), skb);
 509	if (net_xmit_eval(err) == 0) {
 510		dev_sw_netstats_tx_add(dev, 1, length);
 511	} else {
 512		DEV_STATS_INC(dev, tx_errors);
 513		DEV_STATS_INC(dev, tx_aborted_errors);
 514	}
 515
 516	return 0;
 517tx_err_link_failure:
 518	DEV_STATS_INC(dev, tx_carrier_errors);
 519	dst_link_failure(skb);
 520tx_err_dst_release:
 521	dst_release(dst);
 522	return err;
 523}
 524
 525static netdev_tx_t xfrmi_xmit(struct sk_buff *skb, struct net_device *dev)
 526{
 527	struct xfrm_if *xi = netdev_priv(dev);
 528	struct dst_entry *dst = skb_dst(skb);
 529	struct flowi fl;
 530	int ret;
 531
 532	memset(&fl, 0, sizeof(fl));
 533
 534	switch (skb->protocol) {
 535	case htons(ETH_P_IPV6):
 536		memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
 537		xfrm_decode_session(dev_net(dev), skb, &fl, AF_INET6);
 538		if (!dst) {
 539			fl.u.ip6.flowi6_oif = dev->ifindex;
 540			fl.u.ip6.flowi6_flags |= FLOWI_FLAG_ANYSRC;
 541			dst = ip6_route_output(dev_net(dev), NULL, &fl.u.ip6);
 542			if (dst->error) {
 543				dst_release(dst);
 544				DEV_STATS_INC(dev, tx_carrier_errors);
 545				goto tx_err;
 546			}
 547			skb_dst_set(skb, dst);
 548		}
 549		break;
 550	case htons(ETH_P_IP):
 551		memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
 552		xfrm_decode_session(dev_net(dev), skb, &fl, AF_INET);
 553		if (!dst) {
 554			struct rtable *rt;
 555
 556			fl.u.ip4.flowi4_oif = dev->ifindex;
 557			fl.u.ip4.flowi4_flags |= FLOWI_FLAG_ANYSRC;
 558			rt = __ip_route_output_key(dev_net(dev), &fl.u.ip4);
 559			if (IS_ERR(rt)) {
 560				DEV_STATS_INC(dev, tx_carrier_errors);
 561				goto tx_err;
 562			}
 563			skb_dst_set(skb, &rt->dst);
 564		}
 565		break;
 566	default:
 567		goto tx_err;
 568	}
 569
 570	fl.flowi_oif = xi->p.link;
 571
 572	ret = xfrmi_xmit2(skb, dev, &fl);
 573	if (ret < 0)
 574		goto tx_err;
 575
 576	return NETDEV_TX_OK;
 577
 578tx_err:
 579	DEV_STATS_INC(dev, tx_errors);
 580	DEV_STATS_INC(dev, tx_dropped);
 581	kfree_skb(skb);
 582	return NETDEV_TX_OK;
 583}
 584
 585static int xfrmi4_err(struct sk_buff *skb, u32 info)
 586{
 587	const struct iphdr *iph = (const struct iphdr *)skb->data;
 588	struct net *net = dev_net(skb->dev);
 589	int protocol = iph->protocol;
 590	struct ip_comp_hdr *ipch;
 591	struct ip_esp_hdr *esph;
 592	struct ip_auth_hdr *ah ;
 593	struct xfrm_state *x;
 594	struct xfrm_if *xi;
 595	__be32 spi;
 596
 597	switch (protocol) {
 598	case IPPROTO_ESP:
 599		esph = (struct ip_esp_hdr *)(skb->data+(iph->ihl<<2));
 600		spi = esph->spi;
 601		break;
 602	case IPPROTO_AH:
 603		ah = (struct ip_auth_hdr *)(skb->data+(iph->ihl<<2));
 604		spi = ah->spi;
 605		break;
 606	case IPPROTO_COMP:
 607		ipch = (struct ip_comp_hdr *)(skb->data+(iph->ihl<<2));
 608		spi = htonl(ntohs(ipch->cpi));
 609		break;
 610	default:
 611		return 0;
 612	}
 613
 614	switch (icmp_hdr(skb)->type) {
 615	case ICMP_DEST_UNREACH:
 616		if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED)
 617			return 0;
 618		break;
 619	case ICMP_REDIRECT:
 620		break;
 621	default:
 622		return 0;
 623	}
 624
 625	x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
 626			      spi, protocol, AF_INET);
 627	if (!x)
 628		return 0;
 629
 630	xi = xfrmi_lookup(net, x);
 631	if (!xi) {
 632		xfrm_state_put(x);
 633		return -1;
 634	}
 635
 636	if (icmp_hdr(skb)->type == ICMP_DEST_UNREACH)
 637		ipv4_update_pmtu(skb, net, info, 0, protocol);
 638	else
 639		ipv4_redirect(skb, net, 0, protocol);
 640	xfrm_state_put(x);
 641
 642	return 0;
 643}
 644
 645static int xfrmi6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 646		    u8 type, u8 code, int offset, __be32 info)
 647{
 648	const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data;
 649	struct net *net = dev_net(skb->dev);
 650	int protocol = iph->nexthdr;
 651	struct ip_comp_hdr *ipch;
 652	struct ip_esp_hdr *esph;
 653	struct ip_auth_hdr *ah;
 654	struct xfrm_state *x;
 655	struct xfrm_if *xi;
 656	__be32 spi;
 657
 658	switch (protocol) {
 659	case IPPROTO_ESP:
 660		esph = (struct ip_esp_hdr *)(skb->data + offset);
 661		spi = esph->spi;
 662		break;
 663	case IPPROTO_AH:
 664		ah = (struct ip_auth_hdr *)(skb->data + offset);
 665		spi = ah->spi;
 666		break;
 667	case IPPROTO_COMP:
 668		ipch = (struct ip_comp_hdr *)(skb->data + offset);
 669		spi = htonl(ntohs(ipch->cpi));
 670		break;
 671	default:
 672		return 0;
 673	}
 674
 675	if (type != ICMPV6_PKT_TOOBIG &&
 676	    type != NDISC_REDIRECT)
 677		return 0;
 678
 679	x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
 680			      spi, protocol, AF_INET6);
 681	if (!x)
 682		return 0;
 683
 684	xi = xfrmi_lookup(net, x);
 685	if (!xi) {
 686		xfrm_state_put(x);
 687		return -1;
 688	}
 689
 690	if (type == NDISC_REDIRECT)
 691		ip6_redirect(skb, net, skb->dev->ifindex, 0,
 692			     sock_net_uid(net, NULL));
 693	else
 694		ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL));
 695	xfrm_state_put(x);
 696
 697	return 0;
 698}
 699
 700static int xfrmi_change(struct xfrm_if *xi, const struct xfrm_if_parms *p)
 701{
 702	if (xi->p.link != p->link)
 703		return -EINVAL;
 704
 705	xi->p.if_id = p->if_id;
 706
 707	return 0;
 708}
 709
 710static int xfrmi_update(struct xfrm_if *xi, struct xfrm_if_parms *p)
 711{
 712	struct net *net = xi->net;
 713	struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 714	int err;
 715
 716	xfrmi_unlink(xfrmn, xi);
 717	synchronize_net();
 718	err = xfrmi_change(xi, p);
 719	xfrmi_link(xfrmn, xi);
 720	netdev_state_change(xi->dev);
 721	return err;
 722}
 723
 724static int xfrmi_get_iflink(const struct net_device *dev)
 725{
 726	struct xfrm_if *xi = netdev_priv(dev);
 727
 728	return READ_ONCE(xi->p.link);
 729}
 730
 731static const struct net_device_ops xfrmi_netdev_ops = {
 732	.ndo_init	= xfrmi_dev_init,
 733	.ndo_uninit	= xfrmi_dev_uninit,
 734	.ndo_start_xmit = xfrmi_xmit,
 735	.ndo_get_stats64 = dev_get_tstats64,
 736	.ndo_get_iflink = xfrmi_get_iflink,
 737};
 738
 739static void xfrmi_dev_setup(struct net_device *dev)
 740{
 741	dev->netdev_ops 	= &xfrmi_netdev_ops;
 742	dev->header_ops		= &ip_tunnel_header_ops;
 743	dev->type		= ARPHRD_NONE;
 744	dev->mtu		= ETH_DATA_LEN;
 745	dev->min_mtu		= ETH_MIN_MTU;
 746	dev->max_mtu		= IP_MAX_MTU;
 747	dev->flags 		= IFF_NOARP;
 748	dev->needs_free_netdev	= true;
 749	dev->priv_destructor	= xfrmi_dev_free;
 750	dev->pcpu_stat_type	= NETDEV_PCPU_STAT_TSTATS;
 751	netif_keep_dst(dev);
 752
 753	eth_broadcast_addr(dev->broadcast);
 754}
 755
 756#define XFRMI_FEATURES (NETIF_F_SG |		\
 757			NETIF_F_FRAGLIST |	\
 758			NETIF_F_GSO_SOFTWARE |	\
 759			NETIF_F_HW_CSUM)
 760
 761static int xfrmi_dev_init(struct net_device *dev)
 762{
 763	struct xfrm_if *xi = netdev_priv(dev);
 764	struct net_device *phydev = __dev_get_by_index(xi->net, xi->p.link);
 765	int err;
 766
 767	err = gro_cells_init(&xi->gro_cells, dev);
 768	if (err)
 769		return err;
 770
 771	dev->lltx = true;
 772	dev->features |= XFRMI_FEATURES;
 773	dev->hw_features |= XFRMI_FEATURES;
 774
 775	if (phydev) {
 776		dev->needed_headroom = phydev->needed_headroom;
 777		dev->needed_tailroom = phydev->needed_tailroom;
 778
 779		if (is_zero_ether_addr(dev->dev_addr))
 780			eth_hw_addr_inherit(dev, phydev);
 781		if (is_zero_ether_addr(dev->broadcast))
 782			memcpy(dev->broadcast, phydev->broadcast,
 783			       dev->addr_len);
 784	} else {
 785		eth_hw_addr_random(dev);
 786		eth_broadcast_addr(dev->broadcast);
 787	}
 788
 789	return 0;
 790}
 791
 792static int xfrmi_validate(struct nlattr *tb[], struct nlattr *data[],
 793			 struct netlink_ext_ack *extack)
 794{
 795	return 0;
 796}
 797
 798static void xfrmi_netlink_parms(struct nlattr *data[],
 799			       struct xfrm_if_parms *parms)
 800{
 801	memset(parms, 0, sizeof(*parms));
 802
 803	if (!data)
 804		return;
 805
 806	if (data[IFLA_XFRM_LINK])
 807		parms->link = nla_get_u32(data[IFLA_XFRM_LINK]);
 808
 809	if (data[IFLA_XFRM_IF_ID])
 810		parms->if_id = nla_get_u32(data[IFLA_XFRM_IF_ID]);
 811
 812	if (data[IFLA_XFRM_COLLECT_METADATA])
 813		parms->collect_md = true;
 814}
 815
 816static int xfrmi_newlink(struct net_device *dev,
 817			 struct rtnl_newlink_params *params,
 818			 struct netlink_ext_ack *extack)
 819{
 820	struct nlattr **data = params->data;
 821	struct xfrm_if_parms p = {};
 822	struct xfrm_if *xi;
 823	struct net *net;
 824	int err;
 825
 826	net = params->link_net ? : dev_net(dev);
 827	xfrmi_netlink_parms(data, &p);
 828	if (p.collect_md) {
 829		struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 830
 831		if (p.link || p.if_id) {
 832			NL_SET_ERR_MSG(extack, "link and if_id must be zero");
 833			return -EINVAL;
 834		}
 835
 836		if (rtnl_dereference(xfrmn->collect_md_xfrmi))
 837			return -EEXIST;
 838
 839	} else {
 840		if (!p.if_id) {
 841			NL_SET_ERR_MSG(extack, "if_id must be non zero");
 842			return -EINVAL;
 843		}
 844
 845		xi = xfrmi_locate(net, &p);
 846		if (xi)
 847			return -EEXIST;
 848	}
 849
 850	xi = netdev_priv(dev);
 851	xi->p = p;
 852	xi->net = net;
 853	xi->dev = dev;
 854
 855	err = xfrmi_create(net, dev);
 856	return err;
 857}
 858
 859static void xfrmi_dellink(struct net_device *dev, struct list_head *head)
 860{
 861	unregister_netdevice_queue(dev, head);
 862}
 863
 864static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[],
 865			   struct nlattr *data[],
 866			   struct netlink_ext_ack *extack)
 867{
 868	struct xfrm_if *xi = netdev_priv(dev);
 869	struct net *net = xi->net;
 870	struct xfrm_if_parms p = {};
 871
 872	xfrmi_netlink_parms(data, &p);
 873	if (!p.if_id) {
 874		NL_SET_ERR_MSG(extack, "if_id must be non zero");
 875		return -EINVAL;
 876	}
 877
 878	if (p.collect_md || xi->p.collect_md) {
 879		NL_SET_ERR_MSG(extack, "collect_md can't be changed");
 880		return -EINVAL;
 881	}
 882
 883	xi = xfrmi_locate(net, &p);
 884	if (!xi) {
 885		xi = netdev_priv(dev);
 886	} else {
 887		if (xi->dev != dev)
 888			return -EEXIST;
 889	}
 890
 891	return xfrmi_update(xi, &p);
 892}
 893
 894static size_t xfrmi_get_size(const struct net_device *dev)
 895{
 896	return
 897		/* IFLA_XFRM_LINK */
 898		nla_total_size(4) +
 899		/* IFLA_XFRM_IF_ID */
 900		nla_total_size(4) +
 901		/* IFLA_XFRM_COLLECT_METADATA */
 902		nla_total_size(0) +
 903		0;
 904}
 905
 906static int xfrmi_fill_info(struct sk_buff *skb, const struct net_device *dev)
 907{
 908	struct xfrm_if *xi = netdev_priv(dev);
 909	struct xfrm_if_parms *parm = &xi->p;
 910
 911	if (nla_put_u32(skb, IFLA_XFRM_LINK, parm->link) ||
 912	    nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id) ||
 913	    (xi->p.collect_md && nla_put_flag(skb, IFLA_XFRM_COLLECT_METADATA)))
 914		goto nla_put_failure;
 915	return 0;
 916
 917nla_put_failure:
 918	return -EMSGSIZE;
 919}
 920
 921static struct net *xfrmi_get_link_net(const struct net_device *dev)
 922{
 923	struct xfrm_if *xi = netdev_priv(dev);
 924
 925	return READ_ONCE(xi->net);
 926}
 927
 928static const struct nla_policy xfrmi_policy[IFLA_XFRM_MAX + 1] = {
 929	[IFLA_XFRM_UNSPEC]		= { .strict_start_type = IFLA_XFRM_COLLECT_METADATA },
 930	[IFLA_XFRM_LINK]		= { .type = NLA_U32 },
 931	[IFLA_XFRM_IF_ID]		= { .type = NLA_U32 },
 932	[IFLA_XFRM_COLLECT_METADATA]	= { .type = NLA_FLAG },
 933};
 934
 935static struct rtnl_link_ops xfrmi_link_ops __read_mostly = {
 936	.kind		= "xfrm",
 937	.maxtype	= IFLA_XFRM_MAX,
 938	.policy		= xfrmi_policy,
 939	.priv_size	= sizeof(struct xfrm_if),
 940	.setup		= xfrmi_dev_setup,
 941	.validate	= xfrmi_validate,
 942	.newlink	= xfrmi_newlink,
 943	.dellink	= xfrmi_dellink,
 944	.changelink	= xfrmi_changelink,
 945	.get_size	= xfrmi_get_size,
 946	.fill_info	= xfrmi_fill_info,
 947	.get_link_net	= xfrmi_get_link_net,
 948};
 949
 950static void __net_exit xfrmi_exit_rtnl(struct net *net,
 951				       struct list_head *dev_to_kill)
 952{
 953	struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 954	struct xfrm_if __rcu **xip;
 955	struct xfrm_if *xi;
 956	int i;
 957
 958	for (i = 0; i < XFRMI_HASH_SIZE; i++) {
 959		for (xip = &xfrmn->xfrmi[i];
 960		     (xi = rtnl_net_dereference(net, *xip)) != NULL;
 961		     xip = &xi->next)
 962			unregister_netdevice_queue(xi->dev, dev_to_kill);
 963	}
 964
 965	xi = rtnl_net_dereference(net, xfrmn->collect_md_xfrmi);
 966	if (xi)
 967		unregister_netdevice_queue(xi->dev, dev_to_kill);
 968}
 969
 970static struct pernet_operations xfrmi_net_ops = {
 971	.exit_rtnl = xfrmi_exit_rtnl,
 972	.id   = &xfrmi_net_id,
 973	.size = sizeof(struct xfrmi_net),
 974};
 975
 976static struct xfrm6_protocol xfrmi_esp6_protocol __read_mostly = {
 977	.handler	=	xfrmi6_rcv,
 978	.input_handler	=	xfrmi6_input,
 979	.cb_handler	=	xfrmi_rcv_cb,
 980	.err_handler	=	xfrmi6_err,
 981	.priority	=	10,
 982};
 983
 984static struct xfrm6_protocol xfrmi_ah6_protocol __read_mostly = {
 985	.handler	=	xfrm6_rcv,
 986	.input_handler	=	xfrm_input,
 987	.cb_handler	=	xfrmi_rcv_cb,
 988	.err_handler	=	xfrmi6_err,
 989	.priority	=	10,
 990};
 991
 992static struct xfrm6_protocol xfrmi_ipcomp6_protocol __read_mostly = {
 993	.handler	=	xfrm6_rcv,
 994	.input_handler	=	xfrm_input,
 995	.cb_handler	=	xfrmi_rcv_cb,
 996	.err_handler	=	xfrmi6_err,
 997	.priority	=	10,
 998};
 999
1000#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
1001static int xfrmi6_rcv_tunnel(struct sk_buff *skb)
1002{
1003	const xfrm_address_t *saddr;
1004	__be32 spi;
1005
1006	saddr = (const xfrm_address_t *)&ipv6_hdr(skb)->saddr;
1007	spi = xfrm6_tunnel_spi_lookup(dev_net(skb->dev), saddr);
1008
1009	return xfrm6_rcv_spi(skb, IPPROTO_IPV6, spi, NULL);
1010}
1011
1012static struct xfrm6_tunnel xfrmi_ipv6_handler __read_mostly = {
1013	.handler	=	xfrmi6_rcv_tunnel,
1014	.cb_handler	=	xfrmi_rcv_cb,
1015	.err_handler	=	xfrmi6_err,
1016	.priority	=	2,
1017};
1018
1019static struct xfrm6_tunnel xfrmi_ip6ip_handler __read_mostly = {
1020	.handler	=	xfrmi6_rcv_tunnel,
1021	.cb_handler	=	xfrmi_rcv_cb,
1022	.err_handler	=	xfrmi6_err,
1023	.priority	=	2,
1024};
1025#endif
1026
1027static struct xfrm4_protocol xfrmi_esp4_protocol __read_mostly = {
1028	.handler	=	xfrmi4_rcv,
1029	.input_handler	=	xfrmi4_input,
1030	.cb_handler	=	xfrmi_rcv_cb,
1031	.err_handler	=	xfrmi4_err,
1032	.priority	=	10,
1033};
1034
1035static struct xfrm4_protocol xfrmi_ah4_protocol __read_mostly = {
1036	.handler	=	xfrm4_rcv,
1037	.input_handler	=	xfrm_input,
1038	.cb_handler	=	xfrmi_rcv_cb,
1039	.err_handler	=	xfrmi4_err,
1040	.priority	=	10,
1041};
1042
1043static struct xfrm4_protocol xfrmi_ipcomp4_protocol __read_mostly = {
1044	.handler	=	xfrm4_rcv,
1045	.input_handler	=	xfrm_input,
1046	.cb_handler	=	xfrmi_rcv_cb,
1047	.err_handler	=	xfrmi4_err,
1048	.priority	=	10,
1049};
1050
1051#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
1052static int xfrmi4_rcv_tunnel(struct sk_buff *skb)
1053{
1054	return xfrm4_rcv_spi(skb, IPPROTO_IPIP, ip_hdr(skb)->saddr);
1055}
1056
1057static struct xfrm_tunnel xfrmi_ipip_handler __read_mostly = {
1058	.handler	=	xfrmi4_rcv_tunnel,
1059	.cb_handler	=	xfrmi_rcv_cb,
1060	.err_handler	=	xfrmi4_err,
1061	.priority	=	3,
1062};
1063
1064static struct xfrm_tunnel xfrmi_ipip6_handler __read_mostly = {
1065	.handler	=	xfrmi4_rcv_tunnel,
1066	.cb_handler	=	xfrmi_rcv_cb,
1067	.err_handler	=	xfrmi4_err,
1068	.priority	=	2,
1069};
1070#endif
1071
1072static int __init xfrmi4_init(void)
1073{
1074	int err;
1075
1076	err = xfrm4_protocol_register(&xfrmi_esp4_protocol, IPPROTO_ESP);
1077	if (err < 0)
1078		goto xfrm_proto_esp_failed;
1079	err = xfrm4_protocol_register(&xfrmi_ah4_protocol, IPPROTO_AH);
1080	if (err < 0)
1081		goto xfrm_proto_ah_failed;
1082	err = xfrm4_protocol_register(&xfrmi_ipcomp4_protocol, IPPROTO_COMP);
1083	if (err < 0)
1084		goto xfrm_proto_comp_failed;
1085#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
1086	err = xfrm4_tunnel_register(&xfrmi_ipip_handler, AF_INET);
1087	if (err < 0)
1088		goto xfrm_tunnel_ipip_failed;
1089	err = xfrm4_tunnel_register(&xfrmi_ipip6_handler, AF_INET6);
1090	if (err < 0)
1091		goto xfrm_tunnel_ipip6_failed;
1092#endif
1093
1094	return 0;
1095
1096#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
1097xfrm_tunnel_ipip6_failed:
1098	xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET);
1099xfrm_tunnel_ipip_failed:
1100	xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP);
1101#endif
1102xfrm_proto_comp_failed:
1103	xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH);
1104xfrm_proto_ah_failed:
1105	xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP);
1106xfrm_proto_esp_failed:
1107	return err;
1108}
1109
1110static void xfrmi4_fini(void)
1111{
1112#if IS_REACHABLE(CONFIG_INET_XFRM_TUNNEL)
1113	xfrm4_tunnel_deregister(&xfrmi_ipip6_handler, AF_INET6);
1114	xfrm4_tunnel_deregister(&xfrmi_ipip_handler, AF_INET);
1115#endif
1116	xfrm4_protocol_deregister(&xfrmi_ipcomp4_protocol, IPPROTO_COMP);
1117	xfrm4_protocol_deregister(&xfrmi_ah4_protocol, IPPROTO_AH);
1118	xfrm4_protocol_deregister(&xfrmi_esp4_protocol, IPPROTO_ESP);
1119}
1120
1121static int __init xfrmi6_init(void)
1122{
1123	int err;
1124
1125	err = xfrm6_protocol_register(&xfrmi_esp6_protocol, IPPROTO_ESP);
1126	if (err < 0)
1127		goto xfrm_proto_esp_failed;
1128	err = xfrm6_protocol_register(&xfrmi_ah6_protocol, IPPROTO_AH);
1129	if (err < 0)
1130		goto xfrm_proto_ah_failed;
1131	err = xfrm6_protocol_register(&xfrmi_ipcomp6_protocol, IPPROTO_COMP);
1132	if (err < 0)
1133		goto xfrm_proto_comp_failed;
1134#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
1135	err = xfrm6_tunnel_register(&xfrmi_ipv6_handler, AF_INET6);
1136	if (err < 0)
1137		goto xfrm_tunnel_ipv6_failed;
1138	err = xfrm6_tunnel_register(&xfrmi_ip6ip_handler, AF_INET);
1139	if (err < 0)
1140		goto xfrm_tunnel_ip6ip_failed;
1141#endif
1142
1143	return 0;
1144
1145#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
1146xfrm_tunnel_ip6ip_failed:
1147	xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6);
1148xfrm_tunnel_ipv6_failed:
1149	xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP);
1150#endif
1151xfrm_proto_comp_failed:
1152	xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH);
1153xfrm_proto_ah_failed:
1154	xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP);
1155xfrm_proto_esp_failed:
1156	return err;
1157}
1158
1159static void xfrmi6_fini(void)
1160{
1161#if IS_REACHABLE(CONFIG_INET6_XFRM_TUNNEL)
1162	xfrm6_tunnel_deregister(&xfrmi_ip6ip_handler, AF_INET);
1163	xfrm6_tunnel_deregister(&xfrmi_ipv6_handler, AF_INET6);
1164#endif
1165	xfrm6_protocol_deregister(&xfrmi_ipcomp6_protocol, IPPROTO_COMP);
1166	xfrm6_protocol_deregister(&xfrmi_ah6_protocol, IPPROTO_AH);
1167	xfrm6_protocol_deregister(&xfrmi_esp6_protocol, IPPROTO_ESP);
1168}
1169
1170static const struct xfrm_if_cb xfrm_if_cb = {
1171	.decode_session =	xfrmi_decode_session,
1172};
1173
1174static int __init xfrmi_init(void)
1175{
1176	const char *msg;
1177	int err;
1178
1179	pr_info("IPsec XFRM device driver\n");
1180
1181	msg = "tunnel device";
1182	err = register_pernet_device(&xfrmi_net_ops);
1183	if (err < 0)
1184		goto pernet_dev_failed;
1185
1186	msg = "xfrm4 protocols";
1187	err = xfrmi4_init();
1188	if (err < 0)
1189		goto xfrmi4_failed;
1190
1191	msg = "xfrm6 protocols";
1192	err = xfrmi6_init();
1193	if (err < 0)
1194		goto xfrmi6_failed;
1195
1196
1197	msg = "netlink interface";
1198	err = rtnl_link_register(&xfrmi_link_ops);
1199	if (err < 0)
1200		goto rtnl_link_failed;
1201
1202	err = register_xfrm_interface_bpf();
1203	if (err < 0)
1204		goto kfunc_failed;
1205
1206	lwtunnel_encap_add_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM);
1207
1208	xfrm_if_register_cb(&xfrm_if_cb);
1209
1210	return err;
1211
1212kfunc_failed:
1213	rtnl_link_unregister(&xfrmi_link_ops);
1214rtnl_link_failed:
1215	xfrmi6_fini();
1216xfrmi6_failed:
1217	xfrmi4_fini();
1218xfrmi4_failed:
1219	unregister_pernet_device(&xfrmi_net_ops);
1220pernet_dev_failed:
1221	pr_err("xfrmi init: failed to register %s\n", msg);
1222	return err;
1223}
1224
1225static void __exit xfrmi_fini(void)
1226{
1227	xfrm_if_unregister_cb();
1228	lwtunnel_encap_del_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM);
1229	rtnl_link_unregister(&xfrmi_link_ops);
1230	xfrmi4_fini();
1231	xfrmi6_fini();
1232	unregister_pernet_device(&xfrmi_net_ops);
1233}
1234
1235module_init(xfrmi_init);
1236module_exit(xfrmi_fini);
1237MODULE_LICENSE("GPL");
1238MODULE_ALIAS_RTNL_LINK("xfrm");
1239MODULE_ALIAS_NETDEV("xfrm0");
1240MODULE_AUTHOR("Steffen Klassert");
1241MODULE_DESCRIPTION("XFRM virtual interface");