vrf.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * vrf.c: device driver to encapsulate a VRF space
   4 *
   5 * Copyright (c) 2015 Cumulus Networks. All rights reserved.
   6 * Copyright (c) 2015 Shrijeet Mukherjee <shm@cumulusnetworks.com>
   7 * Copyright (c) 2015 David Ahern <dsa@cumulusnetworks.com>
   8 *
   9 * Based on dummy, team and ipvlan drivers
  10 */
  11
  12#include <linux/ethtool.h>
  13#include <linux/module.h>
  14#include <linux/kernel.h>
  15#include <linux/netdevice.h>
  16#include <linux/etherdevice.h>
  17#include <linux/ip.h>
  18#include <linux/init.h>
  19#include <linux/moduleparam.h>
  20#include <linux/netfilter.h>
  21#include <linux/rtnetlink.h>
  22#include <net/rtnetlink.h>
  23#include <linux/u64_stats_sync.h>
  24#include <linux/hashtable.h>
  25#include <linux/spinlock_types.h>
  26
  27#include <linux/inetdevice.h>
  28#include <net/arp.h>
  29#include <net/flow.h>
  30#include <net/ip.h>
  31#include <net/ip_fib.h>
  32#include <net/ip6_fib.h>
  33#include <net/ip6_route.h>
  34#include <net/route.h>
  35#include <net/addrconf.h>
  36#include <net/l3mdev.h>
  37#include <net/fib_rules.h>
  38#include <net/netdev_lock.h>
  39#include <net/sch_generic.h>
  40#include <net/netns/generic.h>
  41#include <net/netfilter/nf_conntrack.h>
  42
  43#define DRV_NAME	"vrf"
  44#define DRV_VERSION	"1.1"
  45
  46#define FIB_RULE_PREF  1000       /* default preference for FIB rules */
  47
  48#define HT_MAP_BITS	4
  49#define HASH_INITVAL	((u32)0xcafef00d)
  50
  51struct  vrf_map {
  52	DECLARE_HASHTABLE(ht, HT_MAP_BITS);
  53	spinlock_t vmap_lock;
  54
  55	/* shared_tables:
  56	 * count how many distinct tables do not comply with the strict mode
  57	 * requirement.
  58	 * shared_tables value must be 0 in order to enable the strict mode.
  59	 *
  60	 * example of the evolution of shared_tables:
  61	 *                                                        | time
  62	 * add  vrf0 --> table 100        shared_tables = 0       | t0
  63	 * add  vrf1 --> table 101        shared_tables = 0       | t1
  64	 * add  vrf2 --> table 100        shared_tables = 1       | t2
  65	 * add  vrf3 --> table 100        shared_tables = 1       | t3
  66	 * add  vrf4 --> table 101        shared_tables = 2       v t4
  67	 *
  68	 * shared_tables is a "step function" (or "staircase function")
  69	 * and it is increased by one when the second vrf is associated to a
  70	 * table.
  71	 *
  72	 * at t2, vrf0 and vrf2 are bound to table 100: shared_tables = 1.
  73	 *
  74	 * at t3, another dev (vrf3) is bound to the same table 100 but the
  75	 * value of shared_tables is still 1.
  76	 * This means that no matter how many new vrfs will register on the
  77	 * table 100, the shared_tables will not increase (considering only
  78	 * table 100).
  79	 *
  80	 * at t4, vrf4 is bound to table 101, and shared_tables = 2.
  81	 *
  82	 * Looking at the value of shared_tables we can immediately know if
  83	 * the strict_mode can or cannot be enforced. Indeed, strict_mode
  84	 * can be enforced iff shared_tables = 0.
  85	 *
  86	 * Conversely, shared_tables is decreased when a vrf is de-associated
  87	 * from a table with exactly two associated vrfs.
  88	 */
  89	u32 shared_tables;
  90
  91	bool strict_mode;
  92};
  93
  94struct vrf_map_elem {
  95	struct hlist_node hnode;
  96	struct list_head vrf_list;  /* VRFs registered to this table */
  97
  98	u32 table_id;
  99	int users;
 100	int ifindex;
 101};
 102
 103static unsigned int vrf_net_id;
 104
 105/* per netns vrf data */
 106struct netns_vrf {
 107	/* protected by rtnl lock */
 108	bool add_fib_rules;
 109
 110	struct vrf_map vmap;
 111	struct ctl_table_header	*ctl_hdr;
 112};
 113
 114struct net_vrf {
 115	struct rtable __rcu	*rth;
 116	struct rt6_info	__rcu	*rt6;
 117#if IS_ENABLED(CONFIG_IPV6)
 118	struct fib6_table	*fib6_table;
 119#endif
 120	u32                     tb_id;
 121
 122	struct list_head	me_list;   /* entry in vrf_map_elem */
 123	int			ifindex;
 124};
 125
 126static void vrf_tx_error(struct net_device *vrf_dev, struct sk_buff *skb)
 127{
 128	vrf_dev->stats.tx_errors++;
 129	kfree_skb(skb);
 130}
 131
 132static struct vrf_map *netns_vrf_map(struct net *net)
 133{
 134	struct netns_vrf *nn_vrf = net_generic(net, vrf_net_id);
 135
 136	return &nn_vrf->vmap;
 137}
 138
 139static struct vrf_map *netns_vrf_map_by_dev(struct net_device *dev)
 140{
 141	return netns_vrf_map(dev_net(dev));
 142}
 143
 144static int vrf_map_elem_get_vrf_ifindex(struct vrf_map_elem *me)
 145{
 146	struct list_head *me_head = &me->vrf_list;
 147	struct net_vrf *vrf;
 148
 149	if (list_empty(me_head))
 150		return -ENODEV;
 151
 152	vrf = list_first_entry(me_head, struct net_vrf, me_list);
 153
 154	return vrf->ifindex;
 155}
 156
 157static struct vrf_map_elem *vrf_map_elem_alloc(gfp_t flags)
 158{
 159	struct vrf_map_elem *me;
 160
 161	me = kmalloc(sizeof(*me), flags);
 162	if (!me)
 163		return NULL;
 164
 165	return me;
 166}
 167
 168static void vrf_map_elem_free(struct vrf_map_elem *me)
 169{
 170	kfree(me);
 171}
 172
 173static void vrf_map_elem_init(struct vrf_map_elem *me, int table_id,
 174			      int ifindex, int users)
 175{
 176	me->table_id = table_id;
 177	me->ifindex = ifindex;
 178	me->users = users;
 179	INIT_LIST_HEAD(&me->vrf_list);
 180}
 181
 182static struct vrf_map_elem *vrf_map_lookup_elem(struct vrf_map *vmap,
 183						u32 table_id)
 184{
 185	struct vrf_map_elem *me;
 186	u32 key;
 187
 188	key = jhash_1word(table_id, HASH_INITVAL);
 189	hash_for_each_possible(vmap->ht, me, hnode, key) {
 190		if (me->table_id == table_id)
 191			return me;
 192	}
 193
 194	return NULL;
 195}
 196
 197static void vrf_map_add_elem(struct vrf_map *vmap, struct vrf_map_elem *me)
 198{
 199	u32 table_id = me->table_id;
 200	u32 key;
 201
 202	key = jhash_1word(table_id, HASH_INITVAL);
 203	hash_add(vmap->ht, &me->hnode, key);
 204}
 205
 206static void vrf_map_del_elem(struct vrf_map_elem *me)
 207{
 208	hash_del(&me->hnode);
 209}
 210
 211static void vrf_map_lock(struct vrf_map *vmap) __acquires(&vmap->vmap_lock)
 212{
 213	spin_lock(&vmap->vmap_lock);
 214}
 215
 216static void vrf_map_unlock(struct vrf_map *vmap) __releases(&vmap->vmap_lock)
 217{
 218	spin_unlock(&vmap->vmap_lock);
 219}
 220
 221/* called with rtnl lock held */
 222static int
 223vrf_map_register_dev(struct net_device *dev, struct netlink_ext_ack *extack)
 224{
 225	struct vrf_map *vmap = netns_vrf_map_by_dev(dev);
 226	struct net_vrf *vrf = netdev_priv(dev);
 227	struct vrf_map_elem *new_me, *me;
 228	u32 table_id = vrf->tb_id;
 229	bool free_new_me = false;
 230	int users;
 231	int res;
 232
 233	/* we pre-allocate elements used in the spin-locked section (so that we
 234	 * keep the spinlock as short as possible).
 235	 */
 236	new_me = vrf_map_elem_alloc(GFP_KERNEL);
 237	if (!new_me)
 238		return -ENOMEM;
 239
 240	vrf_map_elem_init(new_me, table_id, dev->ifindex, 0);
 241
 242	vrf_map_lock(vmap);
 243
 244	me = vrf_map_lookup_elem(vmap, table_id);
 245	if (!me) {
 246		me = new_me;
 247		vrf_map_add_elem(vmap, me);
 248		goto link_vrf;
 249	}
 250
 251	/* we already have an entry in the vrf_map, so it means there is (at
 252	 * least) a vrf registered on the specific table.
 253	 */
 254	free_new_me = true;
 255	if (vmap->strict_mode) {
 256		/* vrfs cannot share the same table */
 257		NL_SET_ERR_MSG(extack, "Table is used by another VRF");
 258		res = -EBUSY;
 259		goto unlock;
 260	}
 261
 262link_vrf:
 263	users = ++me->users;
 264	if (users == 2)
 265		++vmap->shared_tables;
 266
 267	list_add(&vrf->me_list, &me->vrf_list);
 268
 269	res = 0;
 270
 271unlock:
 272	vrf_map_unlock(vmap);
 273
 274	/* clean-up, if needed */
 275	if (free_new_me)
 276		vrf_map_elem_free(new_me);
 277
 278	return res;
 279}
 280
 281/* called with rtnl lock held */
 282static void vrf_map_unregister_dev(struct net_device *dev)
 283{
 284	struct vrf_map *vmap = netns_vrf_map_by_dev(dev);
 285	struct net_vrf *vrf = netdev_priv(dev);
 286	u32 table_id = vrf->tb_id;
 287	struct vrf_map_elem *me;
 288	int users;
 289
 290	vrf_map_lock(vmap);
 291
 292	me = vrf_map_lookup_elem(vmap, table_id);
 293	if (!me)
 294		goto unlock;
 295
 296	list_del(&vrf->me_list);
 297
 298	users = --me->users;
 299	if (users == 1) {
 300		--vmap->shared_tables;
 301	} else if (users == 0) {
 302		vrf_map_del_elem(me);
 303
 304		/* no one will refer to this element anymore */
 305		vrf_map_elem_free(me);
 306	}
 307
 308unlock:
 309	vrf_map_unlock(vmap);
 310}
 311
 312/* return the vrf device index associated with the table_id */
 313static int vrf_ifindex_lookup_by_table_id(struct net *net, u32 table_id)
 314{
 315	struct vrf_map *vmap = netns_vrf_map(net);
 316	struct vrf_map_elem *me;
 317	int ifindex;
 318
 319	vrf_map_lock(vmap);
 320
 321	if (!vmap->strict_mode) {
 322		ifindex = -EPERM;
 323		goto unlock;
 324	}
 325
 326	me = vrf_map_lookup_elem(vmap, table_id);
 327	if (!me) {
 328		ifindex = -ENODEV;
 329		goto unlock;
 330	}
 331
 332	ifindex = vrf_map_elem_get_vrf_ifindex(me);
 333
 334unlock:
 335	vrf_map_unlock(vmap);
 336
 337	return ifindex;
 338}
 339
 340/* by default VRF devices do not have a qdisc and are expected
 341 * to be created with only a single queue.
 342 */
 343static bool qdisc_tx_is_default(const struct net_device *dev)
 344{
 345	struct netdev_queue *txq;
 346
 347	if (dev->num_tx_queues > 1)
 348		return false;
 349
 350	txq = netdev_get_tx_queue(dev, 0);
 351
 352	return qdisc_txq_has_no_queue(txq);
 353}
 354
 355/* Local traffic destined to local address. Reinsert the packet to rx
 356 * path, similar to loopback handling.
 357 */
 358static int vrf_local_xmit(struct sk_buff *skb, struct net_device *dev,
 359			  struct dst_entry *dst)
 360{
 361	unsigned int len = skb->len;
 362
 363	skb_orphan(skb);
 364
 365	skb_dst_set(skb, dst);
 366
 367	/* set pkt_type to avoid skb hitting packet taps twice -
 368	 * once on Tx and again in Rx processing
 369	 */
 370	skb->pkt_type = PACKET_LOOPBACK;
 371
 372	skb->protocol = eth_type_trans(skb, dev);
 373
 374	if (likely(__netif_rx(skb) == NET_RX_SUCCESS))
 375		dev_dstats_rx_add(dev, len);
 376	else
 377		dev_dstats_rx_dropped(dev);
 378
 379	return NETDEV_TX_OK;
 380}
 381
 382static void vrf_nf_set_untracked(struct sk_buff *skb)
 383{
 384	if (skb_get_nfct(skb) == 0)
 385		nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
 386}
 387
 388static void vrf_nf_reset_ct(struct sk_buff *skb)
 389{
 390	if (skb_get_nfct(skb) == IP_CT_UNTRACKED)
 391		nf_reset_ct(skb);
 392}
 393
 394#if IS_ENABLED(CONFIG_IPV6)
 395static int vrf_ip6_local_out(struct net *net, struct sock *sk,
 396			     struct sk_buff *skb)
 397{
 398	int err;
 399
 400	vrf_nf_reset_ct(skb);
 401
 402	err = nf_hook(NFPROTO_IPV6, NF_INET_LOCAL_OUT, net,
 403		      sk, skb, NULL, skb_dst(skb)->dev, dst_output);
 404
 405	if (likely(err == 1))
 406		err = dst_output(net, sk, skb);
 407
 408	return err;
 409}
 410
 411static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 412					   struct net_device *dev)
 413{
 414	const struct ipv6hdr *iph;
 415	struct net *net = dev_net(skb->dev);
 416	struct flowi6 fl6;
 417	int ret = NET_XMIT_DROP;
 418	struct dst_entry *dst;
 419	struct dst_entry *dst_null = &net->ipv6.ip6_null_entry->dst;
 420
 421	if (!pskb_may_pull(skb, ETH_HLEN + sizeof(struct ipv6hdr)))
 422		goto err;
 423
 424	iph = ipv6_hdr(skb);
 425
 426	memset(&fl6, 0, sizeof(fl6));
 427	/* needed to match OIF rule */
 428	fl6.flowi6_l3mdev = dev->ifindex;
 429	fl6.flowi6_iif = LOOPBACK_IFINDEX;
 430	fl6.daddr = iph->daddr;
 431	fl6.saddr = iph->saddr;
 432	fl6.flowlabel = ip6_flowinfo(iph);
 433	fl6.flowi6_mark = skb->mark;
 434	fl6.flowi6_proto = iph->nexthdr;
 435
 436	dst = ip6_dst_lookup_flow(net, NULL, &fl6, NULL);
 437	if (IS_ERR(dst) || dst == dst_null)
 438		goto err;
 439
 440	skb_dst_drop(skb);
 441
 442	/* if dst.dev is the VRF device again this is locally originated traffic
 443	 * destined to a local address. Short circuit to Rx path.
 444	 */
 445	if (dst->dev == dev)
 446		return vrf_local_xmit(skb, dev, dst);
 447
 448	skb_dst_set(skb, dst);
 449
 450	/* strip the ethernet header added for pass through VRF device */
 451	__skb_pull(skb, skb_network_offset(skb));
 452
 453	memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
 454	ret = vrf_ip6_local_out(net, skb->sk, skb);
 455	if (unlikely(net_xmit_eval(ret)))
 456		dev->stats.tx_errors++;
 457	else
 458		ret = NET_XMIT_SUCCESS;
 459
 460	return ret;
 461err:
 462	vrf_tx_error(dev, skb);
 463	return NET_XMIT_DROP;
 464}
 465#else
 466static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 467					   struct net_device *dev)
 468{
 469	vrf_tx_error(dev, skb);
 470	return NET_XMIT_DROP;
 471}
 472#endif
 473
 474/* based on ip_local_out; can't use it b/c the dst is switched pointing to us */
 475static int vrf_ip_local_out(struct net *net, struct sock *sk,
 476			    struct sk_buff *skb)
 477{
 478	int err;
 479
 480	vrf_nf_reset_ct(skb);
 481
 482	err = nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, net, sk,
 483		      skb, NULL, skb_dst(skb)->dev, dst_output);
 484	if (likely(err == 1))
 485		err = dst_output(net, sk, skb);
 486
 487	return err;
 488}
 489
 490static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
 491					   struct net_device *vrf_dev)
 492{
 493	struct iphdr *ip4h;
 494	int ret = NET_XMIT_DROP;
 495	struct flowi4 fl4;
 496	struct net *net = dev_net(vrf_dev);
 497	struct rtable *rt;
 498
 499	if (!pskb_may_pull(skb, ETH_HLEN + sizeof(struct iphdr)))
 500		goto err;
 501
 502	ip4h = ip_hdr(skb);
 503
 504	memset(&fl4, 0, sizeof(fl4));
 505	/* needed to match OIF rule */
 506	fl4.flowi4_l3mdev = vrf_dev->ifindex;
 507	fl4.flowi4_iif = LOOPBACK_IFINDEX;
 508	fl4.flowi4_dscp = ip4h_dscp(ip4h);
 509	fl4.flowi4_flags = FLOWI_FLAG_ANYSRC;
 510	fl4.flowi4_proto = ip4h->protocol;
 511	fl4.daddr = ip4h->daddr;
 512	fl4.saddr = ip4h->saddr;
 513
 514	rt = ip_route_output_flow(net, &fl4, NULL);
 515	if (IS_ERR(rt))
 516		goto err;
 517
 518	skb_dst_drop(skb);
 519
 520	/* if dst.dev is the VRF device again this is locally originated traffic
 521	 * destined to a local address. Short circuit to Rx path.
 522	 */
 523	if (rt->dst.dev == vrf_dev)
 524		return vrf_local_xmit(skb, vrf_dev, &rt->dst);
 525
 526	skb_dst_set(skb, &rt->dst);
 527
 528	/* strip the ethernet header added for pass through VRF device */
 529	__skb_pull(skb, skb_network_offset(skb));
 530
 531	if (!ip4h->saddr) {
 532		ip4h->saddr = inet_select_addr(skb_dst(skb)->dev, 0,
 533					       RT_SCOPE_LINK);
 534	}
 535
 536	memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
 537	ret = vrf_ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
 538	if (unlikely(net_xmit_eval(ret)))
 539		vrf_dev->stats.tx_errors++;
 540	else
 541		ret = NET_XMIT_SUCCESS;
 542
 543out:
 544	return ret;
 545err:
 546	vrf_tx_error(vrf_dev, skb);
 547	goto out;
 548}
 549
 550static netdev_tx_t is_ip_tx_frame(struct sk_buff *skb, struct net_device *dev)
 551{
 552	switch (skb->protocol) {
 553	case htons(ETH_P_IP):
 554		return vrf_process_v4_outbound(skb, dev);
 555	case htons(ETH_P_IPV6):
 556		return vrf_process_v6_outbound(skb, dev);
 557	default:
 558		vrf_tx_error(dev, skb);
 559		return NET_XMIT_DROP;
 560	}
 561}
 562
 563static netdev_tx_t vrf_xmit(struct sk_buff *skb, struct net_device *dev)
 564{
 565	unsigned int len = skb->len;
 566	netdev_tx_t ret;
 567
 568	ret = is_ip_tx_frame(skb, dev);
 569	if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN))
 570		dev_dstats_tx_add(dev, len);
 571	else
 572		dev_dstats_tx_dropped(dev);
 573
 574	return ret;
 575}
 576
 577static void vrf_finish_direct(struct sk_buff *skb)
 578{
 579	struct net_device *vrf_dev = skb->dev;
 580
 581	if (!list_empty(&vrf_dev->ptype_all) &&
 582	    likely(skb_headroom(skb) >= ETH_HLEN)) {
 583		struct ethhdr *eth = skb_push(skb, ETH_HLEN);
 584
 585		ether_addr_copy(eth->h_source, vrf_dev->dev_addr);
 586		eth_zero_addr(eth->h_dest);
 587		eth->h_proto = skb->protocol;
 588
 589		rcu_read_lock_bh();
 590		dev_queue_xmit_nit(skb, vrf_dev);
 591		rcu_read_unlock_bh();
 592
 593		skb_pull(skb, ETH_HLEN);
 594	}
 595
 596	vrf_nf_reset_ct(skb);
 597}
 598
 599#if IS_ENABLED(CONFIG_IPV6)
 600/* modelled after ip6_finish_output2 */
 601static int vrf_finish_output6(struct net *net, struct sock *sk,
 602			      struct sk_buff *skb)
 603{
 604	struct dst_entry *dst = skb_dst(skb);
 605	struct net_device *dev = dst->dev;
 606	const struct in6_addr *nexthop;
 607	struct neighbour *neigh;
 608	int ret;
 609
 610	vrf_nf_reset_ct(skb);
 611
 612	skb->protocol = htons(ETH_P_IPV6);
 613	skb->dev = dev;
 614
 615	rcu_read_lock();
 616	nexthop = rt6_nexthop(dst_rt6_info(dst), &ipv6_hdr(skb)->daddr);
 617	neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
 618	if (unlikely(!neigh))
 619		neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
 620	if (!IS_ERR(neigh)) {
 621		sock_confirm_neigh(skb, neigh);
 622		ret = neigh_output(neigh, skb, false);
 623		rcu_read_unlock();
 624		return ret;
 625	}
 626	rcu_read_unlock();
 627
 628	IP6_INC_STATS(dev_net(dst->dev),
 629		      ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
 630	kfree_skb(skb);
 631	return -EINVAL;
 632}
 633
 634/* modelled after ip6_output */
 635static int vrf_output6(struct net *net, struct sock *sk, struct sk_buff *skb)
 636{
 637	return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
 638			    net, sk, skb, NULL, skb_dst(skb)->dev,
 639			    vrf_finish_output6,
 640			    !(IP6CB(skb)->flags & IP6SKB_REROUTED));
 641}
 642
 643/* set dst on skb to send packet to us via dev_xmit path. Allows
 644 * packet to go through device based features such as qdisc, netfilter
 645 * hooks and packet sockets with skb->dev set to vrf device.
 646 */
 647static struct sk_buff *vrf_ip6_out_redirect(struct net_device *vrf_dev,
 648					    struct sk_buff *skb)
 649{
 650	struct net_vrf *vrf = netdev_priv(vrf_dev);
 651	struct dst_entry *dst = NULL;
 652	struct rt6_info *rt6;
 653
 654	rcu_read_lock();
 655
 656	rt6 = rcu_dereference(vrf->rt6);
 657	if (likely(rt6)) {
 658		dst = &rt6->dst;
 659		dst_hold(dst);
 660	}
 661
 662	rcu_read_unlock();
 663
 664	if (unlikely(!dst)) {
 665		vrf_tx_error(vrf_dev, skb);
 666		return NULL;
 667	}
 668
 669	skb_dst_drop(skb);
 670	skb_dst_set(skb, dst);
 671
 672	return skb;
 673}
 674
 675static int vrf_output6_direct_finish(struct net *net, struct sock *sk,
 676				     struct sk_buff *skb)
 677{
 678	vrf_finish_direct(skb);
 679
 680	return vrf_ip6_local_out(net, sk, skb);
 681}
 682
 683static int vrf_output6_direct(struct net *net, struct sock *sk,
 684			      struct sk_buff *skb)
 685{
 686	int err = 1;
 687
 688	skb->protocol = htons(ETH_P_IPV6);
 689
 690	if (!(IPCB(skb)->flags & IPSKB_REROUTED))
 691		err = nf_hook(NFPROTO_IPV6, NF_INET_POST_ROUTING, net, sk, skb,
 692			      NULL, skb->dev, vrf_output6_direct_finish);
 693
 694	if (likely(err == 1))
 695		vrf_finish_direct(skb);
 696
 697	return err;
 698}
 699
 700static int vrf_ip6_out_direct_finish(struct net *net, struct sock *sk,
 701				     struct sk_buff *skb)
 702{
 703	int err;
 704
 705	err = vrf_output6_direct(net, sk, skb);
 706	if (likely(err == 1))
 707		err = vrf_ip6_local_out(net, sk, skb);
 708
 709	return err;
 710}
 711
 712static struct sk_buff *vrf_ip6_out_direct(struct net_device *vrf_dev,
 713					  struct sock *sk,
 714					  struct sk_buff *skb)
 715{
 716	struct net *net = dev_net(vrf_dev);
 717	int err;
 718
 719	skb->dev = vrf_dev;
 720
 721	err = nf_hook(NFPROTO_IPV6, NF_INET_LOCAL_OUT, net, sk,
 722		      skb, NULL, vrf_dev, vrf_ip6_out_direct_finish);
 723
 724	if (likely(err == 1))
 725		err = vrf_output6_direct(net, sk, skb);
 726
 727	if (likely(err == 1))
 728		return skb;
 729
 730	return NULL;
 731}
 732
 733static struct sk_buff *vrf_ip6_out(struct net_device *vrf_dev,
 734				   struct sock *sk,
 735				   struct sk_buff *skb)
 736{
 737	/* don't divert link scope packets */
 738	if (rt6_need_strict(&ipv6_hdr(skb)->daddr))
 739		return skb;
 740
 741	vrf_nf_set_untracked(skb);
 742
 743	if (qdisc_tx_is_default(vrf_dev) ||
 744	    IP6CB(skb)->flags & IP6SKB_XFRM_TRANSFORMED)
 745		return vrf_ip6_out_direct(vrf_dev, sk, skb);
 746
 747	return vrf_ip6_out_redirect(vrf_dev, skb);
 748}
 749
 750/* holding rtnl */
 751static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 752{
 753	struct rt6_info *rt6 = rtnl_dereference(vrf->rt6);
 754	struct net *net = dev_net(dev);
 755	struct dst_entry *dst;
 756
 757	RCU_INIT_POINTER(vrf->rt6, NULL);
 758	synchronize_rcu();
 759
 760	/* move dev in dst's to loopback so this VRF device can be deleted
 761	 * - based on dst_ifdown
 762	 */
 763	if (rt6) {
 764		dst = &rt6->dst;
 765		netdev_ref_replace(dst->dev, net->loopback_dev,
 766				   &dst->dev_tracker, GFP_KERNEL);
 767		dst->dev = net->loopback_dev;
 768		dst_release(dst);
 769	}
 770}
 771
 772static int vrf_rt6_create(struct net_device *dev)
 773{
 774	int flags = DST_NOPOLICY | DST_NOXFRM;
 775	struct net_vrf *vrf = netdev_priv(dev);
 776	struct net *net = dev_net(dev);
 777	struct rt6_info *rt6;
 778	int rc = -ENOMEM;
 779
 780	/* IPv6 can be CONFIG enabled and then disabled runtime */
 781	if (!ipv6_mod_enabled())
 782		return 0;
 783
 784	vrf->fib6_table = fib6_new_table(net, vrf->tb_id);
 785	if (!vrf->fib6_table)
 786		goto out;
 787
 788	/* create a dst for routing packets out a VRF device */
 789	rt6 = ip6_dst_alloc(net, dev, flags);
 790	if (!rt6)
 791		goto out;
 792
 793	rt6->dst.output	= vrf_output6;
 794
 795	rcu_assign_pointer(vrf->rt6, rt6);
 796
 797	rc = 0;
 798out:
 799	return rc;
 800}
 801#else
 802static struct sk_buff *vrf_ip6_out(struct net_device *vrf_dev,
 803				   struct sock *sk,
 804				   struct sk_buff *skb)
 805{
 806	return skb;
 807}
 808
 809static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 810{
 811}
 812
 813static int vrf_rt6_create(struct net_device *dev)
 814{
 815	return 0;
 816}
 817#endif
 818
 819/* modelled after ip_finish_output2 */
 820static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
 821{
 822	struct dst_entry *dst = skb_dst(skb);
 823	struct rtable *rt = dst_rtable(dst);
 824	struct net_device *dev = dst->dev;
 825	unsigned int hh_len = LL_RESERVED_SPACE(dev);
 826	struct neighbour *neigh;
 827	bool is_v6gw = false;
 828
 829	vrf_nf_reset_ct(skb);
 830
 831	/* Be paranoid, rather than too clever. */
 832	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
 833		skb = skb_expand_head(skb, hh_len);
 834		if (!skb) {
 835			dev->stats.tx_errors++;
 836			return -ENOMEM;
 837		}
 838	}
 839
 840	rcu_read_lock();
 841
 842	neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
 843	if (!IS_ERR(neigh)) {
 844		int ret;
 845
 846		sock_confirm_neigh(skb, neigh);
 847		/* if crossing protocols, can not use the cached header */
 848		ret = neigh_output(neigh, skb, is_v6gw);
 849		rcu_read_unlock();
 850		return ret;
 851	}
 852
 853	rcu_read_unlock();
 854	vrf_tx_error(skb->dev, skb);
 855	return -EINVAL;
 856}
 857
 858static int vrf_output(struct net *net, struct sock *sk, struct sk_buff *skb)
 859{
 860	struct net_device *dev = skb_dst(skb)->dev;
 861
 862	IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
 863
 864	skb->dev = dev;
 865	skb->protocol = htons(ETH_P_IP);
 866
 867	return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
 868			    net, sk, skb, NULL, dev,
 869			    vrf_finish_output,
 870			    !(IPCB(skb)->flags & IPSKB_REROUTED));
 871}
 872
 873/* set dst on skb to send packet to us via dev_xmit path. Allows
 874 * packet to go through device based features such as qdisc, netfilter
 875 * hooks and packet sockets with skb->dev set to vrf device.
 876 */
 877static struct sk_buff *vrf_ip_out_redirect(struct net_device *vrf_dev,
 878					   struct sk_buff *skb)
 879{
 880	struct net_vrf *vrf = netdev_priv(vrf_dev);
 881	struct dst_entry *dst = NULL;
 882	struct rtable *rth;
 883
 884	rcu_read_lock();
 885
 886	rth = rcu_dereference(vrf->rth);
 887	if (likely(rth)) {
 888		dst = &rth->dst;
 889		dst_hold(dst);
 890	}
 891
 892	rcu_read_unlock();
 893
 894	if (unlikely(!dst)) {
 895		vrf_tx_error(vrf_dev, skb);
 896		return NULL;
 897	}
 898
 899	skb_dst_drop(skb);
 900	skb_dst_set(skb, dst);
 901
 902	return skb;
 903}
 904
 905static int vrf_output_direct_finish(struct net *net, struct sock *sk,
 906				    struct sk_buff *skb)
 907{
 908	vrf_finish_direct(skb);
 909
 910	return vrf_ip_local_out(net, sk, skb);
 911}
 912
 913static int vrf_output_direct(struct net *net, struct sock *sk,
 914			     struct sk_buff *skb)
 915{
 916	int err = 1;
 917
 918	skb->protocol = htons(ETH_P_IP);
 919
 920	if (!(IPCB(skb)->flags & IPSKB_REROUTED))
 921		err = nf_hook(NFPROTO_IPV4, NF_INET_POST_ROUTING, net, sk, skb,
 922			      NULL, skb->dev, vrf_output_direct_finish);
 923
 924	if (likely(err == 1))
 925		vrf_finish_direct(skb);
 926
 927	return err;
 928}
 929
 930static int vrf_ip_out_direct_finish(struct net *net, struct sock *sk,
 931				    struct sk_buff *skb)
 932{
 933	int err;
 934
 935	err = vrf_output_direct(net, sk, skb);
 936	if (likely(err == 1))
 937		err = vrf_ip_local_out(net, sk, skb);
 938
 939	return err;
 940}
 941
 942static struct sk_buff *vrf_ip_out_direct(struct net_device *vrf_dev,
 943					 struct sock *sk,
 944					 struct sk_buff *skb)
 945{
 946	struct net *net = dev_net(vrf_dev);
 947	int err;
 948
 949	skb->dev = vrf_dev;
 950
 951	err = nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT, net, sk,
 952		      skb, NULL, vrf_dev, vrf_ip_out_direct_finish);
 953
 954	if (likely(err == 1))
 955		err = vrf_output_direct(net, sk, skb);
 956
 957	if (likely(err == 1))
 958		return skb;
 959
 960	return NULL;
 961}
 962
 963static struct sk_buff *vrf_ip_out(struct net_device *vrf_dev,
 964				  struct sock *sk,
 965				  struct sk_buff *skb)
 966{
 967	/* don't divert multicast or local broadcast */
 968	if (ipv4_is_multicast(ip_hdr(skb)->daddr) ||
 969	    ipv4_is_lbcast(ip_hdr(skb)->daddr))
 970		return skb;
 971
 972	vrf_nf_set_untracked(skb);
 973
 974	if (qdisc_tx_is_default(vrf_dev) ||
 975	    IPCB(skb)->flags & IPSKB_XFRM_TRANSFORMED)
 976		return vrf_ip_out_direct(vrf_dev, sk, skb);
 977
 978	return vrf_ip_out_redirect(vrf_dev, skb);
 979}
 980
 981/* called with rcu lock held */
 982static struct sk_buff *vrf_l3_out(struct net_device *vrf_dev,
 983				  struct sock *sk,
 984				  struct sk_buff *skb,
 985				  u16 proto)
 986{
 987	switch (proto) {
 988	case AF_INET:
 989		return vrf_ip_out(vrf_dev, sk, skb);
 990	case AF_INET6:
 991		return vrf_ip6_out(vrf_dev, sk, skb);
 992	}
 993
 994	return skb;
 995}
 996
 997/* holding rtnl */
 998static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf)
 999{
1000	struct rtable *rth = rtnl_dereference(vrf->rth);
1001	struct net *net = dev_net(dev);
1002	struct dst_entry *dst;
1003
1004	RCU_INIT_POINTER(vrf->rth, NULL);
1005	synchronize_rcu();
1006
1007	/* move dev in dst's to loopback so this VRF device can be deleted
1008	 * - based on dst_ifdown
1009	 */
1010	if (rth) {
1011		dst = &rth->dst;
1012		netdev_ref_replace(dst->dev, net->loopback_dev,
1013				   &dst->dev_tracker, GFP_KERNEL);
1014		dst->dev = net->loopback_dev;
1015		dst_release(dst);
1016	}
1017}
1018
1019static int vrf_rtable_create(struct net_device *dev)
1020{
1021	struct net_vrf *vrf = netdev_priv(dev);
1022	struct rtable *rth;
1023
1024	if (!fib_new_table(dev_net(dev), vrf->tb_id))
1025		return -ENOMEM;
1026
1027	/* create a dst for routing packets out through a VRF device */
1028	rth = rt_dst_alloc(dev, 0, RTN_UNICAST, 1);
1029	if (!rth)
1030		return -ENOMEM;
1031
1032	rth->dst.output	= vrf_output;
1033
1034	rcu_assign_pointer(vrf->rth, rth);
1035
1036	return 0;
1037}
1038
1039/**************************** device handling ********************/
1040
1041/* cycle interface to flush neighbor cache and move routes across tables */
1042static void cycle_netdev(struct net_device *dev,
1043			 struct netlink_ext_ack *extack)
1044{
1045	unsigned int flags = dev->flags;
1046	int ret;
1047
1048	if (!netif_running(dev))
1049		return;
1050
1051	ret = dev_change_flags(dev, flags & ~IFF_UP, extack);
1052	if (ret >= 0)
1053		ret = dev_change_flags(dev, flags, extack);
1054
1055	if (ret < 0) {
1056		netdev_err(dev,
1057			   "Failed to cycle device %s; route tables might be wrong!\n",
1058			   dev->name);
1059	}
1060}
1061
1062static int do_vrf_add_slave(struct net_device *dev, struct net_device *port_dev,
1063			    struct netlink_ext_ack *extack)
1064{
1065	int ret;
1066
1067	/* do not allow loopback device to be enslaved to a VRF.
1068	 * The vrf device acts as the loopback for the vrf.
1069	 */
1070	if (port_dev == dev_net(dev)->loopback_dev) {
1071		NL_SET_ERR_MSG(extack,
1072			       "Can not enslave loopback device to a VRF");
1073		return -EOPNOTSUPP;
1074	}
1075
1076	port_dev->priv_flags |= IFF_L3MDEV_SLAVE;
1077	ret = netdev_master_upper_dev_link(port_dev, dev, NULL, NULL, extack);
1078	if (ret < 0)
1079		goto err;
1080
1081	cycle_netdev(port_dev, extack);
1082
1083	return 0;
1084
1085err:
1086	port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE;
1087	return ret;
1088}
1089
1090static int vrf_add_slave(struct net_device *dev, struct net_device *port_dev,
1091			 struct netlink_ext_ack *extack)
1092{
1093	if (netif_is_l3_master(port_dev)) {
1094		NL_SET_ERR_MSG(extack,
1095			       "Can not enslave an L3 master device to a VRF");
1096		return -EINVAL;
1097	}
1098
1099	if (netif_is_l3_slave(port_dev))
1100		return -EINVAL;
1101
1102	return do_vrf_add_slave(dev, port_dev, extack);
1103}
1104
1105/* inverse of do_vrf_add_slave */
1106static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev)
1107{
1108	netdev_upper_dev_unlink(port_dev, dev);
1109	port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE;
1110
1111	cycle_netdev(port_dev, NULL);
1112
1113	return 0;
1114}
1115
1116static int vrf_del_slave(struct net_device *dev, struct net_device *port_dev)
1117{
1118	return do_vrf_del_slave(dev, port_dev);
1119}
1120
1121static void vrf_dev_uninit(struct net_device *dev)
1122{
1123	struct net_vrf *vrf = netdev_priv(dev);
1124
1125	vrf_rtable_release(dev, vrf);
1126	vrf_rt6_release(dev, vrf);
1127}
1128
1129static int vrf_dev_init(struct net_device *dev)
1130{
1131	struct net_vrf *vrf = netdev_priv(dev);
1132
1133	/* create the default dst which points back to us */
1134	if (vrf_rtable_create(dev) != 0)
1135		goto out_nomem;
1136
1137	if (vrf_rt6_create(dev) != 0)
1138		goto out_rth;
1139
1140	dev->flags = IFF_MASTER | IFF_NOARP;
1141
1142	/* similarly, oper state is irrelevant; set to up to avoid confusion */
1143	dev->operstate = IF_OPER_UP;
1144	netdev_lockdep_set_classes(dev);
1145	return 0;
1146
1147out_rth:
1148	vrf_rtable_release(dev, vrf);
1149out_nomem:
1150	return -ENOMEM;
1151}
1152
1153static const struct net_device_ops vrf_netdev_ops = {
1154	.ndo_init		= vrf_dev_init,
1155	.ndo_uninit		= vrf_dev_uninit,
1156	.ndo_start_xmit		= vrf_xmit,
1157	.ndo_set_mac_address	= eth_mac_addr,
1158	.ndo_add_slave		= vrf_add_slave,
1159	.ndo_del_slave		= vrf_del_slave,
1160};
1161
1162static u32 vrf_fib_table(const struct net_device *dev)
1163{
1164	struct net_vrf *vrf = netdev_priv(dev);
1165
1166	return vrf->tb_id;
1167}
1168
1169static int vrf_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
1170{
1171	kfree_skb(skb);
1172	return 0;
1173}
1174
1175static struct sk_buff *vrf_rcv_nfhook(u8 pf, unsigned int hook,
1176				      struct sk_buff *skb,
1177				      struct net_device *dev)
1178{
1179	struct net *net = dev_net(dev);
1180
1181	if (nf_hook(pf, hook, net, NULL, skb, dev, NULL, vrf_rcv_finish) != 1)
1182		skb = NULL;    /* kfree_skb(skb) handled by nf code */
1183
1184	return skb;
1185}
1186
1187static int vrf_prepare_mac_header(struct sk_buff *skb,
1188				  struct net_device *vrf_dev, u16 proto)
1189{
1190	struct ethhdr *eth;
1191	int err;
1192
1193	/* in general, we do not know if there is enough space in the head of
1194	 * the packet for hosting the mac header.
1195	 */
1196	err = skb_cow_head(skb, LL_RESERVED_SPACE(vrf_dev));
1197	if (unlikely(err))
1198		/* no space in the skb head */
1199		return -ENOBUFS;
1200
1201	__skb_push(skb, ETH_HLEN);
1202	eth = (struct ethhdr *)skb->data;
1203
1204	skb_reset_mac_header(skb);
1205	skb_reset_mac_len(skb);
1206
1207	/* we set the ethernet destination and the source addresses to the
1208	 * address of the VRF device.
1209	 */
1210	ether_addr_copy(eth->h_dest, vrf_dev->dev_addr);
1211	ether_addr_copy(eth->h_source, vrf_dev->dev_addr);
1212	eth->h_proto = htons(proto);
1213
1214	/* the destination address of the Ethernet frame corresponds to the
1215	 * address set on the VRF interface; therefore, the packet is intended
1216	 * to be processed locally.
1217	 */
1218	skb->protocol = eth->h_proto;
1219	skb->pkt_type = PACKET_HOST;
1220
1221	skb_postpush_rcsum(skb, skb->data, ETH_HLEN);
1222
1223	skb_pull_inline(skb, ETH_HLEN);
1224
1225	return 0;
1226}
1227
1228/* prepare and add the mac header to the packet if it was not set previously.
1229 * In this way, packet sniffers such as tcpdump can parse the packet correctly.
1230 * If the mac header was already set, the original mac header is left
1231 * untouched and the function returns immediately.
1232 */
1233static int vrf_add_mac_header_if_unset(struct sk_buff *skb,
1234				       struct net_device *vrf_dev,
1235				       u16 proto, struct net_device *orig_dev)
1236{
1237	if (skb_mac_header_was_set(skb) && dev_has_header(orig_dev))
1238		return 0;
1239
1240	return vrf_prepare_mac_header(skb, vrf_dev, proto);
1241}
1242
1243#if IS_ENABLED(CONFIG_IPV6)
1244/* neighbor handling is done with actual device; do not want
1245 * to flip skb->dev for those ndisc packets. This really fails
1246 * for multiple next protocols (e.g., NEXTHDR_HOP). But it is
1247 * a start.
1248 */
1249static bool ipv6_ndisc_frame(const struct sk_buff *skb)
1250{
1251	const struct ipv6hdr *iph = ipv6_hdr(skb);
1252	bool rc = false;
1253
1254	if (iph->nexthdr == NEXTHDR_ICMP) {
1255		const struct icmp6hdr *icmph;
1256		struct icmp6hdr _icmph;
1257
1258		icmph = skb_header_pointer(skb, sizeof(*iph),
1259					   sizeof(_icmph), &_icmph);
1260		if (!icmph)
1261			goto out;
1262
1263		switch (icmph->icmp6_type) {
1264		case NDISC_ROUTER_SOLICITATION:
1265		case NDISC_ROUTER_ADVERTISEMENT:
1266		case NDISC_NEIGHBOUR_SOLICITATION:
1267		case NDISC_NEIGHBOUR_ADVERTISEMENT:
1268		case NDISC_REDIRECT:
1269			rc = true;
1270			break;
1271		}
1272	}
1273
1274out:
1275	return rc;
1276}
1277
1278static struct rt6_info *vrf_ip6_route_lookup(struct net *net,
1279					     const struct net_device *dev,
1280					     struct flowi6 *fl6,
1281					     int ifindex,
1282					     const struct sk_buff *skb,
1283					     int flags)
1284{
1285	struct net_vrf *vrf = netdev_priv(dev);
1286
1287	return ip6_pol_route(net, vrf->fib6_table, ifindex, fl6, skb, flags);
1288}
1289
1290static void vrf_ip6_input_dst(struct sk_buff *skb, struct net_device *vrf_dev,
1291			      int ifindex)
1292{
1293	const struct ipv6hdr *iph = ipv6_hdr(skb);
1294	struct flowi6 fl6 = {
1295		.flowi6_iif     = ifindex,
1296		.flowi6_mark    = skb->mark,
1297		.flowi6_proto   = iph->nexthdr,
1298		.daddr          = iph->daddr,
1299		.saddr          = iph->saddr,
1300		.flowlabel      = ip6_flowinfo(iph),
1301	};
1302	struct net *net = dev_net(vrf_dev);
1303	struct rt6_info *rt6;
1304
1305	skb_dst_drop(skb);
1306
1307	rt6 = vrf_ip6_route_lookup(net, vrf_dev, &fl6, ifindex, skb,
1308				   RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_IFACE);
1309	if (unlikely(!rt6))
1310		return;
1311
1312	if (unlikely(&rt6->dst == &net->ipv6.ip6_null_entry->dst))
1313		return;
1314
1315	skb_dst_set(skb, &rt6->dst);
1316}
1317
1318static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
1319				   struct sk_buff *skb)
1320{
1321	int orig_iif = skb->skb_iif;
1322	bool need_strict = rt6_need_strict(&ipv6_hdr(skb)->daddr);
1323	bool is_ndisc = ipv6_ndisc_frame(skb);
1324
1325	/* loopback, multicast & non-ND link-local traffic; do not push through
1326	 * packet taps again. Reset pkt_type for upper layers to process skb.
1327	 * For non-loopback strict packets, determine the dst using the original
1328	 * ifindex.
1329	 */
1330	if (skb->pkt_type == PACKET_LOOPBACK || (need_strict && !is_ndisc)) {
1331		skb->dev = vrf_dev;
1332		skb->skb_iif = vrf_dev->ifindex;
1333		IP6CB(skb)->flags |= IP6SKB_L3SLAVE;
1334
1335		if (skb->pkt_type == PACKET_LOOPBACK)
1336			skb->pkt_type = PACKET_HOST;
1337		else
1338			vrf_ip6_input_dst(skb, vrf_dev, orig_iif);
1339
1340		goto out;
1341	}
1342
1343	/* if packet is NDISC then keep the ingress interface */
1344	if (!is_ndisc) {
1345		struct net_device *orig_dev = skb->dev;
1346
1347		dev_dstats_rx_add(vrf_dev, skb->len);
1348		skb->dev = vrf_dev;
1349		skb->skb_iif = vrf_dev->ifindex;
1350
1351		if (!list_empty(&vrf_dev->ptype_all)) {
1352			int err;
1353
1354			err = vrf_add_mac_header_if_unset(skb, vrf_dev,
1355							  ETH_P_IPV6,
1356							  orig_dev);
1357			if (likely(!err)) {
1358				skb_push(skb, skb->mac_len);
1359				dev_queue_xmit_nit(skb, vrf_dev);
1360				skb_pull(skb, skb->mac_len);
1361			}
1362		}
1363
1364		IP6CB(skb)->flags |= IP6SKB_L3SLAVE;
1365	}
1366
1367	if (need_strict)
1368		vrf_ip6_input_dst(skb, vrf_dev, orig_iif);
1369
1370	skb = vrf_rcv_nfhook(NFPROTO_IPV6, NF_INET_PRE_ROUTING, skb, vrf_dev);
1371out:
1372	return skb;
1373}
1374
1375#else
1376static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev,
1377				   struct sk_buff *skb)
1378{
1379	return skb;
1380}
1381#endif
1382
1383static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev,
1384				  struct sk_buff *skb)
1385{
1386	struct net_device *orig_dev = skb->dev;
1387
1388	skb->dev = vrf_dev;
1389	skb->skb_iif = vrf_dev->ifindex;
1390	IPCB(skb)->flags |= IPSKB_L3SLAVE;
1391
1392	if (ipv4_is_multicast(ip_hdr(skb)->daddr))
1393		goto out;
1394
1395	/* loopback traffic; do not push through packet taps again.
1396	 * Reset pkt_type for upper layers to process skb
1397	 */
1398	if (skb->pkt_type == PACKET_LOOPBACK) {
1399		skb->pkt_type = PACKET_HOST;
1400		goto out;
1401	}
1402
1403	dev_dstats_rx_add(vrf_dev, skb->len);
1404
1405	if (!list_empty(&vrf_dev->ptype_all)) {
1406		int err;
1407
1408		err = vrf_add_mac_header_if_unset(skb, vrf_dev, ETH_P_IP,
1409						  orig_dev);
1410		if (likely(!err)) {
1411			skb_push(skb, skb->mac_len);
1412			dev_queue_xmit_nit(skb, vrf_dev);
1413			skb_pull(skb, skb->mac_len);
1414		}
1415	}
1416
1417	skb = vrf_rcv_nfhook(NFPROTO_IPV4, NF_INET_PRE_ROUTING, skb, vrf_dev);
1418out:
1419	return skb;
1420}
1421
1422/* called with rcu lock held */
1423static struct sk_buff *vrf_l3_rcv(struct net_device *vrf_dev,
1424				  struct sk_buff *skb,
1425				  u16 proto)
1426{
1427	switch (proto) {
1428	case AF_INET:
1429		return vrf_ip_rcv(vrf_dev, skb);
1430	case AF_INET6:
1431		return vrf_ip6_rcv(vrf_dev, skb);
1432	}
1433
1434	return skb;
1435}
1436
1437#if IS_ENABLED(CONFIG_IPV6)
1438/* send to link-local or multicast address via interface enslaved to
1439 * VRF device. Force lookup to VRF table without changing flow struct
1440 * Note: Caller to this function must hold rcu_read_lock() and no refcnt
1441 * is taken on the dst by this function.
1442 */
1443static struct dst_entry *vrf_link_scope_lookup(const struct net_device *dev,
1444					      struct flowi6 *fl6)
1445{
1446	struct net *net = dev_net(dev);
1447	int flags = RT6_LOOKUP_F_IFACE | RT6_LOOKUP_F_DST_NOREF;
1448	struct dst_entry *dst = NULL;
1449	struct rt6_info *rt;
1450
1451	/* VRF device does not have a link-local address and
1452	 * sending packets to link-local or mcast addresses over
1453	 * a VRF device does not make sense
1454	 */
1455	if (fl6->flowi6_oif == dev->ifindex) {
1456		dst = &net->ipv6.ip6_null_entry->dst;
1457		return dst;
1458	}
1459
1460	if (!ipv6_addr_any(&fl6->saddr))
1461		flags |= RT6_LOOKUP_F_HAS_SADDR;
1462
1463	rt = vrf_ip6_route_lookup(net, dev, fl6, fl6->flowi6_oif, NULL, flags);
1464	if (rt)
1465		dst = &rt->dst;
1466
1467	return dst;
1468}
1469#endif
1470
1471static const struct l3mdev_ops vrf_l3mdev_ops = {
1472	.l3mdev_fib_table	= vrf_fib_table,
1473	.l3mdev_l3_rcv		= vrf_l3_rcv,
1474	.l3mdev_l3_out		= vrf_l3_out,
1475#if IS_ENABLED(CONFIG_IPV6)
1476	.l3mdev_link_scope_lookup = vrf_link_scope_lookup,
1477#endif
1478};
1479
1480static void vrf_get_drvinfo(struct net_device *dev,
1481			    struct ethtool_drvinfo *info)
1482{
1483	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
1484	strscpy(info->version, DRV_VERSION, sizeof(info->version));
1485}
1486
1487static const struct ethtool_ops vrf_ethtool_ops = {
1488	.get_drvinfo	= vrf_get_drvinfo,
1489};
1490
1491static inline size_t vrf_fib_rule_nl_size(void)
1492{
1493	size_t sz;
1494
1495	sz  = NLMSG_ALIGN(sizeof(struct fib_rule_hdr));
1496	sz += nla_total_size(sizeof(u8));	/* FRA_L3MDEV */
1497	sz += nla_total_size(sizeof(u32));	/* FRA_PRIORITY */
1498	sz += nla_total_size(sizeof(u8));       /* FRA_PROTOCOL */
1499
1500	return sz;
1501}
1502
1503static int vrf_fib_rule(const struct net_device *dev, __u8 family, bool add_it)
1504{
1505	struct fib_rule_hdr *frh;
1506	struct nlmsghdr *nlh;
1507	struct sk_buff *skb;
1508	int err;
1509
1510	if ((family == AF_INET6 || family == RTNL_FAMILY_IP6MR) &&
1511	    !ipv6_mod_enabled())
1512		return 0;
1513
1514	skb = nlmsg_new(vrf_fib_rule_nl_size(), GFP_KERNEL);
1515	if (!skb)
1516		return -ENOMEM;
1517
1518	nlh = nlmsg_put(skb, 0, 0, 0, sizeof(*frh), 0);
1519	if (!nlh)
1520		goto nla_put_failure;
1521
1522	/* rule only needs to appear once */
1523	nlh->nlmsg_flags |= NLM_F_EXCL;
1524
1525	frh = nlmsg_data(nlh);
1526	memset(frh, 0, sizeof(*frh));
1527	frh->family = family;
1528	frh->action = FR_ACT_TO_TBL;
1529
1530	if (nla_put_u8(skb, FRA_PROTOCOL, RTPROT_KERNEL))
1531		goto nla_put_failure;
1532
1533	if (nla_put_u8(skb, FRA_L3MDEV, 1))
1534		goto nla_put_failure;
1535
1536	if (nla_put_u32(skb, FRA_PRIORITY, FIB_RULE_PREF))
1537		goto nla_put_failure;
1538
1539	nlmsg_end(skb, nlh);
1540
1541	if (add_it) {
1542		err = fib_newrule(dev_net(dev), skb, nlh, NULL, true);
1543		if (err == -EEXIST)
1544			err = 0;
1545	} else {
1546		err = fib_delrule(dev_net(dev), skb, nlh, NULL, true);
1547		if (err == -ENOENT)
1548			err = 0;
1549	}
1550	nlmsg_free(skb);
1551
1552	return err;
1553
1554nla_put_failure:
1555	nlmsg_free(skb);
1556
1557	return -EMSGSIZE;
1558}
1559
1560static int vrf_add_fib_rules(const struct net_device *dev)
1561{
1562	int err;
1563
1564	err = vrf_fib_rule(dev, AF_INET,  true);
1565	if (err < 0)
1566		goto out_err;
1567
1568	err = vrf_fib_rule(dev, AF_INET6, true);
1569	if (err < 0)
1570		goto ipv6_err;
1571
1572#if IS_ENABLED(CONFIG_IP_MROUTE_MULTIPLE_TABLES)
1573	err = vrf_fib_rule(dev, RTNL_FAMILY_IPMR, true);
1574	if (err < 0)
1575		goto ipmr_err;
1576#endif
1577
1578#if IS_ENABLED(CONFIG_IPV6_MROUTE_MULTIPLE_TABLES)
1579	err = vrf_fib_rule(dev, RTNL_FAMILY_IP6MR, true);
1580	if (err < 0)
1581		goto ip6mr_err;
1582#endif
1583
1584	return 0;
1585
1586#if IS_ENABLED(CONFIG_IPV6_MROUTE_MULTIPLE_TABLES)
1587ip6mr_err:
1588	vrf_fib_rule(dev, RTNL_FAMILY_IPMR,  false);
1589#endif
1590
1591#if IS_ENABLED(CONFIG_IP_MROUTE_MULTIPLE_TABLES)
1592ipmr_err:
1593	vrf_fib_rule(dev, AF_INET6,  false);
1594#endif
1595
1596ipv6_err:
1597	vrf_fib_rule(dev, AF_INET,  false);
1598
1599out_err:
1600	netdev_err(dev, "Failed to add FIB rules.\n");
1601	return err;
1602}
1603
1604static void vrf_setup(struct net_device *dev)
1605{
1606	ether_setup(dev);
1607
1608	/* Initialize the device structure. */
1609	dev->netdev_ops = &vrf_netdev_ops;
1610	dev->l3mdev_ops = &vrf_l3mdev_ops;
1611	dev->ethtool_ops = &vrf_ethtool_ops;
1612	dev->needs_free_netdev = true;
1613
1614	/* Fill in device structure with ethernet-generic values. */
1615	eth_hw_addr_random(dev);
1616
1617	/* don't acquire vrf device's netif_tx_lock when transmitting */
1618	dev->lltx = true;
1619
1620	/* don't allow vrf devices to change network namespaces. */
1621	dev->netns_immutable = true;
1622
1623	/* does not make sense for a VLAN to be added to a vrf device */
1624	dev->features   |= NETIF_F_VLAN_CHALLENGED;
1625
1626	/* enable offload features */
1627	dev->features   |= NETIF_F_GSO_SOFTWARE;
1628	dev->features   |= NETIF_F_RXCSUM | NETIF_F_HW_CSUM | NETIF_F_SCTP_CRC;
1629	dev->features   |= NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA;
1630
1631	dev->hw_features = dev->features;
1632	dev->hw_enc_features = dev->features;
1633
1634	/* default to no qdisc; user can add if desired */
1635	dev->priv_flags |= IFF_NO_QUEUE;
1636	dev->priv_flags |= IFF_NO_RX_HANDLER;
1637	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
1638
1639	/* VRF devices do not care about MTU, but if the MTU is set
1640	 * too low then the ipv4 and ipv6 protocols are disabled
1641	 * which breaks networking.
1642	 */
1643	dev->min_mtu = IPV6_MIN_MTU;
1644	dev->max_mtu = IP6_MAX_MTU;
1645	dev->mtu = dev->max_mtu;
1646
1647	dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
1648}
1649
1650static int vrf_validate(struct nlattr *tb[], struct nlattr *data[],
1651			struct netlink_ext_ack *extack)
1652{
1653	if (tb[IFLA_ADDRESS]) {
1654		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
1655			NL_SET_ERR_MSG(extack, "Invalid hardware address");
1656			return -EINVAL;
1657		}
1658		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
1659			NL_SET_ERR_MSG(extack, "Invalid hardware address");
1660			return -EADDRNOTAVAIL;
1661		}
1662	}
1663	return 0;
1664}
1665
1666static void vrf_dellink(struct net_device *dev, struct list_head *head)
1667{
1668	struct net_device *port_dev;
1669	struct list_head *iter;
1670
1671	netdev_for_each_lower_dev(dev, port_dev, iter)
1672		vrf_del_slave(dev, port_dev);
1673
1674	vrf_map_unregister_dev(dev);
1675
1676	unregister_netdevice_queue(dev, head);
1677}
1678
1679static int vrf_newlink(struct net_device *dev,
1680		       struct rtnl_newlink_params *params,
1681		       struct netlink_ext_ack *extack)
1682{
1683	struct net_vrf *vrf = netdev_priv(dev);
1684	struct nlattr **data = params->data;
1685	struct netns_vrf *nn_vrf;
1686	bool *add_fib_rules;
1687	struct net *net;
1688	int err;
1689
1690	if (!data || !data[IFLA_VRF_TABLE]) {
1691		NL_SET_ERR_MSG(extack, "VRF table id is missing");
1692		return -EINVAL;
1693	}
1694
1695	vrf->tb_id = nla_get_u32(data[IFLA_VRF_TABLE]);
1696	if (vrf->tb_id == RT_TABLE_UNSPEC) {
1697		NL_SET_ERR_MSG_ATTR(extack, data[IFLA_VRF_TABLE],
1698				    "Invalid VRF table id");
1699		return -EINVAL;
1700	}
1701
1702	dev->priv_flags |= IFF_L3MDEV_MASTER;
1703
1704	err = register_netdevice(dev);
1705	if (err)
1706		goto out;
1707
1708	/* mapping between table_id and vrf;
1709	 * note: such binding could not be done in the dev init function
1710	 * because dev->ifindex id is not available yet.
1711	 */
1712	vrf->ifindex = dev->ifindex;
1713
1714	err = vrf_map_register_dev(dev, extack);
1715	if (err) {
1716		unregister_netdevice(dev);
1717		goto out;
1718	}
1719
1720	net = dev_net(dev);
1721	nn_vrf = net_generic(net, vrf_net_id);
1722
1723	add_fib_rules = &nn_vrf->add_fib_rules;
1724	if (*add_fib_rules) {
1725		err = vrf_add_fib_rules(dev);
1726		if (err) {
1727			vrf_map_unregister_dev(dev);
1728			unregister_netdevice(dev);
1729			goto out;
1730		}
1731		*add_fib_rules = false;
1732	}
1733
1734out:
1735	return err;
1736}
1737
1738static size_t vrf_nl_getsize(const struct net_device *dev)
1739{
1740	return nla_total_size(sizeof(u32));  /* IFLA_VRF_TABLE */
1741}
1742
1743static int vrf_fillinfo(struct sk_buff *skb,
1744			const struct net_device *dev)
1745{
1746	struct net_vrf *vrf = netdev_priv(dev);
1747
1748	return nla_put_u32(skb, IFLA_VRF_TABLE, vrf->tb_id);
1749}
1750
1751static size_t vrf_get_slave_size(const struct net_device *bond_dev,
1752				 const struct net_device *slave_dev)
1753{
1754	return nla_total_size(sizeof(u32));  /* IFLA_VRF_PORT_TABLE */
1755}
1756
1757static int vrf_fill_slave_info(struct sk_buff *skb,
1758			       const struct net_device *vrf_dev,
1759			       const struct net_device *slave_dev)
1760{
1761	struct net_vrf *vrf = netdev_priv(vrf_dev);
1762
1763	if (nla_put_u32(skb, IFLA_VRF_PORT_TABLE, vrf->tb_id))
1764		return -EMSGSIZE;
1765
1766	return 0;
1767}
1768
1769static const struct nla_policy vrf_nl_policy[IFLA_VRF_MAX + 1] = {
1770	[IFLA_VRF_TABLE] = { .type = NLA_U32 },
1771};
1772
1773static struct rtnl_link_ops vrf_link_ops __read_mostly = {
1774	.kind		= DRV_NAME,
1775	.priv_size	= sizeof(struct net_vrf),
1776
1777	.get_size	= vrf_nl_getsize,
1778	.policy		= vrf_nl_policy,
1779	.validate	= vrf_validate,
1780	.fill_info	= vrf_fillinfo,
1781
1782	.get_slave_size  = vrf_get_slave_size,
1783	.fill_slave_info = vrf_fill_slave_info,
1784
1785	.newlink	= vrf_newlink,
1786	.dellink	= vrf_dellink,
1787	.setup		= vrf_setup,
1788	.maxtype	= IFLA_VRF_MAX,
1789};
1790
1791static int vrf_device_event(struct notifier_block *unused,
1792			    unsigned long event, void *ptr)
1793{
1794	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1795
1796	/* only care about unregister events to drop slave references */
1797	if (event == NETDEV_UNREGISTER) {
1798		struct net_device *vrf_dev;
1799
1800		if (!netif_is_l3_slave(dev))
1801			goto out;
1802
1803		vrf_dev = netdev_master_upper_dev_get(dev);
1804		vrf_del_slave(vrf_dev, dev);
1805	}
1806out:
1807	return NOTIFY_DONE;
1808}
1809
1810static struct notifier_block vrf_notifier_block __read_mostly = {
1811	.notifier_call = vrf_device_event,
1812};
1813
1814static int vrf_map_init(struct vrf_map *vmap)
1815{
1816	spin_lock_init(&vmap->vmap_lock);
1817	hash_init(vmap->ht);
1818
1819	vmap->strict_mode = false;
1820
1821	return 0;
1822}
1823
1824#ifdef CONFIG_SYSCTL
1825static bool vrf_strict_mode(struct vrf_map *vmap)
1826{
1827	bool strict_mode;
1828
1829	vrf_map_lock(vmap);
1830	strict_mode = vmap->strict_mode;
1831	vrf_map_unlock(vmap);
1832
1833	return strict_mode;
1834}
1835
1836static int vrf_strict_mode_change(struct vrf_map *vmap, bool new_mode)
1837{
1838	bool *cur_mode;
1839	int res = 0;
1840
1841	vrf_map_lock(vmap);
1842
1843	cur_mode = &vmap->strict_mode;
1844	if (*cur_mode == new_mode)
1845		goto unlock;
1846
1847	if (*cur_mode) {
1848		/* disable strict mode */
1849		*cur_mode = false;
1850	} else {
1851		if (vmap->shared_tables) {
1852			/* we cannot allow strict_mode because there are some
1853			 * vrfs that share one or more tables.
1854			 */
1855			res = -EBUSY;
1856			goto unlock;
1857		}
1858
1859		/* no tables are shared among vrfs, so we can go back
1860		 * to 1:1 association between a vrf with its table.
1861		 */
1862		*cur_mode = true;
1863	}
1864
1865unlock:
1866	vrf_map_unlock(vmap);
1867
1868	return res;
1869}
1870
1871static int vrf_shared_table_handler(const struct ctl_table *table, int write,
1872				    void *buffer, size_t *lenp, loff_t *ppos)
1873{
1874	struct net *net = (struct net *)table->extra1;
1875	struct vrf_map *vmap = netns_vrf_map(net);
1876	int proc_strict_mode = 0;
1877	struct ctl_table tmp = {
1878		.procname	= table->procname,
1879		.data		= &proc_strict_mode,
1880		.maxlen		= sizeof(int),
1881		.mode		= table->mode,
1882		.extra1		= SYSCTL_ZERO,
1883		.extra2		= SYSCTL_ONE,
1884	};
1885	int ret;
1886
1887	if (!write)
1888		proc_strict_mode = vrf_strict_mode(vmap);
1889
1890	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
1891
1892	if (write && ret == 0)
1893		ret = vrf_strict_mode_change(vmap, (bool)proc_strict_mode);
1894
1895	return ret;
1896}
1897
1898static const struct ctl_table vrf_table[] = {
1899	{
1900		.procname	= "strict_mode",
1901		.data		= NULL,
1902		.maxlen		= sizeof(int),
1903		.mode		= 0644,
1904		.proc_handler	= vrf_shared_table_handler,
1905		/* set by the vrf_netns_init */
1906		.extra1		= NULL,
1907	},
1908};
1909
1910static int vrf_netns_init_sysctl(struct net *net, struct netns_vrf *nn_vrf)
1911{
1912	struct ctl_table *table;
1913
1914	table = kmemdup(vrf_table, sizeof(vrf_table), GFP_KERNEL);
1915	if (!table)
1916		return -ENOMEM;
1917
1918	/* init the extra1 parameter with the reference to current netns */
1919	table[0].extra1 = net;
1920
1921	nn_vrf->ctl_hdr = register_net_sysctl_sz(net, "net/vrf", table,
1922						 ARRAY_SIZE(vrf_table));
1923	if (!nn_vrf->ctl_hdr) {
1924		kfree(table);
1925		return -ENOMEM;
1926	}
1927
1928	return 0;
1929}
1930
1931static void vrf_netns_exit_sysctl(struct net *net)
1932{
1933	struct netns_vrf *nn_vrf = net_generic(net, vrf_net_id);
1934	const struct ctl_table *table;
1935
1936	table = nn_vrf->ctl_hdr->ctl_table_arg;
1937	unregister_net_sysctl_table(nn_vrf->ctl_hdr);
1938	kfree(table);
1939}
1940#else
1941static int vrf_netns_init_sysctl(struct net *net, struct netns_vrf *nn_vrf)
1942{
1943	return 0;
1944}
1945
1946static void vrf_netns_exit_sysctl(struct net *net)
1947{
1948}
1949#endif
1950
1951/* Initialize per network namespace state */
1952static int __net_init vrf_netns_init(struct net *net)
1953{
1954	struct netns_vrf *nn_vrf = net_generic(net, vrf_net_id);
1955
1956	nn_vrf->add_fib_rules = true;
1957	vrf_map_init(&nn_vrf->vmap);
1958
1959	return vrf_netns_init_sysctl(net, nn_vrf);
1960}
1961
1962static void __net_exit vrf_netns_exit(struct net *net)
1963{
1964	vrf_netns_exit_sysctl(net);
1965}
1966
1967static struct pernet_operations vrf_net_ops __net_initdata = {
1968	.init = vrf_netns_init,
1969	.exit = vrf_netns_exit,
1970	.id   = &vrf_net_id,
1971	.size = sizeof(struct netns_vrf),
1972};
1973
1974static int __init vrf_init_module(void)
1975{
1976	int rc;
1977
1978	register_netdevice_notifier(&vrf_notifier_block);
1979
1980	rc = register_pernet_subsys(&vrf_net_ops);
1981	if (rc < 0)
1982		goto error;
1983
1984	rc = l3mdev_table_lookup_register(L3MDEV_TYPE_VRF,
1985					  vrf_ifindex_lookup_by_table_id);
1986	if (rc < 0)
1987		goto unreg_pernet;
1988
1989	rc = rtnl_link_register(&vrf_link_ops);
1990	if (rc < 0)
1991		goto table_lookup_unreg;
1992
1993	return 0;
1994
1995table_lookup_unreg:
1996	l3mdev_table_lookup_unregister(L3MDEV_TYPE_VRF,
1997				       vrf_ifindex_lookup_by_table_id);
1998
1999unreg_pernet:
2000	unregister_pernet_subsys(&vrf_net_ops);
2001
2002error:
2003	unregister_netdevice_notifier(&vrf_notifier_block);
2004	return rc;
2005}
2006
2007module_init(vrf_init_module);
2008MODULE_AUTHOR("Shrijeet Mukherjee, David Ahern");
2009MODULE_DESCRIPTION("Device driver to instantiate VRF domains");
2010MODULE_LICENSE("GPL");
2011MODULE_ALIAS_RTNL_LINK(DRV_NAME);
2012MODULE_VERSION(DRV_VERSION);