FILE
netkit.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/netkit.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (c) 2023 Isovalent */ 3 4#include <linux/netdevice.h> 5#include <linux/ethtool.h> 6#include <linux/etherdevice.h> 7#include <linux/filter.h> 8#include <linux/netfilter_netdev.h> 9#include <linux/bpf_mprog.h> 10#include <linux/indirect_call_wrapper.h> 11 12#include <net/netkit.h> 13#include <net/dst.h> 14#include <net/tcx.h> 15 16#define DRV_NAME "netkit" 17 18struct netkit { 19 __cacheline_group_begin(netkit_fastpath); 20 struct net_device __rcu *peer; 21 struct bpf_mprog_entry __rcu *active; 22 enum netkit_action policy; 23 enum netkit_scrub scrub; 24 struct bpf_mprog_bundle bundle; 25 __cacheline_group_end(netkit_fastpath); 26 27 __cacheline_group_begin(netkit_slowpath); 28 enum netkit_mode mode; 29 bool primary; 30 u32 headroom; 31 __cacheline_group_end(netkit_slowpath); 32}; 33 34struct netkit_link { 35 struct bpf_link link; 36 struct net_device *dev; 37}; 38 39static __always_inline int 40netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, 41 enum netkit_action ret) 42{ 43 const struct bpf_mprog_fp *fp; 44 const struct bpf_prog *prog; 45 46 bpf_mprog_foreach_prog(entry, fp, prog) { 47 bpf_compute_data_pointers(skb); 48 ret = bpf_prog_run(prog, skb); 49 if (ret != NETKIT_NEXT) 50 break; 51 } 52 return ret; 53} 54 55static void netkit_xnet(struct sk_buff *skb) 56{ 57 skb->priority = 0; 58 skb->mark = 0; 59} 60 61static void netkit_prep_forward(struct sk_buff *skb, 62 bool xnet, bool xnet_scrub) 63{ 64 skb_scrub_packet(skb, false); 65 nf_skip_egress(skb, true); 66 skb_reset_mac_header(skb); 67 if (!xnet) 68 return; 69 skb_clear_tstamp(skb); 70 if (xnet_scrub) 71 netkit_xnet(skb); 72} 73 74static struct netkit *netkit_priv(const struct net_device *dev) 75{ 76 return netdev_priv(dev); 77} 78 79static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev) 80{ 81 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 82 struct netkit *nk = netkit_priv(dev); 83 enum netkit_action ret = READ_ONCE(nk->policy); 84 netdev_tx_t ret_dev = NET_XMIT_SUCCESS; 85 const struct bpf_mprog_entry *entry; 86 struct net_device *peer; 87 int len = skb->len; 88 89 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 90 rcu_read_lock(); 91 peer = rcu_dereference(nk->peer); 92 if (unlikely(!peer || !(peer->flags & IFF_UP) || 93 !pskb_may_pull(skb, ETH_HLEN) || 94 skb_orphan_frags(skb, GFP_ATOMIC))) 95 goto drop; 96 netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), 97 nk->scrub); 98 eth_skb_pkt_type(skb, peer); 99 skb->dev = peer; 100 entry = rcu_dereference(nk->active); 101 if (entry) 102 ret = netkit_run(entry, skb, ret); 103 switch (ret) { 104 case NETKIT_NEXT: 105 case NETKIT_PASS: 106 eth_skb_pull_mac(skb); 107 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 108 if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { 109 dev_sw_netstats_tx_add(dev, 1, len); 110 dev_sw_netstats_rx_add(peer, len); 111 } else { 112 goto drop_stats; 113 } 114 break; 115 case NETKIT_REDIRECT: 116 dev_sw_netstats_tx_add(dev, 1, len); 117 skb_do_redirect(skb); 118 break; 119 case NETKIT_DROP: 120 default: 121drop: 122 kfree_skb(skb); 123drop_stats: 124 dev_core_stats_tx_dropped_inc(dev); 125 ret_dev = NET_XMIT_DROP; 126 break; 127 } 128 rcu_read_unlock(); 129 bpf_net_ctx_clear(bpf_net_ctx); 130 return ret_dev; 131} 132 133static int netkit_open(struct net_device *dev) 134{ 135 struct netkit *nk = netkit_priv(dev); 136 struct net_device *peer = rtnl_dereference(nk->peer); 137 138 if (!peer) 139 return -ENOTCONN; 140 if (peer->flags & IFF_UP) { 141 netif_carrier_on(dev); 142 netif_carrier_on(peer); 143 } 144 return 0; 145} 146 147static int netkit_close(struct net_device *dev) 148{ 149 struct netkit *nk = netkit_priv(dev); 150 struct net_device *peer = rtnl_dereference(nk->peer); 151 152 netif_carrier_off(dev); 153 if (peer) 154 netif_carrier_off(peer); 155 return 0; 156} 157 158static int netkit_get_iflink(const struct net_device *dev) 159{ 160 struct netkit *nk = netkit_priv(dev); 161 struct net_device *peer; 162 int iflink = 0; 163 164 rcu_read_lock(); 165 peer = rcu_dereference(nk->peer); 166 if (peer) 167 iflink = READ_ONCE(peer->ifindex); 168 rcu_read_unlock(); 169 return iflink; 170} 171 172static void netkit_set_multicast(struct net_device *dev) 173{ 174 /* Nothing to do, we receive whatever gets pushed to us! */ 175} 176 177static int netkit_set_macaddr(struct net_device *dev, void *sa) 178{ 179 struct netkit *nk = netkit_priv(dev); 180 181 if (nk->mode != NETKIT_L2) 182 return -EOPNOTSUPP; 183 184 return eth_mac_addr(dev, sa); 185} 186 187static void netkit_set_headroom(struct net_device *dev, int headroom) 188{ 189 struct netkit *nk = netkit_priv(dev), *nk2; 190 struct net_device *peer; 191 192 if (headroom < 0) 193 headroom = NET_SKB_PAD; 194 195 rcu_read_lock(); 196 peer = rcu_dereference(nk->peer); 197 if (unlikely(!peer)) 198 goto out; 199 200 nk2 = netkit_priv(peer); 201 nk->headroom = headroom; 202 headroom = max(nk->headroom, nk2->headroom); 203 204 peer->needed_headroom = headroom; 205 dev->needed_headroom = headroom; 206out: 207 rcu_read_unlock(); 208} 209 210INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev) 211{ 212 return rcu_dereference(netkit_priv(dev)->peer); 213} 214 215static void netkit_get_stats(struct net_device *dev, 216 struct rtnl_link_stats64 *stats) 217{ 218 dev_fetch_sw_netstats(stats, dev->tstats); 219 stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 220} 221 222static void netkit_uninit(struct net_device *dev); 223 224static const struct net_device_ops netkit_netdev_ops = { 225 .ndo_open = netkit_open, 226 .ndo_stop = netkit_close, 227 .ndo_start_xmit = netkit_xmit, 228 .ndo_set_rx_mode = netkit_set_multicast, 229 .ndo_set_rx_headroom = netkit_set_headroom, 230 .ndo_set_mac_address = netkit_set_macaddr, 231 .ndo_get_iflink = netkit_get_iflink, 232 .ndo_get_peer_dev = netkit_peer_dev, 233 .ndo_get_stats64 = netkit_get_stats, 234 .ndo_uninit = netkit_uninit, 235 .ndo_features_check = passthru_features_check, 236}; 237 238static void netkit_get_drvinfo(struct net_device *dev, 239 struct ethtool_drvinfo *info) 240{ 241 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 242} 243 244static const struct ethtool_ops netkit_ethtool_ops = { 245 .get_drvinfo = netkit_get_drvinfo, 246}; 247 248static void netkit_setup(struct net_device *dev) 249{ 250 static const netdev_features_t netkit_features_hw_vlan = 251 NETIF_F_HW_VLAN_CTAG_TX | 252 NETIF_F_HW_VLAN_CTAG_RX | 253 NETIF_F_HW_VLAN_STAG_TX | 254 NETIF_F_HW_VLAN_STAG_RX; 255 static const netdev_features_t netkit_features = 256 netkit_features_hw_vlan | 257 NETIF_F_SG | 258 NETIF_F_FRAGLIST | 259 NETIF_F_HW_CSUM | 260 NETIF_F_RXCSUM | 261 NETIF_F_SCTP_CRC | 262 NETIF_F_HIGHDMA | 263 NETIF_F_GSO_SOFTWARE | 264 NETIF_F_GSO_ENCAP_ALL; 265 266 ether_setup(dev); 267 dev->max_mtu = ETH_MAX_MTU; 268 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 269 270 dev->flags |= IFF_NOARP; 271 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 272 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 273 dev->priv_flags |= IFF_PHONY_HEADROOM; 274 dev->priv_flags |= IFF_NO_QUEUE; 275 dev->priv_flags |= IFF_DISABLE_NETPOLL; 276 dev->lltx = true; 277 278 dev->ethtool_ops = &netkit_ethtool_ops; 279 dev->netdev_ops = &netkit_netdev_ops; 280 281 dev->features |= netkit_features; 282 dev->hw_features = netkit_features; 283 dev->hw_enc_features = netkit_features; 284 dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; 285 dev->vlan_features = dev->features & ~netkit_features_hw_vlan; 286 287 dev->needs_free_netdev = true; 288 289 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 290} 291 292static struct net *netkit_get_link_net(const struct net_device *dev) 293{ 294 struct netkit *nk = netkit_priv(dev); 295 struct net_device *peer = rtnl_dereference(nk->peer); 296 297 return peer ? dev_net(peer) : dev_net(dev); 298} 299 300static int netkit_check_policy(int policy, struct nlattr *tb, 301 struct netlink_ext_ack *extack) 302{ 303 switch (policy) { 304 case NETKIT_PASS: 305 case NETKIT_DROP: 306 return 0; 307 default: 308 NL_SET_ERR_MSG_ATTR(extack, tb, 309 "Provided default xmit policy not supported"); 310 return -EINVAL; 311 } 312} 313 314static int netkit_validate(struct nlattr *tb[], struct nlattr *data[], 315 struct netlink_ext_ack *extack) 316{ 317 struct nlattr *attr = tb[IFLA_ADDRESS]; 318 319 if (!attr) 320 return 0; 321 if (nla_len(attr) != ETH_ALEN) 322 return -EINVAL; 323 if (!is_valid_ether_addr(nla_data(attr))) 324 return -EADDRNOTAVAIL; 325 return 0; 326} 327 328static struct rtnl_link_ops netkit_link_ops; 329 330static int netkit_new_link(struct net_device *dev, 331 struct rtnl_newlink_params *params, 332 struct netlink_ext_ack *extack) 333{ 334 struct net *peer_net = rtnl_newlink_peer_net(params); 335 enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT; 336 enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT; 337 struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr; 338 enum netkit_action policy_prim = NETKIT_PASS; 339 enum netkit_action policy_peer = NETKIT_PASS; 340 struct nlattr **data = params->data; 341 enum netkit_mode mode = NETKIT_L3; 342 unsigned char ifname_assign_type; 343 struct nlattr **tb = params->tb; 344 u16 headroom = 0, tailroom = 0; 345 struct ifinfomsg *ifmp = NULL; 346 struct net_device *peer; 347 char ifname[IFNAMSIZ]; 348 struct netkit *nk; 349 int err; 350 351 tbp = tb; 352 if (data) { 353 if (data[IFLA_NETKIT_MODE]) 354 mode = nla_get_u32(data[IFLA_NETKIT_MODE]); 355 if (data[IFLA_NETKIT_PEER_INFO]) { 356 attr = data[IFLA_NETKIT_PEER_INFO]; 357 ifmp = nla_data(attr); 358 rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack); 359 tbp = peer_tb; 360 } 361 if (data[IFLA_NETKIT_SCRUB]) 362 scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]); 363 if (data[IFLA_NETKIT_PEER_SCRUB]) 364 scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]); 365 if (data[IFLA_NETKIT_POLICY]) { 366 attr = data[IFLA_NETKIT_POLICY]; 367 policy_prim = nla_get_u32(attr); 368 err = netkit_check_policy(policy_prim, attr, extack); 369 if (err < 0) 370 return err; 371 } 372 if (data[IFLA_NETKIT_PEER_POLICY]) { 373 attr = data[IFLA_NETKIT_PEER_POLICY]; 374 policy_peer = nla_get_u32(attr); 375 err = netkit_check_policy(policy_peer, attr, extack); 376 if (err < 0) 377 return err; 378 } 379 if (data[IFLA_NETKIT_HEADROOM]) 380 headroom = nla_get_u16(data[IFLA_NETKIT_HEADROOM]); 381 if (data[IFLA_NETKIT_TAILROOM]) 382 tailroom = nla_get_u16(data[IFLA_NETKIT_TAILROOM]); 383 } 384 385 if (ifmp && tbp[IFLA_IFNAME]) { 386 nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); 387 ifname_assign_type = NET_NAME_USER; 388 } else { 389 strscpy(ifname, "nk%d", IFNAMSIZ); 390 ifname_assign_type = NET_NAME_ENUM; 391 } 392 if (mode != NETKIT_L2 && 393 (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS])) 394 return -EOPNOTSUPP; 395 396 peer = rtnl_create_link(peer_net, ifname, ifname_assign_type, 397 &netkit_link_ops, tbp, extack); 398 if (IS_ERR(peer)) 399 return PTR_ERR(peer); 400 401 netif_inherit_tso_max(peer, dev); 402 if (headroom) { 403 peer->needed_headroom = headroom; 404 dev->needed_headroom = headroom; 405 } 406 if (tailroom) { 407 peer->needed_tailroom = tailroom; 408 dev->needed_tailroom = tailroom; 409 } 410 411 if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS])) 412 eth_hw_addr_random(peer); 413 if (ifmp && dev->ifindex) 414 peer->ifindex = ifmp->ifi_index; 415 416 nk = netkit_priv(peer); 417 nk->primary = false; 418 nk->policy = policy_peer; 419 nk->scrub = scrub_peer; 420 nk->mode = mode; 421 nk->headroom = headroom; 422 bpf_mprog_bundle_init(&nk->bundle); 423 424 err = register_netdevice(peer); 425 if (err < 0) 426 goto err_register_peer; 427 netif_carrier_off(peer); 428 if (mode == NETKIT_L2) 429 dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL); 430 431 err = rtnl_configure_link(peer, NULL, 0, NULL); 432 if (err < 0) 433 goto err_configure_peer; 434 435 if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS]) 436 eth_hw_addr_random(dev); 437 if (tb[IFLA_IFNAME]) 438 nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); 439 else 440 strscpy(dev->name, "nk%d", IFNAMSIZ); 441 442 nk = netkit_priv(dev); 443 nk->primary = true; 444 nk->policy = policy_prim; 445 nk->scrub = scrub_prim; 446 nk->mode = mode; 447 nk->headroom = headroom; 448 bpf_mprog_bundle_init(&nk->bundle); 449 450 err = register_netdevice(dev); 451 if (err < 0) 452 goto err_configure_peer; 453 netif_carrier_off(dev); 454 if (mode == NETKIT_L2) 455 dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL); 456 457 rcu_assign_pointer(netkit_priv(dev)->peer, peer); 458 rcu_assign_pointer(netkit_priv(peer)->peer, dev); 459 return 0; 460err_configure_peer: 461 unregister_netdevice(peer); 462 return err; 463err_register_peer: 464 free_netdev(peer); 465 return err; 466} 467 468static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev, 469 bool bundle_fallback) 470{ 471 struct netkit *nk = netkit_priv(dev); 472 struct bpf_mprog_entry *entry; 473 474 ASSERT_RTNL(); 475 entry = rcu_dereference_rtnl(nk->active); 476 if (entry) 477 return entry; 478 if (bundle_fallback) 479 return &nk->bundle.a; 480 return NULL; 481} 482 483static void netkit_entry_update(struct net_device *dev, 484 struct bpf_mprog_entry *entry) 485{ 486 struct netkit *nk = netkit_priv(dev); 487 488 ASSERT_RTNL(); 489 rcu_assign_pointer(nk->active, entry); 490} 491 492static void netkit_entry_sync(void) 493{ 494 synchronize_rcu(); 495} 496 497static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which) 498{ 499 struct net_device *dev; 500 struct netkit *nk; 501 502 ASSERT_RTNL(); 503 504 switch (which) { 505 case BPF_NETKIT_PRIMARY: 506 case BPF_NETKIT_PEER: 507 break; 508 default: 509 return ERR_PTR(-EINVAL); 510 } 511 512 dev = __dev_get_by_index(net, ifindex); 513 if (!dev) 514 return ERR_PTR(-ENODEV); 515 if (dev->netdev_ops != &netkit_netdev_ops) 516 return ERR_PTR(-ENXIO); 517 518 nk = netkit_priv(dev); 519 if (!nk->primary) 520 return ERR_PTR(-EACCES); 521 if (which == BPF_NETKIT_PEER) { 522 dev = rcu_dereference_rtnl(nk->peer); 523 if (!dev) 524 return ERR_PTR(-ENODEV); 525 } 526 return dev; 527} 528 529int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) 530{ 531 struct bpf_mprog_entry *entry, *entry_new; 532 struct bpf_prog *replace_prog = NULL; 533 struct net_device *dev; 534 int ret; 535 536 rtnl_lock(); 537 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 538 attr->attach_type); 539 if (IS_ERR(dev)) { 540 ret = PTR_ERR(dev); 541 goto out; 542 } 543 entry = netkit_entry_fetch(dev, true); 544 if (attr->attach_flags & BPF_F_REPLACE) { 545 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, 546 prog->type); 547 if (IS_ERR(replace_prog)) { 548 ret = PTR_ERR(replace_prog); 549 replace_prog = NULL; 550 goto out; 551 } 552 } 553 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, 554 attr->attach_flags, attr->relative_fd, 555 attr->expected_revision); 556 if (!ret) { 557 if (entry != entry_new) { 558 netkit_entry_update(dev, entry_new); 559 netkit_entry_sync(); 560 } 561 bpf_mprog_commit(entry); 562 } 563out: 564 if (replace_prog) 565 bpf_prog_put(replace_prog); 566 rtnl_unlock(); 567 return ret; 568} 569 570int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) 571{ 572 struct bpf_mprog_entry *entry, *entry_new; 573 struct net_device *dev; 574 int ret; 575 576 rtnl_lock(); 577 dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, 578 attr->attach_type); 579 if (IS_ERR(dev)) { 580 ret = PTR_ERR(dev); 581 goto out; 582 } 583 entry = netkit_entry_fetch(dev, false); 584 if (!entry) { 585 ret = -ENOENT; 586 goto out; 587 } 588 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, 589 attr->relative_fd, attr->expected_revision); 590 if (!ret) { 591 if (!bpf_mprog_total(entry_new)) 592 entry_new = NULL; 593 netkit_entry_update(dev, entry_new); 594 netkit_entry_sync(); 595 bpf_mprog_commit(entry); 596 } 597out: 598 rtnl_unlock(); 599 return ret; 600} 601 602int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) 603{ 604 struct net_device *dev; 605 int ret; 606 607 rtnl_lock(); 608 dev = netkit_dev_fetch(current->nsproxy->net_ns, 609 attr->query.target_ifindex, 610 attr->query.attach_type); 611 if (IS_ERR(dev)) { 612 ret = PTR_ERR(dev); 613 goto out; 614 } 615 ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false)); 616out: 617 rtnl_unlock(); 618 return ret; 619} 620 621static struct netkit_link *netkit_link(const struct bpf_link *link) 622{ 623 return container_of(link, struct netkit_link, link); 624} 625 626static int netkit_link_prog_attach(struct bpf_link *link, u32 flags, 627 u32 id_or_fd, u64 revision) 628{ 629 struct netkit_link *nkl = netkit_link(link); 630 struct bpf_mprog_entry *entry, *entry_new; 631 struct net_device *dev = nkl->dev; 632 int ret; 633 634 ASSERT_RTNL(); 635 entry = netkit_entry_fetch(dev, true); 636 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, 637 id_or_fd, revision); 638 if (!ret) { 639 if (entry != entry_new) { 640 netkit_entry_update(dev, entry_new); 641 netkit_entry_sync(); 642 } 643 bpf_mprog_commit(entry); 644 } 645 return ret; 646} 647 648static void netkit_link_release(struct bpf_link *link) 649{ 650 struct netkit_link *nkl = netkit_link(link); 651 struct bpf_mprog_entry *entry, *entry_new; 652 struct net_device *dev; 653 int ret = 0; 654 655 rtnl_lock(); 656 dev = nkl->dev; 657 if (!dev) 658 goto out; 659 entry = netkit_entry_fetch(dev, false); 660 if (!entry) { 661 ret = -ENOENT; 662 goto out; 663 } 664 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); 665 if (!ret) { 666 if (!bpf_mprog_total(entry_new)) 667 entry_new = NULL; 668 netkit_entry_update(dev, entry_new); 669 netkit_entry_sync(); 670 bpf_mprog_commit(entry); 671 nkl->dev = NULL; 672 } 673out: 674 WARN_ON_ONCE(ret); 675 rtnl_unlock(); 676} 677 678static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog, 679 struct bpf_prog *oprog) 680{ 681 struct netkit_link *nkl = netkit_link(link); 682 struct bpf_mprog_entry *entry, *entry_new; 683 struct net_device *dev; 684 int ret = 0; 685 686 rtnl_lock(); 687 dev = nkl->dev; 688 if (!dev) { 689 ret = -ENOLINK; 690 goto out; 691 } 692 if (oprog && link->prog != oprog) { 693 ret = -EPERM; 694 goto out; 695 } 696 oprog = link->prog; 697 if (oprog == nprog) { 698 bpf_prog_put(nprog); 699 goto out; 700 } 701 entry = netkit_entry_fetch(dev, false); 702 if (!entry) { 703 ret = -ENOENT; 704 goto out; 705 } 706 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, 707 BPF_F_REPLACE | BPF_F_ID, 708 link->prog->aux->id, 0); 709 if (!ret) { 710 WARN_ON_ONCE(entry != entry_new); 711 oprog = xchg(&link->prog, nprog); 712 bpf_prog_put(oprog); 713 bpf_mprog_commit(entry); 714 } 715out: 716 rtnl_unlock(); 717 return ret; 718} 719 720static void netkit_link_dealloc(struct bpf_link *link) 721{ 722 kfree(netkit_link(link)); 723} 724 725static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) 726{ 727 const struct netkit_link *nkl = netkit_link(link); 728 u32 ifindex = 0; 729 730 rtnl_lock(); 731 if (nkl->dev) 732 ifindex = nkl->dev->ifindex; 733 rtnl_unlock(); 734 735 seq_printf(seq, "ifindex:\t%u\n", ifindex); 736 seq_printf(seq, "attach_type:\t%u (%s)\n", 737 link->attach_type, 738 link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer"); 739} 740 741static int netkit_link_fill_info(const struct bpf_link *link, 742 struct bpf_link_info *info) 743{ 744 const struct netkit_link *nkl = netkit_link(link); 745 u32 ifindex = 0; 746 747 rtnl_lock(); 748 if (nkl->dev) 749 ifindex = nkl->dev->ifindex; 750 rtnl_unlock(); 751 752 info->netkit.ifindex = ifindex; 753 info->netkit.attach_type = link->attach_type; 754 return 0; 755} 756 757static int netkit_link_detach(struct bpf_link *link) 758{ 759 netkit_link_release(link); 760 return 0; 761} 762 763static const struct bpf_link_ops netkit_link_lops = { 764 .release = netkit_link_release, 765 .detach = netkit_link_detach, 766 .dealloc = netkit_link_dealloc, 767 .update_prog = netkit_link_update, 768 .show_fdinfo = netkit_link_fdinfo, 769 .fill_link_info = netkit_link_fill_info, 770}; 771 772static int netkit_link_init(struct netkit_link *nkl, 773 struct bpf_link_primer *link_primer, 774 const union bpf_attr *attr, 775 struct net_device *dev, 776 struct bpf_prog *prog) 777{ 778 bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT, 779 &netkit_link_lops, prog, attr->link_create.attach_type); 780 nkl->dev = dev; 781 return bpf_link_prime(&nkl->link, link_primer); 782} 783 784int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 785{ 786 struct bpf_link_primer link_primer; 787 struct netkit_link *nkl; 788 struct net_device *dev; 789 int ret; 790 791 rtnl_lock(); 792 dev = netkit_dev_fetch(current->nsproxy->net_ns, 793 attr->link_create.target_ifindex, 794 attr->link_create.attach_type); 795 if (IS_ERR(dev)) { 796 ret = PTR_ERR(dev); 797 goto out; 798 } 799 nkl = kzalloc(sizeof(*nkl), GFP_KERNEL_ACCOUNT); 800 if (!nkl) { 801 ret = -ENOMEM; 802 goto out; 803 } 804 ret = netkit_link_init(nkl, &link_primer, attr, dev, prog); 805 if (ret) { 806 kfree(nkl); 807 goto out; 808 } 809 ret = netkit_link_prog_attach(&nkl->link, 810 attr->link_create.flags, 811 attr->link_create.netkit.relative_fd, 812 attr->link_create.netkit.expected_revision); 813 if (ret) { 814 nkl->dev = NULL; 815 bpf_link_cleanup(&link_primer); 816 goto out; 817 } 818 ret = bpf_link_settle(&link_primer); 819out: 820 rtnl_unlock(); 821 return ret; 822} 823 824static void netkit_release_all(struct net_device *dev) 825{ 826 struct bpf_mprog_entry *entry; 827 struct bpf_tuple tuple = {}; 828 struct bpf_mprog_fp *fp; 829 struct bpf_mprog_cp *cp; 830 831 entry = netkit_entry_fetch(dev, false); 832 if (!entry) 833 return; 834 netkit_entry_update(dev, NULL); 835 netkit_entry_sync(); 836 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { 837 if (tuple.link) 838 netkit_link(tuple.link)->dev = NULL; 839 else 840 bpf_prog_put(tuple.prog); 841 } 842} 843 844static void netkit_uninit(struct net_device *dev) 845{ 846 netkit_release_all(dev); 847} 848 849static void netkit_del_link(struct net_device *dev, struct list_head *head) 850{ 851 struct netkit *nk = netkit_priv(dev); 852 struct net_device *peer = rtnl_dereference(nk->peer); 853 854 RCU_INIT_POINTER(nk->peer, NULL); 855 unregister_netdevice_queue(dev, head); 856 if (peer) { 857 nk = netkit_priv(peer); 858 RCU_INIT_POINTER(nk->peer, NULL); 859 unregister_netdevice_queue(peer, head); 860 } 861} 862 863static int netkit_change_link(struct net_device *dev, struct nlattr *tb[], 864 struct nlattr *data[], 865 struct netlink_ext_ack *extack) 866{ 867 struct netkit *nk = netkit_priv(dev); 868 struct net_device *peer = rtnl_dereference(nk->peer); 869 enum netkit_action policy; 870 struct nlattr *attr; 871 int err, i; 872 static const struct { 873 u32 attr; 874 char *name; 875 } fixed_params[] = { 876 { IFLA_NETKIT_MODE, "operating mode" }, 877 { IFLA_NETKIT_SCRUB, "scrubbing" }, 878 { IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" }, 879 { IFLA_NETKIT_PEER_INFO, "peer info" }, 880 { IFLA_NETKIT_HEADROOM, "headroom" }, 881 { IFLA_NETKIT_TAILROOM, "tailroom" }, 882 }; 883 884 if (!nk->primary) { 885 NL_SET_ERR_MSG(extack, 886 "netkit link settings can be changed only through the primary device"); 887 return -EACCES; 888 } 889 890 for (i = 0; i < ARRAY_SIZE(fixed_params); i++) { 891 attr = data[fixed_params[i].attr]; 892 if (attr) { 893 NL_SET_ERR_MSG_ATTR_FMT(extack, attr, 894 "netkit link %s cannot be changed after device creation", 895 fixed_params[i].name); 896 return -EACCES; 897 } 898 } 899 900 if (data[IFLA_NETKIT_POLICY]) { 901 attr = data[IFLA_NETKIT_POLICY]; 902 policy = nla_get_u32(attr); 903 err = netkit_check_policy(policy, attr, extack); 904 if (err) 905 return err; 906 WRITE_ONCE(nk->policy, policy); 907 } 908 909 if (data[IFLA_NETKIT_PEER_POLICY]) { 910 err = -EOPNOTSUPP; 911 attr = data[IFLA_NETKIT_PEER_POLICY]; 912 policy = nla_get_u32(attr); 913 if (peer) 914 err = netkit_check_policy(policy, attr, extack); 915 if (err) 916 return err; 917 nk = netkit_priv(peer); 918 WRITE_ONCE(nk->policy, policy); 919 } 920 921 return 0; 922} 923 924static size_t netkit_get_size(const struct net_device *dev) 925{ 926 return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */ 927 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */ 928 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */ 929 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */ 930 nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */ 931 nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */ 932 nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */ 933 nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */ 934 0; 935} 936 937static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev) 938{ 939 struct netkit *nk = netkit_priv(dev); 940 struct net_device *peer = rtnl_dereference(nk->peer); 941 942 if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary)) 943 return -EMSGSIZE; 944 if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy)) 945 return -EMSGSIZE; 946 if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode)) 947 return -EMSGSIZE; 948 if (nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub)) 949 return -EMSGSIZE; 950 if (nla_put_u16(skb, IFLA_NETKIT_HEADROOM, dev->needed_headroom)) 951 return -EMSGSIZE; 952 if (nla_put_u16(skb, IFLA_NETKIT_TAILROOM, dev->needed_tailroom)) 953 return -EMSGSIZE; 954 955 if (peer) { 956 nk = netkit_priv(peer); 957 if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy)) 958 return -EMSGSIZE; 959 if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub)) 960 return -EMSGSIZE; 961 } 962 963 return 0; 964} 965 966static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = { 967 [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) }, 968 [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3), 969 [IFLA_NETKIT_POLICY] = { .type = NLA_U32 }, 970 [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 }, 971 [IFLA_NETKIT_HEADROOM] = { .type = NLA_U16 }, 972 [IFLA_NETKIT_TAILROOM] = { .type = NLA_U16 }, 973 [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), 974 [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), 975 [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT, 976 .reject_message = "Primary attribute is read-only" }, 977}; 978 979static struct rtnl_link_ops netkit_link_ops = { 980 .kind = DRV_NAME, 981 .priv_size = sizeof(struct netkit), 982 .setup = netkit_setup, 983 .newlink = netkit_new_link, 984 .dellink = netkit_del_link, 985 .changelink = netkit_change_link, 986 .get_link_net = netkit_get_link_net, 987 .get_size = netkit_get_size, 988 .fill_info = netkit_fill_info, 989 .policy = netkit_policy, 990 .validate = netkit_validate, 991 .peer_type = IFLA_NETKIT_PEER_INFO, 992 .maxtype = IFLA_NETKIT_MAX, 993}; 994 995static __init int netkit_init(void) 996{ 997 BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT || 998 (int)NETKIT_PASS != (int)TCX_PASS || 999 (int)NETKIT_DROP != (int)TCX_DROP || 1000 (int)NETKIT_REDIRECT != (int)TCX_REDIRECT); 1001 1002 return rtnl_link_register(&netkit_link_ops); 1003} 1004 1005static __exit void netkit_exit(void) 1006{ 1007 rtnl_link_unregister(&netkit_link_ops); 1008} 1009 1010module_init(netkit_init); 1011module_exit(netkit_exit); 1012 1013MODULE_DESCRIPTION("BPF-programmable network device"); 1014MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>"); 1015MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>"); 1016MODULE_LICENSE("GPL"); 1017MODULE_ALIAS_RTNL_LINK(DRV_NAME);
~/
Read-only file preview.