FILE
af_packet.c
File preview with syntax highlighting.
linux-6.19.14/net/packet/af_packet.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * PACKET - implements raw packet sockets. 8 * 9 * Authors: Ross Biro 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Alan Cox, <gw4pts@gw4pts.ampr.org> 12 * 13 * Fixes: 14 * Alan Cox : verify_area() now used correctly 15 * Alan Cox : new skbuff lists, look ma no backlogs! 16 * Alan Cox : tidied skbuff lists. 17 * Alan Cox : Now uses generic datagram routines I 18 * added. Also fixed the peek/read crash 19 * from all old Linux datagram code. 20 * Alan Cox : Uses the improved datagram code. 21 * Alan Cox : Added NULL's for socket options. 22 * Alan Cox : Re-commented the code. 23 * Alan Cox : Use new kernel side addressing 24 * Rob Janssen : Correct MTU usage. 25 * Dave Platt : Counter leaks caused by incorrect 26 * interrupt locking and some slightly 27 * dubious gcc output. Can you read 28 * compiler: it said _VOLATILE_ 29 * Richard Kooijman : Timestamp fixes. 30 * Alan Cox : New buffers. Use sk->mac.raw. 31 * Alan Cox : sendmsg/recvmsg support. 32 * Alan Cox : Protocol setting support 33 * Alexey Kuznetsov : Untied from IPv4 stack. 34 * Cyrus Durgin : Fixed kerneld for kmod. 35 * Michal Ostrowski : Module initialization cleanup. 36 * Ulises Alonso : Frame number limit removal and 37 * packet_set_ring memory leak. 38 * Eric Biederman : Allow for > 8 byte hardware addresses. 39 * The convention is that longer addresses 40 * will simply extend the hardware address 41 * byte arrays at the end of sockaddr_ll 42 * and packet_mreq. 43 * Johann Baudy : Added TX RING. 44 * Chetan Loke : Implemented TPACKET_V3 block abstraction 45 * layer. 46 * Copyright (C) 2011, <lokec@ccs.neu.edu> 47 */ 48 49#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 50 51#include <linux/ethtool.h> 52#include <linux/filter.h> 53#include <linux/types.h> 54#include <linux/mm.h> 55#include <linux/capability.h> 56#include <linux/fcntl.h> 57#include <linux/socket.h> 58#include <linux/in.h> 59#include <linux/inet.h> 60#include <linux/netdevice.h> 61#include <linux/if_packet.h> 62#include <linux/wireless.h> 63#include <linux/kernel.h> 64#include <linux/kmod.h> 65#include <linux/slab.h> 66#include <linux/vmalloc.h> 67#include <net/net_namespace.h> 68#include <net/ip.h> 69#include <net/protocol.h> 70#include <linux/skbuff.h> 71#include <net/sock.h> 72#include <linux/errno.h> 73#include <linux/timer.h> 74#include <linux/uaccess.h> 75#include <asm/ioctls.h> 76#include <asm/page.h> 77#include <asm/cacheflush.h> 78#include <asm/io.h> 79#include <linux/proc_fs.h> 80#include <linux/seq_file.h> 81#include <linux/poll.h> 82#include <linux/module.h> 83#include <linux/init.h> 84#include <linux/mutex.h> 85#include <linux/if_vlan.h> 86#include <linux/virtio_net.h> 87#include <linux/errqueue.h> 88#include <linux/net_tstamp.h> 89#include <linux/percpu.h> 90#ifdef CONFIG_INET 91#include <net/inet_common.h> 92#endif 93#include <linux/bpf.h> 94#include <net/compat.h> 95#include <linux/netfilter_netdev.h> 96 97#include "internal.h" 98 99/* 100 Assumptions: 101 - If the device has no dev->header_ops->create, there is no LL header 102 visible above the device. In this case, its hard_header_len should be 0. 103 The device may prepend its own header internally. In this case, its 104 needed_headroom should be set to the space needed for it to add its 105 internal header. 106 For example, a WiFi driver pretending to be an Ethernet driver should 107 set its hard_header_len to be the Ethernet header length, and set its 108 needed_headroom to be (the real WiFi header length - the fake Ethernet 109 header length). 110 - packet socket receives packets with pulled ll header, 111 so that SOCK_RAW should push it back. 112 113On receive: 114----------- 115 116Incoming, dev_has_header(dev) == true 117 mac_header -> ll header 118 data -> data 119 120Outgoing, dev_has_header(dev) == true 121 mac_header -> ll header 122 data -> ll header 123 124Incoming, dev_has_header(dev) == false 125 mac_header -> data 126 However drivers often make it point to the ll header. 127 This is incorrect because the ll header should be invisible to us. 128 data -> data 129 130Outgoing, dev_has_header(dev) == false 131 mac_header -> data. ll header is invisible to us. 132 data -> data 133 134Resume 135 If dev_has_header(dev) == false we are unable to restore the ll header, 136 because it is invisible to us. 137 138 139On transmit: 140------------ 141 142dev_has_header(dev) == true 143 mac_header -> ll header 144 data -> ll header 145 146dev_has_header(dev) == false (ll header is invisible to us) 147 mac_header -> data 148 data -> data 149 150 We should set network_header on output to the correct position, 151 packet classifier depends on it. 152 */ 153 154/* Private packet socket structures. */ 155 156/* identical to struct packet_mreq except it has 157 * a longer address field. 158 */ 159struct packet_mreq_max { 160 int mr_ifindex; 161 unsigned short mr_type; 162 unsigned short mr_alen; 163 unsigned char mr_address[MAX_ADDR_LEN]; 164}; 165 166union tpacket_uhdr { 167 struct tpacket_hdr *h1; 168 struct tpacket2_hdr *h2; 169 struct tpacket3_hdr *h3; 170 void *raw; 171}; 172 173static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, 174 int closing, int tx_ring); 175 176#define V3_ALIGNMENT (8) 177 178#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT)) 179 180#define BLK_PLUS_PRIV(sz_of_priv) \ 181 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT)) 182 183#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status) 184#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts) 185#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt) 186#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len) 187#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num) 188#define BLOCK_O2PRIV(x) ((x)->offset_to_priv) 189 190struct packet_sock; 191static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, 192 struct packet_type *pt, struct net_device *orig_dev); 193 194static void *packet_previous_frame(struct packet_sock *po, 195 struct packet_ring_buffer *rb, 196 int status); 197static void packet_increment_head(struct packet_ring_buffer *buff); 198static int prb_curr_blk_in_use(struct tpacket_block_desc *); 199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *, 200 struct packet_sock *); 201static void prb_retire_current_block(struct tpacket_kbdq_core *, 202 struct packet_sock *, unsigned int status); 203static int prb_queue_frozen(struct tpacket_kbdq_core *); 204static void prb_open_block(struct tpacket_kbdq_core *, 205 struct tpacket_block_desc *); 206static enum hrtimer_restart prb_retire_rx_blk_timer_expired(struct hrtimer *); 207static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *); 208static void prb_clear_rxhash(struct tpacket_kbdq_core *, 209 struct tpacket3_hdr *); 210static void prb_fill_vlan_info(struct tpacket_kbdq_core *, 211 struct tpacket3_hdr *); 212static void packet_flush_mclist(struct sock *sk); 213static u16 packet_pick_tx_queue(struct sk_buff *skb); 214 215struct packet_skb_cb { 216 union { 217 struct sockaddr_pkt pkt; 218 union { 219 /* Trick: alias skb original length with 220 * ll.sll_family and ll.protocol in order 221 * to save room. 222 */ 223 unsigned int origlen; 224 struct sockaddr_ll ll; 225 }; 226 } sa; 227}; 228 229#define vio_le() virtio_legacy_is_little_endian() 230 231#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb)) 232 233#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc)) 234#define GET_PBLOCK_DESC(x, bid) \ 235 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer)) 236#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \ 237 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer)) 238#define GET_NEXT_PRB_BLK_NUM(x) \ 239 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \ 240 ((x)->kactive_blk_num+1) : 0) 241 242static void __fanout_unlink(struct sock *sk, struct packet_sock *po); 243static void __fanout_link(struct sock *sk, struct packet_sock *po); 244 245#ifdef CONFIG_NETFILTER_EGRESS 246static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb) 247{ 248 struct sk_buff *next, *head = NULL, *tail; 249 int rc; 250 251 rcu_read_lock(); 252 for (; skb != NULL; skb = next) { 253 next = skb->next; 254 skb_mark_not_on_list(skb); 255 256 if (!nf_hook_egress(skb, &rc, skb->dev)) 257 continue; 258 259 if (!head) 260 head = skb; 261 else 262 tail->next = skb; 263 264 tail = skb; 265 } 266 rcu_read_unlock(); 267 268 return head; 269} 270#endif 271 272static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb) 273{ 274 if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS)) 275 return dev_queue_xmit(skb); 276 277#ifdef CONFIG_NETFILTER_EGRESS 278 if (nf_hook_egress_active()) { 279 skb = nf_hook_direct_egress(skb); 280 if (!skb) 281 return NET_XMIT_DROP; 282 } 283#endif 284 return dev_direct_xmit(skb, packet_pick_tx_queue(skb)); 285} 286 287static struct net_device *packet_cached_dev_get(struct packet_sock *po) 288{ 289 struct net_device *dev; 290 291 rcu_read_lock(); 292 dev = rcu_dereference(po->cached_dev); 293 dev_hold(dev); 294 rcu_read_unlock(); 295 296 return dev; 297} 298 299static void packet_cached_dev_assign(struct packet_sock *po, 300 struct net_device *dev) 301{ 302 rcu_assign_pointer(po->cached_dev, dev); 303} 304 305static void packet_cached_dev_reset(struct packet_sock *po) 306{ 307 RCU_INIT_POINTER(po->cached_dev, NULL); 308} 309 310static u16 packet_pick_tx_queue(struct sk_buff *skb) 311{ 312 struct net_device *dev = skb->dev; 313 const struct net_device_ops *ops = dev->netdev_ops; 314 int cpu = raw_smp_processor_id(); 315 u16 queue_index; 316 317#ifdef CONFIG_XPS 318 skb->sender_cpu = cpu + 1; 319#endif 320 skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues); 321 if (ops->ndo_select_queue) { 322 queue_index = ops->ndo_select_queue(dev, skb, NULL); 323 queue_index = netdev_cap_txqueue(dev, queue_index); 324 } else { 325 queue_index = netdev_pick_tx(dev, skb, NULL); 326 } 327 328 return queue_index; 329} 330 331/* __register_prot_hook must be invoked through register_prot_hook 332 * or from a context in which asynchronous accesses to the packet 333 * socket is not possible (packet_create()). 334 */ 335static void __register_prot_hook(struct sock *sk) 336{ 337 struct packet_sock *po = pkt_sk(sk); 338 339 if (!packet_sock_flag(po, PACKET_SOCK_RUNNING)) { 340 if (po->fanout) 341 __fanout_link(sk, po); 342 else 343 dev_add_pack(&po->prot_hook); 344 345 sock_hold(sk); 346 packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 1); 347 } 348} 349 350static void register_prot_hook(struct sock *sk) 351{ 352 lockdep_assert_held_once(&pkt_sk(sk)->bind_lock); 353 __register_prot_hook(sk); 354} 355 356/* If the sync parameter is true, we will temporarily drop 357 * the po->bind_lock and do a synchronize_net to make sure no 358 * asynchronous packet processing paths still refer to the elements 359 * of po->prot_hook. If the sync parameter is false, it is the 360 * callers responsibility to take care of this. 361 */ 362static void __unregister_prot_hook(struct sock *sk, bool sync) 363{ 364 struct packet_sock *po = pkt_sk(sk); 365 366 lockdep_assert_held_once(&po->bind_lock); 367 368 packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 0); 369 370 if (po->fanout) 371 __fanout_unlink(sk, po); 372 else 373 __dev_remove_pack(&po->prot_hook); 374 375 __sock_put(sk); 376 377 if (sync) { 378 spin_unlock(&po->bind_lock); 379 synchronize_net(); 380 spin_lock(&po->bind_lock); 381 } 382} 383 384static void unregister_prot_hook(struct sock *sk, bool sync) 385{ 386 struct packet_sock *po = pkt_sk(sk); 387 388 if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) 389 __unregister_prot_hook(sk, sync); 390} 391 392static inline struct page * __pure pgv_to_page(void *addr) 393{ 394 if (is_vmalloc_addr(addr)) 395 return vmalloc_to_page(addr); 396 return virt_to_page(addr); 397} 398 399static void __packet_set_status(struct packet_sock *po, void *frame, int status) 400{ 401 union tpacket_uhdr h; 402 403 /* WRITE_ONCE() are paired with READ_ONCE() in __packet_get_status */ 404 405 h.raw = frame; 406 switch (po->tp_version) { 407 case TPACKET_V1: 408 WRITE_ONCE(h.h1->tp_status, status); 409 flush_dcache_page(pgv_to_page(&h.h1->tp_status)); 410 break; 411 case TPACKET_V2: 412 WRITE_ONCE(h.h2->tp_status, status); 413 flush_dcache_page(pgv_to_page(&h.h2->tp_status)); 414 break; 415 case TPACKET_V3: 416 WRITE_ONCE(h.h3->tp_status, status); 417 flush_dcache_page(pgv_to_page(&h.h3->tp_status)); 418 break; 419 default: 420 WARN(1, "TPACKET version not supported.\n"); 421 BUG(); 422 } 423 424 smp_wmb(); 425} 426 427static int __packet_get_status(const struct packet_sock *po, void *frame) 428{ 429 union tpacket_uhdr h; 430 431 smp_rmb(); 432 433 /* READ_ONCE() are paired with WRITE_ONCE() in __packet_set_status */ 434 435 h.raw = frame; 436 switch (po->tp_version) { 437 case TPACKET_V1: 438 flush_dcache_page(pgv_to_page(&h.h1->tp_status)); 439 return READ_ONCE(h.h1->tp_status); 440 case TPACKET_V2: 441 flush_dcache_page(pgv_to_page(&h.h2->tp_status)); 442 return READ_ONCE(h.h2->tp_status); 443 case TPACKET_V3: 444 flush_dcache_page(pgv_to_page(&h.h3->tp_status)); 445 return READ_ONCE(h.h3->tp_status); 446 default: 447 WARN(1, "TPACKET version not supported.\n"); 448 BUG(); 449 return 0; 450 } 451} 452 453static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts, 454 unsigned int flags) 455{ 456 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 457 458 if (shhwtstamps && 459 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) && 460 ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts)) 461 return TP_STATUS_TS_RAW_HARDWARE; 462 463 if ((flags & SOF_TIMESTAMPING_SOFTWARE) && 464 ktime_to_timespec64_cond(skb_tstamp(skb), ts)) 465 return TP_STATUS_TS_SOFTWARE; 466 467 return 0; 468} 469 470static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame, 471 struct sk_buff *skb) 472{ 473 union tpacket_uhdr h; 474 struct timespec64 ts; 475 __u32 ts_status; 476 477 if (!(ts_status = tpacket_get_timestamp(skb, &ts, READ_ONCE(po->tp_tstamp)))) 478 return 0; 479 480 h.raw = frame; 481 /* 482 * versions 1 through 3 overflow the timestamps in y2106, since they 483 * all store the seconds in a 32-bit unsigned integer. 484 * If we create a version 4, that should have a 64-bit timestamp, 485 * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit 486 * nanoseconds. 487 */ 488 switch (po->tp_version) { 489 case TPACKET_V1: 490 h.h1->tp_sec = ts.tv_sec; 491 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC; 492 break; 493 case TPACKET_V2: 494 h.h2->tp_sec = ts.tv_sec; 495 h.h2->tp_nsec = ts.tv_nsec; 496 break; 497 case TPACKET_V3: 498 h.h3->tp_sec = ts.tv_sec; 499 h.h3->tp_nsec = ts.tv_nsec; 500 break; 501 default: 502 WARN(1, "TPACKET version not supported.\n"); 503 BUG(); 504 } 505 506 /* one flush is safe, as both fields always lie on the same cacheline */ 507 flush_dcache_page(pgv_to_page(&h.h1->tp_sec)); 508 smp_wmb(); 509 510 return ts_status; 511} 512 513static void *packet_lookup_frame(const struct packet_sock *po, 514 const struct packet_ring_buffer *rb, 515 unsigned int position, 516 int status) 517{ 518 unsigned int pg_vec_pos, frame_offset; 519 union tpacket_uhdr h; 520 521 pg_vec_pos = position / rb->frames_per_block; 522 frame_offset = position % rb->frames_per_block; 523 524 h.raw = rb->pg_vec[pg_vec_pos].buffer + 525 (frame_offset * rb->frame_size); 526 527 if (status != __packet_get_status(po, h.raw)) 528 return NULL; 529 530 return h.raw; 531} 532 533static void *packet_current_frame(struct packet_sock *po, 534 struct packet_ring_buffer *rb, 535 int status) 536{ 537 return packet_lookup_frame(po, rb, rb->head, status); 538} 539 540static u16 vlan_get_tci(const struct sk_buff *skb, struct net_device *dev) 541{ 542 struct vlan_hdr vhdr, *vh; 543 unsigned int header_len; 544 545 if (!dev) 546 return 0; 547 548 /* In the SOCK_DGRAM scenario, skb data starts at the network 549 * protocol, which is after the VLAN headers. The outer VLAN 550 * header is at the hard_header_len offset in non-variable 551 * length link layer headers. If it's a VLAN device, the 552 * min_header_len should be used to exclude the VLAN header 553 * size. 554 */ 555 if (dev->min_header_len == dev->hard_header_len) 556 header_len = dev->hard_header_len; 557 else if (is_vlan_dev(dev)) 558 header_len = dev->min_header_len; 559 else 560 return 0; 561 562 vh = skb_header_pointer(skb, skb_mac_offset(skb) + header_len, 563 sizeof(vhdr), &vhdr); 564 if (unlikely(!vh)) 565 return 0; 566 567 return ntohs(vh->h_vlan_TCI); 568} 569 570static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb) 571{ 572 __be16 proto = skb->protocol; 573 574 if (unlikely(eth_type_vlan(proto))) 575 proto = __vlan_get_protocol_offset(skb, proto, 576 skb_mac_offset(skb), NULL); 577 578 return proto; 579} 580 581static void prb_shutdown_retire_blk_timer(struct packet_sock *po, 582 struct sk_buff_head *rb_queue) 583{ 584 struct tpacket_kbdq_core *pkc; 585 586 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); 587 hrtimer_cancel(&pkc->retire_blk_timer); 588} 589 590static int prb_calc_retire_blk_tmo(struct packet_sock *po, 591 int blk_size_in_bytes) 592{ 593 struct net_device *dev; 594 unsigned int mbits, div; 595 struct ethtool_link_ksettings ecmd; 596 int err; 597 598 rtnl_lock(); 599 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex); 600 if (unlikely(!dev)) { 601 rtnl_unlock(); 602 return DEFAULT_PRB_RETIRE_TOV; 603 } 604 err = __ethtool_get_link_ksettings(dev, &ecmd); 605 rtnl_unlock(); 606 if (err) 607 return DEFAULT_PRB_RETIRE_TOV; 608 609 /* If the link speed is so slow you don't really 610 * need to worry about perf anyways 611 */ 612 if (ecmd.base.speed < SPEED_1000 || 613 ecmd.base.speed == SPEED_UNKNOWN) 614 return DEFAULT_PRB_RETIRE_TOV; 615 616 div = ecmd.base.speed / 1000; 617 mbits = (blk_size_in_bytes * 8) / (1024 * 1024); 618 619 if (div) 620 mbits /= div; 621 622 if (div) 623 return mbits + 1; 624 return mbits; 625} 626 627static void prb_init_ft_ops(struct tpacket_kbdq_core *p1, 628 union tpacket_req_u *req_u) 629{ 630 p1->feature_req_word = req_u->req3.tp_feature_req_word; 631} 632 633static void init_prb_bdqc(struct packet_sock *po, 634 struct packet_ring_buffer *rb, 635 struct pgv *pg_vec, 636 union tpacket_req_u *req_u) 637{ 638 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb); 639 struct tpacket_block_desc *pbd; 640 641 memset(p1, 0x0, sizeof(*p1)); 642 643 p1->knxt_seq_num = 1; 644 p1->pkbdq = pg_vec; 645 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer; 646 p1->pkblk_start = pg_vec[0].buffer; 647 p1->kblk_size = req_u->req3.tp_block_size; 648 p1->knum_blocks = req_u->req3.tp_block_nr; 649 p1->hdrlen = po->tp_hdrlen; 650 p1->version = po->tp_version; 651 po->stats.stats3.tp_freeze_q_cnt = 0; 652 if (req_u->req3.tp_retire_blk_tov) 653 p1->interval_ktime = ms_to_ktime(req_u->req3.tp_retire_blk_tov); 654 else 655 p1->interval_ktime = ms_to_ktime(prb_calc_retire_blk_tmo(po, 656 req_u->req3.tp_block_size)); 657 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv; 658 rwlock_init(&p1->blk_fill_in_prog_lock); 659 660 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv); 661 prb_init_ft_ops(p1, req_u); 662 hrtimer_setup(&p1->retire_blk_timer, prb_retire_rx_blk_timer_expired, 663 CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); 664 hrtimer_start(&p1->retire_blk_timer, p1->interval_ktime, 665 HRTIMER_MODE_REL_SOFT); 666 prb_open_block(p1, pbd); 667} 668 669/* 670 * With a 1MB block-size, on a 1Gbps line, it will take 671 * i) ~8 ms to fill a block + ii) memcpy etc. 672 * In this cut we are not accounting for the memcpy time. 673 * 674 * Since the tmo granularity is in msecs, it is not too expensive 675 * to refresh the timer, lets say every '8' msecs. 676 * Either the user can set the 'tmo' or we can derive it based on 677 * a) line-speed and b) block-size. 678 * prb_calc_retire_blk_tmo() calculates the tmo. 679 */ 680static enum hrtimer_restart prb_retire_rx_blk_timer_expired(struct hrtimer *t) 681{ 682 struct packet_sock *po = 683 timer_container_of(po, t, rx_ring.prb_bdqc.retire_blk_timer); 684 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring); 685 unsigned int frozen; 686 struct tpacket_block_desc *pbd; 687 688 spin_lock(&po->sk.sk_receive_queue.lock); 689 690 frozen = prb_queue_frozen(pkc); 691 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc); 692 693 /* We only need to plug the race when the block is partially filled. 694 * tpacket_rcv: 695 * lock(); increment BLOCK_NUM_PKTS; unlock() 696 * copy_bits() is in progress ... 697 * timer fires on other cpu: 698 * we can't retire the current block because copy_bits 699 * is in progress. 700 * 701 */ 702 if (BLOCK_NUM_PKTS(pbd)) { 703 /* Waiting for skb_copy_bits to finish... */ 704 write_lock(&pkc->blk_fill_in_prog_lock); 705 write_unlock(&pkc->blk_fill_in_prog_lock); 706 } 707 708 if (!frozen) { 709 if (BLOCK_NUM_PKTS(pbd)) { 710 /* Not an empty block. Need retire the block. */ 711 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO); 712 prb_dispatch_next_block(pkc, po); 713 } 714 } else { 715 /* Case 1. Queue was frozen because user-space was 716 * lagging behind. 717 */ 718 if (!prb_curr_blk_in_use(pbd)) { 719 /* Case 2. queue was frozen,user-space caught up, 720 * now the link went idle && the timer fired. 721 * We don't have a block to close.So we open this 722 * block and restart the timer. 723 * opening a block thaws the queue,restarts timer 724 * Thawing/timer-refresh is a side effect. 725 */ 726 prb_open_block(pkc, pbd); 727 } 728 } 729 730 hrtimer_forward_now(&pkc->retire_blk_timer, pkc->interval_ktime); 731 spin_unlock(&po->sk.sk_receive_queue.lock); 732 return HRTIMER_RESTART; 733} 734 735static void prb_flush_block(struct tpacket_kbdq_core *pkc1, 736 struct tpacket_block_desc *pbd1, __u32 status) 737{ 738 /* Flush everything minus the block header */ 739 740#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1 741 u8 *start, *end; 742 743 start = (u8 *)pbd1; 744 745 /* Skip the block header(we know header WILL fit in 4K) */ 746 start += PAGE_SIZE; 747 748 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end); 749 for (; start < end; start += PAGE_SIZE) 750 flush_dcache_page(pgv_to_page(start)); 751 752 smp_wmb(); 753#endif 754 755 /* Now update the block status. */ 756 757 BLOCK_STATUS(pbd1) = status; 758 759 /* Flush the block header */ 760 761#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1 762 start = (u8 *)pbd1; 763 flush_dcache_page(pgv_to_page(start)); 764 765 smp_wmb(); 766#endif 767} 768 769/* 770 * Side effect: 771 * 772 * 1) flush the block 773 * 2) Increment active_blk_num 774 * 775 * Note:We DONT refresh the timer on purpose. 776 * Because almost always the next block will be opened. 777 */ 778static void prb_close_block(struct tpacket_kbdq_core *pkc1, 779 struct tpacket_block_desc *pbd1, 780 struct packet_sock *po, unsigned int stat) 781{ 782 __u32 status = TP_STATUS_USER | stat; 783 784 struct tpacket3_hdr *last_pkt; 785 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1; 786 struct sock *sk = &po->sk; 787 788 if (atomic_read(&po->tp_drops)) 789 status |= TP_STATUS_LOSING; 790 791 last_pkt = (struct tpacket3_hdr *)pkc1->prev; 792 last_pkt->tp_next_offset = 0; 793 794 /* Get the ts of the last pkt */ 795 if (BLOCK_NUM_PKTS(pbd1)) { 796 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec; 797 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec; 798 } else { 799 /* Ok, we tmo'd - so get the current time. 800 * 801 * It shouldn't really happen as we don't close empty 802 * blocks. See prb_retire_rx_blk_timer_expired(). 803 */ 804 struct timespec64 ts; 805 ktime_get_real_ts64(&ts); 806 h1->ts_last_pkt.ts_sec = ts.tv_sec; 807 h1->ts_last_pkt.ts_nsec = ts.tv_nsec; 808 } 809 810 smp_wmb(); 811 812 /* Flush the block */ 813 prb_flush_block(pkc1, pbd1, status); 814 815 sk->sk_data_ready(sk); 816 817 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1); 818} 819 820static void prb_thaw_queue(struct tpacket_kbdq_core *pkc) 821{ 822 pkc->reset_pending_on_curr_blk = 0; 823} 824 825/* 826 * prb_open_block is called by tpacket_rcv or timer callback. 827 * 828 * Reasons why NOT update hrtimer in prb_open_block: 829 * 1) It will increase complexity to distinguish the two caller scenario. 830 * 2) hrtimer_cancel and hrtimer_start need to be called if you want to update 831 * TMO of an already enqueued hrtimer, leading to complex shutdown logic. 832 * 833 * One side effect of NOT update hrtimer when called by tpacket_rcv is that 834 * a newly opened block triggered by tpacket_rcv may be retired earlier than 835 * expected. On the other hand, if timeout is updated in prb_open_block, the 836 * frequent reception of network packets that leads to prb_open_block being 837 * called may cause hrtimer to be removed and enqueued repeatedly. 838 */ 839static void prb_open_block(struct tpacket_kbdq_core *pkc1, 840 struct tpacket_block_desc *pbd1) 841{ 842 struct timespec64 ts; 843 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1; 844 845 smp_rmb(); 846 847 /* We could have just memset this but we will lose the 848 * flexibility of making the priv area sticky 849 */ 850 851 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++; 852 BLOCK_NUM_PKTS(pbd1) = 0; 853 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv); 854 855 ktime_get_real_ts64(&ts); 856 857 h1->ts_first_pkt.ts_sec = ts.tv_sec; 858 h1->ts_first_pkt.ts_nsec = ts.tv_nsec; 859 860 pkc1->pkblk_start = (char *)pbd1; 861 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv); 862 863 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv); 864 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN; 865 866 pbd1->version = pkc1->version; 867 pkc1->prev = pkc1->nxt_offset; 868 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size; 869 870 prb_thaw_queue(pkc1); 871 872 smp_wmb(); 873} 874 875/* 876 * Queue freeze logic: 877 * 1) Assume tp_block_nr = 8 blocks. 878 * 2) At time 't0', user opens Rx ring. 879 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7 880 * 4) user-space is either sleeping or processing block '0'. 881 * 5) tpacket_rcv is currently filling block '7', since there is no space left, 882 * it will close block-7,loop around and try to fill block '0'. 883 * call-flow: 884 * __packet_lookup_frame_in_block 885 * prb_retire_current_block() 886 * prb_dispatch_next_block() 887 * |->(BLOCK_STATUS == USER) evaluates to true 888 * 5.1) Since block-0 is currently in-use, we just freeze the queue. 889 * 6) Now there are two cases: 890 * 6.1) Link goes idle right after the queue is frozen. 891 * But remember, the last open_block() refreshed the timer. 892 * When this timer expires,it will refresh itself so that we can 893 * re-open block-0 in near future. 894 * 6.2) Link is busy and keeps on receiving packets. This is a simple 895 * case and __packet_lookup_frame_in_block will check if block-0 896 * is free and can now be re-used. 897 */ 898static void prb_freeze_queue(struct tpacket_kbdq_core *pkc, 899 struct packet_sock *po) 900{ 901 pkc->reset_pending_on_curr_blk = 1; 902 po->stats.stats3.tp_freeze_q_cnt++; 903} 904 905#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT)) 906 907/* 908 * If the next block is free then we will dispatch it 909 * and return a good offset. 910 * Else, we will freeze the queue. 911 * So, caller must check the return value. 912 */ 913static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc, 914 struct packet_sock *po) 915{ 916 struct tpacket_block_desc *pbd; 917 918 smp_rmb(); 919 920 /* 1. Get current block num */ 921 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc); 922 923 /* 2. If this block is currently in_use then freeze the queue */ 924 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) { 925 prb_freeze_queue(pkc, po); 926 return NULL; 927 } 928 929 /* 930 * 3. 931 * open this block and return the offset where the first packet 932 * needs to get stored. 933 */ 934 prb_open_block(pkc, pbd); 935 return (void *)pkc->nxt_offset; 936} 937 938static void prb_retire_current_block(struct tpacket_kbdq_core *pkc, 939 struct packet_sock *po, unsigned int status) 940{ 941 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc); 942 943 /* retire/close the current block */ 944 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) { 945 /* 946 * Plug the case where copy_bits() is in progress on 947 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't 948 * have space to copy the pkt in the current block and 949 * called prb_retire_current_block() 950 * 951 * We don't need to worry about the TMO case because 952 * the timer-handler already handled this case. 953 */ 954 if (!(status & TP_STATUS_BLK_TMO)) { 955 /* Waiting for skb_copy_bits to finish... */ 956 write_lock(&pkc->blk_fill_in_prog_lock); 957 write_unlock(&pkc->blk_fill_in_prog_lock); 958 } 959 prb_close_block(pkc, pbd, po, status); 960 return; 961 } 962} 963 964static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd) 965{ 966 return TP_STATUS_USER & BLOCK_STATUS(pbd); 967} 968 969static int prb_queue_frozen(struct tpacket_kbdq_core *pkc) 970{ 971 return pkc->reset_pending_on_curr_blk; 972} 973 974static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb) 975 __releases(&pkc->blk_fill_in_prog_lock) 976{ 977 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb); 978 979 read_unlock(&pkc->blk_fill_in_prog_lock); 980} 981 982static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc, 983 struct tpacket3_hdr *ppd) 984{ 985 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb); 986} 987 988static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc, 989 struct tpacket3_hdr *ppd) 990{ 991 ppd->hv1.tp_rxhash = 0; 992} 993 994static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc, 995 struct tpacket3_hdr *ppd) 996{ 997 struct packet_sock *po = container_of(pkc, struct packet_sock, rx_ring.prb_bdqc); 998 999 if (skb_vlan_tag_present(pkc->skb)) { 1000 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb); 1001 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto); 1002 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; 1003 } else if (unlikely(po->sk.sk_type == SOCK_DGRAM && eth_type_vlan(pkc->skb->protocol))) { 1004 ppd->hv1.tp_vlan_tci = vlan_get_tci(pkc->skb, pkc->skb->dev); 1005 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->protocol); 1006 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; 1007 } else { 1008 ppd->hv1.tp_vlan_tci = 0; 1009 ppd->hv1.tp_vlan_tpid = 0; 1010 ppd->tp_status = TP_STATUS_AVAILABLE; 1011 } 1012} 1013 1014static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc, 1015 struct tpacket3_hdr *ppd) 1016{ 1017 ppd->hv1.tp_padding = 0; 1018 prb_fill_vlan_info(pkc, ppd); 1019 1020 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH) 1021 prb_fill_rxhash(pkc, ppd); 1022 else 1023 prb_clear_rxhash(pkc, ppd); 1024} 1025 1026static void prb_fill_curr_block(char *curr, 1027 struct tpacket_kbdq_core *pkc, 1028 struct tpacket_block_desc *pbd, 1029 unsigned int len) 1030 __acquires(&pkc->blk_fill_in_prog_lock) 1031{ 1032 struct tpacket3_hdr *ppd; 1033 1034 ppd = (struct tpacket3_hdr *)curr; 1035 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len); 1036 pkc->prev = curr; 1037 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len); 1038 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len); 1039 BLOCK_NUM_PKTS(pbd) += 1; 1040 read_lock(&pkc->blk_fill_in_prog_lock); 1041 prb_run_all_ft_ops(pkc, ppd); 1042} 1043 1044/* Assumes caller has the sk->rx_queue.lock */ 1045static void *__packet_lookup_frame_in_block(struct packet_sock *po, 1046 struct sk_buff *skb, 1047 unsigned int len 1048 ) 1049{ 1050 struct tpacket_kbdq_core *pkc; 1051 struct tpacket_block_desc *pbd; 1052 char *curr, *end; 1053 1054 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); 1055 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc); 1056 1057 /* Queue is frozen when user space is lagging behind */ 1058 if (prb_queue_frozen(pkc)) { 1059 /* 1060 * Check if that last block which caused the queue to freeze, 1061 * is still in_use by user-space. 1062 */ 1063 if (prb_curr_blk_in_use(pbd)) { 1064 /* Can't record this packet */ 1065 return NULL; 1066 } else { 1067 /* 1068 * Ok, the block was released by user-space. 1069 * Now let's open that block. 1070 * opening a block also thaws the queue. 1071 * Thawing is a side effect. 1072 */ 1073 prb_open_block(pkc, pbd); 1074 } 1075 } 1076 1077 smp_mb(); 1078 curr = pkc->nxt_offset; 1079 pkc->skb = skb; 1080 end = (char *)pbd + pkc->kblk_size; 1081 1082 /* first try the current block */ 1083 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) { 1084 prb_fill_curr_block(curr, pkc, pbd, len); 1085 return (void *)curr; 1086 } 1087 1088 /* Ok, close the current block */ 1089 prb_retire_current_block(pkc, po, 0); 1090 1091 /* Now, try to dispatch the next block */ 1092 curr = (char *)prb_dispatch_next_block(pkc, po); 1093 if (curr) { 1094 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc); 1095 prb_fill_curr_block(curr, pkc, pbd, len); 1096 return (void *)curr; 1097 } 1098 1099 /* 1100 * No free blocks are available.user_space hasn't caught up yet. 1101 * Queue was just frozen and now this packet will get dropped. 1102 */ 1103 return NULL; 1104} 1105 1106static void *packet_current_rx_frame(struct packet_sock *po, 1107 struct sk_buff *skb, 1108 int status, unsigned int len) 1109{ 1110 char *curr = NULL; 1111 switch (po->tp_version) { 1112 case TPACKET_V1: 1113 case TPACKET_V2: 1114 curr = packet_lookup_frame(po, &po->rx_ring, 1115 po->rx_ring.head, status); 1116 return curr; 1117 case TPACKET_V3: 1118 return __packet_lookup_frame_in_block(po, skb, len); 1119 default: 1120 WARN(1, "TPACKET version not supported\n"); 1121 BUG(); 1122 return NULL; 1123 } 1124} 1125 1126static void *prb_lookup_block(const struct packet_sock *po, 1127 const struct packet_ring_buffer *rb, 1128 unsigned int idx, 1129 int status) 1130{ 1131 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb); 1132 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx); 1133 1134 if (status != BLOCK_STATUS(pbd)) 1135 return NULL; 1136 return pbd; 1137} 1138 1139static int prb_previous_blk_num(struct packet_ring_buffer *rb) 1140{ 1141 unsigned int prev; 1142 if (rb->prb_bdqc.kactive_blk_num) 1143 prev = rb->prb_bdqc.kactive_blk_num-1; 1144 else 1145 prev = rb->prb_bdqc.knum_blocks-1; 1146 return prev; 1147} 1148 1149/* Assumes caller has held the rx_queue.lock */ 1150static void *__prb_previous_block(struct packet_sock *po, 1151 struct packet_ring_buffer *rb, 1152 int status) 1153{ 1154 unsigned int previous = prb_previous_blk_num(rb); 1155 return prb_lookup_block(po, rb, previous, status); 1156} 1157 1158static void *packet_previous_rx_frame(struct packet_sock *po, 1159 struct packet_ring_buffer *rb, 1160 int status) 1161{ 1162 if (po->tp_version <= TPACKET_V2) 1163 return packet_previous_frame(po, rb, status); 1164 1165 return __prb_previous_block(po, rb, status); 1166} 1167 1168static void packet_increment_rx_head(struct packet_sock *po, 1169 struct packet_ring_buffer *rb) 1170{ 1171 switch (po->tp_version) { 1172 case TPACKET_V1: 1173 case TPACKET_V2: 1174 return packet_increment_head(rb); 1175 case TPACKET_V3: 1176 default: 1177 WARN(1, "TPACKET version not supported.\n"); 1178 BUG(); 1179 return; 1180 } 1181} 1182 1183static void *packet_previous_frame(struct packet_sock *po, 1184 struct packet_ring_buffer *rb, 1185 int status) 1186{ 1187 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max; 1188 return packet_lookup_frame(po, rb, previous, status); 1189} 1190 1191static void packet_increment_head(struct packet_ring_buffer *buff) 1192{ 1193 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0; 1194} 1195 1196static void packet_inc_pending(struct packet_ring_buffer *rb) 1197{ 1198 this_cpu_inc(*rb->pending_refcnt); 1199} 1200 1201static void packet_dec_pending(struct packet_ring_buffer *rb) 1202{ 1203 this_cpu_dec(*rb->pending_refcnt); 1204} 1205 1206static unsigned int packet_read_pending(const struct packet_ring_buffer *rb) 1207{ 1208 unsigned int refcnt = 0; 1209 int cpu; 1210 1211 /* We don't use pending refcount in rx_ring. */ 1212 if (rb->pending_refcnt == NULL) 1213 return 0; 1214 1215 for_each_possible_cpu(cpu) 1216 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu); 1217 1218 return refcnt; 1219} 1220 1221static int packet_alloc_pending(struct packet_sock *po) 1222{ 1223 po->rx_ring.pending_refcnt = NULL; 1224 1225 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int); 1226 if (unlikely(po->tx_ring.pending_refcnt == NULL)) 1227 return -ENOBUFS; 1228 1229 return 0; 1230} 1231 1232static void packet_free_pending(struct packet_sock *po) 1233{ 1234 free_percpu(po->tx_ring.pending_refcnt); 1235} 1236 1237#define ROOM_POW_OFF 2 1238#define ROOM_NONE 0x0 1239#define ROOM_LOW 0x1 1240#define ROOM_NORMAL 0x2 1241 1242static bool __tpacket_has_room(const struct packet_sock *po, int pow_off) 1243{ 1244 int idx, len; 1245 1246 len = READ_ONCE(po->rx_ring.frame_max) + 1; 1247 idx = READ_ONCE(po->rx_ring.head); 1248 if (pow_off) 1249 idx += len >> pow_off; 1250 if (idx >= len) 1251 idx -= len; 1252 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL); 1253} 1254 1255static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off) 1256{ 1257 int idx, len; 1258 1259 len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks); 1260 idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num); 1261 if (pow_off) 1262 idx += len >> pow_off; 1263 if (idx >= len) 1264 idx -= len; 1265 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL); 1266} 1267 1268static int __packet_rcv_has_room(const struct packet_sock *po, 1269 const struct sk_buff *skb) 1270{ 1271 const struct sock *sk = &po->sk; 1272 int ret = ROOM_NONE; 1273 1274 if (po->prot_hook.func != tpacket_rcv) { 1275 int rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1276 int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc) 1277 - (skb ? skb->truesize : 0); 1278 1279 if (avail > (rcvbuf >> ROOM_POW_OFF)) 1280 return ROOM_NORMAL; 1281 else if (avail > 0) 1282 return ROOM_LOW; 1283 else 1284 return ROOM_NONE; 1285 } 1286 1287 if (po->tp_version == TPACKET_V3) { 1288 if (__tpacket_v3_has_room(po, ROOM_POW_OFF)) 1289 ret = ROOM_NORMAL; 1290 else if (__tpacket_v3_has_room(po, 0)) 1291 ret = ROOM_LOW; 1292 } else { 1293 if (__tpacket_has_room(po, ROOM_POW_OFF)) 1294 ret = ROOM_NORMAL; 1295 else if (__tpacket_has_room(po, 0)) 1296 ret = ROOM_LOW; 1297 } 1298 1299 return ret; 1300} 1301 1302static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) 1303{ 1304 bool pressure; 1305 int ret; 1306 1307 ret = __packet_rcv_has_room(po, skb); 1308 pressure = ret != ROOM_NORMAL; 1309 1310 if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) != pressure) 1311 packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, pressure); 1312 1313 return ret; 1314} 1315 1316static void packet_rcv_try_clear_pressure(struct packet_sock *po) 1317{ 1318 if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) && 1319 __packet_rcv_has_room(po, NULL) == ROOM_NORMAL) 1320 packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false); 1321} 1322 1323static void packet_sock_destruct(struct sock *sk) 1324{ 1325 skb_queue_purge(&sk->sk_error_queue); 1326 1327 WARN_ON(atomic_read(&sk->sk_rmem_alloc)); 1328 WARN_ON(refcount_read(&sk->sk_wmem_alloc)); 1329 1330 if (!sock_flag(sk, SOCK_DEAD)) { 1331 pr_err("Attempt to release alive packet socket: %p\n", sk); 1332 return; 1333 } 1334} 1335 1336static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb) 1337{ 1338 u32 *history = po->rollover->history; 1339 u32 victim, rxhash; 1340 int i, count = 0; 1341 1342 rxhash = skb_get_hash(skb); 1343 for (i = 0; i < ROLLOVER_HLEN; i++) 1344 if (READ_ONCE(history[i]) == rxhash) 1345 count++; 1346 1347 victim = get_random_u32_below(ROLLOVER_HLEN); 1348 1349 /* Avoid dirtying the cache line if possible */ 1350 if (READ_ONCE(history[victim]) != rxhash) 1351 WRITE_ONCE(history[victim], rxhash); 1352 1353 return count > (ROLLOVER_HLEN >> 1); 1354} 1355 1356static unsigned int fanout_demux_hash(struct packet_fanout *f, 1357 struct sk_buff *skb, 1358 unsigned int num) 1359{ 1360 return reciprocal_scale(__skb_get_hash_symmetric(skb), num); 1361} 1362 1363static unsigned int fanout_demux_lb(struct packet_fanout *f, 1364 struct sk_buff *skb, 1365 unsigned int num) 1366{ 1367 unsigned int val = atomic_inc_return(&f->rr_cur); 1368 1369 return val % num; 1370} 1371 1372static unsigned int fanout_demux_cpu(struct packet_fanout *f, 1373 struct sk_buff *skb, 1374 unsigned int num) 1375{ 1376 return smp_processor_id() % num; 1377} 1378 1379static unsigned int fanout_demux_rnd(struct packet_fanout *f, 1380 struct sk_buff *skb, 1381 unsigned int num) 1382{ 1383 return get_random_u32_below(num); 1384} 1385 1386static unsigned int fanout_demux_rollover(struct packet_fanout *f, 1387 struct sk_buff *skb, 1388 unsigned int idx, bool try_self, 1389 unsigned int num) 1390{ 1391 struct packet_sock *po, *po_next, *po_skip = NULL; 1392 unsigned int i, j, room = ROOM_NONE; 1393 1394 po = pkt_sk(rcu_dereference(f->arr[idx])); 1395 1396 if (try_self) { 1397 room = packet_rcv_has_room(po, skb); 1398 if (room == ROOM_NORMAL || 1399 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb))) 1400 return idx; 1401 po_skip = po; 1402 } 1403 1404 i = j = min_t(int, po->rollover->sock, num - 1); 1405 do { 1406 po_next = pkt_sk(rcu_dereference(f->arr[i])); 1407 if (po_next != po_skip && 1408 !packet_sock_flag(po_next, PACKET_SOCK_PRESSURE) && 1409 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) { 1410 if (i != j) 1411 po->rollover->sock = i; 1412 atomic_long_inc(&po->rollover->num); 1413 if (room == ROOM_LOW) 1414 atomic_long_inc(&po->rollover->num_huge); 1415 return i; 1416 } 1417 1418 if (++i == num) 1419 i = 0; 1420 } while (i != j); 1421 1422 atomic_long_inc(&po->rollover->num_failed); 1423 return idx; 1424} 1425 1426static unsigned int fanout_demux_qm(struct packet_fanout *f, 1427 struct sk_buff *skb, 1428 unsigned int num) 1429{ 1430 return skb_get_queue_mapping(skb) % num; 1431} 1432 1433static unsigned int fanout_demux_bpf(struct packet_fanout *f, 1434 struct sk_buff *skb, 1435 unsigned int num) 1436{ 1437 struct bpf_prog *prog; 1438 unsigned int ret = 0; 1439 1440 rcu_read_lock(); 1441 prog = rcu_dereference(f->bpf_prog); 1442 if (prog) 1443 ret = bpf_prog_run_clear_cb(prog, skb) % num; 1444 rcu_read_unlock(); 1445 1446 return ret; 1447} 1448 1449static bool fanout_has_flag(struct packet_fanout *f, u16 flag) 1450{ 1451 return f->flags & (flag >> 8); 1452} 1453 1454static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev, 1455 struct packet_type *pt, struct net_device *orig_dev) 1456{ 1457 struct packet_fanout *f = pt->af_packet_priv; 1458 unsigned int num = READ_ONCE(f->num_members); 1459 struct net *net = read_pnet(&f->net); 1460 struct packet_sock *po; 1461 unsigned int idx; 1462 1463 if (!net_eq(dev_net(dev), net) || !num) { 1464 kfree_skb(skb); 1465 return 0; 1466 } 1467 1468 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) { 1469 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET); 1470 if (!skb) 1471 return 0; 1472 } 1473 switch (f->type) { 1474 case PACKET_FANOUT_HASH: 1475 default: 1476 idx = fanout_demux_hash(f, skb, num); 1477 break; 1478 case PACKET_FANOUT_LB: 1479 idx = fanout_demux_lb(f, skb, num); 1480 break; 1481 case PACKET_FANOUT_CPU: 1482 idx = fanout_demux_cpu(f, skb, num); 1483 break; 1484 case PACKET_FANOUT_RND: 1485 idx = fanout_demux_rnd(f, skb, num); 1486 break; 1487 case PACKET_FANOUT_QM: 1488 idx = fanout_demux_qm(f, skb, num); 1489 break; 1490 case PACKET_FANOUT_ROLLOVER: 1491 idx = fanout_demux_rollover(f, skb, 0, false, num); 1492 break; 1493 case PACKET_FANOUT_CBPF: 1494 case PACKET_FANOUT_EBPF: 1495 idx = fanout_demux_bpf(f, skb, num); 1496 break; 1497 } 1498 1499 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER)) 1500 idx = fanout_demux_rollover(f, skb, idx, true, num); 1501 1502 po = pkt_sk(rcu_dereference(f->arr[idx])); 1503 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); 1504} 1505 1506DEFINE_MUTEX(fanout_mutex); 1507EXPORT_SYMBOL_GPL(fanout_mutex); 1508static LIST_HEAD(fanout_list); 1509static u16 fanout_next_id; 1510 1511static void __fanout_link(struct sock *sk, struct packet_sock *po) 1512{ 1513 struct packet_fanout *f = po->fanout; 1514 1515 spin_lock(&f->lock); 1516 rcu_assign_pointer(f->arr[f->num_members], sk); 1517 smp_wmb(); 1518 f->num_members++; 1519 if (f->num_members == 1) 1520 dev_add_pack(&f->prot_hook); 1521 spin_unlock(&f->lock); 1522} 1523 1524static void __fanout_unlink(struct sock *sk, struct packet_sock *po) 1525{ 1526 struct packet_fanout *f = po->fanout; 1527 int i; 1528 1529 spin_lock(&f->lock); 1530 for (i = 0; i < f->num_members; i++) { 1531 if (rcu_dereference_protected(f->arr[i], 1532 lockdep_is_held(&f->lock)) == sk) 1533 break; 1534 } 1535 BUG_ON(i >= f->num_members); 1536 rcu_assign_pointer(f->arr[i], 1537 rcu_dereference_protected(f->arr[f->num_members - 1], 1538 lockdep_is_held(&f->lock))); 1539 f->num_members--; 1540 if (f->num_members == 0) 1541 __dev_remove_pack(&f->prot_hook); 1542 spin_unlock(&f->lock); 1543} 1544 1545static bool match_fanout_group(struct packet_type *ptype, struct sock *sk) 1546{ 1547 if (sk->sk_family != PF_PACKET) 1548 return false; 1549 1550 return ptype->af_packet_priv == pkt_sk(sk)->fanout; 1551} 1552 1553static void fanout_init_data(struct packet_fanout *f) 1554{ 1555 switch (f->type) { 1556 case PACKET_FANOUT_LB: 1557 atomic_set(&f->rr_cur, 0); 1558 break; 1559 case PACKET_FANOUT_CBPF: 1560 case PACKET_FANOUT_EBPF: 1561 RCU_INIT_POINTER(f->bpf_prog, NULL); 1562 break; 1563 } 1564} 1565 1566static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new) 1567{ 1568 struct bpf_prog *old; 1569 1570 spin_lock(&f->lock); 1571 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock)); 1572 rcu_assign_pointer(f->bpf_prog, new); 1573 spin_unlock(&f->lock); 1574 1575 if (old) { 1576 synchronize_net(); 1577 bpf_prog_destroy(old); 1578 } 1579} 1580 1581static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data, 1582 unsigned int len) 1583{ 1584 struct bpf_prog *new; 1585 struct sock_fprog fprog; 1586 int ret; 1587 1588 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) 1589 return -EPERM; 1590 1591 ret = copy_bpf_fprog_from_user(&fprog, data, len); 1592 if (ret) 1593 return ret; 1594 1595 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false); 1596 if (ret) 1597 return ret; 1598 1599 __fanout_set_data_bpf(po->fanout, new); 1600 return 0; 1601} 1602 1603static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data, 1604 unsigned int len) 1605{ 1606 struct bpf_prog *new; 1607 u32 fd; 1608 1609 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED)) 1610 return -EPERM; 1611 if (len != sizeof(fd)) 1612 return -EINVAL; 1613 if (copy_from_sockptr(&fd, data, len)) 1614 return -EFAULT; 1615 1616 new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER); 1617 if (IS_ERR(new)) 1618 return PTR_ERR(new); 1619 1620 __fanout_set_data_bpf(po->fanout, new); 1621 return 0; 1622} 1623 1624static int fanout_set_data(struct packet_sock *po, sockptr_t data, 1625 unsigned int len) 1626{ 1627 switch (po->fanout->type) { 1628 case PACKET_FANOUT_CBPF: 1629 return fanout_set_data_cbpf(po, data, len); 1630 case PACKET_FANOUT_EBPF: 1631 return fanout_set_data_ebpf(po, data, len); 1632 default: 1633 return -EINVAL; 1634 } 1635} 1636 1637static void fanout_release_data(struct packet_fanout *f) 1638{ 1639 switch (f->type) { 1640 case PACKET_FANOUT_CBPF: 1641 case PACKET_FANOUT_EBPF: 1642 __fanout_set_data_bpf(f, NULL); 1643 } 1644} 1645 1646static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id) 1647{ 1648 struct packet_fanout *f; 1649 1650 list_for_each_entry(f, &fanout_list, list) { 1651 if (f->id == candidate_id && 1652 read_pnet(&f->net) == sock_net(sk)) { 1653 return false; 1654 } 1655 } 1656 return true; 1657} 1658 1659static bool fanout_find_new_id(struct sock *sk, u16 *new_id) 1660{ 1661 u16 id = fanout_next_id; 1662 1663 do { 1664 if (__fanout_id_is_free(sk, id)) { 1665 *new_id = id; 1666 fanout_next_id = id + 1; 1667 return true; 1668 } 1669 1670 id++; 1671 } while (id != fanout_next_id); 1672 1673 return false; 1674} 1675 1676static int fanout_add(struct sock *sk, struct fanout_args *args) 1677{ 1678 struct packet_rollover *rollover = NULL; 1679 struct packet_sock *po = pkt_sk(sk); 1680 u16 type_flags = args->type_flags; 1681 struct packet_fanout *f, *match; 1682 u8 type = type_flags & 0xff; 1683 u8 flags = type_flags >> 8; 1684 u16 id = args->id; 1685 int err; 1686 1687 switch (type) { 1688 case PACKET_FANOUT_ROLLOVER: 1689 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER) 1690 return -EINVAL; 1691 break; 1692 case PACKET_FANOUT_HASH: 1693 case PACKET_FANOUT_LB: 1694 case PACKET_FANOUT_CPU: 1695 case PACKET_FANOUT_RND: 1696 case PACKET_FANOUT_QM: 1697 case PACKET_FANOUT_CBPF: 1698 case PACKET_FANOUT_EBPF: 1699 break; 1700 default: 1701 return -EINVAL; 1702 } 1703 1704 mutex_lock(&fanout_mutex); 1705 1706 err = -EALREADY; 1707 if (po->fanout) 1708 goto out; 1709 1710 if (type == PACKET_FANOUT_ROLLOVER || 1711 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) { 1712 err = -ENOMEM; 1713 rollover = kzalloc(sizeof(*rollover), GFP_KERNEL); 1714 if (!rollover) 1715 goto out; 1716 atomic_long_set(&rollover->num, 0); 1717 atomic_long_set(&rollover->num_huge, 0); 1718 atomic_long_set(&rollover->num_failed, 0); 1719 } 1720 1721 if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) { 1722 if (id != 0) { 1723 err = -EINVAL; 1724 goto out; 1725 } 1726 if (!fanout_find_new_id(sk, &id)) { 1727 err = -ENOMEM; 1728 goto out; 1729 } 1730 /* ephemeral flag for the first socket in the group: drop it */ 1731 flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8); 1732 } 1733 1734 match = NULL; 1735 list_for_each_entry(f, &fanout_list, list) { 1736 if (f->id == id && 1737 read_pnet(&f->net) == sock_net(sk)) { 1738 match = f; 1739 break; 1740 } 1741 } 1742 err = -EINVAL; 1743 if (match) { 1744 if (match->flags != flags) 1745 goto out; 1746 if (args->max_num_members && 1747 args->max_num_members != match->max_num_members) 1748 goto out; 1749 } else { 1750 if (args->max_num_members > PACKET_FANOUT_MAX) 1751 goto out; 1752 if (!args->max_num_members) 1753 /* legacy PACKET_FANOUT_MAX */ 1754 args->max_num_members = 256; 1755 err = -ENOMEM; 1756 match = kvzalloc(struct_size(match, arr, args->max_num_members), 1757 GFP_KERNEL); 1758 if (!match) 1759 goto out; 1760 write_pnet(&match->net, sock_net(sk)); 1761 match->id = id; 1762 match->type = type; 1763 match->flags = flags; 1764 INIT_LIST_HEAD(&match->list); 1765 spin_lock_init(&match->lock); 1766 refcount_set(&match->sk_ref, 0); 1767 fanout_init_data(match); 1768 match->prot_hook.type = po->prot_hook.type; 1769 match->prot_hook.dev = po->prot_hook.dev; 1770 match->prot_hook.func = packet_rcv_fanout; 1771 match->prot_hook.af_packet_priv = match; 1772 match->prot_hook.af_packet_net = read_pnet(&match->net); 1773 match->prot_hook.id_match = match_fanout_group; 1774 match->max_num_members = args->max_num_members; 1775 match->prot_hook.ignore_outgoing = type_flags & PACKET_FANOUT_FLAG_IGNORE_OUTGOING; 1776 list_add(&match->list, &fanout_list); 1777 } 1778 err = -EINVAL; 1779 1780 spin_lock(&po->bind_lock); 1781 if (po->num && 1782 match->type == type && 1783 match->prot_hook.type == po->prot_hook.type && 1784 match->prot_hook.dev == po->prot_hook.dev) { 1785 err = -ENOSPC; 1786 if (refcount_read(&match->sk_ref) < match->max_num_members) { 1787 /* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */ 1788 WRITE_ONCE(po->fanout, match); 1789 1790 po->rollover = rollover; 1791 rollover = NULL; 1792 refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1); 1793 if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) { 1794 __dev_remove_pack(&po->prot_hook); 1795 __fanout_link(sk, po); 1796 } 1797 err = 0; 1798 } 1799 } 1800 spin_unlock(&po->bind_lock); 1801 1802 if (err && !refcount_read(&match->sk_ref)) { 1803 list_del(&match->list); 1804 kvfree(match); 1805 } 1806 1807out: 1808 kfree(rollover); 1809 mutex_unlock(&fanout_mutex); 1810 return err; 1811} 1812 1813/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes 1814 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout. 1815 * It is the responsibility of the caller to call fanout_release_data() and 1816 * free the returned packet_fanout (after synchronize_net()) 1817 */ 1818static struct packet_fanout *fanout_release(struct sock *sk) 1819{ 1820 struct packet_sock *po = pkt_sk(sk); 1821 struct packet_fanout *f; 1822 1823 mutex_lock(&fanout_mutex); 1824 f = po->fanout; 1825 if (f) { 1826 po->fanout = NULL; 1827 1828 if (refcount_dec_and_test(&f->sk_ref)) 1829 list_del(&f->list); 1830 else 1831 f = NULL; 1832 } 1833 mutex_unlock(&fanout_mutex); 1834 1835 return f; 1836} 1837 1838static bool packet_extra_vlan_len_allowed(const struct net_device *dev, 1839 struct sk_buff *skb) 1840{ 1841 /* Earlier code assumed this would be a VLAN pkt, double-check 1842 * this now that we have the actual packet in hand. We can only 1843 * do this check on Ethernet devices. 1844 */ 1845 if (unlikely(dev->type != ARPHRD_ETHER)) 1846 return false; 1847 1848 skb_reset_mac_header(skb); 1849 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q)); 1850} 1851 1852static const struct proto_ops packet_ops; 1853 1854static const struct proto_ops packet_ops_spkt; 1855 1856static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev, 1857 struct packet_type *pt, struct net_device *orig_dev) 1858{ 1859 struct sock *sk; 1860 struct sockaddr_pkt *spkt; 1861 1862 /* 1863 * When we registered the protocol we saved the socket in the data 1864 * field for just this event. 1865 */ 1866 1867 sk = pt->af_packet_priv; 1868 1869 /* 1870 * Yank back the headers [hope the device set this 1871 * right or kerboom...] 1872 * 1873 * Incoming packets have ll header pulled, 1874 * push it back. 1875 * 1876 * For outgoing ones skb->data == skb_mac_header(skb) 1877 * so that this procedure is noop. 1878 */ 1879 1880 if (skb->pkt_type == PACKET_LOOPBACK) 1881 goto out; 1882 1883 if (!net_eq(dev_net(dev), sock_net(sk))) 1884 goto out; 1885 1886 skb = skb_share_check(skb, GFP_ATOMIC); 1887 if (skb == NULL) 1888 goto oom; 1889 1890 /* drop any routing info */ 1891 skb_dst_drop(skb); 1892 1893 /* drop conntrack reference */ 1894 nf_reset_ct(skb); 1895 1896 spkt = &PACKET_SKB_CB(skb)->sa.pkt; 1897 1898 skb_push(skb, skb->data - skb_mac_header(skb)); 1899 1900 /* 1901 * The SOCK_PACKET socket receives _all_ frames. 1902 */ 1903 1904 spkt->spkt_family = dev->type; 1905 strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device)); 1906 spkt->spkt_protocol = skb->protocol; 1907 1908 /* 1909 * Charge the memory to the socket. This is done specifically 1910 * to prevent sockets using all the memory up. 1911 */ 1912 1913 if (sock_queue_rcv_skb(sk, skb) == 0) 1914 return 0; 1915 1916out: 1917 kfree_skb(skb); 1918oom: 1919 return 0; 1920} 1921 1922static void packet_parse_headers(struct sk_buff *skb, struct socket *sock) 1923{ 1924 int depth; 1925 1926 if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) && 1927 sock->type == SOCK_RAW) { 1928 skb_reset_mac_header(skb); 1929 skb->protocol = dev_parse_header_protocol(skb); 1930 } 1931 1932 /* Move network header to the right position for VLAN tagged packets */ 1933 if (likely(skb->dev->type == ARPHRD_ETHER) && 1934 eth_type_vlan(skb->protocol) && 1935 vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) 1936 skb_set_network_header(skb, depth); 1937 1938 skb_probe_transport_header(skb); 1939} 1940 1941/* 1942 * Output a raw packet to a device layer. This bypasses all the other 1943 * protocol layers and you must therefore supply it with a complete frame 1944 */ 1945 1946static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg, 1947 size_t len) 1948{ 1949 struct sock *sk = sock->sk; 1950 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name); 1951 struct sk_buff *skb = NULL; 1952 struct net_device *dev; 1953 struct sockcm_cookie sockc; 1954 __be16 proto = 0; 1955 int err; 1956 int extra_len = 0; 1957 1958 /* 1959 * Get and verify the address. 1960 */ 1961 1962 if (saddr) { 1963 if (msg->msg_namelen < sizeof(struct sockaddr)) 1964 return -EINVAL; 1965 if (msg->msg_namelen == sizeof(struct sockaddr_pkt)) 1966 proto = saddr->spkt_protocol; 1967 } else 1968 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */ 1969 1970 /* 1971 * Find the device first to size check it 1972 */ 1973 1974 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0; 1975retry: 1976 rcu_read_lock(); 1977 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device); 1978 err = -ENODEV; 1979 if (dev == NULL) 1980 goto out_unlock; 1981 1982 err = -ENETDOWN; 1983 if (!(dev->flags & IFF_UP)) 1984 goto out_unlock; 1985 1986 /* 1987 * You may not queue a frame bigger than the mtu. This is the lowest level 1988 * raw protocol and you must do your own fragmentation at this level. 1989 */ 1990 1991 if (unlikely(sock_flag(sk, SOCK_NOFCS))) { 1992 if (!netif_supports_nofcs(dev)) { 1993 err = -EPROTONOSUPPORT; 1994 goto out_unlock; 1995 } 1996 extra_len = 4; /* We're doing our own CRC */ 1997 } 1998 1999 err = -EMSGSIZE; 2000 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len) 2001 goto out_unlock; 2002 2003 if (!skb) { 2004 size_t reserved = LL_RESERVED_SPACE(dev); 2005 int tlen = dev->needed_tailroom; 2006 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0; 2007 2008 rcu_read_unlock(); 2009 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL); 2010 if (skb == NULL) 2011 return -ENOBUFS; 2012 /* FIXME: Save some space for broken drivers that write a hard 2013 * header at transmission time by themselves. PPP is the notable 2014 * one here. This should really be fixed at the driver level. 2015 */ 2016 skb_reserve(skb, reserved); 2017 skb_reset_network_header(skb); 2018 2019 /* Try to align data part correctly */ 2020 if (hhlen) { 2021 skb->data -= hhlen; 2022 skb->tail -= hhlen; 2023 if (len < hhlen) 2024 skb_reset_network_header(skb); 2025 } 2026 err = memcpy_from_msg(skb_put(skb, len), msg, len); 2027 if (err) 2028 goto out_free; 2029 goto retry; 2030 } 2031 2032 if (!dev_validate_header(dev, skb->data, len) || !skb->len) { 2033 err = -EINVAL; 2034 goto out_unlock; 2035 } 2036 if (len > (dev->mtu + dev->hard_header_len + extra_len) && 2037 !packet_extra_vlan_len_allowed(dev, skb)) { 2038 err = -EMSGSIZE; 2039 goto out_unlock; 2040 } 2041 2042 sockcm_init(&sockc, sk); 2043 if (msg->msg_controllen) { 2044 err = sock_cmsg_send(sk, msg, &sockc); 2045 if (unlikely(err)) 2046 goto out_unlock; 2047 } 2048 2049 skb->protocol = proto; 2050 skb->dev = dev; 2051 skb->priority = sockc.priority; 2052 skb->mark = sockc.mark; 2053 skb_set_delivery_type_by_clockid(skb, sockc.transmit_time, sk->sk_clockid); 2054 skb_setup_tx_timestamp(skb, &sockc); 2055 2056 if (unlikely(extra_len == 4)) 2057 skb->no_fcs = 1; 2058 2059 packet_parse_headers(skb, sock); 2060 2061 dev_queue_xmit(skb); 2062 rcu_read_unlock(); 2063 return len; 2064 2065out_unlock: 2066 rcu_read_unlock(); 2067out_free: 2068 kfree_skb(skb); 2069 return err; 2070} 2071 2072static unsigned int run_filter(struct sk_buff *skb, 2073 const struct sock *sk, 2074 unsigned int res) 2075{ 2076 struct sk_filter *filter; 2077 2078 rcu_read_lock(); 2079 filter = rcu_dereference(sk->sk_filter); 2080 if (filter != NULL) 2081 res = bpf_prog_run_clear_cb(filter->prog, skb); 2082 rcu_read_unlock(); 2083 2084 return res; 2085} 2086 2087static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb, 2088 size_t *len, int vnet_hdr_sz) 2089{ 2090 struct virtio_net_hdr_mrg_rxbuf vnet_hdr = { .num_buffers = 0 }; 2091 2092 if (*len < vnet_hdr_sz) 2093 return -EINVAL; 2094 *len -= vnet_hdr_sz; 2095 2096 if (virtio_net_hdr_from_skb(skb, (struct virtio_net_hdr *)&vnet_hdr, vio_le(), true, 0)) 2097 return -EINVAL; 2098 2099 return memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_sz); 2100} 2101 2102/* 2103 * This function makes lazy skb cloning in hope that most of packets 2104 * are discarded by BPF. 2105 * 2106 * Note tricky part: we DO mangle shared skb! skb->data, skb->len 2107 * and skb->cb are mangled. It works because (and until) packets 2108 * falling here are owned by current CPU. Output packets are cloned 2109 * by dev_queue_xmit_nit(), input packets are processed by net_bh 2110 * sequentially, so that if we return skb to original state on exit, 2111 * we will not harm anyone. 2112 */ 2113 2114static int packet_rcv(struct sk_buff *skb, struct net_device *dev, 2115 struct packet_type *pt, struct net_device *orig_dev) 2116{ 2117 enum skb_drop_reason drop_reason = SKB_CONSUMED; 2118 struct sock *sk = NULL; 2119 struct sockaddr_ll *sll; 2120 struct packet_sock *po; 2121 u8 *skb_head = skb->data; 2122 int skb_len = skb->len; 2123 unsigned int snaplen, res; 2124 2125 if (skb->pkt_type == PACKET_LOOPBACK) 2126 goto drop; 2127 2128 sk = pt->af_packet_priv; 2129 po = pkt_sk(sk); 2130 2131 if (!net_eq(dev_net(dev), sock_net(sk))) 2132 goto drop; 2133 2134 skb->dev = dev; 2135 2136 if (dev_has_header(dev)) { 2137 /* The device has an explicit notion of ll header, 2138 * exported to higher levels. 2139 * 2140 * Otherwise, the device hides details of its frame 2141 * structure, so that corresponding packet head is 2142 * never delivered to user. 2143 */ 2144 if (sk->sk_type != SOCK_DGRAM) 2145 skb_push(skb, skb->data - skb_mac_header(skb)); 2146 else if (skb->pkt_type == PACKET_OUTGOING) { 2147 /* Special case: outgoing packets have ll header at head */ 2148 skb_pull(skb, skb_network_offset(skb)); 2149 } 2150 } 2151 2152 snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb); 2153 2154 res = run_filter(skb, sk, snaplen); 2155 if (!res) 2156 goto drop_n_restore; 2157 if (snaplen > res) 2158 snaplen = res; 2159 2160 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) 2161 goto drop_n_acct; 2162 2163 if (skb_shared(skb)) { 2164 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 2165 if (nskb == NULL) 2166 goto drop_n_acct; 2167 2168 if (skb_head != skb->data) { 2169 skb->data = skb_head; 2170 skb->len = skb_len; 2171 } 2172 consume_skb(skb); 2173 skb = nskb; 2174 } 2175 2176 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8); 2177 2178 sll = &PACKET_SKB_CB(skb)->sa.ll; 2179 sll->sll_hatype = dev->type; 2180 sll->sll_pkttype = skb->pkt_type; 2181 if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV))) 2182 sll->sll_ifindex = orig_dev->ifindex; 2183 else 2184 sll->sll_ifindex = dev->ifindex; 2185 2186 sll->sll_halen = dev_parse_header(skb, sll->sll_addr); 2187 2188 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg(). 2189 * Use their space for storing the original skb length. 2190 */ 2191 PACKET_SKB_CB(skb)->sa.origlen = skb->len; 2192 2193 if (pskb_trim(skb, snaplen)) 2194 goto drop_n_acct; 2195 2196 skb_set_owner_r(skb, sk); 2197 skb->dev = NULL; 2198 skb_dst_drop(skb); 2199 2200 /* drop conntrack reference */ 2201 nf_reset_ct(skb); 2202 2203 spin_lock(&sk->sk_receive_queue.lock); 2204 po->stats.stats1.tp_packets++; 2205 sock_skb_set_dropcount(sk, skb); 2206 skb_clear_delivery_time(skb); 2207 __skb_queue_tail(&sk->sk_receive_queue, skb); 2208 spin_unlock(&sk->sk_receive_queue.lock); 2209 sk->sk_data_ready(sk); 2210 return 0; 2211 2212drop_n_acct: 2213 atomic_inc(&po->tp_drops); 2214 sk_drops_inc(sk); 2215 drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR; 2216 2217drop_n_restore: 2218 if (skb_head != skb->data && skb_shared(skb)) { 2219 skb->data = skb_head; 2220 skb->len = skb_len; 2221 } 2222drop: 2223 sk_skb_reason_drop(sk, skb, drop_reason); 2224 return 0; 2225} 2226 2227static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, 2228 struct packet_type *pt, struct net_device *orig_dev) 2229{ 2230 enum skb_drop_reason drop_reason = SKB_CONSUMED; 2231 struct sock *sk = NULL; 2232 struct packet_sock *po; 2233 struct sockaddr_ll *sll; 2234 union tpacket_uhdr h; 2235 u8 *skb_head = skb->data; 2236 int skb_len = skb->len; 2237 unsigned int snaplen, res; 2238 unsigned long status = TP_STATUS_USER; 2239 unsigned short macoff, hdrlen; 2240 unsigned int netoff; 2241 struct sk_buff *copy_skb = NULL; 2242 struct timespec64 ts; 2243 __u32 ts_status; 2244 unsigned int slot_id = 0; 2245 int vnet_hdr_sz = 0; 2246 2247 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT. 2248 * We may add members to them until current aligned size without forcing 2249 * userspace to call getsockopt(..., PACKET_HDRLEN, ...). 2250 */ 2251 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32); 2252 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48); 2253 2254 if (skb->pkt_type == PACKET_LOOPBACK) 2255 goto drop; 2256 2257 sk = pt->af_packet_priv; 2258 po = pkt_sk(sk); 2259 2260 if (!net_eq(dev_net(dev), sock_net(sk))) 2261 goto drop; 2262 2263 if (dev_has_header(dev)) { 2264 if (sk->sk_type != SOCK_DGRAM) 2265 skb_push(skb, skb->data - skb_mac_header(skb)); 2266 else if (skb->pkt_type == PACKET_OUTGOING) { 2267 /* Special case: outgoing packets have ll header at head */ 2268 skb_pull(skb, skb_network_offset(skb)); 2269 } 2270 } 2271 2272 snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb); 2273 2274 res = run_filter(skb, sk, snaplen); 2275 if (!res) 2276 goto drop_n_restore; 2277 2278 /* If we are flooded, just give up */ 2279 if (__packet_rcv_has_room(po, skb) == ROOM_NONE) { 2280 atomic_inc(&po->tp_drops); 2281 goto drop_n_restore; 2282 } 2283 2284 if (skb->ip_summed == CHECKSUM_PARTIAL) 2285 status |= TP_STATUS_CSUMNOTREADY; 2286 else if (skb->pkt_type != PACKET_OUTGOING && 2287 skb_csum_unnecessary(skb)) 2288 status |= TP_STATUS_CSUM_VALID; 2289 if (skb_is_gso(skb) && skb_is_gso_tcp(skb)) 2290 status |= TP_STATUS_GSO_TCP; 2291 2292 if (snaplen > res) 2293 snaplen = res; 2294 2295 if (sk->sk_type == SOCK_DGRAM) { 2296 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + 2297 po->tp_reserve; 2298 } else { 2299 unsigned int maclen = skb_network_offset(skb); 2300 netoff = TPACKET_ALIGN(po->tp_hdrlen + 2301 (maclen < 16 ? 16 : maclen)) + 2302 po->tp_reserve; 2303 vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz); 2304 if (vnet_hdr_sz) 2305 netoff += vnet_hdr_sz; 2306 macoff = netoff - maclen; 2307 } 2308 if (netoff > USHRT_MAX) { 2309 atomic_inc(&po->tp_drops); 2310 goto drop_n_restore; 2311 } 2312 if (po->tp_version <= TPACKET_V2) { 2313 if (macoff + snaplen > po->rx_ring.frame_size) { 2314 if (READ_ONCE(po->copy_thresh) && 2315 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) { 2316 if (skb_shared(skb)) { 2317 copy_skb = skb_clone(skb, GFP_ATOMIC); 2318 } else { 2319 copy_skb = skb_get(skb); 2320 skb_head = skb->data; 2321 } 2322 if (copy_skb) { 2323 memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0, 2324 sizeof(PACKET_SKB_CB(copy_skb)->sa.ll)); 2325 skb_set_owner_r(copy_skb, sk); 2326 } 2327 } 2328 snaplen = po->rx_ring.frame_size - macoff; 2329 if ((int)snaplen < 0) { 2330 snaplen = 0; 2331 vnet_hdr_sz = 0; 2332 } 2333 } 2334 } else if (unlikely(macoff + snaplen > 2335 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) { 2336 u32 nval; 2337 2338 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff; 2339 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n", 2340 snaplen, nval, macoff); 2341 snaplen = nval; 2342 if (unlikely((int)snaplen < 0)) { 2343 snaplen = 0; 2344 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len; 2345 vnet_hdr_sz = 0; 2346 } 2347 } 2348 spin_lock(&sk->sk_receive_queue.lock); 2349 h.raw = packet_current_rx_frame(po, skb, 2350 TP_STATUS_KERNEL, (macoff+snaplen)); 2351 if (!h.raw) 2352 goto drop_n_account; 2353 2354 if (po->tp_version <= TPACKET_V2) { 2355 slot_id = po->rx_ring.head; 2356 if (test_bit(slot_id, po->rx_ring.rx_owner_map)) 2357 goto drop_n_account; 2358 __set_bit(slot_id, po->rx_ring.rx_owner_map); 2359 } 2360 2361 if (vnet_hdr_sz && 2362 virtio_net_hdr_from_skb(skb, h.raw + macoff - 2363 sizeof(struct virtio_net_hdr), 2364 vio_le(), true, 0)) { 2365 if (po->tp_version == TPACKET_V3) 2366 prb_clear_blk_fill_status(&po->rx_ring); 2367 goto drop_n_account; 2368 } 2369 2370 if (po->tp_version <= TPACKET_V2) { 2371 packet_increment_rx_head(po, &po->rx_ring); 2372 /* 2373 * LOSING will be reported till you read the stats, 2374 * because it's COR - Clear On Read. 2375 * Anyways, moving it for V1/V2 only as V3 doesn't need this 2376 * at packet level. 2377 */ 2378 if (atomic_read(&po->tp_drops)) 2379 status |= TP_STATUS_LOSING; 2380 } 2381 2382 po->stats.stats1.tp_packets++; 2383 if (copy_skb) { 2384 status |= TP_STATUS_COPY; 2385 skb_clear_delivery_time(copy_skb); 2386 __skb_queue_tail(&sk->sk_receive_queue, copy_skb); 2387 } 2388 spin_unlock(&sk->sk_receive_queue.lock); 2389 2390 skb_copy_bits(skb, 0, h.raw + macoff, snaplen); 2391 2392 /* Always timestamp; prefer an existing software timestamp taken 2393 * closer to the time of capture. 2394 */ 2395 ts_status = tpacket_get_timestamp(skb, &ts, 2396 READ_ONCE(po->tp_tstamp) | 2397 SOF_TIMESTAMPING_SOFTWARE); 2398 if (!ts_status) 2399 ktime_get_real_ts64(&ts); 2400 2401 status |= ts_status; 2402 2403 switch (po->tp_version) { 2404 case TPACKET_V1: 2405 h.h1->tp_len = skb->len; 2406 h.h1->tp_snaplen = snaplen; 2407 h.h1->tp_mac = macoff; 2408 h.h1->tp_net = netoff; 2409 h.h1->tp_sec = ts.tv_sec; 2410 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC; 2411 hdrlen = sizeof(*h.h1); 2412 break; 2413 case TPACKET_V2: 2414 h.h2->tp_len = skb->len; 2415 h.h2->tp_snaplen = snaplen; 2416 h.h2->tp_mac = macoff; 2417 h.h2->tp_net = netoff; 2418 h.h2->tp_sec = ts.tv_sec; 2419 h.h2->tp_nsec = ts.tv_nsec; 2420 if (skb_vlan_tag_present(skb)) { 2421 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb); 2422 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto); 2423 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; 2424 } else if (unlikely(sk->sk_type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) { 2425 h.h2->tp_vlan_tci = vlan_get_tci(skb, skb->dev); 2426 h.h2->tp_vlan_tpid = ntohs(skb->protocol); 2427 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; 2428 } else { 2429 h.h2->tp_vlan_tci = 0; 2430 h.h2->tp_vlan_tpid = 0; 2431 } 2432 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding)); 2433 hdrlen = sizeof(*h.h2); 2434 break; 2435 case TPACKET_V3: 2436 /* tp_nxt_offset,vlan are already populated above. 2437 * So DONT clear those fields here 2438 */ 2439 h.h3->tp_status |= status; 2440 h.h3->tp_len = skb->len; 2441 h.h3->tp_snaplen = snaplen; 2442 h.h3->tp_mac = macoff; 2443 h.h3->tp_net = netoff; 2444 h.h3->tp_sec = ts.tv_sec; 2445 h.h3->tp_nsec = ts.tv_nsec; 2446 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding)); 2447 hdrlen = sizeof(*h.h3); 2448 break; 2449 default: 2450 BUG(); 2451 } 2452 2453 sll = h.raw + TPACKET_ALIGN(hdrlen); 2454 sll->sll_halen = dev_parse_header(skb, sll->sll_addr); 2455 sll->sll_family = AF_PACKET; 2456 sll->sll_hatype = dev->type; 2457 sll->sll_protocol = (sk->sk_type == SOCK_DGRAM) ? 2458 vlan_get_protocol_dgram(skb) : skb->protocol; 2459 sll->sll_pkttype = skb->pkt_type; 2460 if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV))) 2461 sll->sll_ifindex = orig_dev->ifindex; 2462 else 2463 sll->sll_ifindex = dev->ifindex; 2464 2465 smp_mb(); 2466 2467#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1 2468 if (po->tp_version <= TPACKET_V2) { 2469 u8 *start, *end; 2470 2471 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw + 2472 macoff + snaplen); 2473 2474 for (start = h.raw; start < end; start += PAGE_SIZE) 2475 flush_dcache_page(pgv_to_page(start)); 2476 } 2477 smp_wmb(); 2478#endif 2479 2480 if (po->tp_version <= TPACKET_V2) { 2481 spin_lock(&sk->sk_receive_queue.lock); 2482 __packet_set_status(po, h.raw, status); 2483 __clear_bit(slot_id, po->rx_ring.rx_owner_map); 2484 spin_unlock(&sk->sk_receive_queue.lock); 2485 sk->sk_data_ready(sk); 2486 } else if (po->tp_version == TPACKET_V3) { 2487 prb_clear_blk_fill_status(&po->rx_ring); 2488 } 2489 2490drop_n_restore: 2491 if (skb_head != skb->data && skb_shared(skb)) { 2492 skb->data = skb_head; 2493 skb->len = skb_len; 2494 } 2495drop: 2496 sk_skb_reason_drop(sk, skb, drop_reason); 2497 return 0; 2498 2499drop_n_account: 2500 spin_unlock(&sk->sk_receive_queue.lock); 2501 atomic_inc(&po->tp_drops); 2502 drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR; 2503 2504 sk->sk_data_ready(sk); 2505 sk_skb_reason_drop(sk, copy_skb, drop_reason); 2506 goto drop_n_restore; 2507} 2508 2509static void tpacket_destruct_skb(struct sk_buff *skb) 2510{ 2511 struct packet_sock *po = pkt_sk(skb->sk); 2512 2513 if (likely(po->tx_ring.pg_vec)) { 2514 void *ph; 2515 __u32 ts; 2516 2517 ph = skb_zcopy_get_nouarg(skb); 2518 packet_dec_pending(&po->tx_ring); 2519 2520 ts = __packet_set_timestamp(po, ph, skb); 2521 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); 2522 2523 complete(&po->skb_completion); 2524 } 2525 2526 sock_wfree(skb); 2527} 2528 2529static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len) 2530{ 2531 if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 2532 (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) + 2533 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 > 2534 __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len))) 2535 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(), 2536 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) + 2537 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2); 2538 2539 if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len) 2540 return -EINVAL; 2541 2542 return 0; 2543} 2544 2545static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len, 2546 struct virtio_net_hdr *vnet_hdr, int vnet_hdr_sz) 2547{ 2548 int ret; 2549 2550 if (*len < vnet_hdr_sz) 2551 return -EINVAL; 2552 *len -= vnet_hdr_sz; 2553 2554 if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter)) 2555 return -EFAULT; 2556 2557 ret = __packet_snd_vnet_parse(vnet_hdr, *len); 2558 if (ret) 2559 return ret; 2560 2561 /* move iter to point to the start of mac header */ 2562 if (vnet_hdr_sz != sizeof(struct virtio_net_hdr)) 2563 iov_iter_advance(&msg->msg_iter, vnet_hdr_sz - sizeof(struct virtio_net_hdr)); 2564 2565 return 0; 2566} 2567 2568static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, 2569 void *frame, struct net_device *dev, void *data, int tp_len, 2570 __be16 proto, unsigned char *addr, int hlen, int copylen, 2571 const struct sockcm_cookie *sockc) 2572{ 2573 union tpacket_uhdr ph; 2574 int to_write, offset, len, nr_frags, len_max; 2575 struct socket *sock = po->sk.sk_socket; 2576 struct page *page; 2577 int err; 2578 2579 ph.raw = frame; 2580 2581 skb->protocol = proto; 2582 skb->dev = dev; 2583 skb->priority = sockc->priority; 2584 skb->mark = sockc->mark; 2585 skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid); 2586 skb_setup_tx_timestamp(skb, sockc); 2587 skb_zcopy_set_nouarg(skb, ph.raw); 2588 2589 skb_reserve(skb, hlen); 2590 skb_reset_network_header(skb); 2591 2592 to_write = tp_len; 2593 2594 if (sock->type == SOCK_DGRAM) { 2595 err = dev_hard_header(skb, dev, ntohs(proto), addr, 2596 NULL, tp_len); 2597 if (unlikely(err < 0)) 2598 return -EINVAL; 2599 } else if (copylen) { 2600 int hdrlen = min_t(int, copylen, tp_len); 2601 2602 skb_push(skb, dev->hard_header_len); 2603 skb_put(skb, copylen - dev->hard_header_len); 2604 err = skb_store_bits(skb, 0, data, hdrlen); 2605 if (unlikely(err)) 2606 return err; 2607 if (!dev_validate_header(dev, skb->data, hdrlen)) 2608 return -EINVAL; 2609 2610 data += hdrlen; 2611 to_write -= hdrlen; 2612 } 2613 2614 offset = offset_in_page(data); 2615 len_max = PAGE_SIZE - offset; 2616 len = ((to_write > len_max) ? len_max : to_write); 2617 2618 skb->data_len = to_write; 2619 skb->len += to_write; 2620 skb->truesize += to_write; 2621 refcount_add(to_write, &po->sk.sk_wmem_alloc); 2622 2623 while (likely(to_write)) { 2624 nr_frags = skb_shinfo(skb)->nr_frags; 2625 2626 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) { 2627 pr_err("Packet exceed the number of skb frags(%u)\n", 2628 (unsigned int)MAX_SKB_FRAGS); 2629 return -EFAULT; 2630 } 2631 2632 page = pgv_to_page(data); 2633 data += len; 2634 flush_dcache_page(page); 2635 get_page(page); 2636 skb_fill_page_desc(skb, nr_frags, page, offset, len); 2637 to_write -= len; 2638 offset = 0; 2639 len_max = PAGE_SIZE; 2640 len = ((to_write > len_max) ? len_max : to_write); 2641 } 2642 2643 packet_parse_headers(skb, sock); 2644 2645 return tp_len; 2646} 2647 2648static int tpacket_parse_header(struct packet_sock *po, void *frame, 2649 int size_max, void **data) 2650{ 2651 union tpacket_uhdr ph; 2652 int tp_len, off; 2653 2654 ph.raw = frame; 2655 2656 switch (po->tp_version) { 2657 case TPACKET_V3: 2658 if (ph.h3->tp_next_offset != 0) { 2659 pr_warn_once("variable sized slot not supported"); 2660 return -EINVAL; 2661 } 2662 tp_len = ph.h3->tp_len; 2663 break; 2664 case TPACKET_V2: 2665 tp_len = ph.h2->tp_len; 2666 break; 2667 default: 2668 tp_len = ph.h1->tp_len; 2669 break; 2670 } 2671 if (unlikely(tp_len > size_max)) { 2672 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max); 2673 return -EMSGSIZE; 2674 } 2675 2676 if (unlikely(packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF))) { 2677 int off_min, off_max; 2678 2679 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll); 2680 off_max = po->tx_ring.frame_size - tp_len; 2681 if (po->sk.sk_type == SOCK_DGRAM) { 2682 switch (po->tp_version) { 2683 case TPACKET_V3: 2684 off = ph.h3->tp_net; 2685 break; 2686 case TPACKET_V2: 2687 off = ph.h2->tp_net; 2688 break; 2689 default: 2690 off = ph.h1->tp_net; 2691 break; 2692 } 2693 } else { 2694 switch (po->tp_version) { 2695 case TPACKET_V3: 2696 off = ph.h3->tp_mac; 2697 break; 2698 case TPACKET_V2: 2699 off = ph.h2->tp_mac; 2700 break; 2701 default: 2702 off = ph.h1->tp_mac; 2703 break; 2704 } 2705 } 2706 if (unlikely((off < off_min) || (off_max < off))) 2707 return -EINVAL; 2708 } else { 2709 off = po->tp_hdrlen - sizeof(struct sockaddr_ll); 2710 } 2711 2712 *data = frame + off; 2713 return tp_len; 2714} 2715 2716static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) 2717{ 2718 struct sk_buff *skb = NULL; 2719 struct net_device *dev; 2720 struct virtio_net_hdr *vnet_hdr = NULL; 2721 struct sockcm_cookie sockc; 2722 __be16 proto; 2723 int err, reserve = 0; 2724 void *ph; 2725 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name); 2726 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT); 2727 int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz); 2728 unsigned char *addr = NULL; 2729 int tp_len, size_max; 2730 void *data; 2731 int len_sum = 0; 2732 int status = TP_STATUS_AVAILABLE; 2733 int hlen, tlen, copylen = 0; 2734 long timeo; 2735 2736 mutex_lock(&po->pg_vec_lock); 2737 2738 /* packet_sendmsg() check on tx_ring.pg_vec was lockless, 2739 * we need to confirm it under protection of pg_vec_lock. 2740 */ 2741 if (unlikely(!po->tx_ring.pg_vec)) { 2742 err = -EBUSY; 2743 goto out; 2744 } 2745 if (likely(saddr == NULL)) { 2746 dev = packet_cached_dev_get(po); 2747 proto = READ_ONCE(po->num); 2748 } else { 2749 err = -EINVAL; 2750 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) 2751 goto out; 2752 if (msg->msg_namelen < (saddr->sll_halen 2753 + offsetof(struct sockaddr_ll, 2754 sll_addr))) 2755 goto out; 2756 proto = saddr->sll_protocol; 2757 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex); 2758 if (po->sk.sk_socket->type == SOCK_DGRAM) { 2759 if (dev && msg->msg_namelen < dev->addr_len + 2760 offsetof(struct sockaddr_ll, sll_addr)) 2761 goto out_put; 2762 addr = saddr->sll_addr; 2763 } 2764 } 2765 2766 err = -ENXIO; 2767 if (unlikely(dev == NULL)) 2768 goto out; 2769 err = -ENETDOWN; 2770 if (unlikely(!(dev->flags & IFF_UP))) 2771 goto out_put; 2772 2773 sockcm_init(&sockc, &po->sk); 2774 if (msg->msg_controllen) { 2775 err = sock_cmsg_send(&po->sk, msg, &sockc); 2776 if (unlikely(err)) 2777 goto out_put; 2778 } 2779 2780 if (po->sk.sk_socket->type == SOCK_RAW) 2781 reserve = dev->hard_header_len; 2782 size_max = po->tx_ring.frame_size 2783 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); 2784 2785 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz) 2786 size_max = dev->mtu + reserve + VLAN_HLEN; 2787 2788 timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT); 2789 reinit_completion(&po->skb_completion); 2790 2791 do { 2792 ph = packet_current_frame(po, &po->tx_ring, 2793 TP_STATUS_SEND_REQUEST); 2794 if (unlikely(ph == NULL)) { 2795 /* Note: packet_read_pending() might be slow if we 2796 * have to call it as it's per_cpu variable, but in 2797 * fast-path we don't have to call it, only when ph 2798 * is NULL, we need to check the pending_refcnt. 2799 */ 2800 if (need_wait && packet_read_pending(&po->tx_ring)) { 2801 timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo); 2802 if (timeo <= 0) { 2803 err = !timeo ? -ETIMEDOUT : -ERESTARTSYS; 2804 goto out_put; 2805 } 2806 /* check for additional frames */ 2807 continue; 2808 } else 2809 break; 2810 } 2811 2812 skb = NULL; 2813 tp_len = tpacket_parse_header(po, ph, size_max, &data); 2814 if (tp_len < 0) 2815 goto tpacket_error; 2816 2817 status = TP_STATUS_SEND_REQUEST; 2818 hlen = LL_RESERVED_SPACE(dev); 2819 tlen = dev->needed_tailroom; 2820 if (vnet_hdr_sz) { 2821 vnet_hdr = data; 2822 data += vnet_hdr_sz; 2823 tp_len -= vnet_hdr_sz; 2824 if (tp_len < 0 || 2825 __packet_snd_vnet_parse(vnet_hdr, tp_len)) { 2826 tp_len = -EINVAL; 2827 goto tpacket_error; 2828 } 2829 copylen = __virtio16_to_cpu(vio_le(), 2830 vnet_hdr->hdr_len); 2831 } 2832 copylen = max_t(int, copylen, dev->hard_header_len); 2833 skb = sock_alloc_send_skb(&po->sk, 2834 hlen + tlen + sizeof(struct sockaddr_ll) + 2835 (copylen - dev->hard_header_len), 2836 !need_wait, &err); 2837 2838 if (unlikely(skb == NULL)) { 2839 /* we assume the socket was initially writeable ... */ 2840 if (likely(len_sum > 0)) 2841 err = len_sum; 2842 goto out_status; 2843 } 2844 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, 2845 addr, hlen, copylen, &sockc); 2846 if (likely(tp_len >= 0) && 2847 tp_len > dev->mtu + reserve && 2848 !vnet_hdr_sz && 2849 !packet_extra_vlan_len_allowed(dev, skb)) 2850 tp_len = -EMSGSIZE; 2851 2852 if (unlikely(tp_len < 0)) { 2853tpacket_error: 2854 if (packet_sock_flag(po, PACKET_SOCK_TP_LOSS)) { 2855 __packet_set_status(po, ph, 2856 TP_STATUS_AVAILABLE); 2857 packet_increment_head(&po->tx_ring); 2858 kfree_skb(skb); 2859 continue; 2860 } else { 2861 status = TP_STATUS_WRONG_FORMAT; 2862 err = tp_len; 2863 goto out_status; 2864 } 2865 } 2866 2867 if (vnet_hdr_sz) { 2868 if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) { 2869 tp_len = -EINVAL; 2870 goto tpacket_error; 2871 } 2872 virtio_net_hdr_set_proto(skb, vnet_hdr); 2873 } 2874 2875 skb->destructor = tpacket_destruct_skb; 2876 __packet_set_status(po, ph, TP_STATUS_SENDING); 2877 packet_inc_pending(&po->tx_ring); 2878 2879 status = TP_STATUS_SEND_REQUEST; 2880 err = packet_xmit(po, skb); 2881 if (unlikely(err != 0)) { 2882 if (err > 0) 2883 err = net_xmit_errno(err); 2884 if (err && __packet_get_status(po, ph) == 2885 TP_STATUS_AVAILABLE) { 2886 /* skb was destructed already */ 2887 skb = NULL; 2888 goto out_status; 2889 } 2890 /* 2891 * skb was dropped but not destructed yet; 2892 * let's treat it like congestion or err < 0 2893 */ 2894 err = 0; 2895 } 2896 packet_increment_head(&po->tx_ring); 2897 len_sum += tp_len; 2898 } while (1); 2899 2900 err = len_sum; 2901 goto out_put; 2902 2903out_status: 2904 __packet_set_status(po, ph, status); 2905 kfree_skb(skb); 2906out_put: 2907 dev_put(dev); 2908out: 2909 mutex_unlock(&po->pg_vec_lock); 2910 return err; 2911} 2912 2913static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad, 2914 size_t reserve, size_t len, 2915 size_t linear, int noblock, 2916 int *err) 2917{ 2918 struct sk_buff *skb; 2919 2920 /* Under a page? Don't bother with paged skb. */ 2921 if (prepad + len < PAGE_SIZE || !linear) 2922 linear = len; 2923 2924 if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) 2925 linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); 2926 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 2927 err, PAGE_ALLOC_COSTLY_ORDER); 2928 if (!skb) 2929 return NULL; 2930 2931 skb_reserve(skb, reserve); 2932 skb_put(skb, linear); 2933 skb->data_len = len - linear; 2934 skb->len += len - linear; 2935 2936 return skb; 2937} 2938 2939static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) 2940{ 2941 struct sock *sk = sock->sk; 2942 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name); 2943 struct sk_buff *skb; 2944 struct net_device *dev; 2945 __be16 proto; 2946 unsigned char *addr = NULL; 2947 int err, reserve = 0; 2948 struct sockcm_cookie sockc; 2949 struct virtio_net_hdr vnet_hdr = { 0 }; 2950 int offset = 0; 2951 struct packet_sock *po = pkt_sk(sk); 2952 int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz); 2953 int hlen, tlen, linear; 2954 int extra_len = 0; 2955 2956 /* 2957 * Get and verify the address. 2958 */ 2959 2960 if (likely(saddr == NULL)) { 2961 dev = packet_cached_dev_get(po); 2962 proto = READ_ONCE(po->num); 2963 } else { 2964 err = -EINVAL; 2965 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) 2966 goto out; 2967 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr))) 2968 goto out; 2969 proto = saddr->sll_protocol; 2970 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex); 2971 if (sock->type == SOCK_DGRAM) { 2972 if (dev && msg->msg_namelen < dev->addr_len + 2973 offsetof(struct sockaddr_ll, sll_addr)) 2974 goto out_unlock; 2975 addr = saddr->sll_addr; 2976 } 2977 } 2978 2979 err = -ENXIO; 2980 if (unlikely(dev == NULL)) 2981 goto out_unlock; 2982 err = -ENETDOWN; 2983 if (unlikely(!(dev->flags & IFF_UP))) 2984 goto out_unlock; 2985 2986 sockcm_init(&sockc, sk); 2987 if (msg->msg_controllen) { 2988 err = sock_cmsg_send(sk, msg, &sockc); 2989 if (unlikely(err)) 2990 goto out_unlock; 2991 } 2992 2993 if (sock->type == SOCK_RAW) 2994 reserve = dev->hard_header_len; 2995 if (vnet_hdr_sz) { 2996 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz); 2997 if (err) 2998 goto out_unlock; 2999 } 3000 3001 if (unlikely(sock_flag(sk, SOCK_NOFCS))) { 3002 if (!netif_supports_nofcs(dev)) { 3003 err = -EPROTONOSUPPORT; 3004 goto out_unlock; 3005 } 3006 extra_len = 4; /* We're doing our own CRC */ 3007 } 3008 3009 err = -EMSGSIZE; 3010 if (!vnet_hdr.gso_type && 3011 (len > dev->mtu + reserve + VLAN_HLEN + extra_len)) 3012 goto out_unlock; 3013 3014 err = -ENOBUFS; 3015 hlen = LL_RESERVED_SPACE(dev); 3016 tlen = dev->needed_tailroom; 3017 linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len); 3018 linear = max(linear, min_t(int, len, dev->hard_header_len)); 3019 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear, 3020 msg->msg_flags & MSG_DONTWAIT, &err); 3021 if (skb == NULL) 3022 goto out_unlock; 3023 3024 skb_reset_network_header(skb); 3025 3026 err = -EINVAL; 3027 if (sock->type == SOCK_DGRAM) { 3028 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len); 3029 if (unlikely(offset < 0)) 3030 goto out_free; 3031 } else if (reserve) { 3032 skb_reserve(skb, -reserve); 3033 if (len < reserve + sizeof(struct ipv6hdr) && 3034 dev->min_header_len != dev->hard_header_len) 3035 skb_reset_network_header(skb); 3036 } 3037 3038 /* Returns -EFAULT on error */ 3039 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len); 3040 if (err) 3041 goto out_free; 3042 3043 if ((sock->type == SOCK_RAW && 3044 !dev_validate_header(dev, skb->data, len)) || !skb->len) { 3045 err = -EINVAL; 3046 goto out_free; 3047 } 3048 3049 skb_setup_tx_timestamp(skb, &sockc); 3050 3051 if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) && 3052 !packet_extra_vlan_len_allowed(dev, skb)) { 3053 err = -EMSGSIZE; 3054 goto out_free; 3055 } 3056 3057 skb->protocol = proto; 3058 skb->dev = dev; 3059 skb->priority = sockc.priority; 3060 skb->mark = sockc.mark; 3061 skb_set_delivery_type_by_clockid(skb, sockc.transmit_time, sk->sk_clockid); 3062 3063 if (unlikely(extra_len == 4)) 3064 skb->no_fcs = 1; 3065 3066 packet_parse_headers(skb, sock); 3067 3068 if (vnet_hdr_sz) { 3069 err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le()); 3070 if (err) 3071 goto out_free; 3072 len += vnet_hdr_sz; 3073 virtio_net_hdr_set_proto(skb, &vnet_hdr); 3074 } 3075 3076 err = packet_xmit(po, skb); 3077 3078 if (unlikely(err != 0)) { 3079 if (err > 0) 3080 err = net_xmit_errno(err); 3081 if (err) 3082 goto out_unlock; 3083 } 3084 3085 dev_put(dev); 3086 3087 return len; 3088 3089out_free: 3090 kfree_skb(skb); 3091out_unlock: 3092 dev_put(dev); 3093out: 3094 return err; 3095} 3096 3097static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) 3098{ 3099 struct sock *sk = sock->sk; 3100 struct packet_sock *po = pkt_sk(sk); 3101 3102 /* Reading tx_ring.pg_vec without holding pg_vec_lock is racy. 3103 * tpacket_snd() will redo the check safely. 3104 */ 3105 if (data_race(po->tx_ring.pg_vec)) 3106 return tpacket_snd(po, msg); 3107 3108 return packet_snd(sock, msg, len); 3109} 3110 3111/* 3112 * Close a PACKET socket. This is fairly simple. We immediately go 3113 * to 'closed' state and remove our protocol entry in the device list. 3114 */ 3115 3116static int packet_release(struct socket *sock) 3117{ 3118 struct sock *sk = sock->sk; 3119 struct packet_sock *po; 3120 struct packet_fanout *f; 3121 struct net *net; 3122 union tpacket_req_u req_u; 3123 3124 if (!sk) 3125 return 0; 3126 3127 net = sock_net(sk); 3128 po = pkt_sk(sk); 3129 3130 mutex_lock(&net->packet.sklist_lock); 3131 sk_del_node_init_rcu(sk); 3132 mutex_unlock(&net->packet.sklist_lock); 3133 3134 sock_prot_inuse_add(net, sk->sk_prot, -1); 3135 3136 spin_lock(&po->bind_lock); 3137 unregister_prot_hook(sk, false); 3138 WRITE_ONCE(po->num, 0); 3139 packet_cached_dev_reset(po); 3140 3141 if (po->prot_hook.dev) { 3142 netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker); 3143 po->prot_hook.dev = NULL; 3144 } 3145 spin_unlock(&po->bind_lock); 3146 3147 packet_flush_mclist(sk); 3148 3149 lock_sock(sk); 3150 if (po->rx_ring.pg_vec) { 3151 memset(&req_u, 0, sizeof(req_u)); 3152 packet_set_ring(sk, &req_u, 1, 0); 3153 } 3154 3155 if (po->tx_ring.pg_vec) { 3156 memset(&req_u, 0, sizeof(req_u)); 3157 packet_set_ring(sk, &req_u, 1, 1); 3158 } 3159 release_sock(sk); 3160 3161 f = fanout_release(sk); 3162 3163 synchronize_net(); 3164 3165 kfree(po->rollover); 3166 if (f) { 3167 fanout_release_data(f); 3168 kvfree(f); 3169 } 3170 /* 3171 * Now the socket is dead. No more input will appear. 3172 */ 3173 sock_orphan(sk); 3174 sock->sk = NULL; 3175 3176 /* Purge queues */ 3177 3178 skb_queue_purge(&sk->sk_receive_queue); 3179 packet_free_pending(po); 3180 3181 sock_put(sk); 3182 return 0; 3183} 3184 3185/* 3186 * Attach a packet hook. 3187 */ 3188 3189static int packet_do_bind(struct sock *sk, const char *name, int ifindex, 3190 __be16 proto) 3191{ 3192 struct packet_sock *po = pkt_sk(sk); 3193 struct net_device *dev = NULL; 3194 bool unlisted = false; 3195 bool need_rehook; 3196 int ret = 0; 3197 3198 lock_sock(sk); 3199 spin_lock(&po->bind_lock); 3200 if (!proto) 3201 proto = po->num; 3202 3203 rcu_read_lock(); 3204 3205 if (po->fanout) { 3206 ret = -EINVAL; 3207 goto out_unlock; 3208 } 3209 3210 if (name) { 3211 dev = dev_get_by_name_rcu(sock_net(sk), name); 3212 if (!dev) { 3213 ret = -ENODEV; 3214 goto out_unlock; 3215 } 3216 } else if (ifindex) { 3217 dev = dev_get_by_index_rcu(sock_net(sk), ifindex); 3218 if (!dev) { 3219 ret = -ENODEV; 3220 goto out_unlock; 3221 } 3222 } 3223 3224 need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev; 3225 3226 if (need_rehook) { 3227 dev_hold(dev); 3228 if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) { 3229 rcu_read_unlock(); 3230 /* prevents packet_notifier() from calling 3231 * register_prot_hook() 3232 */ 3233 WRITE_ONCE(po->num, 0); 3234 __unregister_prot_hook(sk, true); 3235 rcu_read_lock(); 3236 if (dev) 3237 unlisted = !dev_get_by_index_rcu(sock_net(sk), 3238 dev->ifindex); 3239 } 3240 3241 BUG_ON(packet_sock_flag(po, PACKET_SOCK_RUNNING)); 3242 WRITE_ONCE(po->num, proto); 3243 po->prot_hook.type = proto; 3244 3245 netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker); 3246 3247 if (unlikely(unlisted)) { 3248 po->prot_hook.dev = NULL; 3249 WRITE_ONCE(po->ifindex, -1); 3250 packet_cached_dev_reset(po); 3251 } else { 3252 netdev_hold(dev, &po->prot_hook.dev_tracker, 3253 GFP_ATOMIC); 3254 po->prot_hook.dev = dev; 3255 WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0); 3256 packet_cached_dev_assign(po, dev); 3257 } 3258 dev_put(dev); 3259 } 3260 3261 if (proto == 0 || !need_rehook) 3262 goto out_unlock; 3263 3264 if (!unlisted && (!dev || (dev->flags & IFF_UP))) { 3265 register_prot_hook(sk); 3266 } else { 3267 sk->sk_err = ENETDOWN; 3268 if (!sock_flag(sk, SOCK_DEAD)) 3269 sk_error_report(sk); 3270 } 3271 3272out_unlock: 3273 rcu_read_unlock(); 3274 spin_unlock(&po->bind_lock); 3275 release_sock(sk); 3276 return ret; 3277} 3278 3279/* 3280 * Bind a packet socket to a device 3281 */ 3282 3283static int packet_bind_spkt(struct socket *sock, struct sockaddr_unsized *uaddr, 3284 int addr_len) 3285{ 3286 struct sock *sk = sock->sk; 3287 struct sockaddr *sa = (struct sockaddr *)uaddr; 3288 char name[sizeof(sa->sa_data) + 1]; 3289 3290 /* 3291 * Check legality 3292 */ 3293 3294 if (addr_len != sizeof(struct sockaddr)) 3295 return -EINVAL; 3296 /* uaddr->sa_data comes from the userspace, it's not guaranteed to be 3297 * zero-terminated. 3298 */ 3299 memcpy(name, sa->sa_data, sizeof(sa->sa_data)); 3300 name[sizeof(sa->sa_data)] = 0; 3301 3302 return packet_do_bind(sk, name, 0, 0); 3303} 3304 3305static int packet_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len) 3306{ 3307 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr; 3308 struct sock *sk = sock->sk; 3309 3310 /* 3311 * Check legality 3312 */ 3313 3314 if (addr_len < sizeof(struct sockaddr_ll)) 3315 return -EINVAL; 3316 if (sll->sll_family != AF_PACKET) 3317 return -EINVAL; 3318 3319 return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol); 3320} 3321 3322static struct proto packet_proto = { 3323 .name = "PACKET", 3324 .owner = THIS_MODULE, 3325 .obj_size = sizeof(struct packet_sock), 3326}; 3327 3328/* 3329 * Create a packet of type SOCK_PACKET. 3330 */ 3331 3332static int packet_create(struct net *net, struct socket *sock, int protocol, 3333 int kern) 3334{ 3335 struct sock *sk; 3336 struct packet_sock *po; 3337 __be16 proto = (__force __be16)protocol; /* weird, but documented */ 3338 int err; 3339 3340 if (!ns_capable(net->user_ns, CAP_NET_RAW)) 3341 return -EPERM; 3342 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW && 3343 sock->type != SOCK_PACKET) 3344 return -ESOCKTNOSUPPORT; 3345 3346 sock->state = SS_UNCONNECTED; 3347 3348 err = -ENOBUFS; 3349 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern); 3350 if (sk == NULL) 3351 goto out; 3352 3353 sock->ops = &packet_ops; 3354 if (sock->type == SOCK_PACKET) 3355 sock->ops = &packet_ops_spkt; 3356 3357 po = pkt_sk(sk); 3358 err = packet_alloc_pending(po); 3359 if (err) 3360 goto out_sk_free; 3361 3362 sock_init_data(sock, sk); 3363 3364 init_completion(&po->skb_completion); 3365 sk->sk_family = PF_PACKET; 3366 po->num = proto; 3367 3368 packet_cached_dev_reset(po); 3369 3370 sk->sk_destruct = packet_sock_destruct; 3371 3372 /* 3373 * Attach a protocol block 3374 */ 3375 3376 spin_lock_init(&po->bind_lock); 3377 mutex_init(&po->pg_vec_lock); 3378 po->rollover = NULL; 3379 po->prot_hook.func = packet_rcv; 3380 3381 if (sock->type == SOCK_PACKET) 3382 po->prot_hook.func = packet_rcv_spkt; 3383 3384 po->prot_hook.af_packet_priv = sk; 3385 po->prot_hook.af_packet_net = sock_net(sk); 3386 3387 if (proto) { 3388 po->prot_hook.type = proto; 3389 __register_prot_hook(sk); 3390 } 3391 3392 mutex_lock(&net->packet.sklist_lock); 3393 sk_add_node_tail_rcu(sk, &net->packet.sklist); 3394 mutex_unlock(&net->packet.sklist_lock); 3395 3396 sock_prot_inuse_add(net, &packet_proto, 1); 3397 3398 return 0; 3399out_sk_free: 3400 sk_free(sk); 3401out: 3402 return err; 3403} 3404 3405/* 3406 * Pull a packet from our receive queue and hand it to the user. 3407 * If necessary we block. 3408 */ 3409 3410static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, 3411 int flags) 3412{ 3413 struct sock *sk = sock->sk; 3414 struct sk_buff *skb; 3415 int copied, err; 3416 int vnet_hdr_len = READ_ONCE(pkt_sk(sk)->vnet_hdr_sz); 3417 unsigned int origlen = 0; 3418 3419 err = -EINVAL; 3420 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE)) 3421 goto out; 3422 3423#if 0 3424 /* What error should we return now? EUNATTACH? */ 3425 if (pkt_sk(sk)->ifindex < 0) 3426 return -ENODEV; 3427#endif 3428 3429 if (flags & MSG_ERRQUEUE) { 3430 err = sock_recv_errqueue(sk, msg, len, 3431 SOL_PACKET, PACKET_TX_TIMESTAMP); 3432 goto out; 3433 } 3434 3435 /* 3436 * Call the generic datagram receiver. This handles all sorts 3437 * of horrible races and re-entrancy so we can forget about it 3438 * in the protocol layers. 3439 * 3440 * Now it will return ENETDOWN, if device have just gone down, 3441 * but then it will block. 3442 */ 3443 3444 skb = skb_recv_datagram(sk, flags, &err); 3445 3446 /* 3447 * An error occurred so return it. Because skb_recv_datagram() 3448 * handles the blocking we don't see and worry about blocking 3449 * retries. 3450 */ 3451 3452 if (skb == NULL) 3453 goto out; 3454 3455 packet_rcv_try_clear_pressure(pkt_sk(sk)); 3456 3457 if (vnet_hdr_len) { 3458 err = packet_rcv_vnet(msg, skb, &len, vnet_hdr_len); 3459 if (err) 3460 goto out_free; 3461 } 3462 3463 /* You lose any data beyond the buffer you gave. If it worries 3464 * a user program they can ask the device for its MTU 3465 * anyway. 3466 */ 3467 copied = skb->len; 3468 if (copied > len) { 3469 copied = len; 3470 msg->msg_flags |= MSG_TRUNC; 3471 } 3472 3473 err = skb_copy_datagram_msg(skb, 0, msg, copied); 3474 if (err) 3475 goto out_free; 3476 3477 if (sock->type != SOCK_PACKET) { 3478 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll; 3479 3480 /* Original length was stored in sockaddr_ll fields */ 3481 origlen = PACKET_SKB_CB(skb)->sa.origlen; 3482 sll->sll_family = AF_PACKET; 3483 sll->sll_protocol = (sock->type == SOCK_DGRAM) ? 3484 vlan_get_protocol_dgram(skb) : skb->protocol; 3485 } 3486 3487 sock_recv_cmsgs(msg, sk, skb); 3488 3489 if (msg->msg_name) { 3490 const size_t max_len = min(sizeof(skb->cb), 3491 sizeof(struct sockaddr_storage)); 3492 int copy_len; 3493 3494 /* If the address length field is there to be filled 3495 * in, we fill it in now. 3496 */ 3497 if (sock->type == SOCK_PACKET) { 3498 __sockaddr_check_size(sizeof(struct sockaddr_pkt)); 3499 msg->msg_namelen = sizeof(struct sockaddr_pkt); 3500 copy_len = msg->msg_namelen; 3501 } else { 3502 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll; 3503 3504 msg->msg_namelen = sll->sll_halen + 3505 offsetof(struct sockaddr_ll, sll_addr); 3506 copy_len = msg->msg_namelen; 3507 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) { 3508 memset(msg->msg_name + 3509 offsetof(struct sockaddr_ll, sll_addr), 3510 0, sizeof(sll->sll_addr)); 3511 msg->msg_namelen = sizeof(struct sockaddr_ll); 3512 } 3513 } 3514 if (WARN_ON_ONCE(copy_len > max_len)) { 3515 copy_len = max_len; 3516 msg->msg_namelen = copy_len; 3517 } 3518 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len); 3519 } 3520 3521 if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) { 3522 struct tpacket_auxdata aux; 3523 3524 aux.tp_status = TP_STATUS_USER; 3525 if (skb->ip_summed == CHECKSUM_PARTIAL) 3526 aux.tp_status |= TP_STATUS_CSUMNOTREADY; 3527 else if (skb->pkt_type != PACKET_OUTGOING && 3528 skb_csum_unnecessary(skb)) 3529 aux.tp_status |= TP_STATUS_CSUM_VALID; 3530 if (skb_is_gso(skb) && skb_is_gso_tcp(skb)) 3531 aux.tp_status |= TP_STATUS_GSO_TCP; 3532 3533 aux.tp_len = origlen; 3534 aux.tp_snaplen = skb->len; 3535 aux.tp_mac = 0; 3536 aux.tp_net = skb_network_offset(skb); 3537 if (skb_vlan_tag_present(skb)) { 3538 aux.tp_vlan_tci = skb_vlan_tag_get(skb); 3539 aux.tp_vlan_tpid = ntohs(skb->vlan_proto); 3540 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; 3541 } else if (unlikely(sock->type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) { 3542 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll; 3543 struct net_device *dev; 3544 3545 rcu_read_lock(); 3546 dev = dev_get_by_index_rcu(sock_net(sk), sll->sll_ifindex); 3547 if (dev) { 3548 aux.tp_vlan_tci = vlan_get_tci(skb, dev); 3549 aux.tp_vlan_tpid = ntohs(skb->protocol); 3550 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID; 3551 } else { 3552 aux.tp_vlan_tci = 0; 3553 aux.tp_vlan_tpid = 0; 3554 } 3555 rcu_read_unlock(); 3556 } else { 3557 aux.tp_vlan_tci = 0; 3558 aux.tp_vlan_tpid = 0; 3559 } 3560 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux); 3561 } 3562 3563 /* 3564 * Free or return the buffer as appropriate. Again this 3565 * hides all the races and re-entrancy issues from us. 3566 */ 3567 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied); 3568 3569out_free: 3570 skb_free_datagram(sk, skb); 3571out: 3572 return err; 3573} 3574 3575static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr, 3576 int peer) 3577{ 3578 struct net_device *dev; 3579 struct sock *sk = sock->sk; 3580 3581 if (peer) 3582 return -EOPNOTSUPP; 3583 3584 uaddr->sa_family = AF_PACKET; 3585 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data)); 3586 rcu_read_lock(); 3587 dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex)); 3588 if (dev) 3589 strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data)); 3590 rcu_read_unlock(); 3591 3592 return sizeof(*uaddr); 3593} 3594 3595static int packet_getname(struct socket *sock, struct sockaddr *uaddr, 3596 int peer) 3597{ 3598 struct net_device *dev; 3599 struct sock *sk = sock->sk; 3600 struct packet_sock *po = pkt_sk(sk); 3601 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr); 3602 int ifindex; 3603 3604 if (peer) 3605 return -EOPNOTSUPP; 3606 3607 ifindex = READ_ONCE(po->ifindex); 3608 sll->sll_family = AF_PACKET; 3609 sll->sll_ifindex = ifindex; 3610 sll->sll_protocol = READ_ONCE(po->num); 3611 sll->sll_pkttype = 0; 3612 rcu_read_lock(); 3613 dev = dev_get_by_index_rcu(sock_net(sk), ifindex); 3614 if (dev) { 3615 sll->sll_hatype = dev->type; 3616 sll->sll_halen = dev->addr_len; 3617 3618 /* Let __fortify_memcpy_chk() know the actual buffer size. */ 3619 memcpy(((struct sockaddr_storage *)sll)->__data + 3620 offsetof(struct sockaddr_ll, sll_addr) - 3621 offsetofend(struct sockaddr_ll, sll_family), 3622 dev->dev_addr, dev->addr_len); 3623 } else { 3624 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */ 3625 sll->sll_halen = 0; 3626 } 3627 rcu_read_unlock(); 3628 3629 return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen; 3630} 3631 3632static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i, 3633 int what) 3634{ 3635 switch (i->type) { 3636 case PACKET_MR_MULTICAST: 3637 if (i->alen != dev->addr_len) 3638 return -EINVAL; 3639 if (what > 0) 3640 return dev_mc_add(dev, i->addr); 3641 else 3642 return dev_mc_del(dev, i->addr); 3643 break; 3644 case PACKET_MR_PROMISC: 3645 return dev_set_promiscuity(dev, what); 3646 case PACKET_MR_ALLMULTI: 3647 return dev_set_allmulti(dev, what); 3648 case PACKET_MR_UNICAST: 3649 if (i->alen != dev->addr_len) 3650 return -EINVAL; 3651 if (what > 0) 3652 return dev_uc_add(dev, i->addr); 3653 else 3654 return dev_uc_del(dev, i->addr); 3655 break; 3656 default: 3657 break; 3658 } 3659 return 0; 3660} 3661 3662static void packet_dev_mclist_delete(struct net_device *dev, 3663 struct packet_mclist **mlp, 3664 struct list_head *list) 3665{ 3666 struct packet_mclist *ml; 3667 3668 while ((ml = *mlp) != NULL) { 3669 if (ml->ifindex == dev->ifindex) { 3670 list_add(&ml->remove_list, list); 3671 *mlp = ml->next; 3672 } else 3673 mlp = &ml->next; 3674 } 3675} 3676 3677static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq) 3678{ 3679 struct packet_sock *po = pkt_sk(sk); 3680 struct packet_mclist *ml, *i; 3681 struct net_device *dev; 3682 int err; 3683 3684 rtnl_lock(); 3685 3686 err = -ENODEV; 3687 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex); 3688 if (!dev) 3689 goto done; 3690 3691 err = -EINVAL; 3692 if (mreq->mr_alen > dev->addr_len) 3693 goto done; 3694 3695 err = -ENOBUFS; 3696 i = kmalloc(sizeof(*i), GFP_KERNEL); 3697 if (i == NULL) 3698 goto done; 3699 3700 err = 0; 3701 for (ml = po->mclist; ml; ml = ml->next) { 3702 if (ml->ifindex == mreq->mr_ifindex && 3703 ml->type == mreq->mr_type && 3704 ml->alen == mreq->mr_alen && 3705 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) { 3706 ml->count++; 3707 /* Free the new element ... */ 3708 kfree(i); 3709 goto done; 3710 } 3711 } 3712 3713 i->type = mreq->mr_type; 3714 i->ifindex = mreq->mr_ifindex; 3715 i->alen = mreq->mr_alen; 3716 memcpy(i->addr, mreq->mr_address, i->alen); 3717 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen); 3718 i->count = 1; 3719 INIT_LIST_HEAD(&i->remove_list); 3720 i->next = po->mclist; 3721 po->mclist = i; 3722 err = packet_dev_mc(dev, i, 1); 3723 if (err) { 3724 po->mclist = i->next; 3725 kfree(i); 3726 } 3727 3728done: 3729 rtnl_unlock(); 3730 return err; 3731} 3732 3733static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq) 3734{ 3735 struct packet_mclist *ml, **mlp; 3736 3737 rtnl_lock(); 3738 3739 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) { 3740 if (ml->ifindex == mreq->mr_ifindex && 3741 ml->type == mreq->mr_type && 3742 ml->alen == mreq->mr_alen && 3743 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) { 3744 if (--ml->count == 0) { 3745 struct net_device *dev; 3746 *mlp = ml->next; 3747 dev = __dev_get_by_index(sock_net(sk), ml->ifindex); 3748 if (dev) 3749 packet_dev_mc(dev, ml, -1); 3750 kfree(ml); 3751 } 3752 break; 3753 } 3754 } 3755 rtnl_unlock(); 3756 return 0; 3757} 3758 3759static void packet_flush_mclist(struct sock *sk) 3760{ 3761 struct packet_sock *po = pkt_sk(sk); 3762 struct packet_mclist *ml; 3763 3764 if (!po->mclist) 3765 return; 3766 3767 rtnl_lock(); 3768 while ((ml = po->mclist) != NULL) { 3769 struct net_device *dev; 3770 3771 po->mclist = ml->next; 3772 dev = __dev_get_by_index(sock_net(sk), ml->ifindex); 3773 if (dev != NULL) 3774 packet_dev_mc(dev, ml, -1); 3775 kfree(ml); 3776 } 3777 rtnl_unlock(); 3778} 3779 3780static int 3781packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, 3782 unsigned int optlen) 3783{ 3784 struct sock *sk = sock->sk; 3785 struct packet_sock *po = pkt_sk(sk); 3786 int ret; 3787 3788 if (level != SOL_PACKET) 3789 return -ENOPROTOOPT; 3790 3791 switch (optname) { 3792 case PACKET_ADD_MEMBERSHIP: 3793 case PACKET_DROP_MEMBERSHIP: 3794 { 3795 struct packet_mreq_max mreq; 3796 int len = optlen; 3797 memset(&mreq, 0, sizeof(mreq)); 3798 if (len < sizeof(struct packet_mreq)) 3799 return -EINVAL; 3800 if (len > sizeof(mreq)) 3801 len = sizeof(mreq); 3802 if (copy_from_sockptr(&mreq, optval, len)) 3803 return -EFAULT; 3804 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address))) 3805 return -EINVAL; 3806 if (optname == PACKET_ADD_MEMBERSHIP) 3807 ret = packet_mc_add(sk, &mreq); 3808 else 3809 ret = packet_mc_drop(sk, &mreq); 3810 return ret; 3811 } 3812 3813 case PACKET_RX_RING: 3814 case PACKET_TX_RING: 3815 { 3816 union tpacket_req_u req_u; 3817 3818 ret = -EINVAL; 3819 lock_sock(sk); 3820 switch (po->tp_version) { 3821 case TPACKET_V1: 3822 case TPACKET_V2: 3823 if (optlen < sizeof(req_u.req)) 3824 break; 3825 ret = copy_from_sockptr(&req_u.req, optval, 3826 sizeof(req_u.req)) ? 3827 -EINVAL : 0; 3828 break; 3829 case TPACKET_V3: 3830 default: 3831 if (optlen < sizeof(req_u.req3)) 3832 break; 3833 ret = copy_from_sockptr(&req_u.req3, optval, 3834 sizeof(req_u.req3)) ? 3835 -EINVAL : 0; 3836 break; 3837 } 3838 if (!ret) 3839 ret = packet_set_ring(sk, &req_u, 0, 3840 optname == PACKET_TX_RING); 3841 release_sock(sk); 3842 return ret; 3843 } 3844 case PACKET_COPY_THRESH: 3845 { 3846 int val; 3847 3848 if (optlen != sizeof(val)) 3849 return -EINVAL; 3850 if (copy_from_sockptr(&val, optval, sizeof(val))) 3851 return -EFAULT; 3852 3853 WRITE_ONCE(pkt_sk(sk)->copy_thresh, val); 3854 return 0; 3855 } 3856 case PACKET_VERSION: 3857 { 3858 int val; 3859 3860 if (optlen != sizeof(val)) 3861 return -EINVAL; 3862 if (copy_from_sockptr(&val, optval, sizeof(val))) 3863 return -EFAULT; 3864 switch (val) { 3865 case TPACKET_V1: 3866 case TPACKET_V2: 3867 case TPACKET_V3: 3868 break; 3869 default: 3870 return -EINVAL; 3871 } 3872 lock_sock(sk); 3873 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { 3874 ret = -EBUSY; 3875 } else { 3876 po->tp_version = val; 3877 ret = 0; 3878 } 3879 release_sock(sk); 3880 return ret; 3881 } 3882 case PACKET_RESERVE: 3883 { 3884 unsigned int val; 3885 3886 if (optlen != sizeof(val)) 3887 return -EINVAL; 3888 if (copy_from_sockptr(&val, optval, sizeof(val))) 3889 return -EFAULT; 3890 if (val > INT_MAX) 3891 return -EINVAL; 3892 lock_sock(sk); 3893 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { 3894 ret = -EBUSY; 3895 } else { 3896 po->tp_reserve = val; 3897 ret = 0; 3898 } 3899 release_sock(sk); 3900 return ret; 3901 } 3902 case PACKET_LOSS: 3903 { 3904 unsigned int val; 3905 3906 if (optlen != sizeof(val)) 3907 return -EINVAL; 3908 if (copy_from_sockptr(&val, optval, sizeof(val))) 3909 return -EFAULT; 3910 3911 lock_sock(sk); 3912 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { 3913 ret = -EBUSY; 3914 } else { 3915 packet_sock_flag_set(po, PACKET_SOCK_TP_LOSS, val); 3916 ret = 0; 3917 } 3918 release_sock(sk); 3919 return ret; 3920 } 3921 case PACKET_AUXDATA: 3922 { 3923 int val; 3924 3925 if (optlen < sizeof(val)) 3926 return -EINVAL; 3927 if (copy_from_sockptr(&val, optval, sizeof(val))) 3928 return -EFAULT; 3929 3930 packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val); 3931 return 0; 3932 } 3933 case PACKET_ORIGDEV: 3934 { 3935 int val; 3936 3937 if (optlen < sizeof(val)) 3938 return -EINVAL; 3939 if (copy_from_sockptr(&val, optval, sizeof(val))) 3940 return -EFAULT; 3941 3942 packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val); 3943 return 0; 3944 } 3945 case PACKET_VNET_HDR: 3946 case PACKET_VNET_HDR_SZ: 3947 { 3948 int val, hdr_len; 3949 3950 if (sock->type != SOCK_RAW) 3951 return -EINVAL; 3952 if (optlen < sizeof(val)) 3953 return -EINVAL; 3954 if (copy_from_sockptr(&val, optval, sizeof(val))) 3955 return -EFAULT; 3956 3957 if (optname == PACKET_VNET_HDR_SZ) { 3958 if (val && val != sizeof(struct virtio_net_hdr) && 3959 val != sizeof(struct virtio_net_hdr_mrg_rxbuf)) 3960 return -EINVAL; 3961 hdr_len = val; 3962 } else { 3963 hdr_len = val ? sizeof(struct virtio_net_hdr) : 0; 3964 } 3965 lock_sock(sk); 3966 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) { 3967 ret = -EBUSY; 3968 } else { 3969 WRITE_ONCE(po->vnet_hdr_sz, hdr_len); 3970 ret = 0; 3971 } 3972 release_sock(sk); 3973 return ret; 3974 } 3975 case PACKET_TIMESTAMP: 3976 { 3977 int val; 3978 3979 if (optlen != sizeof(val)) 3980 return -EINVAL; 3981 if (copy_from_sockptr(&val, optval, sizeof(val))) 3982 return -EFAULT; 3983 3984 WRITE_ONCE(po->tp_tstamp, val); 3985 return 0; 3986 } 3987 case PACKET_FANOUT: 3988 { 3989 struct fanout_args args = { 0 }; 3990 3991 if (optlen != sizeof(int) && optlen != sizeof(args)) 3992 return -EINVAL; 3993 if (copy_from_sockptr(&args, optval, optlen)) 3994 return -EFAULT; 3995 3996 return fanout_add(sk, &args); 3997 } 3998 case PACKET_FANOUT_DATA: 3999 { 4000 /* Paired with the WRITE_ONCE() in fanout_add() */ 4001 if (!READ_ONCE(po->fanout)) 4002 return -EINVAL; 4003 4004 return fanout_set_data(po, optval, optlen); 4005 } 4006 case PACKET_IGNORE_OUTGOING: 4007 { 4008 int val; 4009 4010 if (optlen != sizeof(val)) 4011 return -EINVAL; 4012 if (copy_from_sockptr(&val, optval, sizeof(val))) 4013 return -EFAULT; 4014 if (val < 0 || val > 1) 4015 return -EINVAL; 4016 4017 WRITE_ONCE(po->prot_hook.ignore_outgoing, !!val); 4018 return 0; 4019 } 4020 case PACKET_TX_HAS_OFF: 4021 { 4022 unsigned int val; 4023 4024 if (optlen != sizeof(val)) 4025 return -EINVAL; 4026 if (copy_from_sockptr(&val, optval, sizeof(val))) 4027 return -EFAULT; 4028 4029 lock_sock(sk); 4030 if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec) 4031 packet_sock_flag_set(po, PACKET_SOCK_TX_HAS_OFF, val); 4032 4033 release_sock(sk); 4034 return 0; 4035 } 4036 case PACKET_QDISC_BYPASS: 4037 { 4038 int val; 4039 4040 if (optlen != sizeof(val)) 4041 return -EINVAL; 4042 if (copy_from_sockptr(&val, optval, sizeof(val))) 4043 return -EFAULT; 4044 4045 packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val); 4046 return 0; 4047 } 4048 default: 4049 return -ENOPROTOOPT; 4050 } 4051} 4052 4053static int packet_getsockopt(struct socket *sock, int level, int optname, 4054 char __user *optval, int __user *optlen) 4055{ 4056 int len; 4057 int val, lv = sizeof(val); 4058 struct sock *sk = sock->sk; 4059 struct packet_sock *po = pkt_sk(sk); 4060 void *data = &val; 4061 union tpacket_stats_u st; 4062 struct tpacket_rollover_stats rstats; 4063 int drops; 4064 4065 if (level != SOL_PACKET) 4066 return -ENOPROTOOPT; 4067 4068 if (get_user(len, optlen)) 4069 return -EFAULT; 4070 4071 if (len < 0) 4072 return -EINVAL; 4073 4074 switch (optname) { 4075 case PACKET_STATISTICS: 4076 spin_lock_bh(&sk->sk_receive_queue.lock); 4077 memcpy(&st, &po->stats, sizeof(st)); 4078 memset(&po->stats, 0, sizeof(po->stats)); 4079 spin_unlock_bh(&sk->sk_receive_queue.lock); 4080 drops = atomic_xchg(&po->tp_drops, 0); 4081 4082 if (po->tp_version == TPACKET_V3) { 4083 lv = sizeof(struct tpacket_stats_v3); 4084 st.stats3.tp_drops = drops; 4085 st.stats3.tp_packets += drops; 4086 data = &st.stats3; 4087 } else { 4088 lv = sizeof(struct tpacket_stats); 4089 st.stats1.tp_drops = drops; 4090 st.stats1.tp_packets += drops; 4091 data = &st.stats1; 4092 } 4093 4094 break; 4095 case PACKET_AUXDATA: 4096 val = packet_sock_flag(po, PACKET_SOCK_AUXDATA); 4097 break; 4098 case PACKET_ORIGDEV: 4099 val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV); 4100 break; 4101 case PACKET_VNET_HDR: 4102 val = !!READ_ONCE(po->vnet_hdr_sz); 4103 break; 4104 case PACKET_VNET_HDR_SZ: 4105 val = READ_ONCE(po->vnet_hdr_sz); 4106 break; 4107 case PACKET_COPY_THRESH: 4108 val = READ_ONCE(pkt_sk(sk)->copy_thresh); 4109 break; 4110 case PACKET_VERSION: 4111 val = po->tp_version; 4112 break; 4113 case PACKET_HDRLEN: 4114 if (len > sizeof(int)) 4115 len = sizeof(int); 4116 if (len < sizeof(int)) 4117 return -EINVAL; 4118 if (copy_from_user(&val, optval, len)) 4119 return -EFAULT; 4120 switch (val) { 4121 case TPACKET_V1: 4122 val = sizeof(struct tpacket_hdr); 4123 break; 4124 case TPACKET_V2: 4125 val = sizeof(struct tpacket2_hdr); 4126 break; 4127 case TPACKET_V3: 4128 val = sizeof(struct tpacket3_hdr); 4129 break; 4130 default: 4131 return -EINVAL; 4132 } 4133 break; 4134 case PACKET_RESERVE: 4135 val = po->tp_reserve; 4136 break; 4137 case PACKET_LOSS: 4138 val = packet_sock_flag(po, PACKET_SOCK_TP_LOSS); 4139 break; 4140 case PACKET_TIMESTAMP: 4141 val = READ_ONCE(po->tp_tstamp); 4142 break; 4143 case PACKET_FANOUT: 4144 val = (po->fanout ? 4145 ((u32)po->fanout->id | 4146 ((u32)po->fanout->type << 16) | 4147 ((u32)po->fanout->flags << 24)) : 4148 0); 4149 break; 4150 case PACKET_IGNORE_OUTGOING: 4151 val = READ_ONCE(po->prot_hook.ignore_outgoing); 4152 break; 4153 case PACKET_ROLLOVER_STATS: 4154 if (!po->rollover) 4155 return -EINVAL; 4156 rstats.tp_all = atomic_long_read(&po->rollover->num); 4157 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge); 4158 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed); 4159 data = &rstats; 4160 lv = sizeof(rstats); 4161 break; 4162 case PACKET_TX_HAS_OFF: 4163 val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF); 4164 break; 4165 case PACKET_QDISC_BYPASS: 4166 val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS); 4167 break; 4168 default: 4169 return -ENOPROTOOPT; 4170 } 4171 4172 if (len > lv) 4173 len = lv; 4174 if (put_user(len, optlen)) 4175 return -EFAULT; 4176 if (copy_to_user(optval, data, len)) 4177 return -EFAULT; 4178 return 0; 4179} 4180 4181static int packet_notifier(struct notifier_block *this, 4182 unsigned long msg, void *ptr) 4183{ 4184 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 4185 struct net *net = dev_net(dev); 4186 struct packet_mclist *ml, *tmp; 4187 LIST_HEAD(mclist); 4188 struct sock *sk; 4189 4190 rcu_read_lock(); 4191 sk_for_each_rcu(sk, &net->packet.sklist) { 4192 struct packet_sock *po = pkt_sk(sk); 4193 4194 switch (msg) { 4195 case NETDEV_UNREGISTER: 4196 if (po->mclist) 4197 packet_dev_mclist_delete(dev, &po->mclist, 4198 &mclist); 4199 fallthrough; 4200 4201 case NETDEV_DOWN: 4202 if (dev->ifindex == po->ifindex) { 4203 spin_lock(&po->bind_lock); 4204 if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) { 4205 __unregister_prot_hook(sk, false); 4206 sk->sk_err = ENETDOWN; 4207 if (!sock_flag(sk, SOCK_DEAD)) 4208 sk_error_report(sk); 4209 } 4210 if (msg == NETDEV_UNREGISTER) { 4211 packet_cached_dev_reset(po); 4212 WRITE_ONCE(po->ifindex, -1); 4213 netdev_put(po->prot_hook.dev, 4214 &po->prot_hook.dev_tracker); 4215 po->prot_hook.dev = NULL; 4216 } 4217 spin_unlock(&po->bind_lock); 4218 } 4219 break; 4220 case NETDEV_UP: 4221 if (dev->ifindex == po->ifindex) { 4222 spin_lock(&po->bind_lock); 4223 if (po->num) 4224 register_prot_hook(sk); 4225 spin_unlock(&po->bind_lock); 4226 } 4227 break; 4228 } 4229 } 4230 rcu_read_unlock(); 4231 4232 /* packet_dev_mc might grab instance locks so can't run under rcu */ 4233 list_for_each_entry_safe(ml, tmp, &mclist, remove_list) { 4234 packet_dev_mc(dev, ml, -1); 4235 kfree(ml); 4236 } 4237 4238 return NOTIFY_DONE; 4239} 4240 4241 4242static int packet_ioctl(struct socket *sock, unsigned int cmd, 4243 unsigned long arg) 4244{ 4245 struct sock *sk = sock->sk; 4246 4247 switch (cmd) { 4248 case SIOCOUTQ: 4249 { 4250 int amount = sk_wmem_alloc_get(sk); 4251 4252 return put_user(amount, (int __user *)arg); 4253 } 4254 case SIOCINQ: 4255 { 4256 struct sk_buff *skb; 4257 int amount = 0; 4258 4259 spin_lock_bh(&sk->sk_receive_queue.lock); 4260 skb = skb_peek(&sk->sk_receive_queue); 4261 if (skb) 4262 amount = skb->len; 4263 spin_unlock_bh(&sk->sk_receive_queue.lock); 4264 return put_user(amount, (int __user *)arg); 4265 } 4266#ifdef CONFIG_INET 4267 case SIOCADDRT: 4268 case SIOCDELRT: 4269 case SIOCDARP: 4270 case SIOCGARP: 4271 case SIOCSARP: 4272 case SIOCGIFADDR: 4273 case SIOCSIFADDR: 4274 case SIOCGIFBRDADDR: 4275 case SIOCSIFBRDADDR: 4276 case SIOCGIFNETMASK: 4277 case SIOCSIFNETMASK: 4278 case SIOCGIFDSTADDR: 4279 case SIOCSIFDSTADDR: 4280 case SIOCSIFFLAGS: 4281 return inet_dgram_ops.ioctl(sock, cmd, arg); 4282#endif 4283 4284 default: 4285 return -ENOIOCTLCMD; 4286 } 4287 return 0; 4288} 4289 4290static __poll_t packet_poll(struct file *file, struct socket *sock, 4291 poll_table *wait) 4292{ 4293 struct sock *sk = sock->sk; 4294 struct packet_sock *po = pkt_sk(sk); 4295 __poll_t mask = datagram_poll(file, sock, wait); 4296 4297 spin_lock_bh(&sk->sk_receive_queue.lock); 4298 if (po->rx_ring.pg_vec) { 4299 if (!packet_previous_rx_frame(po, &po->rx_ring, 4300 TP_STATUS_KERNEL)) 4301 mask |= EPOLLIN | EPOLLRDNORM; 4302 } 4303 packet_rcv_try_clear_pressure(po); 4304 spin_unlock_bh(&sk->sk_receive_queue.lock); 4305 spin_lock_bh(&sk->sk_write_queue.lock); 4306 if (po->tx_ring.pg_vec) { 4307 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) 4308 mask |= EPOLLOUT | EPOLLWRNORM; 4309 } 4310 spin_unlock_bh(&sk->sk_write_queue.lock); 4311 return mask; 4312} 4313 4314 4315/* Dirty? Well, I still did not learn better way to account 4316 * for user mmaps. 4317 */ 4318 4319static void packet_mm_open(struct vm_area_struct *vma) 4320{ 4321 struct file *file = vma->vm_file; 4322 struct socket *sock = file->private_data; 4323 struct sock *sk = sock->sk; 4324 4325 if (sk) 4326 atomic_long_inc(&pkt_sk(sk)->mapped); 4327} 4328 4329static void packet_mm_close(struct vm_area_struct *vma) 4330{ 4331 struct file *file = vma->vm_file; 4332 struct socket *sock = file->private_data; 4333 struct sock *sk = sock->sk; 4334 4335 if (sk) 4336 atomic_long_dec(&pkt_sk(sk)->mapped); 4337} 4338 4339static const struct vm_operations_struct packet_mmap_ops = { 4340 .open = packet_mm_open, 4341 .close = packet_mm_close, 4342}; 4343 4344static void free_pg_vec(struct pgv *pg_vec, unsigned int order, 4345 unsigned int len) 4346{ 4347 int i; 4348 4349 for (i = 0; i < len; i++) { 4350 if (likely(pg_vec[i].buffer)) { 4351 if (is_vmalloc_addr(pg_vec[i].buffer)) 4352 vfree(pg_vec[i].buffer); 4353 else 4354 free_pages((unsigned long)pg_vec[i].buffer, 4355 order); 4356 pg_vec[i].buffer = NULL; 4357 } 4358 } 4359 kfree(pg_vec); 4360} 4361 4362static char *alloc_one_pg_vec_page(unsigned long order) 4363{ 4364 char *buffer; 4365 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | 4366 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; 4367 4368 buffer = (char *) __get_free_pages(gfp_flags, order); 4369 if (buffer) 4370 return buffer; 4371 4372 /* __get_free_pages failed, fall back to vmalloc */ 4373 buffer = vzalloc(array_size((1 << order), PAGE_SIZE)); 4374 if (buffer) 4375 return buffer; 4376 4377 /* vmalloc failed, lets dig into swap here */ 4378 gfp_flags &= ~__GFP_NORETRY; 4379 buffer = (char *) __get_free_pages(gfp_flags, order); 4380 if (buffer) 4381 return buffer; 4382 4383 /* complete and utter failure */ 4384 return NULL; 4385} 4386 4387static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order) 4388{ 4389 unsigned int block_nr = req->tp_block_nr; 4390 struct pgv *pg_vec; 4391 int i; 4392 4393 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN); 4394 if (unlikely(!pg_vec)) 4395 goto out; 4396 4397 for (i = 0; i < block_nr; i++) { 4398 pg_vec[i].buffer = alloc_one_pg_vec_page(order); 4399 if (unlikely(!pg_vec[i].buffer)) 4400 goto out_free_pgvec; 4401 } 4402 4403out: 4404 return pg_vec; 4405 4406out_free_pgvec: 4407 free_pg_vec(pg_vec, order, block_nr); 4408 pg_vec = NULL; 4409 goto out; 4410} 4411 4412static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, 4413 int closing, int tx_ring) 4414{ 4415 struct pgv *pg_vec = NULL; 4416 struct packet_sock *po = pkt_sk(sk); 4417 unsigned long *rx_owner_map = NULL; 4418 int was_running, order = 0; 4419 struct packet_ring_buffer *rb; 4420 struct sk_buff_head *rb_queue; 4421 __be16 num; 4422 int err; 4423 /* Added to avoid minimal code churn */ 4424 struct tpacket_req *req = &req_u->req; 4425 4426 rb = tx_ring ? &po->tx_ring : &po->rx_ring; 4427 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue; 4428 4429 err = -EBUSY; 4430 if (!closing) { 4431 if (atomic_long_read(&po->mapped)) 4432 goto out; 4433 if (packet_read_pending(rb)) 4434 goto out; 4435 } 4436 4437 if (req->tp_block_nr) { 4438 unsigned int min_frame_size; 4439 4440 /* Sanity tests and some calculations */ 4441 err = -EBUSY; 4442 if (unlikely(rb->pg_vec)) 4443 goto out; 4444 4445 switch (po->tp_version) { 4446 case TPACKET_V1: 4447 po->tp_hdrlen = TPACKET_HDRLEN; 4448 break; 4449 case TPACKET_V2: 4450 po->tp_hdrlen = TPACKET2_HDRLEN; 4451 break; 4452 case TPACKET_V3: 4453 po->tp_hdrlen = TPACKET3_HDRLEN; 4454 break; 4455 } 4456 4457 err = -EINVAL; 4458 if (unlikely((int)req->tp_block_size <= 0)) 4459 goto out; 4460 if (unlikely(!PAGE_ALIGNED(req->tp_block_size))) 4461 goto out; 4462 min_frame_size = po->tp_hdrlen + po->tp_reserve; 4463 if (po->tp_version >= TPACKET_V3 && 4464 req->tp_block_size < 4465 BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size) 4466 goto out; 4467 if (unlikely(req->tp_frame_size < min_frame_size)) 4468 goto out; 4469 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1))) 4470 goto out; 4471 4472 rb->frames_per_block = req->tp_block_size / req->tp_frame_size; 4473 if (unlikely(rb->frames_per_block == 0)) 4474 goto out; 4475 if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr)) 4476 goto out; 4477 if (unlikely((rb->frames_per_block * req->tp_block_nr) != 4478 req->tp_frame_nr)) 4479 goto out; 4480 4481 err = -ENOMEM; 4482 order = get_order(req->tp_block_size); 4483 pg_vec = alloc_pg_vec(req, order); 4484 if (unlikely(!pg_vec)) 4485 goto out; 4486 switch (po->tp_version) { 4487 case TPACKET_V3: 4488 /* Block transmit is not supported yet */ 4489 if (!tx_ring) { 4490 init_prb_bdqc(po, rb, pg_vec, req_u); 4491 } else { 4492 struct tpacket_req3 *req3 = &req_u->req3; 4493 4494 if (req3->tp_retire_blk_tov || 4495 req3->tp_sizeof_priv || 4496 req3->tp_feature_req_word) { 4497 err = -EINVAL; 4498 goto out_free_pg_vec; 4499 } 4500 } 4501 break; 4502 default: 4503 if (!tx_ring) { 4504 rx_owner_map = bitmap_alloc(req->tp_frame_nr, 4505 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); 4506 if (!rx_owner_map) 4507 goto out_free_pg_vec; 4508 } 4509 break; 4510 } 4511 } 4512 /* Done */ 4513 else { 4514 err = -EINVAL; 4515 if (unlikely(req->tp_frame_nr)) 4516 goto out; 4517 } 4518 4519 4520 /* Detach socket from network */ 4521 spin_lock(&po->bind_lock); 4522 was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING); 4523 num = po->num; 4524 WRITE_ONCE(po->num, 0); 4525 if (was_running) 4526 __unregister_prot_hook(sk, false); 4527 4528 spin_unlock(&po->bind_lock); 4529 4530 synchronize_net(); 4531 4532 err = -EBUSY; 4533 mutex_lock(&po->pg_vec_lock); 4534 if (closing || atomic_long_read(&po->mapped) == 0) { 4535 err = 0; 4536 spin_lock_bh(&rb_queue->lock); 4537 swap(rb->pg_vec, pg_vec); 4538 if (po->tp_version <= TPACKET_V2) 4539 swap(rb->rx_owner_map, rx_owner_map); 4540 rb->frame_max = (req->tp_frame_nr - 1); 4541 rb->head = 0; 4542 rb->frame_size = req->tp_frame_size; 4543 spin_unlock_bh(&rb_queue->lock); 4544 4545 swap(rb->pg_vec_order, order); 4546 swap(rb->pg_vec_len, req->tp_block_nr); 4547 4548 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE; 4549 po->prot_hook.func = (po->rx_ring.pg_vec) ? 4550 tpacket_rcv : packet_rcv; 4551 skb_queue_purge(rb_queue); 4552 if (atomic_long_read(&po->mapped)) 4553 pr_err("packet_mmap: vma is busy: %ld\n", 4554 atomic_long_read(&po->mapped)); 4555 } 4556 mutex_unlock(&po->pg_vec_lock); 4557 4558 spin_lock(&po->bind_lock); 4559 WRITE_ONCE(po->num, num); 4560 if (was_running) 4561 register_prot_hook(sk); 4562 4563 spin_unlock(&po->bind_lock); 4564 if (pg_vec && (po->tp_version > TPACKET_V2)) { 4565 /* Because we don't support block-based V3 on tx-ring */ 4566 if (!tx_ring) 4567 prb_shutdown_retire_blk_timer(po, rb_queue); 4568 } 4569 4570out_free_pg_vec: 4571 if (pg_vec) { 4572 bitmap_free(rx_owner_map); 4573 free_pg_vec(pg_vec, order, req->tp_block_nr); 4574 } 4575out: 4576 return err; 4577} 4578 4579static int packet_mmap(struct file *file, struct socket *sock, 4580 struct vm_area_struct *vma) 4581{ 4582 struct sock *sk = sock->sk; 4583 struct packet_sock *po = pkt_sk(sk); 4584 unsigned long size, expected_size; 4585 struct packet_ring_buffer *rb; 4586 unsigned long start; 4587 int err = -EINVAL; 4588 int i; 4589 4590 if (vma->vm_pgoff) 4591 return -EINVAL; 4592 4593 mutex_lock(&po->pg_vec_lock); 4594 4595 expected_size = 0; 4596 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { 4597 if (rb->pg_vec) { 4598 expected_size += rb->pg_vec_len 4599 * rb->pg_vec_pages 4600 * PAGE_SIZE; 4601 } 4602 } 4603 4604 if (expected_size == 0) 4605 goto out; 4606 4607 size = vma->vm_end - vma->vm_start; 4608 if (size != expected_size) 4609 goto out; 4610 4611 start = vma->vm_start; 4612 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { 4613 if (rb->pg_vec == NULL) 4614 continue; 4615 4616 for (i = 0; i < rb->pg_vec_len; i++) { 4617 struct page *page; 4618 void *kaddr = rb->pg_vec[i].buffer; 4619 int pg_num; 4620 4621 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) { 4622 page = pgv_to_page(kaddr); 4623 err = vm_insert_page(vma, start, page); 4624 if (unlikely(err)) 4625 goto out; 4626 start += PAGE_SIZE; 4627 kaddr += PAGE_SIZE; 4628 } 4629 } 4630 } 4631 4632 atomic_long_inc(&po->mapped); 4633 vma->vm_ops = &packet_mmap_ops; 4634 err = 0; 4635 4636out: 4637 mutex_unlock(&po->pg_vec_lock); 4638 return err; 4639} 4640 4641static const struct proto_ops packet_ops_spkt = { 4642 .family = PF_PACKET, 4643 .owner = THIS_MODULE, 4644 .release = packet_release, 4645 .bind = packet_bind_spkt, 4646 .connect = sock_no_connect, 4647 .socketpair = sock_no_socketpair, 4648 .accept = sock_no_accept, 4649 .getname = packet_getname_spkt, 4650 .poll = datagram_poll, 4651 .ioctl = packet_ioctl, 4652 .gettstamp = sock_gettstamp, 4653 .listen = sock_no_listen, 4654 .shutdown = sock_no_shutdown, 4655 .sendmsg = packet_sendmsg_spkt, 4656 .recvmsg = packet_recvmsg, 4657 .mmap = sock_no_mmap, 4658}; 4659 4660static const struct proto_ops packet_ops = { 4661 .family = PF_PACKET, 4662 .owner = THIS_MODULE, 4663 .release = packet_release, 4664 .bind = packet_bind, 4665 .connect = sock_no_connect, 4666 .socketpair = sock_no_socketpair, 4667 .accept = sock_no_accept, 4668 .getname = packet_getname, 4669 .poll = packet_poll, 4670 .ioctl = packet_ioctl, 4671 .gettstamp = sock_gettstamp, 4672 .listen = sock_no_listen, 4673 .shutdown = sock_no_shutdown, 4674 .setsockopt = packet_setsockopt, 4675 .getsockopt = packet_getsockopt, 4676 .sendmsg = packet_sendmsg, 4677 .recvmsg = packet_recvmsg, 4678 .mmap = packet_mmap, 4679}; 4680 4681static const struct net_proto_family packet_family_ops = { 4682 .family = PF_PACKET, 4683 .create = packet_create, 4684 .owner = THIS_MODULE, 4685}; 4686 4687static struct notifier_block packet_netdev_notifier = { 4688 .notifier_call = packet_notifier, 4689}; 4690 4691#ifdef CONFIG_PROC_FS 4692 4693static void *packet_seq_start(struct seq_file *seq, loff_t *pos) 4694 __acquires(RCU) 4695{ 4696 struct net *net = seq_file_net(seq); 4697 4698 rcu_read_lock(); 4699 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos); 4700} 4701 4702static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4703{ 4704 struct net *net = seq_file_net(seq); 4705 return seq_hlist_next_rcu(v, &net->packet.sklist, pos); 4706} 4707 4708static void packet_seq_stop(struct seq_file *seq, void *v) 4709 __releases(RCU) 4710{ 4711 rcu_read_unlock(); 4712} 4713 4714static int packet_seq_show(struct seq_file *seq, void *v) 4715{ 4716 if (v == SEQ_START_TOKEN) 4717 seq_printf(seq, 4718 "%*sRefCnt Type Proto Iface R Rmem User Inode\n", 4719 IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk"); 4720 else { 4721 struct sock *s = sk_entry(v); 4722 const struct packet_sock *po = pkt_sk(s); 4723 4724 seq_printf(seq, 4725 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n", 4726 s, 4727 refcount_read(&s->sk_refcnt), 4728 s->sk_type, 4729 ntohs(READ_ONCE(po->num)), 4730 READ_ONCE(po->ifindex), 4731 packet_sock_flag(po, PACKET_SOCK_RUNNING), 4732 atomic_read(&s->sk_rmem_alloc), 4733 from_kuid_munged(seq_user_ns(seq), sk_uid(s)), 4734 sock_i_ino(s)); 4735 } 4736 4737 return 0; 4738} 4739 4740static const struct seq_operations packet_seq_ops = { 4741 .start = packet_seq_start, 4742 .next = packet_seq_next, 4743 .stop = packet_seq_stop, 4744 .show = packet_seq_show, 4745}; 4746#endif 4747 4748static int __net_init packet_net_init(struct net *net) 4749{ 4750 mutex_init(&net->packet.sklist_lock); 4751 INIT_HLIST_HEAD(&net->packet.sklist); 4752 4753#ifdef CONFIG_PROC_FS 4754 if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops, 4755 sizeof(struct seq_net_private))) 4756 return -ENOMEM; 4757#endif /* CONFIG_PROC_FS */ 4758 4759 return 0; 4760} 4761 4762static void __net_exit packet_net_exit(struct net *net) 4763{ 4764 remove_proc_entry("packet", net->proc_net); 4765 WARN_ON_ONCE(!hlist_empty(&net->packet.sklist)); 4766} 4767 4768static struct pernet_operations packet_net_ops = { 4769 .init = packet_net_init, 4770 .exit = packet_net_exit, 4771}; 4772 4773 4774static void __exit packet_exit(void) 4775{ 4776 sock_unregister(PF_PACKET); 4777 proto_unregister(&packet_proto); 4778 unregister_netdevice_notifier(&packet_netdev_notifier); 4779 unregister_pernet_subsys(&packet_net_ops); 4780} 4781 4782static int __init packet_init(void) 4783{ 4784 int rc; 4785 4786 rc = register_pernet_subsys(&packet_net_ops); 4787 if (rc) 4788 goto out; 4789 rc = register_netdevice_notifier(&packet_netdev_notifier); 4790 if (rc) 4791 goto out_pernet; 4792 rc = proto_register(&packet_proto, 0); 4793 if (rc) 4794 goto out_notifier; 4795 rc = sock_register(&packet_family_ops); 4796 if (rc) 4797 goto out_proto; 4798 4799 return 0; 4800 4801out_proto: 4802 proto_unregister(&packet_proto); 4803out_notifier: 4804 unregister_netdevice_notifier(&packet_netdev_notifier); 4805out_pernet: 4806 unregister_pernet_subsys(&packet_net_ops); 4807out: 4808 return rc; 4809} 4810 4811module_init(packet_init); 4812module_exit(packet_exit); 4813MODULE_DESCRIPTION("Packet socket support (AF_PACKET)"); 4814MODULE_LICENSE("GPL"); 4815MODULE_ALIAS_NETPROTO(PF_PACKET);
~/
Read-only file preview.