FILE
tcp.h
File preview with syntax highlighting.
linux-6.19.14/include/net/tcp.h
1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Definitions for the TCP module. 8 * 9 * Version: @(#)tcp.h 1.0.5 05/23/93 10 * 11 * Authors: Ross Biro 12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 13 */ 14#ifndef _TCP_H 15#define _TCP_H 16 17#define FASTRETRANS_DEBUG 1 18 19#include <linux/list.h> 20#include <linux/tcp.h> 21#include <linux/bug.h> 22#include <linux/slab.h> 23#include <linux/cache.h> 24#include <linux/percpu.h> 25#include <linux/skbuff.h> 26#include <linux/kref.h> 27#include <linux/ktime.h> 28#include <linux/indirect_call_wrapper.h> 29#include <linux/bits.h> 30 31#include <net/inet_connection_sock.h> 32#include <net/inet_timewait_sock.h> 33#include <net/inet_hashtables.h> 34#include <net/checksum.h> 35#include <net/request_sock.h> 36#include <net/sock_reuseport.h> 37#include <net/sock.h> 38#include <net/snmp.h> 39#include <net/ip.h> 40#include <net/tcp_states.h> 41#include <net/tcp_ao.h> 42#include <net/inet_ecn.h> 43#include <net/dst.h> 44#include <net/mptcp.h> 45#include <net/xfrm.h> 46#include <net/secure_seq.h> 47 48#include <linux/seq_file.h> 49#include <linux/memcontrol.h> 50#include <linux/bpf-cgroup.h> 51#include <linux/siphash.h> 52 53extern struct inet_hashinfo tcp_hashinfo; 54 55DECLARE_PER_CPU(unsigned int, tcp_orphan_count); 56int tcp_orphan_count_sum(void); 57 58static inline void tcp_orphan_count_inc(void) 59{ 60 this_cpu_inc(tcp_orphan_count); 61} 62 63static inline void tcp_orphan_count_dec(void) 64{ 65 this_cpu_dec(tcp_orphan_count); 66} 67 68DECLARE_PER_CPU(u32, tcp_tw_isn); 69 70void tcp_time_wait(struct sock *sk, int state, int timeo); 71 72#define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) 73#define MAX_TCP_OPTION_SPACE 40 74#define TCP_MIN_SND_MSS 48 75#define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE) 76 77/* 78 * Never offer a window over 32767 without using window scaling. Some 79 * poor stacks do signed 16bit maths! 80 */ 81#define MAX_TCP_WINDOW 32767U 82 83/* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 84#define TCP_MIN_MSS 88U 85 86/* The initial MTU to use for probing */ 87#define TCP_BASE_MSS 1024 88 89/* probing interval, default to 10 minutes as per RFC4821 */ 90#define TCP_PROBE_INTERVAL 600 91 92/* Specify interval when tcp mtu probing will stop */ 93#define TCP_PROBE_THRESHOLD 8 94 95/* After receiving this amount of duplicate ACKs fast retransmit starts. */ 96#define TCP_FASTRETRANS_THRESH 3 97 98/* Maximal number of ACKs sent quickly to accelerate slow-start. */ 99#define TCP_MAX_QUICKACKS 16U 100 101/* Maximal number of window scale according to RFC1323 */ 102#define TCP_MAX_WSCALE 14U 103 104/* Default sending frequency of accurate ECN option per RTT */ 105#define TCP_ACCECN_OPTION_BEACON 3 106 107/* urg_data states */ 108#define TCP_URG_VALID 0x0100 109#define TCP_URG_NOTYET 0x0200 110#define TCP_URG_READ 0x0400 111 112#define TCP_RETR1 3 /* 113 * This is how many retries it does before it 114 * tries to figure out if the gateway is 115 * down. Minimal RFC value is 3; it corresponds 116 * to ~3sec-8min depending on RTO. 117 */ 118 119#define TCP_RETR2 15 /* 120 * This should take at least 121 * 90 minutes to time out. 122 * RFC1122 says that the limit is 100 sec. 123 * 15 is ~13-30min depending on RTO. 124 */ 125 126#define TCP_SYN_RETRIES 6 /* This is how many retries are done 127 * when active opening a connection. 128 * RFC1122 says the minimum retry MUST 129 * be at least 180secs. Nevertheless 130 * this value is corresponding to 131 * 63secs of retransmission with the 132 * current initial RTO. 133 */ 134 135#define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 136 * when passive opening a connection. 137 * This is corresponding to 31secs of 138 * retransmission with the current 139 * initial RTO. 140 */ 141 142#define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 143 * state, about 60 seconds */ 144#define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 145 /* BSD style FIN_WAIT2 deadlock breaker. 146 * It used to be 3min, new value is 60sec, 147 * to combine FIN-WAIT-2 timeout with 148 * TIME-WAIT timer. 149 */ 150#define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */ 151 152#define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 153static_assert((1 << ATO_BITS) > TCP_DELACK_MAX); 154 155#if HZ >= 100 156#define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 157#define TCP_ATO_MIN ((unsigned)(HZ/25)) 158#else 159#define TCP_DELACK_MIN 4U 160#define TCP_ATO_MIN 4U 161#endif 162#define TCP_RTO_MAX_SEC 120 163#define TCP_RTO_MAX ((unsigned)(TCP_RTO_MAX_SEC * HZ)) 164#define TCP_RTO_MIN ((unsigned)(HZ / 5)) 165#define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 166 167#define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */ 168 169#define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 170#define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 171 * used as a fallback RTO for the 172 * initial data transmission if no 173 * valid RTT sample has been acquired, 174 * most likely due to retrans in 3WHS. 175 */ 176 177#define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 178 * for local resources. 179 */ 180#define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 181#define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 182#define TCP_KEEPALIVE_INTVL (75*HZ) 183 184#define MAX_TCP_KEEPIDLE 32767 185#define MAX_TCP_KEEPINTVL 32767 186#define MAX_TCP_KEEPCNT 127 187#define MAX_TCP_SYNCNT 127 188 189/* Ensure that TCP PAWS checks are relaxed after ~2147 seconds 190 * to avoid overflows. This assumes a clock smaller than 1 Mhz. 191 * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz. 192 */ 193#define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC) 194 195#define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 196 * after this time. It should be equal 197 * (or greater than) TCP_TIMEWAIT_LEN 198 * to provide reliability equal to one 199 * provided by timewait state. 200 */ 201#define TCP_PAWS_WINDOW 1 /* Replay window for per-host 202 * timestamps. It must be less than 203 * minimal timewait lifetime. 204 */ 205/* 206 * TCP option 207 */ 208 209#define TCPOPT_NOP 1 /* Padding */ 210#define TCPOPT_EOL 0 /* End of options */ 211#define TCPOPT_MSS 2 /* Segment size negotiating */ 212#define TCPOPT_WINDOW 3 /* Window scaling */ 213#define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 214#define TCPOPT_SACK 5 /* SACK Block */ 215#define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 216#define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 217#define TCPOPT_AO 29 /* Authentication Option (RFC5925) */ 218#define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */ 219#define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 220#define TCPOPT_ACCECN0 172 /* 0xAC: Accurate ECN Order 0 */ 221#define TCPOPT_ACCECN1 174 /* 0xAE: Accurate ECN Order 1 */ 222#define TCPOPT_EXP 254 /* Experimental */ 223/* Magic number to be after the option value for sharing TCP 224 * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 225 */ 226#define TCPOPT_FASTOPEN_MAGIC 0xF989 227#define TCPOPT_SMC_MAGIC 0xE2D4C3D9 228 229/* 230 * TCP option lengths 231 */ 232 233#define TCPOLEN_MSS 4 234#define TCPOLEN_WINDOW 3 235#define TCPOLEN_SACK_PERM 2 236#define TCPOLEN_TIMESTAMP 10 237#define TCPOLEN_MD5SIG 18 238#define TCPOLEN_FASTOPEN_BASE 2 239#define TCPOLEN_ACCECN_BASE 2 240#define TCPOLEN_EXP_FASTOPEN_BASE 4 241#define TCPOLEN_EXP_SMC_BASE 6 242 243/* But this is what stacks really send out. */ 244#define TCPOLEN_TSTAMP_ALIGNED 12 245#define TCPOLEN_WSCALE_ALIGNED 4 246#define TCPOLEN_SACKPERM_ALIGNED 4 247#define TCPOLEN_SACK_BASE 2 248#define TCPOLEN_SACK_BASE_ALIGNED 4 249#define TCPOLEN_SACK_PERBLOCK 8 250#define TCPOLEN_MD5SIG_ALIGNED 20 251#define TCPOLEN_MSS_ALIGNED 4 252#define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 253#define TCPOLEN_ACCECN_PERFIELD 3 254 255/* Maximum number of byte counters in AccECN option + size */ 256#define TCP_ACCECN_NUMFIELDS 3 257#define TCP_ACCECN_MAXSIZE (TCPOLEN_ACCECN_BASE + \ 258 TCPOLEN_ACCECN_PERFIELD * \ 259 TCP_ACCECN_NUMFIELDS) 260#define TCP_ACCECN_SAFETY_SHIFT 1 /* SAFETY_FACTOR in accecn draft */ 261 262/* Flags in tp->nonagle */ 263#define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 264#define TCP_NAGLE_CORK 2 /* Socket is corked */ 265#define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 266 267/* TCP thin-stream limits */ 268#define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 269 270/* TCP initial congestion window as per rfc6928 */ 271#define TCP_INIT_CWND 10 272 273/* Bit Flags for sysctl_tcp_fastopen */ 274#define TFO_CLIENT_ENABLE 1 275#define TFO_SERVER_ENABLE 2 276#define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 277 278/* Accept SYN data w/o any cookie option */ 279#define TFO_SERVER_COOKIE_NOT_REQD 0x200 280 281/* Force enable TFO on all listeners, i.e., not requiring the 282 * TCP_FASTOPEN socket option. 283 */ 284#define TFO_SERVER_WO_SOCKOPT1 0x400 285 286 287/* sysctl variables for tcp */ 288extern int sysctl_tcp_max_orphans; 289extern long sysctl_tcp_mem[3]; 290 291#define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 292#define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 293#define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 294 295DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 296 297extern struct percpu_counter tcp_sockets_allocated; 298extern unsigned long tcp_memory_pressure; 299 300/* optimized version of sk_under_memory_pressure() for TCP sockets */ 301static inline bool tcp_under_memory_pressure(const struct sock *sk) 302{ 303 if (mem_cgroup_sk_enabled(sk) && 304 mem_cgroup_sk_under_memory_pressure(sk)) 305 return true; 306 307 if (sk->sk_bypass_prot_mem) 308 return false; 309 310 return READ_ONCE(tcp_memory_pressure); 311} 312/* 313 * The next routines deal with comparing 32 bit unsigned ints 314 * and worry about wraparound (automatic with unsigned arithmetic). 315 */ 316 317static inline bool before(__u32 seq1, __u32 seq2) 318{ 319 return (__s32)(seq1-seq2) < 0; 320} 321#define after(seq2, seq1) before(seq1, seq2) 322 323/* is s2<=s1<=s3 ? */ 324static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 325{ 326 return seq3 - seq2 >= seq1 - seq2; 327} 328 329static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 330{ 331 sk_wmem_queued_add(sk, -skb->truesize); 332 if (!skb_zcopy_pure(skb)) 333 sk_mem_uncharge(sk, skb->truesize); 334 else 335 sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); 336 __kfree_skb(skb); 337} 338 339void sk_forced_mem_schedule(struct sock *sk, int size); 340 341bool tcp_check_oom(const struct sock *sk, int shift); 342 343 344extern struct proto tcp_prot; 345 346#define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 347#define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 348#define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 349#define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 350 351void tcp_tsq_work_init(void); 352 353int tcp_v4_err(struct sk_buff *skb, u32); 354 355void tcp_shutdown(struct sock *sk, int how); 356 357int tcp_v4_early_demux(struct sk_buff *skb); 358int tcp_v4_rcv(struct sk_buff *skb); 359 360void tcp_remove_empty_skb(struct sock *sk); 361int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 362int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 363int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 364 size_t size, struct ubuf_info *uarg); 365void tcp_splice_eof(struct socket *sock); 366int tcp_send_mss(struct sock *sk, int *size_goal, int flags); 367int tcp_wmem_schedule(struct sock *sk, int copy); 368void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, 369 int size_goal); 370void tcp_release_cb(struct sock *sk); 371void tcp_wfree(struct sk_buff *skb); 372void tcp_write_timer_handler(struct sock *sk); 373void tcp_delack_timer_handler(struct sock *sk); 374int tcp_ioctl(struct sock *sk, int cmd, int *karg); 375enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 376void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 377void tcp_rcvbuf_grow(struct sock *sk, u32 newval); 378void tcp_rcv_space_adjust(struct sock *sk); 379int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 380void tcp_twsk_destructor(struct sock *sk); 381void tcp_twsk_purge(struct list_head *net_exit_list); 382ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 383 struct pipe_inode_info *pipe, size_t len, 384 unsigned int flags); 385struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 386 bool force_schedule); 387 388static inline void tcp_dec_quickack_mode(struct sock *sk) 389{ 390 struct inet_connection_sock *icsk = inet_csk(sk); 391 392 if (icsk->icsk_ack.quick) { 393 /* How many ACKs S/ACKing new data have we sent? */ 394 const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0; 395 396 if (pkts >= icsk->icsk_ack.quick) { 397 icsk->icsk_ack.quick = 0; 398 /* Leaving quickack mode we deflate ATO. */ 399 icsk->icsk_ack.ato = TCP_ATO_MIN; 400 } else 401 icsk->icsk_ack.quick -= pkts; 402 } 403} 404 405#define TCP_ECN_MODE_RFC3168 BIT(0) 406#define TCP_ECN_QUEUE_CWR BIT(1) 407#define TCP_ECN_DEMAND_CWR BIT(2) 408#define TCP_ECN_SEEN BIT(3) 409#define TCP_ECN_MODE_ACCECN BIT(4) 410 411#define TCP_ECN_DISABLED 0 412#define TCP_ECN_MODE_PENDING (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) 413#define TCP_ECN_MODE_ANY (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) 414 415static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp) 416{ 417 return tp->ecn_flags & TCP_ECN_MODE_ANY; 418} 419 420static inline bool tcp_ecn_mode_rfc3168(const struct tcp_sock *tp) 421{ 422 return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_RFC3168; 423} 424 425static inline bool tcp_ecn_mode_accecn(const struct tcp_sock *tp) 426{ 427 return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_ACCECN; 428} 429 430static inline bool tcp_ecn_disabled(const struct tcp_sock *tp) 431{ 432 return !tcp_ecn_mode_any(tp); 433} 434 435static inline bool tcp_ecn_mode_pending(const struct tcp_sock *tp) 436{ 437 return (tp->ecn_flags & TCP_ECN_MODE_PENDING) == TCP_ECN_MODE_PENDING; 438} 439 440static inline void tcp_ecn_mode_set(struct tcp_sock *tp, u8 mode) 441{ 442 tp->ecn_flags &= ~TCP_ECN_MODE_ANY; 443 tp->ecn_flags |= mode; 444} 445 446enum tcp_tw_status { 447 TCP_TW_SUCCESS = 0, 448 TCP_TW_RST = 1, 449 TCP_TW_ACK = 2, 450 TCP_TW_SYN = 3, 451 TCP_TW_ACK_OOW = 4 452}; 453 454 455enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 456 struct sk_buff *skb, 457 const struct tcphdr *th, 458 u32 *tw_isn, 459 enum skb_drop_reason *drop_reason); 460struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 461 struct request_sock *req, bool fastopen, 462 bool *lost_race, enum skb_drop_reason *drop_reason); 463enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, 464 struct sk_buff *skb); 465void tcp_enter_loss(struct sock *sk); 466void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag); 467void tcp_clear_retrans(struct tcp_sock *tp); 468void tcp_update_pacing_rate(struct sock *sk); 469void tcp_set_rto(struct sock *sk); 470void tcp_update_metrics(struct sock *sk); 471void tcp_init_metrics(struct sock *sk); 472void tcp_metrics_init(void); 473bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 474void __tcp_close(struct sock *sk, long timeout); 475void tcp_close(struct sock *sk, long timeout); 476void tcp_init_sock(struct sock *sk); 477void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); 478__poll_t tcp_poll(struct file *file, struct socket *sock, 479 struct poll_table_struct *wait); 480int do_tcp_getsockopt(struct sock *sk, int level, 481 int optname, sockptr_t optval, sockptr_t optlen); 482int tcp_getsockopt(struct sock *sk, int level, int optname, 483 char __user *optval, int __user *optlen); 484bool tcp_bpf_bypass_getsockopt(int level, int optname); 485int do_tcp_setsockopt(struct sock *sk, int level, int optname, 486 sockptr_t optval, unsigned int optlen); 487int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 488 unsigned int optlen); 489void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout); 490void tcp_set_keepalive(struct sock *sk, int val); 491void tcp_syn_ack_timeout(const struct request_sock *req); 492int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 493 int flags, int *addr_len); 494int tcp_set_rcvlowat(struct sock *sk, int val); 495int tcp_set_window_clamp(struct sock *sk, int val); 496void tcp_update_recv_tstamps(struct sk_buff *skb, 497 struct scm_timestamping_internal *tss); 498void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 499 struct scm_timestamping_internal *tss); 500void tcp_data_ready(struct sock *sk); 501#ifdef CONFIG_MMU 502int tcp_mmap(struct file *file, struct socket *sock, 503 struct vm_area_struct *vma); 504#endif 505void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 506 struct tcp_options_received *opt_rx, 507 int estab, struct tcp_fastopen_cookie *foc); 508 509/* 510 * BPF SKB-less helpers 511 */ 512u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 513 struct tcphdr *th, u32 *cookie); 514u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 515 struct tcphdr *th, u32 *cookie); 516u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss); 517u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, 518 const struct tcp_request_sock_ops *af_ops, 519 struct sock *sk, struct tcphdr *th); 520/* 521 * TCP v4 functions exported for the inet6 API 522 */ 523 524void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); 525void tcp_v4_mtu_reduced(struct sock *sk); 526void tcp_req_err(struct sock *sk, u32 seq, bool abort); 527void tcp_ld_RTO_revert(struct sock *sk, u32 seq); 528int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 529struct sock *tcp_create_openreq_child(const struct sock *sk, 530 struct request_sock *req, 531 struct sk_buff *skb); 532void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 533struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 534 struct request_sock *req, 535 struct dst_entry *dst, 536 struct request_sock *req_unhash, 537 bool *own_req, 538 void (*opt_child_init)(struct sock *newsk, 539 const struct sock *sk)); 540int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 541int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); 542int tcp_connect(struct sock *sk); 543enum tcp_synack_type { 544 TCP_SYNACK_NORMAL, 545 TCP_SYNACK_FASTOPEN, 546 TCP_SYNACK_COOKIE, 547}; 548struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 549 struct request_sock *req, 550 struct tcp_fastopen_cookie *foc, 551 enum tcp_synack_type synack_type, 552 struct sk_buff *syn_skb); 553int tcp_disconnect(struct sock *sk, int flags); 554 555void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 556int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 557void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 558 559/* From syncookies.c */ 560struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 561 struct request_sock *req, 562 struct dst_entry *dst); 563int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th); 564struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 565struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, 566 struct sock *sk, struct sk_buff *skb, 567 struct tcp_options_received *tcp_opt, 568 int mss, u32 tsoff); 569 570#if IS_ENABLED(CONFIG_BPF) 571struct bpf_tcp_req_attrs { 572 u32 rcv_tsval; 573 u32 rcv_tsecr; 574 u16 mss; 575 u8 rcv_wscale; 576 u8 snd_wscale; 577 u8 ecn_ok; 578 u8 wscale_ok; 579 u8 sack_ok; 580 u8 tstamp_ok; 581 u8 usec_ts_ok; 582 u8 reserved[3]; 583}; 584#endif 585 586#ifdef CONFIG_SYN_COOKIES 587 588/* Syncookies use a monotonic timer which increments every 60 seconds. 589 * This counter is used both as a hash input and partially encoded into 590 * the cookie value. A cookie is only validated further if the delta 591 * between the current counter value and the encoded one is less than this, 592 * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 593 * the counter advances immediately after a cookie is generated). 594 */ 595#define MAX_SYNCOOKIE_AGE 2 596#define TCP_SYNCOOKIE_PERIOD (60 * HZ) 597#define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 598 599/* syncookies: remember time of last synqueue overflow 600 * But do not dirty this field too often (once per second is enough) 601 * It is racy as we do not hold a lock, but race is very minor. 602 */ 603static inline void tcp_synq_overflow(const struct sock *sk) 604{ 605 unsigned int last_overflow; 606 unsigned int now = jiffies; 607 608 if (sk->sk_reuseport) { 609 struct sock_reuseport *reuse; 610 611 reuse = rcu_dereference(sk->sk_reuseport_cb); 612 if (likely(reuse)) { 613 last_overflow = READ_ONCE(reuse->synq_overflow_ts); 614 if (!time_between32(now, last_overflow, 615 last_overflow + HZ)) 616 WRITE_ONCE(reuse->synq_overflow_ts, now); 617 return; 618 } 619 } 620 621 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 622 if (!time_between32(now, last_overflow, last_overflow + HZ)) 623 WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); 624} 625 626/* syncookies: no recent synqueue overflow on this listening socket? */ 627static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 628{ 629 unsigned int last_overflow; 630 unsigned int now = jiffies; 631 632 if (sk->sk_reuseport) { 633 struct sock_reuseport *reuse; 634 635 reuse = rcu_dereference(sk->sk_reuseport_cb); 636 if (likely(reuse)) { 637 last_overflow = READ_ONCE(reuse->synq_overflow_ts); 638 return !time_between32(now, last_overflow - HZ, 639 last_overflow + 640 TCP_SYNCOOKIE_VALID); 641 } 642 } 643 644 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 645 646 /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID, 647 * then we're under synflood. However, we have to use 648 * 'last_overflow - HZ' as lower bound. That's because a concurrent 649 * tcp_synq_overflow() could update .ts_recent_stamp after we read 650 * jiffies but before we store .ts_recent_stamp into last_overflow, 651 * which could lead to rejecting a valid syncookie. 652 */ 653 return !time_between32(now, last_overflow - HZ, 654 last_overflow + TCP_SYNCOOKIE_VALID); 655} 656 657static inline u32 tcp_cookie_time(void) 658{ 659 u64 val = get_jiffies_64(); 660 661 do_div(val, TCP_SYNCOOKIE_PERIOD); 662 return val; 663} 664 665/* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */ 666static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val) 667{ 668 if (usec_ts) 669 return div_u64(val, NSEC_PER_USEC); 670 671 return div_u64(val, NSEC_PER_MSEC); 672} 673 674u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 675 u16 *mssp); 676__u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 677u64 cookie_init_timestamp(struct request_sock *req, u64 now); 678bool cookie_timestamp_decode(const struct net *net, 679 struct tcp_options_received *opt); 680 681static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst) 682{ 683 return READ_ONCE(net->ipv4.sysctl_tcp_ecn) || 684 dst_feature(dst, RTAX_FEATURE_ECN); 685} 686 687#if IS_ENABLED(CONFIG_BPF) 688static inline bool cookie_bpf_ok(struct sk_buff *skb) 689{ 690 return skb->sk; 691} 692 693struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb); 694#else 695static inline bool cookie_bpf_ok(struct sk_buff *skb) 696{ 697 return false; 698} 699 700static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk, 701 struct sk_buff *skb) 702{ 703 return NULL; 704} 705#endif 706 707/* From net/ipv6/syncookies.c */ 708int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th); 709struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 710 711u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 712 const struct tcphdr *th, u16 *mssp); 713__u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 714#endif 715/* tcp_output.c */ 716 717void tcp_skb_entail(struct sock *sk, struct sk_buff *skb); 718void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb); 719void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 720 int nonagle); 721int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 722int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 723void tcp_retransmit_timer(struct sock *sk); 724void tcp_xmit_retransmit_queue(struct sock *); 725void tcp_simple_retransmit(struct sock *); 726void tcp_enter_recovery(struct sock *sk, bool ece_ack); 727int tcp_trim_head(struct sock *, struct sk_buff *, u32); 728enum tcp_queue { 729 TCP_FRAG_IN_WRITE_QUEUE, 730 TCP_FRAG_IN_RTX_QUEUE, 731}; 732int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 733 struct sk_buff *skb, u32 len, 734 unsigned int mss_now, gfp_t gfp); 735 736void tcp_send_probe0(struct sock *); 737int tcp_write_wakeup(struct sock *, int mib); 738void tcp_send_fin(struct sock *sk); 739void tcp_send_active_reset(struct sock *sk, gfp_t priority, 740 enum sk_rst_reason reason); 741int tcp_send_synack(struct sock *); 742void tcp_push_one(struct sock *, unsigned int mss_now); 743void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags); 744void tcp_send_ack(struct sock *sk); 745void tcp_send_delayed_ack(struct sock *sk); 746void tcp_send_loss_probe(struct sock *sk); 747bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 748void tcp_skb_collapse_tstamp(struct sk_buff *skb, 749 const struct sk_buff *next_skb); 750 751/* tcp_input.c */ 752void tcp_rearm_rto(struct sock *sk); 753void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 754void tcp_done_with_error(struct sock *sk, int err); 755void tcp_reset(struct sock *sk, struct sk_buff *skb); 756void tcp_fin(struct sock *sk); 757void tcp_check_space(struct sock *sk); 758void tcp_sack_compress_send_ack(struct sock *sk); 759 760static inline void tcp_cleanup_skb(struct sk_buff *skb) 761{ 762 skb_dst_drop(skb); 763 secpath_reset(skb); 764} 765 766static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb) 767{ 768 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb)); 769 DEBUG_NET_WARN_ON_ONCE(secpath_exists(skb)); 770 __skb_queue_tail(&sk->sk_receive_queue, skb); 771} 772 773/* tcp_timer.c */ 774void tcp_init_xmit_timers(struct sock *); 775static inline void tcp_clear_xmit_timers(struct sock *sk) 776{ 777 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 778 __sock_put(sk); 779 780 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 781 __sock_put(sk); 782 783 inet_csk_clear_xmit_timers(sk); 784} 785 786unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 787unsigned int tcp_current_mss(struct sock *sk); 788u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when); 789 790/* Bound MSS / TSO packet size with the half of the window */ 791static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 792{ 793 int cutoff; 794 795 /* When peer uses tiny windows, there is no use in packetizing 796 * to sub-MSS pieces for the sake of SWS or making sure there 797 * are enough packets in the pipe for fast recovery. 798 * 799 * On the other hand, for extremely large MSS devices, handling 800 * smaller than MSS windows in this way does make sense. 801 */ 802 if (tp->max_window > TCP_MSS_DEFAULT) 803 cutoff = (tp->max_window >> 1); 804 else 805 cutoff = tp->max_window; 806 807 if (cutoff && pktsize > cutoff) 808 return max_t(int, cutoff, 68U - tp->tcp_header_len); 809 else 810 return pktsize; 811} 812 813/* tcp.c */ 814void tcp_get_info(struct sock *, struct tcp_info *); 815 816/* Read 'sendfile()'-style from a TCP socket */ 817int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 818 sk_read_actor_t recv_actor); 819int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc, 820 sk_read_actor_t recv_actor, bool noack, 821 u32 *copied_seq); 822int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); 823struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off); 824void tcp_read_done(struct sock *sk, size_t len); 825 826void tcp_initialize_rcv_mss(struct sock *sk); 827 828int tcp_mtu_to_mss(struct sock *sk, int pmtu); 829int tcp_mss_to_mtu(struct sock *sk, int mss); 830void tcp_mtup_init(struct sock *sk); 831 832static inline unsigned int tcp_rto_max(const struct sock *sk) 833{ 834 return READ_ONCE(inet_csk(sk)->icsk_rto_max); 835} 836 837static inline void tcp_bound_rto(struct sock *sk) 838{ 839 inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk)); 840} 841 842static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 843{ 844 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 845} 846 847static inline unsigned long tcp_reqsk_timeout(struct request_sock *req) 848{ 849 u64 timeout = (u64)req->timeout << req->num_timeout; 850 851 return (unsigned long)min_t(u64, timeout, 852 tcp_rto_max(req->rsk_listener)); 853} 854 855u32 tcp_delack_max(const struct sock *sk); 856 857/* Compute the actual rto_min value */ 858static inline u32 tcp_rto_min(const struct sock *sk) 859{ 860 const struct dst_entry *dst = __sk_dst_get(sk); 861 u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min); 862 863 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 864 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN); 865 return rto_min; 866} 867 868static inline u32 tcp_rto_min_us(const struct sock *sk) 869{ 870 return jiffies_to_usecs(tcp_rto_min(sk)); 871} 872 873static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 874{ 875 return dst_metric_locked(dst, RTAX_CC_ALGO); 876} 877 878/* Minimum RTT in usec. ~0 means not available. */ 879static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 880{ 881 return minmax_get(&tp->rtt_min); 882} 883 884/* Compute the actual receive window we are currently advertising. 885 * Rcv_nxt can be after the window if our peer push more data 886 * than the offered window. 887 */ 888static inline u32 tcp_receive_window(const struct tcp_sock *tp) 889{ 890 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 891 892 if (win < 0) 893 win = 0; 894 return (u32) win; 895} 896 897/* Choose a new window, without checks for shrinking, and without 898 * scaling applied to the result. The caller does these things 899 * if necessary. This is a "raw" window selection. 900 */ 901u32 __tcp_select_window(struct sock *sk); 902 903void tcp_send_window_probe(struct sock *sk); 904 905/* TCP uses 32bit jiffies to save some space. 906 * Note that this is different from tcp_time_stamp, which 907 * historically has been the same until linux-4.13. 908 */ 909#define tcp_jiffies32 ((u32)jiffies) 910 911/* 912 * Deliver a 32bit value for TCP timestamp option (RFC 7323) 913 * It is no longer tied to jiffies, but to 1 ms clock. 914 * Note: double check if you want to use tcp_jiffies32 instead of this. 915 */ 916#define TCP_TS_HZ 1000 917 918static inline u64 tcp_clock_ns(void) 919{ 920 return ktime_get_ns(); 921} 922 923static inline u64 tcp_clock_us(void) 924{ 925 return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 926} 927 928static inline u64 tcp_clock_ms(void) 929{ 930 return div_u64(tcp_clock_ns(), NSEC_PER_MSEC); 931} 932 933/* TCP Timestamp included in TS option (RFC 1323) can either use ms 934 * or usec resolution. Each socket carries a flag to select one or other 935 * resolution, as the route attribute could change anytime. 936 * Each flow must stick to initial resolution. 937 */ 938static inline u32 tcp_clock_ts(bool usec_ts) 939{ 940 return usec_ts ? tcp_clock_us() : tcp_clock_ms(); 941} 942 943static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp) 944{ 945 return div_u64(tp->tcp_mstamp, USEC_PER_MSEC); 946} 947 948static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp) 949{ 950 if (tp->tcp_usec_ts) 951 return tp->tcp_mstamp; 952 return tcp_time_stamp_ms(tp); 953} 954 955void tcp_mstamp_refresh(struct tcp_sock *tp); 956 957static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 958{ 959 return max_t(s64, t1 - t0, 0); 960} 961 962/* provide the departure time in us unit */ 963static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) 964{ 965 return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC); 966} 967 968/* Provide skb TSval in usec or ms unit */ 969static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb) 970{ 971 if (usec_ts) 972 return tcp_skb_timestamp_us(skb); 973 974 return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC); 975} 976 977static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw) 978{ 979 return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset; 980} 981 982static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq) 983{ 984 return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off; 985} 986 987#define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 988 989#define TCPHDR_FIN BIT(0) 990#define TCPHDR_SYN BIT(1) 991#define TCPHDR_RST BIT(2) 992#define TCPHDR_PSH BIT(3) 993#define TCPHDR_ACK BIT(4) 994#define TCPHDR_URG BIT(5) 995#define TCPHDR_ECE BIT(6) 996#define TCPHDR_CWR BIT(7) 997#define TCPHDR_AE BIT(8) 998#define TCPHDR_FLAGS_MASK (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \ 999 TCPHDR_PSH | TCPHDR_ACK | TCPHDR_URG | \ 1000 TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE) 1001#define tcp_flags_ntohs(th) (ntohs(*(__be16 *)&tcp_flag_word(th)) & \ 1002 TCPHDR_FLAGS_MASK) 1003 1004#define TCPHDR_ACE (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE) 1005#define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 1006#define TCPHDR_SYNACK_ACCECN (TCPHDR_SYN | TCPHDR_ACK | TCPHDR_CWR) 1007 1008#define TCP_ACCECN_CEP_ACE_MASK 0x7 1009#define TCP_ACCECN_ACE_MAX_DELTA 6 1010 1011/* To avoid/detect middlebox interference, not all counters start at 0. 1012 * See draft-ietf-tcpm-accurate-ecn for the latest values. 1013 */ 1014#define TCP_ACCECN_CEP_INIT_OFFSET 5 1015#define TCP_ACCECN_E1B_INIT_OFFSET 1 1016#define TCP_ACCECN_E0B_INIT_OFFSET 1 1017#define TCP_ACCECN_CEB_INIT_OFFSET 0 1018 1019/* State flags for sacked in struct tcp_skb_cb */ 1020enum tcp_skb_cb_sacked_flags { 1021 TCPCB_SACKED_ACKED = (1 << 0), /* SKB ACK'd by a SACK block */ 1022 TCPCB_SACKED_RETRANS = (1 << 1), /* SKB retransmitted */ 1023 TCPCB_LOST = (1 << 2), /* SKB is lost */ 1024 TCPCB_TAGBITS = (TCPCB_SACKED_ACKED | TCPCB_SACKED_RETRANS | 1025 TCPCB_LOST), /* All tag bits */ 1026 TCPCB_REPAIRED = (1 << 4), /* SKB repaired (no skb_mstamp_ns) */ 1027 TCPCB_EVER_RETRANS = (1 << 7), /* Ever retransmitted frame */ 1028 TCPCB_RETRANS = (TCPCB_SACKED_RETRANS | TCPCB_EVER_RETRANS | 1029 TCPCB_REPAIRED), 1030}; 1031 1032/* This is what the send packet queuing engine uses to pass 1033 * TCP per-packet control information to the transmission code. 1034 * We also store the host-order sequence numbers in here too. 1035 * This is 44 bytes if IPV6 is enabled. 1036 * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 1037 */ 1038struct tcp_skb_cb { 1039 __u32 seq; /* Starting sequence number */ 1040 __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 1041 union { 1042 /* Note : 1043 * tcp_gso_segs/size are used in write queue only, 1044 * cf tcp_skb_pcount()/tcp_skb_mss() 1045 */ 1046 struct { 1047 u16 tcp_gso_segs; 1048 u16 tcp_gso_size; 1049 }; 1050 }; 1051 __u16 tcp_flags; /* TCP header flags (tcp[12-13])*/ 1052 1053 __u8 sacked; /* State flags for SACK. */ 1054 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 1055#define TSTAMP_ACK_SK 0x1 1056#define TSTAMP_ACK_BPF 0x2 1057 __u8 txstamp_ack:2, /* Record TX timestamp for ack? */ 1058 eor:1, /* Is skb MSG_EOR marked? */ 1059 has_rxtstamp:1, /* SKB has a RX timestamp */ 1060 unused:4; 1061 __u32 ack_seq; /* Sequence number ACK'd */ 1062 union { 1063 struct { 1064#define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) 1065 /* There is space for up to 24 bytes */ 1066 __u32 is_app_limited:1, /* cwnd not fully used? */ 1067 delivered_ce:20, 1068 unused:11; 1069 /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 1070 __u32 delivered; 1071 /* start of send pipeline phase */ 1072 u64 first_tx_mstamp; 1073 /* when we reached the "delivered" count */ 1074 u64 delivered_mstamp; 1075 } tx; /* only used for outgoing skbs */ 1076 union { 1077 struct inet_skb_parm h4; 1078#if IS_ENABLED(CONFIG_IPV6) 1079 struct inet6_skb_parm h6; 1080#endif 1081 } header; /* For incoming skbs */ 1082 }; 1083}; 1084 1085#define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 1086 1087extern const struct inet_connection_sock_af_ops ipv4_specific; 1088 1089#if IS_ENABLED(CONFIG_IPV6) 1090/* This is the variant of inet6_iif() that must be used by TCP, 1091 * as TCP moves IP6CB into a different location in skb->cb[] 1092 */ 1093static inline int tcp_v6_iif(const struct sk_buff *skb) 1094{ 1095 return TCP_SKB_CB(skb)->header.h6.iif; 1096} 1097 1098static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 1099{ 1100 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 1101 1102 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 1103} 1104 1105/* TCP_SKB_CB reference means this can not be used from early demux */ 1106static inline int tcp_v6_sdif(const struct sk_buff *skb) 1107{ 1108#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 1109 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 1110 return TCP_SKB_CB(skb)->header.h6.iif; 1111#endif 1112 return 0; 1113} 1114 1115extern const struct inet_connection_sock_af_ops ipv6_specific; 1116 1117INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); 1118INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); 1119void tcp_v6_early_demux(struct sk_buff *skb); 1120 1121#endif 1122 1123/* TCP_SKB_CB reference means this can not be used from early demux */ 1124static inline int tcp_v4_sdif(struct sk_buff *skb) 1125{ 1126#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 1127 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 1128 return TCP_SKB_CB(skb)->header.h4.iif; 1129#endif 1130 return 0; 1131} 1132 1133/* Due to TSO, an SKB can be composed of multiple actual 1134 * packets. To keep these tracked properly, we use this. 1135 */ 1136static inline int tcp_skb_pcount(const struct sk_buff *skb) 1137{ 1138 return TCP_SKB_CB(skb)->tcp_gso_segs; 1139} 1140 1141static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 1142{ 1143 TCP_SKB_CB(skb)->tcp_gso_segs = segs; 1144} 1145 1146static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 1147{ 1148 TCP_SKB_CB(skb)->tcp_gso_segs += segs; 1149} 1150 1151/* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 1152static inline int tcp_skb_mss(const struct sk_buff *skb) 1153{ 1154 return TCP_SKB_CB(skb)->tcp_gso_size; 1155} 1156 1157static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 1158{ 1159 return likely(!TCP_SKB_CB(skb)->eor); 1160} 1161 1162static inline bool tcp_skb_can_collapse(const struct sk_buff *to, 1163 const struct sk_buff *from) 1164{ 1165 /* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */ 1166 return likely(tcp_skb_can_collapse_to(to) && 1167 mptcp_skb_can_collapse(to, from) && 1168 skb_pure_zcopy_same(to, from) && 1169 skb_frags_readable(to) == skb_frags_readable(from)); 1170} 1171 1172static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to, 1173 const struct sk_buff *from) 1174{ 1175 return likely(mptcp_skb_can_collapse(to, from) && 1176 !skb_cmp_decrypted(to, from)); 1177} 1178 1179/* Events passed to congestion control interface */ 1180enum tcp_ca_event { 1181 CA_EVENT_TX_START, /* first transmit when no packets in flight */ 1182 CA_EVENT_CWND_RESTART, /* congestion window restart */ 1183 CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 1184 CA_EVENT_LOSS, /* loss timeout */ 1185 CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 1186 CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 1187}; 1188 1189/* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 1190enum tcp_ca_ack_event_flags { 1191 CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 1192 CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 1193 CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 1194}; 1195 1196/* 1197 * Interface for adding new TCP congestion control handlers 1198 */ 1199#define TCP_CA_NAME_MAX 16 1200#define TCP_CA_MAX 128 1201#define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 1202 1203#define TCP_CA_UNSPEC 0 1204 1205/* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 1206#define TCP_CONG_NON_RESTRICTED BIT(0) 1207/* Requires ECN/ECT set on all packets */ 1208#define TCP_CONG_NEEDS_ECN BIT(1) 1209/* Require successfully negotiated AccECN capability */ 1210#define TCP_CONG_NEEDS_ACCECN BIT(2) 1211/* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ 1212#define TCP_CONG_ECT_1_NEGOTIATION BIT(3) 1213/* Cannot fallback to RFC3168 during AccECN negotiation */ 1214#define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) 1215#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ 1216 TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ 1217 TCP_CONG_NO_FALLBACK_RFC3168) 1218 1219union tcp_cc_info; 1220 1221struct ack_sample { 1222 u32 pkts_acked; 1223 s32 rtt_us; 1224 u32 in_flight; 1225}; 1226 1227/* A rate sample measures the number of (original/retransmitted) data 1228 * packets delivered "delivered" over an interval of time "interval_us". 1229 * The tcp_rate.c code fills in the rate sample, and congestion 1230 * control modules that define a cong_control function to run at the end 1231 * of ACK processing can optionally chose to consult this sample when 1232 * setting cwnd and pacing rate. 1233 * A sample is invalid if "delivered" or "interval_us" is negative. 1234 */ 1235struct rate_sample { 1236 u64 prior_mstamp; /* starting timestamp for interval */ 1237 u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 1238 u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ 1239 s32 delivered; /* number of packets delivered over interval */ 1240 s32 delivered_ce; /* number of packets delivered w/ CE marks*/ 1241 long interval_us; /* time for tp->delivered to incr "delivered" */ 1242 u32 snd_interval_us; /* snd interval for delivered packets */ 1243 u32 rcv_interval_us; /* rcv interval for delivered packets */ 1244 long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 1245 int losses; /* number of packets marked lost upon ACK */ 1246 u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 1247 u32 prior_in_flight; /* in flight before this ACK */ 1248 u32 last_end_seq; /* end_seq of most recently ACKed packet */ 1249 bool is_app_limited; /* is sample from packet with bubble in pipe? */ 1250 bool is_retrans; /* is sample from retransmission? */ 1251 bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 1252}; 1253 1254struct tcp_congestion_ops { 1255/* fast path fields are put first to fill one cache line */ 1256 1257 /* return slow start threshold (required) */ 1258 u32 (*ssthresh)(struct sock *sk); 1259 1260 /* do new cwnd calculation (required) */ 1261 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 1262 1263 /* call before changing ca_state (optional) */ 1264 void (*set_state)(struct sock *sk, u8 new_state); 1265 1266 /* call when cwnd event occurs (optional) */ 1267 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 1268 1269 /* call when ack arrives (optional) */ 1270 void (*in_ack_event)(struct sock *sk, u32 flags); 1271 1272 /* hook for packet ack accounting (optional) */ 1273 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 1274 1275 /* override sysctl_tcp_min_tso_segs */ 1276 u32 (*min_tso_segs)(struct sock *sk); 1277 1278 /* call when packets are delivered to update cwnd and pacing rate, 1279 * after all the ca_state processing. (optional) 1280 */ 1281 void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs); 1282 1283 1284 /* new value of cwnd after loss (required) */ 1285 u32 (*undo_cwnd)(struct sock *sk); 1286 /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 1287 u32 (*sndbuf_expand)(struct sock *sk); 1288 1289/* control/slow paths put last */ 1290 /* get info for inet_diag (optional) */ 1291 size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 1292 union tcp_cc_info *info); 1293 1294 char name[TCP_CA_NAME_MAX]; 1295 struct module *owner; 1296 struct list_head list; 1297 u32 key; 1298 u32 flags; 1299 1300 /* initialize private data (optional) */ 1301 void (*init)(struct sock *sk); 1302 /* cleanup private data (optional) */ 1303 void (*release)(struct sock *sk); 1304} ____cacheline_aligned_in_smp; 1305 1306int tcp_register_congestion_control(struct tcp_congestion_ops *type); 1307void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 1308int tcp_update_congestion_control(struct tcp_congestion_ops *type, 1309 struct tcp_congestion_ops *old_type); 1310int tcp_validate_congestion_control(struct tcp_congestion_ops *ca); 1311 1312void tcp_assign_congestion_control(struct sock *sk); 1313void tcp_init_congestion_control(struct sock *sk); 1314void tcp_cleanup_congestion_control(struct sock *sk); 1315int tcp_set_default_congestion_control(struct net *net, const char *name); 1316void tcp_get_default_congestion_control(struct net *net, char *name); 1317void tcp_get_available_congestion_control(char *buf, size_t len); 1318void tcp_get_allowed_congestion_control(char *buf, size_t len); 1319int tcp_set_allowed_congestion_control(char *allowed); 1320int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, 1321 bool cap_net_admin); 1322u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1323void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1324 1325u32 tcp_reno_ssthresh(struct sock *sk); 1326u32 tcp_reno_undo_cwnd(struct sock *sk); 1327void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1328extern struct tcp_congestion_ops tcp_reno; 1329 1330struct tcp_congestion_ops *tcp_ca_find(const char *name); 1331struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 1332u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca); 1333#ifdef CONFIG_INET 1334char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1335#else 1336static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1337{ 1338 return NULL; 1339} 1340#endif 1341 1342static inline bool tcp_ca_needs_ecn(const struct sock *sk) 1343{ 1344 const struct inet_connection_sock *icsk = inet_csk(sk); 1345 1346 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 1347} 1348 1349static inline bool tcp_ca_needs_accecn(const struct sock *sk) 1350{ 1351 const struct inet_connection_sock *icsk = inet_csk(sk); 1352 1353 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; 1354} 1355 1356static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) 1357{ 1358 const struct inet_connection_sock *icsk = inet_csk(sk); 1359 1360 return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; 1361} 1362 1363static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) 1364{ 1365 const struct inet_connection_sock *icsk = inet_csk(sk); 1366 1367 return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; 1368} 1369 1370static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1371{ 1372 const struct inet_connection_sock *icsk = inet_csk(sk); 1373 1374 if (icsk->icsk_ca_ops->cwnd_event) 1375 icsk->icsk_ca_ops->cwnd_event(sk, event); 1376} 1377 1378/* From tcp_cong.c */ 1379void tcp_set_ca_state(struct sock *sk, const u8 ca_state); 1380 1381/* From tcp_rate.c */ 1382void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); 1383void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, 1384 struct rate_sample *rs); 1385void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, 1386 bool is_sack_reneg, struct rate_sample *rs); 1387void tcp_rate_check_app_limited(struct sock *sk); 1388 1389static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) 1390{ 1391 return t1 > t2 || (t1 == t2 && after(seq1, seq2)); 1392} 1393 1394/* These functions determine how the current flow behaves in respect of SACK 1395 * handling. SACK is negotiated with the peer, and therefore it can vary 1396 * between different flows. 1397 * 1398 * tcp_is_sack - SACK enabled 1399 * tcp_is_reno - No SACK 1400 */ 1401static inline int tcp_is_sack(const struct tcp_sock *tp) 1402{ 1403 return likely(tp->rx_opt.sack_ok); 1404} 1405 1406static inline bool tcp_is_reno(const struct tcp_sock *tp) 1407{ 1408 return !tcp_is_sack(tp); 1409} 1410 1411static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 1412{ 1413 return tp->sacked_out + tp->lost_out; 1414} 1415 1416/* This determines how many packets are "in the network" to the best 1417 * of our knowledge. In many cases it is conservative, but where 1418 * detailed information is available from the receiver (via SACK 1419 * blocks etc.) we can make more aggressive calculations. 1420 * 1421 * Use this for decisions involving congestion control, use just 1422 * tp->packets_out to determine if the send queue is empty or not. 1423 * 1424 * Read this equation as: 1425 * 1426 * "Packets sent once on transmission queue" MINUS 1427 * "Packets left network, but not honestly ACKed yet" PLUS 1428 * "Packets fast retransmitted" 1429 */ 1430static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 1431{ 1432 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 1433} 1434 1435#define TCP_INFINITE_SSTHRESH 0x7fffffff 1436 1437static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) 1438{ 1439 return tp->snd_cwnd; 1440} 1441 1442static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) 1443{ 1444 WARN_ON_ONCE((int)val <= 0); 1445 tp->snd_cwnd = val; 1446} 1447 1448static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1449{ 1450 return tcp_snd_cwnd(tp) < tp->snd_ssthresh; 1451} 1452 1453static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 1454{ 1455 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 1456} 1457 1458static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1459{ 1460 return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1461 (1 << inet_csk(sk)->icsk_ca_state); 1462} 1463 1464/* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1465 * The exception is cwnd reduction phase, when cwnd is decreasing towards 1466 * ssthresh. 1467 */ 1468static inline __u32 tcp_current_ssthresh(const struct sock *sk) 1469{ 1470 const struct tcp_sock *tp = tcp_sk(sk); 1471 1472 if (tcp_in_cwnd_reduction(sk)) 1473 return tp->snd_ssthresh; 1474 else 1475 return max(tp->snd_ssthresh, 1476 ((tcp_snd_cwnd(tp) >> 1) + 1477 (tcp_snd_cwnd(tp) >> 2))); 1478} 1479 1480/* Use define here intentionally to get WARN_ON location shown at the caller */ 1481#define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 1482 1483void tcp_enter_cwr(struct sock *sk); 1484__u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 1485 1486/* The maximum number of MSS of available cwnd for which TSO defers 1487 * sending if not using sysctl_tcp_tso_win_divisor. 1488 */ 1489static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 1490{ 1491 return 3; 1492} 1493 1494/* Returns end sequence number of the receiver's advertised window */ 1495static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 1496{ 1497 return tp->snd_una + tp->snd_wnd; 1498} 1499 1500/* We follow the spirit of RFC2861 to validate cwnd but implement a more 1501 * flexible approach. The RFC suggests cwnd should not be raised unless 1502 * it was fully used previously. And that's exactly what we do in 1503 * congestion avoidance mode. But in slow start we allow cwnd to grow 1504 * as long as the application has used half the cwnd. 1505 * Example : 1506 * cwnd is 10 (IW10), but application sends 9 frames. 1507 * We allow cwnd to reach 18 when all frames are ACKed. 1508 * This check is safe because it's as aggressive as slow start which already 1509 * risks 100% overshoot. The advantage is that we discourage application to 1510 * either send more filler packets or data to artificially blow up the cwnd 1511 * usage, and allow application-limited process to probe bw more aggressively. 1512 */ 1513static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1514{ 1515 const struct tcp_sock *tp = tcp_sk(sk); 1516 1517 if (tp->is_cwnd_limited) 1518 return true; 1519 1520 /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1521 if (tcp_in_slow_start(tp)) 1522 return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; 1523 1524 return false; 1525} 1526 1527/* BBR congestion control needs pacing. 1528 * Same remark for SO_MAX_PACING_RATE. 1529 * sch_fq packet scheduler is efficiently handling pacing, 1530 * but is not always installed/used. 1531 * Return true if TCP stack should pace packets itself. 1532 */ 1533static inline bool tcp_needs_internal_pacing(const struct sock *sk) 1534{ 1535 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 1536} 1537 1538/* Estimates in how many jiffies next packet for this flow can be sent. 1539 * Scheduling a retransmit timer too early would be silly. 1540 */ 1541static inline unsigned long tcp_pacing_delay(const struct sock *sk) 1542{ 1543 s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; 1544 1545 return delay > 0 ? nsecs_to_jiffies(delay) : 0; 1546} 1547 1548static inline void tcp_reset_xmit_timer(struct sock *sk, 1549 const int what, 1550 unsigned long when, 1551 bool pace_delay) 1552{ 1553 if (pace_delay) 1554 when += tcp_pacing_delay(sk); 1555 inet_csk_reset_xmit_timer(sk, what, when, 1556 tcp_rto_max(sk)); 1557} 1558 1559/* Something is really bad, we could not queue an additional packet, 1560 * because qdisc is full or receiver sent a 0 window, or we are paced. 1561 * We do not want to add fuel to the fire, or abort too early, 1562 * so make sure the timer we arm now is at least 200ms in the future, 1563 * regardless of current icsk_rto value (as it could be ~2ms) 1564 */ 1565static inline unsigned long tcp_probe0_base(const struct sock *sk) 1566{ 1567 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 1568} 1569 1570/* Variant of inet_csk_rto_backoff() used for zero window probes */ 1571static inline unsigned long tcp_probe0_when(const struct sock *sk, 1572 unsigned long max_when) 1573{ 1574 u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1, 1575 inet_csk(sk)->icsk_backoff); 1576 u64 when = (u64)tcp_probe0_base(sk) << backoff; 1577 1578 return (unsigned long)min_t(u64, when, max_when); 1579} 1580 1581static inline void tcp_check_probe_timer(struct sock *sk) 1582{ 1583 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 1584 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 1585 tcp_probe0_base(sk), true); 1586} 1587 1588static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 1589{ 1590 tp->snd_wl1 = seq; 1591} 1592 1593static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 1594{ 1595 tp->snd_wl1 = seq; 1596} 1597 1598/* 1599 * Calculate(/check) TCP checksum 1600 */ 1601static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1602 __be32 daddr, __wsum base) 1603{ 1604 return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base); 1605} 1606 1607static inline bool tcp_checksum_complete(struct sk_buff *skb) 1608{ 1609 return !skb_csum_unnecessary(skb) && 1610 __skb_checksum_complete(skb); 1611} 1612 1613bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, 1614 enum skb_drop_reason *reason); 1615 1616 1617int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason); 1618void tcp_set_state(struct sock *sk, int state); 1619void tcp_done(struct sock *sk); 1620int tcp_abort(struct sock *sk, int err); 1621 1622static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 1623{ 1624 rx_opt->dsack = 0; 1625 rx_opt->num_sacks = 0; 1626} 1627 1628void tcp_cwnd_restart(struct sock *sk, s32 delta); 1629 1630static inline void tcp_slow_start_after_idle_check(struct sock *sk) 1631{ 1632 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 1633 struct tcp_sock *tp = tcp_sk(sk); 1634 s32 delta; 1635 1636 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || 1637 tp->packets_out || ca_ops->cong_control) 1638 return; 1639 delta = tcp_jiffies32 - tp->lsndtime; 1640 if (delta > inet_csk(sk)->icsk_rto) 1641 tcp_cwnd_restart(sk, delta); 1642} 1643 1644/* Determine a window scaling and initial window to offer. */ 1645void tcp_select_initial_window(const struct sock *sk, int __space, 1646 __u32 mss, __u32 *rcv_wnd, 1647 __u32 *window_clamp, int wscale_ok, 1648 __u8 *rcv_wscale, __u32 init_rcv_wnd); 1649 1650static inline int __tcp_win_from_space(u8 scaling_ratio, int space) 1651{ 1652 s64 scaled_space = (s64)space * scaling_ratio; 1653 1654 return scaled_space >> TCP_RMEM_TO_WIN_SCALE; 1655} 1656 1657static inline int tcp_win_from_space(const struct sock *sk, int space) 1658{ 1659 return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); 1660} 1661 1662/* inverse of __tcp_win_from_space() */ 1663static inline int __tcp_space_from_win(u8 scaling_ratio, int win) 1664{ 1665 u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE; 1666 1667 do_div(val, scaling_ratio); 1668 return val; 1669} 1670 1671static inline int tcp_space_from_win(const struct sock *sk, int win) 1672{ 1673 return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); 1674} 1675 1676/* Assume a 50% default for skb->len/skb->truesize ratio. 1677 * This may be adjusted later in tcp_measure_rcv_mss(). 1678 */ 1679#define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1)) 1680 1681static inline void tcp_scaling_ratio_init(struct sock *sk) 1682{ 1683 tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; 1684} 1685 1686/* Note: caller must be prepared to deal with negative returns */ 1687static inline int tcp_space(const struct sock *sk) 1688{ 1689 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - 1690 READ_ONCE(sk->sk_backlog.len) - 1691 atomic_read(&sk->sk_rmem_alloc)); 1692} 1693 1694static inline int tcp_full_space(const struct sock *sk) 1695{ 1696 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 1697} 1698 1699static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) 1700{ 1701 int unused_mem = sk_unused_reserved_mem(sk); 1702 struct tcp_sock *tp = tcp_sk(sk); 1703 1704 tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh); 1705 if (unused_mem) 1706 tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh, 1707 tcp_win_from_space(sk, unused_mem)); 1708} 1709 1710static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) 1711{ 1712 __tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss); 1713} 1714 1715void tcp_cleanup_rbuf(struct sock *sk, int copied); 1716void __tcp_cleanup_rbuf(struct sock *sk, int copied); 1717 1718 1719/* We provision sk_rcvbuf around 200% of sk_rcvlowat. 1720 * If 87.5 % (7/8) of the space has been consumed, we want to override 1721 * SO_RCVLOWAT constraint, since we are receiving skbs with too small 1722 * len/truesize ratio. 1723 */ 1724static inline bool tcp_rmem_pressure(const struct sock *sk) 1725{ 1726 int rcvbuf, threshold; 1727 1728 if (tcp_under_memory_pressure(sk)) 1729 return true; 1730 1731 rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1732 threshold = rcvbuf - (rcvbuf >> 3); 1733 1734 return atomic_read(&sk->sk_rmem_alloc) > threshold; 1735} 1736 1737static inline bool tcp_epollin_ready(const struct sock *sk, int target) 1738{ 1739 const struct tcp_sock *tp = tcp_sk(sk); 1740 int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); 1741 1742 if (avail <= 0) 1743 return false; 1744 1745 return (avail >= target) || tcp_rmem_pressure(sk) || 1746 (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); 1747} 1748 1749extern void tcp_openreq_init_rwin(struct request_sock *req, 1750 const struct sock *sk_listener, 1751 const struct dst_entry *dst); 1752 1753void tcp_enter_memory_pressure(struct sock *sk); 1754void tcp_leave_memory_pressure(struct sock *sk); 1755 1756static inline int keepalive_intvl_when(const struct tcp_sock *tp) 1757{ 1758 struct net *net = sock_net((struct sock *)tp); 1759 int val; 1760 1761 /* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl() 1762 * and do_tcp_setsockopt(). 1763 */ 1764 val = READ_ONCE(tp->keepalive_intvl); 1765 1766 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); 1767} 1768 1769static inline int keepalive_time_when(const struct tcp_sock *tp) 1770{ 1771 struct net *net = sock_net((struct sock *)tp); 1772 int val; 1773 1774 /* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */ 1775 val = READ_ONCE(tp->keepalive_time); 1776 1777 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 1778} 1779 1780static inline int keepalive_probes(const struct tcp_sock *tp) 1781{ 1782 struct net *net = sock_net((struct sock *)tp); 1783 int val; 1784 1785 /* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt() 1786 * and do_tcp_setsockopt(). 1787 */ 1788 val = READ_ONCE(tp->keepalive_probes); 1789 1790 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); 1791} 1792 1793static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 1794{ 1795 const struct inet_connection_sock *icsk = &tp->inet_conn; 1796 1797 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 1798 tcp_jiffies32 - tp->rcv_tstamp); 1799} 1800 1801static inline int tcp_fin_time(const struct sock *sk) 1802{ 1803 int fin_timeout = tcp_sk(sk)->linger2 ? : 1804 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); 1805 const int rto = inet_csk(sk)->icsk_rto; 1806 1807 if (fin_timeout < (rto << 2) - (rto >> 1)) 1808 fin_timeout = (rto << 2) - (rto >> 1); 1809 1810 return fin_timeout; 1811} 1812 1813static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1814 int paws_win) 1815{ 1816 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1817 return true; 1818 if (unlikely(!time_before32(ktime_get_seconds(), 1819 rx_opt->ts_recent_stamp + TCP_PAWS_WRAP))) 1820 return true; 1821 /* 1822 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1823 * then following tcp messages have valid values. Ignore 0 value, 1824 * or else 'negative' tsval might forbid us to accept their packets. 1825 */ 1826 if (!rx_opt->ts_recent) 1827 return true; 1828 return false; 1829} 1830 1831static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1832 int rst) 1833{ 1834 if (tcp_paws_check(rx_opt, 0)) 1835 return false; 1836 1837 /* RST segments are not recommended to carry timestamp, 1838 and, if they do, it is recommended to ignore PAWS because 1839 "their cleanup function should take precedence over timestamps." 1840 Certainly, it is mistake. It is necessary to understand the reasons 1841 of this constraint to relax it: if peer reboots, clock may go 1842 out-of-sync and half-open connections will not be reset. 1843 Actually, the problem would be not existing if all 1844 the implementations followed draft about maintaining clock 1845 via reboots. Linux-2.2 DOES NOT! 1846 1847 However, we can relax time bounds for RST segments to MSL. 1848 */ 1849 if (rst && !time_before32(ktime_get_seconds(), 1850 rx_opt->ts_recent_stamp + TCP_PAWS_MSL)) 1851 return false; 1852 return true; 1853} 1854 1855static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 1856{ 1857 u32 ace; 1858 1859 /* mptcp hooks are only on the slow path */ 1860 if (sk_is_mptcp((struct sock *)tp)) 1861 return; 1862 1863 ace = tcp_ecn_mode_accecn(tp) ? 1864 ((tp->delivered_ce + TCP_ACCECN_CEP_INIT_OFFSET) & 1865 TCP_ACCECN_CEP_ACE_MASK) : 0; 1866 1867 tp->pred_flags = htonl((tp->tcp_header_len << 26) | 1868 (ace << 22) | 1869 ntohl(TCP_FLAG_ACK) | 1870 snd_wnd); 1871} 1872 1873static inline void tcp_fast_path_on(struct tcp_sock *tp) 1874{ 1875 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 1876} 1877 1878static inline void tcp_fast_path_check(struct sock *sk) 1879{ 1880 struct tcp_sock *tp = tcp_sk(sk); 1881 1882 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 1883 tp->rcv_wnd && 1884 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 1885 !tp->urg_data) 1886 tcp_fast_path_on(tp); 1887} 1888 1889bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 1890 int mib_idx, u32 *last_oow_ack_time); 1891 1892static inline void tcp_mib_init(struct net *net) 1893{ 1894 /* See RFC 2012 */ 1895 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 1896 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 1897 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 1898 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 1899} 1900 1901/* from STCP */ 1902static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 1903{ 1904 tp->retransmit_skb_hint = NULL; 1905} 1906 1907#define tcp_md5_addr tcp_ao_addr 1908 1909/* - key database */ 1910struct tcp_md5sig_key { 1911 struct hlist_node node; 1912 u8 keylen; 1913 u8 family; /* AF_INET or AF_INET6 */ 1914 u8 prefixlen; 1915 u8 flags; 1916 union tcp_md5_addr addr; 1917 int l3index; /* set if key added with L3 scope */ 1918 u8 key[TCP_MD5SIG_MAXKEYLEN]; 1919 struct rcu_head rcu; 1920}; 1921 1922/* - sock block */ 1923struct tcp_md5sig_info { 1924 struct hlist_head head; 1925 struct rcu_head rcu; 1926}; 1927 1928/* - pseudo header */ 1929struct tcp4_pseudohdr { 1930 __be32 saddr; 1931 __be32 daddr; 1932 __u8 pad; 1933 __u8 protocol; 1934 __be16 len; 1935}; 1936 1937struct tcp6_pseudohdr { 1938 struct in6_addr saddr; 1939 struct in6_addr daddr; 1940 __be32 len; 1941 __be32 protocol; /* including padding */ 1942}; 1943 1944/* 1945 * struct tcp_sigpool - per-CPU pool of ahash_requests 1946 * @scratch: per-CPU temporary area, that can be used between 1947 * tcp_sigpool_start() and tcp_sigpool_end() to perform 1948 * crypto request 1949 * @req: pre-allocated ahash request 1950 */ 1951struct tcp_sigpool { 1952 void *scratch; 1953 struct ahash_request *req; 1954}; 1955 1956int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size); 1957void tcp_sigpool_get(unsigned int id); 1958void tcp_sigpool_release(unsigned int id); 1959int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp, 1960 const struct sk_buff *skb, 1961 unsigned int header_len); 1962 1963/** 1964 * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash 1965 * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash() 1966 * @c: returned tcp_sigpool for usage (uninitialized on failure) 1967 * 1968 * Returns: 0 on success, error otherwise. 1969 */ 1970int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c); 1971/** 1972 * tcp_sigpool_end - enable bh and stop using tcp_sigpool 1973 * @c: tcp_sigpool context that was returned by tcp_sigpool_start() 1974 */ 1975void tcp_sigpool_end(struct tcp_sigpool *c); 1976size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len); 1977/* - functions */ 1978void tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 1979 const struct sock *sk, const struct sk_buff *skb); 1980int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1981 int family, u8 prefixlen, int l3index, u8 flags, 1982 const u8 *newkey, u8 newkeylen); 1983int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 1984 int family, u8 prefixlen, int l3index, 1985 struct tcp_md5sig_key *key); 1986 1987int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 1988 int family, u8 prefixlen, int l3index, u8 flags); 1989void tcp_clear_md5_list(struct sock *sk); 1990struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1991 const struct sock *addr_sk); 1992 1993#ifdef CONFIG_TCP_MD5SIG 1994struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 1995 const union tcp_md5_addr *addr, 1996 int family, bool any_l3index); 1997static inline struct tcp_md5sig_key * 1998tcp_md5_do_lookup(const struct sock *sk, int l3index, 1999 const union tcp_md5_addr *addr, int family) 2000{ 2001 if (!static_branch_unlikely(&tcp_md5_needed.key)) 2002 return NULL; 2003 return __tcp_md5_do_lookup(sk, l3index, addr, family, false); 2004} 2005 2006static inline struct tcp_md5sig_key * 2007tcp_md5_do_lookup_any_l3index(const struct sock *sk, 2008 const union tcp_md5_addr *addr, int family) 2009{ 2010 if (!static_branch_unlikely(&tcp_md5_needed.key)) 2011 return NULL; 2012 return __tcp_md5_do_lookup(sk, 0, addr, family, true); 2013} 2014 2015#define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 2016void tcp_md5_destruct_sock(struct sock *sk); 2017#else 2018static inline struct tcp_md5sig_key * 2019tcp_md5_do_lookup(const struct sock *sk, int l3index, 2020 const union tcp_md5_addr *addr, int family) 2021{ 2022 return NULL; 2023} 2024 2025static inline struct tcp_md5sig_key * 2026tcp_md5_do_lookup_any_l3index(const struct sock *sk, 2027 const union tcp_md5_addr *addr, int family) 2028{ 2029 return NULL; 2030} 2031 2032#define tcp_twsk_md5_key(twsk) NULL 2033static inline void tcp_md5_destruct_sock(struct sock *sk) 2034{ 2035} 2036#endif 2037 2038struct md5_ctx; 2039void tcp_md5_hash_skb_data(struct md5_ctx *ctx, const struct sk_buff *skb, 2040 unsigned int header_len); 2041void tcp_md5_hash_key(struct md5_ctx *ctx, const struct tcp_md5sig_key *key); 2042 2043/* From tcp_fastopen.c */ 2044void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 2045 struct tcp_fastopen_cookie *cookie); 2046void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 2047 struct tcp_fastopen_cookie *cookie, bool syn_lost, 2048 u16 try_exp); 2049struct tcp_fastopen_request { 2050 /* Fast Open cookie. Size 0 means a cookie request */ 2051 struct tcp_fastopen_cookie cookie; 2052 struct msghdr *data; /* data in MSG_FASTOPEN */ 2053 size_t size; 2054 int copied; /* queued in tcp_connect() */ 2055 struct ubuf_info *uarg; 2056}; 2057void tcp_free_fastopen_req(struct tcp_sock *tp); 2058void tcp_fastopen_destroy_cipher(struct sock *sk); 2059void tcp_fastopen_ctx_destroy(struct net *net); 2060int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 2061 void *primary_key, void *backup_key); 2062int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 2063 u64 *key); 2064void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 2065struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 2066 struct request_sock *req, 2067 struct tcp_fastopen_cookie *foc, 2068 const struct dst_entry *dst); 2069void tcp_fastopen_init_key_once(struct net *net); 2070bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 2071 struct tcp_fastopen_cookie *cookie); 2072bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 2073#define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t) 2074#define TCP_FASTOPEN_KEY_MAX 2 2075#define TCP_FASTOPEN_KEY_BUF_LENGTH \ 2076 (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX) 2077 2078/* Fastopen key context */ 2079struct tcp_fastopen_context { 2080 siphash_key_t key[TCP_FASTOPEN_KEY_MAX]; 2081 int num; 2082 struct rcu_head rcu; 2083}; 2084 2085void tcp_fastopen_active_disable(struct sock *sk); 2086bool tcp_fastopen_active_should_disable(struct sock *sk); 2087void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 2088void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 2089 2090/* Caller needs to wrap with rcu_read_(un)lock() */ 2091static inline 2092struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) 2093{ 2094 struct tcp_fastopen_context *ctx; 2095 2096 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); 2097 if (!ctx) 2098 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); 2099 return ctx; 2100} 2101 2102static inline 2103bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc, 2104 const struct tcp_fastopen_cookie *orig) 2105{ 2106 if (orig->len == TCP_FASTOPEN_COOKIE_SIZE && 2107 orig->len == foc->len && 2108 !memcmp(orig->val, foc->val, foc->len)) 2109 return true; 2110 return false; 2111} 2112 2113static inline 2114int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx) 2115{ 2116 return ctx->num; 2117} 2118 2119/* Latencies incurred by various limits for a sender. They are 2120 * chronograph-like stats that are mutually exclusive. 2121 */ 2122enum tcp_chrono { 2123 TCP_CHRONO_UNSPEC, 2124 TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 2125 TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 2126 TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 2127 __TCP_CHRONO_MAX, 2128}; 2129 2130void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); 2131void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 2132 2133/* This helper is needed, because skb->tcp_tsorted_anchor uses 2134 * the same memory storage than skb->destructor/_skb_refdst 2135 */ 2136static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 2137{ 2138 skb->destructor = NULL; 2139 skb->_skb_refdst = 0UL; 2140} 2141 2142#define tcp_skb_tsorted_save(skb) { \ 2143 unsigned long _save = skb->_skb_refdst; \ 2144 skb->_skb_refdst = 0UL; 2145 2146#define tcp_skb_tsorted_restore(skb) \ 2147 skb->_skb_refdst = _save; \ 2148} 2149 2150void tcp_write_queue_purge(struct sock *sk); 2151 2152static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 2153{ 2154 return skb_rb_first(&sk->tcp_rtx_queue); 2155} 2156 2157static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) 2158{ 2159 return skb_rb_last(&sk->tcp_rtx_queue); 2160} 2161 2162static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 2163{ 2164 return skb_peek_tail(&sk->sk_write_queue); 2165} 2166 2167#define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 2168 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 2169 2170static inline struct sk_buff *tcp_send_head(const struct sock *sk) 2171{ 2172 return skb_peek(&sk->sk_write_queue); 2173} 2174 2175static inline bool tcp_skb_is_last(const struct sock *sk, 2176 const struct sk_buff *skb) 2177{ 2178 return skb_queue_is_last(&sk->sk_write_queue, skb); 2179} 2180 2181/** 2182 * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue 2183 * @sk: socket 2184 * 2185 * Since the write queue can have a temporary empty skb in it, 2186 * we must not use "return skb_queue_empty(&sk->sk_write_queue)" 2187 */ 2188static inline bool tcp_write_queue_empty(const struct sock *sk) 2189{ 2190 const struct tcp_sock *tp = tcp_sk(sk); 2191 2192 return tp->write_seq == tp->snd_nxt; 2193} 2194 2195static inline bool tcp_rtx_queue_empty(const struct sock *sk) 2196{ 2197 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 2198} 2199 2200static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 2201{ 2202 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 2203} 2204 2205static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 2206{ 2207 __skb_queue_tail(&sk->sk_write_queue, skb); 2208 2209 /* Queue it, remembering where we must start sending. */ 2210 if (sk->sk_write_queue.next == skb) 2211 tcp_chrono_start(sk, TCP_CHRONO_BUSY); 2212} 2213 2214/* Insert new before skb on the write queue of sk. */ 2215static inline void tcp_insert_write_queue_before(struct sk_buff *new, 2216 struct sk_buff *skb, 2217 struct sock *sk) 2218{ 2219 __skb_queue_before(&sk->sk_write_queue, skb, new); 2220} 2221 2222static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 2223{ 2224 tcp_skb_tsorted_anchor_cleanup(skb); 2225 __skb_unlink(skb, &sk->sk_write_queue); 2226} 2227 2228void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 2229 2230static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 2231{ 2232 tcp_skb_tsorted_anchor_cleanup(skb); 2233 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 2234} 2235 2236static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 2237{ 2238 list_del(&skb->tcp_tsorted_anchor); 2239 tcp_rtx_queue_unlink(skb, sk); 2240 tcp_wmem_free_skb(sk, skb); 2241} 2242 2243static inline void tcp_write_collapse_fence(struct sock *sk) 2244{ 2245 struct sk_buff *skb = tcp_write_queue_tail(sk); 2246 2247 if (skb) 2248 TCP_SKB_CB(skb)->eor = 1; 2249} 2250 2251static inline void tcp_push_pending_frames(struct sock *sk) 2252{ 2253 if (tcp_send_head(sk)) { 2254 struct tcp_sock *tp = tcp_sk(sk); 2255 2256 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 2257 } 2258} 2259 2260/* Start sequence of the skb just after the highest skb with SACKed 2261 * bit, valid only if sacked_out > 0 or when the caller has ensured 2262 * validity by itself. 2263 */ 2264static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 2265{ 2266 if (!tp->sacked_out) 2267 return tp->snd_una; 2268 2269 if (tp->highest_sack == NULL) 2270 return tp->snd_nxt; 2271 2272 return TCP_SKB_CB(tp->highest_sack)->seq; 2273} 2274 2275static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 2276{ 2277 tcp_sk(sk)->highest_sack = skb_rb_next(skb); 2278} 2279 2280static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 2281{ 2282 return tcp_sk(sk)->highest_sack; 2283} 2284 2285static inline void tcp_highest_sack_reset(struct sock *sk) 2286{ 2287 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 2288} 2289 2290/* Called when old skb is about to be deleted and replaced by new skb */ 2291static inline void tcp_highest_sack_replace(struct sock *sk, 2292 struct sk_buff *old, 2293 struct sk_buff *new) 2294{ 2295 if (old == tcp_highest_sack(sk)) 2296 tcp_sk(sk)->highest_sack = new; 2297} 2298 2299/* This helper checks if socket has IP_TRANSPARENT set */ 2300static inline bool inet_sk_transparent(const struct sock *sk) 2301{ 2302 switch (sk->sk_state) { 2303 case TCP_TIME_WAIT: 2304 return inet_twsk(sk)->tw_transparent; 2305 case TCP_NEW_SYN_RECV: 2306 return inet_rsk(inet_reqsk(sk))->no_srccheck; 2307 } 2308 return inet_test_bit(TRANSPARENT, sk); 2309} 2310 2311/* Determines whether this is a thin stream (which may suffer from 2312 * increased latency). Used to trigger latency-reducing mechanisms. 2313 */ 2314static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 2315{ 2316 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 2317} 2318 2319/* /proc */ 2320enum tcp_seq_states { 2321 TCP_SEQ_STATE_LISTENING, 2322 TCP_SEQ_STATE_ESTABLISHED, 2323}; 2324 2325void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 2326void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 2327void tcp_seq_stop(struct seq_file *seq, void *v); 2328 2329struct tcp_seq_afinfo { 2330 sa_family_t family; 2331}; 2332 2333struct tcp_iter_state { 2334 struct seq_net_private p; 2335 enum tcp_seq_states state; 2336 struct sock *syn_wait_sk; 2337 int bucket, offset, sbucket, num; 2338 loff_t last_pos; 2339}; 2340 2341extern struct request_sock_ops tcp_request_sock_ops; 2342extern struct request_sock_ops tcp6_request_sock_ops; 2343 2344void tcp_v4_destroy_sock(struct sock *sk); 2345 2346struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 2347 netdev_features_t features); 2348struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th); 2349struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb, 2350 struct tcphdr *th); 2351INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); 2352INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); 2353INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff)); 2354INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)); 2355#ifdef CONFIG_INET 2356void tcp_gro_complete(struct sk_buff *skb); 2357#else 2358static inline void tcp_gro_complete(struct sk_buff *skb) { } 2359#endif 2360 2361void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); 2362 2363static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 2364{ 2365 struct net *net = sock_net((struct sock *)tp); 2366 u32 val; 2367 2368 val = READ_ONCE(tp->notsent_lowat); 2369 2370 return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); 2371} 2372 2373bool tcp_stream_memory_free(const struct sock *sk, int wake); 2374 2375#ifdef CONFIG_PROC_FS 2376int tcp4_proc_init(void); 2377void tcp4_proc_exit(void); 2378#endif 2379 2380int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 2381int tcp_conn_request(struct request_sock_ops *rsk_ops, 2382 const struct tcp_request_sock_ops *af_ops, 2383 struct sock *sk, struct sk_buff *skb); 2384 2385/* TCP af-specific functions */ 2386struct tcp_sock_af_ops { 2387#ifdef CONFIG_TCP_MD5SIG 2388 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 2389 const struct sock *addr_sk); 2390 void (*calc_md5_hash)(char *location, 2391 const struct tcp_md5sig_key *md5, 2392 const struct sock *sk, 2393 const struct sk_buff *skb); 2394 int (*md5_parse)(struct sock *sk, 2395 int optname, 2396 sockptr_t optval, 2397 int optlen); 2398#endif 2399#ifdef CONFIG_TCP_AO 2400 int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen); 2401 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, 2402 struct sock *addr_sk, 2403 int sndid, int rcvid); 2404 int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key, 2405 const struct sock *sk, 2406 __be32 sisn, __be32 disn, bool send); 2407 int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao, 2408 const struct sock *sk, const struct sk_buff *skb, 2409 const u8 *tkey, int hash_offset, u32 sne); 2410#endif 2411}; 2412 2413struct tcp_request_sock_ops { 2414 u16 mss_clamp; 2415#ifdef CONFIG_TCP_MD5SIG 2416 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 2417 const struct sock *addr_sk); 2418 void (*calc_md5_hash) (char *location, 2419 const struct tcp_md5sig_key *md5, 2420 const struct sock *sk, 2421 const struct sk_buff *skb); 2422#endif 2423#ifdef CONFIG_TCP_AO 2424 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, 2425 struct request_sock *req, 2426 int sndid, int rcvid); 2427 int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk); 2428 int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt, 2429 struct request_sock *req, const struct sk_buff *skb, 2430 int hash_offset, u32 sne); 2431#endif 2432#ifdef CONFIG_SYN_COOKIES 2433 __u32 (*cookie_init_seq)(const struct sk_buff *skb, 2434 __u16 *mss); 2435#endif 2436 struct dst_entry *(*route_req)(const struct sock *sk, 2437 struct sk_buff *skb, 2438 struct flowi *fl, 2439 struct request_sock *req, 2440 u32 tw_isn); 2441 union tcp_seq_and_ts_off (*init_seq_and_ts_off)( 2442 const struct net *net, 2443 const struct sk_buff *skb); 2444 int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 2445 struct flowi *fl, struct request_sock *req, 2446 struct tcp_fastopen_cookie *foc, 2447 enum tcp_synack_type synack_type, 2448 struct sk_buff *syn_skb); 2449}; 2450 2451extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops; 2452#if IS_ENABLED(CONFIG_IPV6) 2453extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; 2454#endif 2455 2456#ifdef CONFIG_SYN_COOKIES 2457static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 2458 const struct sock *sk, struct sk_buff *skb, 2459 __u16 *mss) 2460{ 2461 tcp_synq_overflow(sk); 2462 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 2463 return ops->cookie_init_seq(skb, mss); 2464} 2465#else 2466static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 2467 const struct sock *sk, struct sk_buff *skb, 2468 __u16 *mss) 2469{ 2470 return 0; 2471} 2472#endif 2473 2474struct tcp_key { 2475 union { 2476 struct { 2477 struct tcp_ao_key *ao_key; 2478 char *traffic_key; 2479 u32 sne; 2480 u8 rcv_next; 2481 }; 2482 struct tcp_md5sig_key *md5_key; 2483 }; 2484 enum { 2485 TCP_KEY_NONE = 0, 2486 TCP_KEY_MD5, 2487 TCP_KEY_AO, 2488 } type; 2489}; 2490 2491static inline void tcp_get_current_key(const struct sock *sk, 2492 struct tcp_key *out) 2493{ 2494#if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG) 2495 const struct tcp_sock *tp = tcp_sk(sk); 2496#endif 2497 2498#ifdef CONFIG_TCP_AO 2499 if (static_branch_unlikely(&tcp_ao_needed.key)) { 2500 struct tcp_ao_info *ao; 2501 2502 ao = rcu_dereference_protected(tp->ao_info, 2503 lockdep_sock_is_held(sk)); 2504 if (ao) { 2505 out->ao_key = READ_ONCE(ao->current_key); 2506 out->type = TCP_KEY_AO; 2507 return; 2508 } 2509 } 2510#endif 2511#ifdef CONFIG_TCP_MD5SIG 2512 if (static_branch_unlikely(&tcp_md5_needed.key) && 2513 rcu_access_pointer(tp->md5sig_info)) { 2514 out->md5_key = tp->af_specific->md5_lookup(sk, sk); 2515 if (out->md5_key) { 2516 out->type = TCP_KEY_MD5; 2517 return; 2518 } 2519 } 2520#endif 2521 out->type = TCP_KEY_NONE; 2522} 2523 2524static inline bool tcp_key_is_md5(const struct tcp_key *key) 2525{ 2526 if (static_branch_tcp_md5()) 2527 return key->type == TCP_KEY_MD5; 2528 return false; 2529} 2530 2531static inline bool tcp_key_is_ao(const struct tcp_key *key) 2532{ 2533 if (static_branch_tcp_ao()) 2534 return key->type == TCP_KEY_AO; 2535 return false; 2536} 2537 2538int tcpv4_offload_init(void); 2539 2540void tcp_v4_init(void); 2541void tcp_init(void); 2542 2543/* tcp_recovery.c */ 2544void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 2545void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 2546extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 2547 u32 reo_wnd); 2548extern bool tcp_rack_mark_lost(struct sock *sk); 2549extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, 2550 u64 xmit_time); 2551extern void tcp_rack_reo_timeout(struct sock *sk); 2552extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); 2553 2554/* tcp_plb.c */ 2555 2556/* 2557 * Scaling factor for fractions in PLB. For example, tcp_plb_update_state 2558 * expects cong_ratio which represents fraction of traffic that experienced 2559 * congestion over a single RTT. In order to avoid floating point operations, 2560 * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in. 2561 */ 2562#define TCP_PLB_SCALE 8 2563 2564/* State for PLB (Protective Load Balancing) for a single TCP connection. */ 2565struct tcp_plb_state { 2566 u8 consec_cong_rounds:5, /* consecutive congested rounds */ 2567 unused:3; 2568 u32 pause_until; /* jiffies32 when PLB can resume rerouting */ 2569}; 2570 2571static inline void tcp_plb_init(const struct sock *sk, 2572 struct tcp_plb_state *plb) 2573{ 2574 plb->consec_cong_rounds = 0; 2575 plb->pause_until = 0; 2576} 2577void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, 2578 const int cong_ratio); 2579void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); 2580void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); 2581 2582static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str) 2583{ 2584 WARN_ONCE(cond, 2585 "%scwn:%u out:%u sacked:%u lost:%u retrans:%u tlp_high_seq:%u sk_state:%u ca_state:%u advmss:%u mss_cache:%u pmtu:%u\n", 2586 str, 2587 tcp_snd_cwnd(tcp_sk(sk)), 2588 tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out, 2589 tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out, 2590 tcp_sk(sk)->tlp_high_seq, sk->sk_state, 2591 inet_csk(sk)->icsk_ca_state, 2592 tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache, 2593 inet_csk(sk)->icsk_pmtu_cookie); 2594} 2595 2596/* At how many usecs into the future should the RTO fire? */ 2597static inline s64 tcp_rto_delta_us(const struct sock *sk) 2598{ 2599 const struct sk_buff *skb = tcp_rtx_queue_head(sk); 2600 u32 rto = inet_csk(sk)->icsk_rto; 2601 2602 if (likely(skb)) { 2603 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto); 2604 2605 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 2606 } else { 2607 tcp_warn_once(sk, 1, "rtx queue empty: "); 2608 return jiffies_to_usecs(rto); 2609 } 2610 2611} 2612 2613/* 2614 * Save and compile IPv4 options, return a pointer to it 2615 */ 2616static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 2617 struct sk_buff *skb) 2618{ 2619 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 2620 struct ip_options_rcu *dopt = NULL; 2621 2622 if (opt->optlen) { 2623 int opt_size = sizeof(*dopt) + opt->optlen; 2624 2625 dopt = kmalloc(opt_size, GFP_ATOMIC); 2626 if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 2627 kfree(dopt); 2628 dopt = NULL; 2629 } 2630 } 2631 return dopt; 2632} 2633 2634/* locally generated TCP pure ACKs have skb->truesize == 2 2635 * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 2636 * This is much faster than dissecting the packet to find out. 2637 * (Think of GRE encapsulations, IPv4, IPv6, ...) 2638 */ 2639static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 2640{ 2641 return skb->truesize == 2; 2642} 2643 2644static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 2645{ 2646 skb->truesize = 2; 2647} 2648 2649static inline int tcp_inq(struct sock *sk) 2650{ 2651 struct tcp_sock *tp = tcp_sk(sk); 2652 int answ; 2653 2654 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 2655 answ = 0; 2656 } else if (sock_flag(sk, SOCK_URGINLINE) || 2657 !tp->urg_data || 2658 before(tp->urg_seq, tp->copied_seq) || 2659 !before(tp->urg_seq, tp->rcv_nxt)) { 2660 2661 answ = tp->rcv_nxt - tp->copied_seq; 2662 2663 /* Subtract 1, if FIN was received */ 2664 if (answ && sock_flag(sk, SOCK_DONE)) 2665 answ--; 2666 } else { 2667 answ = tp->urg_seq - tp->copied_seq; 2668 } 2669 2670 return answ; 2671} 2672 2673int tcp_peek_len(struct socket *sock); 2674 2675static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2676{ 2677 u16 segs_in; 2678 2679 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 2680 2681 /* We update these fields while other threads might 2682 * read them from tcp_get_info() 2683 */ 2684 WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); 2685 if (skb->len > tcp_hdrlen(skb)) 2686 WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); 2687} 2688 2689/* 2690 * TCP listen path runs lockless. 2691 * We forced "struct sock" to be const qualified to make sure 2692 * we don't modify one of its field by mistake. 2693 * Here, we increment sk_drops which is an atomic_t, so we can safely 2694 * make sock writable again. 2695 */ 2696static inline void tcp_listendrop(const struct sock *sk) 2697{ 2698 sk_drops_inc((struct sock *)sk); 2699 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 2700} 2701 2702enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2703 2704/* 2705 * Interface for adding Upper Level Protocols over TCP 2706 */ 2707 2708#define TCP_ULP_NAME_MAX 16 2709#define TCP_ULP_MAX 128 2710#define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2711 2712struct tcp_ulp_ops { 2713 struct list_head list; 2714 2715 /* initialize ulp */ 2716 int (*init)(struct sock *sk); 2717 /* update ulp */ 2718 void (*update)(struct sock *sk, struct proto *p, 2719 void (*write_space)(struct sock *sk)); 2720 /* cleanup ulp */ 2721 void (*release)(struct sock *sk); 2722 /* diagnostic */ 2723 int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin); 2724 size_t (*get_info_size)(const struct sock *sk, bool net_admin); 2725 /* clone ulp */ 2726 void (*clone)(const struct request_sock *req, struct sock *newsk, 2727 const gfp_t priority); 2728 2729 char name[TCP_ULP_NAME_MAX]; 2730 struct module *owner; 2731}; 2732int tcp_register_ulp(struct tcp_ulp_ops *type); 2733void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2734int tcp_set_ulp(struct sock *sk, const char *name); 2735void tcp_get_available_ulp(char *buf, size_t len); 2736void tcp_cleanup_ulp(struct sock *sk); 2737void tcp_update_ulp(struct sock *sk, struct proto *p, 2738 void (*write_space)(struct sock *sk)); 2739 2740#define MODULE_ALIAS_TCP_ULP(name) \ 2741 MODULE_INFO(alias, name); \ 2742 MODULE_INFO(alias, "tcp-ulp-" name) 2743 2744#ifdef CONFIG_NET_SOCK_MSG 2745struct sk_msg; 2746struct sk_psock; 2747 2748#ifdef CONFIG_BPF_SYSCALL 2749int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); 2750void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 2751#ifdef CONFIG_BPF_STREAM_PARSER 2752struct strparser; 2753int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc, 2754 sk_read_actor_t recv_actor); 2755#endif /* CONFIG_BPF_STREAM_PARSER */ 2756#endif /* CONFIG_BPF_SYSCALL */ 2757 2758#ifdef CONFIG_INET 2759void tcp_eat_skb(struct sock *sk, struct sk_buff *skb); 2760#else 2761static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 2762{ 2763} 2764#endif 2765 2766int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 2767 struct sk_msg *msg, u32 bytes, int flags); 2768#endif /* CONFIG_NET_SOCK_MSG */ 2769 2770#if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 2771static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 2772{ 2773} 2774#endif 2775 2776#ifdef CONFIG_CGROUP_BPF 2777static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 2778 struct sk_buff *skb, 2779 unsigned int end_offset) 2780{ 2781 skops->skb = skb; 2782 skops->skb_data_end = skb->data + end_offset; 2783} 2784#else 2785static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 2786 struct sk_buff *skb, 2787 unsigned int end_offset) 2788{ 2789} 2790#endif 2791 2792/* Call BPF_SOCK_OPS program that returns an int. If the return value 2793 * is < 0, then the BPF op failed (for example if the loaded BPF 2794 * program does not support the chosen operation or there is no BPF 2795 * program loaded). 2796 */ 2797#ifdef CONFIG_BPF 2798static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 2799{ 2800 struct bpf_sock_ops_kern sock_ops; 2801 int ret; 2802 2803 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2804 if (sk_fullsock(sk)) { 2805 sock_ops.is_fullsock = 1; 2806 sock_ops.is_locked_tcp_sock = 1; 2807 sock_owned_by_me(sk); 2808 } 2809 2810 sock_ops.sk = sk; 2811 sock_ops.op = op; 2812 if (nargs > 0) 2813 memcpy(sock_ops.args, args, nargs * sizeof(*args)); 2814 2815 ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 2816 if (ret == 0) 2817 ret = sock_ops.reply; 2818 else 2819 ret = -1; 2820 return ret; 2821} 2822 2823static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2824{ 2825 u32 args[2] = {arg1, arg2}; 2826 2827 return tcp_call_bpf(sk, op, 2, args); 2828} 2829 2830static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2831 u32 arg3) 2832{ 2833 u32 args[3] = {arg1, arg2, arg3}; 2834 2835 return tcp_call_bpf(sk, op, 3, args); 2836} 2837 2838#else 2839static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 2840{ 2841 return -EPERM; 2842} 2843 2844static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) 2845{ 2846 return -EPERM; 2847} 2848 2849static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, 2850 u32 arg3) 2851{ 2852 return -EPERM; 2853} 2854 2855#endif 2856 2857static inline u32 tcp_timeout_init(struct sock *sk) 2858{ 2859 int timeout; 2860 2861 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); 2862 2863 if (timeout <= 0) 2864 timeout = TCP_TIMEOUT_INIT; 2865 return min_t(int, timeout, TCP_RTO_MAX); 2866} 2867 2868static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 2869{ 2870 int rwnd; 2871 2872 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); 2873 2874 if (rwnd < 0) 2875 rwnd = 0; 2876 return rwnd; 2877} 2878 2879static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 2880{ 2881 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); 2882} 2883 2884static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) 2885{ 2886 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) 2887 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); 2888} 2889 2890#if IS_ENABLED(CONFIG_SMC) 2891extern struct static_key_false tcp_have_smc; 2892#endif 2893 2894#if IS_ENABLED(CONFIG_TLS_DEVICE) 2895void clean_acked_data_enable(struct tcp_sock *tp, 2896 void (*cad)(struct sock *sk, u32 ack_seq)); 2897void clean_acked_data_disable(struct tcp_sock *tp); 2898void clean_acked_data_flush(void); 2899#endif 2900 2901DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 2902static inline void tcp_add_tx_delay(struct sk_buff *skb, 2903 const struct tcp_sock *tp) 2904{ 2905 if (static_branch_unlikely(&tcp_tx_delay_enabled)) 2906 skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC; 2907} 2908 2909/* Compute Earliest Departure Time for some control packets 2910 * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets. 2911 */ 2912static inline u64 tcp_transmit_time(const struct sock *sk) 2913{ 2914 if (static_branch_unlikely(&tcp_tx_delay_enabled)) { 2915 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? 2916 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; 2917 2918 return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC; 2919 } 2920 return 0; 2921} 2922 2923static inline int tcp_parse_auth_options(const struct tcphdr *th, 2924 const u8 **md5_hash, const struct tcp_ao_hdr **aoh) 2925{ 2926 const u8 *md5_tmp, *ao_tmp; 2927 int ret; 2928 2929 ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp); 2930 if (ret) 2931 return ret; 2932 2933 if (md5_hash) 2934 *md5_hash = md5_tmp; 2935 2936 if (aoh) { 2937 if (!ao_tmp) 2938 *aoh = NULL; 2939 else 2940 *aoh = (struct tcp_ao_hdr *)(ao_tmp - 2); 2941 } 2942 2943 return 0; 2944} 2945 2946static inline bool tcp_ao_required(struct sock *sk, const void *saddr, 2947 int family, int l3index, bool stat_inc) 2948{ 2949#ifdef CONFIG_TCP_AO 2950 struct tcp_ao_info *ao_info; 2951 struct tcp_ao_key *ao_key; 2952 2953 if (!static_branch_unlikely(&tcp_ao_needed.key)) 2954 return false; 2955 2956 ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info, 2957 lockdep_sock_is_held(sk)); 2958 if (!ao_info) 2959 return false; 2960 2961 ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1); 2962 if (ao_info->ao_required || ao_key) { 2963 if (stat_inc) { 2964 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED); 2965 atomic64_inc(&ao_info->counters.ao_required); 2966 } 2967 return true; 2968 } 2969#endif 2970 return false; 2971} 2972 2973enum skb_drop_reason tcp_inbound_hash(struct sock *sk, 2974 const struct request_sock *req, const struct sk_buff *skb, 2975 const void *saddr, const void *daddr, 2976 int family, int dif, int sdif); 2977 2978#endif /* _TCP_H */
~/
Read-only file preview.