tcp_output.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * INET		An implementation of the TCP/IP protocol suite for the LINUX
   4 *		operating system.  INET is implemented using the  BSD Socket
   5 *		interface as the means of communication with the user level.
   6 *
   7 *		Implementation of the Transmission Control Protocol(TCP).
   8 *
   9 * Authors:	Ross Biro
  10 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11 *		Mark Evans, <evansmp@uhura.aston.ac.uk>
  12 *		Corey Minyard <wf-rch!minyard@relay.EU.net>
  13 *		Florian La Roche, <flla@stud.uni-sb.de>
  14 *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  15 *		Linus Torvalds, <torvalds@cs.helsinki.fi>
  16 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
  17 *		Matthew Dillon, <dillon@apollo.west.oic.com>
  18 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  19 *		Jorge Cwik, <jorge@laser.satlink.net>
  20 */
  21
  22/*
  23 * Changes:	Pedro Roque	:	Retransmit queue handled by TCP.
  24 *				:	Fragmentation on mtu decrease
  25 *				:	Segment collapse on retransmit
  26 *				:	AF independence
  27 *
  28 *		Linus Torvalds	:	send_delayed_ack
  29 *		David S. Miller	:	Charge memory using the right skb
  30 *					during syn/ack processing.
  31 *		David S. Miller :	Output engine completely rewritten.
  32 *		Andrea Arcangeli:	SYNACK carry ts_recent in tsecr.
  33 *		Cacophonix Gaul :	draft-minshall-nagle-01
  34 *		J Hadi Salim	:	ECN support
  35 *
  36 */
  37
  38#define pr_fmt(fmt) "TCP: " fmt
  39
  40#include <net/tcp.h>
  41#include <net/tcp_ecn.h>
  42#include <net/mptcp.h>
  43#include <net/smc.h>
  44#include <net/proto_memory.h>
  45#include <net/psp.h>
  46
  47#include <linux/compiler.h>
  48#include <linux/gfp.h>
  49#include <linux/module.h>
  50#include <linux/static_key.h>
  51#include <linux/skbuff_ref.h>
  52
  53#include <trace/events/tcp.h>
  54
  55/* Refresh clocks of a TCP socket,
  56 * ensuring monotically increasing values.
  57 */
  58void tcp_mstamp_refresh(struct tcp_sock *tp)
  59{
  60	u64 val = tcp_clock_ns();
  61
  62	tp->tcp_clock_cache = val;
  63	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
  64}
  65
  66static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
  67			   int push_one, gfp_t gfp);
  68
  69/* Account for new data that has been sent to the network. */
  70static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
  71{
  72	struct inet_connection_sock *icsk = inet_csk(sk);
  73	struct tcp_sock *tp = tcp_sk(sk);
  74	unsigned int prior_packets = tp->packets_out;
  75
  76	WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq);
  77
  78	__skb_unlink(skb, &sk->sk_write_queue);
  79	tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);
  80
  81	if (tp->highest_sack == NULL)
  82		tp->highest_sack = skb;
  83
  84	tp->packets_out += tcp_skb_pcount(skb);
  85	if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
  86		tcp_rearm_rto(sk);
  87
  88	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
  89		      tcp_skb_pcount(skb));
  90	tcp_check_space(sk);
  91}
  92
  93/* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
  94 * window scaling factor due to loss of precision.
  95 * If window has been shrunk, what should we make? It is not clear at all.
  96 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
  97 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
  98 * invalid. OK, let's make this for now:
  99 */
 100static inline __u32 tcp_acceptable_seq(const struct sock *sk)
 101{
 102	const struct tcp_sock *tp = tcp_sk(sk);
 103
 104	if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
 105	    (tp->rx_opt.wscale_ok &&
 106	     ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
 107		return tp->snd_nxt;
 108	else
 109		return tcp_wnd_end(tp);
 110}
 111
 112/* Calculate mss to advertise in SYN segment.
 113 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
 114 *
 115 * 1. It is independent of path mtu.
 116 * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
 117 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
 118 *    attached devices, because some buggy hosts are confused by
 119 *    large MSS.
 120 * 4. We do not make 3, we advertise MSS, calculated from first
 121 *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
 122 *    This may be overridden via information stored in routing table.
 123 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
 124 *    probably even Jumbo".
 125 */
 126static __u16 tcp_advertise_mss(struct sock *sk)
 127{
 128	struct tcp_sock *tp = tcp_sk(sk);
 129	const struct dst_entry *dst = __sk_dst_get(sk);
 130	int mss = tp->advmss;
 131
 132	if (dst) {
 133		unsigned int metric = dst_metric_advmss(dst);
 134
 135		if (metric < mss) {
 136			mss = metric;
 137			tp->advmss = mss;
 138		}
 139	}
 140
 141	return (__u16)mss;
 142}
 143
 144/* RFC2861. Reset CWND after idle period longer RTO to "restart window".
 145 * This is the first part of cwnd validation mechanism.
 146 */
 147void tcp_cwnd_restart(struct sock *sk, s32 delta)
 148{
 149	struct tcp_sock *tp = tcp_sk(sk);
 150	u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
 151	u32 cwnd = tcp_snd_cwnd(tp);
 152
 153	tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
 154
 155	tp->snd_ssthresh = tcp_current_ssthresh(sk);
 156	restart_cwnd = min(restart_cwnd, cwnd);
 157
 158	while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
 159		cwnd >>= 1;
 160	tcp_snd_cwnd_set(tp, max(cwnd, restart_cwnd));
 161	tp->snd_cwnd_stamp = tcp_jiffies32;
 162	tp->snd_cwnd_used = 0;
 163}
 164
 165/* Congestion state accounting after a packet has been sent. */
 166static void tcp_event_data_sent(struct tcp_sock *tp,
 167				struct sock *sk)
 168{
 169	struct inet_connection_sock *icsk = inet_csk(sk);
 170	const u32 now = tcp_jiffies32;
 171
 172	if (tcp_packets_in_flight(tp) == 0)
 173		tcp_ca_event(sk, CA_EVENT_TX_START);
 174
 175	tp->lsndtime = now;
 176
 177	/* If it is a reply for ato after last received
 178	 * packet, increase pingpong count.
 179	 */
 180	if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
 181		inet_csk_inc_pingpong_cnt(sk);
 182}
 183
 184/* Account for an ACK we sent. */
 185static inline void tcp_event_ack_sent(struct sock *sk, u32 rcv_nxt)
 186{
 187	struct tcp_sock *tp = tcp_sk(sk);
 188
 189	if (unlikely(tp->compressed_ack)) {
 190		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
 191			      tp->compressed_ack);
 192		tp->compressed_ack = 0;
 193		if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
 194			__sock_put(sk);
 195	}
 196
 197	if (unlikely(rcv_nxt != tp->rcv_nxt))
 198		return;  /* Special ACK sent by DCTCP to reflect ECN */
 199	tcp_dec_quickack_mode(sk);
 200	inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
 201}
 202
 203/* Determine a window scaling and initial window to offer.
 204 * Based on the assumption that the given amount of space
 205 * will be offered. Store the results in the tp structure.
 206 * NOTE: for smooth operation initial space offering should
 207 * be a multiple of mss if possible. We assume here that mss >= 1.
 208 * This MUST be enforced by all callers.
 209 */
 210void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
 211			       __u32 *rcv_wnd, __u32 *__window_clamp,
 212			       int wscale_ok, __u8 *rcv_wscale,
 213			       __u32 init_rcv_wnd)
 214{
 215	unsigned int space = (__space < 0 ? 0 : __space);
 216	u32 window_clamp = READ_ONCE(*__window_clamp);
 217
 218	/* If no clamp set the clamp to the max possible scaled window */
 219	if (window_clamp == 0)
 220		window_clamp = (U16_MAX << TCP_MAX_WSCALE);
 221	space = min(window_clamp, space);
 222
 223	/* Quantize space offering to a multiple of mss if possible. */
 224	if (space > mss)
 225		space = rounddown(space, mss);
 226
 227	/* NOTE: offering an initial window larger than 32767
 228	 * will break some buggy TCP stacks. If the admin tells us
 229	 * it is likely we could be speaking with such a buggy stack
 230	 * we will truncate our initial window offering to 32K-1
 231	 * unless the remote has sent us a window scaling option,
 232	 * which we interpret as a sign the remote TCP is not
 233	 * misinterpreting the window field as a signed quantity.
 234	 */
 235	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows))
 236		(*rcv_wnd) = min(space, MAX_TCP_WINDOW);
 237	else
 238		(*rcv_wnd) = space;
 239
 240	if (init_rcv_wnd)
 241		*rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
 242
 243	*rcv_wscale = 0;
 244	if (wscale_ok) {
 245		/* Set window scaling on max possible window */
 246		space = max_t(u32, space, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
 247		space = max_t(u32, space, READ_ONCE(sysctl_rmem_max));
 248		space = min_t(u32, space, window_clamp);
 249		*rcv_wscale = clamp_t(int, ilog2(space) - 15,
 250				      0, TCP_MAX_WSCALE);
 251	}
 252	/* Set the clamp no higher than max representable value */
 253	WRITE_ONCE(*__window_clamp,
 254		   min_t(__u32, U16_MAX << (*rcv_wscale), window_clamp));
 255}
 256EXPORT_IPV6_MOD(tcp_select_initial_window);
 257
 258/* Chose a new window to advertise, update state in tcp_sock for the
 259 * socket, and return result with RFC1323 scaling applied.  The return
 260 * value can be stuffed directly into th->window for an outgoing
 261 * frame.
 262 */
 263static u16 tcp_select_window(struct sock *sk)
 264{
 265	struct tcp_sock *tp = tcp_sk(sk);
 266	struct net *net = sock_net(sk);
 267	u32 old_win = tp->rcv_wnd;
 268	u32 cur_win, new_win;
 269
 270	/* Make the window 0 if we failed to queue the data because we
 271	 * are out of memory.
 272	 */
 273	if (unlikely(inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOMEM)) {
 274		tp->pred_flags = 0;
 275		tp->rcv_wnd = 0;
 276		tp->rcv_wup = tp->rcv_nxt;
 277		return 0;
 278	}
 279
 280	cur_win = tcp_receive_window(tp);
 281	new_win = __tcp_select_window(sk);
 282	if (new_win < cur_win) {
 283		/* Danger Will Robinson!
 284		 * Don't update rcv_wup/rcv_wnd here or else
 285		 * we will not be able to advertise a zero
 286		 * window in time.  --DaveM
 287		 *
 288		 * Relax Will Robinson.
 289		 */
 290		if (!READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) || !tp->rx_opt.rcv_wscale) {
 291			/* Never shrink the offered window */
 292			if (new_win == 0)
 293				NET_INC_STATS(net, LINUX_MIB_TCPWANTZEROWINDOWADV);
 294			new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
 295		}
 296	}
 297
 298	tp->rcv_wnd = new_win;
 299	tp->rcv_wup = tp->rcv_nxt;
 300
 301	/* Make sure we do not exceed the maximum possible
 302	 * scaled window.
 303	 */
 304	if (!tp->rx_opt.rcv_wscale &&
 305	    READ_ONCE(net->ipv4.sysctl_tcp_workaround_signed_windows))
 306		new_win = min(new_win, MAX_TCP_WINDOW);
 307	else
 308		new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
 309
 310	/* RFC1323 scaling applied */
 311	new_win >>= tp->rx_opt.rcv_wscale;
 312
 313	/* If we advertise zero window, disable fast path. */
 314	if (new_win == 0) {
 315		tp->pred_flags = 0;
 316		if (old_win)
 317			NET_INC_STATS(net, LINUX_MIB_TCPTOZEROWINDOWADV);
 318	} else if (old_win == 0) {
 319		NET_INC_STATS(net, LINUX_MIB_TCPFROMZEROWINDOWADV);
 320	}
 321
 322	return new_win;
 323}
 324
 325/* Set up ECN state for a packet on a ESTABLISHED socket that is about to
 326 * be sent.
 327 */
 328static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
 329			 struct tcphdr *th, int tcp_header_len)
 330{
 331	struct tcp_sock *tp = tcp_sk(sk);
 332
 333	if (!tcp_ecn_mode_any(tp))
 334		return;
 335
 336	if (tcp_ecn_mode_accecn(tp)) {
 337		if (!tcp_accecn_ace_fail_recv(tp))
 338			INET_ECN_xmit(sk);
 339		tcp_accecn_set_ace(tp, skb, th);
 340		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ACCECN;
 341	} else {
 342		/* Not-retransmitted data segment: set ECT and inject CWR. */
 343		if (skb->len != tcp_header_len &&
 344		    !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
 345			INET_ECN_xmit(sk);
 346			if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
 347				tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
 348				th->cwr = 1;
 349				skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
 350			}
 351		} else if (!tcp_ca_needs_ecn(sk)) {
 352			/* ACK or retransmitted segment: clear ECT|CE */
 353			INET_ECN_dontxmit(sk);
 354		}
 355		if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
 356			th->ece = 1;
 357	}
 358}
 359
 360/* Constructs common control bits of non-data skb. If SYN/FIN is present,
 361 * auto increment end seqno.
 362 */
 363static void tcp_init_nondata_skb(struct sk_buff *skb, struct sock *sk,
 364				 u32 seq, u16 flags)
 365{
 366	skb->ip_summed = CHECKSUM_PARTIAL;
 367
 368	TCP_SKB_CB(skb)->tcp_flags = flags;
 369
 370	tcp_skb_pcount_set(skb, 1);
 371	psp_enqueue_set_decrypted(sk, skb);
 372
 373	TCP_SKB_CB(skb)->seq = seq;
 374	if (flags & (TCPHDR_SYN | TCPHDR_FIN))
 375		seq++;
 376	TCP_SKB_CB(skb)->end_seq = seq;
 377}
 378
 379static inline bool tcp_urg_mode(const struct tcp_sock *tp)
 380{
 381	return tp->snd_una != tp->snd_up;
 382}
 383
 384#define OPTION_SACK_ADVERTISE	BIT(0)
 385#define OPTION_TS		BIT(1)
 386#define OPTION_MD5		BIT(2)
 387#define OPTION_WSCALE		BIT(3)
 388#define OPTION_FAST_OPEN_COOKIE	BIT(8)
 389#define OPTION_SMC		BIT(9)
 390#define OPTION_MPTCP		BIT(10)
 391#define OPTION_AO		BIT(11)
 392#define OPTION_ACCECN		BIT(12)
 393
 394static void smc_options_write(__be32 *ptr, u16 *options)
 395{
 396#if IS_ENABLED(CONFIG_SMC)
 397	if (static_branch_unlikely(&tcp_have_smc)) {
 398		if (unlikely(OPTION_SMC & *options)) {
 399			*ptr++ = htonl((TCPOPT_NOP  << 24) |
 400				       (TCPOPT_NOP  << 16) |
 401				       (TCPOPT_EXP <<  8) |
 402				       (TCPOLEN_EXP_SMC_BASE));
 403			*ptr++ = htonl(TCPOPT_SMC_MAGIC);
 404		}
 405	}
 406#endif
 407}
 408
 409struct tcp_out_options {
 410	u16 options;		/* bit field of OPTION_* */
 411	u16 mss;		/* 0 to disable */
 412	u8 ws;			/* window scale, 0 to disable */
 413	u8 num_sack_blocks;	/* number of SACK blocks to include */
 414	u8 num_accecn_fields:7,	/* number of AccECN fields needed */
 415	   use_synack_ecn_bytes:1; /* Use synack_ecn_bytes or not */
 416	u8 hash_size;		/* bytes in hash_location */
 417	u8 bpf_opt_len;		/* length of BPF hdr option */
 418	__u8 *hash_location;	/* temporary pointer, overloaded */
 419	__u32 tsval, tsecr;	/* need to include OPTION_TS */
 420	struct tcp_fastopen_cookie *fastopen_cookie;	/* Fast open cookie */
 421	struct mptcp_out_options mptcp;
 422};
 423
 424static void mptcp_options_write(struct tcphdr *th, __be32 *ptr,
 425				struct tcp_sock *tp,
 426				struct tcp_out_options *opts)
 427{
 428#if IS_ENABLED(CONFIG_MPTCP)
 429	if (unlikely(OPTION_MPTCP & opts->options))
 430		mptcp_write_options(th, ptr, tp, &opts->mptcp);
 431#endif
 432}
 433
 434#ifdef CONFIG_CGROUP_BPF
 435static int bpf_skops_write_hdr_opt_arg0(struct sk_buff *skb,
 436					enum tcp_synack_type synack_type)
 437{
 438	if (unlikely(!skb))
 439		return BPF_WRITE_HDR_TCP_CURRENT_MSS;
 440
 441	if (unlikely(synack_type == TCP_SYNACK_COOKIE))
 442		return BPF_WRITE_HDR_TCP_SYNACK_COOKIE;
 443
 444	return 0;
 445}
 446
 447/* req, syn_skb and synack_type are used when writing synack */
 448static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
 449				  struct request_sock *req,
 450				  struct sk_buff *syn_skb,
 451				  enum tcp_synack_type synack_type,
 452				  struct tcp_out_options *opts,
 453				  unsigned int *remaining)
 454{
 455	struct bpf_sock_ops_kern sock_ops;
 456	int err;
 457
 458	if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
 459					   BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) ||
 460	    !*remaining)
 461		return;
 462
 463	/* *remaining has already been aligned to 4 bytes, so *remaining >= 4 */
 464
 465	/* init sock_ops */
 466	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
 467
 468	sock_ops.op = BPF_SOCK_OPS_HDR_OPT_LEN_CB;
 469
 470	if (req) {
 471		/* The listen "sk" cannot be passed here because
 472		 * it is not locked.  It would not make too much
 473		 * sense to do bpf_setsockopt(listen_sk) based
 474		 * on individual connection request also.
 475		 *
 476		 * Thus, "req" is passed here and the cgroup-bpf-progs
 477		 * of the listen "sk" will be run.
 478		 *
 479		 * "req" is also used here for fastopen even the "sk" here is
 480		 * a fullsock "child" sk.  It is to keep the behavior
 481		 * consistent between fastopen and non-fastopen on
 482		 * the bpf programming side.
 483		 */
 484		sock_ops.sk = (struct sock *)req;
 485		sock_ops.syn_skb = syn_skb;
 486	} else {
 487		sock_owned_by_me(sk);
 488
 489		sock_ops.is_fullsock = 1;
 490		sock_ops.is_locked_tcp_sock = 1;
 491		sock_ops.sk = sk;
 492	}
 493
 494	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
 495	sock_ops.remaining_opt_len = *remaining;
 496	/* tcp_current_mss() does not pass a skb */
 497	if (skb)
 498		bpf_skops_init_skb(&sock_ops, skb, 0);
 499
 500	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
 501
 502	if (err || sock_ops.remaining_opt_len == *remaining)
 503		return;
 504
 505	opts->bpf_opt_len = *remaining - sock_ops.remaining_opt_len;
 506	/* round up to 4 bytes */
 507	opts->bpf_opt_len = (opts->bpf_opt_len + 3) & ~3;
 508
 509	*remaining -= opts->bpf_opt_len;
 510}
 511
 512static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
 513				    struct request_sock *req,
 514				    struct sk_buff *syn_skb,
 515				    enum tcp_synack_type synack_type,
 516				    struct tcp_out_options *opts)
 517{
 518	u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len;
 519	struct bpf_sock_ops_kern sock_ops;
 520	int err;
 521
 522	if (likely(!max_opt_len))
 523		return;
 524
 525	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
 526
 527	sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
 528
 529	if (req) {
 530		sock_ops.sk = (struct sock *)req;
 531		sock_ops.syn_skb = syn_skb;
 532	} else {
 533		sock_owned_by_me(sk);
 534
 535		sock_ops.is_fullsock = 1;
 536		sock_ops.is_locked_tcp_sock = 1;
 537		sock_ops.sk = sk;
 538	}
 539
 540	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
 541	sock_ops.remaining_opt_len = max_opt_len;
 542	first_opt_off = tcp_hdrlen(skb) - max_opt_len;
 543	bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
 544
 545	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
 546
 547	if (err)
 548		nr_written = 0;
 549	else
 550		nr_written = max_opt_len - sock_ops.remaining_opt_len;
 551
 552	if (nr_written < max_opt_len)
 553		memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP,
 554		       max_opt_len - nr_written);
 555}
 556#else
 557static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
 558				  struct request_sock *req,
 559				  struct sk_buff *syn_skb,
 560				  enum tcp_synack_type synack_type,
 561				  struct tcp_out_options *opts,
 562				  unsigned int *remaining)
 563{
 564}
 565
 566static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
 567				    struct request_sock *req,
 568				    struct sk_buff *syn_skb,
 569				    enum tcp_synack_type synack_type,
 570				    struct tcp_out_options *opts)
 571{
 572}
 573#endif
 574
 575static __be32 *process_tcp_ao_options(struct tcp_sock *tp,
 576				      const struct tcp_request_sock *tcprsk,
 577				      struct tcp_out_options *opts,
 578				      struct tcp_key *key, __be32 *ptr)
 579{
 580#ifdef CONFIG_TCP_AO
 581	u8 maclen = tcp_ao_maclen(key->ao_key);
 582
 583	if (tcprsk) {
 584		u8 aolen = maclen + sizeof(struct tcp_ao_hdr);
 585
 586		*ptr++ = htonl((TCPOPT_AO << 24) | (aolen << 16) |
 587			       (tcprsk->ao_keyid << 8) |
 588			       (tcprsk->ao_rcv_next));
 589	} else {
 590		struct tcp_ao_key *rnext_key;
 591		struct tcp_ao_info *ao_info;
 592
 593		ao_info = rcu_dereference_check(tp->ao_info,
 594			lockdep_sock_is_held(&tp->inet_conn.icsk_inet.sk));
 595		rnext_key = READ_ONCE(ao_info->rnext_key);
 596		if (WARN_ON_ONCE(!rnext_key))
 597			return ptr;
 598		*ptr++ = htonl((TCPOPT_AO << 24) |
 599			       (tcp_ao_len(key->ao_key) << 16) |
 600			       (key->ao_key->sndid << 8) |
 601			       (rnext_key->rcvid));
 602	}
 603	opts->hash_location = (__u8 *)ptr;
 604	ptr += maclen / sizeof(*ptr);
 605	if (unlikely(maclen % sizeof(*ptr))) {
 606		memset(ptr, TCPOPT_NOP, sizeof(*ptr));
 607		ptr++;
 608	}
 609#endif
 610	return ptr;
 611}
 612
 613/* Initial values for AccECN option, ordered is based on ECN field bits
 614 * similar to received_ecn_bytes. Used for SYN/ACK AccECN option.
 615 */
 616static const u32 synack_ecn_bytes[3] = { 0, 0, 0 };
 617
 618/* Write previously computed TCP options to the packet.
 619 *
 620 * Beware: Something in the Internet is very sensitive to the ordering of
 621 * TCP options, we learned this through the hard way, so be careful here.
 622 * Luckily we can at least blame others for their non-compliance but from
 623 * inter-operability perspective it seems that we're somewhat stuck with
 624 * the ordering which we have been using if we want to keep working with
 625 * those broken things (not that it currently hurts anybody as there isn't
 626 * particular reason why the ordering would need to be changed).
 627 *
 628 * At least SACK_PERM as the first option is known to lead to a disaster
 629 * (but it may well be that other scenarios fail similarly).
 630 */
 631static void tcp_options_write(struct tcphdr *th, struct tcp_sock *tp,
 632			      const struct tcp_request_sock *tcprsk,
 633			      struct tcp_out_options *opts,
 634			      struct tcp_key *key)
 635{
 636	u8 leftover_highbyte = TCPOPT_NOP; /* replace 1st NOP if avail */
 637	u8 leftover_lowbyte = TCPOPT_NOP;  /* replace 2nd NOP in succession */
 638	__be32 *ptr = (__be32 *)(th + 1);
 639	u16 options = opts->options;	/* mungable copy */
 640
 641	if (tcp_key_is_md5(key)) {
 642		*ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 643			       (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 644		/* overload cookie hash location */
 645		opts->hash_location = (__u8 *)ptr;
 646		ptr += 4;
 647	} else if (tcp_key_is_ao(key)) {
 648		ptr = process_tcp_ao_options(tp, tcprsk, opts, key, ptr);
 649	}
 650	if (unlikely(opts->mss)) {
 651		*ptr++ = htonl((TCPOPT_MSS << 24) |
 652			       (TCPOLEN_MSS << 16) |
 653			       opts->mss);
 654	}
 655
 656	if (likely(OPTION_TS & options)) {
 657		if (unlikely(OPTION_SACK_ADVERTISE & options)) {
 658			*ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
 659				       (TCPOLEN_SACK_PERM << 16) |
 660				       (TCPOPT_TIMESTAMP << 8) |
 661				       TCPOLEN_TIMESTAMP);
 662			options &= ~OPTION_SACK_ADVERTISE;
 663		} else {
 664			*ptr++ = htonl((TCPOPT_NOP << 24) |
 665				       (TCPOPT_NOP << 16) |
 666				       (TCPOPT_TIMESTAMP << 8) |
 667				       TCPOLEN_TIMESTAMP);
 668		}
 669		*ptr++ = htonl(opts->tsval);
 670		*ptr++ = htonl(opts->tsecr);
 671	}
 672
 673	if (OPTION_ACCECN & options) {
 674		const u32 *ecn_bytes = opts->use_synack_ecn_bytes ?
 675				       synack_ecn_bytes :
 676				       tp->received_ecn_bytes;
 677		const u8 ect0_idx = INET_ECN_ECT_0 - 1;
 678		const u8 ect1_idx = INET_ECN_ECT_1 - 1;
 679		const u8 ce_idx = INET_ECN_CE - 1;
 680		u32 e0b;
 681		u32 e1b;
 682		u32 ceb;
 683		u8 len;
 684
 685		e0b = ecn_bytes[ect0_idx] + TCP_ACCECN_E0B_INIT_OFFSET;
 686		e1b = ecn_bytes[ect1_idx] + TCP_ACCECN_E1B_INIT_OFFSET;
 687		ceb = ecn_bytes[ce_idx] + TCP_ACCECN_CEB_INIT_OFFSET;
 688		len = TCPOLEN_ACCECN_BASE +
 689		      opts->num_accecn_fields * TCPOLEN_ACCECN_PERFIELD;
 690
 691		if (opts->num_accecn_fields == 2) {
 692			*ptr++ = htonl((TCPOPT_ACCECN1 << 24) | (len << 16) |
 693				       ((e1b >> 8) & 0xffff));
 694			*ptr++ = htonl(((e1b & 0xff) << 24) |
 695				       (ceb & 0xffffff));
 696		} else if (opts->num_accecn_fields == 1) {
 697			*ptr++ = htonl((TCPOPT_ACCECN1 << 24) | (len << 16) |
 698				       ((e1b >> 8) & 0xffff));
 699			leftover_highbyte = e1b & 0xff;
 700			leftover_lowbyte = TCPOPT_NOP;
 701		} else if (opts->num_accecn_fields == 0) {
 702			leftover_highbyte = TCPOPT_ACCECN1;
 703			leftover_lowbyte = len;
 704		} else if (opts->num_accecn_fields == 3) {
 705			*ptr++ = htonl((TCPOPT_ACCECN1 << 24) | (len << 16) |
 706				       ((e1b >> 8) & 0xffff));
 707			*ptr++ = htonl(((e1b & 0xff) << 24) |
 708				       (ceb & 0xffffff));
 709			*ptr++ = htonl(((e0b & 0xffffff) << 8) |
 710				       TCPOPT_NOP);
 711		}
 712		if (tp) {
 713			tp->accecn_minlen = 0;
 714			tp->accecn_opt_tstamp = tp->tcp_mstamp;
 715			if (tp->accecn_opt_demand)
 716				tp->accecn_opt_demand--;
 717		}
 718	}
 719
 720	if (unlikely(OPTION_SACK_ADVERTISE & options)) {
 721		*ptr++ = htonl((leftover_highbyte << 24) |
 722			       (leftover_lowbyte << 16) |
 723			       (TCPOPT_SACK_PERM << 8) |
 724			       TCPOLEN_SACK_PERM);
 725		leftover_highbyte = TCPOPT_NOP;
 726		leftover_lowbyte = TCPOPT_NOP;
 727	}
 728
 729	if (unlikely(OPTION_WSCALE & options)) {
 730		u8 highbyte = TCPOPT_NOP;
 731
 732		/* Do not split the leftover 2-byte to fit into a single
 733		 * NOP, i.e., replace this NOP only when 1 byte is leftover
 734		 * within leftover_highbyte.
 735		 */
 736		if (unlikely(leftover_highbyte != TCPOPT_NOP &&
 737			     leftover_lowbyte == TCPOPT_NOP)) {
 738			highbyte = leftover_highbyte;
 739			leftover_highbyte = TCPOPT_NOP;
 740		}
 741		*ptr++ = htonl((highbyte << 24) |
 742			       (TCPOPT_WINDOW << 16) |
 743			       (TCPOLEN_WINDOW << 8) |
 744			       opts->ws);
 745	}
 746
 747	if (unlikely(opts->num_sack_blocks)) {
 748		struct tcp_sack_block *sp = tp->rx_opt.dsack ?
 749			tp->duplicate_sack : tp->selective_acks;
 750		int this_sack;
 751
 752		*ptr++ = htonl((leftover_highbyte << 24) |
 753			       (leftover_lowbyte << 16) |
 754			       (TCPOPT_SACK <<  8) |
 755			       (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
 756						     TCPOLEN_SACK_PERBLOCK)));
 757		leftover_highbyte = TCPOPT_NOP;
 758		leftover_lowbyte = TCPOPT_NOP;
 759
 760		for (this_sack = 0; this_sack < opts->num_sack_blocks;
 761		     ++this_sack) {
 762			*ptr++ = htonl(sp[this_sack].start_seq);
 763			*ptr++ = htonl(sp[this_sack].end_seq);
 764		}
 765
 766		tp->rx_opt.dsack = 0;
 767	} else if (unlikely(leftover_highbyte != TCPOPT_NOP ||
 768			    leftover_lowbyte != TCPOPT_NOP)) {
 769		*ptr++ = htonl((leftover_highbyte << 24) |
 770			       (leftover_lowbyte << 16) |
 771			       (TCPOPT_NOP << 8) |
 772			       TCPOPT_NOP);
 773		leftover_highbyte = TCPOPT_NOP;
 774		leftover_lowbyte = TCPOPT_NOP;
 775	}
 776
 777	if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
 778		struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
 779		u8 *p = (u8 *)ptr;
 780		u32 len; /* Fast Open option length */
 781
 782		if (foc->exp) {
 783			len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
 784			*ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
 785				     TCPOPT_FASTOPEN_MAGIC);
 786			p += TCPOLEN_EXP_FASTOPEN_BASE;
 787		} else {
 788			len = TCPOLEN_FASTOPEN_BASE + foc->len;
 789			*p++ = TCPOPT_FASTOPEN;
 790			*p++ = len;
 791		}
 792
 793		memcpy(p, foc->val, foc->len);
 794		if ((len & 3) == 2) {
 795			p[foc->len] = TCPOPT_NOP;
 796			p[foc->len + 1] = TCPOPT_NOP;
 797		}
 798		ptr += (len + 3) >> 2;
 799	}
 800
 801	smc_options_write(ptr, &options);
 802
 803	mptcp_options_write(th, ptr, tp, opts);
 804}
 805
 806static void smc_set_option(struct tcp_sock *tp,
 807			   struct tcp_out_options *opts,
 808			   unsigned int *remaining)
 809{
 810#if IS_ENABLED(CONFIG_SMC)
 811	if (static_branch_unlikely(&tcp_have_smc) && tp->syn_smc) {
 812		tp->syn_smc = !!smc_call_hsbpf(1, tp, syn_option);
 813		/* re-check syn_smc */
 814		if (tp->syn_smc &&
 815		    *remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
 816			opts->options |= OPTION_SMC;
 817			*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
 818		}
 819	}
 820#endif
 821}
 822
 823static void smc_set_option_cond(const struct tcp_sock *tp,
 824				struct inet_request_sock *ireq,
 825				struct tcp_out_options *opts,
 826				unsigned int *remaining)
 827{
 828#if IS_ENABLED(CONFIG_SMC)
 829	if (static_branch_unlikely(&tcp_have_smc) && tp->syn_smc && ireq->smc_ok) {
 830		ireq->smc_ok = !!smc_call_hsbpf(1, tp, synack_option, ireq);
 831		/* re-check smc_ok */
 832		if (ireq->smc_ok &&
 833		    *remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
 834			opts->options |= OPTION_SMC;
 835			*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
 836		}
 837	}
 838#endif
 839}
 840
 841static void mptcp_set_option_cond(const struct request_sock *req,
 842				  struct tcp_out_options *opts,
 843				  unsigned int *remaining)
 844{
 845	if (rsk_is_mptcp(req)) {
 846		unsigned int size;
 847
 848		if (mptcp_synack_options(req, &size, &opts->mptcp)) {
 849			if (*remaining >= size) {
 850				opts->options |= OPTION_MPTCP;
 851				*remaining -= size;
 852			}
 853		}
 854	}
 855}
 856
 857static u32 tcp_synack_options_combine_saving(struct tcp_out_options *opts)
 858{
 859	/* How much there's room for combining with the alignment padding? */
 860	if ((opts->options & (OPTION_SACK_ADVERTISE | OPTION_TS)) ==
 861	    OPTION_SACK_ADVERTISE)
 862		return 2;
 863	else if (opts->options & OPTION_WSCALE)
 864		return 1;
 865	return 0;
 866}
 867
 868/* Calculates how long AccECN option will fit to @remaining option space.
 869 *
 870 * AccECN option can sometimes replace NOPs used for alignment of other
 871 * TCP options (up to @max_combine_saving available).
 872 *
 873 * Only solutions with at least @required AccECN fields are accepted.
 874 *
 875 * Returns: The size of the AccECN option excluding space repurposed from
 876 * the alignment of the other options.
 877 */
 878static int tcp_options_fit_accecn(struct tcp_out_options *opts, int required,
 879				  int remaining)
 880{
 881	int size = TCP_ACCECN_MAXSIZE;
 882	int sack_blocks_reduce = 0;
 883	int max_combine_saving;
 884	int rem = remaining;
 885	int align_size;
 886
 887	if (opts->use_synack_ecn_bytes)
 888		max_combine_saving = tcp_synack_options_combine_saving(opts);
 889	else
 890		max_combine_saving = opts->num_sack_blocks > 0 ? 2 : 0;
 891	opts->num_accecn_fields = TCP_ACCECN_NUMFIELDS;
 892	while (opts->num_accecn_fields >= required) {
 893		/* Pad to dword if cannot combine */
 894		if ((size & 0x3) > max_combine_saving)
 895			align_size = ALIGN(size, 4);
 896		else
 897			align_size = ALIGN_DOWN(size, 4);
 898
 899		if (rem >= align_size) {
 900			size = align_size;
 901			break;
 902		} else if (opts->num_accecn_fields == required &&
 903			   opts->num_sack_blocks > 2 &&
 904			   required > 0) {
 905			/* Try to fit the option by removing one SACK block */
 906			opts->num_sack_blocks--;
 907			sack_blocks_reduce++;
 908			rem = rem + TCPOLEN_SACK_PERBLOCK;
 909
 910			opts->num_accecn_fields = TCP_ACCECN_NUMFIELDS;
 911			size = TCP_ACCECN_MAXSIZE;
 912			continue;
 913		}
 914
 915		opts->num_accecn_fields--;
 916		size -= TCPOLEN_ACCECN_PERFIELD;
 917	}
 918	if (sack_blocks_reduce > 0) {
 919		if (opts->num_accecn_fields >= required)
 920			size -= sack_blocks_reduce * TCPOLEN_SACK_PERBLOCK;
 921		else
 922			opts->num_sack_blocks += sack_blocks_reduce;
 923	}
 924	if (opts->num_accecn_fields < required)
 925		return 0;
 926
 927	opts->options |= OPTION_ACCECN;
 928	return size;
 929}
 930
 931/* Compute TCP options for SYN packets. This is not the final
 932 * network wire format yet.
 933 */
 934static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
 935				struct tcp_out_options *opts,
 936				struct tcp_key *key)
 937{
 938	struct tcp_sock *tp = tcp_sk(sk);
 939	unsigned int remaining = MAX_TCP_OPTION_SPACE;
 940	struct tcp_fastopen_request *fastopen = tp->fastopen_req;
 941	bool timestamps;
 942
 943	/* Better than switch (key.type) as it has static branches */
 944	if (tcp_key_is_md5(key)) {
 945		timestamps = false;
 946		opts->options |= OPTION_MD5;
 947		remaining -= TCPOLEN_MD5SIG_ALIGNED;
 948	} else {
 949		timestamps = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps);
 950		if (tcp_key_is_ao(key)) {
 951			opts->options |= OPTION_AO;
 952			remaining -= tcp_ao_len_aligned(key->ao_key);
 953		}
 954	}
 955
 956	/* We always get an MSS option.  The option bytes which will be seen in
 957	 * normal data packets should timestamps be used, must be in the MSS
 958	 * advertised.  But we subtract them from tp->mss_cache so that
 959	 * calculations in tcp_sendmsg are simpler etc.  So account for this
 960	 * fact here if necessary.  If we don't do this correctly, as a
 961	 * receiver we won't recognize data packets as being full sized when we
 962	 * should, and thus we won't abide by the delayed ACK rules correctly.
 963	 * SACKs don't matter, we never delay an ACK when we have any of those
 964	 * going out.  */
 965	opts->mss = tcp_advertise_mss(sk);
 966	remaining -= TCPOLEN_MSS_ALIGNED;
 967
 968	if (likely(timestamps)) {
 969		opts->options |= OPTION_TS;
 970		opts->tsval = tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb) + tp->tsoffset;
 971		opts->tsecr = tp->rx_opt.ts_recent;
 972		remaining -= TCPOLEN_TSTAMP_ALIGNED;
 973	}
 974	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling))) {
 975		opts->ws = tp->rx_opt.rcv_wscale;
 976		opts->options |= OPTION_WSCALE;
 977		remaining -= TCPOLEN_WSCALE_ALIGNED;
 978	}
 979	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_sack))) {
 980		opts->options |= OPTION_SACK_ADVERTISE;
 981		if (unlikely(!(OPTION_TS & opts->options)))
 982			remaining -= TCPOLEN_SACKPERM_ALIGNED;
 983	}
 984
 985	if (fastopen && fastopen->cookie.len >= 0) {
 986		u32 need = fastopen->cookie.len;
 987
 988		need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
 989					       TCPOLEN_FASTOPEN_BASE;
 990		need = (need + 3) & ~3U;  /* Align to 32 bits */
 991		if (remaining >= need) {
 992			opts->options |= OPTION_FAST_OPEN_COOKIE;
 993			opts->fastopen_cookie = &fastopen->cookie;
 994			remaining -= need;
 995			tp->syn_fastopen = 1;
 996			tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
 997		}
 998	}
 999
1000	smc_set_option(tp, opts, &remaining);
1001
1002	if (sk_is_mptcp(sk)) {
1003		unsigned int size;
1004
1005		if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) {
1006			if (remaining >= size) {
1007				opts->options |= OPTION_MPTCP;
1008				remaining -= size;
1009			}
1010		}
1011	}
1012
1013	/* Simultaneous open SYN/ACK needs AccECN option but not SYN.
1014	 * It is attempted to negotiate the use of AccECN also on the first
1015	 * retransmitted SYN, as mentioned in "3.1.4.1. Retransmitted SYNs"
1016	 * of AccECN draft.
1017	 */
1018	if (unlikely((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK) &&
1019		     tcp_ecn_mode_accecn(tp) &&
1020		     inet_csk(sk)->icsk_retransmits < 2 &&
1021		     READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_option) &&
1022		     remaining >= TCPOLEN_ACCECN_BASE)) {
1023		opts->use_synack_ecn_bytes = 1;
1024		remaining -= tcp_options_fit_accecn(opts, 0, remaining);
1025	}
1026
1027	bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
1028
1029	return MAX_TCP_OPTION_SPACE - remaining;
1030}
1031
1032/* Set up TCP options for SYN-ACKs. */
1033static unsigned int tcp_synack_options(const struct sock *sk,
1034				       struct request_sock *req,
1035				       unsigned int mss, struct sk_buff *skb,
1036				       struct tcp_out_options *opts,
1037				       const struct tcp_key *key,
1038				       struct tcp_fastopen_cookie *foc,
1039				       enum tcp_synack_type synack_type,
1040				       struct sk_buff *syn_skb)
1041{
1042	struct inet_request_sock *ireq = inet_rsk(req);
1043	unsigned int remaining = MAX_TCP_OPTION_SPACE;
1044	struct tcp_request_sock *treq = tcp_rsk(req);
1045
1046	if (tcp_key_is_md5(key)) {
1047		opts->options |= OPTION_MD5;
1048		remaining -= TCPOLEN_MD5SIG_ALIGNED;
1049
1050		/* We can't fit any SACK blocks in a packet with MD5 + TS
1051		 * options. There was discussion about disabling SACK
1052		 * rather than TS in order to fit in better with old,
1053		 * buggy kernels, but that was deemed to be unnecessary.
1054		 */
1055		if (synack_type != TCP_SYNACK_COOKIE)
1056			ireq->tstamp_ok &= !ireq->sack_ok;
1057	} else if (tcp_key_is_ao(key)) {
1058		opts->options |= OPTION_AO;
1059		remaining -= tcp_ao_len_aligned(key->ao_key);
1060		ireq->tstamp_ok &= !ireq->sack_ok;
1061	}
1062
1063	/* We always send an MSS option. */
1064	opts->mss = mss;
1065	remaining -= TCPOLEN_MSS_ALIGNED;
1066
1067	if (likely(ireq->wscale_ok)) {
1068		opts->ws = ireq->rcv_wscale;
1069		opts->options |= OPTION_WSCALE;
1070		remaining -= TCPOLEN_WSCALE_ALIGNED;
1071	}
1072	if (likely(ireq->tstamp_ok)) {
1073		opts->options |= OPTION_TS;
1074		opts->tsval = tcp_skb_timestamp_ts(tcp_rsk(req)->req_usec_ts, skb) +
1075			      tcp_rsk(req)->ts_off;
1076		if (!tcp_rsk(req)->snt_tsval_first) {
1077			if (!opts->tsval)
1078				opts->tsval = ~0U;
1079			tcp_rsk(req)->snt_tsval_first = opts->tsval;
1080		}
1081		WRITE_ONCE(tcp_rsk(req)->snt_tsval_last, opts->tsval);
1082		opts->tsecr = req->ts_recent;
1083		remaining -= TCPOLEN_TSTAMP_ALIGNED;
1084	}
1085	if (likely(ireq->sack_ok)) {
1086		opts->options |= OPTION_SACK_ADVERTISE;
1087		if (unlikely(!ireq->tstamp_ok))
1088			remaining -= TCPOLEN_SACKPERM_ALIGNED;
1089	}
1090	if (foc != NULL && foc->len >= 0) {
1091		u32 need = foc->len;
1092
1093		need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
1094				   TCPOLEN_FASTOPEN_BASE;
1095		need = (need + 3) & ~3U;  /* Align to 32 bits */
1096		if (remaining >= need) {
1097			opts->options |= OPTION_FAST_OPEN_COOKIE;
1098			opts->fastopen_cookie = foc;
1099			remaining -= need;
1100		}
1101	}
1102
1103	mptcp_set_option_cond(req, opts, &remaining);
1104
1105	smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining);
1106
1107	if (treq->accecn_ok &&
1108	    READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_option) &&
1109	    req->num_timeout < 1 && remaining >= TCPOLEN_ACCECN_BASE) {
1110		opts->use_synack_ecn_bytes = 1;
1111		remaining -= tcp_options_fit_accecn(opts, 0, remaining);
1112	}
1113
1114	bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
1115			      synack_type, opts, &remaining);
1116
1117	return MAX_TCP_OPTION_SPACE - remaining;
1118}
1119
1120/* Compute TCP options for ESTABLISHED sockets. This is not the
1121 * final wire format yet.
1122 */
1123static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
1124					struct tcp_out_options *opts,
1125					struct tcp_key *key)
1126{
1127	struct tcp_sock *tp = tcp_sk(sk);
1128	unsigned int size = 0;
1129	unsigned int eff_sacks;
1130
1131	opts->options = 0;
1132
1133	/* Better than switch (key.type) as it has static branches */
1134	if (tcp_key_is_md5(key)) {
1135		opts->options |= OPTION_MD5;
1136		size += TCPOLEN_MD5SIG_ALIGNED;
1137	} else if (tcp_key_is_ao(key)) {
1138		opts->options |= OPTION_AO;
1139		size += tcp_ao_len_aligned(key->ao_key);
1140	}
1141
1142	if (likely(tp->rx_opt.tstamp_ok)) {
1143		opts->options |= OPTION_TS;
1144		opts->tsval = skb ? tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb) +
1145				tp->tsoffset : 0;
1146		opts->tsecr = tp->rx_opt.ts_recent;
1147		size += TCPOLEN_TSTAMP_ALIGNED;
1148	}
1149
1150	/* MPTCP options have precedence over SACK for the limited TCP
1151	 * option space because a MPTCP connection would be forced to
1152	 * fall back to regular TCP if a required multipath option is
1153	 * missing. SACK still gets a chance to use whatever space is
1154	 * left.
1155	 */
1156	if (sk_is_mptcp(sk)) {
1157		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
1158		unsigned int opt_size = 0;
1159
1160		if (mptcp_established_options(sk, skb, &opt_size, remaining,
1161					      &opts->mptcp)) {
1162			opts->options |= OPTION_MPTCP;
1163			size += opt_size;
1164		}
1165	}
1166
1167	eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
1168	if (unlikely(eff_sacks)) {
1169		const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
1170		if (likely(remaining >= TCPOLEN_SACK_BASE_ALIGNED +
1171					TCPOLEN_SACK_PERBLOCK)) {
1172			opts->num_sack_blocks =
1173				min_t(unsigned int, eff_sacks,
1174				      (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
1175				      TCPOLEN_SACK_PERBLOCK);
1176
1177			size += TCPOLEN_SACK_BASE_ALIGNED +
1178				opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
1179		} else {
1180			opts->num_sack_blocks = 0;
1181		}
1182	} else {
1183		opts->num_sack_blocks = 0;
1184	}
1185
1186	if (tcp_ecn_mode_accecn(tp)) {
1187		int ecn_opt = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_option);
1188
1189		if (ecn_opt && tp->saw_accecn_opt && !tcp_accecn_opt_fail_send(tp) &&
1190		    (ecn_opt >= TCP_ACCECN_OPTION_FULL || tp->accecn_opt_demand ||
1191		     tcp_accecn_option_beacon_check(sk))) {
1192			opts->use_synack_ecn_bytes = 0;
1193			size += tcp_options_fit_accecn(opts, tp->accecn_minlen,
1194						       MAX_TCP_OPTION_SPACE - size);
1195		}
1196	}
1197
1198	if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
1199					    BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
1200		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
1201
1202		bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
1203
1204		size = MAX_TCP_OPTION_SPACE - remaining;
1205	}
1206
1207	return size;
1208}
1209
1210
1211/* TCP SMALL QUEUES (TSQ)
1212 *
1213 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
1214 * to reduce RTT and bufferbloat.
1215 * We do this using a special skb destructor (tcp_wfree).
1216 *
1217 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
1218 * needs to be reallocated in a driver.
1219 * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
1220 *
1221 * Since transmit from skb destructor is forbidden, we use a BH work item
1222 * to process all sockets that eventually need to send more skbs.
1223 * We use one work item per cpu, with its own queue of sockets.
1224 */
1225struct tsq_work {
1226	struct work_struct	work;
1227	struct list_head	head; /* queue of tcp sockets */
1228};
1229static DEFINE_PER_CPU(struct tsq_work, tsq_work);
1230
1231static void tcp_tsq_write(struct sock *sk)
1232{
1233	if ((1 << sk->sk_state) &
1234	    (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
1235	     TCPF_CLOSE_WAIT  | TCPF_LAST_ACK)) {
1236		struct tcp_sock *tp = tcp_sk(sk);
1237
1238		if (tp->lost_out > tp->retrans_out &&
1239		    tcp_snd_cwnd(tp) > tcp_packets_in_flight(tp)) {
1240			tcp_mstamp_refresh(tp);
1241			tcp_xmit_retransmit_queue(sk);
1242		}
1243
1244		tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
1245			       0, GFP_ATOMIC);
1246	}
1247}
1248
1249static void tcp_tsq_handler(struct sock *sk)
1250{
1251	bh_lock_sock(sk);
1252	if (!sock_owned_by_user(sk))
1253		tcp_tsq_write(sk);
1254	else if (!test_and_set_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
1255		sock_hold(sk);
1256	bh_unlock_sock(sk);
1257}
1258/*
1259 * One work item per cpu tries to send more skbs.
1260 * We run in BH context but need to disable irqs when
1261 * transferring tsq->head because tcp_wfree() might
1262 * interrupt us (non NAPI drivers)
1263 */
1264static void tcp_tsq_workfn(struct work_struct *work)
1265{
1266	struct tsq_work *tsq = container_of(work, struct tsq_work, work);
1267	LIST_HEAD(list);
1268	unsigned long flags;
1269	struct list_head *q, *n;
1270	struct tcp_sock *tp;
1271	struct sock *sk;
1272
1273	local_irq_save(flags);
1274	list_splice_init(&tsq->head, &list);
1275	local_irq_restore(flags);
1276
1277	list_for_each_safe(q, n, &list) {
1278		tp = list_entry(q, struct tcp_sock, tsq_node);
1279		list_del(&tp->tsq_node);
1280
1281		sk = (struct sock *)tp;
1282		smp_mb__before_atomic();
1283		clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
1284
1285		tcp_tsq_handler(sk);
1286		sk_free(sk);
1287	}
1288}
1289
1290#define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |		\
1291			  TCPF_WRITE_TIMER_DEFERRED |	\
1292			  TCPF_DELACK_TIMER_DEFERRED |	\
1293			  TCPF_MTU_REDUCED_DEFERRED |	\
1294			  TCPF_ACK_DEFERRED)
1295/**
1296 * tcp_release_cb - tcp release_sock() callback
1297 * @sk: socket
1298 *
1299 * called from release_sock() to perform protocol dependent
1300 * actions before socket release.
1301 */
1302void tcp_release_cb(struct sock *sk)
1303{
1304	unsigned long flags = smp_load_acquire(&sk->sk_tsq_flags);
1305	unsigned long nflags;
1306
1307	/* perform an atomic operation only if at least one flag is set */
1308	do {
1309		if (!(flags & TCP_DEFERRED_ALL))
1310			return;
1311		nflags = flags & ~TCP_DEFERRED_ALL;
1312	} while (!try_cmpxchg(&sk->sk_tsq_flags, &flags, nflags));
1313
1314	if (flags & TCPF_TSQ_DEFERRED) {
1315		tcp_tsq_write(sk);
1316		__sock_put(sk);
1317	}
1318
1319	if (flags & TCPF_WRITE_TIMER_DEFERRED) {
1320		tcp_write_timer_handler(sk);
1321		__sock_put(sk);
1322	}
1323	if (flags & TCPF_DELACK_TIMER_DEFERRED) {
1324		tcp_delack_timer_handler(sk);
1325		__sock_put(sk);
1326	}
1327	if (flags & TCPF_MTU_REDUCED_DEFERRED) {
1328		inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
1329		__sock_put(sk);
1330	}
1331	if ((flags & TCPF_ACK_DEFERRED) && inet_csk_ack_scheduled(sk))
1332		tcp_send_ack(sk);
1333}
1334EXPORT_IPV6_MOD(tcp_release_cb);
1335
1336void __init tcp_tsq_work_init(void)
1337{
1338	int i;
1339
1340	for_each_possible_cpu(i) {
1341		struct tsq_work *tsq = &per_cpu(tsq_work, i);
1342
1343		INIT_LIST_HEAD(&tsq->head);
1344		INIT_WORK(&tsq->work, tcp_tsq_workfn);
1345	}
1346}
1347
1348/*
1349 * Write buffer destructor automatically called from kfree_skb.
1350 * We can't xmit new skbs from this context, as we might already
1351 * hold qdisc lock.
1352 */
1353void tcp_wfree(struct sk_buff *skb)
1354{
1355	struct sock *sk = skb->sk;
1356	struct tcp_sock *tp = tcp_sk(sk);
1357	unsigned long flags, nval, oval;
1358	struct tsq_work *tsq;
1359	bool empty;
1360
1361	/* Keep one reference on sk_wmem_alloc.
1362	 * Will be released by sk_free() from here or tcp_tsq_workfn()
1363	 */
1364	WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
1365
1366	/* If this softirq is serviced by ksoftirqd, we are likely under stress.
1367	 * Wait until our queues (qdisc + devices) are drained.
1368	 * This gives :
1369	 * - less callbacks to tcp_write_xmit(), reducing stress (batches)
1370	 * - chance for incoming ACK (processed by another cpu maybe)
1371	 *   to migrate this flow (skb->ooo_okay will be eventually set)
1372	 */
1373	if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
1374		goto out;
1375
1376	oval = smp_load_acquire(&sk->sk_tsq_flags);
1377	do {
1378		if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
1379			goto out;
1380
1381		nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
1382	} while (!try_cmpxchg(&sk->sk_tsq_flags, &oval, nval));
1383
1384	/* queue this socket to BH workqueue */
1385	local_irq_save(flags);
1386	tsq = this_cpu_ptr(&tsq_work);
1387	empty = list_empty(&tsq->head);
1388	list_add(&tp->tsq_node, &tsq->head);
1389	if (empty)
1390		queue_work(system_bh_wq, &tsq->work);
1391	local_irq_restore(flags);
1392	return;
1393out:
1394	sk_free(sk);
1395}
1396
1397/* Note: Called under soft irq.
1398 * We can call TCP stack right away, unless socket is owned by user.
1399 */
1400enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
1401{
1402	struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
1403	struct sock *sk = (struct sock *)tp;
1404
1405	tcp_tsq_handler(sk);
1406	sock_put(sk);
1407
1408	return HRTIMER_NORESTART;
1409}
1410
1411static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,
1412				      u64 prior_wstamp)
1413{
1414	struct tcp_sock *tp = tcp_sk(sk);
1415
1416	if (sk->sk_pacing_status != SK_PACING_NONE) {
1417		unsigned long rate = READ_ONCE(sk->sk_pacing_rate);
1418
1419		/* Original sch_fq does not pace first 10 MSS
1420		 * Note that tp->data_segs_out overflows after 2^32 packets,
1421		 * this is a minor annoyance.
1422		 */
1423		if (rate != ~0UL && rate && tp->data_segs_out >= 10) {
1424			u64 len_ns = div64_ul((u64)skb->len * NSEC_PER_SEC, rate);
1425			u64 credit = tp->tcp_wstamp_ns - prior_wstamp;
1426
1427			/* take into account OS jitter */
1428			len_ns -= min_t(u64, len_ns / 2, credit);
1429			tp->tcp_wstamp_ns += len_ns;
1430		}
1431	}
1432	list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1433}
1434
1435INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1436INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1437INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));
1438
1439/* This routine actually transmits TCP packets queued in by
1440 * tcp_do_sendmsg().  This is used by both the initial
1441 * transmission and possible later retransmissions.
1442 * All SKB's seen here are completely headerless.  It is our
1443 * job to build the TCP header, and pass the packet down to
1444 * IP so it can do the same plus pass the packet off to the
1445 * device.
1446 *
1447 * We are working here with either a clone of the original
1448 * SKB, or a fresh unique copy made by the retransmit engine.
1449 */
1450static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
1451			      int clone_it, gfp_t gfp_mask, u32 rcv_nxt)
1452{
1453	const struct inet_connection_sock *icsk = inet_csk(sk);
1454	struct inet_sock *inet;
1455	struct tcp_sock *tp;
1456	struct tcp_skb_cb *tcb;
1457	struct tcp_out_options opts;
1458	unsigned int tcp_options_size, tcp_header_size;
1459	struct sk_buff *oskb = NULL;
1460	struct tcp_key key;
1461	struct tcphdr *th;
1462	u64 prior_wstamp;
1463	int err;
1464
1465	BUG_ON(!skb || !tcp_skb_pcount(skb));
1466	tp = tcp_sk(sk);
1467	prior_wstamp = tp->tcp_wstamp_ns;
1468	tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache);
1469	skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC);
1470	if (clone_it) {
1471		oskb = skb;
1472
1473		tcp_skb_tsorted_save(oskb) {
1474			if (unlikely(skb_cloned(oskb)))
1475				skb = pskb_copy(oskb, gfp_mask);
1476			else
1477				skb = skb_clone(oskb, gfp_mask);
1478		} tcp_skb_tsorted_restore(oskb);
1479
1480		if (unlikely(!skb))
1481			return -ENOBUFS;
1482		/* retransmit skbs might have a non zero value in skb->dev
1483		 * because skb->dev is aliased with skb->rbnode.rb_left
1484		 */
1485		skb->dev = NULL;
1486	}
1487
1488	inet = inet_sk(sk);
1489	tcb = TCP_SKB_CB(skb);
1490	memset(&opts, 0, sizeof(opts));
1491
1492	tcp_get_current_key(sk, &key);
1493	if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
1494		tcp_options_size = tcp_syn_options(sk, skb, &opts, &key);
1495	} else {
1496		tcp_options_size = tcp_established_options(sk, skb, &opts, &key);
1497		/* Force a PSH flag on all (GSO) packets to expedite GRO flush
1498		 * at receiver : This slightly improve GRO performance.
1499		 * Note that we do not force the PSH flag for non GSO packets,
1500		 * because they might be sent under high congestion events,
1501		 * and in this case it is better to delay the delivery of 1-MSS
1502		 * packets and thus the corresponding ACK packet that would
1503		 * release the following packet.
1504		 */
1505		if (tcp_skb_pcount(skb) > 1)
1506			tcb->tcp_flags |= TCPHDR_PSH;
1507	}
1508	tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
1509
1510	/* We set skb->ooo_okay to one if this packet can select
1511	 * a different TX queue than prior packets of this flow,
1512	 * to avoid self inflicted reorders.
1513	 * The 'other' queue decision is based on current cpu number
1514	 * if XPS is enabled, or sk->sk_txhash otherwise.
1515	 * We can switch to another (and better) queue if:
1516	 * 1) No packet with payload is in qdisc/device queues.
1517	 *    Delays in TX completion can defeat the test
1518	 *    even if packets were already sent.
1519	 * 2) Or rtx queue is empty.
1520	 *    This mitigates above case if ACK packets for
1521	 *    all prior packets were already processed.
1522	 */
1523	skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) ||
1524			tcp_rtx_queue_empty(sk);
1525
1526	/* If we had to use memory reserve to allocate this skb,
1527	 * this might cause drops if packet is looped back :
1528	 * Other socket might not have SOCK_MEMALLOC.
1529	 * Packets not looped back do not care about pfmemalloc.
1530	 */
1531	skb->pfmemalloc = 0;
1532
1533	skb_push(skb, tcp_header_size);
1534	skb_reset_transport_header(skb);
1535
1536	skb_orphan(skb);
1537	skb->sk = sk;
1538	skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
1539	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
1540
1541	skb_set_dst_pending_confirm(skb, READ_ONCE(sk->sk_dst_pending_confirm));
1542
1543	/* Build TCP header and checksum it. */
1544	th = (struct tcphdr *)skb->data;
1545	th->source		= inet->inet_sport;
1546	th->dest		= inet->inet_dport;
1547	th->seq			= htonl(tcb->seq);
1548	th->ack_seq		= htonl(rcv_nxt);
1549	*(((__be16 *)th) + 6)	= htons(((tcp_header_size >> 2) << 12) |
1550					(tcb->tcp_flags & TCPHDR_FLAGS_MASK));
1551
1552	th->check		= 0;
1553	th->urg_ptr		= 0;
1554
1555	/* The urg_mode check is necessary during a below snd_una win probe */
1556	if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
1557		if (before(tp->snd_up, tcb->seq + 0x10000)) {
1558			th->urg_ptr = htons(tp->snd_up - tcb->seq);
1559			th->urg = 1;
1560		} else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
1561			th->urg_ptr = htons(0xFFFF);
1562			th->urg = 1;
1563		}
1564	}
1565
1566	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1567	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
1568		th->window      = htons(tcp_select_window(sk));
1569		tcp_ecn_send(sk, skb, th, tcp_header_size);
1570	} else {
1571		/* RFC1323: The window in SYN & SYN/ACK segments
1572		 * is never scaled.
1573		 */
1574		th->window	= htons(min(tp->rcv_wnd, 65535U));
1575	}
1576
1577	tcp_options_write(th, tp, NULL, &opts, &key);
1578
1579	if (tcp_key_is_md5(&key)) {
1580#ifdef CONFIG_TCP_MD5SIG
1581		/* Calculate the MD5 hash, as we have all we need now */
1582		sk_gso_disable(sk);
1583		tp->af_specific->calc_md5_hash(opts.hash_location,
1584					       key.md5_key, sk, skb);
1585#endif
1586	} else if (tcp_key_is_ao(&key)) {
1587		int err;
1588
1589		err = tcp_ao_transmit_skb(sk, skb, key.ao_key, th,
1590					  opts.hash_location);
1591		if (err) {
1592			sk_skb_reason_drop(sk, skb, SKB_DROP_REASON_NOT_SPECIFIED);
1593			return -ENOMEM;
1594		}
1595	}
1596
1597	/* BPF prog is the last one writing header option */
1598	bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts);
1599
1600	INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check,
1601			   tcp_v6_send_check, tcp_v4_send_check,
1602			   sk, skb);
1603
1604	if (likely(tcb->tcp_flags & TCPHDR_ACK))
1605		tcp_event_ack_sent(sk, rcv_nxt);
1606
1607	if (skb->len != tcp_header_size) {
1608		tcp_event_data_sent(tp, sk);
1609		tp->data_segs_out += tcp_skb_pcount(skb);
1610		tp->bytes_sent += skb->len - tcp_header_size;
1611	}
1612
1613	if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1614		TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1615			      tcp_skb_pcount(skb));
1616
1617	tp->segs_out += tcp_skb_pcount(skb);
1618	skb_set_hash_from_sk(skb, sk);
1619	/* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
1620	skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1621	skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
1622
1623	/* Leave earliest departure time in skb->tstamp (skb->skb_mstamp_ns) */
1624
1625	/* Cleanup our debris for IP stacks */
1626	memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1627			       sizeof(struct inet6_skb_parm)));
1628
1629	tcp_add_tx_delay(skb, tp);
1630
1631	err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit,
1632				 inet6_csk_xmit, ip_queue_xmit,
1633				 sk, skb, &inet->cork.fl);
1634
1635	if (unlikely(err > 0)) {
1636		tcp_enter_cwr(sk);
1637		err = net_xmit_eval(err);
1638	}
1639	if (!err && oskb) {
1640		tcp_update_skb_after_send(sk, oskb, prior_wstamp);
1641		tcp_rate_skb_sent(sk, oskb);
1642	}
1643	return err;
1644}
1645
1646static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
1647			    gfp_t gfp_mask)
1648{
1649	return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask,
1650				  tcp_sk(sk)->rcv_nxt);
1651}
1652
1653/* This routine just queues the buffer for sending.
1654 *
1655 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
1656 * otherwise socket can stall.
1657 */
1658static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1659{
1660	struct tcp_sock *tp = tcp_sk(sk);
1661
1662	/* Advance write_seq and place onto the write_queue. */
1663	WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);
1664	__skb_header_release(skb);
1665	psp_enqueue_set_decrypted(sk, skb);
1666	tcp_add_write_queue_tail(sk, skb);
1667	sk_wmem_queued_add(sk, skb->truesize);
1668	sk_mem_charge(sk, skb->truesize);
1669}
1670
1671/* Initialize TSO segments for a packet. */
1672static int tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1673{
1674	int tso_segs;
1675
1676	if (skb->len <= mss_now) {
1677		/* Avoid the costly divide in the normal
1678		 * non-TSO case.
1679		 */
1680		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1681		tcp_skb_pcount_set(skb, 1);
1682		return 1;
1683	}
1684	TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
1685	tso_segs = DIV_ROUND_UP(skb->len, mss_now);
1686	tcp_skb_pcount_set(skb, tso_segs);
1687	return tso_segs;
1688}
1689
1690/* Pcount in the middle of the write queue got changed, we need to do various
1691 * tweaks to fix counters
1692 */
1693static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1694{
1695	struct tcp_sock *tp = tcp_sk(sk);
1696
1697	tp->packets_out -= decr;
1698
1699	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1700		tp->sacked_out -= decr;
1701	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1702		tp->retrans_out -= decr;
1703	if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1704		tp->lost_out -= decr;
1705
1706	/* Reno case is special. Sigh... */
1707	if (tcp_is_reno(tp) && decr > 0)
1708		tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1709
1710	tcp_verify_left_out(tp);
1711}
1712
1713static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
1714{
1715	return TCP_SKB_CB(skb)->txstamp_ack ||
1716		(skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
1717}
1718
1719static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1720{
1721	struct skb_shared_info *shinfo = skb_shinfo(skb);
1722
1723	if (unlikely(tcp_has_tx_tstamp(skb)) &&
1724	    !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1725		struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1726		u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1727
1728		shinfo->tx_flags &= ~tsflags;
1729		shinfo2->tx_flags |= tsflags;
1730		swap(shinfo->tskey, shinfo2->tskey);
1731		TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
1732		TCP_SKB_CB(skb)->txstamp_ack = 0;
1733	}
1734}
1735
1736static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
1737{
1738	TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
1739	TCP_SKB_CB(skb)->eor = 0;
1740}
1741
1742/* Insert buff after skb on the write or rtx queue of sk.  */
1743static void tcp_insert_write_queue_after(struct sk_buff *skb,
1744					 struct sk_buff *buff,
1745					 struct sock *sk,
1746					 enum tcp_queue tcp_queue)
1747{
1748	if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
1749		__skb_queue_after(&sk->sk_write_queue, skb, buff);
1750	else
1751		tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
1752}
1753
1754/* Function to create two new TCP segments.  Shrinks the given segment
1755 * to the specified size and appends a new segment with the rest of the
1756 * packet to the list.  This won't be called frequently, I hope.
1757 * Remember, these are still headerless SKBs at this point.
1758 */
1759int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
1760		 struct sk_buff *skb, u32 len,
1761		 unsigned int mss_now, gfp_t gfp)
1762{
1763	struct tcp_sock *tp = tcp_sk(sk);
1764	struct sk_buff *buff;
1765	int old_factor;
1766	long limit;
1767	u16 flags;
1768	int nlen;
1769
1770	if (WARN_ON(len > skb->len))
1771		return -EINVAL;
1772
1773	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
1774
1775	/* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb.
1776	 * We need some allowance to not penalize applications setting small
1777	 * SO_SNDBUF values.
1778	 * Also allow first and last skb in retransmit queue to be split.
1779	 */
1780	limit = sk->sk_sndbuf + 2 * SKB_TRUESIZE(GSO_LEGACY_MAX_SIZE);
1781	if (unlikely((sk->sk_wmem_queued >> 1) > limit &&
1782		     tcp_queue != TCP_FRAG_IN_WRITE_QUEUE &&
1783		     skb != tcp_rtx_queue_head(sk) &&
1784		     skb != tcp_rtx_queue_tail(sk))) {
1785		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPWQUEUETOOBIG);
1786		return -ENOMEM;
1787	}
1788
1789	if (skb_unclone_keeptruesize(skb, gfp))
1790		return -ENOMEM;
1791
1792	/* Get a new skb... force flag on. */
1793	buff = tcp_stream_alloc_skb(sk, gfp, true);
1794	if (!buff)
1795		return -ENOMEM; /* We'll just try again later. */
1796	skb_copy_decrypted(buff, skb);
1797	mptcp_skb_ext_copy(buff, skb);
1798
1799	sk_wmem_queued_add(sk, buff->truesize);
1800	sk_mem_charge(sk, buff->truesize);
1801	nlen = skb->len - len;
1802	buff->truesize += nlen;
1803	skb->truesize -= nlen;
1804
1805	/* Correct the sequence numbers. */
1806	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1807	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1808	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1809
1810	/* PSH and FIN should only be set in the second packet. */
1811	flags = TCP_SKB_CB(skb)->tcp_flags;
1812	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1813	TCP_SKB_CB(buff)->tcp_flags = flags;
1814	TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1815	tcp_skb_fragment_eor(skb, buff);
1816
1817	skb_split(skb, buff, len);
1818
1819	skb_set_delivery_time(buff, skb->tstamp, SKB_CLOCK_MONOTONIC);
1820	tcp_fragment_tstamp(skb, buff);
1821
1822	old_factor = tcp_skb_pcount(skb);
1823
1824	/* Fix up tso_factor for both original and new SKB.  */
1825	tcp_set_skb_tso_segs(skb, mss_now);
1826	tcp_set_skb_tso_segs(buff, mss_now);
1827
1828	/* Update delivered info for the new segment */
1829	TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
1830
1831	/* If this packet has been sent out already, we must
1832	 * adjust the various packet counters.
1833	 */
1834	if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1835		int diff = old_factor - tcp_skb_pcount(skb) -
1836			tcp_skb_pcount(buff);
1837
1838		if (diff)
1839			tcp_adjust_pcount(sk, skb, diff);
1840	}
1841
1842	/* Link BUFF into the send queue. */
1843	__skb_header_release(buff);
1844	tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
1845	if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
1846		list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor);
1847
1848	return 0;
1849}
1850
1851/* This is similar to __pskb_pull_tail(). The difference is that pulled
1852 * data is not copied, but immediately discarded.
1853 */
1854static int __pskb_trim_head(struct sk_buff *skb, int len)
1855{
1856	struct skb_shared_info *shinfo;
1857	int i, k, eat;
1858
1859	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
1860	eat = len;
1861	k = 0;
1862	shinfo = skb_shinfo(skb);
1863	for (i = 0; i < shinfo->nr_frags; i++) {
1864		int size = skb_frag_size(&shinfo->frags[i]);
1865
1866		if (size <= eat) {
1867			skb_frag_unref(skb, i);
1868			eat -= size;
1869		} else {
1870			shinfo->frags[k] = shinfo->frags[i];
1871			if (eat) {
1872				skb_frag_off_add(&shinfo->frags[k], eat);
1873				skb_frag_size_sub(&shinfo->frags[k], eat);
1874				eat = 0;
1875			}
1876			k++;
1877		}
1878	}
1879	shinfo->nr_frags = k;
1880
1881	skb->data_len -= len;
1882	skb->len = skb->data_len;
1883	return len;
1884}
1885
1886/* Remove acked data from a packet in the transmit queue. */
1887int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1888{
1889	u32 delta_truesize;
1890
1891	if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
1892		return -ENOMEM;
1893
1894	delta_truesize = __pskb_trim_head(skb, len);
1895
1896	TCP_SKB_CB(skb)->seq += len;
1897
1898	skb->truesize	   -= delta_truesize;
1899	sk_wmem_queued_add(sk, -delta_truesize);
1900	if (!skb_zcopy_pure(skb))
1901		sk_mem_uncharge(sk, delta_truesize);
1902
1903	/* Any change of skb->len requires recalculation of tso factor. */
1904	if (tcp_skb_pcount(skb) > 1)
1905		tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
1906
1907	return 0;
1908}
1909
1910/* Calculate MSS not accounting any TCP options.  */
1911static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1912{
1913	const struct tcp_sock *tp = tcp_sk(sk);
1914	const struct inet_connection_sock *icsk = inet_csk(sk);
1915	int mss_now;
1916
1917	/* Calculate base mss without TCP options:
1918	   It is MMS_S - sizeof(tcphdr) of rfc1122
1919	 */
1920	mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1921
1922	/* Clamp it (mss_clamp does not include tcp options) */
1923	if (mss_now > tp->rx_opt.mss_clamp)
1924		mss_now = tp->rx_opt.mss_clamp;
1925
1926	/* Now subtract optional transport overhead */
1927	mss_now -= icsk->icsk_ext_hdr_len;
1928
1929	/* Then reserve room for full set of TCP options and 8 bytes of data */
1930	mss_now = max(mss_now,
1931		      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss));
1932	return mss_now;
1933}
1934
1935/* Calculate MSS. Not accounting for SACKs here.  */
1936int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1937{
1938	/* Subtract TCP options size, not including SACKs */
1939	return __tcp_mtu_to_mss(sk, pmtu) -
1940	       (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1941}
1942EXPORT_IPV6_MOD(tcp_mtu_to_mss);
1943
1944/* Inverse of above */
1945int tcp_mss_to_mtu(struct sock *sk, int mss)
1946{
1947	const struct tcp_sock *tp = tcp_sk(sk);
1948	const struct inet_connection_sock *icsk = inet_csk(sk);
1949
1950	return mss +
1951	      tp->tcp_header_len +
1952	      icsk->icsk_ext_hdr_len +
1953	      icsk->icsk_af_ops->net_header_len;
1954}
1955EXPORT_SYMBOL(tcp_mss_to_mtu);
1956
1957/* MTU probing init per socket */
1958void tcp_mtup_init(struct sock *sk)
1959{
1960	struct tcp_sock *tp = tcp_sk(sk);
1961	struct inet_connection_sock *icsk = inet_csk(sk);
1962	struct net *net = sock_net(sk);
1963
1964	icsk->icsk_mtup.enabled = READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing) > 1;
1965	icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1966			       icsk->icsk_af_ops->net_header_len;
1967	icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, READ_ONCE(net->ipv4.sysctl_tcp_base_mss));
1968	icsk->icsk_mtup.probe_size = 0;
1969	if (icsk->icsk_mtup.enabled)
1970		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
1971}
1972
1973/* This function synchronize snd mss to current pmtu/exthdr set.
1974
1975   tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1976   for TCP options, but includes only bare TCP header.
1977
1978   tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1979   It is minimum of user_mss and mss received with SYN.
1980   It also does not include TCP options.
1981
1982   inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1983
1984   tp->mss_cache is current effective sending mss, including
1985   all tcp options except for SACKs. It is evaluated,
1986   taking into account current pmtu, but never exceeds
1987   tp->rx_opt.mss_clamp.
1988
1989   NOTE1. rfc1122 clearly states that advertised MSS
1990   DOES NOT include either tcp or ip options.
1991
1992   NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1993   are READ ONLY outside this function.		--ANK (980731)
1994 */
1995unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1996{
1997	struct tcp_sock *tp = tcp_sk(sk);
1998	struct inet_connection_sock *icsk = inet_csk(sk);
1999	int mss_now;
2000
2001	if (icsk->icsk_mtup.search_high > pmtu)
2002		icsk->icsk_mtup.search_high = pmtu;
2003
2004	mss_now = tcp_mtu_to_mss(sk, pmtu);
2005	mss_now = tcp_bound_to_half_wnd(tp, mss_now);
2006
2007	/* And store cached results */
2008	icsk->icsk_pmtu_cookie = pmtu;
2009	if (icsk->icsk_mtup.enabled)
2010		mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
2011	tp->mss_cache = mss_now;
2012
2013	return mss_now;
2014}
2015EXPORT_IPV6_MOD(tcp_sync_mss);
2016
2017/* Compute the current effective MSS, taking SACKs and IP options,
2018 * and even PMTU discovery events into account.
2019 */
2020unsigned int tcp_current_mss(struct sock *sk)
2021{
2022	const struct tcp_sock *tp = tcp_sk(sk);
2023	const struct dst_entry *dst = __sk_dst_get(sk);
2024	u32 mss_now;
2025	unsigned int header_len;
2026	struct tcp_out_options opts;
2027	struct tcp_key key;
2028
2029	mss_now = tp->mss_cache;
2030
2031	if (dst) {
2032		u32 mtu = dst_mtu(dst);
2033		if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
2034			mss_now = tcp_sync_mss(sk, mtu);
2035	}
2036	tcp_get_current_key(sk, &key);
2037	header_len = tcp_established_options(sk, NULL, &opts, &key) +
2038		     sizeof(struct tcphdr);
2039	/* The mss_cache is sized based on tp->tcp_header_len, which assumes
2040	 * some common options. If this is an odd packet (because we have SACK
2041	 * blocks etc) then our calculated header_len will be different, and
2042	 * we have to adjust mss_now correspondingly */
2043	if (header_len != tp->tcp_header_len) {
2044		int delta = (int) header_len - tp->tcp_header_len;
2045		mss_now -= delta;
2046	}
2047
2048	return mss_now;
2049}
2050
2051/* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
2052 * As additional protections, we do not touch cwnd in retransmission phases,
2053 * and if application hit its sndbuf limit recently.
2054 */
2055static void tcp_cwnd_application_limited(struct sock *sk)
2056{
2057	struct tcp_sock *tp = tcp_sk(sk);
2058
2059	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
2060	    sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
2061		/* Limited by application or receiver window. */
2062		u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
2063		u32 win_used = max(tp->snd_cwnd_used, init_win);
2064		if (win_used < tcp_snd_cwnd(tp)) {
2065			tp->snd_ssthresh = tcp_current_ssthresh(sk);
2066			tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1);
2067		}
2068		tp->snd_cwnd_used = 0;
2069	}
2070	tp->snd_cwnd_stamp = tcp_jiffies32;
2071}
2072
2073static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
2074{
2075	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
2076	struct tcp_sock *tp = tcp_sk(sk);
2077
2078	/* Track the strongest available signal of the degree to which the cwnd
2079	 * is fully utilized. If cwnd-limited then remember that fact for the
2080	 * current window. If not cwnd-limited then track the maximum number of
2081	 * outstanding packets in the current window. (If cwnd-limited then we
2082	 * chose to not update tp->max_packets_out to avoid an extra else
2083	 * clause with no functional impact.)
2084	 */
2085	if (!before(tp->snd_una, tp->cwnd_usage_seq) ||
2086	    is_cwnd_limited ||
2087	    (!tp->is_cwnd_limited &&
2088	     tp->packets_out > tp->max_packets_out)) {
2089		tp->is_cwnd_limited = is_cwnd_limited;
2090		tp->max_packets_out = tp->packets_out;
2091		tp->cwnd_usage_seq = tp->snd_nxt;
2092	}
2093
2094	if (tcp_is_cwnd_limited(sk)) {
2095		/* Network is feed fully. */
2096		tp->snd_cwnd_used = 0;
2097		tp->snd_cwnd_stamp = tcp_jiffies32;
2098	} else {
2099		/* Network starves. */
2100		if (tp->packets_out > tp->snd_cwnd_used)
2101			tp->snd_cwnd_used = tp->packets_out;
2102
2103		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) &&
2104		    (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
2105		    !ca_ops->cong_control)
2106			tcp_cwnd_application_limited(sk);
2107
2108		/* The following conditions together indicate the starvation
2109		 * is caused by insufficient sender buffer:
2110		 * 1) just sent some data (see tcp_write_xmit)
2111		 * 2) not cwnd limited (this else condition)
2112		 * 3) no more data to send (tcp_write_queue_empty())
2113		 * 4) application is hitting buffer limit (SOCK_NOSPACE)
2114		 */
2115		if (tcp_write_queue_empty(sk) && sk->sk_socket &&
2116		    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
2117		    (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
2118			tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
2119	}
2120}
2121
2122/* Minshall's variant of the Nagle send check. */
2123static bool tcp_minshall_check(const struct tcp_sock *tp)
2124{
2125	return after(tp->snd_sml, tp->snd_una) &&
2126		!after(tp->snd_sml, tp->snd_nxt);
2127}
2128
2129/* Update snd_sml if this skb is under mss
2130 * Note that a TSO packet might end with a sub-mss segment
2131 * The test is really :
2132 * if ((skb->len % mss) != 0)
2133 *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
2134 * But we can avoid doing the divide again given we already have
2135 *  skb_pcount = skb->len / mss_now
2136 */
2137static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
2138				const struct sk_buff *skb)
2139{
2140	if (skb->len < tcp_skb_pcount(skb) * mss_now)
2141		tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
2142}
2143
2144/* Return false, if packet can be sent now without violation Nagle's rules:
2145 * 1. It is full sized. (provided by caller in %partial bool)
2146 * 2. Or it contains FIN. (already checked by caller)
2147 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
2148 * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
2149 *    With Minshall's modification: all sent small packets are ACKed.
2150 */
2151static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
2152			    int nonagle)
2153{
2154	return partial &&
2155		((nonagle & TCP_NAGLE_CORK) ||
2156		 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
2157}
2158
2159/* Return how many segs we'd like on a TSO packet,
2160 * depending on current pacing rate, and how close the peer is.
2161 *
2162 * Rationale is:
2163 * - For close peers, we rather send bigger packets to reduce
2164 *   cpu costs, because occasional losses will be repaired fast.
2165 * - For long distance/rtt flows, we would like to get ACK clocking
2166 *   with 1 ACK per ms.
2167 *
2168 * Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting
2169 * in bigger TSO bursts. We we cut the RTT-based allowance in half
2170 * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance
2171 * is below 1500 bytes after 6 * ~500 usec = 3ms.
2172 */
2173static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
2174			    int min_tso_segs)
2175{
2176	unsigned long bytes;
2177	u32 r;
2178
2179	bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift);
2180
2181	r = tcp_min_rtt(tcp_sk(sk)) >> READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log);
2182	if (r < BITS_PER_TYPE(sk->sk_gso_max_size))
2183		bytes += sk->sk_gso_max_size >> r;
2184
2185	bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size);
2186
2187	return max_t(u32, bytes / mss_now, min_tso_segs);
2188}
2189
2190/* Return the number of segments we want in the skb we are transmitting.
2191 * See if congestion control module wants to decide; otherwise, autosize.
2192 */
2193static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
2194{
2195	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
2196	u32 min_tso, tso_segs;
2197
2198	min_tso = ca_ops->min_tso_segs ?
2199			ca_ops->min_tso_segs(sk) :
2200			READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);
2201
2202	tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
2203	return min_t(u32, tso_segs, sk->sk_gso_max_segs);
2204}
2205
2206/* Returns the portion of skb which can be sent right away */
2207static unsigned int tcp_mss_split_point(const struct sock *sk,
2208					const struct sk_buff *skb,
2209					unsigned int mss_now,
2210					unsigned int max_segs,
2211					int nonagle)
2212{
2213	const struct tcp_sock *tp = tcp_sk(sk);
2214	u32 partial, needed, window, max_len;
2215
2216	window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2217	max_len = mss_now * max_segs;
2218
2219	if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
2220		return max_len;
2221
2222	needed = min(skb->len, window);
2223
2224	if (max_len <= needed)
2225		return max_len;
2226
2227	partial = needed % mss_now;
2228	/* If last segment is not a full MSS, check if Nagle rules allow us
2229	 * to include this last segment in this skb.
2230	 * Otherwise, we'll split the skb at last MSS boundary
2231	 */
2232	if (tcp_nagle_check(partial != 0, tp, nonagle))
2233		return needed - partial;
2234
2235	return needed;
2236}
2237
2238/* Can at least one segment of SKB be sent right now, according to the
2239 * congestion window rules?  If so, return how many segments are allowed.
2240 */
2241static u32 tcp_cwnd_test(const struct tcp_sock *tp)
2242{
2243	u32 in_flight, cwnd, halfcwnd;
2244
2245	in_flight = tcp_packets_in_flight(tp);
2246	cwnd = tcp_snd_cwnd(tp);
2247	if (in_flight >= cwnd)
2248		return 0;
2249
2250	/* For better scheduling, ensure we have at least
2251	 * 2 GSO packets in flight.
2252	 */
2253	halfcwnd = max(cwnd >> 1, 1U);
2254	return min(halfcwnd, cwnd - in_flight);
2255}
2256
2257/* Initialize TSO state of a skb.
2258 * This must be invoked the first time we consider transmitting
2259 * SKB onto the wire.
2260 */
2261static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
2262{
2263	int tso_segs = tcp_skb_pcount(skb);
2264
2265	if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now))
2266		return tcp_set_skb_tso_segs(skb, mss_now);
2267
2268	return tso_segs;
2269}
2270
2271
2272/* Return true if the Nagle test allows this packet to be
2273 * sent now.
2274 */
2275static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
2276				  unsigned int cur_mss, int nonagle)
2277{
2278	/* Nagle rule does not apply to frames, which sit in the middle of the
2279	 * write_queue (they have no chances to get new data).
2280	 *
2281	 * This is implemented in the callers, where they modify the 'nonagle'
2282	 * argument based upon the location of SKB in the send queue.
2283	 */
2284	if (nonagle & TCP_NAGLE_PUSH)
2285		return true;
2286
2287	/* Don't use the nagle rule for urgent data (or for the final FIN). */
2288	if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
2289		return true;
2290
2291	if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
2292		return true;
2293
2294	return false;
2295}
2296
2297/* Does at least the first segment of SKB fit into the send window? */
2298static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
2299			     const struct sk_buff *skb,
2300			     unsigned int cur_mss)
2301{
2302	u32 end_seq = TCP_SKB_CB(skb)->end_seq;
2303
2304	if (skb->len > cur_mss)
2305		end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
2306
2307	return !after(end_seq, tcp_wnd_end(tp));
2308}
2309
2310/* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
2311 * which is put after SKB on the list.  It is very much like
2312 * tcp_fragment() except that it may make several kinds of assumptions
2313 * in order to speed up the splitting operation.  In particular, we
2314 * know that all the data is in scatter-gather pages, and that the
2315 * packet has never been sent out before (and thus is not cloned).
2316 */
2317static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
2318			unsigned int mss_now, gfp_t gfp)
2319{
2320	int nlen = skb->len - len;
2321	struct sk_buff *buff;
2322	u16 flags;
2323
2324	/* All of a TSO frame must be composed of paged data.  */
2325	DEBUG_NET_WARN_ON_ONCE(skb->len != skb->data_len);
2326
2327	buff = tcp_stream_alloc_skb(sk, gfp, true);
2328	if (unlikely(!buff))
2329		return -ENOMEM;
2330	skb_copy_decrypted(buff, skb);
2331	mptcp_skb_ext_copy(buff, skb);
2332
2333	sk_wmem_queued_add(sk, buff->truesize);
2334	sk_mem_charge(sk, buff->truesize);
2335	buff->truesize += nlen;
2336	skb->truesize -= nlen;
2337
2338	/* Correct the sequence numbers. */
2339	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
2340	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
2341	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
2342
2343	/* PSH and FIN should only be set in the second packet. */
2344	flags = TCP_SKB_CB(skb)->tcp_flags;
2345	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
2346	TCP_SKB_CB(buff)->tcp_flags = flags;
2347
2348	tcp_skb_fragment_eor(skb, buff);
2349
2350	skb_split(skb, buff, len);
2351	tcp_fragment_tstamp(skb, buff);
2352
2353	/* Fix up tso_factor for both original and new SKB.  */
2354	tcp_set_skb_tso_segs(skb, mss_now);
2355	tcp_set_skb_tso_segs(buff, mss_now);
2356
2357	/* Link BUFF into the send queue. */
2358	__skb_header_release(buff);
2359	tcp_insert_write_queue_after(skb, buff, sk, TCP_FRAG_IN_WRITE_QUEUE);
2360
2361	return 0;
2362}
2363
2364/* Try to defer sending, if possible, in order to minimize the amount
2365 * of TSO splitting we do.  View it as a kind of TSO Nagle test.
2366 *
2367 * This algorithm is from John Heffner.
2368 */
2369static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
2370				 bool *is_cwnd_limited,
2371				 bool *is_rwnd_limited,
2372				 u32 max_segs)
2373{
2374	const struct inet_connection_sock *icsk = inet_csk(sk);
2375	u32 send_win, cong_win, limit, in_flight, threshold;
2376	u64 srtt_in_ns, expected_ack, how_far_is_the_ack;
2377	struct tcp_sock *tp = tcp_sk(sk);
2378	struct sk_buff *head;
2379	int win_divisor;
2380	s64 delta;
2381
2382	if (icsk->icsk_ca_state >= TCP_CA_Recovery)
2383		goto send_now;
2384
2385	/* Avoid bursty behavior by allowing defer
2386	 * only if the last write was recent (1 ms).
2387	 * Note that tp->tcp_wstamp_ns can be in the future if we have
2388	 * packets waiting in a qdisc or device for EDT delivery.
2389	 */
2390	delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC;
2391	if (delta > 0)
2392		goto send_now;
2393
2394	in_flight = tcp_packets_in_flight(tp);
2395
2396	BUG_ON(tcp_skb_pcount(skb) <= 1);
2397	BUG_ON(tcp_snd_cwnd(tp) <= in_flight);
2398
2399	send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2400
2401	/* From in_flight test above, we know that cwnd > in_flight.  */
2402	cong_win = (tcp_snd_cwnd(tp) - in_flight) * tp->mss_cache;
2403
2404	limit = min(send_win, cong_win);
2405
2406	/* If a full-sized TSO skb can be sent, do it. */
2407	if (limit >= max_segs * tp->mss_cache)
2408		goto send_now;
2409
2410	/* Middle in queue won't get any more data, full sendable already? */
2411	if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
2412		goto send_now;
2413
2414	win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor);
2415	if (win_divisor) {
2416		u32 chunk = min(tp->snd_wnd, tcp_snd_cwnd(tp) * tp->mss_cache);
2417
2418		/* If at least some fraction of a window is available,
2419		 * just use it.
2420		 */
2421		chunk /= win_divisor;
2422		if (limit >= chunk)
2423			goto send_now;
2424	} else {
2425		/* Different approach, try not to defer past a single
2426		 * ACK.  Receiver should ACK every other full sized
2427		 * frame, so if we have space for more than 3 frames
2428		 * then send now.
2429		 */
2430		if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
2431			goto send_now;
2432	}
2433
2434	/* TODO : use tsorted_sent_queue ? */
2435	head = tcp_rtx_queue_head(sk);
2436	if (!head)
2437		goto send_now;
2438
2439	srtt_in_ns = (u64)(NSEC_PER_USEC >> 3) * tp->srtt_us;
2440	/* When is the ACK expected ? */
2441	expected_ack = head->tstamp + srtt_in_ns;
2442	/* How far from now is the ACK expected ? */
2443	how_far_is_the_ack = expected_ack - tp->tcp_clock_cache;
2444
2445	/* If next ACK is likely to come too late,
2446	 * ie in more than min(1ms, half srtt), do not defer.
2447	 */
2448	threshold = min(srtt_in_ns >> 1, NSEC_PER_MSEC);
2449
2450	if ((s64)(how_far_is_the_ack - threshold) > 0)
2451		goto send_now;
2452
2453	/* Ok, it looks like it is advisable to defer.
2454	 * Three cases are tracked :
2455	 * 1) We are cwnd-limited
2456	 * 2) We are rwnd-limited
2457	 * 3) We are application limited.
2458	 */
2459	if (cong_win < send_win) {
2460		if (cong_win <= skb->len) {
2461			*is_cwnd_limited = true;
2462			return true;
2463		}
2464	} else {
2465		if (send_win <= skb->len) {
2466			*is_rwnd_limited = true;
2467			return true;
2468		}
2469	}
2470
2471	/* If this packet won't get more data, do not wait. */
2472	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) ||
2473	    TCP_SKB_CB(skb)->eor)
2474		goto send_now;
2475
2476	return true;
2477
2478send_now:
2479	return false;
2480}
2481
2482static inline void tcp_mtu_check_reprobe(struct sock *sk)
2483{
2484	struct inet_connection_sock *icsk = inet_csk(sk);
2485	struct tcp_sock *tp = tcp_sk(sk);
2486	struct net *net = sock_net(sk);
2487	u32 interval;
2488	s32 delta;
2489
2490	interval = READ_ONCE(net->ipv4.sysctl_tcp_probe_interval);
2491	delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
2492	if (unlikely(delta >= interval * HZ)) {
2493		int mss = tcp_current_mss(sk);
2494
2495		/* Update current search range */
2496		icsk->icsk_mtup.probe_size = 0;
2497		icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
2498			sizeof(struct tcphdr) +
2499			icsk->icsk_af_ops->net_header_len;
2500		icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
2501
2502		/* Update probe time stamp */
2503		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
2504	}
2505}
2506
2507static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
2508{
2509	struct sk_buff *skb, *next;
2510
2511	skb = tcp_send_head(sk);
2512	tcp_for_write_queue_from_safe(skb, next, sk) {
2513		if (len <= skb->len)
2514			break;
2515
2516		if (tcp_has_tx_tstamp(skb) || !tcp_skb_can_collapse(skb, next))
2517			return false;
2518
2519		len -= skb->len;
2520	}
2521
2522	return true;
2523}
2524
2525static int tcp_clone_payload(struct sock *sk, struct sk_buff *to,
2526			     int probe_size)
2527{
2528	skb_frag_t *lastfrag = NULL, *fragto = skb_shinfo(to)->frags;
2529	int i, todo, len = 0, nr_frags = 0;
2530	const struct sk_buff *skb;
2531
2532	if (!sk_wmem_schedule(sk, to->truesize + probe_size))
2533		return -ENOMEM;
2534
2535	skb_queue_walk(&sk->sk_write_queue, skb) {
2536		const skb_frag_t *fragfrom = skb_shinfo(skb)->frags;
2537
2538		if (skb_headlen(skb))
2539			return -EINVAL;
2540
2541		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, fragfrom++) {
2542			if (len >= probe_size)
2543				goto commit;
2544			todo = min_t(int, skb_frag_size(fragfrom),
2545				     probe_size - len);
2546			len += todo;
2547			if (lastfrag &&
2548			    skb_frag_page(fragfrom) == skb_frag_page(lastfrag) &&
2549			    skb_frag_off(fragfrom) == skb_frag_off(lastfrag) +
2550						      skb_frag_size(lastfrag)) {
2551				skb_frag_size_add(lastfrag, todo);
2552				continue;
2553			}
2554			if (unlikely(nr_frags == MAX_SKB_FRAGS))
2555				return -E2BIG;
2556			skb_frag_page_copy(fragto, fragfrom);
2557			skb_frag_off_copy(fragto, fragfrom);
2558			skb_frag_size_set(fragto, todo);
2559			nr_frags++;
2560			lastfrag = fragto++;
2561		}
2562	}
2563commit:
2564	WARN_ON_ONCE(len != probe_size);
2565	for (i = 0; i < nr_frags; i++)
2566		skb_frag_ref(to, i);
2567
2568	skb_shinfo(to)->nr_frags = nr_frags;
2569	to->truesize += probe_size;
2570	to->len += probe_size;
2571	to->data_len += probe_size;
2572	__skb_header_release(to);
2573	return 0;
2574}
2575
2576/* tcp_mtu_probe() and tcp_grow_skb() can both eat an skb (src) if
2577 * all its payload was moved to another one (dst).
2578 * Make sure to transfer tcp_flags, eor, and tstamp.
2579 */
2580static void tcp_eat_one_skb(struct sock *sk,
2581			    struct sk_buff *dst,
2582			    struct sk_buff *src)
2583{
2584	TCP_SKB_CB(dst)->tcp_flags |= TCP_SKB_CB(src)->tcp_flags;
2585	TCP_SKB_CB(dst)->eor = TCP_SKB_CB(src)->eor;
2586	tcp_skb_collapse_tstamp(dst, src);
2587	tcp_unlink_write_queue(src, sk);
2588	tcp_wmem_free_skb(sk, src);
2589}
2590
2591/* Create a new MTU probe if we are ready.
2592 * MTU probe is regularly attempting to increase the path MTU by
2593 * deliberately sending larger packets.  This discovers routing
2594 * changes resulting in larger path MTUs.
2595 *
2596 * Returns 0 if we should wait to probe (no cwnd available),
2597 *         1 if a probe was sent,
2598 *         -1 otherwise
2599 */
2600static int tcp_mtu_probe(struct sock *sk)
2601{
2602	struct inet_connection_sock *icsk = inet_csk(sk);
2603	struct tcp_sock *tp = tcp_sk(sk);
2604	struct sk_buff *skb, *nskb, *next;
2605	struct net *net = sock_net(sk);
2606	int probe_size;
2607	int size_needed;
2608	int copy, len;
2609	int mss_now;
2610	int interval;
2611
2612	/* Not currently probing/verifying,
2613	 * not in recovery,
2614	 * have enough cwnd, and
2615	 * not SACKing (the variable headers throw things off)
2616	 */
2617	if (likely(!icsk->icsk_mtup.enabled ||
2618		   icsk->icsk_mtup.probe_size ||
2619		   inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
2620		   tcp_snd_cwnd(tp) < 11 ||
2621		   tp->rx_opt.num_sacks || tp->rx_opt.dsack))
2622		return -1;
2623
2624	/* Use binary search for probe_size between tcp_mss_base,
2625	 * and current mss_clamp. if (search_high - search_low)
2626	 * smaller than a threshold, backoff from probing.
2627	 */
2628	mss_now = tcp_current_mss(sk);
2629	probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
2630				    icsk->icsk_mtup.search_low) >> 1);
2631	size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
2632	interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
2633	/* When misfortune happens, we are reprobing actively,
2634	 * and then reprobe timer has expired. We stick with current
2635	 * probing process by not resetting search range to its orignal.
2636	 */
2637	if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
2638	    interval < READ_ONCE(net->ipv4.sysctl_tcp_probe_threshold)) {
2639		/* Check whether enough time has elaplased for
2640		 * another round of probing.
2641		 */
2642		tcp_mtu_check_reprobe(sk);
2643		return -1;
2644	}
2645
2646	/* Have enough data in the send queue to probe? */
2647	if (tp->write_seq - tp->snd_nxt < size_needed)
2648		return -1;
2649
2650	if (tp->snd_wnd < size_needed)
2651		return -1;
2652	if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
2653		return 0;
2654
2655	/* Do we need to wait to drain cwnd? With none in flight, don't stall */
2656	if (tcp_packets_in_flight(tp) + 2 > tcp_snd_cwnd(tp)) {
2657		if (!tcp_packets_in_flight(tp))
2658			return -1;
2659		else
2660			return 0;
2661	}
2662
2663	if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
2664		return -1;
2665
2666	/* We're allowed to probe.  Build it now. */
2667	nskb = tcp_stream_alloc_skb(sk, GFP_ATOMIC, false);
2668	if (!nskb)
2669		return -1;
2670
2671	/* build the payload, and be prepared to abort if this fails. */
2672	if (tcp_clone_payload(sk, nskb, probe_size)) {
2673		tcp_skb_tsorted_anchor_cleanup(nskb);
2674		consume_skb(nskb);
2675		return -1;
2676	}
2677	sk_wmem_queued_add(sk, nskb->truesize);
2678	sk_mem_charge(sk, nskb->truesize);
2679
2680	skb = tcp_send_head(sk);
2681	skb_copy_decrypted(nskb, skb);
2682	mptcp_skb_ext_copy(nskb, skb);
2683
2684	TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
2685	TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
2686	TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
2687
2688	tcp_insert_write_queue_before(nskb, skb, sk);
2689	tcp_highest_sack_replace(sk, skb, nskb);
2690
2691	len = 0;
2692	tcp_for_write_queue_from_safe(skb, next, sk) {
2693		copy = min_t(int, skb->len, probe_size - len);
2694
2695		if (skb->len <= copy) {
2696			tcp_eat_one_skb(sk, nskb, skb);
2697		} else {
2698			TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
2699						   ~(TCPHDR_FIN|TCPHDR_PSH);
2700			__pskb_trim_head(skb, copy);
2701			tcp_set_skb_tso_segs(skb, mss_now);
2702			TCP_SKB_CB(skb)->seq += copy;
2703		}
2704
2705		len += copy;
2706
2707		if (len >= probe_size)
2708			break;
2709	}
2710	tcp_init_tso_segs(nskb, nskb->len);
2711
2712	/* We're ready to send.  If this fails, the probe will
2713	 * be resegmented into mss-sized pieces by tcp_write_xmit().
2714	 */
2715	if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
2716		/* Decrement cwnd here because we are sending
2717		 * effectively two packets. */
2718		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1);
2719		tcp_event_new_data_sent(sk, nskb);
2720
2721		icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
2722		tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
2723		tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
2724
2725		return 1;
2726	}
2727
2728	return -1;
2729}
2730
2731static bool tcp_pacing_check(struct sock *sk)
2732{
2733	struct tcp_sock *tp = tcp_sk(sk);
2734
2735	if (!tcp_needs_internal_pacing(sk))
2736		return false;
2737
2738	if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache)
2739		return false;
2740
2741	if (!hrtimer_is_queued(&tp->pacing_timer)) {
2742		hrtimer_start(&tp->pacing_timer,
2743			      ns_to_ktime(tp->tcp_wstamp_ns),
2744			      HRTIMER_MODE_ABS_PINNED_SOFT);
2745		sock_hold(sk);
2746	}
2747	return true;
2748}
2749
2750static bool tcp_rtx_queue_empty_or_single_skb(const struct sock *sk)
2751{
2752	const struct rb_node *node = sk->tcp_rtx_queue.rb_node;
2753
2754	/* No skb in the rtx queue. */
2755	if (!node)
2756		return true;
2757
2758	/* Only one skb in rtx queue. */
2759	return !node->rb_left && !node->rb_right;
2760}
2761
2762/* TCP Small Queues :
2763 * Control number of packets in qdisc/devices to two packets / or ~1 ms.
2764 * (These limits are doubled for retransmits)
2765 * This allows for :
2766 *  - better RTT estimation and ACK scheduling
2767 *  - faster recovery
2768 *  - high rates
2769 * Alas, some drivers / subsystems require a fair amount
2770 * of queued bytes to ensure line rate.
2771 * One example is wifi aggregation (802.11 AMPDU)
2772 */
2773static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2774				  unsigned int factor)
2775{
2776	unsigned long limit;
2777
2778	limit = max_t(unsigned long,
2779		      2 * skb->truesize,
2780		      READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift));
2781	limit = min_t(unsigned long, limit,
2782		      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes));
2783	limit <<= factor;
2784
2785	if (static_branch_unlikely(&tcp_tx_delay_enabled) &&
2786	    tcp_sk(sk)->tcp_tx_delay) {
2787		u64 extra_bytes = (u64)READ_ONCE(sk->sk_pacing_rate) *
2788				  tcp_sk(sk)->tcp_tx_delay;
2789
2790		/* TSQ is based on skb truesize sum (sk_wmem_alloc), so we
2791		 * approximate our needs assuming an ~100% skb->truesize overhead.
2792		 * USEC_PER_SEC is approximated by 2^20.
2793		 * do_div(extra_bytes, USEC_PER_SEC/2) is replaced by a right shift.
2794		 */
2795		extra_bytes >>= (20 - 1);
2796		limit += extra_bytes;
2797	}
2798	if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2799		/* Always send skb if rtx queue is empty or has one skb.
2800		 * No need to wait for TX completion to call us back,
2801		 * after softirq schedule.
2802		 * This helps when TX completions are delayed too much.
2803		 */
2804		if (tcp_rtx_queue_empty_or_single_skb(sk))
2805			return false;
2806
2807		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2808		/* It is possible TX completion already happened
2809		 * before we set TSQ_THROTTLED, so we must
2810		 * test again the condition.
2811		 */
2812		smp_mb__after_atomic();
2813		if (refcount_read(&sk->sk_wmem_alloc) > limit)
2814			return true;
2815	}
2816	return false;
2817}
2818
2819static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2820{
2821	const u32 now = tcp_jiffies32;
2822	enum tcp_chrono old = tp->chrono_type;
2823
2824	if (old > TCP_CHRONO_UNSPEC)
2825		tp->chrono_stat[old - 1] += now - tp->chrono_start;
2826	tp->chrono_start = now;
2827	tp->chrono_type = new;
2828}
2829
2830void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2831{
2832	struct tcp_sock *tp = tcp_sk(sk);
2833
2834	/* If there are multiple conditions worthy of tracking in a
2835	 * chronograph then the highest priority enum takes precedence
2836	 * over the other conditions. So that if something "more interesting"
2837	 * starts happening, stop the previous chrono and start a new one.
2838	 */
2839	if (type > tp->chrono_type)
2840		tcp_chrono_set(tp, type);
2841}
2842
2843void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
2844{
2845	struct tcp_sock *tp = tcp_sk(sk);
2846
2847
2848	/* There are multiple conditions worthy of tracking in a
2849	 * chronograph, so that the highest priority enum takes
2850	 * precedence over the other conditions (see tcp_chrono_start).
2851	 * If a condition stops, we only stop chrono tracking if
2852	 * it's the "most interesting" or current chrono we are
2853	 * tracking and starts busy chrono if we have pending data.
2854	 */
2855	if (tcp_rtx_and_write_queues_empty(sk))
2856		tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
2857	else if (type == tp->chrono_type)
2858		tcp_chrono_set(tp, TCP_CHRONO_BUSY);
2859}
2860
2861/* First skb in the write queue is smaller than ideal packet size.
2862 * Check if we can move payload from the second skb in the queue.
2863 */
2864static void tcp_grow_skb(struct sock *sk, struct sk_buff *skb, int amount)
2865{
2866	struct sk_buff *next_skb = skb->next;
2867	unsigned int nlen;
2868
2869	if (tcp_skb_is_last(sk, skb))
2870		return;
2871
2872	if (!tcp_skb_can_collapse(skb, next_skb))
2873		return;
2874
2875	nlen = min_t(u32, amount, next_skb->len);
2876	if (!nlen || !skb_shift(skb, next_skb, nlen))
2877		return;
2878
2879	TCP_SKB_CB(skb)->end_seq += nlen;
2880	TCP_SKB_CB(next_skb)->seq += nlen;
2881
2882	if (!next_skb->len) {
2883		/* In case FIN is set, we need to update end_seq */
2884		TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
2885
2886		tcp_eat_one_skb(sk, skb, next_skb);
2887	}
2888}
2889
2890/* This routine writes packets to the network.  It advances the
2891 * send_head.  This happens as incoming acks open up the remote
2892 * window for us.
2893 *
2894 * LARGESEND note: !tcp_urg_mode is overkill, only frames between
2895 * snd_up-64k-mss .. snd_up cannot be large. However, taking into
2896 * account rare use of URG, this is not a big flaw.
2897 *
2898 * Send at most one packet when push_one > 0. Temporarily ignore
2899 * cwnd limit to force at most one packet out when push_one == 2.
2900
2901 * Returns true, if no segments are in flight and we have queued segments,
2902 * but cannot send anything now because of SWS or another problem.
2903 */
2904static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
2905			   int push_one, gfp_t gfp)
2906{
2907	struct tcp_sock *tp = tcp_sk(sk);
2908	struct sk_buff *skb;
2909	unsigned int tso_segs, sent_pkts;
2910	u32 cwnd_quota, max_segs;
2911	int result;
2912	bool is_cwnd_limited = false, is_rwnd_limited = false;
2913
2914	sent_pkts = 0;
2915
2916	tcp_mstamp_refresh(tp);
2917
2918	/* AccECN option beacon depends on mstamp, it may change mss */
2919	if (tcp_ecn_mode_accecn(tp) && tcp_accecn_option_beacon_check(sk))
2920		mss_now = tcp_current_mss(sk);
2921
2922	if (!push_one) {
2923		/* Do MTU probing. */
2924		result = tcp_mtu_probe(sk);
2925		if (!result) {
2926			return false;
2927		} else if (result > 0) {
2928			sent_pkts = 1;
2929		}
2930	}
2931
2932	max_segs = tcp_tso_segs(sk, mss_now);
2933	while ((skb = tcp_send_head(sk))) {
2934		unsigned int limit;
2935		int missing_bytes;
2936
2937		if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
2938			/* "skb_mstamp_ns" is used as a start point for the retransmit timer */
2939			tp->tcp_wstamp_ns = tp->tcp_clock_cache;
2940			skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC);
2941			list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
2942			tcp_init_tso_segs(skb, mss_now);
2943			goto repair; /* Skip network transmission */
2944		}
2945
2946		if (tcp_pacing_check(sk))
2947			break;
2948
2949		cwnd_quota = tcp_cwnd_test(tp);
2950		if (!cwnd_quota) {
2951			if (push_one == 2)
2952				/* Force out a loss probe pkt. */
2953				cwnd_quota = 1;
2954			else
2955				break;
2956		}
2957		cwnd_quota = min(cwnd_quota, max_segs);
2958		missing_bytes = cwnd_quota * mss_now - skb->len;
2959		if (missing_bytes > 0)
2960			tcp_grow_skb(sk, skb, missing_bytes);
2961
2962		tso_segs = tcp_set_skb_tso_segs(skb, mss_now);
2963
2964		if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
2965			is_rwnd_limited = true;
2966			break;
2967		}
2968
2969		if (tso_segs == 1) {
2970			if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
2971						     (tcp_skb_is_last(sk, skb) ?
2972						      nonagle : TCP_NAGLE_PUSH))))
2973				break;
2974		} else {
2975			if (!push_one &&
2976			    tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
2977						 &is_rwnd_limited, max_segs))
2978				break;
2979		}
2980
2981		limit = mss_now;
2982		if (tso_segs > 1 && !tcp_urg_mode(tp))
2983			limit = tcp_mss_split_point(sk, skb, mss_now,
2984						    cwnd_quota,
2985						    nonagle);
2986
2987		if (skb->len > limit &&
2988		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
2989			break;
2990
2991		if (tcp_small_queue_check(sk, skb, 0))
2992			break;
2993
2994		/* Argh, we hit an empty skb(), presumably a thread
2995		 * is sleeping in sendmsg()/sk_stream_wait_memory().
2996		 * We do not want to send a pure-ack packet and have
2997		 * a strange looking rtx queue with empty packet(s).
2998		 */
2999		if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq)
3000			break;
3001
3002		if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
3003			break;
3004
3005repair:
3006		/* Advance the send_head.  This one is sent out.
3007		 * This call will increment packets_out.
3008		 */
3009		tcp_event_new_data_sent(sk, skb);
3010
3011		tcp_minshall_update(tp, mss_now, skb);
3012		sent_pkts += tcp_skb_pcount(skb);
3013
3014		if (push_one)
3015			break;
3016	}
3017
3018	if (is_rwnd_limited)
3019		tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
3020	else
3021		tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
3022
3023	is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp));
3024	if (likely(sent_pkts || is_cwnd_limited))
3025		tcp_cwnd_validate(sk, is_cwnd_limited);
3026
3027	if (likely(sent_pkts)) {
3028		if (tcp_in_cwnd_reduction(sk))
3029			tp->prr_out += sent_pkts;
3030
3031		/* Send one loss probe per tail loss episode. */
3032		if (push_one != 2)
3033			tcp_schedule_loss_probe(sk, false);
3034		return false;
3035	}
3036	return !tp->packets_out && !tcp_write_queue_empty(sk);
3037}
3038
3039bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
3040{
3041	struct inet_connection_sock *icsk = inet_csk(sk);
3042	struct tcp_sock *tp = tcp_sk(sk);
3043	u32 timeout, timeout_us, rto_delta_us;
3044	int early_retrans;
3045
3046	/* Don't do any loss probe on a Fast Open connection before 3WHS
3047	 * finishes.
3048	 */
3049	if (rcu_access_pointer(tp->fastopen_rsk))
3050		return false;
3051
3052	early_retrans = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_early_retrans);
3053	/* Schedule a loss probe in 2*RTT for SACK capable connections
3054	 * not in loss recovery, that are either limited by cwnd or application.
3055	 */
3056	if ((early_retrans != 3 && early_retrans != 4) ||
3057	    !tp->packets_out || !tcp_is_sack(tp) ||
3058	    (icsk->icsk_ca_state != TCP_CA_Open &&
3059	     icsk->icsk_ca_state != TCP_CA_CWR))
3060		return false;
3061
3062	/* Probe timeout is 2*rtt. Add minimum RTO to account
3063	 * for delayed ack when there's one outstanding packet. If no RTT
3064	 * sample is available then probe after TCP_TIMEOUT_INIT.
3065	 */
3066	if (tp->srtt_us) {
3067		timeout_us = tp->srtt_us >> 2;
3068		if (tp->packets_out == 1)
3069			timeout_us += tcp_rto_min_us(sk);
3070		else
3071			timeout_us += TCP_TIMEOUT_MIN_US;
3072		timeout = usecs_to_jiffies(timeout_us);
3073	} else {
3074		timeout = TCP_TIMEOUT_INIT;
3075	}
3076
3077	/* If the RTO formula yields an earlier time, then use that time. */
3078	rto_delta_us = advancing_rto ?
3079			jiffies_to_usecs(inet_csk(sk)->icsk_rto) :
3080			tcp_rto_delta_us(sk);  /* How far in future is RTO? */
3081	if (rto_delta_us > 0)
3082		timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
3083
3084	tcp_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, true);
3085	return true;
3086}
3087
3088/* Thanks to skb fast clones, we can detect if a prior transmit of
3089 * a packet is still in a qdisc or driver queue.
3090 * In this case, there is very little point doing a retransmit !
3091 */
3092static bool skb_still_in_host_queue(struct sock *sk,
3093				    const struct sk_buff *skb)
3094{
3095	if (unlikely(skb_fclone_busy(sk, skb))) {
3096		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
3097		smp_mb__after_atomic();
3098		if (skb_fclone_busy(sk, skb)) {
3099			NET_INC_STATS(sock_net(sk),
3100				      LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
3101			return true;
3102		}
3103	}
3104	return false;
3105}
3106
3107/* When probe timeout (PTO) fires, try send a new segment if possible, else
3108 * retransmit the last segment.
3109 */
3110void tcp_send_loss_probe(struct sock *sk)
3111{
3112	struct tcp_sock *tp = tcp_sk(sk);
3113	struct sk_buff *skb;
3114	int pcount;
3115	int mss = tcp_current_mss(sk);
3116
3117	/* At most one outstanding TLP */
3118	if (tp->tlp_high_seq)
3119		goto rearm_timer;
3120
3121	tp->tlp_retrans = 0;
3122	skb = tcp_send_head(sk);
3123	if (skb && tcp_snd_wnd_test(tp, skb, mss)) {
3124		pcount = tp->packets_out;
3125		tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
3126		if (tp->packets_out > pcount)
3127			goto probe_sent;
3128		goto rearm_timer;
3129	}
3130	skb = skb_rb_last(&sk->tcp_rtx_queue);
3131	if (unlikely(!skb)) {
3132		tcp_warn_once(sk, tp->packets_out, "invalid inflight: ");
3133		smp_store_release(&inet_csk(sk)->icsk_pending, 0);
3134		return;
3135	}
3136
3137	if (skb_still_in_host_queue(sk, skb))
3138		goto rearm_timer;
3139
3140	pcount = tcp_skb_pcount(skb);
3141	if (WARN_ON(!pcount))
3142		goto rearm_timer;
3143
3144	if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
3145		if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
3146					  (pcount - 1) * mss, mss,
3147					  GFP_ATOMIC)))
3148			goto rearm_timer;
3149		skb = skb_rb_next(skb);
3150	}
3151
3152	if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
3153		goto rearm_timer;
3154
3155	if (__tcp_retransmit_skb(sk, skb, 1))
3156		goto rearm_timer;
3157
3158	tp->tlp_retrans = 1;
3159
3160probe_sent:
3161	/* Record snd_nxt for loss detection. */
3162	tp->tlp_high_seq = tp->snd_nxt;
3163
3164	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
3165	/* Reset s.t. tcp_rearm_rto will restart timer from now */
3166	smp_store_release(&inet_csk(sk)->icsk_pending, 0);
3167rearm_timer:
3168	tcp_rearm_rto(sk);
3169}
3170
3171/* Push out any pending frames which were held back due to
3172 * TCP_CORK or attempt at coalescing tiny packets.
3173 * The socket must be locked by the caller.
3174 */
3175void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
3176			       int nonagle)
3177{
3178	/* If we are closed, the bytes will have to remain here.
3179	 * In time closedown will finish, we empty the write queue and
3180	 * all will be happy.
3181	 */
3182	if (unlikely(sk->sk_state == TCP_CLOSE))
3183		return;
3184
3185	if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
3186			   sk_gfp_mask(sk, GFP_ATOMIC)))
3187		tcp_check_probe_timer(sk);
3188}
3189
3190/* Send _single_ skb sitting at the send head. This function requires
3191 * true push pending frames to setup probe timer etc.
3192 */
3193void tcp_push_one(struct sock *sk, unsigned int mss_now)
3194{
3195	struct sk_buff *skb = tcp_send_head(sk);
3196
3197	BUG_ON(!skb || skb->len < mss_now);
3198
3199	tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
3200}
3201
3202/* This function returns the amount that we can raise the
3203 * usable window based on the following constraints
3204 *
3205 * 1. The window can never be shrunk once it is offered (RFC 793)
3206 * 2. We limit memory per socket
3207 *
3208 * RFC 1122:
3209 * "the suggested [SWS] avoidance algorithm for the receiver is to keep
3210 *  RECV.NEXT + RCV.WIN fixed until:
3211 *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
3212 *
3213 * i.e. don't raise the right edge of the window until you can raise
3214 * it at least MSS bytes.
3215 *
3216 * Unfortunately, the recommended algorithm breaks header prediction,
3217 * since header prediction assumes th->window stays fixed.
3218 *
3219 * Strictly speaking, keeping th->window fixed violates the receiver
3220 * side SWS prevention criteria. The problem is that under this rule
3221 * a stream of single byte packets will cause the right side of the
3222 * window to always advance by a single byte.
3223 *
3224 * Of course, if the sender implements sender side SWS prevention
3225 * then this will not be a problem.
3226 *
3227 * BSD seems to make the following compromise:
3228 *
3229 *	If the free space is less than the 1/4 of the maximum
3230 *	space available and the free space is less than 1/2 mss,
3231 *	then set the window to 0.
3232 *	[ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
3233 *	Otherwise, just prevent the window from shrinking
3234 *	and from being larger than the largest representable value.
3235 *
3236 * This prevents incremental opening of the window in the regime
3237 * where TCP is limited by the speed of the reader side taking
3238 * data out of the TCP receive queue. It does nothing about
3239 * those cases where the window is constrained on the sender side
3240 * because the pipeline is full.
3241 *
3242 * BSD also seems to "accidentally" limit itself to windows that are a
3243 * multiple of MSS, at least until the free space gets quite small.
3244 * This would appear to be a side effect of the mbuf implementation.
3245 * Combining these two algorithms results in the observed behavior
3246 * of having a fixed window size at almost all times.
3247 *
3248 * Below we obtain similar behavior by forcing the offered window to
3249 * a multiple of the mss when it is feasible to do so.
3250 *
3251 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
3252 * Regular options like TIMESTAMP are taken into account.
3253 */
3254u32 __tcp_select_window(struct sock *sk)
3255{
3256	struct inet_connection_sock *icsk = inet_csk(sk);
3257	struct tcp_sock *tp = tcp_sk(sk);
3258	struct net *net = sock_net(sk);
3259	/* MSS for the peer's data.  Previous versions used mss_clamp
3260	 * here.  I don't know if the value based on our guesses
3261	 * of peer's MSS is better for the performance.  It's more correct
3262	 * but may be worse for the performance because of rcv_mss
3263	 * fluctuations.  --SAW  1998/11/1
3264	 */
3265	int mss = icsk->icsk_ack.rcv_mss;
3266	int free_space = tcp_space(sk);
3267	int allowed_space = tcp_full_space(sk);
3268	int full_space, window;
3269
3270	if (sk_is_mptcp(sk))
3271		mptcp_space(sk, &free_space, &allowed_space);
3272
3273	full_space = min_t(int, tp->window_clamp, allowed_space);
3274
3275	if (unlikely(mss > full_space)) {
3276		mss = full_space;
3277		if (mss <= 0)
3278			return 0;
3279	}
3280
3281	/* Only allow window shrink if the sysctl is enabled and we have
3282	 * a non-zero scaling factor in effect.
3283	 */
3284	if (READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) && tp->rx_opt.rcv_wscale)
3285		goto shrink_window_allowed;
3286
3287	/* do not allow window to shrink */
3288
3289	if (free_space < (full_space >> 1)) {
3290		icsk->icsk_ack.quick = 0;
3291
3292		if (tcp_under_memory_pressure(sk))
3293			tcp_adjust_rcv_ssthresh(sk);
3294
3295		/* free_space might become our new window, make sure we don't
3296		 * increase it due to wscale.
3297		 */
3298		free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
3299
3300		/* if free space is less than mss estimate, or is below 1/16th
3301		 * of the maximum allowed, try to move to zero-window, else
3302		 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
3303		 * new incoming data is dropped due to memory limits.
3304		 * With large window, mss test triggers way too late in order
3305		 * to announce zero window in time before rmem limit kicks in.
3306		 */
3307		if (free_space < (allowed_space >> 4) || free_space < mss)
3308			return 0;
3309	}
3310
3311	if (free_space > tp->rcv_ssthresh)
3312		free_space = tp->rcv_ssthresh;
3313
3314	/* Don't do rounding if we are using window scaling, since the
3315	 * scaled window will not line up with the MSS boundary anyway.
3316	 */
3317	if (tp->rx_opt.rcv_wscale) {
3318		window = free_space;
3319
3320		/* Advertise enough space so that it won't get scaled away.
3321		 * Import case: prevent zero window announcement if
3322		 * 1<<rcv_wscale > mss.
3323		 */
3324		window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
3325	} else {
3326		window = tp->rcv_wnd;
3327		/* Get the largest window that is a nice multiple of mss.
3328		 * Window clamp already applied above.
3329		 * If our current window offering is within 1 mss of the
3330		 * free space we just keep it. This prevents the divide
3331		 * and multiply from happening most of the time.
3332		 * We also don't do any window rounding when the free space
3333		 * is too small.
3334		 */
3335		if (window <= free_space - mss || window > free_space)
3336			window = rounddown(free_space, mss);
3337		else if (mss == full_space &&
3338			 free_space > window + (full_space >> 1))
3339			window = free_space;
3340	}
3341
3342	return window;
3343
3344shrink_window_allowed:
3345	/* new window should always be an exact multiple of scaling factor */
3346	free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
3347
3348	if (free_space < (full_space >> 1)) {
3349		icsk->icsk_ack.quick = 0;
3350
3351		if (tcp_under_memory_pressure(sk))
3352			tcp_adjust_rcv_ssthresh(sk);
3353
3354		/* if free space is too low, return a zero window */
3355		if (free_space < (allowed_space >> 4) || free_space < mss ||
3356			free_space < (1 << tp->rx_opt.rcv_wscale))
3357			return 0;
3358	}
3359
3360	if (free_space > tp->rcv_ssthresh) {
3361		free_space = tp->rcv_ssthresh;
3362		/* new window should always be an exact multiple of scaling factor
3363		 *
3364		 * For this case, we ALIGN "up" (increase free_space) because
3365		 * we know free_space is not zero here, it has been reduced from
3366		 * the memory-based limit, and rcv_ssthresh is not a hard limit
3367		 * (unlike sk_rcvbuf).
3368		 */
3369		free_space = ALIGN(free_space, (1 << tp->rx_opt.rcv_wscale));
3370	}
3371
3372	return free_space;
3373}
3374
3375void tcp_skb_collapse_tstamp(struct sk_buff *skb,
3376			     const struct sk_buff *next_skb)
3377{
3378	if (unlikely(tcp_has_tx_tstamp(next_skb))) {
3379		const struct skb_shared_info *next_shinfo =
3380			skb_shinfo(next_skb);
3381		struct skb_shared_info *shinfo = skb_shinfo(skb);
3382
3383		shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
3384		shinfo->tskey = next_shinfo->tskey;
3385		TCP_SKB_CB(skb)->txstamp_ack |=
3386			TCP_SKB_CB(next_skb)->txstamp_ack;
3387	}
3388}
3389
3390/* Collapses two adjacent SKB's during retransmission. */
3391static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
3392{
3393	struct tcp_sock *tp = tcp_sk(sk);
3394	struct sk_buff *next_skb = skb_rb_next(skb);
3395	int next_skb_size;
3396
3397	next_skb_size = next_skb->len;
3398
3399	BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
3400
3401	if (next_skb_size && !tcp_skb_shift(skb, next_skb, 1, next_skb_size))
3402		return false;
3403
3404	tcp_highest_sack_replace(sk, next_skb, skb);
3405
3406	/* Update sequence range on original skb. */
3407	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
3408
3409	/* Merge over control information. This moves PSH/FIN etc. over */
3410	TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
3411
3412	/* All done, get rid of second SKB and account for it so
3413	 * packet counting does not break.
3414	 */
3415	TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
3416	TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
3417
3418	/* changed transmit queue under us so clear hints */
3419	if (next_skb == tp->retransmit_skb_hint)
3420		tp->retransmit_skb_hint = skb;
3421
3422	tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
3423
3424	tcp_skb_collapse_tstamp(skb, next_skb);
3425
3426	tcp_rtx_queue_unlink_and_free(next_skb, sk);
3427	return true;
3428}
3429
3430/* Check if coalescing SKBs is legal. */
3431static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
3432{
3433	if (tcp_skb_pcount(skb) > 1)
3434		return false;
3435	if (skb_cloned(skb))
3436		return false;
3437	if (!skb_frags_readable(skb))
3438		return false;
3439	/* Some heuristics for collapsing over SACK'd could be invented */
3440	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3441		return false;
3442
3443	return true;
3444}
3445
3446/* Collapse packets in the retransmit queue to make to create
3447 * less packets on the wire. This is only done on retransmission.
3448 */
3449static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
3450				     int space)
3451{
3452	struct tcp_sock *tp = tcp_sk(sk);
3453	struct sk_buff *skb = to, *tmp;
3454	bool first = true;
3455
3456	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse))
3457		return;
3458	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3459		return;
3460
3461	skb_rbtree_walk_from_safe(skb, tmp) {
3462		if (!tcp_can_collapse(sk, skb))
3463			break;
3464
3465		if (!tcp_skb_can_collapse(to, skb))
3466			break;
3467
3468		space -= skb->len;
3469
3470		if (first) {
3471			first = false;
3472			continue;
3473		}
3474
3475		if (space < 0)
3476			break;
3477
3478		if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
3479			break;
3480
3481		if (!tcp_collapse_retrans(sk, to))
3482			break;
3483	}
3484}
3485
3486/* This retransmits one SKB.  Policy decisions and retransmit queue
3487 * state updates are done by the caller.  Returns non-zero if an
3488 * error occurred which prevented the send.
3489 */
3490int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3491{
3492	struct inet_connection_sock *icsk = inet_csk(sk);
3493	struct tcp_sock *tp = tcp_sk(sk);
3494	unsigned int cur_mss;
3495	int diff, len, err;
3496	int avail_wnd;
3497
3498	/* Inconclusive MTU probe */
3499	if (icsk->icsk_mtup.probe_size)
3500		icsk->icsk_mtup.probe_size = 0;
3501
3502	if (skb_still_in_host_queue(sk, skb)) {
3503		err = -EBUSY;
3504		goto out;
3505	}
3506
3507start:
3508	if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
3509		if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3510			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN;
3511			TCP_SKB_CB(skb)->seq++;
3512			goto start;
3513		}
3514		if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) {
3515			WARN_ON_ONCE(1);
3516			err = -EINVAL;
3517			goto out;
3518		}
3519		if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) {
3520			err = -ENOMEM;
3521			goto out;
3522		}
3523	}
3524
3525	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) {
3526		err = -EHOSTUNREACH; /* Routing failure or similar. */
3527		goto out;
3528	}
3529
3530	cur_mss = tcp_current_mss(sk);
3531	avail_wnd = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3532
3533	/* If receiver has shrunk his window, and skb is out of
3534	 * new window, do not retransmit it. The exception is the
3535	 * case, when window is shrunk to zero. In this case
3536	 * our retransmit of one segment serves as a zero window probe.
3537	 */
3538	if (avail_wnd <= 0) {
3539		if (TCP_SKB_CB(skb)->seq != tp->snd_una) {
3540			err = -EAGAIN;
3541			goto out;
3542		}
3543		avail_wnd = cur_mss;
3544	}
3545
3546	len = cur_mss * segs;
3547	if (len > avail_wnd) {
3548		len = rounddown(avail_wnd, cur_mss);
3549		if (!len)
3550			len = avail_wnd;
3551	}
3552	if (skb->len > len) {
3553		if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len,
3554				 cur_mss, GFP_ATOMIC)) {
3555			err = -ENOMEM;  /* We'll try again later. */
3556			goto out;
3557		}
3558	} else {
3559		if (skb_unclone_keeptruesize(skb, GFP_ATOMIC)) {
3560			err = -ENOMEM;
3561			goto out;
3562		}
3563
3564		diff = tcp_skb_pcount(skb);
3565		tcp_set_skb_tso_segs(skb, cur_mss);
3566		diff -= tcp_skb_pcount(skb);
3567		if (diff)
3568			tcp_adjust_pcount(sk, skb, diff);
3569		avail_wnd = min_t(int, avail_wnd, cur_mss);
3570		if (skb->len < avail_wnd)
3571			tcp_retrans_try_collapse(sk, skb, avail_wnd);
3572	}
3573
3574	/* RFC3168, section 6.1.1.1. ECN fallback
3575	 * As AccECN uses the same SYN flags (+ AE), this check covers both
3576	 * cases.
3577	 */
3578	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
3579		tcp_ecn_clear_syn(sk, skb);
3580
3581	/* Update global and local TCP statistics. */
3582	segs = tcp_skb_pcount(skb);
3583	TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
3584	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3585		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3586	tp->total_retrans += segs;
3587	tp->bytes_retrans += skb->len;
3588
3589	/* make sure skb->data is aligned on arches that require it
3590	 * and check if ack-trimming & collapsing extended the headroom
3591	 * beyond what csum_start can cover.
3592	 */
3593	if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
3594		     skb_headroom(skb) >= 0xFFFF)) {
3595		struct sk_buff *nskb;
3596
3597		tcp_skb_tsorted_save(skb) {
3598			nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
3599			if (nskb) {
3600				nskb->dev = NULL;
3601				err = tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC);
3602			} else {
3603				err = -ENOBUFS;
3604			}
3605		} tcp_skb_tsorted_restore(skb);
3606
3607		if (!err) {
3608			tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns);
3609			tcp_rate_skb_sent(sk, skb);
3610		}
3611	} else {
3612		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3613	}
3614
3615	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
3616		tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
3617				  TCP_SKB_CB(skb)->seq, segs, err);
3618
3619	if (unlikely(err) && err != -EBUSY)
3620		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs);
3621
3622	/* To avoid taking spuriously low RTT samples based on a timestamp
3623	 * for a transmit that never happened, always mark EVER_RETRANS
3624	 */
3625	TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
3626
3627out:
3628	trace_tcp_retransmit_skb(sk, skb, err);
3629	return err;
3630}
3631
3632int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3633{
3634	struct tcp_sock *tp = tcp_sk(sk);
3635	int err = __tcp_retransmit_skb(sk, skb, segs);
3636
3637	if (err == 0) {
3638#if FASTRETRANS_DEBUG > 0
3639		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
3640			net_dbg_ratelimited("retrans_out leaked\n");
3641		}
3642#endif
3643		TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
3644		tp->retrans_out += tcp_skb_pcount(skb);
3645	}
3646
3647	/* Save stamp of the first (attempted) retransmit. */
3648	if (!tp->retrans_stamp)
3649		tp->retrans_stamp = tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb);
3650
3651	if (tp->undo_retrans < 0)
3652		tp->undo_retrans = 0;
3653	tp->undo_retrans += tcp_skb_pcount(skb);
3654	return err;
3655}
3656
3657/* This gets called after a retransmit timeout, and the initially
3658 * retransmitted data is acknowledged.  It tries to continue
3659 * resending the rest of the retransmit queue, until either
3660 * we've sent it all or the congestion window limit is reached.
3661 */
3662void tcp_xmit_retransmit_queue(struct sock *sk)
3663{
3664	const struct inet_connection_sock *icsk = inet_csk(sk);
3665	struct sk_buff *skb, *rtx_head, *hole = NULL;
3666	struct tcp_sock *tp = tcp_sk(sk);
3667	bool rearm_timer = false;
3668	u32 max_segs;
3669	int mib_idx;
3670
3671	if (!tp->packets_out)
3672		return;
3673
3674	rtx_head = tcp_rtx_queue_head(sk);
3675	skb = tp->retransmit_skb_hint ?: rtx_head;
3676	max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
3677	skb_rbtree_walk_from(skb) {
3678		__u8 sacked;
3679		int segs;
3680
3681		if (tcp_pacing_check(sk))
3682			break;
3683
3684		/* we could do better than to assign each time */
3685		if (!hole)
3686			tp->retransmit_skb_hint = skb;
3687
3688		segs = tcp_snd_cwnd(tp) - tcp_packets_in_flight(tp);
3689		if (segs <= 0)
3690			break;
3691		sacked = TCP_SKB_CB(skb)->sacked;
3692		/* In case tcp_shift_skb_data() have aggregated large skbs,
3693		 * we need to make sure not sending too bigs TSO packets
3694		 */
3695		segs = min_t(int, segs, max_segs);
3696
3697		if (tp->retrans_out >= tp->lost_out) {
3698			break;
3699		} else if (!(sacked & TCPCB_LOST)) {
3700			if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
3701				hole = skb;
3702			continue;
3703
3704		} else {
3705			if (icsk->icsk_ca_state != TCP_CA_Loss)
3706				mib_idx = LINUX_MIB_TCPFASTRETRANS;
3707			else
3708				mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
3709		}
3710
3711		if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
3712			continue;
3713
3714		if (tcp_small_queue_check(sk, skb, 1))
3715			break;
3716
3717		if (tcp_retransmit_skb(sk, skb, segs))
3718			break;
3719
3720		NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
3721
3722		if (tcp_in_cwnd_reduction(sk))
3723			tp->prr_out += tcp_skb_pcount(skb);
3724
3725		if (skb == rtx_head &&
3726		    icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
3727			rearm_timer = true;
3728
3729	}
3730	if (rearm_timer)
3731		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3732				     inet_csk(sk)->icsk_rto, true);
3733}
3734
3735/* We allow to exceed memory limits for FIN packets to expedite
3736 * connection tear down and (memory) recovery.
3737 * Otherwise tcp_send_fin() could be tempted to either delay FIN
3738 * or even be forced to close flow without any FIN.
3739 * In general, we want to allow one skb per socket to avoid hangs
3740 * with edge trigger epoll()
3741 */
3742void sk_forced_mem_schedule(struct sock *sk, int size)
3743{
3744	int delta, amt;
3745
3746	delta = size - sk->sk_forward_alloc;
3747	if (delta <= 0)
3748		return;
3749
3750	amt = sk_mem_pages(delta);
3751	sk_forward_alloc_add(sk, amt << PAGE_SHIFT);
3752
3753	if (mem_cgroup_sk_enabled(sk))
3754		mem_cgroup_sk_charge(sk, amt, gfp_memcg_charge() | __GFP_NOFAIL);
3755
3756	if (sk->sk_bypass_prot_mem)
3757		return;
3758
3759	sk_memory_allocated_add(sk, amt);
3760}
3761
3762/* Send a FIN. The caller locks the socket for us.
3763 * We should try to send a FIN packet really hard, but eventually give up.
3764 */
3765void tcp_send_fin(struct sock *sk)
3766{
3767	struct sk_buff *skb, *tskb, *tail = tcp_write_queue_tail(sk);
3768	struct tcp_sock *tp = tcp_sk(sk);
3769
3770	/* Optimization, tack on the FIN if we have one skb in write queue and
3771	 * this skb was not yet sent, or we are under memory pressure.
3772	 * Note: in the latter case, FIN packet will be sent after a timeout,
3773	 * as TCP stack thinks it has already been transmitted.
3774	 */
3775	tskb = tail;
3776	if (!tskb && tcp_under_memory_pressure(sk))
3777		tskb = skb_rb_last(&sk->tcp_rtx_queue);
3778
3779	if (tskb) {
3780		TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
3781		TCP_SKB_CB(tskb)->end_seq++;
3782		tp->write_seq++;
3783		if (!tail) {
3784			/* This means tskb was already sent.
3785			 * Pretend we included the FIN on previous transmit.
3786			 * We need to set tp->snd_nxt to the value it would have
3787			 * if FIN had been sent. This is because retransmit path
3788			 * does not change tp->snd_nxt.
3789			 */
3790			WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);
3791			return;
3792		}
3793	} else {
3794		skb = alloc_skb_fclone(MAX_TCP_HEADER,
3795				       sk_gfp_mask(sk, GFP_ATOMIC |
3796						       __GFP_NOWARN));
3797		if (unlikely(!skb))
3798			return;
3799
3800		INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
3801		skb_reserve(skb, MAX_TCP_HEADER);
3802		sk_forced_mem_schedule(sk, skb->truesize);
3803		/* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
3804		tcp_init_nondata_skb(skb, sk, tp->write_seq,
3805				     TCPHDR_ACK | TCPHDR_FIN);
3806		tcp_queue_skb(sk, skb);
3807	}
3808	__tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
3809}
3810
3811/* We get here when a process closes a file descriptor (either due to
3812 * an explicit close() or as a byproduct of exit()'ing) and there
3813 * was unread data in the receive queue.  This behavior is recommended
3814 * by RFC 2525, section 2.17.  -DaveM
3815 */
3816void tcp_send_active_reset(struct sock *sk, gfp_t priority,
3817			   enum sk_rst_reason reason)
3818{
3819	struct sk_buff *skb;
3820
3821	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
3822
3823	/* NOTE: No TCP options attached and we never retransmit this. */
3824	skb = alloc_skb(MAX_TCP_HEADER, priority);
3825	if (!skb) {
3826		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3827		return;
3828	}
3829
3830	/* Reserve space for headers and prepare control bits. */
3831	skb_reserve(skb, MAX_TCP_HEADER);
3832	tcp_init_nondata_skb(skb, sk, tcp_acceptable_seq(sk),
3833			     TCPHDR_ACK | TCPHDR_RST);
3834	tcp_mstamp_refresh(tcp_sk(sk));
3835	/* Send it off. */
3836	if (tcp_transmit_skb(sk, skb, 0, priority))
3837		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3838
3839	/* skb of trace_tcp_send_reset() keeps the skb that caused RST,
3840	 * skb here is different to the troublesome skb, so use NULL
3841	 */
3842	trace_tcp_send_reset(sk, NULL, reason);
3843}
3844
3845/* Send a crossed SYN-ACK during socket establishment.
3846 * WARNING: This routine must only be called when we have already sent
3847 * a SYN packet that crossed the incoming SYN that caused this routine
3848 * to get called. If this assumption fails then the initial rcv_wnd
3849 * and rcv_wscale values will not be correct.
3850 */
3851int tcp_send_synack(struct sock *sk)
3852{
3853	struct sk_buff *skb;
3854
3855	skb = tcp_rtx_queue_head(sk);
3856	if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3857		pr_err("%s: wrong queue state\n", __func__);
3858		return -EFAULT;
3859	}
3860	if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
3861		if (skb_cloned(skb)) {
3862			struct sk_buff *nskb;
3863
3864			tcp_skb_tsorted_save(skb) {
3865				nskb = skb_copy(skb, GFP_ATOMIC);
3866			} tcp_skb_tsorted_restore(skb);
3867			if (!nskb)
3868				return -ENOMEM;
3869			INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
3870			tcp_highest_sack_replace(sk, skb, nskb);
3871			tcp_rtx_queue_unlink_and_free(skb, sk);
3872			__skb_header_release(nskb);
3873			tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
3874			sk_wmem_queued_add(sk, nskb->truesize);
3875			sk_mem_charge(sk, nskb->truesize);
3876			skb = nskb;
3877		}
3878
3879		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
3880		tcp_ecn_send_synack(sk, skb);
3881	}
3882	return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3883}
3884
3885/**
3886 * tcp_make_synack - Allocate one skb and build a SYNACK packet.
3887 * @sk: listener socket
3888 * @dst: dst entry attached to the SYNACK. It is consumed and caller
3889 *       should not use it again.
3890 * @req: request_sock pointer
3891 * @foc: cookie for tcp fast open
3892 * @synack_type: Type of synack to prepare
3893 * @syn_skb: SYN packet just received.  It could be NULL for rtx case.
3894 */
3895struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
3896				struct request_sock *req,
3897				struct tcp_fastopen_cookie *foc,
3898				enum tcp_synack_type synack_type,
3899				struct sk_buff *syn_skb)
3900{
3901	struct inet_request_sock *ireq = inet_rsk(req);
3902	const struct tcp_sock *tp = tcp_sk(sk);
3903	struct tcp_out_options opts;
3904	struct tcp_key key = {};
3905	struct sk_buff *skb;
3906	int tcp_header_size;
3907	struct tcphdr *th;
3908	int mss;
3909	u64 now;
3910
3911	skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
3912	if (unlikely(!skb)) {
3913		dst_release(dst);
3914		return NULL;
3915	}
3916	/* Reserve space for headers. */
3917	skb_reserve(skb, MAX_TCP_HEADER);
3918
3919	switch (synack_type) {
3920	case TCP_SYNACK_NORMAL:
3921		skb_set_owner_edemux(skb, req_to_sk(req));
3922		break;
3923	case TCP_SYNACK_COOKIE:
3924		/* Under synflood, we do not attach skb to a socket,
3925		 * to avoid false sharing.
3926		 */
3927		break;
3928	case TCP_SYNACK_FASTOPEN:
3929		/* sk is a const pointer, because we want to express multiple
3930		 * cpu might call us concurrently.
3931		 * sk->sk_wmem_alloc in an atomic, we can promote to rw.
3932		 */
3933		skb_set_owner_w(skb, (struct sock *)sk);
3934		break;
3935	}
3936	skb_dst_set(skb, dst);
3937
3938	mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3939
3940	memset(&opts, 0, sizeof(opts));
3941	now = tcp_clock_ns();
3942#ifdef CONFIG_SYN_COOKIES
3943	if (unlikely(synack_type == TCP_SYNACK_COOKIE && ireq->tstamp_ok))
3944		skb_set_delivery_time(skb, cookie_init_timestamp(req, now),
3945				      SKB_CLOCK_MONOTONIC);
3946	else
3947#endif
3948	{
3949		skb_set_delivery_time(skb, now, SKB_CLOCK_MONOTONIC);
3950		if (!tcp_rsk(req)->snt_synack) /* Timestamp first SYNACK */
3951			tcp_rsk(req)->snt_synack = tcp_skb_timestamp_us(skb);
3952	}
3953
3954#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
3955	rcu_read_lock();
3956#endif
3957	if (tcp_rsk_used_ao(req)) {
3958#ifdef CONFIG_TCP_AO
3959		struct tcp_ao_key *ao_key = NULL;
3960		u8 keyid = tcp_rsk(req)->ao_keyid;
3961		u8 rnext = tcp_rsk(req)->ao_rcv_next;
3962
3963		ao_key = tcp_sk(sk)->af_specific->ao_lookup(sk, req_to_sk(req),
3964							    keyid, -1);
3965		/* If there is no matching key - avoid sending anything,
3966		 * especially usigned segments. It could try harder and lookup
3967		 * for another peer-matching key, but the peer has requested
3968		 * ao_keyid (RFC5925 RNextKeyID), so let's keep it simple here.
3969		 */
3970		if (unlikely(!ao_key)) {
3971			trace_tcp_ao_synack_no_key(sk, keyid, rnext);
3972			rcu_read_unlock();
3973			kfree_skb(skb);
3974			net_warn_ratelimited("TCP-AO: the keyid %u from SYN packet is not present - not sending SYNACK\n",
3975					     keyid);
3976			return NULL;
3977		}
3978		key.ao_key = ao_key;
3979		key.type = TCP_KEY_AO;
3980#endif
3981	} else {
3982#ifdef CONFIG_TCP_MD5SIG
3983		key.md5_key = tcp_rsk(req)->af_specific->req_md5_lookup(sk,
3984					req_to_sk(req));
3985		if (key.md5_key)
3986			key.type = TCP_KEY_MD5;
3987#endif
3988	}
3989	skb_set_hash(skb, READ_ONCE(tcp_rsk(req)->txhash), PKT_HASH_TYPE_L4);
3990	/* bpf program will be interested in the tcp_flags */
3991	TCP_SKB_CB(skb)->tcp_flags = TCPHDR_SYN | TCPHDR_ACK;
3992	tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts,
3993					     &key, foc, synack_type, syn_skb)
3994					+ sizeof(*th);
3995
3996	skb_push(skb, tcp_header_size);
3997	skb_reset_transport_header(skb);
3998
3999	th = (struct tcphdr *)skb->data;
4000	memset(th, 0, sizeof(struct tcphdr));
4001	th->syn = 1;
4002	th->ack = 1;
4003	tcp_ecn_make_synack(req, th);
4004	th->source = htons(ireq->ir_num);
4005	th->dest = ireq->ir_rmt_port;
4006	skb->mark = ireq->ir_mark;
4007	skb->ip_summed = CHECKSUM_PARTIAL;
4008	th->seq = htonl(tcp_rsk(req)->snt_isn);
4009	/* XXX data is queued and acked as is. No buffer/window check */
4010	th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
4011
4012	/* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
4013	th->window = htons(min(req->rsk_rcv_wnd, 65535U));
4014	tcp_options_write(th, NULL, tcp_rsk(req), &opts, &key);
4015	th->doff = (tcp_header_size >> 2);
4016	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
4017
4018	/* Okay, we have all we need - do the md5 hash if needed */
4019	if (tcp_key_is_md5(&key)) {
4020#ifdef CONFIG_TCP_MD5SIG
4021		tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
4022					key.md5_key, req_to_sk(req), skb);
4023#endif
4024	} else if (tcp_key_is_ao(&key)) {
4025#ifdef CONFIG_TCP_AO
4026		tcp_rsk(req)->af_specific->ao_synack_hash(opts.hash_location,
4027					key.ao_key, req, skb,
4028					opts.hash_location - (u8 *)th, 0);
4029#endif
4030	}
4031#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
4032	rcu_read_unlock();
4033#endif
4034
4035	bpf_skops_write_hdr_opt((struct sock *)sk, skb, req, syn_skb,
4036				synack_type, &opts);
4037
4038	skb_set_delivery_time(skb, now, SKB_CLOCK_MONOTONIC);
4039	tcp_add_tx_delay(skb, tp);
4040
4041	return skb;
4042}
4043EXPORT_IPV6_MOD(tcp_make_synack);
4044
4045static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
4046{
4047	struct inet_connection_sock *icsk = inet_csk(sk);
4048	const struct tcp_congestion_ops *ca;
4049	u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
4050
4051	if (ca_key == TCP_CA_UNSPEC)
4052		return;
4053
4054	rcu_read_lock();
4055	ca = tcp_ca_find_key(ca_key);
4056	if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
4057		bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
4058		icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
4059		icsk->icsk_ca_ops = ca;
4060	}
4061	rcu_read_unlock();
4062}
4063
4064/* Do all connect socket setups that can be done AF independent. */
4065static void tcp_connect_init(struct sock *sk)
4066{
4067	const struct dst_entry *dst = __sk_dst_get(sk);
4068	struct tcp_sock *tp = tcp_sk(sk);
4069	__u8 rcv_wscale;
4070	u16 user_mss;
4071	u32 rcv_wnd;
4072
4073	/* We'll fix this up when we get a response from the other end.
4074	 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
4075	 */
4076	tp->tcp_header_len = sizeof(struct tcphdr);
4077	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps))
4078		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
4079
4080	tcp_ao_connect_init(sk);
4081
4082	/* If user gave his TCP_MAXSEG, record it to clamp */
4083	user_mss = READ_ONCE(tp->rx_opt.user_mss);
4084	if (user_mss)
4085		tp->rx_opt.mss_clamp = user_mss;
4086	tp->max_window = 0;
4087	tcp_mtup_init(sk);
4088	tcp_sync_mss(sk, dst_mtu(dst));
4089
4090	tcp_ca_dst_init(sk, dst);
4091
4092	if (!tp->window_clamp)
4093		WRITE_ONCE(tp->window_clamp, dst_metric(dst, RTAX_WINDOW));
4094	tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
4095
4096	tcp_initialize_rcv_mss(sk);
4097
4098	/* limit the window selection if the user enforce a smaller rx buffer */
4099	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
4100	    (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
4101		WRITE_ONCE(tp->window_clamp, tcp_full_space(sk));
4102
4103	rcv_wnd = tcp_rwnd_init_bpf(sk);
4104	if (rcv_wnd == 0)
4105		rcv_wnd = dst_metric(dst, RTAX_INITRWND);
4106
4107	tcp_select_initial_window(sk, tcp_full_space(sk),
4108				  tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
4109				  &tp->rcv_wnd,
4110				  &tp->window_clamp,
4111				  READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling),
4112				  &rcv_wscale,
4113				  rcv_wnd);
4114
4115	tp->rx_opt.rcv_wscale = rcv_wscale;
4116	tp->rcv_ssthresh = tp->rcv_wnd;
4117
4118	WRITE_ONCE(sk->sk_err, 0);
4119	sock_reset_flag(sk, SOCK_DONE);
4120	tp->snd_wnd = 0;
4121	tcp_init_wl(tp, 0);
4122	tcp_write_queue_purge(sk);
4123	tp->snd_una = tp->write_seq;
4124	tp->snd_sml = tp->write_seq;
4125	tp->snd_up = tp->write_seq;
4126	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
4127
4128	if (likely(!tp->repair))
4129		tp->rcv_nxt = 0;
4130	else
4131		tp->rcv_tstamp = tcp_jiffies32;
4132	tp->rcv_wup = tp->rcv_nxt;
4133	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
4134
4135	inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
4136	WRITE_ONCE(inet_csk(sk)->icsk_retransmits, 0);
4137	tcp_clear_retrans(tp);
4138}
4139
4140static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
4141{
4142	struct tcp_sock *tp = tcp_sk(sk);
4143	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
4144
4145	tcb->end_seq += skb->len;
4146	__skb_header_release(skb);
4147	sk_wmem_queued_add(sk, skb->truesize);
4148	sk_mem_charge(sk, skb->truesize);
4149	WRITE_ONCE(tp->write_seq, tcb->end_seq);
4150	tp->packets_out += tcp_skb_pcount(skb);
4151}
4152
4153/* Build and send a SYN with data and (cached) Fast Open cookie. However,
4154 * queue a data-only packet after the regular SYN, such that regular SYNs
4155 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
4156 * only the SYN sequence, the data are retransmitted in the first ACK.
4157 * If cookie is not cached or other error occurs, falls back to send a
4158 * regular SYN with Fast Open cookie request option.
4159 */
4160static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
4161{
4162	struct inet_connection_sock *icsk = inet_csk(sk);
4163	struct tcp_sock *tp = tcp_sk(sk);
4164	struct tcp_fastopen_request *fo = tp->fastopen_req;
4165	struct page_frag *pfrag = sk_page_frag(sk);
4166	struct sk_buff *syn_data;
4167	int space, err = 0;
4168
4169	tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
4170	if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
4171		goto fallback;
4172
4173	/* MSS for SYN-data is based on cached MSS and bounded by PMTU and
4174	 * user-MSS. Reserve maximum option space for middleboxes that add
4175	 * private TCP options. The cost is reduced data space in SYN :(
4176	 */
4177	tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
4178	/* Sync mss_cache after updating the mss_clamp */
4179	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
4180
4181	space = __tcp_mtu_to_mss(sk, icsk->icsk_pmtu_cookie) -
4182		MAX_TCP_OPTION_SPACE;
4183
4184	space = min_t(size_t, space, fo->size);
4185
4186	if (space &&
4187	    !skb_page_frag_refill(min_t(size_t, space, PAGE_SIZE),
4188				  pfrag, sk->sk_allocation))
4189		goto fallback;
4190	syn_data = tcp_stream_alloc_skb(sk, sk->sk_allocation, false);
4191	if (!syn_data)
4192		goto fallback;
4193	memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
4194	if (space) {
4195		space = min_t(size_t, space, pfrag->size - pfrag->offset);
4196		space = tcp_wmem_schedule(sk, space);
4197	}
4198	if (space) {
4199		space = copy_page_from_iter(pfrag->page, pfrag->offset,
4200					    space, &fo->data->msg_iter);
4201		if (unlikely(!space)) {
4202			tcp_skb_tsorted_anchor_cleanup(syn_data);
4203			kfree_skb(syn_data);
4204			goto fallback;
4205		}
4206		skb_fill_page_desc(syn_data, 0, pfrag->page,
4207				   pfrag->offset, space);
4208		page_ref_inc(pfrag->page);
4209		pfrag->offset += space;
4210		skb_len_add(syn_data, space);
4211		skb_zcopy_set(syn_data, fo->uarg, NULL);
4212	}
4213	/* No more data pending in inet_wait_for_connect() */
4214	if (space == fo->size)
4215		fo->data = NULL;
4216	fo->copied = space;
4217
4218	tcp_connect_queue_skb(sk, syn_data);
4219	if (syn_data->len)
4220		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
4221
4222	err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
4223
4224	skb_set_delivery_time(syn, syn_data->skb_mstamp_ns, SKB_CLOCK_MONOTONIC);
4225
4226	/* Now full SYN+DATA was cloned and sent (or not),
4227	 * remove the SYN from the original skb (syn_data)
4228	 * we keep in write queue in case of a retransmit, as we
4229	 * also have the SYN packet (with no data) in the same queue.
4230	 */
4231	TCP_SKB_CB(syn_data)->seq++;
4232	TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
4233	if (!err) {
4234		tp->syn_data = (fo->copied > 0);
4235		tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data);
4236		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
4237		goto done;
4238	}
4239
4240	/* data was not sent, put it in write_queue */
4241	__skb_queue_tail(&sk->sk_write_queue, syn_data);
4242	tp->packets_out -= tcp_skb_pcount(syn_data);
4243
4244fallback:
4245	/* Send a regular SYN with Fast Open cookie request option */
4246	if (fo->cookie.len > 0)
4247		fo->cookie.len = 0;
4248	err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
4249	if (err)
4250		tp->syn_fastopen = 0;
4251done:
4252	fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
4253	return err;
4254}
4255
4256/* Build a SYN and send it off. */
4257int tcp_connect(struct sock *sk)
4258{
4259	struct tcp_sock *tp = tcp_sk(sk);
4260	struct sk_buff *buff;
4261	int err;
4262
4263	tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
4264
4265#if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO)
4266	/* Has to be checked late, after setting daddr/saddr/ops.
4267	 * Return error if the peer has both a md5 and a tcp-ao key
4268	 * configured as this is ambiguous.
4269	 */
4270	if (unlikely(rcu_dereference_protected(tp->md5sig_info,
4271					       lockdep_sock_is_held(sk)))) {
4272		bool needs_ao = !!tp->af_specific->ao_lookup(sk, sk, -1, -1);
4273		bool needs_md5 = !!tp->af_specific->md5_lookup(sk, sk);
4274		struct tcp_ao_info *ao_info;
4275
4276		ao_info = rcu_dereference_check(tp->ao_info,
4277						lockdep_sock_is_held(sk));
4278		if (ao_info) {
4279			/* This is an extra check: tcp_ao_required() in
4280			 * tcp_v{4,6}_parse_md5_keys() should prevent adding
4281			 * md5 keys on ao_required socket.
4282			 */
4283			needs_ao |= ao_info->ao_required;
4284			WARN_ON_ONCE(ao_info->ao_required && needs_md5);
4285		}
4286		if (needs_md5 && needs_ao)
4287			return -EKEYREJECTED;
4288
4289		/* If we have a matching md5 key and no matching tcp-ao key
4290		 * then free up ao_info if allocated.
4291		 */
4292		if (needs_md5) {
4293			tcp_ao_destroy_sock(sk, false);
4294		} else if (needs_ao) {
4295			tcp_clear_md5_list(sk);
4296			kfree(rcu_replace_pointer(tp->md5sig_info, NULL,
4297						  lockdep_sock_is_held(sk)));
4298		}
4299	}
4300#endif
4301#ifdef CONFIG_TCP_AO
4302	if (unlikely(rcu_dereference_protected(tp->ao_info,
4303					       lockdep_sock_is_held(sk)))) {
4304		/* Don't allow connecting if ao is configured but no
4305		 * matching key is found.
4306		 */
4307		if (!tp->af_specific->ao_lookup(sk, sk, -1, -1))
4308			return -EKEYREJECTED;
4309	}
4310#endif
4311
4312	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
4313		return -EHOSTUNREACH; /* Routing failure or similar. */
4314
4315	tcp_connect_init(sk);
4316
4317	if (unlikely(tp->repair)) {
4318		tcp_finish_connect(sk, NULL);
4319		return 0;
4320	}
4321
4322	buff = tcp_stream_alloc_skb(sk, sk->sk_allocation, true);
4323	if (unlikely(!buff))
4324		return -ENOBUFS;
4325
4326	/* SYN eats a sequence byte, write_seq updated by
4327	 * tcp_connect_queue_skb().
4328	 */
4329	tcp_init_nondata_skb(buff, sk, tp->write_seq, TCPHDR_SYN);
4330	tcp_mstamp_refresh(tp);
4331	tp->retrans_stamp = tcp_time_stamp_ts(tp);
4332	tcp_connect_queue_skb(sk, buff);
4333	tcp_ecn_send_syn(sk, buff);
4334	tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
4335
4336	/* Send off SYN; include data in Fast Open. */
4337	err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
4338	      tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
4339	if (err == -ECONNREFUSED)
4340		return err;
4341
4342	/* We change tp->snd_nxt after the tcp_transmit_skb() call
4343	 * in order to make this packet get counted in tcpOutSegs.
4344	 */
4345	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
4346	tp->pushed_seq = tp->write_seq;
4347	buff = tcp_send_head(sk);
4348	if (unlikely(buff)) {
4349		WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq);
4350		tp->pushed_seq	= TCP_SKB_CB(buff)->seq;
4351	}
4352	TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
4353
4354	/* Timer for repeating the SYN until an answer. */
4355	tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
4356			     inet_csk(sk)->icsk_rto, false);
4357	return 0;
4358}
4359EXPORT_SYMBOL(tcp_connect);
4360
4361u32 tcp_delack_max(const struct sock *sk)
4362{
4363	u32 delack_from_rto_min = max(tcp_rto_min(sk), 2) - 1;
4364
4365	return min(READ_ONCE(inet_csk(sk)->icsk_delack_max), delack_from_rto_min);
4366}
4367
4368/* Send out a delayed ack, the caller does the policy checking
4369 * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
4370 * for details.
4371 */
4372void tcp_send_delayed_ack(struct sock *sk)
4373{
4374	struct inet_connection_sock *icsk = inet_csk(sk);
4375	int ato = icsk->icsk_ack.ato;
4376	unsigned long timeout;
4377
4378	if (ato > TCP_DELACK_MIN) {
4379		const struct tcp_sock *tp = tcp_sk(sk);
4380		int max_ato = HZ / 2;
4381
4382		if (inet_csk_in_pingpong_mode(sk) ||
4383		    (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
4384			max_ato = TCP_DELACK_MAX;
4385
4386		/* Slow path, intersegment interval is "high". */
4387
4388		/* If some rtt estimate is known, use it to bound delayed ack.
4389		 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
4390		 * directly.
4391		 */
4392		if (tp->srtt_us) {
4393			int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
4394					TCP_DELACK_MIN);
4395
4396			if (rtt < max_ato)
4397				max_ato = rtt;
4398		}
4399
4400		ato = min(ato, max_ato);
4401	}
4402
4403	ato = min_t(u32, ato, tcp_delack_max(sk));
4404
4405	/* Stay within the limit we were given */
4406	timeout = jiffies + ato;
4407
4408	/* Use new timeout only if there wasn't a older one earlier. */
4409	if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
4410		/* If delack timer is about to expire, send ACK now. */
4411		if (time_before_eq(icsk_delack_timeout(icsk), jiffies + (ato >> 2))) {
4412			tcp_send_ack(sk);
4413			return;
4414		}
4415
4416		if (!time_before(timeout, icsk_delack_timeout(icsk)))
4417			timeout = icsk_delack_timeout(icsk);
4418	}
4419	smp_store_release(&icsk->icsk_ack.pending,
4420			  icsk->icsk_ack.pending | ICSK_ACK_SCHED | ICSK_ACK_TIMER);
4421	sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
4422}
4423
4424/* This routine sends an ack and also updates the window. */
4425void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags)
4426{
4427	struct sk_buff *buff;
4428
4429	/* If we have been reset, we may not send again. */
4430	if (sk->sk_state == TCP_CLOSE)
4431		return;
4432
4433	/* We are not putting this on the write queue, so
4434	 * tcp_transmit_skb() will set the ownership to this
4435	 * sock.
4436	 */
4437	buff = alloc_skb(MAX_TCP_HEADER,
4438			 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4439	if (unlikely(!buff)) {
4440		struct inet_connection_sock *icsk = inet_csk(sk);
4441		unsigned long delay;
4442
4443		delay = TCP_DELACK_MAX << icsk->icsk_ack.retry;
4444		if (delay < tcp_rto_max(sk))
4445			icsk->icsk_ack.retry++;
4446		inet_csk_schedule_ack(sk);
4447		icsk->icsk_ack.ato = TCP_ATO_MIN;
4448		tcp_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, false);
4449		return;
4450	}
4451
4452	/* Reserve space for headers and prepare control bits. */
4453	skb_reserve(buff, MAX_TCP_HEADER);
4454	tcp_init_nondata_skb(buff, sk,
4455			     tcp_acceptable_seq(sk), TCPHDR_ACK | flags);
4456
4457	/* We do not want pure acks influencing TCP Small Queues or fq/pacing
4458	 * too much.
4459	 * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
4460	 */
4461	skb_set_tcp_pure_ack(buff);
4462
4463	/* Send it off, this clears delayed acks for us. */
4464	__tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0, rcv_nxt);
4465}
4466EXPORT_SYMBOL_GPL(__tcp_send_ack);
4467
4468void tcp_send_ack(struct sock *sk)
4469{
4470	__tcp_send_ack(sk, tcp_sk(sk)->rcv_nxt, 0);
4471}
4472
4473/* This routine sends a packet with an out of date sequence
4474 * number. It assumes the other end will try to ack it.
4475 *
4476 * Question: what should we make while urgent mode?
4477 * 4.4BSD forces sending single byte of data. We cannot send
4478 * out of window data, because we have SND.NXT==SND.MAX...
4479 *
4480 * Current solution: to send TWO zero-length segments in urgent mode:
4481 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
4482 * out-of-date with SND.UNA-1 to probe window.
4483 */
4484static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
4485{
4486	struct tcp_sock *tp = tcp_sk(sk);
4487	struct sk_buff *skb;
4488
4489	/* We don't queue it, tcp_transmit_skb() sets ownership. */
4490	skb = alloc_skb(MAX_TCP_HEADER,
4491			sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4492	if (!skb)
4493		return -1;
4494
4495	/* Reserve space for headers and set control bits. */
4496	skb_reserve(skb, MAX_TCP_HEADER);
4497	/* Use a previous sequence.  This should cause the other
4498	 * end to send an ack.  Don't queue or clone SKB, just
4499	 * send it.
4500	 */
4501	tcp_init_nondata_skb(skb, sk, tp->snd_una - !urgent, TCPHDR_ACK);
4502	NET_INC_STATS(sock_net(sk), mib);
4503	return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
4504}
4505
4506/* Called from setsockopt( ... TCP_REPAIR ) */
4507void tcp_send_window_probe(struct sock *sk)
4508{
4509	if (sk->sk_state == TCP_ESTABLISHED) {
4510		tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
4511		tcp_mstamp_refresh(tcp_sk(sk));
4512		tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
4513	}
4514}
4515
4516/* Initiate keepalive or window probe from timer. */
4517int tcp_write_wakeup(struct sock *sk, int mib)
4518{
4519	struct tcp_sock *tp = tcp_sk(sk);
4520	struct sk_buff *skb;
4521
4522	if (sk->sk_state == TCP_CLOSE)
4523		return -1;
4524
4525	skb = tcp_send_head(sk);
4526	if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
4527		int err;
4528		unsigned int mss = tcp_current_mss(sk);
4529		unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
4530
4531		if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
4532			tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
4533
4534		/* We are probing the opening of a window
4535		 * but the window size is != 0
4536		 * must have been a result SWS avoidance ( sender )
4537		 */
4538		if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
4539		    skb->len > mss) {
4540			seg_size = min(seg_size, mss);
4541			TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4542			if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
4543					 skb, seg_size, mss, GFP_ATOMIC))
4544				return -1;
4545		} else if (!tcp_skb_pcount(skb))
4546			tcp_set_skb_tso_segs(skb, mss);
4547
4548		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4549		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
4550		if (!err)
4551			tcp_event_new_data_sent(sk, skb);
4552		return err;
4553	} else {
4554		if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
4555			tcp_xmit_probe_skb(sk, 1, mib);
4556		return tcp_xmit_probe_skb(sk, 0, mib);
4557	}
4558}
4559
4560/* A window probe timeout has occurred.  If window is not closed send
4561 * a partial packet else a zero probe.
4562 */
4563void tcp_send_probe0(struct sock *sk)
4564{
4565	struct inet_connection_sock *icsk = inet_csk(sk);
4566	struct tcp_sock *tp = tcp_sk(sk);
4567	struct net *net = sock_net(sk);
4568	unsigned long timeout;
4569	int err;
4570
4571	err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
4572
4573	if (tp->packets_out || tcp_write_queue_empty(sk)) {
4574		/* Cancel probe timer, if it is not required. */
4575		WRITE_ONCE(icsk->icsk_probes_out, 0);
4576		icsk->icsk_backoff = 0;
4577		icsk->icsk_probes_tstamp = 0;
4578		return;
4579	}
4580
4581	WRITE_ONCE(icsk->icsk_probes_out, icsk->icsk_probes_out + 1);
4582	if (err <= 0) {
4583		if (icsk->icsk_backoff < READ_ONCE(net->ipv4.sysctl_tcp_retries2))
4584			icsk->icsk_backoff++;
4585		timeout = tcp_probe0_when(sk, tcp_rto_max(sk));
4586	} else {
4587		/* If packet was not sent due to local congestion,
4588		 * Let senders fight for local resources conservatively.
4589		 */
4590		timeout = TCP_RESOURCE_PROBE_INTERVAL;
4591	}
4592
4593	timeout = tcp_clamp_probe0_to_user_timeout(sk, timeout);
4594	tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, timeout, true);
4595}
4596
4597int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
4598{
4599	const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
4600	struct flowi fl;
4601	int res;
4602
4603	/* Paired with WRITE_ONCE() in sock_setsockopt() */
4604	if (READ_ONCE(sk->sk_txrehash) == SOCK_TXREHASH_ENABLED)
4605		WRITE_ONCE(tcp_rsk(req)->txhash, net_tx_rndhash());
4606	res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL,
4607				  NULL);
4608	if (!res) {
4609		TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
4610		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
4611		if (unlikely(tcp_passive_fastopen(sk))) {
4612			/* sk has const attribute because listeners are lockless.
4613			 * However in this case, we are dealing with a passive fastopen
4614			 * socket thus we can change total_retrans value.
4615			 */
4616			tcp_sk_rw(sk)->total_retrans++;
4617		}
4618		trace_tcp_retransmit_synack(sk, req);
4619		WRITE_ONCE(req->num_retrans, req->num_retrans + 1);
4620	}
4621	return res;
4622}
4623EXPORT_IPV6_MOD(tcp_rtx_synack);