tcp.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * INET		An implementation of the TCP/IP protocol suite for the LINUX
   4 *		operating system.  INET is implemented using the  BSD Socket
   5 *		interface as the means of communication with the user level.
   6 *
   7 *		Implementation of the Transmission Control Protocol(TCP).
   8 *
   9 * Authors:	Ross Biro
  10 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11 *		Mark Evans, <evansmp@uhura.aston.ac.uk>
  12 *		Corey Minyard <wf-rch!minyard@relay.EU.net>
  13 *		Florian La Roche, <flla@stud.uni-sb.de>
  14 *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  15 *		Linus Torvalds, <torvalds@cs.helsinki.fi>
  16 *		Alan Cox, <gw4pts@gw4pts.ampr.org>
  17 *		Matthew Dillon, <dillon@apollo.west.oic.com>
  18 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  19 *		Jorge Cwik, <jorge@laser.satlink.net>
  20 *
  21 * Fixes:
  22 *		Alan Cox	:	Numerous verify_area() calls
  23 *		Alan Cox	:	Set the ACK bit on a reset
  24 *		Alan Cox	:	Stopped it crashing if it closed while
  25 *					sk->inuse=1 and was trying to connect
  26 *					(tcp_err()).
  27 *		Alan Cox	:	All icmp error handling was broken
  28 *					pointers passed where wrong and the
  29 *					socket was looked up backwards. Nobody
  30 *					tested any icmp error code obviously.
  31 *		Alan Cox	:	tcp_err() now handled properly. It
  32 *					wakes people on errors. poll
  33 *					behaves and the icmp error race
  34 *					has gone by moving it into sock.c
  35 *		Alan Cox	:	tcp_send_reset() fixed to work for
  36 *					everything not just packets for
  37 *					unknown sockets.
  38 *		Alan Cox	:	tcp option processing.
  39 *		Alan Cox	:	Reset tweaked (still not 100%) [Had
  40 *					syn rule wrong]
  41 *		Herp Rosmanith  :	More reset fixes
  42 *		Alan Cox	:	No longer acks invalid rst frames.
  43 *					Acking any kind of RST is right out.
  44 *		Alan Cox	:	Sets an ignore me flag on an rst
  45 *					receive otherwise odd bits of prattle
  46 *					escape still
  47 *		Alan Cox	:	Fixed another acking RST frame bug.
  48 *					Should stop LAN workplace lockups.
  49 *		Alan Cox	: 	Some tidyups using the new skb list
  50 *					facilities
  51 *		Alan Cox	:	sk->keepopen now seems to work
  52 *		Alan Cox	:	Pulls options out correctly on accepts
  53 *		Alan Cox	:	Fixed assorted sk->rqueue->next errors
  54 *		Alan Cox	:	PSH doesn't end a TCP read. Switched a
  55 *					bit to skb ops.
  56 *		Alan Cox	:	Tidied tcp_data to avoid a potential
  57 *					nasty.
  58 *		Alan Cox	:	Added some better commenting, as the
  59 *					tcp is hard to follow
  60 *		Alan Cox	:	Removed incorrect check for 20 * psh
  61 *	Michael O'Reilly	:	ack < copied bug fix.
  62 *	Johannes Stille		:	Misc tcp fixes (not all in yet).
  63 *		Alan Cox	:	FIN with no memory -> CRASH
  64 *		Alan Cox	:	Added socket option proto entries.
  65 *					Also added awareness of them to accept.
  66 *		Alan Cox	:	Added TCP options (SOL_TCP)
  67 *		Alan Cox	:	Switched wakeup calls to callbacks,
  68 *					so the kernel can layer network
  69 *					sockets.
  70 *		Alan Cox	:	Use ip_tos/ip_ttl settings.
  71 *		Alan Cox	:	Handle FIN (more) properly (we hope).
  72 *		Alan Cox	:	RST frames sent on unsynchronised
  73 *					state ack error.
  74 *		Alan Cox	:	Put in missing check for SYN bit.
  75 *		Alan Cox	:	Added tcp_select_window() aka NET2E
  76 *					window non shrink trick.
  77 *		Alan Cox	:	Added a couple of small NET2E timer
  78 *					fixes
  79 *		Charles Hedrick :	TCP fixes
  80 *		Toomas Tamm	:	TCP window fixes
  81 *		Alan Cox	:	Small URG fix to rlogin ^C ack fight
  82 *		Charles Hedrick	:	Rewrote most of it to actually work
  83 *		Linus		:	Rewrote tcp_read() and URG handling
  84 *					completely
  85 *		Gerhard Koerting:	Fixed some missing timer handling
  86 *		Matthew Dillon  :	Reworked TCP machine states as per RFC
  87 *		Gerhard Koerting:	PC/TCP workarounds
  88 *		Adam Caldwell	:	Assorted timer/timing errors
  89 *		Matthew Dillon	:	Fixed another RST bug
  90 *		Alan Cox	:	Move to kernel side addressing changes.
  91 *		Alan Cox	:	Beginning work on TCP fastpathing
  92 *					(not yet usable)
  93 *		Arnt Gulbrandsen:	Turbocharged tcp_check() routine.
  94 *		Alan Cox	:	TCP fast path debugging
  95 *		Alan Cox	:	Window clamping
  96 *		Michael Riepe	:	Bug in tcp_check()
  97 *		Matt Dillon	:	More TCP improvements and RST bug fixes
  98 *		Matt Dillon	:	Yet more small nasties remove from the
  99 *					TCP code (Be very nice to this man if
 100 *					tcp finally works 100%) 8)
 101 *		Alan Cox	:	BSD accept semantics.
 102 *		Alan Cox	:	Reset on closedown bug.
 103 *	Peter De Schrijver	:	ENOTCONN check missing in tcp_sendto().
 104 *		Michael Pall	:	Handle poll() after URG properly in
 105 *					all cases.
 106 *		Michael Pall	:	Undo the last fix in tcp_read_urg()
 107 *					(multi URG PUSH broke rlogin).
 108 *		Michael Pall	:	Fix the multi URG PUSH problem in
 109 *					tcp_readable(), poll() after URG
 110 *					works now.
 111 *		Michael Pall	:	recv(...,MSG_OOB) never blocks in the
 112 *					BSD api.
 113 *		Alan Cox	:	Changed the semantics of sk->socket to
 114 *					fix a race and a signal problem with
 115 *					accept() and async I/O.
 116 *		Alan Cox	:	Relaxed the rules on tcp_sendto().
 117 *		Yury Shevchuk	:	Really fixed accept() blocking problem.
 118 *		Craig I. Hagan  :	Allow for BSD compatible TIME_WAIT for
 119 *					clients/servers which listen in on
 120 *					fixed ports.
 121 *		Alan Cox	:	Cleaned the above up and shrank it to
 122 *					a sensible code size.
 123 *		Alan Cox	:	Self connect lockup fix.
 124 *		Alan Cox	:	No connect to multicast.
 125 *		Ross Biro	:	Close unaccepted children on master
 126 *					socket close.
 127 *		Alan Cox	:	Reset tracing code.
 128 *		Alan Cox	:	Spurious resets on shutdown.
 129 *		Alan Cox	:	Giant 15 minute/60 second timer error
 130 *		Alan Cox	:	Small whoops in polling before an
 131 *					accept.
 132 *		Alan Cox	:	Kept the state trace facility since
 133 *					it's handy for debugging.
 134 *		Alan Cox	:	More reset handler fixes.
 135 *		Alan Cox	:	Started rewriting the code based on
 136 *					the RFC's for other useful protocol
 137 *					references see: Comer, KA9Q NOS, and
 138 *					for a reference on the difference
 139 *					between specifications and how BSD
 140 *					works see the 4.4lite source.
 141 *		A.N.Kuznetsov	:	Don't time wait on completion of tidy
 142 *					close.
 143 *		Linus Torvalds	:	Fin/Shutdown & copied_seq changes.
 144 *		Linus Torvalds	:	Fixed BSD port reuse to work first syn
 145 *		Alan Cox	:	Reimplemented timers as per the RFC
 146 *					and using multiple timers for sanity.
 147 *		Alan Cox	:	Small bug fixes, and a lot of new
 148 *					comments.
 149 *		Alan Cox	:	Fixed dual reader crash by locking
 150 *					the buffers (much like datagram.c)
 151 *		Alan Cox	:	Fixed stuck sockets in probe. A probe
 152 *					now gets fed up of retrying without
 153 *					(even a no space) answer.
 154 *		Alan Cox	:	Extracted closing code better
 155 *		Alan Cox	:	Fixed the closing state machine to
 156 *					resemble the RFC.
 157 *		Alan Cox	:	More 'per spec' fixes.
 158 *		Jorge Cwik	:	Even faster checksumming.
 159 *		Alan Cox	:	tcp_data() doesn't ack illegal PSH
 160 *					only frames. At least one pc tcp stack
 161 *					generates them.
 162 *		Alan Cox	:	Cache last socket.
 163 *		Alan Cox	:	Per route irtt.
 164 *		Matt Day	:	poll()->select() match BSD precisely on error
 165 *		Alan Cox	:	New buffers
 166 *		Marc Tamsky	:	Various sk->prot->retransmits and
 167 *					sk->retransmits misupdating fixed.
 168 *					Fixed tcp_write_timeout: stuck close,
 169 *					and TCP syn retries gets used now.
 170 *		Mark Yarvis	:	In tcp_read_wakeup(), don't send an
 171 *					ack if state is TCP_CLOSED.
 172 *		Alan Cox	:	Look up device on a retransmit - routes may
 173 *					change. Doesn't yet cope with MSS shrink right
 174 *					but it's a start!
 175 *		Marc Tamsky	:	Closing in closing fixes.
 176 *		Mike Shaver	:	RFC1122 verifications.
 177 *		Alan Cox	:	rcv_saddr errors.
 178 *		Alan Cox	:	Block double connect().
 179 *		Alan Cox	:	Small hooks for enSKIP.
 180 *		Alexey Kuznetsov:	Path MTU discovery.
 181 *		Alan Cox	:	Support soft errors.
 182 *		Alan Cox	:	Fix MTU discovery pathological case
 183 *					when the remote claims no mtu!
 184 *		Marc Tamsky	:	TCP_CLOSE fix.
 185 *		Colin (G3TNE)	:	Send a reset on syn ack replies in
 186 *					window but wrong (fixes NT lpd problems)
 187 *		Pedro Roque	:	Better TCP window handling, delayed ack.
 188 *		Joerg Reuter	:	No modification of locked buffers in
 189 *					tcp_do_retransmit()
 190 *		Eric Schenk	:	Changed receiver side silly window
 191 *					avoidance algorithm to BSD style
 192 *					algorithm. This doubles throughput
 193 *					against machines running Solaris,
 194 *					and seems to result in general
 195 *					improvement.
 196 *	Stefan Magdalinski	:	adjusted tcp_readable() to fix FIONREAD
 197 *	Willy Konynenberg	:	Transparent proxying support.
 198 *	Mike McLagan		:	Routing by source
 199 *		Keith Owens	:	Do proper merging with partial SKB's in
 200 *					tcp_do_sendmsg to avoid burstiness.
 201 *		Eric Schenk	:	Fix fast close down bug with
 202 *					shutdown() followed by close().
 203 *		Andi Kleen 	:	Make poll agree with SIGIO
 204 *	Salvatore Sanfilippo	:	Support SO_LINGER with linger == 1 and
 205 *					lingertime == 0 (RFC 793 ABORT Call)
 206 *	Hirokazu Takahashi	:	Use copy_from_user() instead of
 207 *					csum_and_copy_from_user() if possible.
 208 *
 209 * Description of States:
 210 *
 211 *	TCP_SYN_SENT		sent a connection request, waiting for ack
 212 *
 213 *	TCP_SYN_RECV		received a connection request, sent ack,
 214 *				waiting for final ack in three-way handshake.
 215 *
 216 *	TCP_ESTABLISHED		connection established
 217 *
 218 *	TCP_FIN_WAIT1		our side has shutdown, waiting to complete
 219 *				transmission of remaining buffered data
 220 *
 221 *	TCP_FIN_WAIT2		all buffered data sent, waiting for remote
 222 *				to shutdown
 223 *
 224 *	TCP_CLOSING		both sides have shutdown but we still have
 225 *				data we have to finish sending
 226 *
 227 *	TCP_TIME_WAIT		timeout to catch resent junk before entering
 228 *				closed, can only be entered from FIN_WAIT2
 229 *				or CLOSING.  Required because the other end
 230 *				may not have gotten our last ACK causing it
 231 *				to retransmit the data packet (which we ignore)
 232 *
 233 *	TCP_CLOSE_WAIT		remote side has shutdown and is waiting for
 234 *				us to finish writing our data and to shutdown
 235 *				(we have to close() to move on to LAST_ACK)
 236 *
 237 *	TCP_LAST_ACK		out side has shutdown after remote has
 238 *				shutdown.  There may still be data in our
 239 *				buffer that we have to finish sending
 240 *
 241 *	TCP_CLOSE		socket is finished
 242 */
 243
 244#define pr_fmt(fmt) "TCP: " fmt
 245
 246#include <crypto/md5.h>
 247#include <crypto/utils.h>
 248#include <linux/kernel.h>
 249#include <linux/module.h>
 250#include <linux/types.h>
 251#include <linux/fcntl.h>
 252#include <linux/poll.h>
 253#include <linux/inet_diag.h>
 254#include <linux/init.h>
 255#include <linux/fs.h>
 256#include <linux/skbuff.h>
 257#include <linux/splice.h>
 258#include <linux/net.h>
 259#include <linux/socket.h>
 260#include <linux/random.h>
 261#include <linux/memblock.h>
 262#include <linux/highmem.h>
 263#include <linux/cache.h>
 264#include <linux/err.h>
 265#include <linux/time.h>
 266#include <linux/slab.h>
 267#include <linux/errqueue.h>
 268#include <linux/static_key.h>
 269#include <linux/btf.h>
 270
 271#include <net/icmp.h>
 272#include <net/inet_common.h>
 273#include <net/inet_ecn.h>
 274#include <net/tcp.h>
 275#include <net/tcp_ecn.h>
 276#include <net/mptcp.h>
 277#include <net/proto_memory.h>
 278#include <net/xfrm.h>
 279#include <net/ip.h>
 280#include <net/psp.h>
 281#include <net/sock.h>
 282#include <net/rstreason.h>
 283
 284#include <linux/uaccess.h>
 285#include <asm/ioctls.h>
 286#include <net/busy_poll.h>
 287#include <net/hotdata.h>
 288#include <trace/events/tcp.h>
 289#include <net/rps.h>
 290
 291#include "../core/devmem.h"
 292
 293/* Track pending CMSGs. */
 294enum {
 295	TCP_CMSG_INQ = 1,
 296	TCP_CMSG_TS = 2
 297};
 298
 299DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
 300EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
 301
 302DEFINE_PER_CPU(u32, tcp_tw_isn);
 303EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
 304
 305long sysctl_tcp_mem[3] __read_mostly;
 306EXPORT_IPV6_MOD(sysctl_tcp_mem);
 307
 308DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
 309EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
 310
 311#if IS_ENABLED(CONFIG_SMC)
 312DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
 313EXPORT_SYMBOL(tcp_have_smc);
 314#endif
 315
 316/*
 317 * Current number of TCP sockets.
 318 */
 319struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp;
 320EXPORT_IPV6_MOD(tcp_sockets_allocated);
 321
 322/*
 323 * TCP splice context
 324 */
 325struct tcp_splice_state {
 326	struct pipe_inode_info *pipe;
 327	size_t len;
 328	unsigned int flags;
 329};
 330
 331/*
 332 * Pressure flag: try to collapse.
 333 * Technical note: it is used by multiple contexts non atomically.
 334 * All the __sk_mem_schedule() is of this nature: accounting
 335 * is strict, actions are advisory and have some latency.
 336 */
 337unsigned long tcp_memory_pressure __read_mostly;
 338EXPORT_SYMBOL_GPL(tcp_memory_pressure);
 339
 340void tcp_enter_memory_pressure(struct sock *sk)
 341{
 342	unsigned long val;
 343
 344	if (READ_ONCE(tcp_memory_pressure))
 345		return;
 346	val = jiffies;
 347
 348	if (!val)
 349		val--;
 350	if (!cmpxchg(&tcp_memory_pressure, 0, val))
 351		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
 352}
 353EXPORT_IPV6_MOD_GPL(tcp_enter_memory_pressure);
 354
 355void tcp_leave_memory_pressure(struct sock *sk)
 356{
 357	unsigned long val;
 358
 359	if (!READ_ONCE(tcp_memory_pressure))
 360		return;
 361	val = xchg(&tcp_memory_pressure, 0);
 362	if (val)
 363		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
 364			      jiffies_to_msecs(jiffies - val));
 365}
 366EXPORT_IPV6_MOD_GPL(tcp_leave_memory_pressure);
 367
 368/* Convert seconds to retransmits based on initial and max timeout */
 369static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
 370{
 371	u8 res = 0;
 372
 373	if (seconds > 0) {
 374		int period = timeout;
 375
 376		res = 1;
 377		while (seconds > period && res < 255) {
 378			res++;
 379			timeout <<= 1;
 380			if (timeout > rto_max)
 381				timeout = rto_max;
 382			period += timeout;
 383		}
 384	}
 385	return res;
 386}
 387
 388/* Convert retransmits to seconds based on initial and max timeout */
 389static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
 390{
 391	int period = 0;
 392
 393	if (retrans > 0) {
 394		period = timeout;
 395		while (--retrans) {
 396			timeout <<= 1;
 397			if (timeout > rto_max)
 398				timeout = rto_max;
 399			period += timeout;
 400		}
 401	}
 402	return period;
 403}
 404
 405static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
 406{
 407	u32 rate = READ_ONCE(tp->rate_delivered);
 408	u32 intv = READ_ONCE(tp->rate_interval_us);
 409	u64 rate64 = 0;
 410
 411	if (rate && intv) {
 412		rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
 413		do_div(rate64, intv);
 414	}
 415	return rate64;
 416}
 417
 418#ifdef CONFIG_TCP_MD5SIG
 419void tcp_md5_destruct_sock(struct sock *sk)
 420{
 421	struct tcp_sock *tp = tcp_sk(sk);
 422
 423	if (tp->md5sig_info) {
 424
 425		tcp_clear_md5_list(sk);
 426		kfree(rcu_replace_pointer(tp->md5sig_info, NULL, 1));
 427		static_branch_slow_dec_deferred(&tcp_md5_needed);
 428	}
 429}
 430EXPORT_IPV6_MOD_GPL(tcp_md5_destruct_sock);
 431#endif
 432
 433/* Address-family independent initialization for a tcp_sock.
 434 *
 435 * NOTE: A lot of things set to zero explicitly by call to
 436 *       sk_alloc() so need not be done here.
 437 */
 438void tcp_init_sock(struct sock *sk)
 439{
 440	struct inet_connection_sock *icsk = inet_csk(sk);
 441	struct tcp_sock *tp = tcp_sk(sk);
 442	int rto_min_us, rto_max_ms;
 443
 444	tp->out_of_order_queue = RB_ROOT;
 445	sk->tcp_rtx_queue = RB_ROOT;
 446	tcp_init_xmit_timers(sk);
 447	INIT_LIST_HEAD(&tp->tsq_node);
 448	INIT_LIST_HEAD(&tp->tsorted_sent_queue);
 449
 450	icsk->icsk_rto = TCP_TIMEOUT_INIT;
 451
 452	rto_max_ms = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_max_ms);
 453	icsk->icsk_rto_max = msecs_to_jiffies(rto_max_ms);
 454
 455	rto_min_us = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rto_min_us);
 456	icsk->icsk_rto_min = usecs_to_jiffies(rto_min_us);
 457	icsk->icsk_delack_max = TCP_DELACK_MAX;
 458	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
 459	minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
 460
 461	/* So many TCP implementations out there (incorrectly) count the
 462	 * initial SYN frame in their delayed-ACK and congestion control
 463	 * algorithms that we must have the following bandaid to talk
 464	 * efficiently to them.  -DaveM
 465	 */
 466	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
 467
 468	/* There's a bubble in the pipe until at least the first ACK. */
 469	tp->app_limited = ~0U;
 470	tp->rate_app_limited = 1;
 471
 472	/* See draft-stevens-tcpca-spec-01 for discussion of the
 473	 * initialization of these values.
 474	 */
 475	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
 476	tp->snd_cwnd_clamp = ~0;
 477	tp->mss_cache = TCP_MSS_DEFAULT;
 478
 479	tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);
 480	tcp_assign_congestion_control(sk);
 481
 482	tp->tsoffset = 0;
 483	tp->rack.reo_wnd_steps = 1;
 484
 485	sk->sk_write_space = sk_stream_write_space;
 486	sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
 487
 488	icsk->icsk_sync_mss = tcp_sync_mss;
 489
 490	WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1]));
 491	WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1]));
 492	tcp_scaling_ratio_init(sk);
 493
 494	set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
 495	sk_sockets_allocated_inc(sk);
 496	xa_init_flags(&sk->sk_user_frags, XA_FLAGS_ALLOC1);
 497}
 498EXPORT_IPV6_MOD(tcp_init_sock);
 499
 500static void tcp_tx_timestamp(struct sock *sk, struct sockcm_cookie *sockc)
 501{
 502	struct sk_buff *skb = tcp_write_queue_tail(sk);
 503	u32 tsflags = sockc->tsflags;
 504
 505	if (unlikely(!skb))
 506		skb = skb_rb_last(&sk->tcp_rtx_queue);
 507
 508	if (tsflags && skb) {
 509		struct skb_shared_info *shinfo = skb_shinfo(skb);
 510		struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 511
 512		sock_tx_timestamp(sk, sockc, &shinfo->tx_flags);
 513		if (tsflags & SOF_TIMESTAMPING_TX_ACK)
 514			tcb->txstamp_ack |= TSTAMP_ACK_SK;
 515		if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
 516			shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
 517	}
 518
 519	if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) &&
 520	    SK_BPF_CB_FLAG_TEST(sk, SK_BPF_CB_TX_TIMESTAMPING) && skb)
 521		bpf_skops_tx_timestamping(sk, skb, BPF_SOCK_OPS_TSTAMP_SENDMSG_CB);
 522}
 523
 524static bool tcp_stream_is_readable(struct sock *sk, int target)
 525{
 526	if (tcp_epollin_ready(sk, target))
 527		return true;
 528	return sk_is_readable(sk);
 529}
 530
 531/*
 532 *	Wait for a TCP event.
 533 *
 534 *	Note that we don't need to lock the socket, as the upper poll layers
 535 *	take care of normal races (between the test and the event) and we don't
 536 *	go look at any of the socket buffers directly.
 537 */
 538__poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 539{
 540	__poll_t mask;
 541	struct sock *sk = sock->sk;
 542	const struct tcp_sock *tp = tcp_sk(sk);
 543	u8 shutdown;
 544	int state;
 545
 546	sock_poll_wait(file, sock, wait);
 547
 548	state = inet_sk_state_load(sk);
 549	if (state == TCP_LISTEN)
 550		return inet_csk_listen_poll(sk);
 551
 552	/* Socket is not locked. We are protected from async events
 553	 * by poll logic and correct handling of state changes
 554	 * made by other threads is impossible in any case.
 555	 */
 556
 557	mask = 0;
 558
 559	/*
 560	 * EPOLLHUP is certainly not done right. But poll() doesn't
 561	 * have a notion of HUP in just one direction, and for a
 562	 * socket the read side is more interesting.
 563	 *
 564	 * Some poll() documentation says that EPOLLHUP is incompatible
 565	 * with the EPOLLOUT/POLLWR flags, so somebody should check this
 566	 * all. But careful, it tends to be safer to return too many
 567	 * bits than too few, and you can easily break real applications
 568	 * if you don't tell them that something has hung up!
 569	 *
 570	 * Check-me.
 571	 *
 572	 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
 573	 * our fs/select.c). It means that after we received EOF,
 574	 * poll always returns immediately, making impossible poll() on write()
 575	 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
 576	 * if and only if shutdown has been made in both directions.
 577	 * Actually, it is interesting to look how Solaris and DUX
 578	 * solve this dilemma. I would prefer, if EPOLLHUP were maskable,
 579	 * then we could set it on SND_SHUTDOWN. BTW examples given
 580	 * in Stevens' books assume exactly this behaviour, it explains
 581	 * why EPOLLHUP is incompatible with EPOLLOUT.	--ANK
 582	 *
 583	 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 584	 * blocking on fresh not-connected or disconnected socket. --ANK
 585	 */
 586	shutdown = READ_ONCE(sk->sk_shutdown);
 587	if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
 588		mask |= EPOLLHUP;
 589	if (shutdown & RCV_SHUTDOWN)
 590		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
 591
 592	/* Connected or passive Fast Open socket? */
 593	if (state != TCP_SYN_SENT &&
 594	    (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
 595		int target = sock_rcvlowat(sk, 0, INT_MAX);
 596		u16 urg_data = READ_ONCE(tp->urg_data);
 597
 598		if (unlikely(urg_data) &&
 599		    READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
 600		    !sock_flag(sk, SOCK_URGINLINE))
 601			target++;
 602
 603		if (tcp_stream_is_readable(sk, target))
 604			mask |= EPOLLIN | EPOLLRDNORM;
 605
 606		if (!(shutdown & SEND_SHUTDOWN)) {
 607			if (__sk_stream_is_writeable(sk, 1)) {
 608				mask |= EPOLLOUT | EPOLLWRNORM;
 609			} else {  /* send SIGIO later */
 610				sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 611				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 612
 613				/* Race breaker. If space is freed after
 614				 * wspace test but before the flags are set,
 615				 * IO signal will be lost. Memory barrier
 616				 * pairs with the input side.
 617				 */
 618				smp_mb__after_atomic();
 619				if (__sk_stream_is_writeable(sk, 1))
 620					mask |= EPOLLOUT | EPOLLWRNORM;
 621			}
 622		} else
 623			mask |= EPOLLOUT | EPOLLWRNORM;
 624
 625		if (urg_data & TCP_URG_VALID)
 626			mask |= EPOLLPRI;
 627	} else if (state == TCP_SYN_SENT &&
 628		   inet_test_bit(DEFER_CONNECT, sk)) {
 629		/* Active TCP fastopen socket with defer_connect
 630		 * Return EPOLLOUT so application can call write()
 631		 * in order for kernel to generate SYN+data
 632		 */
 633		mask |= EPOLLOUT | EPOLLWRNORM;
 634	}
 635	/* This barrier is coupled with smp_wmb() in tcp_done_with_error() */
 636	smp_rmb();
 637	if (READ_ONCE(sk->sk_err) ||
 638	    !skb_queue_empty_lockless(&sk->sk_error_queue))
 639		mask |= EPOLLERR;
 640
 641	return mask;
 642}
 643EXPORT_SYMBOL(tcp_poll);
 644
 645int tcp_ioctl(struct sock *sk, int cmd, int *karg)
 646{
 647	struct tcp_sock *tp = tcp_sk(sk);
 648	int answ;
 649	bool slow;
 650
 651	switch (cmd) {
 652	case SIOCINQ:
 653		if (sk->sk_state == TCP_LISTEN)
 654			return -EINVAL;
 655
 656		slow = lock_sock_fast(sk);
 657		answ = tcp_inq(sk);
 658		unlock_sock_fast(sk, slow);
 659		break;
 660	case SIOCATMARK:
 661		answ = READ_ONCE(tp->urg_data) &&
 662		       READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
 663		break;
 664	case SIOCOUTQ:
 665		if (sk->sk_state == TCP_LISTEN)
 666			return -EINVAL;
 667
 668		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 669			answ = 0;
 670		else
 671			answ = READ_ONCE(tp->write_seq) - tp->snd_una;
 672		break;
 673	case SIOCOUTQNSD:
 674		if (sk->sk_state == TCP_LISTEN)
 675			return -EINVAL;
 676
 677		if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 678			answ = 0;
 679		else
 680			answ = READ_ONCE(tp->write_seq) -
 681			       READ_ONCE(tp->snd_nxt);
 682		break;
 683	default:
 684		return -ENOIOCTLCMD;
 685	}
 686
 687	*karg = answ;
 688	return 0;
 689}
 690EXPORT_IPV6_MOD(tcp_ioctl);
 691
 692void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 693{
 694	TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
 695	tp->pushed_seq = tp->write_seq;
 696}
 697
 698static inline bool forced_push(const struct tcp_sock *tp)
 699{
 700	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 701}
 702
 703void tcp_skb_entail(struct sock *sk, struct sk_buff *skb)
 704{
 705	struct tcp_sock *tp = tcp_sk(sk);
 706	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 707
 708	tcb->seq     = tcb->end_seq = tp->write_seq;
 709	tcb->tcp_flags = TCPHDR_ACK;
 710	__skb_header_release(skb);
 711	psp_enqueue_set_decrypted(sk, skb);
 712	tcp_add_write_queue_tail(sk, skb);
 713	sk_wmem_queued_add(sk, skb->truesize);
 714	sk_mem_charge(sk, skb->truesize);
 715	if (tp->nonagle & TCP_NAGLE_PUSH)
 716		tp->nonagle &= ~TCP_NAGLE_PUSH;
 717
 718	tcp_slow_start_after_idle_check(sk);
 719}
 720
 721static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
 722{
 723	if (flags & MSG_OOB)
 724		tp->snd_up = tp->write_seq;
 725}
 726
 727/* If a not yet filled skb is pushed, do not send it if
 728 * we have data packets in Qdisc or NIC queues :
 729 * Because TX completion will happen shortly, it gives a chance
 730 * to coalesce future sendmsg() payload into this skb, without
 731 * need for a timer, and with no latency trade off.
 732 * As packets containing data payload have a bigger truesize
 733 * than pure acks (dataless) packets, the last checks prevent
 734 * autocorking if we only have an ACK in Qdisc/NIC queues,
 735 * or if TX completion was delayed after we processed ACK packet.
 736 */
 737static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
 738				int size_goal)
 739{
 740	return skb->len < size_goal &&
 741	       READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) &&
 742	       !tcp_rtx_queue_empty(sk) &&
 743	       refcount_read(&sk->sk_wmem_alloc) > skb->truesize &&
 744	       tcp_skb_can_collapse_to(skb);
 745}
 746
 747void tcp_push(struct sock *sk, int flags, int mss_now,
 748	      int nonagle, int size_goal)
 749{
 750	struct tcp_sock *tp = tcp_sk(sk);
 751	struct sk_buff *skb;
 752
 753	skb = tcp_write_queue_tail(sk);
 754	if (!skb)
 755		return;
 756	if (!(flags & MSG_MORE) || forced_push(tp))
 757		tcp_mark_push(tp, skb);
 758
 759	tcp_mark_urg(tp, flags);
 760
 761	if (tcp_should_autocork(sk, skb, size_goal)) {
 762
 763		/* avoid atomic op if TSQ_THROTTLED bit is already set */
 764		if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
 765			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
 766			set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
 767			smp_mb__after_atomic();
 768		}
 769		/* It is possible TX completion already happened
 770		 * before we set TSQ_THROTTLED.
 771		 */
 772		if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
 773			return;
 774	}
 775
 776	if (flags & MSG_MORE)
 777		nonagle = TCP_NAGLE_CORK;
 778
 779	__tcp_push_pending_frames(sk, mss_now, nonagle);
 780}
 781
 782static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 783				unsigned int offset, size_t len)
 784{
 785	struct tcp_splice_state *tss = rd_desc->arg.data;
 786	int ret;
 787
 788	ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
 789			      min(rd_desc->count, len), tss->flags);
 790	if (ret > 0)
 791		rd_desc->count -= ret;
 792	return ret;
 793}
 794
 795static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 796{
 797	/* Store TCP splice context information in read_descriptor_t. */
 798	read_descriptor_t rd_desc = {
 799		.arg.data = tss,
 800		.count	  = tss->len,
 801	};
 802
 803	return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 804}
 805
 806/**
 807 *  tcp_splice_read - splice data from TCP socket to a pipe
 808 * @sock:	socket to splice from
 809 * @ppos:	position (not valid)
 810 * @pipe:	pipe to splice to
 811 * @len:	number of bytes to splice
 812 * @flags:	splice modifier flags
 813 *
 814 * Description:
 815 *    Will read pages from given socket and fill them into a pipe.
 816 *
 817 **/
 818ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 819			struct pipe_inode_info *pipe, size_t len,
 820			unsigned int flags)
 821{
 822	struct sock *sk = sock->sk;
 823	struct tcp_splice_state tss = {
 824		.pipe = pipe,
 825		.len = len,
 826		.flags = flags,
 827	};
 828	long timeo;
 829	ssize_t spliced;
 830	int ret;
 831
 832	sock_rps_record_flow(sk);
 833	/*
 834	 * We can't seek on a socket input
 835	 */
 836	if (unlikely(*ppos))
 837		return -ESPIPE;
 838
 839	ret = spliced = 0;
 840
 841	lock_sock(sk);
 842
 843	timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
 844	while (tss.len) {
 845		ret = __tcp_splice_read(sk, &tss);
 846		if (ret < 0)
 847			break;
 848		else if (!ret) {
 849			if (spliced)
 850				break;
 851			if (sock_flag(sk, SOCK_DONE))
 852				break;
 853			if (sk->sk_err) {
 854				ret = sock_error(sk);
 855				break;
 856			}
 857			if (sk->sk_shutdown & RCV_SHUTDOWN)
 858				break;
 859			if (sk->sk_state == TCP_CLOSE) {
 860				/*
 861				 * This occurs when user tries to read
 862				 * from never connected socket.
 863				 */
 864				ret = -ENOTCONN;
 865				break;
 866			}
 867			if (!timeo) {
 868				ret = -EAGAIN;
 869				break;
 870			}
 871			/* if __tcp_splice_read() got nothing while we have
 872			 * an skb in receive queue, we do not want to loop.
 873			 * This might happen with URG data.
 874			 */
 875			if (!skb_queue_empty(&sk->sk_receive_queue))
 876				break;
 877			ret = sk_wait_data(sk, &timeo, NULL);
 878			if (ret < 0)
 879				break;
 880			if (signal_pending(current)) {
 881				ret = sock_intr_errno(timeo);
 882				break;
 883			}
 884			continue;
 885		}
 886		tss.len -= ret;
 887		spliced += ret;
 888
 889		if (!tss.len || !timeo)
 890			break;
 891		release_sock(sk);
 892		lock_sock(sk);
 893
 894		if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 895		    (sk->sk_shutdown & RCV_SHUTDOWN) ||
 896		    signal_pending(current))
 897			break;
 898	}
 899
 900	release_sock(sk);
 901
 902	if (spliced)
 903		return spliced;
 904
 905	return ret;
 906}
 907EXPORT_IPV6_MOD(tcp_splice_read);
 908
 909struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
 910				     bool force_schedule)
 911{
 912	struct sk_buff *skb;
 913
 914	skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
 915	if (likely(skb)) {
 916		bool mem_scheduled;
 917
 918		skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
 919		if (force_schedule) {
 920			mem_scheduled = true;
 921			sk_forced_mem_schedule(sk, skb->truesize);
 922		} else {
 923			mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
 924		}
 925		if (likely(mem_scheduled)) {
 926			skb_reserve(skb, MAX_TCP_HEADER);
 927			skb->ip_summed = CHECKSUM_PARTIAL;
 928			INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
 929			return skb;
 930		}
 931		__kfree_skb(skb);
 932	} else {
 933		if (!sk->sk_bypass_prot_mem)
 934			tcp_enter_memory_pressure(sk);
 935		sk_stream_moderate_sndbuf(sk);
 936	}
 937	return NULL;
 938}
 939
 940static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
 941				       int large_allowed)
 942{
 943	struct tcp_sock *tp = tcp_sk(sk);
 944	u32 new_size_goal, size_goal;
 945
 946	if (!large_allowed)
 947		return mss_now;
 948
 949	/* Note : tcp_tso_autosize() will eventually split this later */
 950	new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size);
 951
 952	/* We try hard to avoid divides here */
 953	size_goal = tp->gso_segs * mss_now;
 954	if (unlikely(new_size_goal < size_goal ||
 955		     new_size_goal >= size_goal + mss_now)) {
 956		tp->gso_segs = min_t(u16, new_size_goal / mss_now,
 957				     sk->sk_gso_max_segs);
 958		size_goal = tp->gso_segs * mss_now;
 959	}
 960
 961	return max(size_goal, mss_now);
 962}
 963
 964int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
 965{
 966	int mss_now;
 967
 968	mss_now = tcp_current_mss(sk);
 969	*size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
 970
 971	return mss_now;
 972}
 973
 974/* In some cases, sendmsg() could have added an skb to the write queue,
 975 * but failed adding payload on it. We need to remove it to consume less
 976 * memory, but more importantly be able to generate EPOLLOUT for Edge Trigger
 977 * epoll() users. Another reason is that tcp_write_xmit() does not like
 978 * finding an empty skb in the write queue.
 979 */
 980void tcp_remove_empty_skb(struct sock *sk)
 981{
 982	struct sk_buff *skb = tcp_write_queue_tail(sk);
 983
 984	if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
 985		tcp_unlink_write_queue(skb, sk);
 986		if (tcp_write_queue_empty(sk))
 987			tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
 988		tcp_wmem_free_skb(sk, skb);
 989	}
 990}
 991
 992/* skb changing from pure zc to mixed, must charge zc */
 993static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb)
 994{
 995	if (unlikely(skb_zcopy_pure(skb))) {
 996		u32 extra = skb->truesize -
 997			    SKB_TRUESIZE(skb_end_offset(skb));
 998
 999		if (!sk_wmem_schedule(sk, extra))
1000			return -ENOMEM;
1001
1002		sk_mem_charge(sk, extra);
1003		skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
1004	}
1005	return 0;
1006}
1007
1008
1009int tcp_wmem_schedule(struct sock *sk, int copy)
1010{
1011	int left;
1012
1013	if (likely(sk_wmem_schedule(sk, copy)))
1014		return copy;
1015
1016	/* We could be in trouble if we have nothing queued.
1017	 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
1018	 * to guarantee some progress.
1019	 */
1020	left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued;
1021	if (left > 0)
1022		sk_forced_mem_schedule(sk, min(left, copy));
1023	return min(copy, sk->sk_forward_alloc);
1024}
1025
1026void tcp_free_fastopen_req(struct tcp_sock *tp)
1027{
1028	if (tp->fastopen_req) {
1029		kfree(tp->fastopen_req);
1030		tp->fastopen_req = NULL;
1031	}
1032}
1033
1034int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
1035			 size_t size, struct ubuf_info *uarg)
1036{
1037	struct tcp_sock *tp = tcp_sk(sk);
1038	struct inet_sock *inet = inet_sk(sk);
1039	struct sockaddr *uaddr = msg->msg_name;
1040	int err, flags;
1041
1042	if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) &
1043	      TFO_CLIENT_ENABLE) ||
1044	    (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1045	     uaddr->sa_family == AF_UNSPEC))
1046		return -EOPNOTSUPP;
1047	if (tp->fastopen_req)
1048		return -EALREADY; /* Another Fast Open is in progress */
1049
1050	tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1051				   sk->sk_allocation);
1052	if (unlikely(!tp->fastopen_req))
1053		return -ENOBUFS;
1054	tp->fastopen_req->data = msg;
1055	tp->fastopen_req->size = size;
1056	tp->fastopen_req->uarg = uarg;
1057
1058	if (inet_test_bit(DEFER_CONNECT, sk)) {
1059		err = tcp_connect(sk);
1060		/* Same failure procedure as in tcp_v4/6_connect */
1061		if (err) {
1062			tcp_set_state(sk, TCP_CLOSE);
1063			inet->inet_dport = 0;
1064			sk->sk_route_caps = 0;
1065		}
1066	}
1067	flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1068	err = __inet_stream_connect(sk->sk_socket, (struct sockaddr_unsized *)uaddr,
1069				    msg->msg_namelen, flags, 1);
1070	/* fastopen_req could already be freed in __inet_stream_connect
1071	 * if the connection times out or gets rst
1072	 */
1073	if (tp->fastopen_req) {
1074		*copied = tp->fastopen_req->copied;
1075		tcp_free_fastopen_req(tp);
1076		inet_clear_bit(DEFER_CONNECT, sk);
1077	}
1078	return err;
1079}
1080
1081int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1082{
1083	struct net_devmem_dmabuf_binding *binding = NULL;
1084	struct tcp_sock *tp = tcp_sk(sk);
1085	struct ubuf_info *uarg = NULL;
1086	struct sk_buff *skb;
1087	struct sockcm_cookie sockc;
1088	int flags, err, copied = 0;
1089	int mss_now = 0, size_goal, copied_syn = 0;
1090	int process_backlog = 0;
1091	int sockc_err = 0;
1092	int zc = 0;
1093	long timeo;
1094
1095	flags = msg->msg_flags;
1096
1097	sockc = (struct sockcm_cookie){ .tsflags = READ_ONCE(sk->sk_tsflags) };
1098	if (msg->msg_controllen) {
1099		sockc_err = sock_cmsg_send(sk, msg, &sockc);
1100		/* Don't return error until MSG_FASTOPEN has been processed;
1101		 * that may succeed even if the cmsg is invalid.
1102		 */
1103	}
1104
1105	if ((flags & MSG_ZEROCOPY) && size) {
1106		if (msg->msg_ubuf) {
1107			uarg = msg->msg_ubuf;
1108			if (sk->sk_route_caps & NETIF_F_SG)
1109				zc = MSG_ZEROCOPY;
1110		} else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1111			skb = tcp_write_queue_tail(sk);
1112			uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb),
1113						    !sockc_err && sockc.dmabuf_id);
1114			if (!uarg) {
1115				err = -ENOBUFS;
1116				goto out_err;
1117			}
1118			if (sk->sk_route_caps & NETIF_F_SG)
1119				zc = MSG_ZEROCOPY;
1120			else
1121				uarg_to_msgzc(uarg)->zerocopy = 0;
1122
1123			if (!sockc_err && sockc.dmabuf_id) {
1124				binding = net_devmem_get_binding(sk, sockc.dmabuf_id);
1125				if (IS_ERR(binding)) {
1126					err = PTR_ERR(binding);
1127					binding = NULL;
1128					goto out_err;
1129				}
1130			}
1131		}
1132	} else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) {
1133		if (sk->sk_route_caps & NETIF_F_SG)
1134			zc = MSG_SPLICE_PAGES;
1135	}
1136
1137	if (!sockc_err && sockc.dmabuf_id &&
1138	    (!(flags & MSG_ZEROCOPY) || !sock_flag(sk, SOCK_ZEROCOPY))) {
1139		err = -EINVAL;
1140		goto out_err;
1141	}
1142
1143	if (unlikely(flags & MSG_FASTOPEN ||
1144		     inet_test_bit(DEFER_CONNECT, sk)) &&
1145	    !tp->repair) {
1146		err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1147		if (err == -EINPROGRESS && copied_syn > 0)
1148			goto out;
1149		else if (err)
1150			goto out_err;
1151	}
1152
1153	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1154
1155	tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1156
1157	/* Wait for a connection to finish. One exception is TCP Fast Open
1158	 * (passive side) where data is allowed to be sent before a connection
1159	 * is fully established.
1160	 */
1161	if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1162	    !tcp_passive_fastopen(sk)) {
1163		err = sk_stream_wait_connect(sk, &timeo);
1164		if (err != 0)
1165			goto do_error;
1166	}
1167
1168	if (unlikely(tp->repair)) {
1169		if (tp->repair_queue == TCP_RECV_QUEUE) {
1170			copied = tcp_send_rcvq(sk, msg, size);
1171			goto out_nopush;
1172		}
1173
1174		err = -EINVAL;
1175		if (tp->repair_queue == TCP_NO_QUEUE)
1176			goto out_err;
1177
1178		/* 'common' sending to sendq */
1179	}
1180
1181	if (sockc_err) {
1182		err = sockc_err;
1183		goto out_err;
1184	}
1185
1186	/* This should be in poll */
1187	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1188
1189	/* Ok commence sending. */
1190	copied = 0;
1191
1192restart:
1193	mss_now = tcp_send_mss(sk, &size_goal, flags);
1194
1195	err = -EPIPE;
1196	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1197		goto do_error;
1198
1199	while (msg_data_left(msg)) {
1200		int copy = 0;
1201
1202		skb = tcp_write_queue_tail(sk);
1203		if (skb)
1204			copy = size_goal - skb->len;
1205
1206		trace_tcp_sendmsg_locked(sk, msg, skb, size_goal);
1207
1208		if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1209			bool first_skb;
1210
1211new_segment:
1212			if (!sk_stream_memory_free(sk))
1213				goto wait_for_space;
1214
1215			if (unlikely(process_backlog >= 16)) {
1216				process_backlog = 0;
1217				if (sk_flush_backlog(sk))
1218					goto restart;
1219			}
1220			first_skb = tcp_rtx_and_write_queues_empty(sk);
1221			skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
1222						   first_skb);
1223			if (!skb)
1224				goto wait_for_space;
1225
1226			process_backlog++;
1227
1228#ifdef CONFIG_SKB_DECRYPTED
1229			skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
1230#endif
1231			tcp_skb_entail(sk, skb);
1232			copy = size_goal;
1233
1234			/* All packets are restored as if they have
1235			 * already been sent. skb_mstamp_ns isn't set to
1236			 * avoid wrong rtt estimation.
1237			 */
1238			if (tp->repair)
1239				TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1240		}
1241
1242		/* Try to append data to the end of skb. */
1243		if (copy > msg_data_left(msg))
1244			copy = msg_data_left(msg);
1245
1246		if (zc == 0) {
1247			bool merge = true;
1248			int i = skb_shinfo(skb)->nr_frags;
1249			struct page_frag *pfrag = sk_page_frag(sk);
1250
1251			if (!sk_page_frag_refill(sk, pfrag))
1252				goto wait_for_space;
1253
1254			if (!skb_can_coalesce(skb, i, pfrag->page,
1255					      pfrag->offset)) {
1256				if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
1257					tcp_mark_push(tp, skb);
1258					goto new_segment;
1259				}
1260				merge = false;
1261			}
1262
1263			copy = min_t(int, copy, pfrag->size - pfrag->offset);
1264
1265			if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) {
1266				if (tcp_downgrade_zcopy_pure(sk, skb))
1267					goto wait_for_space;
1268				skb_zcopy_downgrade_managed(skb);
1269			}
1270
1271			copy = tcp_wmem_schedule(sk, copy);
1272			if (!copy)
1273				goto wait_for_space;
1274
1275			err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1276						       pfrag->page,
1277						       pfrag->offset,
1278						       copy);
1279			if (err)
1280				goto do_error;
1281
1282			/* Update the skb. */
1283			if (merge) {
1284				skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1285			} else {
1286				skb_fill_page_desc(skb, i, pfrag->page,
1287						   pfrag->offset, copy);
1288				page_ref_inc(pfrag->page);
1289			}
1290			pfrag->offset += copy;
1291		} else if (zc == MSG_ZEROCOPY)  {
1292			/* First append to a fragless skb builds initial
1293			 * pure zerocopy skb
1294			 */
1295			if (!skb->len)
1296				skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
1297
1298			if (!skb_zcopy_pure(skb)) {
1299				copy = tcp_wmem_schedule(sk, copy);
1300				if (!copy)
1301					goto wait_for_space;
1302			}
1303
1304			err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg,
1305						       binding);
1306			if (err == -EMSGSIZE || err == -EEXIST) {
1307				tcp_mark_push(tp, skb);
1308				goto new_segment;
1309			}
1310			if (err < 0)
1311				goto do_error;
1312			copy = err;
1313		} else if (zc == MSG_SPLICE_PAGES) {
1314			/* Splice in data if we can; copy if we can't. */
1315			if (tcp_downgrade_zcopy_pure(sk, skb))
1316				goto wait_for_space;
1317			copy = tcp_wmem_schedule(sk, copy);
1318			if (!copy)
1319				goto wait_for_space;
1320
1321			err = skb_splice_from_iter(skb, &msg->msg_iter, copy);
1322			if (err < 0) {
1323				if (err == -EMSGSIZE) {
1324					tcp_mark_push(tp, skb);
1325					goto new_segment;
1326				}
1327				goto do_error;
1328			}
1329			copy = err;
1330
1331			if (!(flags & MSG_NO_SHARED_FRAGS))
1332				skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1333
1334			sk_wmem_queued_add(sk, copy);
1335			sk_mem_charge(sk, copy);
1336		}
1337
1338		if (!copied)
1339			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1340
1341		WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1342		TCP_SKB_CB(skb)->end_seq += copy;
1343		tcp_skb_pcount_set(skb, 0);
1344
1345		copied += copy;
1346		if (!msg_data_left(msg)) {
1347			if (unlikely(flags & MSG_EOR))
1348				TCP_SKB_CB(skb)->eor = 1;
1349			goto out;
1350		}
1351
1352		if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1353			continue;
1354
1355		if (forced_push(tp)) {
1356			tcp_mark_push(tp, skb);
1357			__tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1358		} else if (skb == tcp_send_head(sk))
1359			tcp_push_one(sk, mss_now);
1360		continue;
1361
1362wait_for_space:
1363		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1364		tcp_remove_empty_skb(sk);
1365		if (copied)
1366			tcp_push(sk, flags & ~MSG_MORE, mss_now,
1367				 TCP_NAGLE_PUSH, size_goal);
1368
1369		err = sk_stream_wait_memory(sk, &timeo);
1370		if (err != 0)
1371			goto do_error;
1372
1373		mss_now = tcp_send_mss(sk, &size_goal, flags);
1374	}
1375
1376out:
1377	if (copied) {
1378		tcp_tx_timestamp(sk, &sockc);
1379		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1380	}
1381out_nopush:
1382	/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1383	if (uarg && !msg->msg_ubuf)
1384		net_zcopy_put(uarg);
1385	if (binding)
1386		net_devmem_dmabuf_binding_put(binding);
1387	return copied + copied_syn;
1388
1389do_error:
1390	tcp_remove_empty_skb(sk);
1391
1392	if (copied + copied_syn)
1393		goto out;
1394out_err:
1395	/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1396	if (uarg && !msg->msg_ubuf)
1397		net_zcopy_put_abort(uarg, true);
1398	err = sk_stream_error(sk, flags, err);
1399	/* make sure we wake any epoll edge trigger waiter */
1400	if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1401		READ_ONCE(sk->sk_write_space)(sk);
1402		tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1403	}
1404	if (binding)
1405		net_devmem_dmabuf_binding_put(binding);
1406
1407	return err;
1408}
1409EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
1410
1411int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1412{
1413	int ret;
1414
1415	lock_sock(sk);
1416	ret = tcp_sendmsg_locked(sk, msg, size);
1417	release_sock(sk);
1418
1419	return ret;
1420}
1421EXPORT_SYMBOL(tcp_sendmsg);
1422
1423void tcp_splice_eof(struct socket *sock)
1424{
1425	struct sock *sk = sock->sk;
1426	struct tcp_sock *tp = tcp_sk(sk);
1427	int mss_now, size_goal;
1428
1429	if (!tcp_write_queue_tail(sk))
1430		return;
1431
1432	lock_sock(sk);
1433	mss_now = tcp_send_mss(sk, &size_goal, 0);
1434	tcp_push(sk, 0, mss_now, tp->nonagle, size_goal);
1435	release_sock(sk);
1436}
1437EXPORT_IPV6_MOD_GPL(tcp_splice_eof);
1438
1439/*
1440 *	Handle reading urgent data. BSD has very simple semantics for
1441 *	this, no blocking and very strange errors 8)
1442 */
1443
1444static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1445{
1446	struct tcp_sock *tp = tcp_sk(sk);
1447
1448	/* No URG data to read. */
1449	if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1450	    tp->urg_data == TCP_URG_READ)
1451		return -EINVAL;	/* Yes this is right ! */
1452
1453	if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1454		return -ENOTCONN;
1455
1456	if (tp->urg_data & TCP_URG_VALID) {
1457		int err = 0;
1458		char c = tp->urg_data;
1459
1460		if (!(flags & MSG_PEEK))
1461			WRITE_ONCE(tp->urg_data, TCP_URG_READ);
1462
1463		/* Read urgent data. */
1464		msg->msg_flags |= MSG_OOB;
1465
1466		if (len > 0) {
1467			if (!(flags & MSG_TRUNC))
1468				err = memcpy_to_msg(msg, &c, 1);
1469			len = 1;
1470		} else
1471			msg->msg_flags |= MSG_TRUNC;
1472
1473		return err ? -EFAULT : len;
1474	}
1475
1476	if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1477		return 0;
1478
1479	/* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1480	 * the available implementations agree in this case:
1481	 * this call should never block, independent of the
1482	 * blocking state of the socket.
1483	 * Mike <pall@rz.uni-karlsruhe.de>
1484	 */
1485	return -EAGAIN;
1486}
1487
1488static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1489{
1490	struct sk_buff *skb;
1491	int copied = 0, err = 0;
1492
1493	skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1494		err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1495		if (err)
1496			return err;
1497		copied += skb->len;
1498	}
1499
1500	skb_queue_walk(&sk->sk_write_queue, skb) {
1501		err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1502		if (err)
1503			break;
1504
1505		copied += skb->len;
1506	}
1507
1508	return err ?: copied;
1509}
1510
1511/* Clean up the receive buffer for full frames taken by the user,
1512 * then send an ACK if necessary.  COPIED is the number of bytes
1513 * tcp_recvmsg has given to the user so far, it speeds up the
1514 * calculation of whether or not we must ACK for the sake of
1515 * a window update.
1516 */
1517void __tcp_cleanup_rbuf(struct sock *sk, int copied)
1518{
1519	struct tcp_sock *tp = tcp_sk(sk);
1520	bool time_to_ack = false;
1521
1522	if (inet_csk_ack_scheduled(sk)) {
1523		const struct inet_connection_sock *icsk = inet_csk(sk);
1524
1525		if (/* Once-per-two-segments ACK was not sent by tcp_input.c */
1526		    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1527		    /*
1528		     * If this read emptied read buffer, we send ACK, if
1529		     * connection is not bidirectional, user drained
1530		     * receive buffer and there was a small segment
1531		     * in queue.
1532		     */
1533		    (copied > 0 &&
1534		     ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1535		      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1536		       !inet_csk_in_pingpong_mode(sk))) &&
1537		      !atomic_read(&sk->sk_rmem_alloc)))
1538			time_to_ack = true;
1539	}
1540
1541	/* We send an ACK if we can now advertise a non-zero window
1542	 * which has been raised "significantly".
1543	 *
1544	 * Even if window raised up to infinity, do not send window open ACK
1545	 * in states, where we will not receive more. It is useless.
1546	 */
1547	if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1548		__u32 rcv_window_now = tcp_receive_window(tp);
1549
1550		/* Optimize, __tcp_select_window() is not cheap. */
1551		if (2*rcv_window_now <= tp->window_clamp) {
1552			__u32 new_window = __tcp_select_window(sk);
1553
1554			/* Send ACK now, if this read freed lots of space
1555			 * in our buffer. Certainly, new_window is new window.
1556			 * We can advertise it now, if it is not less than current one.
1557			 * "Lots" means "at least twice" here.
1558			 */
1559			if (new_window && new_window >= 2 * rcv_window_now)
1560				time_to_ack = true;
1561		}
1562	}
1563	if (time_to_ack) {
1564		tcp_mstamp_refresh(tp);
1565		tcp_send_ack(sk);
1566	}
1567}
1568
1569void tcp_cleanup_rbuf(struct sock *sk, int copied)
1570{
1571	struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1572	struct tcp_sock *tp = tcp_sk(sk);
1573
1574	WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1575	     "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1576	     tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1577	__tcp_cleanup_rbuf(sk, copied);
1578}
1579
1580static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
1581{
1582	__skb_unlink(skb, &sk->sk_receive_queue);
1583	if (likely(skb->destructor == sock_rfree)) {
1584		sock_rfree(skb);
1585		skb->destructor = NULL;
1586		skb->sk = NULL;
1587		return skb_attempt_defer_free(skb);
1588	}
1589	__kfree_skb(skb);
1590}
1591
1592struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1593{
1594	struct sk_buff *skb;
1595	u32 offset;
1596
1597	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1598		offset = seq - TCP_SKB_CB(skb)->seq;
1599		if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1600			pr_err_once("%s: found a SYN, please report !\n", __func__);
1601			offset--;
1602		}
1603		if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1604			*off = offset;
1605			return skb;
1606		}
1607		/* This looks weird, but this can happen if TCP collapsing
1608		 * splitted a fat GRO packet, while we released socket lock
1609		 * in skb_splice_bits()
1610		 */
1611		tcp_eat_recv_skb(sk, skb);
1612	}
1613	return NULL;
1614}
1615EXPORT_SYMBOL(tcp_recv_skb);
1616
1617/*
1618 * This routine provides an alternative to tcp_recvmsg() for routines
1619 * that would like to handle copying from skbuffs directly in 'sendfile'
1620 * fashion.
1621 * Note:
1622 *	- It is assumed that the socket was locked by the caller.
1623 *	- The routine does not block.
1624 *	- At present, there is no support for reading OOB data
1625 *	  or for 'peeking' the socket using this routine
1626 *	  (although both would be easy to implement).
1627 */
1628static int __tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1629			   sk_read_actor_t recv_actor, bool noack,
1630			   u32 *copied_seq)
1631{
1632	struct sk_buff *skb;
1633	struct tcp_sock *tp = tcp_sk(sk);
1634	u32 seq = *copied_seq;
1635	u32 offset;
1636	int copied = 0;
1637
1638	if (sk->sk_state == TCP_LISTEN)
1639		return -ENOTCONN;
1640	while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1641		if (offset < skb->len) {
1642			int used;
1643			size_t len;
1644
1645			len = skb->len - offset;
1646			/* Stop reading if we hit a patch of urgent data */
1647			if (unlikely(tp->urg_data)) {
1648				u32 urg_offset = tp->urg_seq - seq;
1649				if (urg_offset < len)
1650					len = urg_offset;
1651				if (!len)
1652					break;
1653			}
1654			used = recv_actor(desc, skb, offset, len);
1655			if (used <= 0) {
1656				if (!copied)
1657					copied = used;
1658				break;
1659			}
1660			if (WARN_ON_ONCE(used > len))
1661				used = len;
1662			seq += used;
1663			copied += used;
1664			offset += used;
1665
1666			/* If recv_actor drops the lock (e.g. TCP splice
1667			 * receive) the skb pointer might be invalid when
1668			 * getting here: tcp_collapse might have deleted it
1669			 * while aggregating skbs from the socket queue.
1670			 */
1671			skb = tcp_recv_skb(sk, seq - 1, &offset);
1672			if (!skb)
1673				break;
1674			/* TCP coalescing might have appended data to the skb.
1675			 * Try to splice more frags
1676			 */
1677			if (offset + 1 != skb->len)
1678				continue;
1679		}
1680		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1681			tcp_eat_recv_skb(sk, skb);
1682			++seq;
1683			break;
1684		}
1685		tcp_eat_recv_skb(sk, skb);
1686		if (!desc->count)
1687			break;
1688		WRITE_ONCE(*copied_seq, seq);
1689	}
1690	WRITE_ONCE(*copied_seq, seq);
1691
1692	if (noack)
1693		goto out;
1694
1695	tcp_rcv_space_adjust(sk);
1696
1697	/* Clean up data we have read: This will do ACK frames. */
1698	if (copied > 0) {
1699		tcp_recv_skb(sk, seq, &offset);
1700		tcp_cleanup_rbuf(sk, copied);
1701	}
1702out:
1703	return copied;
1704}
1705
1706int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1707		  sk_read_actor_t recv_actor)
1708{
1709	return __tcp_read_sock(sk, desc, recv_actor, false,
1710			       &tcp_sk(sk)->copied_seq);
1711}
1712EXPORT_SYMBOL(tcp_read_sock);
1713
1714int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
1715			sk_read_actor_t recv_actor, bool noack,
1716			u32 *copied_seq)
1717{
1718	return __tcp_read_sock(sk, desc, recv_actor, noack, copied_seq);
1719}
1720
1721int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
1722{
1723	struct sk_buff *skb;
1724	int copied = 0;
1725
1726	if (sk->sk_state == TCP_LISTEN)
1727		return -ENOTCONN;
1728
1729	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1730		u8 tcp_flags;
1731		int used;
1732
1733		__skb_unlink(skb, &sk->sk_receive_queue);
1734		WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1735		tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
1736		used = recv_actor(sk, skb);
1737		if (used < 0) {
1738			if (!copied)
1739				copied = used;
1740			break;
1741		}
1742		copied += used;
1743
1744		if (tcp_flags & TCPHDR_FIN)
1745			break;
1746	}
1747	return copied;
1748}
1749EXPORT_IPV6_MOD(tcp_read_skb);
1750
1751void tcp_read_done(struct sock *sk, size_t len)
1752{
1753	struct tcp_sock *tp = tcp_sk(sk);
1754	u32 seq = tp->copied_seq;
1755	struct sk_buff *skb;
1756	size_t left;
1757	u32 offset;
1758
1759	if (sk->sk_state == TCP_LISTEN)
1760		return;
1761
1762	left = len;
1763	while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1764		int used;
1765
1766		used = min_t(size_t, skb->len - offset, left);
1767		seq += used;
1768		left -= used;
1769
1770		if (skb->len > offset + used)
1771			break;
1772
1773		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1774			tcp_eat_recv_skb(sk, skb);
1775			++seq;
1776			break;
1777		}
1778		tcp_eat_recv_skb(sk, skb);
1779	}
1780	WRITE_ONCE(tp->copied_seq, seq);
1781
1782	tcp_rcv_space_adjust(sk);
1783
1784	/* Clean up data we have read: This will do ACK frames. */
1785	if (left != len)
1786		tcp_cleanup_rbuf(sk, len - left);
1787}
1788EXPORT_SYMBOL(tcp_read_done);
1789
1790int tcp_peek_len(struct socket *sock)
1791{
1792	return tcp_inq(sock->sk);
1793}
1794EXPORT_IPV6_MOD(tcp_peek_len);
1795
1796/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */
1797int tcp_set_rcvlowat(struct sock *sk, int val)
1798{
1799	struct tcp_sock *tp = tcp_sk(sk);
1800	int space, cap;
1801
1802	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1803		cap = sk->sk_rcvbuf >> 1;
1804	else
1805		cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
1806	val = min(val, cap);
1807	WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1808
1809	/* Check if we need to signal EPOLLIN right now */
1810	tcp_data_ready(sk);
1811
1812	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1813		return 0;
1814
1815	space = tcp_space_from_win(sk, val);
1816	if (space > sk->sk_rcvbuf) {
1817		WRITE_ONCE(sk->sk_rcvbuf, space);
1818
1819		if (tp->window_clamp && tp->window_clamp < val)
1820			WRITE_ONCE(tp->window_clamp, val);
1821	}
1822	return 0;
1823}
1824EXPORT_IPV6_MOD(tcp_set_rcvlowat);
1825
1826void tcp_update_recv_tstamps(struct sk_buff *skb,
1827			     struct scm_timestamping_internal *tss)
1828{
1829	if (skb->tstamp)
1830		tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1831	else
1832		tss->ts[0] = (struct timespec64) {0};
1833
1834	if (skb_hwtstamps(skb)->hwtstamp)
1835		tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1836	else
1837		tss->ts[2] = (struct timespec64) {0};
1838}
1839
1840#ifdef CONFIG_MMU
1841static const struct vm_operations_struct tcp_vm_ops = {
1842};
1843
1844int tcp_mmap(struct file *file, struct socket *sock,
1845	     struct vm_area_struct *vma)
1846{
1847	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1848		return -EPERM;
1849	vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC);
1850
1851	/* Instruct vm_insert_page() to not mmap_read_lock(mm) */
1852	vm_flags_set(vma, VM_MIXEDMAP);
1853
1854	vma->vm_ops = &tcp_vm_ops;
1855	return 0;
1856}
1857EXPORT_IPV6_MOD(tcp_mmap);
1858
1859static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
1860				       u32 *offset_frag)
1861{
1862	skb_frag_t *frag;
1863
1864	if (unlikely(offset_skb >= skb->len))
1865		return NULL;
1866
1867	offset_skb -= skb_headlen(skb);
1868	if ((int)offset_skb < 0 || skb_has_frag_list(skb))
1869		return NULL;
1870
1871	frag = skb_shinfo(skb)->frags;
1872	while (offset_skb) {
1873		if (skb_frag_size(frag) > offset_skb) {
1874			*offset_frag = offset_skb;
1875			return frag;
1876		}
1877		offset_skb -= skb_frag_size(frag);
1878		++frag;
1879	}
1880	*offset_frag = 0;
1881	return frag;
1882}
1883
1884static bool can_map_frag(const skb_frag_t *frag)
1885{
1886	struct page *page;
1887
1888	if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag))
1889		return false;
1890
1891	page = skb_frag_page(frag);
1892
1893	if (PageCompound(page) || page->mapping)
1894		return false;
1895
1896	return true;
1897}
1898
1899static int find_next_mappable_frag(const skb_frag_t *frag,
1900				   int remaining_in_skb)
1901{
1902	int offset = 0;
1903
1904	if (likely(can_map_frag(frag)))
1905		return 0;
1906
1907	while (offset < remaining_in_skb && !can_map_frag(frag)) {
1908		offset += skb_frag_size(frag);
1909		++frag;
1910	}
1911	return offset;
1912}
1913
1914static void tcp_zerocopy_set_hint_for_skb(struct sock *sk,
1915					  struct tcp_zerocopy_receive *zc,
1916					  struct sk_buff *skb, u32 offset)
1917{
1918	u32 frag_offset, partial_frag_remainder = 0;
1919	int mappable_offset;
1920	skb_frag_t *frag;
1921
1922	/* worst case: skip to next skb. try to improve on this case below */
1923	zc->recv_skip_hint = skb->len - offset;
1924
1925	/* Find the frag containing this offset (and how far into that frag) */
1926	frag = skb_advance_to_frag(skb, offset, &frag_offset);
1927	if (!frag)
1928		return;
1929
1930	if (frag_offset) {
1931		struct skb_shared_info *info = skb_shinfo(skb);
1932
1933		/* We read part of the last frag, must recvmsg() rest of skb. */
1934		if (frag == &info->frags[info->nr_frags - 1])
1935			return;
1936
1937		/* Else, we must at least read the remainder in this frag. */
1938		partial_frag_remainder = skb_frag_size(frag) - frag_offset;
1939		zc->recv_skip_hint -= partial_frag_remainder;
1940		++frag;
1941	}
1942
1943	/* partial_frag_remainder: If part way through a frag, must read rest.
1944	 * mappable_offset: Bytes till next mappable frag, *not* counting bytes
1945	 * in partial_frag_remainder.
1946	 */
1947	mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);
1948	zc->recv_skip_hint = mappable_offset + partial_frag_remainder;
1949}
1950
1951static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
1952			      int flags, struct scm_timestamping_internal *tss,
1953			      int *cmsg_flags);
1954static int receive_fallback_to_copy(struct sock *sk,
1955				    struct tcp_zerocopy_receive *zc, int inq,
1956				    struct scm_timestamping_internal *tss)
1957{
1958	unsigned long copy_address = (unsigned long)zc->copybuf_address;
1959	struct msghdr msg = {};
1960	int err;
1961
1962	zc->length = 0;
1963	zc->recv_skip_hint = 0;
1964
1965	if (copy_address != zc->copybuf_address)
1966		return -EINVAL;
1967
1968	err = import_ubuf(ITER_DEST, (void __user *)copy_address, inq,
1969			  &msg.msg_iter);
1970	if (err)
1971		return err;
1972
1973	err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT,
1974				 tss, &zc->msg_flags);
1975	if (err < 0)
1976		return err;
1977
1978	zc->copybuf_len = err;
1979	if (likely(zc->copybuf_len)) {
1980		struct sk_buff *skb;
1981		u32 offset;
1982
1983		skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset);
1984		if (skb)
1985			tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);
1986	}
1987	return 0;
1988}
1989
1990static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
1991				   struct sk_buff *skb, u32 copylen,
1992				   u32 *offset, u32 *seq)
1993{
1994	unsigned long copy_address = (unsigned long)zc->copybuf_address;
1995	struct msghdr msg = {};
1996	int err;
1997
1998	if (copy_address != zc->copybuf_address)
1999		return -EINVAL;
2000
2001	err = import_ubuf(ITER_DEST, (void __user *)copy_address, copylen,
2002			  &msg.msg_iter);
2003	if (err)
2004		return err;
2005	err = skb_copy_datagram_msg(skb, *offset, &msg, copylen);
2006	if (err)
2007		return err;
2008	zc->recv_skip_hint -= copylen;
2009	*offset += copylen;
2010	*seq += copylen;
2011	return (__s32)copylen;
2012}
2013
2014static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,
2015				  struct sock *sk,
2016				  struct sk_buff *skb,
2017				  u32 *seq,
2018				  s32 copybuf_len,
2019				  struct scm_timestamping_internal *tss)
2020{
2021	u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
2022
2023	if (!copylen)
2024		return 0;
2025	/* skb is null if inq < PAGE_SIZE. */
2026	if (skb) {
2027		offset = *seq - TCP_SKB_CB(skb)->seq;
2028	} else {
2029		skb = tcp_recv_skb(sk, *seq, &offset);
2030		if (TCP_SKB_CB(skb)->has_rxtstamp) {
2031			tcp_update_recv_tstamps(skb, tss);
2032			zc->msg_flags |= TCP_CMSG_TS;
2033		}
2034	}
2035
2036	zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,
2037						  seq);
2038	return zc->copybuf_len < 0 ? 0 : copylen;
2039}
2040
2041static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma,
2042					      struct page **pending_pages,
2043					      unsigned long pages_remaining,
2044					      unsigned long *address,
2045					      u32 *length,
2046					      u32 *seq,
2047					      struct tcp_zerocopy_receive *zc,
2048					      u32 total_bytes_to_map,
2049					      int err)
2050{
2051	/* At least one page did not map. Try zapping if we skipped earlier. */
2052	if (err == -EBUSY &&
2053	    zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {
2054		u32 maybe_zap_len;
2055
2056		maybe_zap_len = total_bytes_to_map -  /* All bytes to map */
2057				*length + /* Mapped or pending */
2058				(pages_remaining * PAGE_SIZE); /* Failed map. */
2059		zap_page_range_single(vma, *address, maybe_zap_len, NULL);
2060		err = 0;
2061	}
2062
2063	if (!err) {
2064		unsigned long leftover_pages = pages_remaining;
2065		int bytes_mapped;
2066
2067		/* We called zap_page_range_single, try to reinsert. */
2068		err = vm_insert_pages(vma, *address,
2069				      pending_pages,
2070				      &pages_remaining);
2071		bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining);
2072		*seq += bytes_mapped;
2073		*address += bytes_mapped;
2074	}
2075	if (err) {
2076		/* Either we were unable to zap, OR we zapped, retried an
2077		 * insert, and still had an issue. Either ways, pages_remaining
2078		 * is the number of pages we were unable to map, and we unroll
2079		 * some state we speculatively touched before.
2080		 */
2081		const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
2082
2083		*length -= bytes_not_mapped;
2084		zc->recv_skip_hint += bytes_not_mapped;
2085	}
2086	return err;
2087}
2088
2089static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
2090					struct page **pages,
2091					unsigned int pages_to_map,
2092					unsigned long *address,
2093					u32 *length,
2094					u32 *seq,
2095					struct tcp_zerocopy_receive *zc,
2096					u32 total_bytes_to_map)
2097{
2098	unsigned long pages_remaining = pages_to_map;
2099	unsigned int pages_mapped;
2100	unsigned int bytes_mapped;
2101	int err;
2102
2103	err = vm_insert_pages(vma, *address, pages, &pages_remaining);
2104	pages_mapped = pages_to_map - (unsigned int)pages_remaining;
2105	bytes_mapped = PAGE_SIZE * pages_mapped;
2106	/* Even if vm_insert_pages fails, it may have partially succeeded in
2107	 * mapping (some but not all of the pages).
2108	 */
2109	*seq += bytes_mapped;
2110	*address += bytes_mapped;
2111
2112	if (likely(!err))
2113		return 0;
2114
2115	/* Error: maybe zap and retry + rollback state for failed inserts. */
2116	return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped,
2117		pages_remaining, address, length, seq, zc, total_bytes_to_map,
2118		err);
2119}
2120
2121#define TCP_VALID_ZC_MSG_FLAGS   (TCP_CMSG_TS)
2122static void tcp_zc_finalize_rx_tstamp(struct sock *sk,
2123				      struct tcp_zerocopy_receive *zc,
2124				      struct scm_timestamping_internal *tss)
2125{
2126	unsigned long msg_control_addr;
2127	struct msghdr cmsg_dummy;
2128
2129	msg_control_addr = (unsigned long)zc->msg_control;
2130	cmsg_dummy.msg_control_user = (void __user *)msg_control_addr;
2131	cmsg_dummy.msg_controllen =
2132		(__kernel_size_t)zc->msg_controllen;
2133	cmsg_dummy.msg_flags = in_compat_syscall()
2134		? MSG_CMSG_COMPAT : 0;
2135	cmsg_dummy.msg_control_is_user = true;
2136	zc->msg_flags = 0;
2137	if (zc->msg_control == msg_control_addr &&
2138	    zc->msg_controllen == cmsg_dummy.msg_controllen) {
2139		tcp_recv_timestamp(&cmsg_dummy, sk, tss);
2140		zc->msg_control = (__u64)
2141			((uintptr_t)cmsg_dummy.msg_control_user);
2142		zc->msg_controllen =
2143			(__u64)cmsg_dummy.msg_controllen;
2144		zc->msg_flags = (__u32)cmsg_dummy.msg_flags;
2145	}
2146}
2147
2148static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm,
2149					   unsigned long address,
2150					   bool *mmap_locked)
2151{
2152	struct vm_area_struct *vma = lock_vma_under_rcu(mm, address);
2153
2154	if (vma) {
2155		if (vma->vm_ops != &tcp_vm_ops) {
2156			vma_end_read(vma);
2157			return NULL;
2158		}
2159		*mmap_locked = false;
2160		return vma;
2161	}
2162
2163	mmap_read_lock(mm);
2164	vma = vma_lookup(mm, address);
2165	if (!vma || vma->vm_ops != &tcp_vm_ops) {
2166		mmap_read_unlock(mm);
2167		return NULL;
2168	}
2169	*mmap_locked = true;
2170	return vma;
2171}
2172
2173#define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32
2174static int tcp_zerocopy_receive(struct sock *sk,
2175				struct tcp_zerocopy_receive *zc,
2176				struct scm_timestamping_internal *tss)
2177{
2178	u32 length = 0, offset, vma_len, avail_len, copylen = 0;
2179	unsigned long address = (unsigned long)zc->address;
2180	struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE];
2181	s32 copybuf_len = zc->copybuf_len;
2182	struct tcp_sock *tp = tcp_sk(sk);
2183	const skb_frag_t *frags = NULL;
2184	unsigned int pages_to_map = 0;
2185	struct vm_area_struct *vma;
2186	struct sk_buff *skb = NULL;
2187	u32 seq = tp->copied_seq;
2188	u32 total_bytes_to_map;
2189	int inq = tcp_inq(sk);
2190	bool mmap_locked;
2191	int ret;
2192
2193	zc->copybuf_len = 0;
2194	zc->msg_flags = 0;
2195
2196	if (address & (PAGE_SIZE - 1) || address != zc->address)
2197		return -EINVAL;
2198
2199	if (sk->sk_state == TCP_LISTEN)
2200		return -ENOTCONN;
2201
2202	sock_rps_record_flow(sk);
2203
2204	if (inq && inq <= copybuf_len)
2205		return receive_fallback_to_copy(sk, zc, inq, tss);
2206
2207	if (inq < PAGE_SIZE) {
2208		zc->length = 0;
2209		zc->recv_skip_hint = inq;
2210		if (!inq && sock_flag(sk, SOCK_DONE))
2211			return -EIO;
2212		return 0;
2213	}
2214
2215	vma = find_tcp_vma(current->mm, address, &mmap_locked);
2216	if (!vma)
2217		return -EINVAL;
2218
2219	vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
2220	avail_len = min_t(u32, vma_len, inq);
2221	total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1);
2222	if (total_bytes_to_map) {
2223		if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))
2224			zap_page_range_single(vma, address, total_bytes_to_map,
2225					      NULL);
2226		zc->length = total_bytes_to_map;
2227		zc->recv_skip_hint = 0;
2228	} else {
2229		zc->length = avail_len;
2230		zc->recv_skip_hint = avail_len;
2231	}
2232	ret = 0;
2233	while (length + PAGE_SIZE <= zc->length) {
2234		int mappable_offset;
2235		struct page *page;
2236
2237		if (zc->recv_skip_hint < PAGE_SIZE) {
2238			u32 offset_frag;
2239
2240			if (skb) {
2241				if (zc->recv_skip_hint > 0)
2242					break;
2243				skb = skb->next;
2244				offset = seq - TCP_SKB_CB(skb)->seq;
2245			} else {
2246				skb = tcp_recv_skb(sk, seq, &offset);
2247			}
2248
2249			if (!skb_frags_readable(skb))
2250				break;
2251
2252			if (TCP_SKB_CB(skb)->has_rxtstamp) {
2253				tcp_update_recv_tstamps(skb, tss);
2254				zc->msg_flags |= TCP_CMSG_TS;
2255			}
2256			zc->recv_skip_hint = skb->len - offset;
2257			frags = skb_advance_to_frag(skb, offset, &offset_frag);
2258			if (!frags || offset_frag)
2259				break;
2260		}
2261
2262		mappable_offset = find_next_mappable_frag(frags,
2263							  zc->recv_skip_hint);
2264		if (mappable_offset) {
2265			zc->recv_skip_hint = mappable_offset;
2266			break;
2267		}
2268		page = skb_frag_page(frags);
2269		if (WARN_ON_ONCE(!page))
2270			break;
2271
2272		prefetchw(page);
2273		pages[pages_to_map++] = page;
2274		length += PAGE_SIZE;
2275		zc->recv_skip_hint -= PAGE_SIZE;
2276		frags++;
2277		if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE ||
2278		    zc->recv_skip_hint < PAGE_SIZE) {
2279			/* Either full batch, or we're about to go to next skb
2280			 * (and we cannot unroll failed ops across skbs).
2281			 */
2282			ret = tcp_zerocopy_vm_insert_batch(vma, pages,
2283							   pages_to_map,
2284							   &address, &length,
2285							   &seq, zc,
2286							   total_bytes_to_map);
2287			if (ret)
2288				goto out;
2289			pages_to_map = 0;
2290		}
2291	}
2292	if (pages_to_map) {
2293		ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map,
2294						   &address, &length, &seq,
2295						   zc, total_bytes_to_map);
2296	}
2297out:
2298	if (mmap_locked)
2299		mmap_read_unlock(current->mm);
2300	else
2301		vma_end_read(vma);
2302	/* Try to copy straggler data. */
2303	if (!ret)
2304		copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);
2305
2306	if (length + copylen) {
2307		WRITE_ONCE(tp->copied_seq, seq);
2308		tcp_rcv_space_adjust(sk);
2309
2310		/* Clean up data we have read: This will do ACK frames. */
2311		tcp_recv_skb(sk, seq, &offset);
2312		tcp_cleanup_rbuf(sk, length + copylen);
2313		ret = 0;
2314		if (length == zc->length)
2315			zc->recv_skip_hint = 0;
2316	} else {
2317		if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
2318			ret = -EIO;
2319	}
2320	zc->length = length;
2321	return ret;
2322}
2323#endif
2324
2325/* Similar to __sock_recv_timestamp, but does not require an skb */
2326void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
2327			struct scm_timestamping_internal *tss)
2328{
2329	int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
2330	u32 tsflags = READ_ONCE(sk->sk_tsflags);
2331	bool has_timestamping = false;
2332
2333	if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
2334		if (sock_flag(sk, SOCK_RCVTSTAMP)) {
2335			if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
2336				if (new_tstamp) {
2337					struct __kernel_timespec kts = {
2338						.tv_sec = tss->ts[0].tv_sec,
2339						.tv_nsec = tss->ts[0].tv_nsec,
2340					};
2341					put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
2342						 sizeof(kts), &kts);
2343				} else {
2344					struct __kernel_old_timespec ts_old = {
2345						.tv_sec = tss->ts[0].tv_sec,
2346						.tv_nsec = tss->ts[0].tv_nsec,
2347					};
2348					put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
2349						 sizeof(ts_old), &ts_old);
2350				}
2351			} else {
2352				if (new_tstamp) {
2353					struct __kernel_sock_timeval stv = {
2354						.tv_sec = tss->ts[0].tv_sec,
2355						.tv_usec = tss->ts[0].tv_nsec / 1000,
2356					};
2357					put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
2358						 sizeof(stv), &stv);
2359				} else {
2360					struct __kernel_old_timeval tv = {
2361						.tv_sec = tss->ts[0].tv_sec,
2362						.tv_usec = tss->ts[0].tv_nsec / 1000,
2363					};
2364					put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
2365						 sizeof(tv), &tv);
2366				}
2367			}
2368		}
2369
2370		if (tsflags & SOF_TIMESTAMPING_SOFTWARE &&
2371		    (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE ||
2372		     !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
2373			has_timestamping = true;
2374		else
2375			tss->ts[0] = (struct timespec64) {0};
2376	}
2377
2378	if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
2379		if (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE &&
2380		    (tsflags & SOF_TIMESTAMPING_RX_HARDWARE ||
2381		     !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
2382			has_timestamping = true;
2383		else
2384			tss->ts[2] = (struct timespec64) {0};
2385	}
2386
2387	if (has_timestamping) {
2388		tss->ts[1] = (struct timespec64) {0};
2389		if (sock_flag(sk, SOCK_TSTAMP_NEW))
2390			put_cmsg_scm_timestamping64(msg, tss);
2391		else
2392			put_cmsg_scm_timestamping(msg, tss);
2393	}
2394}
2395
2396static int tcp_inq_hint(struct sock *sk)
2397{
2398	const struct tcp_sock *tp = tcp_sk(sk);
2399	u32 copied_seq = READ_ONCE(tp->copied_seq);
2400	u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
2401	int inq;
2402
2403	inq = rcv_nxt - copied_seq;
2404	if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
2405		lock_sock(sk);
2406		inq = tp->rcv_nxt - tp->copied_seq;
2407		release_sock(sk);
2408	}
2409	/* After receiving a FIN, tell the user-space to continue reading
2410	 * by returning a non-zero inq.
2411	 */
2412	if (inq == 0 && sock_flag(sk, SOCK_DONE))
2413		inq = 1;
2414	return inq;
2415}
2416
2417/* batch __xa_alloc() calls and reduce xa_lock()/xa_unlock() overhead. */
2418struct tcp_xa_pool {
2419	u8		max; /* max <= MAX_SKB_FRAGS */
2420	u8		idx; /* idx <= max */
2421	__u32		tokens[MAX_SKB_FRAGS];
2422	netmem_ref	netmems[MAX_SKB_FRAGS];
2423};
2424
2425static void tcp_xa_pool_commit_locked(struct sock *sk, struct tcp_xa_pool *p)
2426{
2427	int i;
2428
2429	/* Commit part that has been copied to user space. */
2430	for (i = 0; i < p->idx; i++)
2431		__xa_cmpxchg(&sk->sk_user_frags, p->tokens[i], XA_ZERO_ENTRY,
2432			     (__force void *)p->netmems[i], GFP_KERNEL);
2433	/* Rollback what has been pre-allocated and is no longer needed. */
2434	for (; i < p->max; i++)
2435		__xa_erase(&sk->sk_user_frags, p->tokens[i]);
2436
2437	p->max = 0;
2438	p->idx = 0;
2439}
2440
2441static void tcp_xa_pool_commit(struct sock *sk, struct tcp_xa_pool *p)
2442{
2443	if (!p->max)
2444		return;
2445
2446	xa_lock_bh(&sk->sk_user_frags);
2447
2448	tcp_xa_pool_commit_locked(sk, p);
2449
2450	xa_unlock_bh(&sk->sk_user_frags);
2451}
2452
2453static int tcp_xa_pool_refill(struct sock *sk, struct tcp_xa_pool *p,
2454			      unsigned int max_frags)
2455{
2456	int err, k;
2457
2458	if (p->idx < p->max)
2459		return 0;
2460
2461	xa_lock_bh(&sk->sk_user_frags);
2462
2463	tcp_xa_pool_commit_locked(sk, p);
2464
2465	for (k = 0; k < max_frags; k++) {
2466		err = __xa_alloc(&sk->sk_user_frags, &p->tokens[k],
2467				 XA_ZERO_ENTRY, xa_limit_31b, GFP_KERNEL);
2468		if (err)
2469			break;
2470	}
2471
2472	xa_unlock_bh(&sk->sk_user_frags);
2473
2474	p->max = k;
2475	p->idx = 0;
2476	return k ? 0 : err;
2477}
2478
2479/* On error, returns the -errno. On success, returns number of bytes sent to the
2480 * user. May not consume all of @remaining_len.
2481 */
2482static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
2483			      unsigned int offset, struct msghdr *msg,
2484			      int remaining_len)
2485{
2486	struct dmabuf_cmsg dmabuf_cmsg = { 0 };
2487	struct tcp_xa_pool tcp_xa_pool;
2488	unsigned int start;
2489	int i, copy, n;
2490	int sent = 0;
2491	int err = 0;
2492
2493	tcp_xa_pool.max = 0;
2494	tcp_xa_pool.idx = 0;
2495	do {
2496		start = skb_headlen(skb);
2497
2498		if (skb_frags_readable(skb)) {
2499			err = -ENODEV;
2500			goto out;
2501		}
2502
2503		/* Copy header. */
2504		copy = start - offset;
2505		if (copy > 0) {
2506			copy = min(copy, remaining_len);
2507
2508			n = copy_to_iter(skb->data + offset, copy,
2509					 &msg->msg_iter);
2510			if (n != copy) {
2511				err = -EFAULT;
2512				goto out;
2513			}
2514
2515			offset += copy;
2516			remaining_len -= copy;
2517
2518			/* First a dmabuf_cmsg for # bytes copied to user
2519			 * buffer.
2520			 */
2521			memset(&dmabuf_cmsg, 0, sizeof(dmabuf_cmsg));
2522			dmabuf_cmsg.frag_size = copy;
2523			err = put_cmsg_notrunc(msg, SOL_SOCKET,
2524					       SO_DEVMEM_LINEAR,
2525					       sizeof(dmabuf_cmsg),
2526					       &dmabuf_cmsg);
2527			if (err)
2528				goto out;
2529
2530			sent += copy;
2531
2532			if (remaining_len == 0)
2533				goto out;
2534		}
2535
2536		/* after that, send information of dmabuf pages through a
2537		 * sequence of cmsg
2538		 */
2539		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2540			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2541			struct net_iov *niov;
2542			u64 frag_offset;
2543			int end;
2544
2545			/* !skb_frags_readable() should indicate that ALL the
2546			 * frags in this skb are dmabuf net_iovs. We're checking
2547			 * for that flag above, but also check individual frags
2548			 * here. If the tcp stack is not setting
2549			 * skb_frags_readable() correctly, we still don't want
2550			 * to crash here.
2551			 */
2552			if (!skb_frag_net_iov(frag)) {
2553				net_err_ratelimited("Found non-dmabuf skb with net_iov");
2554				err = -ENODEV;
2555				goto out;
2556			}
2557
2558			niov = skb_frag_net_iov(frag);
2559			if (!net_is_devmem_iov(niov)) {
2560				err = -ENODEV;
2561				goto out;
2562			}
2563
2564			end = start + skb_frag_size(frag);
2565			copy = end - offset;
2566
2567			if (copy > 0) {
2568				copy = min(copy, remaining_len);
2569
2570				frag_offset = net_iov_virtual_addr(niov) +
2571					      skb_frag_off(frag) + offset -
2572					      start;
2573				dmabuf_cmsg.frag_offset = frag_offset;
2574				dmabuf_cmsg.frag_size = copy;
2575				err = tcp_xa_pool_refill(sk, &tcp_xa_pool,
2576							 skb_shinfo(skb)->nr_frags - i);
2577				if (err)
2578					goto out;
2579
2580				/* Will perform the exchange later */
2581				dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx];
2582				dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov);
2583
2584				offset += copy;
2585				remaining_len -= copy;
2586
2587				err = put_cmsg_notrunc(msg, SOL_SOCKET,
2588						       SO_DEVMEM_DMABUF,
2589						       sizeof(dmabuf_cmsg),
2590						       &dmabuf_cmsg);
2591				if (err)
2592					goto out;
2593
2594				atomic_long_inc(&niov->desc.pp_ref_count);
2595				tcp_xa_pool.netmems[tcp_xa_pool.idx++] = skb_frag_netmem(frag);
2596
2597				sent += copy;
2598
2599				if (remaining_len == 0)
2600					goto out;
2601			}
2602			start = end;
2603		}
2604
2605		tcp_xa_pool_commit(sk, &tcp_xa_pool);
2606		if (!remaining_len)
2607			goto out;
2608
2609		/* if remaining_len is not satisfied yet, we need to go to the
2610		 * next frag in the frag_list to satisfy remaining_len.
2611		 */
2612		skb = skb_shinfo(skb)->frag_list ?: skb->next;
2613
2614		offset = offset - start;
2615	} while (skb);
2616
2617	if (remaining_len) {
2618		err = -EFAULT;
2619		goto out;
2620	}
2621
2622out:
2623	tcp_xa_pool_commit(sk, &tcp_xa_pool);
2624	if (!sent)
2625		sent = err;
2626
2627	return sent;
2628}
2629
2630/*
2631 *	This routine copies from a sock struct into the user buffer.
2632 *
2633 *	Technical note: in 2.3 we work on _locked_ socket, so that
2634 *	tricks with *seq access order and skb->users are not required.
2635 *	Probably, code can be easily improved even more.
2636 */
2637
2638static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
2639			      int flags, struct scm_timestamping_internal *tss,
2640			      int *cmsg_flags)
2641{
2642	struct tcp_sock *tp = tcp_sk(sk);
2643	int last_copied_dmabuf = -1; /* uninitialized */
2644	int copied = 0;
2645	u32 peek_seq;
2646	u32 *seq;
2647	unsigned long used;
2648	int err;
2649	int target;		/* Read at least this many bytes */
2650	long timeo;
2651	struct sk_buff *skb, *last;
2652	u32 peek_offset = 0;
2653	u32 urg_hole = 0;
2654
2655	err = -ENOTCONN;
2656	if (sk->sk_state == TCP_LISTEN)
2657		goto out;
2658
2659	if (tp->recvmsg_inq)
2660		*cmsg_flags = TCP_CMSG_INQ;
2661	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
2662
2663	/* Urgent data needs to be handled specially. */
2664	if (flags & MSG_OOB)
2665		goto recv_urg;
2666
2667	if (unlikely(tp->repair)) {
2668		err = -EPERM;
2669		if (!(flags & MSG_PEEK))
2670			goto out;
2671
2672		if (tp->repair_queue == TCP_SEND_QUEUE)
2673			goto recv_sndq;
2674
2675		err = -EINVAL;
2676		if (tp->repair_queue == TCP_NO_QUEUE)
2677			goto out;
2678
2679		/* 'common' recv queue MSG_PEEK-ing */
2680	}
2681
2682	seq = &tp->copied_seq;
2683	if (flags & MSG_PEEK) {
2684		peek_offset = max(sk_peek_offset(sk, flags), 0);
2685		peek_seq = tp->copied_seq + peek_offset;
2686		seq = &peek_seq;
2687	}
2688
2689	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2690
2691	do {
2692		u32 offset;
2693
2694		/* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
2695		if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {
2696			if (copied)
2697				break;
2698			if (signal_pending(current)) {
2699				copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2700				break;
2701			}
2702		}
2703
2704		/* Next get a buffer. */
2705
2706		last = skb_peek_tail(&sk->sk_receive_queue);
2707		skb_queue_walk(&sk->sk_receive_queue, skb) {
2708			last = skb;
2709			/* Now that we have two receive queues this
2710			 * shouldn't happen.
2711			 */
2712			if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
2713				 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2714				 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2715				 flags))
2716				break;
2717
2718			offset = *seq - TCP_SKB_CB(skb)->seq;
2719			if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2720				pr_err_once("%s: found a SYN, please report !\n", __func__);
2721				offset--;
2722			}
2723			if (offset < skb->len)
2724				goto found_ok_skb;
2725			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2726				goto found_fin_ok;
2727			WARN(!(flags & MSG_PEEK),
2728			     "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2729			     *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
2730		}
2731
2732		/* Well, if we have backlog, try to process it now yet. */
2733
2734		if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
2735			break;
2736
2737		if (copied) {
2738			if (!timeo ||
2739			    sk->sk_err ||
2740			    sk->sk_state == TCP_CLOSE ||
2741			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
2742			    signal_pending(current))
2743				break;
2744		} else {
2745			if (sock_flag(sk, SOCK_DONE))
2746				break;
2747
2748			if (sk->sk_err) {
2749				copied = sock_error(sk);
2750				break;
2751			}
2752
2753			if (sk->sk_shutdown & RCV_SHUTDOWN)
2754				break;
2755
2756			if (sk->sk_state == TCP_CLOSE) {
2757				/* This occurs when user tries to read
2758				 * from never connected socket.
2759				 */
2760				copied = -ENOTCONN;
2761				break;
2762			}
2763
2764			if (!timeo) {
2765				copied = -EAGAIN;
2766				break;
2767			}
2768
2769			if (signal_pending(current)) {
2770				copied = sock_intr_errno(timeo);
2771				break;
2772			}
2773		}
2774
2775		if (copied >= target) {
2776			/* Do not sleep, just process backlog. */
2777			__sk_flush_backlog(sk);
2778		} else {
2779			tcp_cleanup_rbuf(sk, copied);
2780			err = sk_wait_data(sk, &timeo, last);
2781			if (err < 0) {
2782				err = copied ? : err;
2783				goto out;
2784			}
2785		}
2786
2787		if ((flags & MSG_PEEK) &&
2788		    (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) {
2789			net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2790					    current->comm,
2791					    task_pid_nr(current));
2792			peek_seq = tp->copied_seq + peek_offset;
2793		}
2794		continue;
2795
2796found_ok_skb:
2797		/* Ok so how much can we use? */
2798		used = skb->len - offset;
2799		if (len < used)
2800			used = len;
2801
2802		/* Do we have urgent data here? */
2803		if (unlikely(tp->urg_data)) {
2804			u32 urg_offset = tp->urg_seq - *seq;
2805			if (urg_offset < used) {
2806				if (!urg_offset) {
2807					if (!sock_flag(sk, SOCK_URGINLINE)) {
2808						WRITE_ONCE(*seq, *seq + 1);
2809						urg_hole++;
2810						offset++;
2811						used--;
2812						if (!used)
2813							goto skip_copy;
2814					}
2815				} else
2816					used = urg_offset;
2817			}
2818		}
2819
2820		if (!(flags & MSG_TRUNC)) {
2821			if (last_copied_dmabuf != -1 &&
2822			    last_copied_dmabuf != !skb_frags_readable(skb))
2823				break;
2824
2825			if (skb_frags_readable(skb)) {
2826				err = skb_copy_datagram_msg(skb, offset, msg,
2827							    used);
2828				if (err) {
2829					/* Exception. Bailout! */
2830					if (!copied)
2831						copied = -EFAULT;
2832					break;
2833				}
2834			} else {
2835				if (!(flags & MSG_SOCK_DEVMEM)) {
2836					/* dmabuf skbs can only be received
2837					 * with the MSG_SOCK_DEVMEM flag.
2838					 */
2839					if (!copied)
2840						copied = -EFAULT;
2841
2842					break;
2843				}
2844
2845				err = tcp_recvmsg_dmabuf(sk, skb, offset, msg,
2846							 used);
2847				if (err < 0) {
2848					if (!copied)
2849						copied = err;
2850
2851					break;
2852				}
2853				used = err;
2854			}
2855		}
2856
2857		last_copied_dmabuf = !skb_frags_readable(skb);
2858
2859		WRITE_ONCE(*seq, *seq + used);
2860		copied += used;
2861		len -= used;
2862		if (flags & MSG_PEEK)
2863			sk_peek_offset_fwd(sk, used);
2864		else
2865			sk_peek_offset_bwd(sk, used);
2866		tcp_rcv_space_adjust(sk);
2867
2868skip_copy:
2869		if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {
2870			WRITE_ONCE(tp->urg_data, 0);
2871			tcp_fast_path_check(sk);
2872		}
2873
2874		if (TCP_SKB_CB(skb)->has_rxtstamp) {
2875			tcp_update_recv_tstamps(skb, tss);
2876			*cmsg_flags |= TCP_CMSG_TS;
2877		}
2878
2879		if (used + offset < skb->len)
2880			continue;
2881
2882		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2883			goto found_fin_ok;
2884		if (!(flags & MSG_PEEK))
2885			tcp_eat_recv_skb(sk, skb);
2886		continue;
2887
2888found_fin_ok:
2889		/* Process the FIN. */
2890		WRITE_ONCE(*seq, *seq + 1);
2891		if (!(flags & MSG_PEEK))
2892			tcp_eat_recv_skb(sk, skb);
2893		break;
2894	} while (len > 0);
2895
2896	/* According to UNIX98, msg_name/msg_namelen are ignored
2897	 * on connected socket. I was just happy when found this 8) --ANK
2898	 */
2899
2900	/* Clean up data we have read: This will do ACK frames. */
2901	tcp_cleanup_rbuf(sk, copied);
2902	return copied;
2903
2904out:
2905	return err;
2906
2907recv_urg:
2908	err = tcp_recv_urg(sk, msg, len, flags);
2909	goto out;
2910
2911recv_sndq:
2912	err = tcp_peek_sndq(sk, msg, len);
2913	goto out;
2914}
2915
2916int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
2917		int *addr_len)
2918{
2919	int cmsg_flags = 0, ret;
2920	struct scm_timestamping_internal tss;
2921
2922	if (unlikely(flags & MSG_ERRQUEUE))
2923		return inet_recv_error(sk, msg, len, addr_len);
2924
2925	if (sk_can_busy_loop(sk) &&
2926	    skb_queue_empty_lockless(&sk->sk_receive_queue) &&
2927	    sk->sk_state == TCP_ESTABLISHED)
2928		sk_busy_loop(sk, flags & MSG_DONTWAIT);
2929
2930	lock_sock(sk);
2931	ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags);
2932	release_sock(sk);
2933
2934	if ((cmsg_flags | msg->msg_get_inq) && ret >= 0) {
2935		if (cmsg_flags & TCP_CMSG_TS)
2936			tcp_recv_timestamp(msg, sk, &tss);
2937		if ((cmsg_flags & TCP_CMSG_INQ) | msg->msg_get_inq) {
2938			msg->msg_inq = tcp_inq_hint(sk);
2939			if (cmsg_flags & TCP_CMSG_INQ)
2940				put_cmsg(msg, SOL_TCP, TCP_CM_INQ,
2941					 sizeof(msg->msg_inq), &msg->msg_inq);
2942		}
2943	}
2944	return ret;
2945}
2946EXPORT_IPV6_MOD(tcp_recvmsg);
2947
2948void tcp_set_state(struct sock *sk, int state)
2949{
2950	int oldstate = sk->sk_state;
2951
2952	/* We defined a new enum for TCP states that are exported in BPF
2953	 * so as not force the internal TCP states to be frozen. The
2954	 * following checks will detect if an internal state value ever
2955	 * differs from the BPF value. If this ever happens, then we will
2956	 * need to remap the internal value to the BPF value before calling
2957	 * tcp_call_bpf_2arg.
2958	 */
2959	BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2960	BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2961	BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2962	BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2963	BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2964	BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2965	BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2966	BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2967	BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2968	BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2969	BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2970	BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
2971	BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE);
2972	BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2973
2974	/* bpf uapi header bpf.h defines an anonymous enum with values
2975	 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux
2976	 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON.
2977	 * But clang built vmlinux does not have this enum in DWARF
2978	 * since clang removes the above code before generating IR/debuginfo.
2979	 * Let us explicitly emit the type debuginfo to ensure the
2980	 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF
2981	 * regardless of which compiler is used.
2982	 */
2983	BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED);
2984
2985	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2986		tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
2987
2988	switch (state) {
2989	case TCP_ESTABLISHED:
2990		if (oldstate != TCP_ESTABLISHED)
2991			TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2992		break;
2993	case TCP_CLOSE_WAIT:
2994		if (oldstate == TCP_SYN_RECV)
2995			TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2996		break;
2997
2998	case TCP_CLOSE:
2999		if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
3000			TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
3001
3002		sk->sk_prot->unhash(sk);
3003		if (inet_csk(sk)->icsk_bind_hash &&
3004		    !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
3005			inet_put_port(sk);
3006		fallthrough;
3007	default:
3008		if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT)
3009			TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
3010	}
3011
3012	/* Change state AFTER socket is unhashed to avoid closed
3013	 * socket sitting in hash tables.
3014	 */
3015	inet_sk_state_store(sk, state);
3016}
3017EXPORT_SYMBOL_GPL(tcp_set_state);
3018
3019/*
3020 *	State processing on a close. This implements the state shift for
3021 *	sending our FIN frame. Note that we only send a FIN for some
3022 *	states. A shutdown() may have already sent the FIN, or we may be
3023 *	closed.
3024 */
3025
3026static const unsigned char new_state[16] = {
3027  /* current state:        new state:      action:	*/
3028  [0 /* (Invalid) */]	= TCP_CLOSE,
3029  [TCP_ESTABLISHED]	= TCP_FIN_WAIT1 | TCP_ACTION_FIN,
3030  [TCP_SYN_SENT]	= TCP_CLOSE,
3031  [TCP_SYN_RECV]	= TCP_FIN_WAIT1 | TCP_ACTION_FIN,
3032  [TCP_FIN_WAIT1]	= TCP_FIN_WAIT1,
3033  [TCP_FIN_WAIT2]	= TCP_FIN_WAIT2,
3034  [TCP_TIME_WAIT]	= TCP_CLOSE,
3035  [TCP_CLOSE]		= TCP_CLOSE,
3036  [TCP_CLOSE_WAIT]	= TCP_LAST_ACK  | TCP_ACTION_FIN,
3037  [TCP_LAST_ACK]	= TCP_LAST_ACK,
3038  [TCP_LISTEN]		= TCP_CLOSE,
3039  [TCP_CLOSING]		= TCP_CLOSING,
3040  [TCP_NEW_SYN_RECV]	= TCP_CLOSE,	/* should not happen ! */
3041};
3042
3043static int tcp_close_state(struct sock *sk)
3044{
3045	int next = (int)new_state[sk->sk_state];
3046	int ns = next & TCP_STATE_MASK;
3047
3048	tcp_set_state(sk, ns);
3049
3050	return next & TCP_ACTION_FIN;
3051}
3052
3053/*
3054 *	Shutdown the sending side of a connection. Much like close except
3055 *	that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
3056 */
3057
3058void tcp_shutdown(struct sock *sk, int how)
3059{
3060	/*	We need to grab some memory, and put together a FIN,
3061	 *	and then put it into the queue to be sent.
3062	 *		Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
3063	 */
3064	if (!(how & SEND_SHUTDOWN))
3065		return;
3066
3067	/* If we've already sent a FIN, or it's a closed state, skip this. */
3068	if ((1 << sk->sk_state) &
3069	    (TCPF_ESTABLISHED | TCPF_SYN_SENT |
3070	     TCPF_CLOSE_WAIT)) {
3071		/* Clear out any half completed packets.  FIN if needed. */
3072		if (tcp_close_state(sk))
3073			tcp_send_fin(sk);
3074	}
3075}
3076EXPORT_IPV6_MOD(tcp_shutdown);
3077
3078int tcp_orphan_count_sum(void)
3079{
3080	int i, total = 0;
3081
3082	for_each_possible_cpu(i)
3083		total += per_cpu(tcp_orphan_count, i);
3084
3085	return max(total, 0);
3086}
3087
3088static int tcp_orphan_cache;
3089static struct timer_list tcp_orphan_timer;
3090#define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100)
3091
3092static void tcp_orphan_update(struct timer_list *unused)
3093{
3094	WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum());
3095	mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
3096}
3097
3098static bool tcp_too_many_orphans(int shift)
3099{
3100	return READ_ONCE(tcp_orphan_cache) << shift >
3101		READ_ONCE(sysctl_tcp_max_orphans);
3102}
3103
3104static bool tcp_out_of_memory(const struct sock *sk)
3105{
3106	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
3107	    sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
3108		return true;
3109	return false;
3110}
3111
3112bool tcp_check_oom(const struct sock *sk, int shift)
3113{
3114	bool too_many_orphans, out_of_socket_memory;
3115
3116	too_many_orphans = tcp_too_many_orphans(shift);
3117	out_of_socket_memory = tcp_out_of_memory(sk);
3118
3119	if (too_many_orphans)
3120		net_info_ratelimited("too many orphaned sockets\n");
3121	if (out_of_socket_memory)
3122		net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
3123	return too_many_orphans || out_of_socket_memory;
3124}
3125
3126void __tcp_close(struct sock *sk, long timeout)
3127{
3128	bool data_was_unread = false;
3129	struct sk_buff *skb;
3130	int state;
3131
3132	WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
3133
3134	if (sk->sk_state == TCP_LISTEN) {
3135		tcp_set_state(sk, TCP_CLOSE);
3136
3137		/* Special case. */
3138		inet_csk_listen_stop(sk);
3139
3140		goto adjudge_to_death;
3141	}
3142
3143	/*  We need to flush the recv. buffs.  We do this only on the
3144	 *  descriptor close, not protocol-sourced closes, because the
3145	 *  reader process may not have drained the data yet!
3146	 */
3147	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
3148		u32 end_seq = TCP_SKB_CB(skb)->end_seq;
3149
3150		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
3151			end_seq--;
3152		if (after(end_seq, tcp_sk(sk)->copied_seq))
3153			data_was_unread = true;
3154		tcp_eat_recv_skb(sk, skb);
3155	}
3156
3157	/* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
3158	if (sk->sk_state == TCP_CLOSE)
3159		goto adjudge_to_death;
3160
3161	/* As outlined in RFC 2525, section 2.17, we send a RST here because
3162	 * data was lost. To witness the awful effects of the old behavior of
3163	 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
3164	 * GET in an FTP client, suspend the process, wait for the client to
3165	 * advertise a zero window, then kill -9 the FTP client, wheee...
3166	 * Note: timeout is always zero in such a case.
3167	 */
3168	if (unlikely(tcp_sk(sk)->repair)) {
3169		sk->sk_prot->disconnect(sk, 0);
3170	} else if (data_was_unread) {
3171		/* Unread data was tossed, zap the connection. */
3172		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
3173		tcp_set_state(sk, TCP_CLOSE);
3174		tcp_send_active_reset(sk, sk->sk_allocation,
3175				      SK_RST_REASON_TCP_ABORT_ON_CLOSE);
3176	} else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
3177		/* Check zero linger _after_ checking for unread data. */
3178		sk->sk_prot->disconnect(sk, 0);
3179		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
3180	} else if (tcp_close_state(sk)) {
3181		/* We FIN if the application ate all the data before
3182		 * zapping the connection.
3183		 */
3184
3185		/* RED-PEN. Formally speaking, we have broken TCP state
3186		 * machine. State transitions:
3187		 *
3188		 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
3189		 * TCP_SYN_RECV	-> TCP_FIN_WAIT1 (it is difficult)
3190		 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
3191		 *
3192		 * are legal only when FIN has been sent (i.e. in window),
3193		 * rather than queued out of window. Purists blame.
3194		 *
3195		 * F.e. "RFC state" is ESTABLISHED,
3196		 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
3197		 *
3198		 * The visible declinations are that sometimes
3199		 * we enter time-wait state, when it is not required really
3200		 * (harmless), do not send active resets, when they are
3201		 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
3202		 * they look as CLOSING or LAST_ACK for Linux)
3203		 * Probably, I missed some more holelets.
3204		 * 						--ANK
3205		 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
3206		 * in a single packet! (May consider it later but will
3207		 * probably need API support or TCP_CORK SYN-ACK until
3208		 * data is written and socket is closed.)
3209		 */
3210		tcp_send_fin(sk);
3211	}
3212
3213	sk_stream_wait_close(sk, timeout);
3214
3215adjudge_to_death:
3216	state = sk->sk_state;
3217	sock_hold(sk);
3218	sock_orphan(sk);
3219
3220	local_bh_disable();
3221	bh_lock_sock(sk);
3222	/* remove backlog if any, without releasing ownership. */
3223	__release_sock(sk);
3224
3225	tcp_orphan_count_inc();
3226
3227	/* Have we already been destroyed by a softirq or backlog? */
3228	if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
3229		goto out;
3230
3231	/*	This is a (useful) BSD violating of the RFC. There is a
3232	 *	problem with TCP as specified in that the other end could
3233	 *	keep a socket open forever with no application left this end.
3234	 *	We use a 1 minute timeout (about the same as BSD) then kill
3235	 *	our end. If they send after that then tough - BUT: long enough
3236	 *	that we won't make the old 4*rto = almost no time - whoops
3237	 *	reset mistake.
3238	 *
3239	 *	Nope, it was not mistake. It is really desired behaviour
3240	 *	f.e. on http servers, when such sockets are useless, but
3241	 *	consume significant resources. Let's do it with special
3242	 *	linger2	option.					--ANK
3243	 */
3244
3245	if (sk->sk_state == TCP_FIN_WAIT2) {
3246		struct tcp_sock *tp = tcp_sk(sk);
3247		if (READ_ONCE(tp->linger2) < 0) {
3248			tcp_set_state(sk, TCP_CLOSE);
3249			tcp_send_active_reset(sk, GFP_ATOMIC,
3250					      SK_RST_REASON_TCP_ABORT_ON_LINGER);
3251			__NET_INC_STATS(sock_net(sk),
3252					LINUX_MIB_TCPABORTONLINGER);
3253		} else {
3254			const int tmo = tcp_fin_time(sk);
3255
3256			if (tmo > TCP_TIMEWAIT_LEN) {
3257				tcp_reset_keepalive_timer(sk,
3258						tmo - TCP_TIMEWAIT_LEN);
3259			} else {
3260				tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
3261				goto out;
3262			}
3263		}
3264	}
3265	if (sk->sk_state != TCP_CLOSE) {
3266		if (tcp_check_oom(sk, 0)) {
3267			tcp_set_state(sk, TCP_CLOSE);
3268			tcp_send_active_reset(sk, GFP_ATOMIC,
3269					      SK_RST_REASON_TCP_ABORT_ON_MEMORY);
3270			__NET_INC_STATS(sock_net(sk),
3271					LINUX_MIB_TCPABORTONMEMORY);
3272		} else if (!check_net(sock_net(sk))) {
3273			/* Not possible to send reset; just close */
3274			tcp_set_state(sk, TCP_CLOSE);
3275		}
3276	}
3277
3278	if (sk->sk_state == TCP_CLOSE) {
3279		struct request_sock *req;
3280
3281		req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
3282						lockdep_sock_is_held(sk));
3283		/* We could get here with a non-NULL req if the socket is
3284		 * aborted (e.g., closed with unread data) before 3WHS
3285		 * finishes.
3286		 */
3287		if (req)
3288			reqsk_fastopen_remove(sk, req, false);
3289		inet_csk_destroy_sock(sk);
3290	}
3291	/* Otherwise, socket is reprieved until protocol close. */
3292
3293out:
3294	bh_unlock_sock(sk);
3295	local_bh_enable();
3296}
3297
3298void tcp_close(struct sock *sk, long timeout)
3299{
3300	lock_sock(sk);
3301	__tcp_close(sk, timeout);
3302	release_sock(sk);
3303	if (!sk->sk_net_refcnt)
3304		inet_csk_clear_xmit_timers_sync(sk);
3305	sock_put(sk);
3306}
3307EXPORT_SYMBOL(tcp_close);
3308
3309/* These states need RST on ABORT according to RFC793 */
3310
3311static inline bool tcp_need_reset(int state)
3312{
3313	return (1 << state) &
3314	       (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
3315		TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
3316}
3317
3318static void tcp_rtx_queue_purge(struct sock *sk)
3319{
3320	struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
3321
3322	tcp_sk(sk)->highest_sack = NULL;
3323	while (p) {
3324		struct sk_buff *skb = rb_to_skb(p);
3325
3326		p = rb_next(p);
3327		/* Since we are deleting whole queue, no need to
3328		 * list_del(&skb->tcp_tsorted_anchor)
3329		 */
3330		tcp_rtx_queue_unlink(skb, sk);
3331		tcp_wmem_free_skb(sk, skb);
3332	}
3333}
3334
3335void tcp_write_queue_purge(struct sock *sk)
3336{
3337	struct sk_buff *skb;
3338
3339	tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
3340	while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
3341		tcp_skb_tsorted_anchor_cleanup(skb);
3342		tcp_wmem_free_skb(sk, skb);
3343	}
3344	tcp_rtx_queue_purge(sk);
3345	INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
3346	tcp_clear_all_retrans_hints(tcp_sk(sk));
3347	tcp_sk(sk)->packets_out = 0;
3348	inet_csk(sk)->icsk_backoff = 0;
3349}
3350
3351int tcp_disconnect(struct sock *sk, int flags)
3352{
3353	struct inet_sock *inet = inet_sk(sk);
3354	struct inet_connection_sock *icsk = inet_csk(sk);
3355	struct tcp_sock *tp = tcp_sk(sk);
3356	int old_state = sk->sk_state;
3357	struct request_sock *req;
3358	u32 seq;
3359
3360	if (old_state != TCP_CLOSE)
3361		tcp_set_state(sk, TCP_CLOSE);
3362
3363	/* ABORT function of RFC793 */
3364	if (old_state == TCP_LISTEN) {
3365		inet_csk_listen_stop(sk);
3366	} else if (unlikely(tp->repair)) {
3367		WRITE_ONCE(sk->sk_err, ECONNABORTED);
3368	} else if (tcp_need_reset(old_state)) {
3369		tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_TCP_STATE);
3370		WRITE_ONCE(sk->sk_err, ECONNRESET);
3371	} else if (tp->snd_nxt != tp->write_seq &&
3372		   (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) {
3373		/* The last check adjusts for discrepancy of Linux wrt. RFC
3374		 * states
3375		 */
3376		tcp_send_active_reset(sk, gfp_any(),
3377				      SK_RST_REASON_TCP_DISCONNECT_WITH_DATA);
3378		WRITE_ONCE(sk->sk_err, ECONNRESET);
3379	} else if (old_state == TCP_SYN_SENT)
3380		WRITE_ONCE(sk->sk_err, ECONNRESET);
3381
3382	tcp_clear_xmit_timers(sk);
3383	__skb_queue_purge(&sk->sk_receive_queue);
3384	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
3385	WRITE_ONCE(tp->urg_data, 0);
3386	sk_set_peek_off(sk, -1);
3387	tcp_write_queue_purge(sk);
3388	tcp_fastopen_active_disable_ofo_check(sk);
3389	skb_rbtree_purge(&tp->out_of_order_queue);
3390
3391	inet->inet_dport = 0;
3392
3393	inet_bhash2_reset_saddr(sk);
3394
3395	WRITE_ONCE(sk->sk_shutdown, 0);
3396	sock_reset_flag(sk, SOCK_DONE);
3397	tp->srtt_us = 0;
3398	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
3399	tp->rcv_rtt_last_tsecr = 0;
3400
3401	seq = tp->write_seq + tp->max_window + 2;
3402	if (!seq)
3403		seq = 1;
3404	WRITE_ONCE(tp->write_seq, seq);
3405
3406	icsk->icsk_backoff = 0;
3407	WRITE_ONCE(icsk->icsk_probes_out, 0);
3408	icsk->icsk_probes_tstamp = 0;
3409	icsk->icsk_rto = TCP_TIMEOUT_INIT;
3410	WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN);
3411	WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX);
3412	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
3413	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
3414	tp->snd_cwnd_cnt = 0;
3415	tp->is_cwnd_limited = 0;
3416	tp->max_packets_out = 0;
3417	tp->window_clamp = 0;
3418	tp->delivered = 0;
3419	tp->delivered_ce = 0;
3420	tp->accecn_fail_mode = 0;
3421	tp->saw_accecn_opt = TCP_ACCECN_OPT_NOT_SEEN;
3422	tcp_accecn_init_counters(tp);
3423	tp->prev_ecnfield = 0;
3424	tp->accecn_opt_tstamp = 0;
3425	if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release)
3426		icsk->icsk_ca_ops->release(sk);
3427	memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
3428	icsk->icsk_ca_initialized = 0;
3429	tcp_set_ca_state(sk, TCP_CA_Open);
3430	tp->is_sack_reneg = 0;
3431	tcp_clear_retrans(tp);
3432	tp->total_retrans = 0;
3433	inet_csk_delack_init(sk);
3434	/* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
3435	 * issue in __tcp_select_window()
3436	 */
3437	icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
3438	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
3439	__sk_dst_reset(sk);
3440	dst_release(unrcu_pointer(xchg(&sk->sk_rx_dst, NULL)));
3441	tcp_saved_syn_free(tp);
3442	tp->compressed_ack = 0;
3443	tp->segs_in = 0;
3444	tp->segs_out = 0;
3445	tp->bytes_sent = 0;
3446	tp->bytes_acked = 0;
3447	tp->bytes_received = 0;
3448	tp->bytes_retrans = 0;
3449	tp->data_segs_in = 0;
3450	tp->data_segs_out = 0;
3451	tp->duplicate_sack[0].start_seq = 0;
3452	tp->duplicate_sack[0].end_seq = 0;
3453	tp->dsack_dups = 0;
3454	tp->reord_seen = 0;
3455	tp->retrans_out = 0;
3456	tp->sacked_out = 0;
3457	tp->tlp_high_seq = 0;
3458	tp->last_oow_ack_time = 0;
3459	tp->plb_rehash = 0;
3460	/* There's a bubble in the pipe until at least the first ACK. */
3461	tp->app_limited = ~0U;
3462	tp->rate_app_limited = 1;
3463	tp->rack.mstamp = 0;
3464	tp->rack.advanced = 0;
3465	tp->rack.reo_wnd_steps = 1;
3466	tp->rack.last_delivered = 0;
3467	tp->rack.reo_wnd_persist = 0;
3468	tp->rack.dsack_seen = 0;
3469	tp->syn_data_acked = 0;
3470	tp->syn_fastopen_child = 0;
3471	tp->rx_opt.saw_tstamp = 0;
3472	tp->rx_opt.dsack = 0;
3473	tp->rx_opt.num_sacks = 0;
3474	tp->rcv_ooopack = 0;
3475
3476
3477	/* Clean up fastopen related fields */
3478	req = rcu_dereference_protected(tp->fastopen_rsk,
3479					lockdep_sock_is_held(sk));
3480	if (req)
3481		reqsk_fastopen_remove(sk, req, false);
3482	tcp_free_fastopen_req(tp);
3483	inet_clear_bit(DEFER_CONNECT, sk);
3484	tp->fastopen_client_fail = 0;
3485
3486	WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
3487
3488	if (sk->sk_frag.page) {
3489		put_page(sk->sk_frag.page);
3490		sk->sk_frag.page = NULL;
3491		sk->sk_frag.offset = 0;
3492	}
3493	sk_error_report(sk);
3494	return 0;
3495}
3496EXPORT_SYMBOL(tcp_disconnect);
3497
3498static inline bool tcp_can_repair_sock(const struct sock *sk)
3499{
3500	return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
3501		(sk->sk_state != TCP_LISTEN);
3502}
3503
3504static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
3505{
3506	struct tcp_repair_window opt;
3507
3508	if (!tp->repair)
3509		return -EPERM;
3510
3511	if (len != sizeof(opt))
3512		return -EINVAL;
3513
3514	if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
3515		return -EFAULT;
3516
3517	if (opt.max_window < opt.snd_wnd)
3518		return -EINVAL;
3519
3520	if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
3521		return -EINVAL;
3522
3523	if (after(opt.rcv_wup, tp->rcv_nxt))
3524		return -EINVAL;
3525
3526	tp->snd_wl1	= opt.snd_wl1;
3527	tp->snd_wnd	= opt.snd_wnd;
3528	tp->max_window	= opt.max_window;
3529
3530	tp->rcv_wnd	= opt.rcv_wnd;
3531	tp->rcv_wup	= opt.rcv_wup;
3532
3533	return 0;
3534}
3535
3536static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
3537		unsigned int len)
3538{
3539	struct tcp_sock *tp = tcp_sk(sk);
3540	struct tcp_repair_opt opt;
3541	size_t offset = 0;
3542
3543	while (len >= sizeof(opt)) {
3544		if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
3545			return -EFAULT;
3546
3547		offset += sizeof(opt);
3548		len -= sizeof(opt);
3549
3550		switch (opt.opt_code) {
3551		case TCPOPT_MSS:
3552			tp->rx_opt.mss_clamp = opt.opt_val;
3553			tcp_mtup_init(sk);
3554			break;
3555		case TCPOPT_WINDOW:
3556			{
3557				u16 snd_wscale = opt.opt_val & 0xFFFF;
3558				u16 rcv_wscale = opt.opt_val >> 16;
3559
3560				if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
3561					return -EFBIG;
3562
3563				tp->rx_opt.snd_wscale = snd_wscale;
3564				tp->rx_opt.rcv_wscale = rcv_wscale;
3565				tp->rx_opt.wscale_ok = 1;
3566			}
3567			break;
3568		case TCPOPT_SACK_PERM:
3569			if (opt.opt_val != 0)
3570				return -EINVAL;
3571
3572			tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
3573			break;
3574		case TCPOPT_TIMESTAMP:
3575			if (opt.opt_val != 0)
3576				return -EINVAL;
3577
3578			tp->rx_opt.tstamp_ok = 1;
3579			break;
3580		}
3581	}
3582
3583	return 0;
3584}
3585
3586DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
3587EXPORT_IPV6_MOD(tcp_tx_delay_enabled);
3588
3589static void tcp_enable_tx_delay(struct sock *sk, int val)
3590{
3591	struct tcp_sock *tp = tcp_sk(sk);
3592	s32 delta = (val - tp->tcp_tx_delay) << 3;
3593
3594	if (val && !static_branch_unlikely(&tcp_tx_delay_enabled)) {
3595		static int __tcp_tx_delay_enabled = 0;
3596
3597		if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
3598			static_branch_enable(&tcp_tx_delay_enabled);
3599			pr_info("TCP_TX_DELAY enabled\n");
3600		}
3601	}
3602	/* If we change tcp_tx_delay on a live flow, adjust tp->srtt_us,
3603	 * tp->rtt_min, icsk_rto and sk->sk_pacing_rate.
3604	 * This is best effort.
3605	 */
3606	if (delta && sk->sk_state == TCP_ESTABLISHED) {
3607		s64 srtt = (s64)tp->srtt_us + delta;
3608
3609		tp->srtt_us = clamp_t(s64, srtt, 1, ~0U);
3610
3611		/* Note: does not deal with non zero icsk_backoff */
3612		tcp_set_rto(sk);
3613
3614		minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
3615
3616		tcp_update_pacing_rate(sk);
3617	}
3618}
3619
3620/* When set indicates to always queue non-full frames.  Later the user clears
3621 * this option and we transmit any pending partial frames in the queue.  This is
3622 * meant to be used alongside sendfile() to get properly filled frames when the
3623 * user (for example) must write out headers with a write() call first and then
3624 * use sendfile to send out the data parts.
3625 *
3626 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than
3627 * TCP_NODELAY.
3628 */
3629void __tcp_sock_set_cork(struct sock *sk, bool on)
3630{
3631	struct tcp_sock *tp = tcp_sk(sk);
3632
3633	if (on) {
3634		tp->nonagle |= TCP_NAGLE_CORK;
3635	} else {
3636		tp->nonagle &= ~TCP_NAGLE_CORK;
3637		if (tp->nonagle & TCP_NAGLE_OFF)
3638			tp->nonagle |= TCP_NAGLE_PUSH;
3639		tcp_push_pending_frames(sk);
3640	}
3641}
3642
3643void tcp_sock_set_cork(struct sock *sk, bool on)
3644{
3645	lock_sock(sk);
3646	__tcp_sock_set_cork(sk, on);
3647	release_sock(sk);
3648}
3649EXPORT_SYMBOL(tcp_sock_set_cork);
3650
3651/* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is
3652 * remembered, but it is not activated until cork is cleared.
3653 *
3654 * However, when TCP_NODELAY is set we make an explicit push, which overrides
3655 * even TCP_CORK for currently queued segments.
3656 */
3657void __tcp_sock_set_nodelay(struct sock *sk, bool on)
3658{
3659	if (on) {
3660		tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
3661		tcp_push_pending_frames(sk);
3662	} else {
3663		tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
3664	}
3665}
3666
3667void tcp_sock_set_nodelay(struct sock *sk)
3668{
3669	lock_sock(sk);
3670	__tcp_sock_set_nodelay(sk, true);
3671	release_sock(sk);
3672}
3673EXPORT_SYMBOL(tcp_sock_set_nodelay);
3674
3675static void __tcp_sock_set_quickack(struct sock *sk, int val)
3676{
3677	if (!val) {
3678		inet_csk_enter_pingpong_mode(sk);
3679		return;
3680	}
3681
3682	inet_csk_exit_pingpong_mode(sk);
3683	if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
3684	    inet_csk_ack_scheduled(sk)) {
3685		inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
3686		tcp_cleanup_rbuf(sk, 1);
3687		if (!(val & 1))
3688			inet_csk_enter_pingpong_mode(sk);
3689	}
3690}
3691
3692void tcp_sock_set_quickack(struct sock *sk, int val)
3693{
3694	lock_sock(sk);
3695	__tcp_sock_set_quickack(sk, val);
3696	release_sock(sk);
3697}
3698EXPORT_SYMBOL(tcp_sock_set_quickack);
3699
3700int tcp_sock_set_syncnt(struct sock *sk, int val)
3701{
3702	if (val < 1 || val > MAX_TCP_SYNCNT)
3703		return -EINVAL;
3704
3705	WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val);
3706	return 0;
3707}
3708EXPORT_SYMBOL(tcp_sock_set_syncnt);
3709
3710int tcp_sock_set_user_timeout(struct sock *sk, int val)
3711{
3712	/* Cap the max time in ms TCP will retry or probe the window
3713	 * before giving up and aborting (ETIMEDOUT) a connection.
3714	 */
3715	if (val < 0)
3716		return -EINVAL;
3717
3718	WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val);
3719	return 0;
3720}
3721EXPORT_SYMBOL(tcp_sock_set_user_timeout);
3722
3723int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
3724{
3725	struct tcp_sock *tp = tcp_sk(sk);
3726
3727	if (val < 1 || val > MAX_TCP_KEEPIDLE)
3728		return -EINVAL;
3729
3730	/* Paired with WRITE_ONCE() in keepalive_time_when() */
3731	WRITE_ONCE(tp->keepalive_time, val * HZ);
3732	if (sock_flag(sk, SOCK_KEEPOPEN) &&
3733	    !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
3734		u32 elapsed = keepalive_time_elapsed(tp);
3735
3736		if (tp->keepalive_time > elapsed)
3737			elapsed = tp->keepalive_time - elapsed;
3738		else
3739			elapsed = 0;
3740		tcp_reset_keepalive_timer(sk, elapsed);
3741	}
3742
3743	return 0;
3744}
3745
3746int tcp_sock_set_keepidle(struct sock *sk, int val)
3747{
3748	int err;
3749
3750	lock_sock(sk);
3751	err = tcp_sock_set_keepidle_locked(sk, val);
3752	release_sock(sk);
3753	return err;
3754}
3755EXPORT_SYMBOL(tcp_sock_set_keepidle);
3756
3757int tcp_sock_set_keepintvl(struct sock *sk, int val)
3758{
3759	if (val < 1 || val > MAX_TCP_KEEPINTVL)
3760		return -EINVAL;
3761
3762	WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ);
3763	return 0;
3764}
3765EXPORT_SYMBOL(tcp_sock_set_keepintvl);
3766
3767int tcp_sock_set_keepcnt(struct sock *sk, int val)
3768{
3769	if (val < 1 || val > MAX_TCP_KEEPCNT)
3770		return -EINVAL;
3771
3772	/* Paired with READ_ONCE() in keepalive_probes() */
3773	WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val);
3774	return 0;
3775}
3776EXPORT_SYMBOL(tcp_sock_set_keepcnt);
3777
3778int tcp_set_window_clamp(struct sock *sk, int val)
3779{
3780	u32 old_window_clamp, new_window_clamp, new_rcv_ssthresh;
3781	struct tcp_sock *tp = tcp_sk(sk);
3782
3783	if (!val) {
3784		if (sk->sk_state != TCP_CLOSE)
3785			return -EINVAL;
3786		WRITE_ONCE(tp->window_clamp, 0);
3787		return 0;
3788	}
3789
3790	old_window_clamp = tp->window_clamp;
3791	new_window_clamp = max_t(int, SOCK_MIN_RCVBUF / 2, val);
3792
3793	if (new_window_clamp == old_window_clamp)
3794		return 0;
3795
3796	WRITE_ONCE(tp->window_clamp, new_window_clamp);
3797
3798	/* Need to apply the reserved mem provisioning only
3799	 * when shrinking the window clamp.
3800	 */
3801	if (new_window_clamp < old_window_clamp) {
3802		__tcp_adjust_rcv_ssthresh(sk, new_window_clamp);
3803	} else {
3804		new_rcv_ssthresh = min(tp->rcv_wnd, new_window_clamp);
3805		tp->rcv_ssthresh = max(new_rcv_ssthresh, tp->rcv_ssthresh);
3806	}
3807	return 0;
3808}
3809
3810int tcp_sock_set_maxseg(struct sock *sk, int val)
3811{
3812	/* Values greater than interface MTU won't take effect. However
3813	 * at the point when this call is done we typically don't yet
3814	 * know which interface is going to be used
3815	 */
3816	if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW))
3817		return -EINVAL;
3818
3819	WRITE_ONCE(tcp_sk(sk)->rx_opt.user_mss, val);
3820	return 0;
3821}
3822
3823/*
3824 *	Socket option code for TCP.
3825 */
3826int do_tcp_setsockopt(struct sock *sk, int level, int optname,
3827		      sockptr_t optval, unsigned int optlen)
3828{
3829	struct tcp_sock *tp = tcp_sk(sk);
3830	struct inet_connection_sock *icsk = inet_csk(sk);
3831	struct net *net = sock_net(sk);
3832	int val;
3833	int err = 0;
3834
3835	/* These are data/string values, all the others are ints */
3836	switch (optname) {
3837	case TCP_CONGESTION: {
3838		char name[TCP_CA_NAME_MAX];
3839
3840		if (optlen < 1)
3841			return -EINVAL;
3842
3843		val = strncpy_from_sockptr(name, optval,
3844					min_t(long, TCP_CA_NAME_MAX-1, optlen));
3845		if (val < 0)
3846			return -EFAULT;
3847		name[val] = 0;
3848
3849		sockopt_lock_sock(sk);
3850		err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(),
3851						 sockopt_ns_capable(sock_net(sk)->user_ns,
3852								    CAP_NET_ADMIN));
3853		sockopt_release_sock(sk);
3854		return err;
3855	}
3856	case TCP_ULP: {
3857		char name[TCP_ULP_NAME_MAX];
3858
3859		if (optlen < 1)
3860			return -EINVAL;
3861
3862		val = strncpy_from_sockptr(name, optval,
3863					min_t(long, TCP_ULP_NAME_MAX - 1,
3864					      optlen));
3865		if (val < 0)
3866			return -EFAULT;
3867		name[val] = 0;
3868
3869		sockopt_lock_sock(sk);
3870		err = tcp_set_ulp(sk, name);
3871		sockopt_release_sock(sk);
3872		return err;
3873	}
3874	case TCP_FASTOPEN_KEY: {
3875		__u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3876		__u8 *backup_key = NULL;
3877
3878		/* Allow a backup key as well to facilitate key rotation
3879		 * First key is the active one.
3880		 */
3881		if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
3882		    optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
3883			return -EINVAL;
3884
3885		if (copy_from_sockptr(key, optval, optlen))
3886			return -EFAULT;
3887
3888		if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
3889			backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
3890
3891		return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
3892	}
3893	default:
3894		/* fallthru */
3895		break;
3896	}
3897
3898	if (optlen < sizeof(int))
3899		return -EINVAL;
3900
3901	if (copy_from_sockptr(&val, optval, sizeof(val)))
3902		return -EFAULT;
3903
3904	/* Handle options that can be set without locking the socket. */
3905	switch (optname) {
3906	case TCP_SYNCNT:
3907		return tcp_sock_set_syncnt(sk, val);
3908	case TCP_USER_TIMEOUT:
3909		return tcp_sock_set_user_timeout(sk, val);
3910	case TCP_KEEPINTVL:
3911		return tcp_sock_set_keepintvl(sk, val);
3912	case TCP_KEEPCNT:
3913		return tcp_sock_set_keepcnt(sk, val);
3914	case TCP_LINGER2:
3915		if (val < 0)
3916			WRITE_ONCE(tp->linger2, -1);
3917		else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
3918			WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX);
3919		else
3920			WRITE_ONCE(tp->linger2, val * HZ);
3921		return 0;
3922	case TCP_DEFER_ACCEPT:
3923		/* Translate value in seconds to number of retransmits */
3924		WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept,
3925			   secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3926					   TCP_RTO_MAX / HZ));
3927		return 0;
3928	case TCP_RTO_MAX_MS:
3929		if (val < MSEC_PER_SEC || val > TCP_RTO_MAX_SEC * MSEC_PER_SEC)
3930			return -EINVAL;
3931		WRITE_ONCE(inet_csk(sk)->icsk_rto_max, msecs_to_jiffies(val));
3932		return 0;
3933	case TCP_RTO_MIN_US: {
3934		int rto_min = usecs_to_jiffies(val);
3935
3936		if (rto_min > TCP_RTO_MIN || rto_min < TCP_TIMEOUT_MIN)
3937			return -EINVAL;
3938		WRITE_ONCE(inet_csk(sk)->icsk_rto_min, rto_min);
3939		return 0;
3940	}
3941	case TCP_DELACK_MAX_US: {
3942		int delack_max = usecs_to_jiffies(val);
3943
3944		if (delack_max > TCP_DELACK_MAX || delack_max < TCP_TIMEOUT_MIN)
3945			return -EINVAL;
3946		WRITE_ONCE(inet_csk(sk)->icsk_delack_max, delack_max);
3947		return 0;
3948	}
3949	case TCP_MAXSEG:
3950		return tcp_sock_set_maxseg(sk, val);
3951	}
3952
3953	sockopt_lock_sock(sk);
3954
3955	switch (optname) {
3956	case TCP_NODELAY:
3957		__tcp_sock_set_nodelay(sk, val);
3958		break;
3959
3960	case TCP_THIN_LINEAR_TIMEOUTS:
3961		if (val < 0 || val > 1)
3962			err = -EINVAL;
3963		else
3964			tp->thin_lto = val;
3965		break;
3966
3967	case TCP_THIN_DUPACK:
3968		if (val < 0 || val > 1)
3969			err = -EINVAL;
3970		break;
3971
3972	case TCP_REPAIR:
3973		if (!tcp_can_repair_sock(sk))
3974			err = -EPERM;
3975		else if (val == TCP_REPAIR_ON) {
3976			tp->repair = 1;
3977			sk->sk_reuse = SK_FORCE_REUSE;
3978			tp->repair_queue = TCP_NO_QUEUE;
3979		} else if (val == TCP_REPAIR_OFF) {
3980			tp->repair = 0;
3981			sk->sk_reuse = SK_NO_REUSE;
3982			tcp_send_window_probe(sk);
3983		} else if (val == TCP_REPAIR_OFF_NO_WP) {
3984			tp->repair = 0;
3985			sk->sk_reuse = SK_NO_REUSE;
3986		} else
3987			err = -EINVAL;
3988
3989		break;
3990
3991	case TCP_REPAIR_QUEUE:
3992		if (!tp->repair)
3993			err = -EPERM;
3994		else if ((unsigned int)val < TCP_QUEUES_NR)
3995			tp->repair_queue = val;
3996		else
3997			err = -EINVAL;
3998		break;
3999
4000	case TCP_QUEUE_SEQ:
4001		if (sk->sk_state != TCP_CLOSE) {
4002			err = -EPERM;
4003		} else if (tp->repair_queue == TCP_SEND_QUEUE) {
4004			if (!tcp_rtx_queue_empty(sk))
4005				err = -EPERM;
4006			else
4007				WRITE_ONCE(tp->write_seq, val);
4008		} else if (tp->repair_queue == TCP_RECV_QUEUE) {
4009			if (tp->rcv_nxt != tp->copied_seq) {
4010				err = -EPERM;
4011			} else {
4012				WRITE_ONCE(tp->rcv_nxt, val);
4013				WRITE_ONCE(tp->copied_seq, val);
4014			}
4015		} else {
4016			err = -EINVAL;
4017		}
4018		break;
4019
4020	case TCP_REPAIR_OPTIONS:
4021		if (!tp->repair)
4022			err = -EINVAL;
4023		else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)
4024			err = tcp_repair_options_est(sk, optval, optlen);
4025		else
4026			err = -EPERM;
4027		break;
4028
4029	case TCP_CORK:
4030		__tcp_sock_set_cork(sk, val);
4031		break;
4032
4033	case TCP_KEEPIDLE:
4034		err = tcp_sock_set_keepidle_locked(sk, val);
4035		break;
4036	case TCP_SAVE_SYN:
4037		/* 0: disable, 1: enable, 2: start from ether_header */
4038		if (val < 0 || val > 2)
4039			err = -EINVAL;
4040		else
4041			tp->save_syn = val;
4042		break;
4043
4044	case TCP_WINDOW_CLAMP:
4045		err = tcp_set_window_clamp(sk, val);
4046		break;
4047
4048	case TCP_QUICKACK:
4049		__tcp_sock_set_quickack(sk, val);
4050		break;
4051
4052	case TCP_AO_REPAIR:
4053		if (!tcp_can_repair_sock(sk)) {
4054			err = -EPERM;
4055			break;
4056		}
4057		err = tcp_ao_set_repair(sk, optval, optlen);
4058		break;
4059#ifdef CONFIG_TCP_AO
4060	case TCP_AO_ADD_KEY:
4061	case TCP_AO_DEL_KEY:
4062	case TCP_AO_INFO: {
4063		/* If this is the first TCP-AO setsockopt() on the socket,
4064		 * sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR
4065		 * in any state.
4066		 */
4067		if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
4068			goto ao_parse;
4069		if (rcu_dereference_protected(tcp_sk(sk)->ao_info,
4070					      lockdep_sock_is_held(sk)))
4071			goto ao_parse;
4072		if (tp->repair)
4073			goto ao_parse;
4074		err = -EISCONN;
4075		break;
4076ao_parse:
4077		err = tp->af_specific->ao_parse(sk, optname, optval, optlen);
4078		break;
4079	}
4080#endif
4081#ifdef CONFIG_TCP_MD5SIG
4082	case TCP_MD5SIG:
4083	case TCP_MD5SIG_EXT:
4084		err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
4085		break;
4086#endif
4087	case TCP_FASTOPEN:
4088		if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
4089		    TCPF_LISTEN))) {
4090			tcp_fastopen_init_key_once(net);
4091
4092			fastopen_queue_tune(sk, val);
4093		} else {
4094			err = -EINVAL;
4095		}
4096		break;
4097	case TCP_FASTOPEN_CONNECT:
4098		if (val > 1 || val < 0) {
4099			err = -EINVAL;
4100		} else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) &
4101			   TFO_CLIENT_ENABLE) {
4102			if (sk->sk_state == TCP_CLOSE)
4103				tp->fastopen_connect = val;
4104			else
4105				err = -EINVAL;
4106		} else {
4107			err = -EOPNOTSUPP;
4108		}
4109		break;
4110	case TCP_FASTOPEN_NO_COOKIE:
4111		if (val > 1 || val < 0)
4112			err = -EINVAL;
4113		else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
4114			err = -EINVAL;
4115		else
4116			tp->fastopen_no_cookie = val;
4117		break;
4118	case TCP_TIMESTAMP:
4119		if (!tp->repair) {
4120			err = -EPERM;
4121			break;
4122		}
4123		/* val is an opaque field,
4124		 * and low order bit contains usec_ts enable bit.
4125		 * Its a best effort, and we do not care if user makes an error.
4126		 */
4127		tp->tcp_usec_ts = val & 1;
4128		WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts));
4129		break;
4130	case TCP_REPAIR_WINDOW:
4131		err = tcp_repair_set_window(tp, optval, optlen);
4132		break;
4133	case TCP_NOTSENT_LOWAT:
4134		WRITE_ONCE(tp->notsent_lowat, val);
4135		READ_ONCE(sk->sk_write_space)(sk);
4136		break;
4137	case TCP_INQ:
4138		if (val > 1 || val < 0)
4139			err = -EINVAL;
4140		else
4141			tp->recvmsg_inq = val;
4142		break;
4143	case TCP_TX_DELAY:
4144		/* tp->srtt_us is u32, and is shifted by 3 */
4145		if (val < 0 || val >= (1U << (31 - 3))) {
4146			err = -EINVAL;
4147			break;
4148		}
4149		tcp_enable_tx_delay(sk, val);
4150		WRITE_ONCE(tp->tcp_tx_delay, val);
4151		break;
4152	default:
4153		err = -ENOPROTOOPT;
4154		break;
4155	}
4156
4157	sockopt_release_sock(sk);
4158	return err;
4159}
4160
4161int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
4162		   unsigned int optlen)
4163{
4164	const struct inet_connection_sock *icsk = inet_csk(sk);
4165
4166	if (level != SOL_TCP)
4167		/* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
4168		return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname,
4169								optval, optlen);
4170	return do_tcp_setsockopt(sk, level, optname, optval, optlen);
4171}
4172EXPORT_IPV6_MOD(tcp_setsockopt);
4173
4174static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
4175				      struct tcp_info *info)
4176{
4177	u64 stats[__TCP_CHRONO_MAX], total = 0;
4178	enum tcp_chrono i;
4179
4180	for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
4181		stats[i] = tp->chrono_stat[i - 1];
4182		if (i == tp->chrono_type)
4183			stats[i] += tcp_jiffies32 - tp->chrono_start;
4184		stats[i] *= USEC_PER_SEC / HZ;
4185		total += stats[i];
4186	}
4187
4188	info->tcpi_busy_time = total;
4189	info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
4190	info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
4191}
4192
4193/* Return information about state of tcp endpoint in API format. */
4194void tcp_get_info(struct sock *sk, struct tcp_info *info)
4195{
4196	const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
4197	const struct inet_connection_sock *icsk = inet_csk(sk);
4198	const u8 ect1_idx = INET_ECN_ECT_1 - 1;
4199	const u8 ect0_idx = INET_ECN_ECT_0 - 1;
4200	const u8 ce_idx = INET_ECN_CE - 1;
4201	unsigned long rate;
4202	u32 now;
4203	u64 rate64;
4204	bool slow;
4205
4206	memset(info, 0, sizeof(*info));
4207	if (sk->sk_type != SOCK_STREAM)
4208		return;
4209
4210	info->tcpi_state = inet_sk_state_load(sk);
4211
4212	/* Report meaningful fields for all TCP states, including listeners */
4213	rate = READ_ONCE(sk->sk_pacing_rate);
4214	rate64 = (rate != ~0UL) ? rate : ~0ULL;
4215	info->tcpi_pacing_rate = rate64;
4216
4217	rate = READ_ONCE(sk->sk_max_pacing_rate);
4218	rate64 = (rate != ~0UL) ? rate : ~0ULL;
4219	info->tcpi_max_pacing_rate = rate64;
4220
4221	info->tcpi_reordering = tp->reordering;
4222	info->tcpi_snd_cwnd = tcp_snd_cwnd(tp);
4223
4224	if (info->tcpi_state == TCP_LISTEN) {
4225		/* listeners aliased fields :
4226		 * tcpi_unacked -> Number of children ready for accept()
4227		 * tcpi_sacked  -> max backlog
4228		 */
4229		info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
4230		info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
4231		return;
4232	}
4233
4234	slow = lock_sock_fast(sk);
4235
4236	info->tcpi_ca_state = icsk->icsk_ca_state;
4237	info->tcpi_retransmits = icsk->icsk_retransmits;
4238	info->tcpi_probes = icsk->icsk_probes_out;
4239	info->tcpi_backoff = icsk->icsk_backoff;
4240
4241	if (tp->rx_opt.tstamp_ok)
4242		info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
4243	if (tcp_is_sack(tp))
4244		info->tcpi_options |= TCPI_OPT_SACK;
4245	if (tp->rx_opt.wscale_ok) {
4246		info->tcpi_options |= TCPI_OPT_WSCALE;
4247		info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
4248		info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
4249	}
4250
4251	if (tcp_ecn_mode_any(tp))
4252		info->tcpi_options |= TCPI_OPT_ECN;
4253	if (tp->ecn_flags & TCP_ECN_SEEN)
4254		info->tcpi_options |= TCPI_OPT_ECN_SEEN;
4255	if (tp->syn_data_acked)
4256		info->tcpi_options |= TCPI_OPT_SYN_DATA;
4257	if (tp->tcp_usec_ts)
4258		info->tcpi_options |= TCPI_OPT_USEC_TS;
4259	if (tp->syn_fastopen_child)
4260		info->tcpi_options |= TCPI_OPT_TFO_CHILD;
4261
4262	info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
4263	info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato,
4264						tcp_delack_max(sk)));
4265	info->tcpi_snd_mss = tp->mss_cache;
4266	info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
4267
4268	info->tcpi_unacked = tp->packets_out;
4269	info->tcpi_sacked = tp->sacked_out;
4270
4271	info->tcpi_lost = tp->lost_out;
4272	info->tcpi_retrans = tp->retrans_out;
4273
4274	now = tcp_jiffies32;
4275	info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
4276	info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
4277	info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
4278
4279	info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
4280	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
4281	info->tcpi_rtt = tp->srtt_us >> 3;
4282	info->tcpi_rttvar = tp->mdev_us >> 2;
4283	info->tcpi_snd_ssthresh = tp->snd_ssthresh;
4284	info->tcpi_advmss = tp->advmss;
4285
4286	info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
4287	info->tcpi_rcv_space = tp->rcvq_space.space;
4288
4289	info->tcpi_total_retrans = tp->total_retrans;
4290
4291	info->tcpi_bytes_acked = tp->bytes_acked;
4292	info->tcpi_bytes_received = tp->bytes_received;
4293	info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
4294	tcp_get_info_chrono_stats(tp, info);
4295
4296	info->tcpi_segs_out = tp->segs_out;
4297
4298	/* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */
4299	info->tcpi_segs_in = READ_ONCE(tp->segs_in);
4300	info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in);
4301
4302	info->tcpi_min_rtt = tcp_min_rtt(tp);
4303	info->tcpi_data_segs_out = tp->data_segs_out;
4304
4305	info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
4306	rate64 = tcp_compute_delivery_rate(tp);
4307	if (rate64)
4308		info->tcpi_delivery_rate = rate64;
4309	info->tcpi_delivered = tp->delivered;
4310	info->tcpi_delivered_ce = tp->delivered_ce;
4311	info->tcpi_bytes_sent = tp->bytes_sent;
4312	info->tcpi_bytes_retrans = tp->bytes_retrans;
4313	info->tcpi_dsack_dups = tp->dsack_dups;
4314	info->tcpi_reord_seen = tp->reord_seen;
4315	info->tcpi_rcv_ooopack = tp->rcv_ooopack;
4316	info->tcpi_snd_wnd = tp->snd_wnd;
4317	info->tcpi_rcv_wnd = tp->rcv_wnd;
4318	info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash;
4319	info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
4320
4321	info->tcpi_total_rto = tp->total_rto;
4322	info->tcpi_total_rto_recoveries = tp->total_rto_recoveries;
4323	info->tcpi_total_rto_time = tp->total_rto_time;
4324	if (tp->rto_stamp)
4325		info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp;
4326
4327	info->tcpi_accecn_fail_mode = tp->accecn_fail_mode;
4328	info->tcpi_accecn_opt_seen = tp->saw_accecn_opt;
4329	info->tcpi_received_ce = tp->received_ce;
4330	info->tcpi_delivered_e1_bytes = tp->delivered_ecn_bytes[ect1_idx];
4331	info->tcpi_delivered_e0_bytes = tp->delivered_ecn_bytes[ect0_idx];
4332	info->tcpi_delivered_ce_bytes = tp->delivered_ecn_bytes[ce_idx];
4333	info->tcpi_received_e1_bytes = tp->received_ecn_bytes[ect1_idx];
4334	info->tcpi_received_e0_bytes = tp->received_ecn_bytes[ect0_idx];
4335	info->tcpi_received_ce_bytes = tp->received_ecn_bytes[ce_idx];
4336
4337	unlock_sock_fast(sk, slow);
4338}
4339EXPORT_SYMBOL_GPL(tcp_get_info);
4340
4341static size_t tcp_opt_stats_get_size(void)
4342{
4343	return
4344		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */
4345		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */
4346		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */
4347		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */
4348		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */
4349		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */
4350		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */
4351		nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */
4352		nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */
4353		nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */
4354		nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */
4355		nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */
4356		nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */
4357		nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */
4358		nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */
4359		nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */
4360		nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */
4361		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */
4362		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
4363		nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
4364		nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
4365		nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */
4366		nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */
4367		nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */
4368		nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */
4369		nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */
4370		nla_total_size(sizeof(u32)) + /* TCP_NLA_REHASH */
4371		0;
4372}
4373
4374/* Returns TTL or hop limit of an incoming packet from skb. */
4375static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb)
4376{
4377	if (skb->protocol == htons(ETH_P_IP))
4378		return ip_hdr(skb)->ttl;
4379	else if (skb->protocol == htons(ETH_P_IPV6))
4380		return ipv6_hdr(skb)->hop_limit;
4381	else
4382		return 0;
4383}
4384
4385struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
4386					       const struct sk_buff *orig_skb,
4387					       const struct sk_buff *ack_skb)
4388{
4389	const struct tcp_sock *tp = tcp_sk(sk);
4390	struct sk_buff *stats;
4391	struct tcp_info info;
4392	unsigned long rate;
4393	u64 rate64;
4394
4395	stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
4396	if (!stats)
4397		return NULL;
4398
4399	tcp_get_info_chrono_stats(tp, &info);
4400	nla_put_u64_64bit(stats, TCP_NLA_BUSY,
4401			  info.tcpi_busy_time, TCP_NLA_PAD);
4402	nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
4403			  info.tcpi_rwnd_limited, TCP_NLA_PAD);
4404	nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
4405			  info.tcpi_sndbuf_limited, TCP_NLA_PAD);
4406	nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
4407			  tp->data_segs_out, TCP_NLA_PAD);
4408	nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
4409			  tp->total_retrans, TCP_NLA_PAD);
4410
4411	rate = READ_ONCE(sk->sk_pacing_rate);
4412	rate64 = (rate != ~0UL) ? rate : ~0ULL;
4413	nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
4414
4415	rate64 = tcp_compute_delivery_rate(tp);
4416	nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
4417
4418	nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp));
4419	nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
4420	nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
4421
4422	nla_put_u8(stats, TCP_NLA_RECUR_RETRANS,
4423		   READ_ONCE(inet_csk(sk)->icsk_retransmits));
4424	nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
4425	nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
4426	nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
4427	nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
4428
4429	nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
4430	nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
4431
4432	nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
4433			  TCP_NLA_PAD);
4434	nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
4435			  TCP_NLA_PAD);
4436	nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
4437	nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
4438	nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
4439	nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
4440	nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
4441		    max_t(int, 0, tp->write_seq - tp->snd_nxt));
4442	nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
4443			  TCP_NLA_PAD);
4444	if (ack_skb)
4445		nla_put_u8(stats, TCP_NLA_TTL,
4446			   tcp_skb_ttl_or_hop_limit(ack_skb));
4447
4448	nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash);
4449	return stats;
4450}
4451
4452int do_tcp_getsockopt(struct sock *sk, int level,
4453		      int optname, sockptr_t optval, sockptr_t optlen)
4454{
4455	struct inet_connection_sock *icsk = inet_csk(sk);
4456	struct tcp_sock *tp = tcp_sk(sk);
4457	struct net *net = sock_net(sk);
4458	int user_mss;
4459	int val, len;
4460
4461	if (copy_from_sockptr(&len, optlen, sizeof(int)))
4462		return -EFAULT;
4463
4464	if (len < 0)
4465		return -EINVAL;
4466
4467	len = min_t(unsigned int, len, sizeof(int));
4468
4469	switch (optname) {
4470	case TCP_MAXSEG:
4471		val = tp->mss_cache;
4472		user_mss = READ_ONCE(tp->rx_opt.user_mss);
4473		if (user_mss &&
4474		    ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
4475			val = user_mss;
4476		if (tp->repair)
4477			val = tp->rx_opt.mss_clamp;
4478		break;
4479	case TCP_NODELAY:
4480		val = !!(tp->nonagle&TCP_NAGLE_OFF);
4481		break;
4482	case TCP_CORK:
4483		val = !!(tp->nonagle&TCP_NAGLE_CORK);
4484		break;
4485	case TCP_KEEPIDLE:
4486		val = keepalive_time_when(tp) / HZ;
4487		break;
4488	case TCP_KEEPINTVL:
4489		val = keepalive_intvl_when(tp) / HZ;
4490		break;
4491	case TCP_KEEPCNT:
4492		val = keepalive_probes(tp);
4493		break;
4494	case TCP_SYNCNT:
4495		val = READ_ONCE(icsk->icsk_syn_retries) ? :
4496			READ_ONCE(net->ipv4.sysctl_tcp_syn_retries);
4497		break;
4498	case TCP_LINGER2:
4499		val = READ_ONCE(tp->linger2);
4500		if (val >= 0)
4501			val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ;
4502		break;
4503	case TCP_DEFER_ACCEPT:
4504		val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept);
4505		val = retrans_to_secs(val, TCP_TIMEOUT_INIT / HZ,
4506				      TCP_RTO_MAX / HZ);
4507		break;
4508	case TCP_WINDOW_CLAMP:
4509		val = READ_ONCE(tp->window_clamp);
4510		break;
4511	case TCP_INFO: {
4512		struct tcp_info info;
4513
4514		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4515			return -EFAULT;
4516
4517		tcp_get_info(sk, &info);
4518
4519		len = min_t(unsigned int, len, sizeof(info));
4520		if (copy_to_sockptr(optlen, &len, sizeof(int)))
4521			return -EFAULT;
4522		if (copy_to_sockptr(optval, &info, len))
4523			return -EFAULT;
4524		return 0;
4525	}
4526	case TCP_CC_INFO: {
4527		const struct tcp_congestion_ops *ca_ops;
4528		union tcp_cc_info info;
4529		size_t sz = 0;
4530		int attr;
4531
4532		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4533			return -EFAULT;
4534
4535		ca_ops = icsk->icsk_ca_ops;
4536		if (ca_ops && ca_ops->get_info)
4537			sz = ca_ops->get_info(sk, ~0U, &attr, &info);
4538
4539		len = min_t(unsigned int, len, sz);
4540		if (copy_to_sockptr(optlen, &len, sizeof(int)))
4541			return -EFAULT;
4542		if (copy_to_sockptr(optval, &info, len))
4543			return -EFAULT;
4544		return 0;
4545	}
4546	case TCP_QUICKACK:
4547		val = !inet_csk_in_pingpong_mode(sk);
4548		break;
4549
4550	case TCP_CONGESTION:
4551		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4552			return -EFAULT;
4553		len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
4554		if (copy_to_sockptr(optlen, &len, sizeof(int)))
4555			return -EFAULT;
4556		if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len))
4557			return -EFAULT;
4558		return 0;
4559
4560	case TCP_ULP:
4561		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4562			return -EFAULT;
4563		len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
4564		if (!icsk->icsk_ulp_ops) {
4565			len = 0;
4566			if (copy_to_sockptr(optlen, &len, sizeof(int)))
4567				return -EFAULT;
4568			return 0;
4569		}
4570		if (copy_to_sockptr(optlen, &len, sizeof(int)))
4571			return -EFAULT;
4572		if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len))
4573			return -EFAULT;
4574		return 0;
4575
4576	case TCP_FASTOPEN_KEY: {
4577		u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
4578		unsigned int key_len;
4579
4580		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4581			return -EFAULT;
4582
4583		key_len = tcp_fastopen_get_cipher(net, icsk, key) *
4584				TCP_FASTOPEN_KEY_LENGTH;
4585		len = min_t(unsigned int, len, key_len);
4586		if (copy_to_sockptr(optlen, &len, sizeof(int)))
4587			return -EFAULT;
4588		if (copy_to_sockptr(optval, key, len))
4589			return -EFAULT;
4590		return 0;
4591	}
4592	case TCP_THIN_LINEAR_TIMEOUTS:
4593		val = tp->thin_lto;
4594		break;
4595
4596	case TCP_THIN_DUPACK:
4597		val = 0;
4598		break;
4599
4600	case TCP_REPAIR:
4601		val = tp->repair;
4602		break;
4603
4604	case TCP_REPAIR_QUEUE:
4605		if (tp->repair)
4606			val = tp->repair_queue;
4607		else
4608			return -EINVAL;
4609		break;
4610
4611	case TCP_REPAIR_WINDOW: {
4612		struct tcp_repair_window opt;
4613
4614		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4615			return -EFAULT;
4616
4617		if (len != sizeof(opt))
4618			return -EINVAL;
4619
4620		if (!tp->repair)
4621			return -EPERM;
4622
4623		opt.snd_wl1	= tp->snd_wl1;
4624		opt.snd_wnd	= tp->snd_wnd;
4625		opt.max_window	= tp->max_window;
4626		opt.rcv_wnd	= tp->rcv_wnd;
4627		opt.rcv_wup	= tp->rcv_wup;
4628
4629		if (copy_to_sockptr(optval, &opt, len))
4630			return -EFAULT;
4631		return 0;
4632	}
4633	case TCP_QUEUE_SEQ:
4634		if (tp->repair_queue == TCP_SEND_QUEUE)
4635			val = tp->write_seq;
4636		else if (tp->repair_queue == TCP_RECV_QUEUE)
4637			val = tp->rcv_nxt;
4638		else
4639			return -EINVAL;
4640		break;
4641
4642	case TCP_USER_TIMEOUT:
4643		val = READ_ONCE(icsk->icsk_user_timeout);
4644		break;
4645
4646	case TCP_FASTOPEN:
4647		val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen);
4648		break;
4649
4650	case TCP_FASTOPEN_CONNECT:
4651		val = tp->fastopen_connect;
4652		break;
4653
4654	case TCP_FASTOPEN_NO_COOKIE:
4655		val = tp->fastopen_no_cookie;
4656		break;
4657
4658	case TCP_TX_DELAY:
4659		val = READ_ONCE(tp->tcp_tx_delay);
4660		break;
4661
4662	case TCP_TIMESTAMP:
4663		val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset);
4664		if (tp->tcp_usec_ts)
4665			val |= 1;
4666		else
4667			val &= ~1;
4668		break;
4669	case TCP_NOTSENT_LOWAT:
4670		val = READ_ONCE(tp->notsent_lowat);
4671		break;
4672	case TCP_INQ:
4673		val = tp->recvmsg_inq;
4674		break;
4675	case TCP_SAVE_SYN:
4676		val = tp->save_syn;
4677		break;
4678	case TCP_SAVED_SYN: {
4679		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4680			return -EFAULT;
4681
4682		sockopt_lock_sock(sk);
4683		if (tp->saved_syn) {
4684			if (len < tcp_saved_syn_len(tp->saved_syn)) {
4685				len = tcp_saved_syn_len(tp->saved_syn);
4686				if (copy_to_sockptr(optlen, &len, sizeof(int))) {
4687					sockopt_release_sock(sk);
4688					return -EFAULT;
4689				}
4690				sockopt_release_sock(sk);
4691				return -EINVAL;
4692			}
4693			len = tcp_saved_syn_len(tp->saved_syn);
4694			if (copy_to_sockptr(optlen, &len, sizeof(int))) {
4695				sockopt_release_sock(sk);
4696				return -EFAULT;
4697			}
4698			if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {
4699				sockopt_release_sock(sk);
4700				return -EFAULT;
4701			}
4702			tcp_saved_syn_free(tp);
4703			sockopt_release_sock(sk);
4704		} else {
4705			sockopt_release_sock(sk);
4706			len = 0;
4707			if (copy_to_sockptr(optlen, &len, sizeof(int)))
4708				return -EFAULT;
4709		}
4710		return 0;
4711	}
4712#ifdef CONFIG_MMU
4713	case TCP_ZEROCOPY_RECEIVE: {
4714		struct scm_timestamping_internal tss;
4715		struct tcp_zerocopy_receive zc = {};
4716		int err;
4717
4718		if (copy_from_sockptr(&len, optlen, sizeof(int)))
4719			return -EFAULT;
4720		if (len < 0 ||
4721		    len < offsetofend(struct tcp_zerocopy_receive, length))
4722			return -EINVAL;
4723		if (unlikely(len > sizeof(zc))) {
4724			err = check_zeroed_sockptr(optval, sizeof(zc),
4725						   len - sizeof(zc));
4726			if (err < 1)
4727				return err == 0 ? -EINVAL : err;
4728			len = sizeof(zc);
4729			if (copy_to_sockptr(optlen, &len, sizeof(int)))
4730				return -EFAULT;
4731		}
4732		if (copy_from_sockptr(&zc, optval, len))
4733			return -EFAULT;
4734		if (zc.reserved)
4735			return -EINVAL;
4736		if (zc.msg_flags &  ~(TCP_VALID_ZC_MSG_FLAGS))
4737			return -EINVAL;
4738		sockopt_lock_sock(sk);
4739		err = tcp_zerocopy_receive(sk, &zc, &tss);
4740		err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
4741							  &zc, &len, err);
4742		sockopt_release_sock(sk);
4743		if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
4744			goto zerocopy_rcv_cmsg;
4745		switch (len) {
4746		case offsetofend(struct tcp_zerocopy_receive, msg_flags):
4747			goto zerocopy_rcv_cmsg;
4748		case offsetofend(struct tcp_zerocopy_receive, msg_controllen):
4749		case offsetofend(struct tcp_zerocopy_receive, msg_control):
4750		case offsetofend(struct tcp_zerocopy_receive, flags):
4751		case offsetofend(struct tcp_zerocopy_receive, copybuf_len):
4752		case offsetofend(struct tcp_zerocopy_receive, copybuf_address):
4753		case offsetofend(struct tcp_zerocopy_receive, err):
4754			goto zerocopy_rcv_sk_err;
4755		case offsetofend(struct tcp_zerocopy_receive, inq):
4756			goto zerocopy_rcv_inq;
4757		case offsetofend(struct tcp_zerocopy_receive, length):
4758		default:
4759			goto zerocopy_rcv_out;
4760		}
4761zerocopy_rcv_cmsg:
4762		if (zc.msg_flags & TCP_CMSG_TS)
4763			tcp_zc_finalize_rx_tstamp(sk, &zc, &tss);
4764		else
4765			zc.msg_flags = 0;
4766zerocopy_rcv_sk_err:
4767		if (!err)
4768			zc.err = sock_error(sk);
4769zerocopy_rcv_inq:
4770		zc.inq = tcp_inq_hint(sk);
4771zerocopy_rcv_out:
4772		if (!err && copy_to_sockptr(optval, &zc, len))
4773			err = -EFAULT;
4774		return err;
4775	}
4776#endif
4777	case TCP_AO_REPAIR:
4778		if (!tcp_can_repair_sock(sk))
4779			return -EPERM;
4780		return tcp_ao_get_repair(sk, optval, optlen);
4781	case TCP_AO_GET_KEYS:
4782	case TCP_AO_INFO: {
4783		int err;
4784
4785		sockopt_lock_sock(sk);
4786		if (optname == TCP_AO_GET_KEYS)
4787			err = tcp_ao_get_mkts(sk, optval, optlen);
4788		else
4789			err = tcp_ao_get_sock_info(sk, optval, optlen);
4790		sockopt_release_sock(sk);
4791
4792		return err;
4793	}
4794	case TCP_IS_MPTCP:
4795		val = 0;
4796		break;
4797	case TCP_RTO_MAX_MS:
4798		val = jiffies_to_msecs(tcp_rto_max(sk));
4799		break;
4800	case TCP_RTO_MIN_US:
4801		val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_rto_min));
4802		break;
4803	case TCP_DELACK_MAX_US:
4804		val = jiffies_to_usecs(READ_ONCE(inet_csk(sk)->icsk_delack_max));
4805		break;
4806	default:
4807		return -ENOPROTOOPT;
4808	}
4809
4810	if (copy_to_sockptr(optlen, &len, sizeof(int)))
4811		return -EFAULT;
4812	if (copy_to_sockptr(optval, &val, len))
4813		return -EFAULT;
4814	return 0;
4815}
4816
4817bool tcp_bpf_bypass_getsockopt(int level, int optname)
4818{
4819	/* TCP do_tcp_getsockopt has optimized getsockopt implementation
4820	 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
4821	 */
4822	if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
4823		return true;
4824
4825	return false;
4826}
4827EXPORT_IPV6_MOD(tcp_bpf_bypass_getsockopt);
4828
4829int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
4830		   int __user *optlen)
4831{
4832	struct inet_connection_sock *icsk = inet_csk(sk);
4833
4834	if (level != SOL_TCP)
4835		/* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
4836		return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname,
4837								optval, optlen);
4838	return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
4839				 USER_SOCKPTR(optlen));
4840}
4841EXPORT_IPV6_MOD(tcp_getsockopt);
4842
4843#ifdef CONFIG_TCP_MD5SIG
4844void tcp_md5_hash_skb_data(struct md5_ctx *ctx, const struct sk_buff *skb,
4845			   unsigned int header_len)
4846{
4847	const unsigned int head_data_len = skb_headlen(skb) > header_len ?
4848					   skb_headlen(skb) - header_len : 0;
4849	const struct skb_shared_info *shi = skb_shinfo(skb);
4850	struct sk_buff *frag_iter;
4851	unsigned int i;
4852
4853	md5_update(ctx, (const u8 *)tcp_hdr(skb) + header_len, head_data_len);
4854
4855	for (i = 0; i < shi->nr_frags; ++i) {
4856		const skb_frag_t *f = &shi->frags[i];
4857		u32 p_off, p_len, copied;
4858		const void *vaddr;
4859		struct page *p;
4860
4861		skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
4862				      p, p_off, p_len, copied) {
4863			vaddr = kmap_local_page(p);
4864			md5_update(ctx, vaddr + p_off, p_len);
4865			kunmap_local(vaddr);
4866		}
4867	}
4868
4869	skb_walk_frags(skb, frag_iter)
4870		tcp_md5_hash_skb_data(ctx, frag_iter, 0);
4871}
4872EXPORT_IPV6_MOD(tcp_md5_hash_skb_data);
4873
4874void tcp_md5_hash_key(struct md5_ctx *ctx,
4875		      const struct tcp_md5sig_key *key)
4876{
4877	u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
4878
4879	/* We use data_race() because tcp_md5_do_add() might change
4880	 * key->key under us
4881	 */
4882	data_race(({ md5_update(ctx, key->key, keylen), 0; }));
4883}
4884EXPORT_IPV6_MOD(tcp_md5_hash_key);
4885
4886/* Called with rcu_read_lock() */
4887static enum skb_drop_reason
4888tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
4889		     const void *saddr, const void *daddr,
4890		     int family, int l3index, const __u8 *hash_location)
4891{
4892	/* This gets called for each TCP segment that has TCP-MD5 option.
4893	 * We have 2 drop cases:
4894	 * o An MD5 signature is present, but we're not expecting one.
4895	 * o The MD5 signature is wrong.
4896	 */
4897	const struct tcp_sock *tp = tcp_sk(sk);
4898	struct tcp_md5sig_key *key;
4899	u8 newhash[16];
4900
4901	key = tcp_md5_do_lookup(sk, l3index, saddr, family);
4902	if (!key) {
4903		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
4904		trace_tcp_hash_md5_unexpected(sk, skb);
4905		return SKB_DROP_REASON_TCP_MD5UNEXPECTED;
4906	}
4907
4908	/* Check the signature.
4909	 * To support dual stack listeners, we need to handle
4910	 * IPv4-mapped case.
4911	 */
4912	if (family == AF_INET)
4913		tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
4914	else
4915		tp->af_specific->calc_md5_hash(newhash, key, NULL, skb);
4916	if (crypto_memneq(hash_location, newhash, 16)) {
4917		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
4918		trace_tcp_hash_md5_mismatch(sk, skb);
4919		return SKB_DROP_REASON_TCP_MD5FAILURE;
4920	}
4921	return SKB_NOT_DROPPED_YET;
4922}
4923#else
4924static inline enum skb_drop_reason
4925tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
4926		     const void *saddr, const void *daddr,
4927		     int family, int l3index, const __u8 *hash_location)
4928{
4929	return SKB_NOT_DROPPED_YET;
4930}
4931
4932#endif
4933
4934/* Called with rcu_read_lock() */
4935enum skb_drop_reason
4936tcp_inbound_hash(struct sock *sk, const struct request_sock *req,
4937		 const struct sk_buff *skb,
4938		 const void *saddr, const void *daddr,
4939		 int family, int dif, int sdif)
4940{
4941	const struct tcphdr *th = tcp_hdr(skb);
4942	const struct tcp_ao_hdr *aoh;
4943	const __u8 *md5_location;
4944	int l3index;
4945
4946	/* Invalid option or two times meet any of auth options */
4947	if (tcp_parse_auth_options(th, &md5_location, &aoh)) {
4948		trace_tcp_hash_bad_header(sk, skb);
4949		return SKB_DROP_REASON_TCP_AUTH_HDR;
4950	}
4951
4952	if (req) {
4953		if (tcp_rsk_used_ao(req) != !!aoh) {
4954			u8 keyid, rnext, maclen;
4955
4956			if (aoh) {
4957				keyid = aoh->keyid;
4958				rnext = aoh->rnext_keyid;
4959				maclen = tcp_ao_hdr_maclen(aoh);
4960			} else {
4961				keyid = rnext = maclen = 0;
4962			}
4963
4964			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
4965			trace_tcp_ao_handshake_failure(sk, skb, keyid, rnext, maclen);
4966			return SKB_DROP_REASON_TCP_AOFAILURE;
4967		}
4968	}
4969
4970	/* sdif set, means packet ingressed via a device
4971	 * in an L3 domain and dif is set to the l3mdev
4972	 */
4973	l3index = sdif ? dif : 0;
4974
4975	/* Fast path: unsigned segments */
4976	if (likely(!md5_location && !aoh)) {
4977		/* Drop if there's TCP-MD5 or TCP-AO key with any rcvid/sndid
4978		 * for the remote peer. On TCP-AO established connection
4979		 * the last key is impossible to remove, so there's
4980		 * always at least one current_key.
4981		 */
4982		if (tcp_ao_required(sk, saddr, family, l3index, true)) {
4983			trace_tcp_hash_ao_required(sk, skb);
4984			return SKB_DROP_REASON_TCP_AONOTFOUND;
4985		}
4986		if (unlikely(tcp_md5_do_lookup(sk, l3index, saddr, family))) {
4987			NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5NOTFOUND);
4988			trace_tcp_hash_md5_required(sk, skb);
4989			return SKB_DROP_REASON_TCP_MD5NOTFOUND;
4990		}
4991		return SKB_NOT_DROPPED_YET;
4992	}
4993
4994	if (aoh)
4995		return tcp_inbound_ao_hash(sk, skb, family, req, l3index, aoh);
4996
4997	return tcp_inbound_md5_hash(sk, skb, saddr, daddr, family,
4998				    l3index, md5_location);
4999}
5000EXPORT_IPV6_MOD_GPL(tcp_inbound_hash);
5001
5002void tcp_done(struct sock *sk)
5003{
5004	struct request_sock *req;
5005
5006	/* We might be called with a new socket, after
5007	 * inet_csk_prepare_forced_close() has been called
5008	 * so we can not use lockdep_sock_is_held(sk)
5009	 */
5010	req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
5011
5012	if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
5013		TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
5014
5015	tcp_set_state(sk, TCP_CLOSE);
5016	tcp_clear_xmit_timers(sk);
5017	if (req)
5018		reqsk_fastopen_remove(sk, req, false);
5019
5020	WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
5021
5022	if (!sock_flag(sk, SOCK_DEAD))
5023		sk->sk_state_change(sk);
5024	else
5025		inet_csk_destroy_sock(sk);
5026}
5027EXPORT_SYMBOL_GPL(tcp_done);
5028
5029int tcp_abort(struct sock *sk, int err)
5030{
5031	int state = inet_sk_state_load(sk);
5032
5033	if (state == TCP_NEW_SYN_RECV) {
5034		struct request_sock *req = inet_reqsk(sk);
5035
5036		local_bh_disable();
5037		inet_csk_reqsk_queue_drop(req->rsk_listener, req);
5038		local_bh_enable();
5039		return 0;
5040	}
5041	if (state == TCP_TIME_WAIT) {
5042		struct inet_timewait_sock *tw = inet_twsk(sk);
5043
5044		refcount_inc(&tw->tw_refcnt);
5045		local_bh_disable();
5046		inet_twsk_deschedule_put(tw);
5047		local_bh_enable();
5048		return 0;
5049	}
5050
5051	/* BPF context ensures sock locking. */
5052	if (!has_current_bpf_ctx())
5053		/* Don't race with userspace socket closes such as tcp_close. */
5054		lock_sock(sk);
5055
5056	/* Avoid closing the same socket twice. */
5057	if (sk->sk_state == TCP_CLOSE) {
5058		if (!has_current_bpf_ctx())
5059			release_sock(sk);
5060		return -ENOENT;
5061	}
5062
5063	if (sk->sk_state == TCP_LISTEN) {
5064		tcp_set_state(sk, TCP_CLOSE);
5065		inet_csk_listen_stop(sk);
5066	}
5067
5068	/* Don't race with BH socket closes such as inet_csk_listen_stop. */
5069	local_bh_disable();
5070	bh_lock_sock(sk);
5071
5072	if (tcp_need_reset(sk->sk_state))
5073		tcp_send_active_reset(sk, GFP_ATOMIC,
5074				      SK_RST_REASON_TCP_STATE);
5075	tcp_done_with_error(sk, err);
5076
5077	bh_unlock_sock(sk);
5078	local_bh_enable();
5079	if (!has_current_bpf_ctx())
5080		release_sock(sk);
5081	return 0;
5082}
5083EXPORT_SYMBOL_GPL(tcp_abort);
5084
5085extern struct tcp_congestion_ops tcp_reno;
5086
5087static __initdata unsigned long thash_entries;
5088static int __init set_thash_entries(char *str)
5089{
5090	ssize_t ret;
5091
5092	if (!str)
5093		return 0;
5094
5095	ret = kstrtoul(str, 0, &thash_entries);
5096	if (ret)
5097		return 0;
5098
5099	return 1;
5100}
5101__setup("thash_entries=", set_thash_entries);
5102
5103static void __init tcp_init_mem(void)
5104{
5105	unsigned long limit = nr_free_buffer_pages() / 16;
5106
5107	limit = max(limit, 128UL);
5108	sysctl_tcp_mem[0] = limit / 4 * 3;		/* 4.68 % */
5109	sysctl_tcp_mem[1] = limit;			/* 6.25 % */
5110	sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;	/* 9.37 % */
5111}
5112
5113static void __init tcp_struct_check(void)
5114{
5115	/* TX read-mostly hotpath cache lines */
5116	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window);
5117	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh);
5118	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering);
5119	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat);
5120	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs);
5121	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint);
5122#if IS_ENABLED(CONFIG_TLS_DEVICE)
5123	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, tcp_clean_acked);
5124#endif
5125
5126	/* TXRX read-mostly hotpath cache lines */
5127	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset);
5128	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd);
5129	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache);
5130	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd);
5131	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out);
5132	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out);
5133	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out);
5134	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio);
5135
5136	/* RX read-mostly hotpath cache lines */
5137	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq);
5138	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1);
5139	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq);
5140	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us);
5141	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out);
5142	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss);
5143	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data);
5144	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost);
5145	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min);
5146	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue);
5147	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
5148
5149	/* TX read-write hotpath cache lines */
5150	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out);
5151	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
5152	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
5153	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
5154	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
5155	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
5156	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq);
5157	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq);
5158	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime);
5159	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us);
5160	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
5161	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, accecn_opt_tstamp);
5162	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
5163	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
5164	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
5165	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
5166
5167	/* TXRX read-write hotpath cache lines */
5168	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags);
5169	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache);
5170	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp);
5171	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt);
5172	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt);
5173	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una);
5174	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp);
5175	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
5176	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
5177	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
5178	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered);
5179	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce);
5180	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ce);
5181	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ecn_bytes);
5182	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
5183	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd);
5184	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_tstamp);
5185	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
5186
5187	/* RX read-write hotpath cache lines */
5188	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
5189	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in);
5190	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
5191	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
5192	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out);
5193	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq);
5194	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered);
5195	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us);
5196	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
5197	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_ecn_bytes);
5198	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp);
5199	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp);
5200	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked);
5201	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
5202	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
5203}
5204
5205void __init tcp_init(void)
5206{
5207	int max_rshare, max_wshare, cnt;
5208	unsigned long limit;
5209	unsigned int i;
5210
5211	BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
5212	BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
5213		     sizeof_field(struct sk_buff, cb));
5214
5215	tcp_struct_check();
5216
5217	percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
5218
5219	timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE);
5220	mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
5221
5222	inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
5223			    thash_entries, 21,  /* one slot per 2 MB*/
5224			    0, 64 * 1024);
5225	tcp_hashinfo.bind_bucket_cachep =
5226		kmem_cache_create("tcp_bind_bucket",
5227				  sizeof(struct inet_bind_bucket), 0,
5228				  SLAB_HWCACHE_ALIGN | SLAB_PANIC |
5229				  SLAB_ACCOUNT,
5230				  NULL);
5231	tcp_hashinfo.bind2_bucket_cachep =
5232		kmem_cache_create("tcp_bind2_bucket",
5233				  sizeof(struct inet_bind2_bucket), 0,
5234				  SLAB_HWCACHE_ALIGN | SLAB_PANIC |
5235				  SLAB_ACCOUNT,
5236				  NULL);
5237
5238	/* Size and allocate the main established and bind bucket
5239	 * hash tables.
5240	 *
5241	 * The methodology is similar to that of the buffer cache.
5242	 */
5243	tcp_hashinfo.ehash =
5244		alloc_large_system_hash("TCP established",
5245					sizeof(struct inet_ehash_bucket),
5246					thash_entries,
5247					17, /* one slot per 128 KB of memory */
5248					0,
5249					NULL,
5250					&tcp_hashinfo.ehash_mask,
5251					0,
5252					thash_entries ? 0 : 512 * 1024);
5253	for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
5254		INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
5255
5256	if (inet_ehash_locks_alloc(&tcp_hashinfo))
5257		panic("TCP: failed to alloc ehash_locks");
5258	tcp_hashinfo.bhash =
5259		alloc_large_system_hash("TCP bind",
5260					2 * sizeof(struct inet_bind_hashbucket),
5261					tcp_hashinfo.ehash_mask + 1,
5262					17, /* one slot per 128 KB of memory */
5263					0,
5264					&tcp_hashinfo.bhash_size,
5265					NULL,
5266					0,
5267					64 * 1024);
5268	tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
5269	tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;
5270	for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
5271		spin_lock_init(&tcp_hashinfo.bhash[i].lock);
5272		INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
5273		spin_lock_init(&tcp_hashinfo.bhash2[i].lock);
5274		INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
5275	}
5276
5277	tcp_hashinfo.pernet = false;
5278
5279	cnt = tcp_hashinfo.ehash_mask + 1;
5280	sysctl_tcp_max_orphans = cnt / 2;
5281
5282	tcp_init_mem();
5283	/* Set per-socket limits to no more than 1/128 the pressure threshold */
5284	limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
5285	max_wshare = min(4UL*1024*1024, limit);
5286	max_rshare = min(32UL*1024*1024, limit);
5287
5288	init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;
5289	init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
5290	init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
5291
5292	init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE;
5293	init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
5294	init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
5295
5296	pr_info("Hash tables configured (established %u bind %u)\n",
5297		tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
5298
5299	tcp_v4_init();
5300	tcp_metrics_init();
5301	BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
5302	tcp_tsq_work_init();
5303	mptcp_init();
5304}