tcp_ipv4.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * INET		An implementation of the TCP/IP protocol suite for the LINUX
   4 *		operating system.  INET is implemented using the  BSD Socket
   5 *		interface as the means of communication with the user level.
   6 *
   7 *		Implementation of the Transmission Control Protocol(TCP).
   8 *
   9 *		IPv4 specific functions
  10 *
  11 *		code split from:
  12 *		linux/ipv4/tcp.c
  13 *		linux/ipv4/tcp_input.c
  14 *		linux/ipv4/tcp_output.c
  15 *
  16 *		See tcp.c for author information
  17 */
  18
  19/*
  20 * Changes:
  21 *		David S. Miller	:	New socket lookup architecture.
  22 *					This code is dedicated to John Dyson.
  23 *		David S. Miller :	Change semantics of established hash,
  24 *					half is devoted to TIME_WAIT sockets
  25 *					and the rest go in the other half.
  26 *		Andi Kleen :		Add support for syncookies and fixed
  27 *					some bugs: ip options weren't passed to
  28 *					the TCP layer, missed a check for an
  29 *					ACK bit.
  30 *		Andi Kleen :		Implemented fast path mtu discovery.
  31 *	     				Fixed many serious bugs in the
  32 *					request_sock handling and moved
  33 *					most of it into the af independent code.
  34 *					Added tail drop and some other bugfixes.
  35 *					Added new listen semantics.
  36 *		Mike McLagan	:	Routing by source
  37 *	Juan Jose Ciarlante:		ip_dynaddr bits
  38 *		Andi Kleen:		various fixes.
  39 *	Vitaly E. Lavrov	:	Transparent proxy revived after year
  40 *					coma.
  41 *	Andi Kleen		:	Fix new listen.
  42 *	Andi Kleen		:	Fix accept error reporting.
  43 *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
  44 *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
  45 *					a single port at the same time.
  46 */
  47
  48#define pr_fmt(fmt) "TCP: " fmt
  49
  50#include <linux/bottom_half.h>
  51#include <linux/types.h>
  52#include <linux/fcntl.h>
  53#include <linux/module.h>
  54#include <linux/random.h>
  55#include <linux/cache.h>
  56#include <linux/fips.h>
  57#include <linux/jhash.h>
  58#include <linux/init.h>
  59#include <linux/times.h>
  60#include <linux/slab.h>
  61#include <linux/sched.h>
  62#include <linux/sock_diag.h>
  63
  64#include <net/aligned_data.h>
  65#include <net/net_namespace.h>
  66#include <net/icmp.h>
  67#include <net/inet_hashtables.h>
  68#include <net/tcp.h>
  69#include <net/tcp_ecn.h>
  70#include <net/transp_v6.h>
  71#include <net/ipv6.h>
  72#include <net/inet_common.h>
  73#include <net/inet_ecn.h>
  74#include <net/timewait_sock.h>
  75#include <net/xfrm.h>
  76#include <net/secure_seq.h>
  77#include <net/busy_poll.h>
  78#include <net/rstreason.h>
  79#include <net/psp.h>
  80
  81#include <linux/inet.h>
  82#include <linux/ipv6.h>
  83#include <linux/stddef.h>
  84#include <linux/proc_fs.h>
  85#include <linux/seq_file.h>
  86#include <linux/inetdevice.h>
  87#include <linux/btf_ids.h>
  88#include <linux/skbuff_ref.h>
  89
  90#include <crypto/md5.h>
  91#include <crypto/utils.h>
  92
  93#include <trace/events/tcp.h>
  94
  95#ifdef CONFIG_TCP_MD5SIG
  96static void tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
  97				__be32 daddr, __be32 saddr, const struct tcphdr *th);
  98#endif
  99
 100struct inet_hashinfo tcp_hashinfo;
 101
 102static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = {
 103	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
 104};
 105
 106static DEFINE_MUTEX(tcp_exit_batch_mutex);
 107
 108static union tcp_seq_and_ts_off
 109tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
 110{
 111	return secure_tcp_seq_and_ts_off(net,
 112					 ip_hdr(skb)->daddr,
 113					 ip_hdr(skb)->saddr,
 114					 tcp_hdr(skb)->dest,
 115					 tcp_hdr(skb)->source);
 116}
 117
 118int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
 119{
 120	int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
 121	const struct inet_timewait_sock *tw = inet_twsk(sktw);
 122	const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
 123	struct tcp_sock *tp = tcp_sk(sk);
 124	int ts_recent_stamp;
 125	u32 reuse_thresh;
 126
 127	if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2)
 128		reuse = 0;
 129
 130	if (reuse == 2) {
 131		/* Still does not detect *everything* that goes through
 132		 * lo, since we require a loopback src or dst address
 133		 * or direct binding to 'lo' interface.
 134		 */
 135		bool loopback = false;
 136		if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
 137			loopback = true;
 138#if IS_ENABLED(CONFIG_IPV6)
 139		if (tw->tw_family == AF_INET6) {
 140			if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
 141			    ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
 142			    ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
 143			    ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
 144				loopback = true;
 145		} else
 146#endif
 147		{
 148			if (ipv4_is_loopback(tw->tw_daddr) ||
 149			    ipv4_is_loopback(tw->tw_rcv_saddr))
 150				loopback = true;
 151		}
 152		if (!loopback)
 153			reuse = 0;
 154	}
 155
 156	/* With PAWS, it is safe from the viewpoint
 157	   of data integrity. Even without PAWS it is safe provided sequence
 158	   spaces do not overlap i.e. at data rates <= 80Mbit/sec.
 159
 160	   Actually, the idea is close to VJ's one, only timestamp cache is
 161	   held not per host, but per port pair and TW bucket is used as state
 162	   holder.
 163
 164	   If TW bucket has been already destroyed we fall back to VJ's scheme
 165	   and use initial timestamp retrieved from peer table.
 166	 */
 167	ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
 168	reuse_thresh = READ_ONCE(tw->tw_entry_stamp) +
 169		       READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay);
 170	if (ts_recent_stamp &&
 171	    (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) {
 172		/* inet_twsk_hashdance_schedule() sets sk_refcnt after putting twsk
 173		 * and releasing the bucket lock.
 174		 */
 175		if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt)))
 176			return 0;
 177
 178		/* In case of repair and re-using TIME-WAIT sockets we still
 179		 * want to be sure that it is safe as above but honor the
 180		 * sequence numbers and time stamps set as part of the repair
 181		 * process.
 182		 *
 183		 * Without this check re-using a TIME-WAIT socket with TCP
 184		 * repair would accumulate a -1 on the repair assigned
 185		 * sequence number. The first time it is reused the sequence
 186		 * is -1, the second time -2, etc. This fixes that issue
 187		 * without appearing to create any others.
 188		 */
 189		if (likely(!tp->repair)) {
 190			u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
 191
 192			if (!seq)
 193				seq = 1;
 194			WRITE_ONCE(tp->write_seq, seq);
 195			tp->rx_opt.ts_recent	   = READ_ONCE(tcptw->tw_ts_recent);
 196			tp->rx_opt.ts_recent_stamp = ts_recent_stamp;
 197		}
 198
 199		return 1;
 200	}
 201
 202	return 0;
 203}
 204EXPORT_IPV6_MOD_GPL(tcp_twsk_unique);
 205
 206static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
 207			      int addr_len)
 208{
 209	/* This check is replicated from tcp_v4_connect() and intended to
 210	 * prevent BPF program called below from accessing bytes that are out
 211	 * of the bound specified by user in addr_len.
 212	 */
 213	if (addr_len < sizeof(struct sockaddr_in))
 214		return -EINVAL;
 215
 216	sock_owned_by_me(sk);
 217
 218	return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, &addr_len);
 219}
 220
 221/* This will initiate an outgoing connection. */
 222int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len)
 223{
 224	struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
 225	struct inet_timewait_death_row *tcp_death_row;
 226	struct inet_sock *inet = inet_sk(sk);
 227	struct tcp_sock *tp = tcp_sk(sk);
 228	struct ip_options_rcu *inet_opt;
 229	struct net *net = sock_net(sk);
 230	__be16 orig_sport, orig_dport;
 231	__be32 daddr, nexthop;
 232	struct flowi4 *fl4;
 233	struct rtable *rt;
 234	int err;
 235
 236	if (addr_len < sizeof(struct sockaddr_in))
 237		return -EINVAL;
 238
 239	if (usin->sin_family != AF_INET)
 240		return -EAFNOSUPPORT;
 241
 242	nexthop = daddr = usin->sin_addr.s_addr;
 243	inet_opt = rcu_dereference_protected(inet->inet_opt,
 244					     lockdep_sock_is_held(sk));
 245	if (inet_opt && inet_opt->opt.srr) {
 246		if (!daddr)
 247			return -EINVAL;
 248		nexthop = inet_opt->opt.faddr;
 249	}
 250
 251	orig_sport = inet->inet_sport;
 252	orig_dport = usin->sin_port;
 253	fl4 = &inet->cork.fl.u.ip4;
 254	rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
 255			      sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
 256			      orig_dport, sk);
 257	if (IS_ERR(rt)) {
 258		err = PTR_ERR(rt);
 259		if (err == -ENETUNREACH)
 260			IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
 261		return err;
 262	}
 263
 264	if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
 265		ip_rt_put(rt);
 266		return -ENETUNREACH;
 267	}
 268
 269	if (!inet_opt || !inet_opt->opt.srr)
 270		daddr = fl4->daddr;
 271
 272	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 273
 274	if (!inet->inet_saddr) {
 275		err = inet_bhash2_update_saddr(sk,  &fl4->saddr, AF_INET);
 276		if (err) {
 277			ip_rt_put(rt);
 278			return err;
 279		}
 280	} else {
 281		sk_rcv_saddr_set(sk, inet->inet_saddr);
 282	}
 283
 284	if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
 285		/* Reset inherited state */
 286		tp->rx_opt.ts_recent	   = 0;
 287		tp->rx_opt.ts_recent_stamp = 0;
 288		if (likely(!tp->repair))
 289			WRITE_ONCE(tp->write_seq, 0);
 290	}
 291
 292	inet->inet_dport = usin->sin_port;
 293	sk_daddr_set(sk, daddr);
 294
 295	inet_csk(sk)->icsk_ext_hdr_len = psp_sk_overhead(sk);
 296	if (inet_opt)
 297		inet_csk(sk)->icsk_ext_hdr_len += inet_opt->opt.optlen;
 298
 299	tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
 300
 301	/* Socket identity is still unknown (sport may be zero).
 302	 * However we set state to SYN-SENT and not releasing socket
 303	 * lock select source port, enter ourselves into the hash tables and
 304	 * complete initialization after this.
 305	 */
 306	tcp_set_state(sk, TCP_SYN_SENT);
 307	err = inet_hash_connect(tcp_death_row, sk);
 308	if (err)
 309		goto failure;
 310
 311	sk_set_txhash(sk);
 312
 313	rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
 314			       inet->inet_sport, inet->inet_dport, sk);
 315	if (IS_ERR(rt)) {
 316		err = PTR_ERR(rt);
 317		rt = NULL;
 318		goto failure;
 319	}
 320	tp->tcp_usec_ts = dst_tcp_usec_ts(&rt->dst);
 321	/* OK, now commit destination to socket.  */
 322	sk->sk_gso_type = SKB_GSO_TCPV4;
 323	sk_setup_caps(sk, &rt->dst);
 324	rt = NULL;
 325
 326	if (likely(!tp->repair)) {
 327		union tcp_seq_and_ts_off st;
 328
 329		st = secure_tcp_seq_and_ts_off(net,
 330					       inet->inet_saddr,
 331					       inet->inet_daddr,
 332					       inet->inet_sport,
 333					       usin->sin_port);
 334		if (!tp->write_seq)
 335			WRITE_ONCE(tp->write_seq, st.seq);
 336		WRITE_ONCE(tp->tsoffset, st.ts_off);
 337	}
 338
 339	atomic_set(&inet->inet_id, get_random_u16());
 340
 341	if (tcp_fastopen_defer_connect(sk, &err))
 342		return err;
 343	if (err)
 344		goto failure;
 345
 346	err = tcp_connect(sk);
 347
 348	if (err)
 349		goto failure;
 350
 351	return 0;
 352
 353failure:
 354	/*
 355	 * This unhashes the socket and releases the local port,
 356	 * if necessary.
 357	 */
 358	tcp_set_state(sk, TCP_CLOSE);
 359	inet_bhash2_reset_saddr(sk);
 360	ip_rt_put(rt);
 361	sk->sk_route_caps = 0;
 362	inet->inet_dport = 0;
 363	return err;
 364}
 365EXPORT_IPV6_MOD(tcp_v4_connect);
 366
 367/*
 368 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
 369 * It can be called through tcp_release_cb() if socket was owned by user
 370 * at the time tcp_v4_err() was called to handle ICMP message.
 371 */
 372void tcp_v4_mtu_reduced(struct sock *sk)
 373{
 374	struct inet_sock *inet = inet_sk(sk);
 375	struct dst_entry *dst;
 376	u32 mtu;
 377
 378	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 379		return;
 380	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 381	dst = inet_csk_update_pmtu(sk, mtu);
 382	if (!dst)
 383		return;
 384
 385	/* Something is about to be wrong... Remember soft error
 386	 * for the case, if this connection will not able to recover.
 387	 */
 388	if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
 389		WRITE_ONCE(sk->sk_err_soft, EMSGSIZE);
 390
 391	mtu = dst_mtu(dst);
 392
 393	if (inet->pmtudisc != IP_PMTUDISC_DONT &&
 394	    ip_sk_accept_pmtu(sk) &&
 395	    inet_csk(sk)->icsk_pmtu_cookie > mtu) {
 396		tcp_sync_mss(sk, mtu);
 397
 398		/* Resend the TCP packet because it's
 399		 * clear that the old packet has been
 400		 * dropped. This is the new "fast" path mtu
 401		 * discovery.
 402		 */
 403		tcp_simple_retransmit(sk);
 404	} /* else let the usual retransmit timer handle it */
 405}
 406EXPORT_IPV6_MOD(tcp_v4_mtu_reduced);
 407
 408static void do_redirect(struct sk_buff *skb, struct sock *sk)
 409{
 410	struct dst_entry *dst = __sk_dst_check(sk, 0);
 411
 412	if (dst)
 413		dst->ops->redirect(dst, sk, skb);
 414}
 415
 416
 417/* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
 418void tcp_req_err(struct sock *sk, u32 seq, bool abort)
 419{
 420	struct request_sock *req = inet_reqsk(sk);
 421	struct net *net = sock_net(sk);
 422
 423	/* ICMPs are not backlogged, hence we cannot get
 424	 * an established socket here.
 425	 */
 426	if (seq != tcp_rsk(req)->snt_isn) {
 427		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 428	} else if (abort) {
 429		/*
 430		 * Still in SYN_RECV, just remove it silently.
 431		 * There is no good way to pass the error to the newly
 432		 * created socket, and POSIX does not want network
 433		 * errors returned from accept().
 434		 */
 435		inet_csk_reqsk_queue_drop(req->rsk_listener, req);
 436		tcp_listendrop(req->rsk_listener);
 437	}
 438	reqsk_put(req);
 439}
 440EXPORT_IPV6_MOD(tcp_req_err);
 441
 442/* TCP-LD (RFC 6069) logic */
 443void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
 444{
 445	struct inet_connection_sock *icsk = inet_csk(sk);
 446	struct tcp_sock *tp = tcp_sk(sk);
 447	struct sk_buff *skb;
 448	s32 remaining;
 449	u32 delta_us;
 450
 451	if (sock_owned_by_user(sk))
 452		return;
 453
 454	if (seq != tp->snd_una  || !icsk->icsk_retransmits ||
 455	    !icsk->icsk_backoff)
 456		return;
 457
 458	skb = tcp_rtx_queue_head(sk);
 459	if (WARN_ON_ONCE(!skb))
 460		return;
 461
 462	icsk->icsk_backoff--;
 463	icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
 464	icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk));
 465
 466	tcp_mstamp_refresh(tp);
 467	delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
 468	remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
 469
 470	if (remaining > 0) {
 471		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false);
 472	} else {
 473		/* RTO revert clocked out retransmission.
 474		 * Will retransmit now.
 475		 */
 476		tcp_retransmit_timer(sk);
 477	}
 478}
 479EXPORT_IPV6_MOD(tcp_ld_RTO_revert);
 480
 481/*
 482 * This routine is called by the ICMP module when it gets some
 483 * sort of error condition.  If err < 0 then the socket should
 484 * be closed and the error returned to the user.  If err > 0
 485 * it's just the icmp type << 8 | icmp code.  After adjustment
 486 * header points to the first 8 bytes of the tcp header.  We need
 487 * to find the appropriate port.
 488 *
 489 * The locking strategy used here is very "optimistic". When
 490 * someone else accesses the socket the ICMP is just dropped
 491 * and for some paths there is no check at all.
 492 * A more general error queue to queue errors for later handling
 493 * is probably better.
 494 *
 495 */
 496
 497int tcp_v4_err(struct sk_buff *skb, u32 info)
 498{
 499	const struct iphdr *iph = (const struct iphdr *)skb->data;
 500	struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
 501	struct net *net = dev_net_rcu(skb->dev);
 502	const int type = icmp_hdr(skb)->type;
 503	const int code = icmp_hdr(skb)->code;
 504	struct request_sock *fastopen;
 505	struct tcp_sock *tp;
 506	u32 seq, snd_una;
 507	struct sock *sk;
 508	int err;
 509
 510	sk = __inet_lookup_established(net, iph->daddr, th->dest, iph->saddr,
 511				       ntohs(th->source), inet_iif(skb), 0);
 512	if (!sk) {
 513		__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
 514		return -ENOENT;
 515	}
 516	if (sk->sk_state == TCP_TIME_WAIT) {
 517		/* To increase the counter of ignored icmps for TCP-AO */
 518		tcp_ao_ignore_icmp(sk, AF_INET, type, code);
 519		inet_twsk_put(inet_twsk(sk));
 520		return 0;
 521	}
 522	seq = ntohl(th->seq);
 523	if (sk->sk_state == TCP_NEW_SYN_RECV) {
 524		tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
 525				     type == ICMP_TIME_EXCEEDED ||
 526				     (type == ICMP_DEST_UNREACH &&
 527				      (code == ICMP_NET_UNREACH ||
 528				       code == ICMP_HOST_UNREACH)));
 529		return 0;
 530	}
 531
 532	if (tcp_ao_ignore_icmp(sk, AF_INET, type, code)) {
 533		sock_put(sk);
 534		return 0;
 535	}
 536
 537	bh_lock_sock(sk);
 538	/* If too many ICMPs get dropped on busy
 539	 * servers this needs to be solved differently.
 540	 * We do take care of PMTU discovery (RFC1191) special case :
 541	 * we can receive locally generated ICMP messages while socket is held.
 542	 */
 543	if (sock_owned_by_user(sk)) {
 544		if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
 545			__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 546	}
 547	if (sk->sk_state == TCP_CLOSE)
 548		goto out;
 549
 550	if (static_branch_unlikely(&ip4_min_ttl)) {
 551		/* min_ttl can be changed concurrently from do_ip_setsockopt() */
 552		if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
 553			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 554			goto out;
 555		}
 556	}
 557
 558	tp = tcp_sk(sk);
 559	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 560	fastopen = rcu_dereference(tp->fastopen_rsk);
 561	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 562	if (sk->sk_state != TCP_LISTEN &&
 563	    !between(seq, snd_una, tp->snd_nxt)) {
 564		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 565		goto out;
 566	}
 567
 568	switch (type) {
 569	case ICMP_REDIRECT:
 570		if (!sock_owned_by_user(sk))
 571			do_redirect(skb, sk);
 572		goto out;
 573	case ICMP_SOURCE_QUENCH:
 574		/* Just silently ignore these. */
 575		goto out;
 576	case ICMP_PARAMETERPROB:
 577		err = EPROTO;
 578		break;
 579	case ICMP_DEST_UNREACH:
 580		if (code > NR_ICMP_UNREACH)
 581			goto out;
 582
 583		if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
 584			/* We are not interested in TCP_LISTEN and open_requests
 585			 * (SYN-ACKs send out by Linux are always <576bytes so
 586			 * they should go through unfragmented).
 587			 */
 588			if (sk->sk_state == TCP_LISTEN)
 589				goto out;
 590
 591			WRITE_ONCE(tp->mtu_info, info);
 592			if (!sock_owned_by_user(sk)) {
 593				tcp_v4_mtu_reduced(sk);
 594			} else {
 595				if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
 596					sock_hold(sk);
 597			}
 598			goto out;
 599		}
 600
 601		err = icmp_err_convert[code].errno;
 602		/* check if this ICMP message allows revert of backoff.
 603		 * (see RFC 6069)
 604		 */
 605		if (!fastopen &&
 606		    (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
 607			tcp_ld_RTO_revert(sk, seq);
 608		break;
 609	case ICMP_TIME_EXCEEDED:
 610		err = EHOSTUNREACH;
 611		break;
 612	default:
 613		goto out;
 614	}
 615
 616	switch (sk->sk_state) {
 617	case TCP_SYN_SENT:
 618	case TCP_SYN_RECV:
 619		/* Only in fast or simultaneous open. If a fast open socket is
 620		 * already accepted it is treated as a connected one below.
 621		 */
 622		if (fastopen && !fastopen->sk)
 623			break;
 624
 625		ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
 626
 627		if (!sock_owned_by_user(sk))
 628			tcp_done_with_error(sk, err);
 629		else
 630			WRITE_ONCE(sk->sk_err_soft, err);
 631		goto out;
 632	}
 633
 634	/* If we've already connected we will keep trying
 635	 * until we time out, or the user gives up.
 636	 *
 637	 * rfc1122 4.2.3.9 allows to consider as hard errors
 638	 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
 639	 * but it is obsoleted by pmtu discovery).
 640	 *
 641	 * Note, that in modern internet, where routing is unreliable
 642	 * and in each dark corner broken firewalls sit, sending random
 643	 * errors ordered by their masters even this two messages finally lose
 644	 * their original sense (even Linux sends invalid PORT_UNREACHs)
 645	 *
 646	 * Now we are in compliance with RFCs.
 647	 *							--ANK (980905)
 648	 */
 649
 650	if (!sock_owned_by_user(sk) &&
 651	    inet_test_bit(RECVERR, sk)) {
 652		WRITE_ONCE(sk->sk_err, err);
 653		sk_error_report(sk);
 654	} else	{ /* Only an error on timeout */
 655		WRITE_ONCE(sk->sk_err_soft, err);
 656	}
 657
 658out:
 659	bh_unlock_sock(sk);
 660	sock_put(sk);
 661	return 0;
 662}
 663
 664void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
 665{
 666	struct tcphdr *th = tcp_hdr(skb);
 667
 668	th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
 669	skb->csum_start = skb_transport_header(skb) - skb->head;
 670	skb->csum_offset = offsetof(struct tcphdr, check);
 671}
 672
 673/* This routine computes an IPv4 TCP checksum. */
 674void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
 675{
 676	const struct inet_sock *inet = inet_sk(sk);
 677
 678	__tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
 679}
 680EXPORT_IPV6_MOD(tcp_v4_send_check);
 681
 682#define REPLY_OPTIONS_LEN      (MAX_TCP_OPTION_SPACE / sizeof(__be32))
 683
 684static bool tcp_v4_ao_sign_reset(const struct sock *sk, struct sk_buff *skb,
 685				 const struct tcp_ao_hdr *aoh,
 686				 struct ip_reply_arg *arg, struct tcphdr *reply,
 687				 __be32 reply_options[REPLY_OPTIONS_LEN])
 688{
 689#ifdef CONFIG_TCP_AO
 690	int sdif = tcp_v4_sdif(skb);
 691	int dif = inet_iif(skb);
 692	int l3index = sdif ? dif : 0;
 693	bool allocated_traffic_key;
 694	struct tcp_ao_key *key;
 695	char *traffic_key;
 696	bool drop = true;
 697	u32 ao_sne = 0;
 698	u8 keyid;
 699
 700	rcu_read_lock();
 701	if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, ntohl(reply->seq),
 702				 &key, &traffic_key, &allocated_traffic_key,
 703				 &keyid, &ao_sne))
 704		goto out;
 705
 706	reply_options[0] = htonl((TCPOPT_AO << 24) | (tcp_ao_len(key) << 16) |
 707				 (aoh->rnext_keyid << 8) | keyid);
 708	arg->iov[0].iov_len += tcp_ao_len_aligned(key);
 709	reply->doff = arg->iov[0].iov_len / 4;
 710
 711	if (tcp_ao_hash_hdr(AF_INET, (char *)&reply_options[1],
 712			    key, traffic_key,
 713			    (union tcp_ao_addr *)&ip_hdr(skb)->saddr,
 714			    (union tcp_ao_addr *)&ip_hdr(skb)->daddr,
 715			    reply, ao_sne))
 716		goto out;
 717	drop = false;
 718out:
 719	rcu_read_unlock();
 720	if (allocated_traffic_key)
 721		kfree(traffic_key);
 722	return drop;
 723#else
 724	return true;
 725#endif
 726}
 727
 728/*
 729 *	This routine will send an RST to the other tcp.
 730 *
 731 *	Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
 732 *		      for reset.
 733 *	Answer: if a packet caused RST, it is not for a socket
 734 *		existing in our system, if it is matched to a socket,
 735 *		it is just duplicate segment or bug in other side's TCP.
 736 *		So that we build reply only basing on parameters
 737 *		arrived with segment.
 738 *	Exception: precedence violation. We do not implement it in any case.
 739 */
 740
 741static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb,
 742			      enum sk_rst_reason reason)
 743{
 744	const struct tcphdr *th = tcp_hdr(skb);
 745	struct {
 746		struct tcphdr th;
 747		__be32 opt[REPLY_OPTIONS_LEN];
 748	} rep;
 749	const __u8 *md5_hash_location = NULL;
 750	const struct tcp_ao_hdr *aoh;
 751	struct ip_reply_arg arg;
 752#ifdef CONFIG_TCP_MD5SIG
 753	struct tcp_md5sig_key *key = NULL;
 754	unsigned char newhash[16];
 755	struct sock *sk1 = NULL;
 756#endif
 757	u64 transmit_time = 0;
 758	struct sock *ctl_sk;
 759	struct net *net;
 760	u32 txhash = 0;
 761
 762	/* Never send a reset in response to a reset. */
 763	if (th->rst)
 764		return;
 765
 766	/* If sk not NULL, it means we did a successful lookup and incoming
 767	 * route had to be correct. prequeue might have dropped our dst.
 768	 */
 769	if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
 770		return;
 771
 772	/* Swap the send and the receive. */
 773	memset(&rep, 0, sizeof(rep));
 774	rep.th.dest   = th->source;
 775	rep.th.source = th->dest;
 776	rep.th.doff   = sizeof(struct tcphdr) / 4;
 777	rep.th.rst    = 1;
 778
 779	if (th->ack) {
 780		rep.th.seq = th->ack_seq;
 781	} else {
 782		rep.th.ack = 1;
 783		rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
 784				       skb->len - (th->doff << 2));
 785	}
 786
 787	memset(&arg, 0, sizeof(arg));
 788	arg.iov[0].iov_base = (unsigned char *)&rep;
 789	arg.iov[0].iov_len  = sizeof(rep.th);
 790
 791	net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
 792
 793	/* Invalid TCP option size or twice included auth */
 794	if (tcp_parse_auth_options(tcp_hdr(skb), &md5_hash_location, &aoh))
 795		return;
 796
 797	if (aoh && tcp_v4_ao_sign_reset(sk, skb, aoh, &arg, &rep.th, rep.opt))
 798		return;
 799
 800#ifdef CONFIG_TCP_MD5SIG
 801	rcu_read_lock();
 802	if (sk && sk_fullsock(sk)) {
 803		const union tcp_md5_addr *addr;
 804		int l3index;
 805
 806		/* sdif set, means packet ingressed via a device
 807		 * in an L3 domain and inet_iif is set to it.
 808		 */
 809		l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
 810		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
 811		key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
 812	} else if (md5_hash_location) {
 813		const union tcp_md5_addr *addr;
 814		int sdif = tcp_v4_sdif(skb);
 815		int dif = inet_iif(skb);
 816		int l3index;
 817
 818		/*
 819		 * active side is lost. Try to find listening socket through
 820		 * source port, and then find md5 key through listening socket.
 821		 * we are not loose security here:
 822		 * Incoming packet is checked with md5 hash with finding key,
 823		 * no RST generated if md5 hash doesn't match.
 824		 */
 825		sk1 = __inet_lookup_listener(net, NULL, 0, ip_hdr(skb)->saddr,
 826					     th->source, ip_hdr(skb)->daddr,
 827					     ntohs(th->source), dif, sdif);
 828		/* don't send rst if it can't find key */
 829		if (!sk1)
 830			goto out;
 831
 832		/* sdif set, means packet ingressed via a device
 833		 * in an L3 domain and dif is set to it.
 834		 */
 835		l3index = sdif ? dif : 0;
 836		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
 837		key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
 838		if (!key)
 839			goto out;
 840
 841		tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
 842		if (crypto_memneq(md5_hash_location, newhash, 16))
 843			goto out;
 844	}
 845
 846	if (key) {
 847		rep.opt[0] = htonl((TCPOPT_NOP << 24) |
 848				   (TCPOPT_NOP << 16) |
 849				   (TCPOPT_MD5SIG << 8) |
 850				   TCPOLEN_MD5SIG);
 851		/* Update length and the length the header thinks exists */
 852		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
 853		rep.th.doff = arg.iov[0].iov_len / 4;
 854
 855		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
 856				     key, ip_hdr(skb)->saddr,
 857				     ip_hdr(skb)->daddr, &rep.th);
 858	}
 859#endif
 860	/* Can't co-exist with TCPMD5, hence check rep.opt[0] */
 861	if (rep.opt[0] == 0) {
 862		__be32 mrst = mptcp_reset_option(skb);
 863
 864		if (mrst) {
 865			rep.opt[0] = mrst;
 866			arg.iov[0].iov_len += sizeof(mrst);
 867			rep.th.doff = arg.iov[0].iov_len / 4;
 868		}
 869	}
 870
 871	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
 872				      ip_hdr(skb)->saddr, /* XXX */
 873				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
 874	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
 875	arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
 876
 877	/* When socket is gone, all binding information is lost.
 878	 * routing might fail in this case. No choice here, if we choose to force
 879	 * input interface, we will misroute in case of asymmetric route.
 880	 */
 881	if (sk)
 882		arg.bound_dev_if = sk->sk_bound_dev_if;
 883
 884	trace_tcp_send_reset(sk, skb, reason);
 885
 886	BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
 887		     offsetof(struct inet_timewait_sock, tw_bound_dev_if));
 888
 889	/* ECN bits of TW reset are cleared */
 890	arg.tos = ip_hdr(skb)->tos & ~INET_ECN_MASK;
 891	arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 892	local_bh_disable();
 893	local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
 894	ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
 895
 896	sock_net_set(ctl_sk, net);
 897	if (sk) {
 898		ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
 899				   inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
 900		ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
 901				   inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
 902		transmit_time = tcp_transmit_time(sk);
 903		xfrm_sk_clone_policy(ctl_sk, sk);
 904		txhash = (sk->sk_state == TCP_TIME_WAIT) ?
 905			 inet_twsk(sk)->tw_txhash : sk->sk_txhash;
 906	} else {
 907		ctl_sk->sk_mark = 0;
 908		ctl_sk->sk_priority = 0;
 909	}
 910	ip_send_unicast_reply(ctl_sk, sk,
 911			      skb, &TCP_SKB_CB(skb)->header.h4.opt,
 912			      ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
 913			      &arg, arg.iov[0].iov_len,
 914			      transmit_time, txhash);
 915
 916	xfrm_sk_free_policy(ctl_sk);
 917	sock_net_set(ctl_sk, &init_net);
 918	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
 919	__TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
 920	local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
 921	local_bh_enable();
 922
 923#ifdef CONFIG_TCP_MD5SIG
 924out:
 925	rcu_read_unlock();
 926#endif
 927}
 928
 929/* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
 930   outside socket context is ugly, certainly. What can I do?
 931 */
 932
 933static void tcp_v4_send_ack(const struct sock *sk,
 934			    struct sk_buff *skb, u32 seq, u32 ack,
 935			    u32 win, u32 tsval, u32 tsecr, int oif,
 936			    struct tcp_key *key,
 937			    int reply_flags, u8 tos, u32 txhash)
 938{
 939	const struct tcphdr *th = tcp_hdr(skb);
 940	struct {
 941		struct tcphdr th;
 942		__be32 opt[(MAX_TCP_OPTION_SPACE  >> 2)];
 943	} rep;
 944	struct net *net = sock_net(sk);
 945	struct ip_reply_arg arg;
 946	struct sock *ctl_sk;
 947	u64 transmit_time;
 948
 949	memset(&rep.th, 0, sizeof(struct tcphdr));
 950	memset(&arg, 0, sizeof(arg));
 951
 952	arg.iov[0].iov_base = (unsigned char *)&rep;
 953	arg.iov[0].iov_len  = sizeof(rep.th);
 954	if (tsecr) {
 955		rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 956				   (TCPOPT_TIMESTAMP << 8) |
 957				   TCPOLEN_TIMESTAMP);
 958		rep.opt[1] = htonl(tsval);
 959		rep.opt[2] = htonl(tsecr);
 960		arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
 961	}
 962
 963	/* Swap the send and the receive. */
 964	rep.th.dest    = th->source;
 965	rep.th.source  = th->dest;
 966	rep.th.doff    = arg.iov[0].iov_len / 4;
 967	rep.th.seq     = htonl(seq);
 968	rep.th.ack_seq = htonl(ack);
 969	rep.th.ack     = 1;
 970	rep.th.window  = htons(win);
 971
 972#ifdef CONFIG_TCP_MD5SIG
 973	if (tcp_key_is_md5(key)) {
 974		int offset = (tsecr) ? 3 : 0;
 975
 976		rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
 977					  (TCPOPT_NOP << 16) |
 978					  (TCPOPT_MD5SIG << 8) |
 979					  TCPOLEN_MD5SIG);
 980		arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
 981		rep.th.doff = arg.iov[0].iov_len/4;
 982
 983		tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
 984				    key->md5_key, ip_hdr(skb)->saddr,
 985				    ip_hdr(skb)->daddr, &rep.th);
 986	}
 987#endif
 988#ifdef CONFIG_TCP_AO
 989	if (tcp_key_is_ao(key)) {
 990		int offset = (tsecr) ? 3 : 0;
 991
 992		rep.opt[offset++] = htonl((TCPOPT_AO << 24) |
 993					  (tcp_ao_len(key->ao_key) << 16) |
 994					  (key->ao_key->sndid << 8) |
 995					  key->rcv_next);
 996		arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key);
 997		rep.th.doff = arg.iov[0].iov_len / 4;
 998
 999		tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset],
1000				key->ao_key, key->traffic_key,
1001				(union tcp_ao_addr *)&ip_hdr(skb)->saddr,
1002				(union tcp_ao_addr *)&ip_hdr(skb)->daddr,
1003				&rep.th, key->sne);
1004	}
1005#endif
1006	arg.flags = reply_flags;
1007	arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
1008				      ip_hdr(skb)->saddr, /* XXX */
1009				      arg.iov[0].iov_len, IPPROTO_TCP, 0);
1010	arg.csumoffset = offsetof(struct tcphdr, check) / 2;
1011	if (oif)
1012		arg.bound_dev_if = oif;
1013	arg.tos = tos;
1014	arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
1015	local_bh_disable();
1016	local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
1017	ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
1018	sock_net_set(ctl_sk, net);
1019	ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
1020			   inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
1021	ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
1022			   inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
1023	transmit_time = tcp_transmit_time(sk);
1024	ip_send_unicast_reply(ctl_sk, sk,
1025			      skb, &TCP_SKB_CB(skb)->header.h4.opt,
1026			      ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
1027			      &arg, arg.iov[0].iov_len,
1028			      transmit_time, txhash);
1029
1030	sock_net_set(ctl_sk, &init_net);
1031	__TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
1032	local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
1033	local_bh_enable();
1034}
1035
1036static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb,
1037				enum tcp_tw_status tw_status)
1038{
1039	struct inet_timewait_sock *tw = inet_twsk(sk);
1040	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1041	struct tcp_key key = {};
1042	u8 tos = tw->tw_tos;
1043
1044	/* Cleaning only ECN bits of TW ACKs of oow data or is paws_reject,
1045	 * while not cleaning ECN bits of other TW ACKs to avoid these ACKs
1046	 * being placed in a different service queues (Classic rather than L4S)
1047	 */
1048	if (tw_status == TCP_TW_ACK_OOW)
1049		tos &= ~INET_ECN_MASK;
1050
1051#ifdef CONFIG_TCP_AO
1052	struct tcp_ao_info *ao_info;
1053
1054	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1055		/* FIXME: the segment to-be-acked is not verified yet */
1056		ao_info = rcu_dereference(tcptw->ao_info);
1057		if (ao_info) {
1058			const struct tcp_ao_hdr *aoh;
1059
1060			if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) {
1061				inet_twsk_put(tw);
1062				return;
1063			}
1064
1065			if (aoh)
1066				key.ao_key = tcp_ao_established_key(sk, ao_info,
1067								    aoh->rnext_keyid, -1);
1068		}
1069	}
1070	if (key.ao_key) {
1071		struct tcp_ao_key *rnext_key;
1072
1073		key.traffic_key = snd_other_key(key.ao_key);
1074		key.sne = READ_ONCE(ao_info->snd_sne);
1075		rnext_key = READ_ONCE(ao_info->rnext_key);
1076		key.rcv_next = rnext_key->rcvid;
1077		key.type = TCP_KEY_AO;
1078#else
1079	if (0) {
1080#endif
1081	} else if (static_branch_tcp_md5()) {
1082		key.md5_key = tcp_twsk_md5_key(tcptw);
1083		if (key.md5_key)
1084			key.type = TCP_KEY_MD5;
1085	}
1086
1087	tcp_v4_send_ack(sk, skb,
1088			tcptw->tw_snd_nxt, READ_ONCE(tcptw->tw_rcv_nxt),
1089			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1090			tcp_tw_tsval(tcptw),
1091			READ_ONCE(tcptw->tw_ts_recent),
1092			tw->tw_bound_dev_if, &key,
1093			tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
1094			tos,
1095			tw->tw_txhash);
1096
1097	inet_twsk_put(tw);
1098}
1099
1100static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1101				  struct request_sock *req)
1102{
1103	struct tcp_key key = {};
1104
1105	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1106	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1107	 */
1108	u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
1109					     tcp_sk(sk)->snd_nxt;
1110
1111#ifdef CONFIG_TCP_AO
1112	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1113	    tcp_rsk_used_ao(req)) {
1114		const union tcp_md5_addr *addr;
1115		const struct tcp_ao_hdr *aoh;
1116		int l3index;
1117
1118		/* Invalid TCP option size or twice included auth */
1119		if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1120			return;
1121		if (!aoh)
1122			return;
1123
1124		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
1125		l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
1126		key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET,
1127					      aoh->rnext_keyid, -1);
1128		if (unlikely(!key.ao_key)) {
1129			/* Send ACK with any matching MKT for the peer */
1130			key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET, -1, -1);
1131			/* Matching key disappeared (user removed the key?)
1132			 * let the handshake timeout.
1133			 */
1134			if (!key.ao_key) {
1135				net_info_ratelimited("TCP-AO key for (%pI4, %d)->(%pI4, %d) suddenly disappeared, won't ACK new connection\n",
1136						     addr,
1137						     ntohs(tcp_hdr(skb)->source),
1138						     &ip_hdr(skb)->daddr,
1139						     ntohs(tcp_hdr(skb)->dest));
1140				return;
1141			}
1142		}
1143		key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1144		if (!key.traffic_key)
1145			return;
1146
1147		key.type = TCP_KEY_AO;
1148		key.rcv_next = aoh->keyid;
1149		tcp_v4_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1150#else
1151	if (0) {
1152#endif
1153	} else if (static_branch_tcp_md5()) {
1154		const union tcp_md5_addr *addr;
1155		int l3index;
1156
1157		addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
1158		l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
1159		key.md5_key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1160		if (key.md5_key)
1161			key.type = TCP_KEY_MD5;
1162	}
1163
1164	/* Cleaning ECN bits of TW ACKs of oow data or is paws_reject */
1165	tcp_v4_send_ack(sk, skb, seq,
1166			tcp_rsk(req)->rcv_nxt,
1167			tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1168			tcp_rsk_tsval(tcp_rsk(req)),
1169			req->ts_recent,
1170			0, &key,
1171			inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
1172			ip_hdr(skb)->tos & ~INET_ECN_MASK,
1173			READ_ONCE(tcp_rsk(req)->txhash));
1174	if (tcp_key_is_ao(&key))
1175		kfree(key.traffic_key);
1176}
1177
1178/*
1179 *	Send a SYN-ACK after having received a SYN.
1180 *	This still operates on a request_sock only, not on a big
1181 *	socket.
1182 */
1183static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
1184			      struct flowi *fl,
1185			      struct request_sock *req,
1186			      struct tcp_fastopen_cookie *foc,
1187			      enum tcp_synack_type synack_type,
1188			      struct sk_buff *syn_skb)
1189{
1190	struct inet_request_sock *ireq = inet_rsk(req);
1191	struct flowi4 fl4;
1192	int err = -1;
1193	struct sk_buff *skb;
1194	u8 tos;
1195
1196	/* First, grab a route. */
1197	if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
1198		return -1;
1199
1200	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
1201
1202	if (skb) {
1203		tcp_rsk(req)->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK;
1204		__tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
1205
1206		tos = READ_ONCE(inet_sk(sk)->tos);
1207
1208		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1209			tos = (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
1210			      (tos & INET_ECN_MASK);
1211
1212		if (!INET_ECN_is_capable(tos) &&
1213		    tcp_bpf_ca_needs_ecn((struct sock *)req))
1214			tos |= INET_ECN_ECT_0;
1215
1216		rcu_read_lock();
1217		err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
1218					    ireq->ir_rmt_addr,
1219					    rcu_dereference(ireq->ireq_opt),
1220					    tos);
1221		rcu_read_unlock();
1222		err = net_xmit_eval(err);
1223	}
1224
1225	return err;
1226}
1227
1228/*
1229 *	IPv4 request_sock destructor.
1230 */
1231static void tcp_v4_reqsk_destructor(struct request_sock *req)
1232{
1233	kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
1234}
1235
1236#ifdef CONFIG_TCP_MD5SIG
1237/*
1238 * RFC2385 MD5 checksumming requires a mapping of
1239 * IP address->MD5 Key.
1240 * We need to maintain these in the sk structure.
1241 */
1242
1243DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ);
1244EXPORT_IPV6_MOD(tcp_md5_needed);
1245
1246static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
1247{
1248	if (!old)
1249		return true;
1250
1251	/* l3index always overrides non-l3index */
1252	if (old->l3index && new->l3index == 0)
1253		return false;
1254	if (old->l3index == 0 && new->l3index)
1255		return true;
1256
1257	return old->prefixlen < new->prefixlen;
1258}
1259
1260/* Find the Key structure for an address.  */
1261struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1262					   const union tcp_md5_addr *addr,
1263					   int family, bool any_l3index)
1264{
1265	const struct tcp_sock *tp = tcp_sk(sk);
1266	struct tcp_md5sig_key *key;
1267	const struct tcp_md5sig_info *md5sig;
1268	__be32 mask;
1269	struct tcp_md5sig_key *best_match = NULL;
1270	bool match;
1271
1272	/* caller either holds rcu_read_lock() or socket lock */
1273	md5sig = rcu_dereference_check(tp->md5sig_info,
1274				       lockdep_sock_is_held(sk));
1275	if (!md5sig)
1276		return NULL;
1277
1278	hlist_for_each_entry_rcu(key, &md5sig->head, node,
1279				 lockdep_sock_is_held(sk)) {
1280		if (key->family != family)
1281			continue;
1282		if (!any_l3index && key->flags & TCP_MD5SIG_FLAG_IFINDEX &&
1283		    key->l3index != l3index)
1284			continue;
1285		if (family == AF_INET) {
1286			mask = inet_make_mask(key->prefixlen);
1287			match = (key->addr.a4.s_addr & mask) ==
1288				(addr->a4.s_addr & mask);
1289#if IS_ENABLED(CONFIG_IPV6)
1290		} else if (family == AF_INET6) {
1291			match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1292						  key->prefixlen);
1293#endif
1294		} else {
1295			match = false;
1296		}
1297
1298		if (match && better_md5_match(best_match, key))
1299			best_match = key;
1300	}
1301	return best_match;
1302}
1303EXPORT_IPV6_MOD(__tcp_md5_do_lookup);
1304
1305static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1306						      const union tcp_md5_addr *addr,
1307						      int family, u8 prefixlen,
1308						      int l3index, u8 flags)
1309{
1310	const struct tcp_sock *tp = tcp_sk(sk);
1311	struct tcp_md5sig_key *key;
1312	unsigned int size = sizeof(struct in_addr);
1313	const struct tcp_md5sig_info *md5sig;
1314
1315	/* caller either holds rcu_read_lock() or socket lock */
1316	md5sig = rcu_dereference_check(tp->md5sig_info,
1317				       lockdep_sock_is_held(sk));
1318	if (!md5sig)
1319		return NULL;
1320#if IS_ENABLED(CONFIG_IPV6)
1321	if (family == AF_INET6)
1322		size = sizeof(struct in6_addr);
1323#endif
1324	hlist_for_each_entry_rcu(key, &md5sig->head, node,
1325				 lockdep_sock_is_held(sk)) {
1326		if (key->family != family)
1327			continue;
1328		if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
1329			continue;
1330		if (key->l3index != l3index)
1331			continue;
1332		if (!memcmp(&key->addr, addr, size) &&
1333		    key->prefixlen == prefixlen)
1334			return key;
1335	}
1336	return NULL;
1337}
1338
1339struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1340					 const struct sock *addr_sk)
1341{
1342	const union tcp_md5_addr *addr;
1343	int l3index;
1344
1345	l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1346						 addr_sk->sk_bound_dev_if);
1347	addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1348	return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1349}
1350EXPORT_IPV6_MOD(tcp_v4_md5_lookup);
1351
1352static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp)
1353{
1354	struct tcp_sock *tp = tcp_sk(sk);
1355	struct tcp_md5sig_info *md5sig;
1356
1357	md5sig = kmalloc(sizeof(*md5sig), gfp);
1358	if (!md5sig)
1359		return -ENOMEM;
1360
1361	sk_gso_disable(sk);
1362	INIT_HLIST_HEAD(&md5sig->head);
1363	rcu_assign_pointer(tp->md5sig_info, md5sig);
1364	return 0;
1365}
1366
1367/* This can be called on a newly created socket, from other files */
1368static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1369			    int family, u8 prefixlen, int l3index, u8 flags,
1370			    const u8 *newkey, u8 newkeylen, gfp_t gfp)
1371{
1372	/* Add Key to the list */
1373	struct tcp_md5sig_key *key;
1374	struct tcp_sock *tp = tcp_sk(sk);
1375	struct tcp_md5sig_info *md5sig;
1376
1377	key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1378	if (key) {
1379		/* Pre-existing entry - just update that one.
1380		 * Note that the key might be used concurrently.
1381		 * data_race() is telling kcsan that we do not care of
1382		 * key mismatches, since changing MD5 key on live flows
1383		 * can lead to packet drops.
1384		 */
1385		data_race(memcpy(key->key, newkey, newkeylen));
1386
1387		/* Pairs with READ_ONCE() in tcp_md5_hash_key().
1388		 * Also note that a reader could catch new key->keylen value
1389		 * but old key->key[], this is the reason we use __GFP_ZERO
1390		 * at sock_kmalloc() time below these lines.
1391		 */
1392		WRITE_ONCE(key->keylen, newkeylen);
1393
1394		return 0;
1395	}
1396
1397	md5sig = rcu_dereference_protected(tp->md5sig_info,
1398					   lockdep_sock_is_held(sk));
1399
1400	key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1401	if (!key)
1402		return -ENOMEM;
1403
1404	memcpy(key->key, newkey, newkeylen);
1405	key->keylen = newkeylen;
1406	key->family = family;
1407	key->prefixlen = prefixlen;
1408	key->l3index = l3index;
1409	key->flags = flags;
1410	memcpy(&key->addr, addr,
1411	       (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
1412								 sizeof(struct in_addr));
1413	hlist_add_head_rcu(&key->node, &md5sig->head);
1414	return 0;
1415}
1416
1417int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1418		   int family, u8 prefixlen, int l3index, u8 flags,
1419		   const u8 *newkey, u8 newkeylen)
1420{
1421	struct tcp_sock *tp = tcp_sk(sk);
1422
1423	if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
1424		if (fips_enabled) {
1425			pr_warn_once("TCP-MD5 support is disabled due to FIPS\n");
1426			return -EOPNOTSUPP;
1427		}
1428
1429		if (tcp_md5sig_info_add(sk, GFP_KERNEL))
1430			return -ENOMEM;
1431
1432		if (!static_branch_inc(&tcp_md5_needed.key)) {
1433			struct tcp_md5sig_info *md5sig;
1434
1435			md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
1436			rcu_assign_pointer(tp->md5sig_info, NULL);
1437			kfree_rcu(md5sig, rcu);
1438			return -EUSERS;
1439		}
1440	}
1441
1442	return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags,
1443				newkey, newkeylen, GFP_KERNEL);
1444}
1445EXPORT_IPV6_MOD(tcp_md5_do_add);
1446
1447int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1448		     int family, u8 prefixlen, int l3index,
1449		     struct tcp_md5sig_key *key)
1450{
1451	struct tcp_sock *tp = tcp_sk(sk);
1452
1453	if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
1454
1455		if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC)))
1456			return -ENOMEM;
1457
1458		if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) {
1459			struct tcp_md5sig_info *md5sig;
1460
1461			md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
1462			net_warn_ratelimited("Too many TCP-MD5 keys in the system\n");
1463			rcu_assign_pointer(tp->md5sig_info, NULL);
1464			kfree_rcu(md5sig, rcu);
1465			return -EUSERS;
1466		}
1467	}
1468
1469	return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index,
1470				key->flags, key->key, key->keylen,
1471				sk_gfp_mask(sk, GFP_ATOMIC));
1472}
1473EXPORT_IPV6_MOD(tcp_md5_key_copy);
1474
1475int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1476		   u8 prefixlen, int l3index, u8 flags)
1477{
1478	struct tcp_md5sig_key *key;
1479
1480	key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1481	if (!key)
1482		return -ENOENT;
1483	hlist_del_rcu(&key->node);
1484	atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1485	kfree_rcu(key, rcu);
1486	return 0;
1487}
1488EXPORT_IPV6_MOD(tcp_md5_do_del);
1489
1490void tcp_clear_md5_list(struct sock *sk)
1491{
1492	struct tcp_sock *tp = tcp_sk(sk);
1493	struct tcp_md5sig_key *key;
1494	struct hlist_node *n;
1495	struct tcp_md5sig_info *md5sig;
1496
1497	md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1498
1499	hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1500		hlist_del(&key->node);
1501		atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1502		kfree(key);
1503	}
1504}
1505
1506static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1507				 sockptr_t optval, int optlen)
1508{
1509	struct tcp_md5sig cmd;
1510	struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1511	const union tcp_md5_addr *addr;
1512	u8 prefixlen = 32;
1513	int l3index = 0;
1514	bool l3flag;
1515	u8 flags;
1516
1517	if (optlen < sizeof(cmd))
1518		return -EINVAL;
1519
1520	if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1521		return -EFAULT;
1522
1523	if (sin->sin_family != AF_INET)
1524		return -EINVAL;
1525
1526	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1527	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1528
1529	if (optname == TCP_MD5SIG_EXT &&
1530	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1531		prefixlen = cmd.tcpm_prefixlen;
1532		if (prefixlen > 32)
1533			return -EINVAL;
1534	}
1535
1536	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
1537	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1538		struct net_device *dev;
1539
1540		rcu_read_lock();
1541		dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1542		if (dev && netif_is_l3_master(dev))
1543			l3index = dev->ifindex;
1544
1545		rcu_read_unlock();
1546
1547		/* ok to reference set/not set outside of rcu;
1548		 * right now device MUST be an L3 master
1549		 */
1550		if (!dev || !l3index)
1551			return -EINVAL;
1552	}
1553
1554	addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1555
1556	if (!cmd.tcpm_keylen)
1557		return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);
1558
1559	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1560		return -EINVAL;
1561
1562	/* Don't allow keys for peers that have a matching TCP-AO key.
1563	 * See the comment in tcp_ao_add_cmd()
1564	 */
1565	if (tcp_ao_required(sk, addr, AF_INET, l3flag ? l3index : -1, false))
1566		return -EKEYREJECTED;
1567
1568	return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
1569			      cmd.tcpm_key, cmd.tcpm_keylen);
1570}
1571
1572static void tcp_v4_md5_hash_headers(struct md5_ctx *ctx,
1573				    __be32 daddr, __be32 saddr,
1574				    const struct tcphdr *th, int nbytes)
1575{
1576	struct {
1577		struct tcp4_pseudohdr ip;
1578		struct tcphdr tcp;
1579	} h;
1580
1581	h.ip.saddr = saddr;
1582	h.ip.daddr = daddr;
1583	h.ip.pad = 0;
1584	h.ip.protocol = IPPROTO_TCP;
1585	h.ip.len = cpu_to_be16(nbytes);
1586	h.tcp = *th;
1587	h.tcp.check = 0;
1588	md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp));
1589}
1590
1591static noinline_for_stack void
1592tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1593		    __be32 daddr, __be32 saddr, const struct tcphdr *th)
1594{
1595	struct md5_ctx ctx;
1596
1597	md5_init(&ctx);
1598	tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2);
1599	tcp_md5_hash_key(&ctx, key);
1600	md5_final(&ctx, md5_hash);
1601}
1602
1603noinline_for_stack void
1604tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1605		    const struct sock *sk, const struct sk_buff *skb)
1606{
1607	const struct tcphdr *th = tcp_hdr(skb);
1608	__be32 saddr, daddr;
1609	struct md5_ctx ctx;
1610
1611	if (sk) { /* valid for establish/request sockets */
1612		saddr = sk->sk_rcv_saddr;
1613		daddr = sk->sk_daddr;
1614	} else {
1615		const struct iphdr *iph = ip_hdr(skb);
1616		saddr = iph->saddr;
1617		daddr = iph->daddr;
1618	}
1619
1620	md5_init(&ctx);
1621	tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, skb->len);
1622	tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2);
1623	tcp_md5_hash_key(&ctx, key);
1624	md5_final(&ctx, md5_hash);
1625}
1626EXPORT_IPV6_MOD(tcp_v4_md5_hash_skb);
1627
1628#endif
1629
1630static void tcp_v4_init_req(struct request_sock *req,
1631			    const struct sock *sk_listener,
1632			    struct sk_buff *skb)
1633{
1634	struct inet_request_sock *ireq = inet_rsk(req);
1635	struct net *net = sock_net(sk_listener);
1636
1637	sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1638	sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1639	RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1640}
1641
1642static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1643					  struct sk_buff *skb,
1644					  struct flowi *fl,
1645					  struct request_sock *req,
1646					  u32 tw_isn)
1647{
1648	tcp_v4_init_req(req, sk, skb);
1649
1650	if (security_inet_conn_request(sk, skb, req))
1651		return NULL;
1652
1653	return inet_csk_route_req(sk, &fl->u.ip4, req);
1654}
1655
1656struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1657	.family		=	PF_INET,
1658	.obj_size	=	sizeof(struct tcp_request_sock),
1659	.send_ack	=	tcp_v4_reqsk_send_ack,
1660	.destructor	=	tcp_v4_reqsk_destructor,
1661	.send_reset	=	tcp_v4_send_reset,
1662};
1663
1664const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1665	.mss_clamp	=	TCP_MSS_DEFAULT,
1666#ifdef CONFIG_TCP_MD5SIG
1667	.req_md5_lookup	=	tcp_v4_md5_lookup,
1668	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
1669#endif
1670#ifdef CONFIG_TCP_AO
1671	.ao_lookup	=	tcp_v4_ao_lookup_rsk,
1672	.ao_calc_key	=	tcp_v4_ao_calc_key_rsk,
1673	.ao_synack_hash	=	tcp_v4_ao_synack_hash,
1674#endif
1675#ifdef CONFIG_SYN_COOKIES
1676	.cookie_init_seq =	cookie_v4_init_sequence,
1677#endif
1678	.route_req	=	tcp_v4_route_req,
1679	.init_seq_and_ts_off	=	tcp_v4_init_seq_and_ts_off,
1680	.send_synack	=	tcp_v4_send_synack,
1681};
1682
1683int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1684{
1685	/* Never answer to SYNs send to broadcast or multicast */
1686	if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1687		goto drop;
1688
1689	return tcp_conn_request(&tcp_request_sock_ops,
1690				&tcp_request_sock_ipv4_ops, sk, skb);
1691
1692drop:
1693	tcp_listendrop(sk);
1694	return 0;
1695}
1696EXPORT_IPV6_MOD(tcp_v4_conn_request);
1697
1698
1699/*
1700 * The three way handshake has completed - we got a valid synack -
1701 * now create the new socket.
1702 */
1703struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1704				  struct request_sock *req,
1705				  struct dst_entry *dst,
1706				  struct request_sock *req_unhash,
1707				  bool *own_req,
1708				  void (*opt_child_init)(struct sock *newsk,
1709							 const struct sock *sk))
1710{
1711	struct inet_request_sock *ireq;
1712	bool found_dup_sk = false;
1713	struct inet_sock *newinet;
1714	struct tcp_sock *newtp;
1715	struct sock *newsk;
1716#ifdef CONFIG_TCP_MD5SIG
1717	const union tcp_md5_addr *addr;
1718	struct tcp_md5sig_key *key;
1719	int l3index;
1720#endif
1721	struct ip_options_rcu *inet_opt;
1722
1723	if (sk_acceptq_is_full(sk))
1724		goto exit_overflow;
1725
1726	newsk = tcp_create_openreq_child(sk, req, skb);
1727	if (!newsk)
1728		goto exit_nonewsk;
1729
1730	newsk->sk_gso_type = SKB_GSO_TCPV4;
1731	inet_sk_rx_dst_set(newsk, skb);
1732
1733	newtp		      = tcp_sk(newsk);
1734	newinet		      = inet_sk(newsk);
1735	ireq		      = inet_rsk(req);
1736	inet_opt	      = rcu_dereference(ireq->ireq_opt);
1737	RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1738	newinet->mc_index     = inet_iif(skb);
1739	newinet->mc_ttl	      = ip_hdr(skb)->ttl;
1740	newinet->rcv_tos      = ip_hdr(skb)->tos;
1741	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1742	if (inet_opt)
1743		inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1744	atomic_set(&newinet->inet_id, get_random_u16());
1745
1746	/* Set ToS of the new socket based upon the value of incoming SYN.
1747	 * ECT bits are set later in tcp_init_transfer().
1748	 */
1749	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1750		newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1751
1752	if (!dst) {
1753		dst = inet_csk_route_child_sock(sk, newsk, req);
1754		if (!dst)
1755			goto put_and_exit;
1756	} else {
1757		/* syncookie case : see end of cookie_v4_check() */
1758	}
1759	sk_setup_caps(newsk, dst);
1760
1761#if IS_ENABLED(CONFIG_IPV6)
1762	if (opt_child_init)
1763		opt_child_init(newsk, sk);
1764#endif
1765	tcp_ca_openreq_child(newsk, dst);
1766
1767	tcp_sync_mss(newsk, dst_mtu(dst));
1768	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1769
1770	tcp_initialize_rcv_mss(newsk);
1771
1772#ifdef CONFIG_TCP_MD5SIG
1773	l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1774	/* Copy over the MD5 key from the original socket */
1775	addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1776	key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1777	if (key && !tcp_rsk_used_ao(req)) {
1778		if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key))
1779			goto put_and_exit;
1780		sk_gso_disable(newsk);
1781	}
1782#endif
1783#ifdef CONFIG_TCP_AO
1784	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET))
1785		goto put_and_exit; /* OOM, release back memory */
1786#endif
1787
1788	if (__inet_inherit_port(sk, newsk) < 0)
1789		goto put_and_exit;
1790	*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1791				       &found_dup_sk);
1792	if (likely(*own_req)) {
1793		tcp_move_syn(newtp, req);
1794		ireq->ireq_opt = NULL;
1795	} else {
1796		newinet->inet_opt = NULL;
1797
1798		if (!req_unhash && found_dup_sk) {
1799			/* This code path should only be executed in the
1800			 * syncookie case only
1801			 */
1802			bh_unlock_sock(newsk);
1803			sock_put(newsk);
1804			newsk = NULL;
1805		}
1806	}
1807	return newsk;
1808
1809exit_overflow:
1810	NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1811exit_nonewsk:
1812	dst_release(dst);
1813exit:
1814	tcp_listendrop(sk);
1815	return NULL;
1816put_and_exit:
1817	newinet->inet_opt = NULL;
1818	inet_csk_prepare_forced_close(newsk);
1819	tcp_done(newsk);
1820	goto exit;
1821}
1822EXPORT_IPV6_MOD(tcp_v4_syn_recv_sock);
1823
1824static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1825{
1826#ifdef CONFIG_SYN_COOKIES
1827	const struct tcphdr *th = tcp_hdr(skb);
1828
1829	if (!th->syn)
1830		sk = cookie_v4_check(sk, skb);
1831#endif
1832	return sk;
1833}
1834
1835u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1836			 struct tcphdr *th, u32 *cookie)
1837{
1838	u16 mss = 0;
1839#ifdef CONFIG_SYN_COOKIES
1840	mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1841				    &tcp_request_sock_ipv4_ops, sk, th);
1842	if (mss) {
1843		*cookie = __cookie_v4_init_sequence(iph, th, &mss);
1844		tcp_synq_overflow(sk);
1845	}
1846#endif
1847	return mss;
1848}
1849
1850INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1851							   u32));
1852/* The socket must have it's spinlock held when we get
1853 * here, unless it is a TCP_LISTEN socket.
1854 *
1855 * We have a potential double-lock case here, so even when
1856 * doing backlog processing we use the BH locking scheme.
1857 * This is because we cannot sleep with the original spinlock
1858 * held.
1859 */
1860int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1861{
1862	enum skb_drop_reason reason;
1863	struct sock *rsk;
1864
1865	reason = psp_sk_rx_policy_check(sk, skb);
1866	if (reason)
1867		goto err_discard;
1868
1869	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1870		struct dst_entry *dst;
1871
1872		dst = rcu_dereference_protected(sk->sk_rx_dst,
1873						lockdep_sock_is_held(sk));
1874
1875		sock_rps_save_rxhash(sk, skb);
1876		sk_mark_napi_id(sk, skb);
1877		if (dst) {
1878			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1879			    !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
1880					     dst, 0)) {
1881				RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1882				dst_release(dst);
1883			}
1884		}
1885		tcp_rcv_established(sk, skb);
1886		return 0;
1887	}
1888
1889	if (tcp_checksum_complete(skb))
1890		goto csum_err;
1891
1892	if (sk->sk_state == TCP_LISTEN) {
1893		struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1894
1895		if (!nsk)
1896			return 0;
1897		if (nsk != sk) {
1898			reason = tcp_child_process(sk, nsk, skb);
1899			if (reason) {
1900				rsk = nsk;
1901				goto reset;
1902			}
1903			return 0;
1904		}
1905	} else
1906		sock_rps_save_rxhash(sk, skb);
1907
1908	reason = tcp_rcv_state_process(sk, skb);
1909	if (reason) {
1910		rsk = sk;
1911		goto reset;
1912	}
1913	return 0;
1914
1915reset:
1916	tcp_v4_send_reset(rsk, skb, sk_rst_convert_drop_reason(reason));
1917discard:
1918	sk_skb_reason_drop(sk, skb, reason);
1919	/* Be careful here. If this function gets more complicated and
1920	 * gcc suffers from register pressure on the x86, sk (in %ebx)
1921	 * might be destroyed here. This current version compiles correctly,
1922	 * but you have been warned.
1923	 */
1924	return 0;
1925
1926csum_err:
1927	reason = SKB_DROP_REASON_TCP_CSUM;
1928	trace_tcp_bad_csum(skb);
1929	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1930err_discard:
1931	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1932	goto discard;
1933}
1934EXPORT_SYMBOL(tcp_v4_do_rcv);
1935
1936int tcp_v4_early_demux(struct sk_buff *skb)
1937{
1938	struct net *net = dev_net_rcu(skb->dev);
1939	const struct iphdr *iph;
1940	const struct tcphdr *th;
1941	struct sock *sk;
1942
1943	if (skb->pkt_type != PACKET_HOST)
1944		return 0;
1945
1946	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1947		return 0;
1948
1949	iph = ip_hdr(skb);
1950	th = tcp_hdr(skb);
1951
1952	if (th->doff < sizeof(struct tcphdr) / 4)
1953		return 0;
1954
1955	sk = __inet_lookup_established(net, iph->saddr, th->source,
1956				       iph->daddr, ntohs(th->dest),
1957				       skb->skb_iif, inet_sdif(skb));
1958	if (sk) {
1959		skb->sk = sk;
1960		skb->destructor = sock_edemux;
1961		if (sk_fullsock(sk)) {
1962			struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
1963
1964			if (dst)
1965				dst = dst_check(dst, 0);
1966			if (dst &&
1967			    sk->sk_rx_dst_ifindex == skb->skb_iif)
1968				skb_dst_set_noref(skb, dst);
1969		}
1970	}
1971	return 0;
1972}
1973
1974bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1975		     enum skb_drop_reason *reason)
1976{
1977	u32 tail_gso_size, tail_gso_segs;
1978	struct skb_shared_info *shinfo;
1979	const struct tcphdr *th;
1980	struct tcphdr *thtail;
1981	struct sk_buff *tail;
1982	unsigned int hdrlen;
1983	bool fragstolen;
1984	u32 gso_segs;
1985	u32 gso_size;
1986	u64 limit;
1987	int delta;
1988	int err;
1989
1990	/* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1991	 * we can fix skb->truesize to its real value to avoid future drops.
1992	 * This is valid because skb is not yet charged to the socket.
1993	 * It has been noticed pure SACK packets were sometimes dropped
1994	 * (if cooked by drivers without copybreak feature).
1995	 */
1996	skb_condense(skb);
1997
1998	tcp_cleanup_skb(skb);
1999
2000	if (unlikely(tcp_checksum_complete(skb))) {
2001		bh_unlock_sock(sk);
2002		trace_tcp_bad_csum(skb);
2003		*reason = SKB_DROP_REASON_TCP_CSUM;
2004		__TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
2005		__TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
2006		return true;
2007	}
2008
2009	/* Attempt coalescing to last skb in backlog, even if we are
2010	 * above the limits.
2011	 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
2012	 */
2013	th = (const struct tcphdr *)skb->data;
2014	hdrlen = th->doff * 4;
2015
2016	tail = sk->sk_backlog.tail;
2017	if (!tail)
2018		goto no_coalesce;
2019	thtail = (struct tcphdr *)tail->data;
2020
2021	if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
2022	    TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
2023	    ((TCP_SKB_CB(tail)->tcp_flags |
2024	      TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
2025	    !((TCP_SKB_CB(tail)->tcp_flags &
2026	      TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
2027	    ((TCP_SKB_CB(tail)->tcp_flags ^
2028	      TCP_SKB_CB(skb)->tcp_flags) &
2029	     (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) ||
2030	    !tcp_skb_can_collapse_rx(tail, skb) ||
2031	    thtail->doff != th->doff ||
2032	    memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) ||
2033	    /* prior to PSP Rx policy check, retain exact PSP metadata */
2034	    psp_skb_coalesce_diff(tail, skb))
2035		goto no_coalesce;
2036
2037	__skb_pull(skb, hdrlen);
2038
2039	shinfo = skb_shinfo(skb);
2040	gso_size = shinfo->gso_size ?: skb->len;
2041	gso_segs = shinfo->gso_segs ?: 1;
2042
2043	shinfo = skb_shinfo(tail);
2044	tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
2045	tail_gso_segs = shinfo->gso_segs ?: 1;
2046
2047	if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
2048		TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
2049
2050		if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
2051			TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
2052			thtail->window = th->window;
2053		}
2054
2055		/* We have to update both TCP_SKB_CB(tail)->tcp_flags and
2056		 * thtail->fin, so that the fast path in tcp_rcv_established()
2057		 * is not entered if we append a packet with a FIN.
2058		 * SYN, RST, URG are not present.
2059		 * ACK is set on both packets.
2060		 * PSH : we do not really care in TCP stack,
2061		 *       at least for 'GRO' packets.
2062		 */
2063		thtail->fin |= th->fin;
2064		TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
2065
2066		if (TCP_SKB_CB(skb)->has_rxtstamp) {
2067			TCP_SKB_CB(tail)->has_rxtstamp = true;
2068			tail->tstamp = skb->tstamp;
2069			skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
2070		}
2071
2072		/* Not as strict as GRO. We only need to carry mss max value */
2073		shinfo->gso_size = max(gso_size, tail_gso_size);
2074		shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
2075
2076		sk->sk_backlog.len += delta;
2077		__NET_INC_STATS(sock_net(sk),
2078				LINUX_MIB_TCPBACKLOGCOALESCE);
2079		kfree_skb_partial(skb, fragstolen);
2080		return false;
2081	}
2082	__skb_push(skb, hdrlen);
2083
2084no_coalesce:
2085	/* sk->sk_backlog.len is reset only at the end of __release_sock().
2086	 * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach
2087	 * sk_rcvbuf in normal conditions.
2088	 */
2089	limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1;
2090
2091	limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1;
2092
2093	/* Only socket owner can try to collapse/prune rx queues
2094	 * to reduce memory overhead, so add a little headroom here.
2095	 * Few sockets backlog are possibly concurrently non empty.
2096	 */
2097	limit += 64 * 1024;
2098
2099	limit = min_t(u64, limit, UINT_MAX);
2100
2101	err = sk_add_backlog(sk, skb, limit);
2102	if (unlikely(err)) {
2103		bh_unlock_sock(sk);
2104		if (err == -ENOMEM) {
2105			*reason = SKB_DROP_REASON_PFMEMALLOC;
2106			__NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
2107		} else {
2108			*reason = SKB_DROP_REASON_SOCKET_BACKLOG;
2109			__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
2110		}
2111		return true;
2112	}
2113	return false;
2114}
2115EXPORT_IPV6_MOD(tcp_add_backlog);
2116
2117int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason)
2118{
2119	struct tcphdr *th = (struct tcphdr *)skb->data;
2120
2121	return sk_filter_trim_cap(sk, skb, th->doff * 4, reason);
2122}
2123EXPORT_IPV6_MOD(tcp_filter);
2124
2125static void tcp_v4_restore_cb(struct sk_buff *skb)
2126{
2127	memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
2128		sizeof(struct inet_skb_parm));
2129}
2130
2131static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
2132			   const struct tcphdr *th)
2133{
2134	/* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
2135	 * barrier() makes sure compiler wont play fool^Waliasing games.
2136	 */
2137	memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
2138		sizeof(struct inet_skb_parm));
2139	barrier();
2140
2141	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
2142	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
2143				    skb->len - th->doff * 4);
2144	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
2145	TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
2146	TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
2147	TCP_SKB_CB(skb)->sacked	 = 0;
2148	TCP_SKB_CB(skb)->has_rxtstamp =
2149			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
2150}
2151
2152/*
2153 *	From tcp_input.c
2154 */
2155
2156int tcp_v4_rcv(struct sk_buff *skb)
2157{
2158	struct net *net = dev_net_rcu(skb->dev);
2159	enum skb_drop_reason drop_reason;
2160	enum tcp_tw_status tw_status;
2161	int sdif = inet_sdif(skb);
2162	int dif = inet_iif(skb);
2163	const struct iphdr *iph;
2164	const struct tcphdr *th;
2165	struct sock *sk = NULL;
2166	bool refcounted;
2167	int ret;
2168	u32 isn;
2169
2170	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
2171	if (skb->pkt_type != PACKET_HOST)
2172		goto discard_it;
2173
2174	/* Count it even if it's bad */
2175	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
2176
2177	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
2178		goto discard_it;
2179
2180	th = (const struct tcphdr *)skb->data;
2181
2182	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
2183		drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
2184		goto bad_packet;
2185	}
2186	if (!pskb_may_pull(skb, th->doff * 4))
2187		goto discard_it;
2188
2189	/* An explanation is required here, I think.
2190	 * Packet length and doff are validated by header prediction,
2191	 * provided case of th->doff==0 is eliminated.
2192	 * So, we defer the checks. */
2193
2194	if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
2195		goto csum_error;
2196
2197	th = (const struct tcphdr *)skb->data;
2198	iph = ip_hdr(skb);
2199lookup:
2200	sk = __inet_lookup_skb(skb, __tcp_hdrlen(th), th->source,
2201			       th->dest, sdif, &refcounted);
2202	if (!sk)
2203		goto no_tcp_socket;
2204
2205	if (sk->sk_state == TCP_TIME_WAIT)
2206		goto do_time_wait;
2207
2208	if (sk->sk_state == TCP_NEW_SYN_RECV) {
2209		struct request_sock *req = inet_reqsk(sk);
2210		bool req_stolen = false;
2211		struct sock *nsk;
2212
2213		sk = req->rsk_listener;
2214		if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2215			drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2216		else
2217			drop_reason = tcp_inbound_hash(sk, req, skb,
2218						       &iph->saddr, &iph->daddr,
2219						       AF_INET, dif, sdif);
2220		if (unlikely(drop_reason)) {
2221			sk_drops_skbadd(sk, skb);
2222			reqsk_put(req);
2223			goto discard_it;
2224		}
2225		if (tcp_checksum_complete(skb)) {
2226			reqsk_put(req);
2227			goto csum_error;
2228		}
2229		if (unlikely(sk->sk_state != TCP_LISTEN)) {
2230			nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
2231			if (!nsk) {
2232				inet_csk_reqsk_queue_drop_and_put(sk, req);
2233				goto lookup;
2234			}
2235			sk = nsk;
2236			/* reuseport_migrate_sock() has already held one sk_refcnt
2237			 * before returning.
2238			 */
2239		} else {
2240			/* We own a reference on the listener, increase it again
2241			 * as we might lose it too soon.
2242			 */
2243			sock_hold(sk);
2244		}
2245		refcounted = true;
2246		nsk = NULL;
2247		if (!tcp_filter(sk, skb, &drop_reason)) {
2248			th = (const struct tcphdr *)skb->data;
2249			iph = ip_hdr(skb);
2250			tcp_v4_fill_cb(skb, iph, th);
2251			nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
2252					    &drop_reason);
2253		}
2254		if (!nsk) {
2255			reqsk_put(req);
2256			if (req_stolen) {
2257				/* Another cpu got exclusive access to req
2258				 * and created a full blown socket.
2259				 * Try to feed this packet to this socket
2260				 * instead of discarding it.
2261				 */
2262				tcp_v4_restore_cb(skb);
2263				sock_put(sk);
2264				goto lookup;
2265			}
2266			goto discard_and_relse;
2267		}
2268		nf_reset_ct(skb);
2269		if (nsk == sk) {
2270			reqsk_put(req);
2271			tcp_v4_restore_cb(skb);
2272		} else {
2273			drop_reason = tcp_child_process(sk, nsk, skb);
2274			if (drop_reason) {
2275				enum sk_rst_reason rst_reason;
2276
2277				rst_reason = sk_rst_convert_drop_reason(drop_reason);
2278				tcp_v4_send_reset(nsk, skb, rst_reason);
2279				goto discard_and_relse;
2280			}
2281			sock_put(sk);
2282			return 0;
2283		}
2284	}
2285
2286process:
2287	if (static_branch_unlikely(&ip4_min_ttl)) {
2288		/* min_ttl can be changed concurrently from do_ip_setsockopt() */
2289		if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
2290			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2291			drop_reason = SKB_DROP_REASON_TCP_MINTTL;
2292			goto discard_and_relse;
2293		}
2294	}
2295
2296	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
2297		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2298		goto discard_and_relse;
2299	}
2300
2301	drop_reason = tcp_inbound_hash(sk, NULL, skb, &iph->saddr, &iph->daddr,
2302				       AF_INET, dif, sdif);
2303	if (drop_reason)
2304		goto discard_and_relse;
2305
2306	nf_reset_ct(skb);
2307
2308	if (tcp_filter(sk, skb, &drop_reason))
2309		goto discard_and_relse;
2310
2311	th = (const struct tcphdr *)skb->data;
2312	iph = ip_hdr(skb);
2313	tcp_v4_fill_cb(skb, iph, th);
2314
2315	skb->dev = NULL;
2316
2317	if (sk->sk_state == TCP_LISTEN) {
2318		ret = tcp_v4_do_rcv(sk, skb);
2319		goto put_and_return;
2320	}
2321
2322	sk_incoming_cpu_update(sk);
2323
2324	bh_lock_sock_nested(sk);
2325	tcp_segs_in(tcp_sk(sk), skb);
2326	ret = 0;
2327	if (!sock_owned_by_user(sk)) {
2328		ret = tcp_v4_do_rcv(sk, skb);
2329	} else {
2330		if (tcp_add_backlog(sk, skb, &drop_reason))
2331			goto discard_and_relse;
2332	}
2333	bh_unlock_sock(sk);
2334
2335put_and_return:
2336	if (refcounted)
2337		sock_put(sk);
2338
2339	return ret;
2340
2341no_tcp_socket:
2342	drop_reason = SKB_DROP_REASON_NO_SOCKET;
2343	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2344		goto discard_it;
2345
2346	tcp_v4_fill_cb(skb, iph, th);
2347
2348	if (tcp_checksum_complete(skb)) {
2349csum_error:
2350		drop_reason = SKB_DROP_REASON_TCP_CSUM;
2351		trace_tcp_bad_csum(skb);
2352		__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2353bad_packet:
2354		__TCP_INC_STATS(net, TCP_MIB_INERRS);
2355	} else {
2356		tcp_v4_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
2357	}
2358
2359discard_it:
2360	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
2361	/* Discard frame. */
2362	sk_skb_reason_drop(sk, skb, drop_reason);
2363	return 0;
2364
2365discard_and_relse:
2366	sk_drops_skbadd(sk, skb);
2367	if (refcounted)
2368		sock_put(sk);
2369	goto discard_it;
2370
2371do_time_wait:
2372	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2373		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2374		inet_twsk_put(inet_twsk(sk));
2375		goto discard_it;
2376	}
2377
2378	tcp_v4_fill_cb(skb, iph, th);
2379
2380	if (tcp_checksum_complete(skb)) {
2381		inet_twsk_put(inet_twsk(sk));
2382		goto csum_error;
2383	}
2384
2385	tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
2386					       &drop_reason);
2387	switch (tw_status) {
2388	case TCP_TW_SYN: {
2389		struct sock *sk2 = inet_lookup_listener(net, skb, __tcp_hdrlen(th),
2390							iph->saddr, th->source,
2391							iph->daddr, th->dest,
2392							inet_iif(skb),
2393							sdif);
2394		if (sk2) {
2395			inet_twsk_deschedule_put(inet_twsk(sk));
2396			sk = sk2;
2397			tcp_v4_restore_cb(skb);
2398			refcounted = false;
2399			__this_cpu_write(tcp_tw_isn, isn);
2400			goto process;
2401		}
2402
2403		drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
2404		if (drop_reason)
2405			break;
2406	}
2407		/* to ACK */
2408		fallthrough;
2409	case TCP_TW_ACK:
2410	case TCP_TW_ACK_OOW:
2411		tcp_v4_timewait_ack(sk, skb, tw_status);
2412		break;
2413	case TCP_TW_RST:
2414		tcp_v4_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
2415		inet_twsk_deschedule_put(inet_twsk(sk));
2416		goto discard_it;
2417	case TCP_TW_SUCCESS:;
2418	}
2419	goto discard_it;
2420}
2421
2422static struct timewait_sock_ops tcp_timewait_sock_ops = {
2423	.twsk_obj_size	= sizeof(struct tcp_timewait_sock),
2424};
2425
2426void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2427{
2428	struct dst_entry *dst = skb_dst(skb);
2429
2430	if (dst && dst_hold_safe(dst)) {
2431		rcu_assign_pointer(sk->sk_rx_dst, dst);
2432		sk->sk_rx_dst_ifindex = skb->skb_iif;
2433	}
2434}
2435EXPORT_IPV6_MOD(inet_sk_rx_dst_set);
2436
2437const struct inet_connection_sock_af_ops ipv4_specific = {
2438	.queue_xmit	   = ip_queue_xmit,
2439	.send_check	   = tcp_v4_send_check,
2440	.rebuild_header	   = inet_sk_rebuild_header,
2441	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
2442	.conn_request	   = tcp_v4_conn_request,
2443	.syn_recv_sock	   = tcp_v4_syn_recv_sock,
2444	.net_header_len	   = sizeof(struct iphdr),
2445	.setsockopt	   = ip_setsockopt,
2446	.getsockopt	   = ip_getsockopt,
2447	.mtu_reduced	   = tcp_v4_mtu_reduced,
2448};
2449EXPORT_IPV6_MOD(ipv4_specific);
2450
2451#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2452static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2453#ifdef CONFIG_TCP_MD5SIG
2454	.md5_lookup		= tcp_v4_md5_lookup,
2455	.calc_md5_hash		= tcp_v4_md5_hash_skb,
2456	.md5_parse		= tcp_v4_parse_md5_keys,
2457#endif
2458#ifdef CONFIG_TCP_AO
2459	.ao_lookup		= tcp_v4_ao_lookup,
2460	.calc_ao_hash		= tcp_v4_ao_hash_skb,
2461	.ao_parse		= tcp_v4_parse_ao,
2462	.ao_calc_key_sk		= tcp_v4_ao_calc_key_sk,
2463#endif
2464};
2465
2466static void tcp4_destruct_sock(struct sock *sk)
2467{
2468	tcp_md5_destruct_sock(sk);
2469	tcp_ao_destroy_sock(sk, false);
2470	inet_sock_destruct(sk);
2471}
2472#endif
2473
2474/* NOTE: A lot of things set to zero explicitly by call to
2475 *       sk_alloc() so need not be done here.
2476 */
2477static int tcp_v4_init_sock(struct sock *sk)
2478{
2479	struct inet_connection_sock *icsk = inet_csk(sk);
2480
2481	tcp_init_sock(sk);
2482
2483	icsk->icsk_af_ops = &ipv4_specific;
2484
2485#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2486	tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2487	sk->sk_destruct = tcp4_destruct_sock;
2488#endif
2489
2490	return 0;
2491}
2492
2493static void tcp_release_user_frags(struct sock *sk)
2494{
2495#ifdef CONFIG_PAGE_POOL
2496	unsigned long index;
2497	void *netmem;
2498
2499	xa_for_each(&sk->sk_user_frags, index, netmem)
2500		WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
2501#endif
2502}
2503
2504void tcp_v4_destroy_sock(struct sock *sk)
2505{
2506	struct tcp_sock *tp = tcp_sk(sk);
2507
2508	tcp_release_user_frags(sk);
2509
2510	xa_destroy(&sk->sk_user_frags);
2511
2512	trace_tcp_destroy_sock(sk);
2513
2514	tcp_clear_xmit_timers(sk);
2515
2516	tcp_cleanup_congestion_control(sk);
2517
2518	tcp_cleanup_ulp(sk);
2519
2520	/* Cleanup up the write buffer. */
2521	tcp_write_queue_purge(sk);
2522
2523	/* Check if we want to disable active TFO */
2524	tcp_fastopen_active_disable_ofo_check(sk);
2525
2526	/* Cleans up our, hopefully empty, out_of_order_queue. */
2527	skb_rbtree_purge(&tp->out_of_order_queue);
2528
2529	/* Clean up a referenced TCP bind bucket. */
2530	if (inet_csk(sk)->icsk_bind_hash)
2531		inet_put_port(sk);
2532
2533	BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2534
2535	/* If socket is aborted during connect operation */
2536	tcp_free_fastopen_req(tp);
2537	tcp_fastopen_destroy_cipher(sk);
2538	tcp_saved_syn_free(tp);
2539
2540	sk_sockets_allocated_dec(sk);
2541}
2542EXPORT_IPV6_MOD(tcp_v4_destroy_sock);
2543
2544#ifdef CONFIG_PROC_FS
2545/* Proc filesystem TCP sock list dumping. */
2546
2547static unsigned short seq_file_family(const struct seq_file *seq);
2548
2549static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
2550{
2551	unsigned short family = seq_file_family(seq);
2552
2553	/* AF_UNSPEC is used as a match all */
2554	return ((family == AF_UNSPEC || family == sk->sk_family) &&
2555		net_eq(sock_net(sk), seq_file_net(seq)));
2556}
2557
2558/* Find a non empty bucket (starting from st->bucket)
2559 * and return the first sk from it.
2560 */
2561static void *listening_get_first(struct seq_file *seq)
2562{
2563	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2564	struct tcp_iter_state *st = seq->private;
2565
2566	st->offset = 0;
2567	for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
2568		struct inet_listen_hashbucket *ilb2;
2569		struct hlist_nulls_node *node;
2570		struct sock *sk;
2571
2572		ilb2 = &hinfo->lhash2[st->bucket];
2573		if (hlist_nulls_empty(&ilb2->nulls_head))
2574			continue;
2575
2576		spin_lock(&ilb2->lock);
2577		sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
2578			if (seq_sk_match(seq, sk))
2579				return sk;
2580		}
2581		spin_unlock(&ilb2->lock);
2582	}
2583
2584	return NULL;
2585}
2586
2587/* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
2588 * If "cur" is the last one in the st->bucket,
2589 * call listening_get_first() to return the first sk of the next
2590 * non empty bucket.
2591 */
2592static void *listening_get_next(struct seq_file *seq, void *cur)
2593{
2594	struct tcp_iter_state *st = seq->private;
2595	struct inet_listen_hashbucket *ilb2;
2596	struct hlist_nulls_node *node;
2597	struct inet_hashinfo *hinfo;
2598	struct sock *sk = cur;
2599
2600	++st->num;
2601	++st->offset;
2602
2603	sk = sk_nulls_next(sk);
2604	sk_nulls_for_each_from(sk, node) {
2605		if (seq_sk_match(seq, sk))
2606			return sk;
2607	}
2608
2609	hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2610	ilb2 = &hinfo->lhash2[st->bucket];
2611	spin_unlock(&ilb2->lock);
2612	++st->bucket;
2613	return listening_get_first(seq);
2614}
2615
2616static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2617{
2618	struct tcp_iter_state *st = seq->private;
2619	void *rc;
2620
2621	st->bucket = 0;
2622	st->offset = 0;
2623	rc = listening_get_first(seq);
2624
2625	while (rc && *pos) {
2626		rc = listening_get_next(seq, rc);
2627		--*pos;
2628	}
2629	return rc;
2630}
2631
2632static inline bool empty_bucket(struct inet_hashinfo *hinfo,
2633				const struct tcp_iter_state *st)
2634{
2635	return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
2636}
2637
2638/*
2639 * Get first established socket starting from bucket given in st->bucket.
2640 * If st->bucket is zero, the very first socket in the hash is returned.
2641 */
2642static void *established_get_first(struct seq_file *seq)
2643{
2644	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2645	struct tcp_iter_state *st = seq->private;
2646
2647	st->offset = 0;
2648	for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
2649		struct sock *sk;
2650		struct hlist_nulls_node *node;
2651		spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
2652
2653		cond_resched();
2654
2655		/* Lockless fast path for the common case of empty buckets */
2656		if (empty_bucket(hinfo, st))
2657			continue;
2658
2659		spin_lock_bh(lock);
2660		sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
2661			if (seq_sk_match(seq, sk))
2662				return sk;
2663		}
2664		spin_unlock_bh(lock);
2665	}
2666
2667	return NULL;
2668}
2669
2670static void *established_get_next(struct seq_file *seq, void *cur)
2671{
2672	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2673	struct tcp_iter_state *st = seq->private;
2674	struct hlist_nulls_node *node;
2675	struct sock *sk = cur;
2676
2677	++st->num;
2678	++st->offset;
2679
2680	sk = sk_nulls_next(sk);
2681
2682	sk_nulls_for_each_from(sk, node) {
2683		if (seq_sk_match(seq, sk))
2684			return sk;
2685	}
2686
2687	spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2688	++st->bucket;
2689	return established_get_first(seq);
2690}
2691
2692static void *established_get_idx(struct seq_file *seq, loff_t pos)
2693{
2694	struct tcp_iter_state *st = seq->private;
2695	void *rc;
2696
2697	st->bucket = 0;
2698	rc = established_get_first(seq);
2699
2700	while (rc && pos) {
2701		rc = established_get_next(seq, rc);
2702		--pos;
2703	}
2704	return rc;
2705}
2706
2707static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2708{
2709	void *rc;
2710	struct tcp_iter_state *st = seq->private;
2711
2712	st->state = TCP_SEQ_STATE_LISTENING;
2713	rc	  = listening_get_idx(seq, &pos);
2714
2715	if (!rc) {
2716		st->state = TCP_SEQ_STATE_ESTABLISHED;
2717		rc	  = established_get_idx(seq, pos);
2718	}
2719
2720	return rc;
2721}
2722
2723static void *tcp_seek_last_pos(struct seq_file *seq)
2724{
2725	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2726	struct tcp_iter_state *st = seq->private;
2727	int bucket = st->bucket;
2728	int offset = st->offset;
2729	int orig_num = st->num;
2730	void *rc = NULL;
2731
2732	switch (st->state) {
2733	case TCP_SEQ_STATE_LISTENING:
2734		if (st->bucket > hinfo->lhash2_mask)
2735			break;
2736		rc = listening_get_first(seq);
2737		while (offset-- && rc && bucket == st->bucket)
2738			rc = listening_get_next(seq, rc);
2739		if (rc)
2740			break;
2741		st->bucket = 0;
2742		st->state = TCP_SEQ_STATE_ESTABLISHED;
2743		fallthrough;
2744	case TCP_SEQ_STATE_ESTABLISHED:
2745		if (st->bucket > hinfo->ehash_mask)
2746			break;
2747		rc = established_get_first(seq);
2748		while (offset-- && rc && bucket == st->bucket)
2749			rc = established_get_next(seq, rc);
2750	}
2751
2752	st->num = orig_num;
2753
2754	return rc;
2755}
2756
2757void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2758{
2759	struct tcp_iter_state *st = seq->private;
2760	void *rc;
2761
2762	if (*pos && *pos == st->last_pos) {
2763		rc = tcp_seek_last_pos(seq);
2764		if (rc)
2765			goto out;
2766	}
2767
2768	st->state = TCP_SEQ_STATE_LISTENING;
2769	st->num = 0;
2770	st->bucket = 0;
2771	st->offset = 0;
2772	rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2773
2774out:
2775	st->last_pos = *pos;
2776	return rc;
2777}
2778EXPORT_IPV6_MOD(tcp_seq_start);
2779
2780void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2781{
2782	struct tcp_iter_state *st = seq->private;
2783	void *rc = NULL;
2784
2785	if (v == SEQ_START_TOKEN) {
2786		rc = tcp_get_idx(seq, 0);
2787		goto out;
2788	}
2789
2790	switch (st->state) {
2791	case TCP_SEQ_STATE_LISTENING:
2792		rc = listening_get_next(seq, v);
2793		if (!rc) {
2794			st->state = TCP_SEQ_STATE_ESTABLISHED;
2795			st->bucket = 0;
2796			st->offset = 0;
2797			rc	  = established_get_first(seq);
2798		}
2799		break;
2800	case TCP_SEQ_STATE_ESTABLISHED:
2801		rc = established_get_next(seq, v);
2802		break;
2803	}
2804out:
2805	++*pos;
2806	st->last_pos = *pos;
2807	return rc;
2808}
2809EXPORT_IPV6_MOD(tcp_seq_next);
2810
2811void tcp_seq_stop(struct seq_file *seq, void *v)
2812{
2813	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2814	struct tcp_iter_state *st = seq->private;
2815
2816	switch (st->state) {
2817	case TCP_SEQ_STATE_LISTENING:
2818		if (v != SEQ_START_TOKEN)
2819			spin_unlock(&hinfo->lhash2[st->bucket].lock);
2820		break;
2821	case TCP_SEQ_STATE_ESTABLISHED:
2822		if (v)
2823			spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2824		break;
2825	}
2826}
2827EXPORT_IPV6_MOD(tcp_seq_stop);
2828
2829static void get_openreq4(const struct request_sock *req,
2830			 struct seq_file *f, int i)
2831{
2832	const struct inet_request_sock *ireq = inet_rsk(req);
2833	long delta = req->rsk_timer.expires - jiffies;
2834
2835	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2836		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2837		i,
2838		ireq->ir_loc_addr,
2839		ireq->ir_num,
2840		ireq->ir_rmt_addr,
2841		ntohs(ireq->ir_rmt_port),
2842		TCP_SYN_RECV,
2843		0, 0, /* could print option size, but that is af dependent. */
2844		1,    /* timers active (only the expire timer) */
2845		jiffies_delta_to_clock_t(delta),
2846		req->num_timeout,
2847		from_kuid_munged(seq_user_ns(f),
2848				 sk_uid(req->rsk_listener)),
2849		0,  /* non standard timer */
2850		0, /* open_requests have no inode */
2851		0,
2852		req);
2853}
2854
2855static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2856{
2857	int timer_active;
2858	unsigned long timer_expires;
2859	const struct tcp_sock *tp = tcp_sk(sk);
2860	const struct inet_connection_sock *icsk = inet_csk(sk);
2861	const struct inet_sock *inet = inet_sk(sk);
2862	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2863	__be32 dest = inet->inet_daddr;
2864	__be32 src = inet->inet_rcv_saddr;
2865	__u16 destp = ntohs(inet->inet_dport);
2866	__u16 srcp = ntohs(inet->inet_sport);
2867	u8 icsk_pending;
2868	int rx_queue;
2869	int state;
2870
2871	icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2872	if (icsk_pending == ICSK_TIME_RETRANS ||
2873	    icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2874	    icsk_pending == ICSK_TIME_LOSS_PROBE) {
2875		timer_active	= 1;
2876		timer_expires	= tcp_timeout_expires(sk);
2877	} else if (icsk_pending == ICSK_TIME_PROBE0) {
2878		timer_active	= 4;
2879		timer_expires	= tcp_timeout_expires(sk);
2880	} else if (timer_pending(&icsk->icsk_keepalive_timer)) {
2881		timer_active	= 2;
2882		timer_expires	= icsk->icsk_keepalive_timer.expires;
2883	} else {
2884		timer_active	= 0;
2885		timer_expires = jiffies;
2886	}
2887
2888	state = inet_sk_state_load(sk);
2889	if (state == TCP_LISTEN)
2890		rx_queue = READ_ONCE(sk->sk_ack_backlog);
2891	else
2892		/* Because we don't lock the socket,
2893		 * we might find a transient negative value.
2894		 */
2895		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2896				      READ_ONCE(tp->copied_seq), 0);
2897
2898	seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2899			"%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2900		i, src, srcp, dest, destp, state,
2901		READ_ONCE(tp->write_seq) - tp->snd_una,
2902		rx_queue,
2903		timer_active,
2904		jiffies_delta_to_clock_t(timer_expires - jiffies),
2905		READ_ONCE(icsk->icsk_retransmits),
2906		from_kuid_munged(seq_user_ns(f), sk_uid(sk)),
2907		READ_ONCE(icsk->icsk_probes_out),
2908		sock_i_ino(sk),
2909		refcount_read(&sk->sk_refcnt), sk,
2910		jiffies_to_clock_t(icsk->icsk_rto),
2911		jiffies_to_clock_t(icsk->icsk_ack.ato),
2912		(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2913		tcp_snd_cwnd(tp),
2914		state == TCP_LISTEN ?
2915		    fastopenq->max_qlen :
2916		    (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2917}
2918
2919static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2920			       struct seq_file *f, int i)
2921{
2922	long delta = tw->tw_timer.expires - jiffies;
2923	__be32 dest, src;
2924	__u16 destp, srcp;
2925
2926	dest  = tw->tw_daddr;
2927	src   = tw->tw_rcv_saddr;
2928	destp = ntohs(tw->tw_dport);
2929	srcp  = ntohs(tw->tw_sport);
2930
2931	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2932		" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2933		i, src, srcp, dest, destp, READ_ONCE(tw->tw_substate), 0, 0,
2934		3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2935		refcount_read(&tw->tw_refcnt), tw);
2936}
2937
2938#define TMPSZ 150
2939
2940static int tcp4_seq_show(struct seq_file *seq, void *v)
2941{
2942	struct tcp_iter_state *st;
2943	struct sock *sk = v;
2944
2945	seq_setwidth(seq, TMPSZ - 1);
2946	if (v == SEQ_START_TOKEN) {
2947		seq_puts(seq, "  sl  local_address rem_address   st tx_queue "
2948			   "rx_queue tr tm->when retrnsmt   uid  timeout "
2949			   "inode");
2950		goto out;
2951	}
2952	st = seq->private;
2953
2954	if (sk->sk_state == TCP_TIME_WAIT)
2955		get_timewait4_sock(v, seq, st->num);
2956	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2957		get_openreq4(v, seq, st->num);
2958	else
2959		get_tcp4_sock(v, seq, st->num);
2960out:
2961	seq_pad(seq, '\n');
2962	return 0;
2963}
2964
2965#ifdef CONFIG_BPF_SYSCALL
2966union bpf_tcp_iter_batch_item {
2967	struct sock *sk;
2968	__u64 cookie;
2969};
2970
2971struct bpf_tcp_iter_state {
2972	struct tcp_iter_state state;
2973	unsigned int cur_sk;
2974	unsigned int end_sk;
2975	unsigned int max_sk;
2976	union bpf_tcp_iter_batch_item *batch;
2977};
2978
2979struct bpf_iter__tcp {
2980	__bpf_md_ptr(struct bpf_iter_meta *, meta);
2981	__bpf_md_ptr(struct sock_common *, sk_common);
2982	uid_t uid __aligned(8);
2983};
2984
2985static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2986			     struct sock_common *sk_common, uid_t uid)
2987{
2988	struct bpf_iter__tcp ctx;
2989
2990	meta->seq_num--;  /* skip SEQ_START_TOKEN */
2991	ctx.meta = meta;
2992	ctx.sk_common = sk_common;
2993	ctx.uid = uid;
2994	return bpf_iter_run_prog(prog, &ctx);
2995}
2996
2997static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
2998{
2999	union bpf_tcp_iter_batch_item *item;
3000	unsigned int cur_sk = iter->cur_sk;
3001	__u64 cookie;
3002
3003	/* Remember the cookies of the sockets we haven't seen yet, so we can
3004	 * pick up where we left off next time around.
3005	 */
3006	while (cur_sk < iter->end_sk) {
3007		item = &iter->batch[cur_sk++];
3008		cookie = sock_gen_cookie(item->sk);
3009		sock_gen_put(item->sk);
3010		item->cookie = cookie;
3011	}
3012}
3013
3014static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
3015				      unsigned int new_batch_sz, gfp_t flags)
3016{
3017	union bpf_tcp_iter_batch_item *new_batch;
3018
3019	new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
3020			     flags | __GFP_NOWARN);
3021	if (!new_batch)
3022		return -ENOMEM;
3023
3024	memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
3025	kvfree(iter->batch);
3026	iter->batch = new_batch;
3027	iter->max_sk = new_batch_sz;
3028
3029	return 0;
3030}
3031
3032static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk,
3033					       union bpf_tcp_iter_batch_item *cookies,
3034					       int n_cookies)
3035{
3036	struct hlist_nulls_node *node;
3037	struct sock *sk;
3038	int i;
3039
3040	for (i = 0; i < n_cookies; i++) {
3041		sk = first_sk;
3042		sk_nulls_for_each_from(sk, node)
3043			if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
3044				return sk;
3045	}
3046
3047	return NULL;
3048}
3049
3050static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq)
3051{
3052	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
3053	struct bpf_tcp_iter_state *iter = seq->private;
3054	struct tcp_iter_state *st = &iter->state;
3055	unsigned int find_cookie = iter->cur_sk;
3056	unsigned int end_cookie = iter->end_sk;
3057	int resume_bucket = st->bucket;
3058	struct sock *sk;
3059
3060	if (end_cookie && find_cookie == end_cookie)
3061		++st->bucket;
3062
3063	sk = listening_get_first(seq);
3064	iter->cur_sk = 0;
3065	iter->end_sk = 0;
3066
3067	if (sk && st->bucket == resume_bucket && end_cookie) {
3068		sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
3069						end_cookie - find_cookie);
3070		if (!sk) {
3071			spin_unlock(&hinfo->lhash2[st->bucket].lock);
3072			++st->bucket;
3073			sk = listening_get_first(seq);
3074		}
3075	}
3076
3077	return sk;
3078}
3079
3080static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq)
3081{
3082	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
3083	struct bpf_tcp_iter_state *iter = seq->private;
3084	struct tcp_iter_state *st = &iter->state;
3085	unsigned int find_cookie = iter->cur_sk;
3086	unsigned int end_cookie = iter->end_sk;
3087	int resume_bucket = st->bucket;
3088	struct sock *sk;
3089
3090	if (end_cookie && find_cookie == end_cookie)
3091		++st->bucket;
3092
3093	sk = established_get_first(seq);
3094	iter->cur_sk = 0;
3095	iter->end_sk = 0;
3096
3097	if (sk && st->bucket == resume_bucket && end_cookie) {
3098		sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
3099						end_cookie - find_cookie);
3100		if (!sk) {
3101			spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
3102			++st->bucket;
3103			sk = established_get_first(seq);
3104		}
3105	}
3106
3107	return sk;
3108}
3109
3110static struct sock *bpf_iter_tcp_resume(struct seq_file *seq)
3111{
3112	struct bpf_tcp_iter_state *iter = seq->private;
3113	struct tcp_iter_state *st = &iter->state;
3114	struct sock *sk = NULL;
3115
3116	switch (st->state) {
3117	case TCP_SEQ_STATE_LISTENING:
3118		sk = bpf_iter_tcp_resume_listening(seq);
3119		if (sk)
3120			break;
3121		st->bucket = 0;
3122		st->state = TCP_SEQ_STATE_ESTABLISHED;
3123		fallthrough;
3124	case TCP_SEQ_STATE_ESTABLISHED:
3125		sk = bpf_iter_tcp_resume_established(seq);
3126		break;
3127	}
3128
3129	return sk;
3130}
3131
3132static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
3133						 struct sock **start_sk)
3134{
3135	struct bpf_tcp_iter_state *iter = seq->private;
3136	struct hlist_nulls_node *node;
3137	unsigned int expected = 1;
3138	struct sock *sk;
3139
3140	sock_hold(*start_sk);
3141	iter->batch[iter->end_sk++].sk = *start_sk;
3142
3143	sk = sk_nulls_next(*start_sk);
3144	*start_sk = NULL;
3145	sk_nulls_for_each_from(sk, node) {
3146		if (seq_sk_match(seq, sk)) {
3147			if (iter->end_sk < iter->max_sk) {
3148				sock_hold(sk);
3149				iter->batch[iter->end_sk++].sk = sk;
3150			} else if (!*start_sk) {
3151				/* Remember where we left off. */
3152				*start_sk = sk;
3153			}
3154			expected++;
3155		}
3156	}
3157
3158	return expected;
3159}
3160
3161static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
3162						   struct sock **start_sk)
3163{
3164	struct bpf_tcp_iter_state *iter = seq->private;
3165	struct hlist_nulls_node *node;
3166	unsigned int expected = 1;
3167	struct sock *sk;
3168
3169	sock_hold(*start_sk);
3170	iter->batch[iter->end_sk++].sk = *start_sk;
3171
3172	sk = sk_nulls_next(*start_sk);
3173	*start_sk = NULL;
3174	sk_nulls_for_each_from(sk, node) {
3175		if (seq_sk_match(seq, sk)) {
3176			if (iter->end_sk < iter->max_sk) {
3177				sock_hold(sk);
3178				iter->batch[iter->end_sk++].sk = sk;
3179			} else if (!*start_sk) {
3180				/* Remember where we left off. */
3181				*start_sk = sk;
3182			}
3183			expected++;
3184		}
3185	}
3186
3187	return expected;
3188}
3189
3190static unsigned int bpf_iter_fill_batch(struct seq_file *seq,
3191					struct sock **start_sk)
3192{
3193	struct bpf_tcp_iter_state *iter = seq->private;
3194	struct tcp_iter_state *st = &iter->state;
3195
3196	if (st->state == TCP_SEQ_STATE_LISTENING)
3197		return bpf_iter_tcp_listening_batch(seq, start_sk);
3198	else
3199		return bpf_iter_tcp_established_batch(seq, start_sk);
3200}
3201
3202static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq)
3203{
3204	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
3205	struct bpf_tcp_iter_state *iter = seq->private;
3206	struct tcp_iter_state *st = &iter->state;
3207
3208	if (st->state == TCP_SEQ_STATE_LISTENING)
3209		spin_unlock(&hinfo->lhash2[st->bucket].lock);
3210	else
3211		spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
3212}
3213
3214static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
3215{
3216	struct bpf_tcp_iter_state *iter = seq->private;
3217	unsigned int expected;
3218	struct sock *sk;
3219	int err;
3220
3221	sk = bpf_iter_tcp_resume(seq);
3222	if (!sk)
3223		return NULL; /* Done */
3224
3225	expected = bpf_iter_fill_batch(seq, &sk);
3226	if (likely(iter->end_sk == expected))
3227		goto done;
3228
3229	/* Batch size was too small. */
3230	bpf_iter_tcp_unlock_bucket(seq);
3231	bpf_iter_tcp_put_batch(iter);
3232	err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2,
3233					 GFP_USER);
3234	if (err)
3235		return ERR_PTR(err);
3236
3237	sk = bpf_iter_tcp_resume(seq);
3238	if (!sk)
3239		return NULL; /* Done */
3240
3241	expected = bpf_iter_fill_batch(seq, &sk);
3242	if (likely(iter->end_sk == expected))
3243		goto done;
3244
3245	/* Batch size was still too small. Hold onto the lock while we try
3246	 * again with a larger batch to make sure the current bucket's size
3247	 * does not change in the meantime.
3248	 */
3249	err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT);
3250	if (err) {
3251		bpf_iter_tcp_unlock_bucket(seq);
3252		return ERR_PTR(err);
3253	}
3254
3255	expected = bpf_iter_fill_batch(seq, &sk);
3256	WARN_ON_ONCE(iter->end_sk != expected);
3257done:
3258	bpf_iter_tcp_unlock_bucket(seq);
3259	return iter->batch[0].sk;
3260}
3261
3262static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
3263{
3264	/* bpf iter does not support lseek, so it always
3265	 * continue from where it was stop()-ped.
3266	 */
3267	if (*pos)
3268		return bpf_iter_tcp_batch(seq);
3269
3270	return SEQ_START_TOKEN;
3271}
3272
3273static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3274{
3275	struct bpf_tcp_iter_state *iter = seq->private;
3276	struct tcp_iter_state *st = &iter->state;
3277	struct sock *sk;
3278
3279	/* Whenever seq_next() is called, the iter->cur_sk is
3280	 * done with seq_show(), so advance to the next sk in
3281	 * the batch.
3282	 */
3283	if (iter->cur_sk < iter->end_sk) {
3284		/* Keeping st->num consistent in tcp_iter_state.
3285		 * bpf_iter_tcp does not use st->num.
3286		 * meta.seq_num is used instead.
3287		 */
3288		st->num++;
3289		sock_gen_put(iter->batch[iter->cur_sk++].sk);
3290	}
3291
3292	if (iter->cur_sk < iter->end_sk)
3293		sk = iter->batch[iter->cur_sk].sk;
3294	else
3295		sk = bpf_iter_tcp_batch(seq);
3296
3297	++*pos;
3298	/* Keeping st->last_pos consistent in tcp_iter_state.
3299	 * bpf iter does not do lseek, so st->last_pos always equals to *pos.
3300	 */
3301	st->last_pos = *pos;
3302	return sk;
3303}
3304
3305static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
3306{
3307	struct bpf_iter_meta meta;
3308	struct bpf_prog *prog;
3309	struct sock *sk = v;
3310	uid_t uid;
3311	int ret;
3312
3313	if (v == SEQ_START_TOKEN)
3314		return 0;
3315
3316	if (sk_fullsock(sk))
3317		lock_sock(sk);
3318
3319	if (unlikely(sk_unhashed(sk))) {
3320		ret = SEQ_SKIP;
3321		goto unlock;
3322	}
3323
3324	if (sk->sk_state == TCP_TIME_WAIT) {
3325		uid = 0;
3326	} else if (sk->sk_state == TCP_NEW_SYN_RECV) {
3327		const struct request_sock *req = v;
3328
3329		uid = from_kuid_munged(seq_user_ns(seq),
3330				       sk_uid(req->rsk_listener));
3331	} else {
3332		uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
3333	}
3334
3335	meta.seq = seq;
3336	prog = bpf_iter_get_info(&meta, false);
3337	ret = tcp_prog_seq_show(prog, &meta, v, uid);
3338
3339unlock:
3340	if (sk_fullsock(sk))
3341		release_sock(sk);
3342	return ret;
3343
3344}
3345
3346static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
3347{
3348	struct bpf_tcp_iter_state *iter = seq->private;
3349	struct bpf_iter_meta meta;
3350	struct bpf_prog *prog;
3351
3352	if (!v) {
3353		meta.seq = seq;
3354		prog = bpf_iter_get_info(&meta, true);
3355		if (prog)
3356			(void)tcp_prog_seq_show(prog, &meta, v, 0);
3357	}
3358
3359	if (iter->cur_sk < iter->end_sk)
3360		bpf_iter_tcp_put_batch(iter);
3361}
3362
3363static const struct seq_operations bpf_iter_tcp_seq_ops = {
3364	.show		= bpf_iter_tcp_seq_show,
3365	.start		= bpf_iter_tcp_seq_start,
3366	.next		= bpf_iter_tcp_seq_next,
3367	.stop		= bpf_iter_tcp_seq_stop,
3368};
3369#endif
3370static unsigned short seq_file_family(const struct seq_file *seq)
3371{
3372	const struct tcp_seq_afinfo *afinfo;
3373
3374#ifdef CONFIG_BPF_SYSCALL
3375	/* Iterated from bpf_iter.  Let the bpf prog to filter instead. */
3376	if (seq->op == &bpf_iter_tcp_seq_ops)
3377		return AF_UNSPEC;
3378#endif
3379
3380	/* Iterated from proc fs */
3381	afinfo = pde_data(file_inode(seq->file));
3382	return afinfo->family;
3383}
3384
3385static const struct seq_operations tcp4_seq_ops = {
3386	.show		= tcp4_seq_show,
3387	.start		= tcp_seq_start,
3388	.next		= tcp_seq_next,
3389	.stop		= tcp_seq_stop,
3390};
3391
3392static struct tcp_seq_afinfo tcp4_seq_afinfo = {
3393	.family		= AF_INET,
3394};
3395
3396static int __net_init tcp4_proc_init_net(struct net *net)
3397{
3398	if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
3399			sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
3400		return -ENOMEM;
3401	return 0;
3402}
3403
3404static void __net_exit tcp4_proc_exit_net(struct net *net)
3405{
3406	remove_proc_entry("tcp", net->proc_net);
3407}
3408
3409static struct pernet_operations tcp4_net_ops = {
3410	.init = tcp4_proc_init_net,
3411	.exit = tcp4_proc_exit_net,
3412};
3413
3414int __init tcp4_proc_init(void)
3415{
3416	return register_pernet_subsys(&tcp4_net_ops);
3417}
3418
3419void tcp4_proc_exit(void)
3420{
3421	unregister_pernet_subsys(&tcp4_net_ops);
3422}
3423#endif /* CONFIG_PROC_FS */
3424
3425/* @wake is one when sk_stream_write_space() calls us.
3426 * This sends EPOLLOUT only if notsent_bytes is half the limit.
3427 * This mimics the strategy used in sock_def_write_space().
3428 */
3429bool tcp_stream_memory_free(const struct sock *sk, int wake)
3430{
3431	const struct tcp_sock *tp = tcp_sk(sk);
3432	u32 notsent_bytes = READ_ONCE(tp->write_seq) -
3433			    READ_ONCE(tp->snd_nxt);
3434
3435	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
3436}
3437EXPORT_SYMBOL(tcp_stream_memory_free);
3438
3439struct proto tcp_prot = {
3440	.name			= "TCP",
3441	.owner			= THIS_MODULE,
3442	.close			= tcp_close,
3443	.pre_connect		= tcp_v4_pre_connect,
3444	.connect		= tcp_v4_connect,
3445	.disconnect		= tcp_disconnect,
3446	.accept			= inet_csk_accept,
3447	.ioctl			= tcp_ioctl,
3448	.init			= tcp_v4_init_sock,
3449	.destroy		= tcp_v4_destroy_sock,
3450	.shutdown		= tcp_shutdown,
3451	.setsockopt		= tcp_setsockopt,
3452	.getsockopt		= tcp_getsockopt,
3453	.bpf_bypass_getsockopt	= tcp_bpf_bypass_getsockopt,
3454	.keepalive		= tcp_set_keepalive,
3455	.recvmsg		= tcp_recvmsg,
3456	.sendmsg		= tcp_sendmsg,
3457	.splice_eof		= tcp_splice_eof,
3458	.backlog_rcv		= tcp_v4_do_rcv,
3459	.release_cb		= tcp_release_cb,
3460	.hash			= inet_hash,
3461	.unhash			= inet_unhash,
3462	.get_port		= inet_csk_get_port,
3463	.put_port		= inet_put_port,
3464#ifdef CONFIG_BPF_SYSCALL
3465	.psock_update_sk_prot	= tcp_bpf_update_proto,
3466#endif
3467	.enter_memory_pressure	= tcp_enter_memory_pressure,
3468	.leave_memory_pressure	= tcp_leave_memory_pressure,
3469	.stream_memory_free	= tcp_stream_memory_free,
3470	.sockets_allocated	= &tcp_sockets_allocated,
3471
3472	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,
3473	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
3474
3475	.memory_pressure	= &tcp_memory_pressure,
3476	.sysctl_mem		= sysctl_tcp_mem,
3477	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
3478	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
3479	.max_header		= MAX_TCP_HEADER,
3480	.obj_size		= sizeof(struct tcp_sock),
3481	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
3482	.twsk_prot		= &tcp_timewait_sock_ops,
3483	.rsk_prot		= &tcp_request_sock_ops,
3484	.h.hashinfo		= NULL,
3485	.no_autobind		= true,
3486	.diag_destroy		= tcp_abort,
3487};
3488EXPORT_SYMBOL(tcp_prot);
3489
3490static void __net_exit tcp_sk_exit(struct net *net)
3491{
3492	if (net->ipv4.tcp_congestion_control)
3493		bpf_module_put(net->ipv4.tcp_congestion_control,
3494			       net->ipv4.tcp_congestion_control->owner);
3495}
3496
3497static void __net_init tcp_set_hashinfo(struct net *net)
3498{
3499	struct inet_hashinfo *hinfo;
3500	unsigned int ehash_entries;
3501	struct net *old_net;
3502
3503	if (net_eq(net, &init_net))
3504		goto fallback;
3505
3506	old_net = current->nsproxy->net_ns;
3507	ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
3508	if (!ehash_entries)
3509		goto fallback;
3510
3511	ehash_entries = roundup_pow_of_two(ehash_entries);
3512	hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
3513	if (!hinfo) {
3514		pr_warn("Failed to allocate TCP ehash (entries: %u) "
3515			"for a netns, fallback to the global one\n",
3516			ehash_entries);
3517fallback:
3518		hinfo = &tcp_hashinfo;
3519		ehash_entries = tcp_hashinfo.ehash_mask + 1;
3520	}
3521
3522	net->ipv4.tcp_death_row.hashinfo = hinfo;
3523	net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
3524	net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
3525}
3526
3527static int __net_init tcp_sk_init(struct net *net)
3528{
3529	net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN;
3530	net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL;
3531	net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON;
3532	net->ipv4.sysctl_tcp_ecn_fallback = 1;
3533
3534	net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
3535	net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
3536	net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
3537	net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
3538	net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
3539
3540	net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
3541	net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
3542	net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
3543
3544	net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
3545	net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
3546	net->ipv4.sysctl_tcp_syncookies = 1;
3547	net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
3548	net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
3549	net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
3550	net->ipv4.sysctl_tcp_orphan_retries = 0;
3551	net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
3552	net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
3553	net->ipv4.sysctl_tcp_tw_reuse = 2;
3554	net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC;
3555	net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
3556
3557	refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
3558	tcp_set_hashinfo(net);
3559
3560	net->ipv4.sysctl_tcp_sack = 1;
3561	net->ipv4.sysctl_tcp_window_scaling = 1;
3562	net->ipv4.sysctl_tcp_timestamps = 1;
3563	net->ipv4.sysctl_tcp_early_retrans = 3;
3564	net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
3565	net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior.  */
3566	net->ipv4.sysctl_tcp_retrans_collapse = 1;
3567	net->ipv4.sysctl_tcp_max_reordering = 300;
3568	net->ipv4.sysctl_tcp_dsack = 1;
3569	net->ipv4.sysctl_tcp_app_win = 31;
3570	net->ipv4.sysctl_tcp_adv_win_scale = 1;
3571	net->ipv4.sysctl_tcp_frto = 2;
3572	net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
3573	net->ipv4.sysctl_tcp_rcvbuf_low_rtt = USEC_PER_MSEC;
3574	/* This limits the percentage of the congestion window which we
3575	 * will allow a single TSO frame to consume.  Building TSO frames
3576	 * which are too large can cause TCP streams to be bursty.
3577	 */
3578	net->ipv4.sysctl_tcp_tso_win_divisor = 3;
3579	/* Default TSQ limit of 4 MB */
3580	net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;
3581
3582	/* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
3583	net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
3584
3585	net->ipv4.sysctl_tcp_min_tso_segs = 2;
3586	net->ipv4.sysctl_tcp_tso_rtt_log = 9;  /* 2^9 = 512 usec */
3587	net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
3588	net->ipv4.sysctl_tcp_autocorking = 1;
3589	net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
3590	net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
3591	net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
3592	if (net != &init_net) {
3593		memcpy(net->ipv4.sysctl_tcp_rmem,
3594		       init_net.ipv4.sysctl_tcp_rmem,
3595		       sizeof(init_net.ipv4.sysctl_tcp_rmem));
3596		memcpy(net->ipv4.sysctl_tcp_wmem,
3597		       init_net.ipv4.sysctl_tcp_wmem,
3598		       sizeof(init_net.ipv4.sysctl_tcp_wmem));
3599	}
3600	net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
3601	net->ipv4.sysctl_tcp_comp_sack_slack_ns = 10 * NSEC_PER_USEC;
3602	net->ipv4.sysctl_tcp_comp_sack_nr = 44;
3603	net->ipv4.sysctl_tcp_comp_sack_rtt_percent = 33;
3604	net->ipv4.sysctl_tcp_backlog_ack_defer = 1;
3605	net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
3606	net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
3607	atomic_set(&net->ipv4.tfo_active_disable_times, 0);
3608
3609	/* Set default values for PLB */
3610	net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */
3611	net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
3612	net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
3613	net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
3614	/* Default congestion threshold for PLB to mark a round is 50% */
3615	net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
3616
3617	/* Reno is always built in */
3618	if (!net_eq(net, &init_net) &&
3619	    bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
3620			       init_net.ipv4.tcp_congestion_control->owner))
3621		net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
3622	else
3623		net->ipv4.tcp_congestion_control = &tcp_reno;
3624
3625	net->ipv4.sysctl_tcp_syn_linear_timeouts = 4;
3626	net->ipv4.sysctl_tcp_shrink_window = 0;
3627
3628	net->ipv4.sysctl_tcp_pingpong_thresh = 1;
3629	net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN);
3630	net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC;
3631
3632	return 0;
3633}
3634
3635static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
3636{
3637	struct net *net;
3638
3639	/* make sure concurrent calls to tcp_sk_exit_batch from net_cleanup_work
3640	 * and failed setup_net error unwinding path are serialized.
3641	 *
3642	 * tcp_twsk_purge() handles twsk in any dead netns, not just those in
3643	 * net_exit_list, the thread that dismantles a particular twsk must
3644	 * do so without other thread progressing to refcount_dec_and_test() of
3645	 * tcp_death_row.tw_refcount.
3646	 */
3647	mutex_lock(&tcp_exit_batch_mutex);
3648
3649	tcp_twsk_purge(net_exit_list);
3650
3651	list_for_each_entry(net, net_exit_list, exit_list) {
3652		inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
3653		WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
3654		tcp_fastopen_ctx_destroy(net);
3655	}
3656
3657	mutex_unlock(&tcp_exit_batch_mutex);
3658}
3659
3660static struct pernet_operations __net_initdata tcp_sk_ops = {
3661       .init	   = tcp_sk_init,
3662       .exit	   = tcp_sk_exit,
3663       .exit_batch = tcp_sk_exit_batch,
3664};
3665
3666#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3667DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
3668		     struct sock_common *sk_common, uid_t uid)
3669
3670#define INIT_BATCH_SZ 16
3671
3672static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
3673{
3674	struct bpf_tcp_iter_state *iter = priv_data;
3675	int err;
3676
3677	err = bpf_iter_init_seq_net(priv_data, aux);
3678	if (err)
3679		return err;
3680
3681	err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
3682	if (err) {
3683		bpf_iter_fini_seq_net(priv_data);
3684		return err;
3685	}
3686
3687	return 0;
3688}
3689
3690static void bpf_iter_fini_tcp(void *priv_data)
3691{
3692	struct bpf_tcp_iter_state *iter = priv_data;
3693
3694	bpf_iter_fini_seq_net(priv_data);
3695	kvfree(iter->batch);
3696}
3697
3698static const struct bpf_iter_seq_info tcp_seq_info = {
3699	.seq_ops		= &bpf_iter_tcp_seq_ops,
3700	.init_seq_private	= bpf_iter_init_tcp,
3701	.fini_seq_private	= bpf_iter_fini_tcp,
3702	.seq_priv_size		= sizeof(struct bpf_tcp_iter_state),
3703};
3704
3705static const struct bpf_func_proto *
3706bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
3707			    const struct bpf_prog *prog)
3708{
3709	switch (func_id) {
3710	case BPF_FUNC_setsockopt:
3711		return &bpf_sk_setsockopt_proto;
3712	case BPF_FUNC_getsockopt:
3713		return &bpf_sk_getsockopt_proto;
3714	default:
3715		return NULL;
3716	}
3717}
3718
3719static struct bpf_iter_reg tcp_reg_info = {
3720	.target			= "tcp",
3721	.ctx_arg_info_size	= 1,
3722	.ctx_arg_info		= {
3723		{ offsetof(struct bpf_iter__tcp, sk_common),
3724		  PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
3725	},
3726	.get_func_proto		= bpf_iter_tcp_get_func_proto,
3727	.seq_info		= &tcp_seq_info,
3728};
3729
3730static void __init bpf_iter_register(void)
3731{
3732	tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3733	if (bpf_iter_reg_target(&tcp_reg_info))
3734		pr_warn("Warning: could not register bpf iterator tcp\n");
3735}
3736
3737#endif
3738
3739void __init tcp_v4_init(void)
3740{
3741	int cpu, res;
3742
3743	for_each_possible_cpu(cpu) {
3744		struct sock *sk;
3745
3746		res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
3747					   IPPROTO_TCP, &init_net);
3748		if (res)
3749			panic("Failed to create the TCP control socket.\n");
3750		sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
3751
3752		/* Please enforce IP_DF and IPID==0 for RST and
3753		 * ACK sent in SYN-RECV and TIME-WAIT state.
3754		 */
3755		inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
3756
3757		sk->sk_clockid = CLOCK_MONOTONIC;
3758
3759		per_cpu(ipv4_tcp_sk.sock, cpu) = sk;
3760	}
3761	if (register_pernet_subsys(&tcp_sk_ops))
3762		panic("Failed to create the TCP control socket.\n");
3763
3764#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3765	bpf_iter_register();
3766#endif
3767}