tcp_ipv6.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 *	TCP over IPv6
   4 *	Linux INET6 implementation
   5 *
   6 *	Authors:
   7 *	Pedro Roque		<roque@di.fc.ul.pt>
   8 *
   9 *	Based on:
  10 *	linux/net/ipv4/tcp.c
  11 *	linux/net/ipv4/tcp_input.c
  12 *	linux/net/ipv4/tcp_output.c
  13 *
  14 *	Fixes:
  15 *	Hideaki YOSHIFUJI	:	sin6_scope_id support
  16 *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
  17 *	Alexey Kuznetsov		allow both IPv4 and IPv6 sockets to bind
  18 *					a single port at the same time.
  19 *	YOSHIFUJI Hideaki @USAGI:	convert /proc/net/tcp6 to seq_file.
  20 */
  21
  22#include <linux/bottom_half.h>
  23#include <linux/module.h>
  24#include <linux/errno.h>
  25#include <linux/types.h>
  26#include <linux/socket.h>
  27#include <linux/sockios.h>
  28#include <linux/net.h>
  29#include <linux/jiffies.h>
  30#include <linux/in.h>
  31#include <linux/in6.h>
  32#include <linux/netdevice.h>
  33#include <linux/init.h>
  34#include <linux/jhash.h>
  35#include <linux/ipsec.h>
  36#include <linux/times.h>
  37#include <linux/slab.h>
  38#include <linux/uaccess.h>
  39#include <linux/ipv6.h>
  40#include <linux/icmpv6.h>
  41#include <linux/random.h>
  42#include <linux/indirect_call_wrapper.h>
  43
  44#include <net/aligned_data.h>
  45#include <net/tcp.h>
  46#include <net/ndisc.h>
  47#include <net/inet6_hashtables.h>
  48#include <net/inet6_connection_sock.h>
  49#include <net/ipv6.h>
  50#include <net/transp_v6.h>
  51#include <net/addrconf.h>
  52#include <net/ip6_route.h>
  53#include <net/ip6_checksum.h>
  54#include <net/inet_ecn.h>
  55#include <net/protocol.h>
  56#include <net/xfrm.h>
  57#include <net/snmp.h>
  58#include <net/dsfield.h>
  59#include <net/timewait_sock.h>
  60#include <net/inet_common.h>
  61#include <net/secure_seq.h>
  62#include <net/hotdata.h>
  63#include <net/busy_poll.h>
  64#include <net/rstreason.h>
  65#include <net/psp.h>
  66
  67#include <linux/proc_fs.h>
  68#include <linux/seq_file.h>
  69
  70#include <crypto/md5.h>
  71#include <crypto/utils.h>
  72
  73#include <trace/events/tcp.h>
  74
  75static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
  76			      enum sk_rst_reason reason);
  77static void	tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
  78				      struct request_sock *req);
  79
  80INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb);
  81
  82static const struct inet_connection_sock_af_ops ipv6_mapped;
  83const struct inet_connection_sock_af_ops ipv6_specific;
  84#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
  85static const struct tcp_sock_af_ops tcp_sock_ipv6_specific;
  86static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific;
  87#endif
  88
  89/* Helper returning the inet6 address from a given tcp socket.
  90 * It can be used in TCP stack instead of inet6_sk(sk).
  91 * This avoids a dereference and allow compiler optimizations.
  92 * It is a specialized version of inet6_sk_generic().
  93 */
  94#define tcp_inet6_sk(sk) (&container_of_const(tcp_sk(sk), \
  95					      struct tcp6_sock, tcp)->inet6)
  96
  97static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
  98{
  99	struct dst_entry *dst = skb_dst(skb);
 100
 101	if (dst && dst_hold_safe(dst)) {
 102		rcu_assign_pointer(sk->sk_rx_dst, dst);
 103		sk->sk_rx_dst_ifindex = skb->skb_iif;
 104		sk->sk_rx_dst_cookie = rt6_get_cookie(dst_rt6_info(dst));
 105	}
 106}
 107
 108static union tcp_seq_and_ts_off
 109tcp_v6_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
 110{
 111	return secure_tcpv6_seq_and_ts_off(net,
 112					   ipv6_hdr(skb)->daddr.s6_addr32,
 113					   ipv6_hdr(skb)->saddr.s6_addr32,
 114					   tcp_hdr(skb)->dest,
 115					   tcp_hdr(skb)->source);
 116}
 117
 118static int tcp_v6_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
 119			      int addr_len)
 120{
 121	/* This check is replicated from tcp_v6_connect() and intended to
 122	 * prevent BPF program called below from accessing bytes that are out
 123	 * of the bound specified by user in addr_len.
 124	 */
 125	if (addr_len < SIN6_LEN_RFC2133)
 126		return -EINVAL;
 127
 128	sock_owned_by_me(sk);
 129
 130	return BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr, &addr_len);
 131}
 132
 133static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
 134			  int addr_len)
 135{
 136	struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
 137	struct inet_connection_sock *icsk = inet_csk(sk);
 138	struct in6_addr *saddr = NULL, *final_p, final;
 139	struct inet_timewait_death_row *tcp_death_row;
 140	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 141	struct inet_sock *inet = inet_sk(sk);
 142	struct tcp_sock *tp = tcp_sk(sk);
 143	struct net *net = sock_net(sk);
 144	struct ipv6_txoptions *opt;
 145	struct dst_entry *dst;
 146	struct flowi6 fl6;
 147	int addr_type;
 148	int err;
 149
 150	if (addr_len < SIN6_LEN_RFC2133)
 151		return -EINVAL;
 152
 153	if (usin->sin6_family != AF_INET6)
 154		return -EAFNOSUPPORT;
 155
 156	memset(&fl6, 0, sizeof(fl6));
 157
 158	if (inet6_test_bit(SNDFLOW, sk)) {
 159		fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
 160		IP6_ECN_flow_init(fl6.flowlabel);
 161		if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
 162			struct ip6_flowlabel *flowlabel;
 163			flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
 164			if (IS_ERR(flowlabel))
 165				return -EINVAL;
 166			fl6_sock_release(flowlabel);
 167		}
 168	}
 169
 170	/*
 171	 *	connect() to INADDR_ANY means loopback (BSD'ism).
 172	 */
 173
 174	if (ipv6_addr_any(&usin->sin6_addr)) {
 175		if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
 176			ipv6_addr_set_v4mapped(htonl(INADDR_LOOPBACK),
 177					       &usin->sin6_addr);
 178		else
 179			usin->sin6_addr = in6addr_loopback;
 180	}
 181
 182	addr_type = ipv6_addr_type(&usin->sin6_addr);
 183
 184	if (addr_type & IPV6_ADDR_MULTICAST)
 185		return -ENETUNREACH;
 186
 187	if (addr_type&IPV6_ADDR_LINKLOCAL) {
 188		if (addr_len >= sizeof(struct sockaddr_in6) &&
 189		    usin->sin6_scope_id) {
 190			/* If interface is set while binding, indices
 191			 * must coincide.
 192			 */
 193			if (!sk_dev_equal_l3scope(sk, usin->sin6_scope_id))
 194				return -EINVAL;
 195
 196			sk->sk_bound_dev_if = usin->sin6_scope_id;
 197		}
 198
 199		/* Connect to link-local address requires an interface */
 200		if (!sk->sk_bound_dev_if)
 201			return -EINVAL;
 202	}
 203
 204	if (tp->rx_opt.ts_recent_stamp &&
 205	    !ipv6_addr_equal(&sk->sk_v6_daddr, &usin->sin6_addr)) {
 206		tp->rx_opt.ts_recent = 0;
 207		tp->rx_opt.ts_recent_stamp = 0;
 208		WRITE_ONCE(tp->write_seq, 0);
 209	}
 210
 211	sk->sk_v6_daddr = usin->sin6_addr;
 212	np->flow_label = fl6.flowlabel;
 213
 214	/*
 215	 *	TCP over IPv4
 216	 */
 217
 218	if (addr_type & IPV6_ADDR_MAPPED) {
 219		u32 exthdrlen = icsk->icsk_ext_hdr_len;
 220		struct sockaddr_in sin;
 221
 222		if (ipv6_only_sock(sk))
 223			return -ENETUNREACH;
 224
 225		sin.sin_family = AF_INET;
 226		sin.sin_port = usin->sin6_port;
 227		sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3];
 228
 229		/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 230		WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped);
 231		if (sk_is_mptcp(sk))
 232			mptcpv6_handle_mapped(sk, true);
 233		sk->sk_backlog_rcv = tcp_v4_do_rcv;
 234#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 235		tp->af_specific = &tcp_sock_ipv6_mapped_specific;
 236#endif
 237
 238		err = tcp_v4_connect(sk, (struct sockaddr_unsized *)&sin, sizeof(sin));
 239
 240		if (err) {
 241			icsk->icsk_ext_hdr_len = exthdrlen;
 242			/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
 243			WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific);
 244			if (sk_is_mptcp(sk))
 245				mptcpv6_handle_mapped(sk, false);
 246			sk->sk_backlog_rcv = tcp_v6_do_rcv;
 247#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 248			tp->af_specific = &tcp_sock_ipv6_specific;
 249#endif
 250			goto failure;
 251		}
 252		np->saddr = sk->sk_v6_rcv_saddr;
 253
 254		return err;
 255	}
 256
 257	if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
 258		saddr = &sk->sk_v6_rcv_saddr;
 259
 260	fl6.flowi6_proto = IPPROTO_TCP;
 261	fl6.daddr = sk->sk_v6_daddr;
 262	fl6.saddr = saddr ? *saddr : np->saddr;
 263	fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
 264	fl6.flowi6_oif = sk->sk_bound_dev_if;
 265	fl6.flowi6_mark = sk->sk_mark;
 266	fl6.fl6_dport = usin->sin6_port;
 267	fl6.fl6_sport = inet->inet_sport;
 268	if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport)
 269		fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT;
 270	fl6.flowi6_uid = sk_uid(sk);
 271
 272	opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
 273	final_p = fl6_update_dst(&fl6, opt, &final);
 274
 275	security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 276
 277	dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 278	if (IS_ERR(dst)) {
 279		err = PTR_ERR(dst);
 280		goto failure;
 281	}
 282
 283	tp->tcp_usec_ts = dst_tcp_usec_ts(dst);
 284	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
 285
 286	if (!saddr) {
 287		saddr = &fl6.saddr;
 288
 289		err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
 290		if (err)
 291			goto failure;
 292	}
 293
 294	/* set the source address */
 295	np->saddr = *saddr;
 296	inet->inet_rcv_saddr = LOOPBACK4_IPV6;
 297
 298	sk->sk_gso_type = SKB_GSO_TCPV6;
 299	ip6_dst_store(sk, dst, false, false);
 300
 301	icsk->icsk_ext_hdr_len = psp_sk_overhead(sk);
 302	if (opt)
 303		icsk->icsk_ext_hdr_len += opt->opt_flen +
 304					  opt->opt_nflen;
 305
 306	tp->rx_opt.mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) - sizeof(struct ipv6hdr);
 307
 308	inet->inet_dport = usin->sin6_port;
 309
 310	tcp_set_state(sk, TCP_SYN_SENT);
 311	err = inet6_hash_connect(tcp_death_row, sk);
 312	if (err)
 313		goto late_failure;
 314
 315	sk_set_txhash(sk);
 316
 317	if (likely(!tp->repair)) {
 318		union tcp_seq_and_ts_off st;
 319
 320		st = secure_tcpv6_seq_and_ts_off(net,
 321						 np->saddr.s6_addr32,
 322						 sk->sk_v6_daddr.s6_addr32,
 323						 inet->inet_sport,
 324						 inet->inet_dport);
 325		if (!tp->write_seq)
 326			WRITE_ONCE(tp->write_seq, st.seq);
 327		tp->tsoffset = st.ts_off;
 328	}
 329
 330	if (tcp_fastopen_defer_connect(sk, &err))
 331		return err;
 332	if (err)
 333		goto late_failure;
 334
 335	err = tcp_connect(sk);
 336	if (err)
 337		goto late_failure;
 338
 339	return 0;
 340
 341late_failure:
 342	tcp_set_state(sk, TCP_CLOSE);
 343	inet_bhash2_reset_saddr(sk);
 344failure:
 345	inet->inet_dport = 0;
 346	sk->sk_route_caps = 0;
 347	return err;
 348}
 349
 350static void tcp_v6_mtu_reduced(struct sock *sk)
 351{
 352	struct dst_entry *dst;
 353	u32 mtu;
 354
 355	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
 356		return;
 357
 358	mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
 359
 360	/* Drop requests trying to increase our current mss.
 361	 * Check done in __ip6_rt_update_pmtu() is too late.
 362	 */
 363	if (tcp_mtu_to_mss(sk, mtu) >= tcp_sk(sk)->mss_cache)
 364		return;
 365
 366	dst = inet6_csk_update_pmtu(sk, mtu);
 367	if (!dst)
 368		return;
 369
 370	if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) {
 371		tcp_sync_mss(sk, dst_mtu(dst));
 372		tcp_simple_retransmit(sk);
 373	}
 374}
 375
 376static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 377		u8 type, u8 code, int offset, __be32 info)
 378{
 379	const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
 380	const struct tcphdr *th = (struct tcphdr *)(skb->data+offset);
 381	struct net *net = dev_net_rcu(skb->dev);
 382	struct request_sock *fastopen;
 383	struct ipv6_pinfo *np;
 384	struct tcp_sock *tp;
 385	__u32 seq, snd_una;
 386	struct sock *sk;
 387	bool fatal;
 388	int err;
 389
 390	sk = __inet6_lookup_established(net, &hdr->daddr, th->dest,
 391					&hdr->saddr, ntohs(th->source),
 392					skb->dev->ifindex, inet6_sdif(skb));
 393
 394	if (!sk) {
 395		__ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
 396				  ICMP6_MIB_INERRORS);
 397		return -ENOENT;
 398	}
 399
 400	if (sk->sk_state == TCP_TIME_WAIT) {
 401		/* To increase the counter of ignored icmps for TCP-AO */
 402		tcp_ao_ignore_icmp(sk, AF_INET6, type, code);
 403		inet_twsk_put(inet_twsk(sk));
 404		return 0;
 405	}
 406	seq = ntohl(th->seq);
 407	fatal = icmpv6_err_convert(type, code, &err);
 408	if (sk->sk_state == TCP_NEW_SYN_RECV) {
 409		tcp_req_err(sk, seq, fatal);
 410		return 0;
 411	}
 412
 413	if (tcp_ao_ignore_icmp(sk, AF_INET6, type, code)) {
 414		sock_put(sk);
 415		return 0;
 416	}
 417
 418	bh_lock_sock(sk);
 419	if (sock_owned_by_user(sk) && type != ICMPV6_PKT_TOOBIG)
 420		__NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
 421
 422	if (sk->sk_state == TCP_CLOSE)
 423		goto out;
 424
 425	if (static_branch_unlikely(&ip6_min_hopcount)) {
 426		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
 427		if (ipv6_hdr(skb)->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount)) {
 428			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
 429			goto out;
 430		}
 431	}
 432
 433	tp = tcp_sk(sk);
 434	/* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
 435	fastopen = rcu_dereference(tp->fastopen_rsk);
 436	snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
 437	if (sk->sk_state != TCP_LISTEN &&
 438	    !between(seq, snd_una, tp->snd_nxt)) {
 439		__NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
 440		goto out;
 441	}
 442
 443	np = tcp_inet6_sk(sk);
 444
 445	if (type == NDISC_REDIRECT) {
 446		if (!sock_owned_by_user(sk)) {
 447			struct dst_entry *dst = __sk_dst_check(sk, np->dst_cookie);
 448
 449			if (dst)
 450				dst->ops->redirect(dst, sk, skb);
 451		}
 452		goto out;
 453	}
 454
 455	if (type == ICMPV6_PKT_TOOBIG) {
 456		u32 mtu = ntohl(info);
 457
 458		/* We are not interested in TCP_LISTEN and open_requests
 459		 * (SYN-ACKs send out by Linux are always <576bytes so
 460		 * they should go through unfragmented).
 461		 */
 462		if (sk->sk_state == TCP_LISTEN)
 463			goto out;
 464
 465		if (!ip6_sk_accept_pmtu(sk))
 466			goto out;
 467
 468		if (mtu < IPV6_MIN_MTU)
 469			goto out;
 470
 471		WRITE_ONCE(tp->mtu_info, mtu);
 472
 473		if (!sock_owned_by_user(sk))
 474			tcp_v6_mtu_reduced(sk);
 475		else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
 476					   &sk->sk_tsq_flags))
 477			sock_hold(sk);
 478		goto out;
 479	}
 480
 481
 482	/* Might be for an request_sock */
 483	switch (sk->sk_state) {
 484	case TCP_SYN_SENT:
 485	case TCP_SYN_RECV:
 486		/* Only in fast or simultaneous open. If a fast open socket is
 487		 * already accepted it is treated as a connected one below.
 488		 */
 489		if (fastopen && !fastopen->sk)
 490			break;
 491
 492		ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 493
 494		if (!sock_owned_by_user(sk))
 495			tcp_done_with_error(sk, err);
 496		else
 497			WRITE_ONCE(sk->sk_err_soft, err);
 498		goto out;
 499	case TCP_LISTEN:
 500		break;
 501	default:
 502		/* check if this ICMP message allows revert of backoff.
 503		 * (see RFC 6069)
 504		 */
 505		if (!fastopen && type == ICMPV6_DEST_UNREACH &&
 506		    code == ICMPV6_NOROUTE)
 507			tcp_ld_RTO_revert(sk, seq);
 508	}
 509
 510	if (!sock_owned_by_user(sk) && inet6_test_bit(RECVERR6, sk)) {
 511		WRITE_ONCE(sk->sk_err, err);
 512		sk_error_report(sk);
 513	} else {
 514		WRITE_ONCE(sk->sk_err_soft, err);
 515	}
 516out:
 517	bh_unlock_sock(sk);
 518	sock_put(sk);
 519	return 0;
 520}
 521
 522
 523static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
 524			      struct flowi *fl,
 525			      struct request_sock *req,
 526			      struct tcp_fastopen_cookie *foc,
 527			      enum tcp_synack_type synack_type,
 528			      struct sk_buff *syn_skb)
 529{
 530	struct inet_request_sock *ireq = inet_rsk(req);
 531	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
 532	struct ipv6_txoptions *opt;
 533	struct flowi6 *fl6 = &fl->u.ip6;
 534	struct sk_buff *skb;
 535	int err = -ENOMEM;
 536	u8 tclass;
 537
 538	/* First, grab a route. */
 539	if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
 540					       IPPROTO_TCP)) == NULL)
 541		goto done;
 542
 543	skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
 544
 545	if (skb) {
 546		tcp_rsk(req)->syn_ect_snt = np->tclass & INET_ECN_MASK;
 547		__tcp_v6_send_check(skb, &ireq->ir_v6_loc_addr,
 548				    &ireq->ir_v6_rmt_addr);
 549
 550		fl6->daddr = ireq->ir_v6_rmt_addr;
 551		if (inet6_test_bit(REPFLOW, sk) && ireq->pktopts)
 552			fl6->flowlabel = ip6_flowlabel(ipv6_hdr(ireq->pktopts));
 553
 554		tclass = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
 555				(tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
 556				(np->tclass & INET_ECN_MASK) :
 557				np->tclass;
 558
 559		if (!INET_ECN_is_capable(tclass) &&
 560		    tcp_bpf_ca_needs_ecn((struct sock *)req))
 561			tclass |= INET_ECN_ECT_0;
 562
 563		rcu_read_lock();
 564		opt = ireq->ipv6_opt;
 565		if (!opt)
 566			opt = rcu_dereference(np->opt);
 567		err = ip6_xmit(sk, skb, fl6, skb->mark ? : READ_ONCE(sk->sk_mark),
 568			       opt, tclass, READ_ONCE(sk->sk_priority));
 569		rcu_read_unlock();
 570		err = net_xmit_eval(err);
 571	}
 572
 573done:
 574	return err;
 575}
 576
 577
 578static void tcp_v6_reqsk_destructor(struct request_sock *req)
 579{
 580	kfree(inet_rsk(req)->ipv6_opt);
 581	consume_skb(inet_rsk(req)->pktopts);
 582}
 583
 584#ifdef CONFIG_TCP_MD5SIG
 585static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
 586						   const struct in6_addr *addr,
 587						   int l3index)
 588{
 589	return tcp_md5_do_lookup(sk, l3index,
 590				 (union tcp_md5_addr *)addr, AF_INET6);
 591}
 592
 593static struct tcp_md5sig_key *tcp_v6_md5_lookup(const struct sock *sk,
 594						const struct sock *addr_sk)
 595{
 596	int l3index;
 597
 598	l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
 599						 addr_sk->sk_bound_dev_if);
 600	return tcp_v6_md5_do_lookup(sk, &addr_sk->sk_v6_daddr,
 601				    l3index);
 602}
 603
 604static int tcp_v6_parse_md5_keys(struct sock *sk, int optname,
 605				 sockptr_t optval, int optlen)
 606{
 607	struct tcp_md5sig cmd;
 608	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.tcpm_addr;
 609	union tcp_ao_addr *addr;
 610	int l3index = 0;
 611	u8 prefixlen;
 612	bool l3flag;
 613	u8 flags;
 614
 615	if (optlen < sizeof(cmd))
 616		return -EINVAL;
 617
 618	if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
 619		return -EFAULT;
 620
 621	if (sin6->sin6_family != AF_INET6)
 622		return -EINVAL;
 623
 624	flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 625	l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
 626
 627	if (optname == TCP_MD5SIG_EXT &&
 628	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
 629		prefixlen = cmd.tcpm_prefixlen;
 630		if (prefixlen > 128 || (ipv6_addr_v4mapped(&sin6->sin6_addr) &&
 631					prefixlen > 32))
 632			return -EINVAL;
 633	} else {
 634		prefixlen = ipv6_addr_v4mapped(&sin6->sin6_addr) ? 32 : 128;
 635	}
 636
 637	if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
 638	    cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
 639		struct net_device *dev;
 640
 641		rcu_read_lock();
 642		dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
 643		if (dev && netif_is_l3_master(dev))
 644			l3index = dev->ifindex;
 645		rcu_read_unlock();
 646
 647		/* ok to reference set/not set outside of rcu;
 648		 * right now device MUST be an L3 master
 649		 */
 650		if (!dev || !l3index)
 651			return -EINVAL;
 652	}
 653
 654	if (!cmd.tcpm_keylen) {
 655		if (ipv6_addr_v4mapped(&sin6->sin6_addr))
 656			return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3],
 657					      AF_INET, prefixlen,
 658					      l3index, flags);
 659		return tcp_md5_do_del(sk, (union tcp_md5_addr *)&sin6->sin6_addr,
 660				      AF_INET6, prefixlen, l3index, flags);
 661	}
 662
 663	if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
 664		return -EINVAL;
 665
 666	if (ipv6_addr_v4mapped(&sin6->sin6_addr)) {
 667		addr = (union tcp_md5_addr *)&sin6->sin6_addr.s6_addr32[3];
 668
 669		/* Don't allow keys for peers that have a matching TCP-AO key.
 670		 * See the comment in tcp_ao_add_cmd()
 671		 */
 672		if (tcp_ao_required(sk, addr, AF_INET,
 673				    l3flag ? l3index : -1, false))
 674			return -EKEYREJECTED;
 675		return tcp_md5_do_add(sk, addr,
 676				      AF_INET, prefixlen, l3index, flags,
 677				      cmd.tcpm_key, cmd.tcpm_keylen);
 678	}
 679
 680	addr = (union tcp_md5_addr *)&sin6->sin6_addr;
 681
 682	/* Don't allow keys for peers that have a matching TCP-AO key.
 683	 * See the comment in tcp_ao_add_cmd()
 684	 */
 685	if (tcp_ao_required(sk, addr, AF_INET6, l3flag ? l3index : -1, false))
 686		return -EKEYREJECTED;
 687
 688	return tcp_md5_do_add(sk, addr, AF_INET6, prefixlen, l3index, flags,
 689			      cmd.tcpm_key, cmd.tcpm_keylen);
 690}
 691
 692static void tcp_v6_md5_hash_headers(struct md5_ctx *ctx,
 693				    const struct in6_addr *daddr,
 694				    const struct in6_addr *saddr,
 695				    const struct tcphdr *th, int nbytes)
 696{
 697	struct {
 698		struct tcp6_pseudohdr ip; /* TCP pseudo-header (RFC2460) */
 699		struct tcphdr tcp;
 700	} h;
 701
 702	h.ip.saddr = *saddr;
 703	h.ip.daddr = *daddr;
 704	h.ip.protocol = cpu_to_be32(IPPROTO_TCP);
 705	h.ip.len = cpu_to_be32(nbytes);
 706	h.tcp = *th;
 707	h.tcp.check = 0;
 708	md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp));
 709}
 710
 711static noinline_for_stack void
 712tcp_v6_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
 713		    const struct in6_addr *daddr, struct in6_addr *saddr,
 714		    const struct tcphdr *th)
 715{
 716	struct md5_ctx ctx;
 717
 718	md5_init(&ctx);
 719	tcp_v6_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2);
 720	tcp_md5_hash_key(&ctx, key);
 721	md5_final(&ctx, md5_hash);
 722}
 723
 724static noinline_for_stack void
 725tcp_v6_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
 726		    const struct sock *sk, const struct sk_buff *skb)
 727{
 728	const struct tcphdr *th = tcp_hdr(skb);
 729	const struct in6_addr *saddr, *daddr;
 730	struct md5_ctx ctx;
 731
 732	if (sk) { /* valid for establish/request sockets */
 733		saddr = &sk->sk_v6_rcv_saddr;
 734		daddr = &sk->sk_v6_daddr;
 735	} else {
 736		const struct ipv6hdr *ip6h = ipv6_hdr(skb);
 737		saddr = &ip6h->saddr;
 738		daddr = &ip6h->daddr;
 739	}
 740
 741	md5_init(&ctx);
 742	tcp_v6_md5_hash_headers(&ctx, daddr, saddr, th, skb->len);
 743	tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2);
 744	tcp_md5_hash_key(&ctx, key);
 745	md5_final(&ctx, md5_hash);
 746}
 747#endif
 748
 749static void tcp_v6_init_req(struct request_sock *req,
 750			    const struct sock *sk_listener,
 751			    struct sk_buff *skb,
 752			    u32 tw_isn)
 753{
 754	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
 755	struct inet_request_sock *ireq = inet_rsk(req);
 756	const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
 757
 758	ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 759	ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 760	ireq->ir_rmt_addr = LOOPBACK4_IPV6;
 761	ireq->ir_loc_addr = LOOPBACK4_IPV6;
 762
 763	/* So that link locals have meaning */
 764	if ((!sk_listener->sk_bound_dev_if || l3_slave) &&
 765	    ipv6_addr_type(&ireq->ir_v6_rmt_addr) & IPV6_ADDR_LINKLOCAL)
 766		ireq->ir_iif = tcp_v6_iif(skb);
 767
 768	if (!tw_isn &&
 769	    (ipv6_opt_accepted(sk_listener, skb, &TCP_SKB_CB(skb)->header.h6) ||
 770	     np->rxopt.bits.rxinfo ||
 771	     np->rxopt.bits.rxoinfo || np->rxopt.bits.rxhlim ||
 772	     np->rxopt.bits.rxohlim || inet6_test_bit(REPFLOW, sk_listener))) {
 773		refcount_inc(&skb->users);
 774		ireq->pktopts = skb;
 775	}
 776}
 777
 778static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
 779					  struct sk_buff *skb,
 780					  struct flowi *fl,
 781					  struct request_sock *req,
 782					  u32 tw_isn)
 783{
 784	tcp_v6_init_req(req, sk, skb, tw_isn);
 785
 786	if (security_inet_conn_request(sk, skb, req))
 787		return NULL;
 788
 789	return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
 790}
 791
 792struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
 793	.family		=	AF_INET6,
 794	.obj_size	=	sizeof(struct tcp6_request_sock),
 795	.send_ack	=	tcp_v6_reqsk_send_ack,
 796	.destructor	=	tcp_v6_reqsk_destructor,
 797	.send_reset	=	tcp_v6_send_reset,
 798};
 799
 800const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 801	.mss_clamp	=	IPV6_MIN_MTU - sizeof(struct tcphdr) -
 802				sizeof(struct ipv6hdr),
 803#ifdef CONFIG_TCP_MD5SIG
 804	.req_md5_lookup	=	tcp_v6_md5_lookup,
 805	.calc_md5_hash	=	tcp_v6_md5_hash_skb,
 806#endif
 807#ifdef CONFIG_TCP_AO
 808	.ao_lookup	=	tcp_v6_ao_lookup_rsk,
 809	.ao_calc_key	=	tcp_v6_ao_calc_key_rsk,
 810	.ao_synack_hash =	tcp_v6_ao_synack_hash,
 811#endif
 812#ifdef CONFIG_SYN_COOKIES
 813	.cookie_init_seq =	cookie_v6_init_sequence,
 814#endif
 815	.route_req	=	tcp_v6_route_req,
 816	.init_seq_and_ts_off	= tcp_v6_init_seq_and_ts_off,
 817	.send_synack	=	tcp_v6_send_synack,
 818};
 819
 820static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 821				 u32 ack, u32 win, u32 tsval, u32 tsecr,
 822				 int oif, int rst, u8 tclass, __be32 label,
 823				 u32 priority, u32 txhash, struct tcp_key *key)
 824{
 825	struct net *net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
 826	unsigned int tot_len = sizeof(struct tcphdr);
 827	struct sock *ctl_sk = net->ipv6.tcp_sk;
 828	const struct tcphdr *th = tcp_hdr(skb);
 829	__be32 mrst = 0, *topt;
 830	struct dst_entry *dst;
 831	struct sk_buff *buff;
 832	struct tcphdr *t1;
 833	struct flowi6 fl6;
 834	u32 mark = 0;
 835
 836	if (tsecr)
 837		tot_len += TCPOLEN_TSTAMP_ALIGNED;
 838	if (tcp_key_is_md5(key))
 839		tot_len += TCPOLEN_MD5SIG_ALIGNED;
 840	if (tcp_key_is_ao(key))
 841		tot_len += tcp_ao_len_aligned(key->ao_key);
 842
 843#ifdef CONFIG_MPTCP
 844	if (rst && !tcp_key_is_md5(key)) {
 845		mrst = mptcp_reset_option(skb);
 846
 847		if (mrst)
 848			tot_len += sizeof(__be32);
 849	}
 850#endif
 851
 852	buff = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
 853	if (!buff)
 854		return;
 855
 856	skb_reserve(buff, MAX_TCP_HEADER);
 857
 858	t1 = skb_push(buff, tot_len);
 859	skb_reset_transport_header(buff);
 860
 861	/* Swap the send and the receive. */
 862	memset(t1, 0, sizeof(*t1));
 863	t1->dest = th->source;
 864	t1->source = th->dest;
 865	t1->doff = tot_len / 4;
 866	t1->seq = htonl(seq);
 867	t1->ack_seq = htonl(ack);
 868	t1->ack = !rst || !th->ack;
 869	t1->rst = rst;
 870	t1->window = htons(win);
 871
 872	topt = (__be32 *)(t1 + 1);
 873
 874	if (tsecr) {
 875		*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 876				(TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP);
 877		*topt++ = htonl(tsval);
 878		*topt++ = htonl(tsecr);
 879	}
 880
 881	if (mrst)
 882		*topt++ = mrst;
 883
 884#ifdef CONFIG_TCP_MD5SIG
 885	if (tcp_key_is_md5(key)) {
 886		*topt++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
 887				(TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
 888		tcp_v6_md5_hash_hdr((__u8 *)topt, key->md5_key,
 889				    &ipv6_hdr(skb)->saddr,
 890				    &ipv6_hdr(skb)->daddr, t1);
 891	}
 892#endif
 893#ifdef CONFIG_TCP_AO
 894	if (tcp_key_is_ao(key)) {
 895		*topt++ = htonl((TCPOPT_AO << 24) |
 896				(tcp_ao_len(key->ao_key) << 16) |
 897				(key->ao_key->sndid << 8) |
 898				(key->rcv_next));
 899
 900		tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key,
 901				key->traffic_key,
 902				(union tcp_ao_addr *)&ipv6_hdr(skb)->saddr,
 903				(union tcp_ao_addr *)&ipv6_hdr(skb)->daddr,
 904				t1, key->sne);
 905	}
 906#endif
 907
 908	memset(&fl6, 0, sizeof(fl6));
 909	fl6.daddr = ipv6_hdr(skb)->saddr;
 910	fl6.saddr = ipv6_hdr(skb)->daddr;
 911	fl6.flowlabel = label;
 912
 913	buff->ip_summed = CHECKSUM_PARTIAL;
 914
 915	__tcp_v6_send_check(buff, &fl6.saddr, &fl6.daddr);
 916
 917	fl6.flowi6_proto = IPPROTO_TCP;
 918	if (rt6_need_strict(&fl6.daddr) && !oif)
 919		fl6.flowi6_oif = tcp_v6_iif(skb);
 920	else {
 921		if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
 922			oif = skb->skb_iif;
 923
 924		fl6.flowi6_oif = oif;
 925	}
 926
 927	if (sk) {
 928		/* unconstify the socket only to attach it to buff with care. */
 929		skb_set_owner_edemux(buff, (struct sock *)sk);
 930		psp_reply_set_decrypted(sk, buff);
 931
 932		if (sk->sk_state == TCP_TIME_WAIT)
 933			mark = inet_twsk(sk)->tw_mark;
 934		else
 935			mark = READ_ONCE(sk->sk_mark);
 936		skb_set_delivery_time(buff, tcp_transmit_time(sk), SKB_CLOCK_MONOTONIC);
 937	}
 938	if (txhash) {
 939		/* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
 940		skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
 941	}
 942	fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 943	fl6.fl6_dport = t1->dest;
 944	fl6.fl6_sport = t1->source;
 945	fl6.flowi6_uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
 946	security_skb_classify_flow(skb, flowi6_to_flowi_common(&fl6));
 947
 948	/* Pass a socket to ip6_dst_lookup either it is for RST
 949	 * Underlying function will use this to retrieve the network
 950	 * namespace
 951	 */
 952	if (sk && sk->sk_state != TCP_TIME_WAIT)
 953		dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); /*sk's xfrm_policy can be referred*/
 954	else
 955		dst = ip6_dst_lookup_flow(net, ctl_sk, &fl6, NULL);
 956	if (!IS_ERR(dst)) {
 957		skb_dst_set(buff, dst);
 958		ip6_xmit(ctl_sk, buff, &fl6, fl6.flowi6_mark, NULL,
 959			 tclass, priority);
 960		TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
 961		if (rst)
 962			TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
 963		return;
 964	}
 965
 966	kfree_skb(buff);
 967}
 968
 969static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
 970			      enum sk_rst_reason reason)
 971{
 972	const struct tcphdr *th = tcp_hdr(skb);
 973	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
 974	const __u8 *md5_hash_location = NULL;
 975#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
 976	bool allocated_traffic_key = false;
 977#endif
 978	const struct tcp_ao_hdr *aoh;
 979	struct tcp_key key = {};
 980	u32 seq = 0, ack_seq = 0;
 981	__be32 label = 0;
 982	u32 priority = 0;
 983	struct net *net;
 984	u32 txhash = 0;
 985	int oif = 0;
 986#ifdef CONFIG_TCP_MD5SIG
 987	unsigned char newhash[16];
 988	struct sock *sk1 = NULL;
 989#endif
 990
 991	if (th->rst)
 992		return;
 993
 994	/* If sk not NULL, it means we did a successful lookup and incoming
 995	 * route had to be correct. prequeue might have dropped our dst.
 996	 */
 997	if (!sk && !ipv6_unicast_destination(skb))
 998		return;
 999
1000	net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);
1001	/* Invalid TCP option size or twice included auth */
1002	if (tcp_parse_auth_options(th, &md5_hash_location, &aoh))
1003		return;
1004#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1005	rcu_read_lock();
1006#endif
1007#ifdef CONFIG_TCP_MD5SIG
1008	if (sk && sk_fullsock(sk)) {
1009		int l3index;
1010
1011		/* sdif set, means packet ingressed via a device
1012		 * in an L3 domain and inet_iif is set to it.
1013		 */
1014		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1015		key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6h->saddr, l3index);
1016		if (key.md5_key)
1017			key.type = TCP_KEY_MD5;
1018	} else if (md5_hash_location) {
1019		int dif = tcp_v6_iif_l3_slave(skb);
1020		int sdif = tcp_v6_sdif(skb);
1021		int l3index;
1022
1023		/*
1024		 * active side is lost. Try to find listening socket through
1025		 * source port, and then find md5 key through listening socket.
1026		 * we are not loose security here:
1027		 * Incoming packet is checked with md5 hash with finding key,
1028		 * no RST generated if md5 hash doesn't match.
1029		 */
1030		sk1 = inet6_lookup_listener(net, NULL, 0, &ipv6h->saddr, th->source,
1031					    &ipv6h->daddr, ntohs(th->source),
1032					    dif, sdif);
1033		if (!sk1)
1034			goto out;
1035
1036		/* sdif set, means packet ingressed via a device
1037		 * in an L3 domain and dif is set to it.
1038		 */
1039		l3index = tcp_v6_sdif(skb) ? dif : 0;
1040
1041		key.md5_key = tcp_v6_md5_do_lookup(sk1, &ipv6h->saddr, l3index);
1042		if (!key.md5_key)
1043			goto out;
1044		key.type = TCP_KEY_MD5;
1045
1046		tcp_v6_md5_hash_skb(newhash, key.md5_key, NULL, skb);
1047		if (crypto_memneq(md5_hash_location, newhash, 16))
1048			goto out;
1049	}
1050#endif
1051
1052	if (th->ack)
1053		seq = ntohl(th->ack_seq);
1054	else
1055		ack_seq = ntohl(th->seq) + th->syn + th->fin + skb->len -
1056			  (th->doff << 2);
1057
1058#ifdef CONFIG_TCP_AO
1059	if (aoh) {
1060		int l3index;
1061
1062		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1063		if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, seq,
1064					 &key.ao_key, &key.traffic_key,
1065					 &allocated_traffic_key,
1066					 &key.rcv_next, &key.sne))
1067			goto out;
1068		key.type = TCP_KEY_AO;
1069	}
1070#endif
1071
1072	if (sk) {
1073		oif = sk->sk_bound_dev_if;
1074		if (sk_fullsock(sk)) {
1075			if (inet6_test_bit(REPFLOW, sk))
1076				label = ip6_flowlabel(ipv6h);
1077			priority = READ_ONCE(sk->sk_priority);
1078			txhash = sk->sk_txhash;
1079		}
1080		if (sk->sk_state == TCP_TIME_WAIT) {
1081			label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
1082			priority = inet_twsk(sk)->tw_priority;
1083			txhash = inet_twsk(sk)->tw_txhash;
1084		}
1085	} else {
1086		if (READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) &
1087		    FLOWLABEL_REFLECT_TCP_RESET)
1088			label = ip6_flowlabel(ipv6h);
1089	}
1090
1091	trace_tcp_send_reset(sk, skb, reason);
1092
1093	tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, 1,
1094			     ipv6_get_dsfield(ipv6h) & ~INET_ECN_MASK,
1095			     label, priority, txhash,
1096			     &key);
1097
1098#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1099out:
1100	if (allocated_traffic_key)
1101		kfree(key.traffic_key);
1102	rcu_read_unlock();
1103#endif
1104}
1105
1106static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
1107			    u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
1108			    struct tcp_key *key, u8 tclass,
1109			    __be32 label, u32 priority, u32 txhash)
1110{
1111	tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, 0,
1112			     tclass, label, priority, txhash, key);
1113}
1114
1115static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb,
1116				enum tcp_tw_status tw_status)
1117{
1118	struct inet_timewait_sock *tw = inet_twsk(sk);
1119	struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
1120	u8 tclass = tw->tw_tclass;
1121	struct tcp_key key = {};
1122
1123	if (tw_status == TCP_TW_ACK_OOW)
1124		tclass &= ~INET_ECN_MASK;
1125#ifdef CONFIG_TCP_AO
1126	struct tcp_ao_info *ao_info;
1127
1128	if (static_branch_unlikely(&tcp_ao_needed.key)) {
1129
1130		/* FIXME: the segment to-be-acked is not verified yet */
1131		ao_info = rcu_dereference(tcptw->ao_info);
1132		if (ao_info) {
1133			const struct tcp_ao_hdr *aoh;
1134
1135			/* Invalid TCP option size or twice included auth */
1136			if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1137				goto out;
1138			if (aoh)
1139				key.ao_key = tcp_ao_established_key(sk, ao_info,
1140								    aoh->rnext_keyid, -1);
1141		}
1142	}
1143	if (key.ao_key) {
1144		struct tcp_ao_key *rnext_key;
1145
1146		key.traffic_key = snd_other_key(key.ao_key);
1147		/* rcv_next switches to our rcv_next */
1148		rnext_key = READ_ONCE(ao_info->rnext_key);
1149		key.rcv_next = rnext_key->rcvid;
1150		key.sne = READ_ONCE(ao_info->snd_sne);
1151		key.type = TCP_KEY_AO;
1152#else
1153	if (0) {
1154#endif
1155#ifdef CONFIG_TCP_MD5SIG
1156	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1157		key.md5_key = tcp_twsk_md5_key(tcptw);
1158		if (key.md5_key)
1159			key.type = TCP_KEY_MD5;
1160#endif
1161	}
1162
1163	tcp_v6_send_ack(sk, skb, tcptw->tw_snd_nxt,
1164			READ_ONCE(tcptw->tw_rcv_nxt),
1165			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
1166			tcp_tw_tsval(tcptw),
1167			READ_ONCE(tcptw->tw_ts_recent), tw->tw_bound_dev_if,
1168			&key, tclass, cpu_to_be32(tw->tw_flowlabel),
1169			tw->tw_priority, tw->tw_txhash);
1170
1171#ifdef CONFIG_TCP_AO
1172out:
1173#endif
1174	inet_twsk_put(tw);
1175}
1176
1177static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
1178				  struct request_sock *req)
1179{
1180	struct tcp_key key = {};
1181
1182#ifdef CONFIG_TCP_AO
1183	if (static_branch_unlikely(&tcp_ao_needed.key) &&
1184	    tcp_rsk_used_ao(req)) {
1185		const struct in6_addr *addr = &ipv6_hdr(skb)->saddr;
1186		const struct tcp_ao_hdr *aoh;
1187		int l3index;
1188
1189		l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1190		/* Invalid TCP option size or twice included auth */
1191		if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
1192			return;
1193		if (!aoh)
1194			return;
1195		key.ao_key = tcp_ao_do_lookup(sk, l3index,
1196					      (union tcp_ao_addr *)addr,
1197					      AF_INET6, aoh->rnext_keyid, -1);
1198		if (unlikely(!key.ao_key)) {
1199			/* Send ACK with any matching MKT for the peer */
1200			key.ao_key = tcp_ao_do_lookup(sk, l3index,
1201						      (union tcp_ao_addr *)addr,
1202						      AF_INET6, -1, -1);
1203			/* Matching key disappeared (user removed the key?)
1204			 * let the handshake timeout.
1205			 */
1206			if (!key.ao_key) {
1207				net_info_ratelimited("TCP-AO key for (%pI6, %d)->(%pI6, %d) suddenly disappeared, won't ACK new connection\n",
1208						     addr,
1209						     ntohs(tcp_hdr(skb)->source),
1210						     &ipv6_hdr(skb)->daddr,
1211						     ntohs(tcp_hdr(skb)->dest));
1212				return;
1213			}
1214		}
1215		key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
1216		if (!key.traffic_key)
1217			return;
1218
1219		key.type = TCP_KEY_AO;
1220		key.rcv_next = aoh->keyid;
1221		tcp_v6_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
1222#else
1223	if (0) {
1224#endif
1225#ifdef CONFIG_TCP_MD5SIG
1226	} else if (static_branch_unlikely(&tcp_md5_needed.key)) {
1227		int l3index = tcp_v6_sdif(skb) ? tcp_v6_iif_l3_slave(skb) : 0;
1228
1229		key.md5_key = tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr,
1230						   l3index);
1231		if (key.md5_key)
1232			key.type = TCP_KEY_MD5;
1233#endif
1234	}
1235
1236	/* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
1237	 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
1238	 */
1239	tcp_v6_send_ack(sk, skb, (sk->sk_state == TCP_LISTEN) ?
1240			tcp_rsk(req)->snt_isn + 1 : tcp_sk(sk)->snd_nxt,
1241			tcp_rsk(req)->rcv_nxt,
1242			tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
1243			tcp_rsk_tsval(tcp_rsk(req)),
1244			req->ts_recent, sk->sk_bound_dev_if,
1245			&key, ipv6_get_dsfield(ipv6_hdr(skb)) & ~INET_ECN_MASK,
1246			0,
1247			READ_ONCE(sk->sk_priority),
1248			READ_ONCE(tcp_rsk(req)->txhash));
1249	if (tcp_key_is_ao(&key))
1250		kfree(key.traffic_key);
1251}
1252
1253
1254static struct sock *tcp_v6_cookie_check(struct sock *sk, struct sk_buff *skb)
1255{
1256#ifdef CONFIG_SYN_COOKIES
1257	const struct tcphdr *th = tcp_hdr(skb);
1258
1259	if (!th->syn)
1260		sk = cookie_v6_check(sk, skb);
1261#endif
1262	return sk;
1263}
1264
1265u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
1266			 struct tcphdr *th, u32 *cookie)
1267{
1268	u16 mss = 0;
1269#ifdef CONFIG_SYN_COOKIES
1270	mss = tcp_get_syncookie_mss(&tcp6_request_sock_ops,
1271				    &tcp_request_sock_ipv6_ops, sk, th);
1272	if (mss) {
1273		*cookie = __cookie_v6_init_sequence(iph, th, &mss);
1274		tcp_synq_overflow(sk);
1275	}
1276#endif
1277	return mss;
1278}
1279
1280static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
1281{
1282	if (skb->protocol == htons(ETH_P_IP))
1283		return tcp_v4_conn_request(sk, skb);
1284
1285	if (!ipv6_unicast_destination(skb))
1286		goto drop;
1287
1288	if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
1289		__IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
1290		return 0;
1291	}
1292
1293	return tcp_conn_request(&tcp6_request_sock_ops,
1294				&tcp_request_sock_ipv6_ops, sk, skb);
1295
1296drop:
1297	tcp_listendrop(sk);
1298	return 0; /* don't send reset */
1299}
1300
1301static void tcp_v6_restore_cb(struct sk_buff *skb)
1302{
1303	/* We need to move header back to the beginning if xfrm6_policy_check()
1304	 * and tcp_v6_fill_cb() are going to be called again.
1305	 * ip6_datagram_recv_specific_ctl() also expects IP6CB to be there.
1306	 */
1307	memmove(IP6CB(skb), &TCP_SKB_CB(skb)->header.h6,
1308		sizeof(struct inet6_skb_parm));
1309}
1310
1311/* Called from tcp_v4_syn_recv_sock() for v6_mapped children. */
1312static void tcp_v6_mapped_child_init(struct sock *newsk, const struct sock *sk)
1313{
1314	struct inet_sock *newinet = inet_sk(newsk);
1315	struct ipv6_pinfo *newnp;
1316
1317	newinet->pinet6 = newnp = tcp_inet6_sk(newsk);
1318	newinet->ipv6_fl_list = NULL;
1319
1320	memcpy(newnp, tcp_inet6_sk(sk), sizeof(struct ipv6_pinfo));
1321
1322	newnp->saddr = newsk->sk_v6_rcv_saddr;
1323
1324	inet_csk(newsk)->icsk_af_ops = &ipv6_mapped;
1325	if (sk_is_mptcp(newsk))
1326		mptcpv6_handle_mapped(newsk, true);
1327	newsk->sk_backlog_rcv = tcp_v4_do_rcv;
1328#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
1329	tcp_sk(newsk)->af_specific = &tcp_sock_ipv6_mapped_specific;
1330#endif
1331
1332	newnp->ipv6_mc_list = NULL;
1333	newnp->ipv6_ac_list = NULL;
1334	newnp->pktoptions  = NULL;
1335	newnp->opt	   = NULL;
1336
1337	/* tcp_v4_syn_recv_sock() has initialized newinet->mc_{index,ttl} */
1338	newnp->mcast_oif   = newinet->mc_index;
1339	newnp->mcast_hops  = newinet->mc_ttl;
1340
1341	newnp->rcv_flowinfo = 0;
1342	if (inet6_test_bit(REPFLOW, sk))
1343		newnp->flow_label = 0;
1344}
1345
1346static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1347					 struct request_sock *req,
1348					 struct dst_entry *dst,
1349					 struct request_sock *req_unhash,
1350					 bool *own_req,
1351					 void (*opt_child_init)(struct sock *newsk,
1352								const struct sock *sk))
1353{
1354	struct inet_request_sock *ireq;
1355	struct ipv6_pinfo *newnp;
1356	const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1357	struct ipv6_txoptions *opt;
1358	struct inet_sock *newinet;
1359	bool found_dup_sk = false;
1360	struct tcp_sock *newtp;
1361	struct sock *newsk;
1362#ifdef CONFIG_TCP_MD5SIG
1363	struct tcp_md5sig_key *key;
1364	int l3index;
1365#endif
1366	struct flowi6 fl6;
1367
1368	if (skb->protocol == htons(ETH_P_IP))
1369		return tcp_v4_syn_recv_sock(sk, skb, req, dst,
1370					    req_unhash, own_req,
1371					    tcp_v6_mapped_child_init);
1372	ireq = inet_rsk(req);
1373
1374	if (sk_acceptq_is_full(sk))
1375		goto exit_overflow;
1376
1377	if (!dst) {
1378		dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1379		if (!dst)
1380			goto exit;
1381	}
1382
1383	newsk = tcp_create_openreq_child(sk, req, skb);
1384	if (!newsk)
1385		goto exit_nonewsk;
1386
1387	/*
1388	 * No need to charge this sock to the relevant IPv6 refcnt debug socks
1389	 * count here, tcp_create_openreq_child now does this for us, see the
1390	 * comment in that function for the gory details. -acme
1391	 */
1392
1393	newsk->sk_gso_type = SKB_GSO_TCPV6;
1394	inet6_sk_rx_dst_set(newsk, skb);
1395
1396	newinet = inet_sk(newsk);
1397	newinet->pinet6 = tcp_inet6_sk(newsk);
1398	newinet->ipv6_fl_list = NULL;
1399	newinet->inet_opt = NULL;
1400
1401	newtp = tcp_sk(newsk);
1402	newnp = tcp_inet6_sk(newsk);
1403
1404	memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1405
1406	ip6_dst_store(newsk, dst, false, false);
1407
1408	newnp->saddr = ireq->ir_v6_loc_addr;
1409
1410	/* Now IPv6 options...
1411
1412	   First: no IPv4 options.
1413	 */
1414	newnp->ipv6_mc_list = NULL;
1415	newnp->ipv6_ac_list = NULL;
1416
1417	/* Clone RX bits */
1418	newnp->rxopt.all = np->rxopt.all;
1419
1420	newnp->pktoptions = NULL;
1421	newnp->opt	  = NULL;
1422	newnp->mcast_oif  = tcp_v6_iif(skb);
1423	newnp->mcast_hops = ipv6_hdr(skb)->hop_limit;
1424	newnp->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(skb));
1425	if (inet6_test_bit(REPFLOW, sk))
1426		newnp->flow_label = ip6_flowlabel(ipv6_hdr(skb));
1427
1428	/* Set ToS of the new socket based upon the value of incoming SYN.
1429	 * ECT bits are set later in tcp_init_transfer().
1430	 */
1431	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1432		newnp->tclass = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1433
1434	/* Clone native IPv6 options from listening socket (if any)
1435
1436	   Yes, keeping reference count would be much more clever,
1437	   but we make one more one thing there: reattach optmem
1438	   to newsk.
1439	 */
1440	opt = ireq->ipv6_opt;
1441	if (!opt)
1442		opt = rcu_dereference(np->opt);
1443	if (opt) {
1444		opt = ipv6_dup_options(newsk, opt);
1445		RCU_INIT_POINTER(newnp->opt, opt);
1446	}
1447	inet_csk(newsk)->icsk_ext_hdr_len = 0;
1448	if (opt)
1449		inet_csk(newsk)->icsk_ext_hdr_len = opt->opt_nflen +
1450						    opt->opt_flen;
1451
1452	tcp_ca_openreq_child(newsk, dst);
1453
1454	tcp_sync_mss(newsk, dst_mtu(dst));
1455	newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1456
1457	tcp_initialize_rcv_mss(newsk);
1458
1459#ifdef CONFIG_TCP_MD5SIG
1460	l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1461
1462	if (!tcp_rsk_used_ao(req)) {
1463		/* Copy over the MD5 key from the original socket */
1464		key = tcp_v6_md5_do_lookup(sk, &newsk->sk_v6_daddr, l3index);
1465		if (key) {
1466			const union tcp_md5_addr *addr;
1467
1468			addr = (union tcp_md5_addr *)&newsk->sk_v6_daddr;
1469			if (tcp_md5_key_copy(newsk, addr, AF_INET6, 128, l3index, key))
1470				goto put_and_exit;
1471		}
1472	}
1473#endif
1474#ifdef CONFIG_TCP_AO
1475	/* Copy over tcp_ao_info if any */
1476	if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET6))
1477		goto put_and_exit; /* OOM */
1478#endif
1479
1480	if (__inet_inherit_port(sk, newsk) < 0)
1481		goto put_and_exit;
1482	*own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1483				       &found_dup_sk);
1484	if (*own_req) {
1485		tcp_move_syn(newtp, req);
1486
1487		/* Clone pktoptions received with SYN, if we own the req */
1488		if (ireq->pktopts) {
1489			newnp->pktoptions = skb_clone_and_charge_r(ireq->pktopts, newsk);
1490			consume_skb(ireq->pktopts);
1491			ireq->pktopts = NULL;
1492			if (newnp->pktoptions)
1493				tcp_v6_restore_cb(newnp->pktoptions);
1494		}
1495	} else {
1496		if (!req_unhash && found_dup_sk) {
1497			/* This code path should only be executed in the
1498			 * syncookie case only
1499			 */
1500			bh_unlock_sock(newsk);
1501			sock_put(newsk);
1502			newsk = NULL;
1503		}
1504	}
1505
1506	return newsk;
1507
1508exit_overflow:
1509	__NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1510exit_nonewsk:
1511	dst_release(dst);
1512exit:
1513	tcp_listendrop(sk);
1514	return NULL;
1515put_and_exit:
1516	inet_csk_prepare_forced_close(newsk);
1517	tcp_done(newsk);
1518	goto exit;
1519}
1520
1521INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1522							   u32));
1523/* The socket must have it's spinlock held when we get
1524 * here, unless it is a TCP_LISTEN socket.
1525 *
1526 * We have a potential double-lock case here, so even when
1527 * doing backlog processing we use the BH locking scheme.
1528 * This is because we cannot sleep with the original spinlock
1529 * held.
1530 */
1531INDIRECT_CALLABLE_SCOPE
1532int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
1533{
1534	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1535	struct sk_buff *opt_skb = NULL;
1536	enum skb_drop_reason reason;
1537	struct tcp_sock *tp;
1538
1539	/* Imagine: socket is IPv6. IPv4 packet arrives,
1540	   goes to IPv4 receive handler and backlogged.
1541	   From backlog it always goes here. Kerboom...
1542	   Fortunately, tcp_rcv_established and rcv_established
1543	   handle them correctly, but it is not case with
1544	   tcp_v6_hnd_req and tcp_v6_send_reset().   --ANK
1545	 */
1546
1547	if (skb->protocol == htons(ETH_P_IP))
1548		return tcp_v4_do_rcv(sk, skb);
1549
1550	reason = psp_sk_rx_policy_check(sk, skb);
1551	if (reason)
1552		goto err_discard;
1553
1554	/*
1555	 *	socket locking is here for SMP purposes as backlog rcv
1556	 *	is currently called with bh processing disabled.
1557	 */
1558
1559	/* Do Stevens' IPV6_PKTOPTIONS.
1560
1561	   Yes, guys, it is the only place in our code, where we
1562	   may make it not affecting IPv4.
1563	   The rest of code is protocol independent,
1564	   and I do not like idea to uglify IPv4.
1565
1566	   Actually, all the idea behind IPV6_PKTOPTIONS
1567	   looks not very well thought. For now we latch
1568	   options, received in the last packet, enqueued
1569	   by tcp. Feel free to propose better solution.
1570					       --ANK (980728)
1571	 */
1572	if (np->rxopt.all && sk->sk_state != TCP_LISTEN)
1573		opt_skb = skb_clone_and_charge_r(skb, sk);
1574
1575	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1576		struct dst_entry *dst;
1577
1578		dst = rcu_dereference_protected(sk->sk_rx_dst,
1579						lockdep_sock_is_held(sk));
1580
1581		sock_rps_save_rxhash(sk, skb);
1582		sk_mark_napi_id(sk, skb);
1583		if (dst) {
1584			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1585			    INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,
1586					    dst, sk->sk_rx_dst_cookie) == NULL) {
1587				RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1588				dst_release(dst);
1589			}
1590		}
1591
1592		tcp_rcv_established(sk, skb);
1593		if (opt_skb)
1594			goto ipv6_pktoptions;
1595		return 0;
1596	}
1597
1598	if (tcp_checksum_complete(skb))
1599		goto csum_err;
1600
1601	if (sk->sk_state == TCP_LISTEN) {
1602		struct sock *nsk = tcp_v6_cookie_check(sk, skb);
1603
1604		if (nsk != sk) {
1605			if (nsk) {
1606				reason = tcp_child_process(sk, nsk, skb);
1607				if (reason)
1608					goto reset;
1609			}
1610			return 0;
1611		}
1612	} else
1613		sock_rps_save_rxhash(sk, skb);
1614
1615	reason = tcp_rcv_state_process(sk, skb);
1616	if (reason)
1617		goto reset;
1618	if (opt_skb)
1619		goto ipv6_pktoptions;
1620	return 0;
1621
1622reset:
1623	tcp_v6_send_reset(sk, skb, sk_rst_convert_drop_reason(reason));
1624discard:
1625	if (opt_skb)
1626		__kfree_skb(opt_skb);
1627	sk_skb_reason_drop(sk, skb, reason);
1628	return 0;
1629csum_err:
1630	reason = SKB_DROP_REASON_TCP_CSUM;
1631	trace_tcp_bad_csum(skb);
1632	TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1633err_discard:
1634	TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1635	goto discard;
1636
1637
1638ipv6_pktoptions:
1639	/* Do you ask, what is it?
1640
1641	   1. skb was enqueued by tcp.
1642	   2. skb is added to tail of read queue, rather than out of order.
1643	   3. socket is not in passive state.
1644	   4. Finally, it really contains options, which user wants to receive.
1645	 */
1646	tp = tcp_sk(sk);
1647	if (TCP_SKB_CB(opt_skb)->end_seq == tp->rcv_nxt &&
1648	    !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
1649		if (np->rxopt.bits.rxinfo || np->rxopt.bits.rxoinfo)
1650			WRITE_ONCE(np->mcast_oif, tcp_v6_iif(opt_skb));
1651		if (np->rxopt.bits.rxhlim || np->rxopt.bits.rxohlim)
1652			WRITE_ONCE(np->mcast_hops,
1653				   ipv6_hdr(opt_skb)->hop_limit);
1654		if (np->rxopt.bits.rxflow || np->rxopt.bits.rxtclass)
1655			np->rcv_flowinfo = ip6_flowinfo(ipv6_hdr(opt_skb));
1656		if (inet6_test_bit(REPFLOW, sk))
1657			np->flow_label = ip6_flowlabel(ipv6_hdr(opt_skb));
1658		if (ipv6_opt_accepted(sk, opt_skb, &TCP_SKB_CB(opt_skb)->header.h6)) {
1659			tcp_v6_restore_cb(opt_skb);
1660			opt_skb = xchg(&np->pktoptions, opt_skb);
1661		} else {
1662			__kfree_skb(opt_skb);
1663			opt_skb = xchg(&np->pktoptions, NULL);
1664		}
1665	}
1666
1667	consume_skb(opt_skb);
1668	return 0;
1669}
1670
1671static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr,
1672			   const struct tcphdr *th)
1673{
1674	/* This is tricky: we move IP6CB at its correct location into
1675	 * TCP_SKB_CB(). It must be done after xfrm6_policy_check(), because
1676	 * _decode_session6() uses IP6CB().
1677	 * barrier() makes sure compiler won't play aliasing games.
1678	 */
1679	memmove(&TCP_SKB_CB(skb)->header.h6, IP6CB(skb),
1680		sizeof(struct inet6_skb_parm));
1681	barrier();
1682
1683	TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1684	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1685				    skb->len - th->doff*4);
1686	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1687	TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
1688	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
1689	TCP_SKB_CB(skb)->sacked = 0;
1690	TCP_SKB_CB(skb)->has_rxtstamp =
1691			skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1692}
1693
1694INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
1695{
1696	struct net *net = dev_net_rcu(skb->dev);
1697	enum skb_drop_reason drop_reason;
1698	enum tcp_tw_status tw_status;
1699	int sdif = inet6_sdif(skb);
1700	int dif = inet6_iif(skb);
1701	const struct tcphdr *th;
1702	const struct ipv6hdr *hdr;
1703	struct sock *sk = NULL;
1704	bool refcounted;
1705	int ret;
1706	u32 isn;
1707
1708	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1709	if (skb->pkt_type != PACKET_HOST)
1710		goto discard_it;
1711
1712	/*
1713	 *	Count it even if it's bad.
1714	 */
1715	__TCP_INC_STATS(net, TCP_MIB_INSEGS);
1716
1717	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1718		goto discard_it;
1719
1720	th = (const struct tcphdr *)skb->data;
1721
1722	if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1723		drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1724		goto bad_packet;
1725	}
1726	if (!pskb_may_pull(skb, th->doff*4))
1727		goto discard_it;
1728
1729	if (skb_checksum_init(skb, IPPROTO_TCP, ip6_compute_pseudo))
1730		goto csum_error;
1731
1732	th = (const struct tcphdr *)skb->data;
1733	hdr = ipv6_hdr(skb);
1734
1735lookup:
1736	sk = __inet6_lookup_skb(skb, __tcp_hdrlen(th),
1737				th->source, th->dest, inet6_iif(skb), sdif,
1738				&refcounted);
1739	if (!sk)
1740		goto no_tcp_socket;
1741
1742	if (sk->sk_state == TCP_TIME_WAIT)
1743		goto do_time_wait;
1744
1745	if (sk->sk_state == TCP_NEW_SYN_RECV) {
1746		struct request_sock *req = inet_reqsk(sk);
1747		bool req_stolen = false;
1748		struct sock *nsk;
1749
1750		sk = req->rsk_listener;
1751		if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
1752			drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1753		else
1754			drop_reason = tcp_inbound_hash(sk, req, skb,
1755						       &hdr->saddr, &hdr->daddr,
1756						       AF_INET6, dif, sdif);
1757		if (drop_reason) {
1758			sk_drops_skbadd(sk, skb);
1759			reqsk_put(req);
1760			goto discard_it;
1761		}
1762		if (tcp_checksum_complete(skb)) {
1763			reqsk_put(req);
1764			goto csum_error;
1765		}
1766		if (unlikely(sk->sk_state != TCP_LISTEN)) {
1767			nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1768			if (!nsk) {
1769				inet_csk_reqsk_queue_drop_and_put(sk, req);
1770				goto lookup;
1771			}
1772			sk = nsk;
1773			/* reuseport_migrate_sock() has already held one sk_refcnt
1774			 * before returning.
1775			 */
1776		} else {
1777			sock_hold(sk);
1778		}
1779		refcounted = true;
1780		nsk = NULL;
1781		if (!tcp_filter(sk, skb, &drop_reason)) {
1782			th = (const struct tcphdr *)skb->data;
1783			hdr = ipv6_hdr(skb);
1784			tcp_v6_fill_cb(skb, hdr, th);
1785			nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
1786					    &drop_reason);
1787		}
1788		if (!nsk) {
1789			reqsk_put(req);
1790			if (req_stolen) {
1791				/* Another cpu got exclusive access to req
1792				 * and created a full blown socket.
1793				 * Try to feed this packet to this socket
1794				 * instead of discarding it.
1795				 */
1796				tcp_v6_restore_cb(skb);
1797				sock_put(sk);
1798				goto lookup;
1799			}
1800			goto discard_and_relse;
1801		}
1802		nf_reset_ct(skb);
1803		if (nsk == sk) {
1804			reqsk_put(req);
1805			tcp_v6_restore_cb(skb);
1806		} else {
1807			drop_reason = tcp_child_process(sk, nsk, skb);
1808			if (drop_reason) {
1809				enum sk_rst_reason rst_reason;
1810
1811				rst_reason = sk_rst_convert_drop_reason(drop_reason);
1812				tcp_v6_send_reset(nsk, skb, rst_reason);
1813				goto discard_and_relse;
1814			}
1815			sock_put(sk);
1816			return 0;
1817		}
1818	}
1819
1820process:
1821	if (static_branch_unlikely(&ip6_min_hopcount)) {
1822		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
1823		if (unlikely(hdr->hop_limit < READ_ONCE(tcp_inet6_sk(sk)->min_hopcount))) {
1824			__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
1825			drop_reason = SKB_DROP_REASON_TCP_MINTTL;
1826			goto discard_and_relse;
1827		}
1828	}
1829
1830	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
1831		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1832		goto discard_and_relse;
1833	}
1834
1835	drop_reason = tcp_inbound_hash(sk, NULL, skb, &hdr->saddr, &hdr->daddr,
1836				       AF_INET6, dif, sdif);
1837	if (drop_reason)
1838		goto discard_and_relse;
1839
1840	nf_reset_ct(skb);
1841
1842	if (tcp_filter(sk, skb, &drop_reason))
1843		goto discard_and_relse;
1844
1845	th = (const struct tcphdr *)skb->data;
1846	hdr = ipv6_hdr(skb);
1847	tcp_v6_fill_cb(skb, hdr, th);
1848
1849	skb->dev = NULL;
1850
1851	if (sk->sk_state == TCP_LISTEN) {
1852		ret = tcp_v6_do_rcv(sk, skb);
1853		goto put_and_return;
1854	}
1855
1856	sk_incoming_cpu_update(sk);
1857
1858	bh_lock_sock_nested(sk);
1859	tcp_segs_in(tcp_sk(sk), skb);
1860	ret = 0;
1861	if (!sock_owned_by_user(sk)) {
1862		ret = tcp_v6_do_rcv(sk, skb);
1863	} else {
1864		if (tcp_add_backlog(sk, skb, &drop_reason))
1865			goto discard_and_relse;
1866	}
1867	bh_unlock_sock(sk);
1868put_and_return:
1869	if (refcounted)
1870		sock_put(sk);
1871	return ret ? -1 : 0;
1872
1873no_tcp_socket:
1874	drop_reason = SKB_DROP_REASON_NO_SOCKET;
1875	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
1876		goto discard_it;
1877
1878	tcp_v6_fill_cb(skb, hdr, th);
1879
1880	if (tcp_checksum_complete(skb)) {
1881csum_error:
1882		drop_reason = SKB_DROP_REASON_TCP_CSUM;
1883		trace_tcp_bad_csum(skb);
1884		__TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
1885bad_packet:
1886		__TCP_INC_STATS(net, TCP_MIB_INERRS);
1887	} else {
1888		tcp_v6_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
1889	}
1890
1891discard_it:
1892	SKB_DR_OR(drop_reason, NOT_SPECIFIED);
1893	sk_skb_reason_drop(sk, skb, drop_reason);
1894	return 0;
1895
1896discard_and_relse:
1897	sk_drops_skbadd(sk, skb);
1898	if (refcounted)
1899		sock_put(sk);
1900	goto discard_it;
1901
1902do_time_wait:
1903	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) {
1904		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1905		inet_twsk_put(inet_twsk(sk));
1906		goto discard_it;
1907	}
1908
1909	tcp_v6_fill_cb(skb, hdr, th);
1910
1911	if (tcp_checksum_complete(skb)) {
1912		inet_twsk_put(inet_twsk(sk));
1913		goto csum_error;
1914	}
1915
1916	tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
1917					       &drop_reason);
1918	switch (tw_status) {
1919	case TCP_TW_SYN:
1920	{
1921		struct sock *sk2;
1922
1923		sk2 = inet6_lookup_listener(net, skb, __tcp_hdrlen(th),
1924					    &ipv6_hdr(skb)->saddr, th->source,
1925					    &ipv6_hdr(skb)->daddr,
1926					    ntohs(th->dest),
1927					    tcp_v6_iif_l3_slave(skb),
1928					    sdif);
1929		if (sk2) {
1930			struct inet_timewait_sock *tw = inet_twsk(sk);
1931			inet_twsk_deschedule_put(tw);
1932			sk = sk2;
1933			tcp_v6_restore_cb(skb);
1934			refcounted = false;
1935			__this_cpu_write(tcp_tw_isn, isn);
1936			goto process;
1937		}
1938
1939		drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
1940		if (drop_reason)
1941			break;
1942	}
1943		/* to ACK */
1944		fallthrough;
1945	case TCP_TW_ACK:
1946	case TCP_TW_ACK_OOW:
1947		tcp_v6_timewait_ack(sk, skb, tw_status);
1948		break;
1949	case TCP_TW_RST:
1950		tcp_v6_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
1951		inet_twsk_deschedule_put(inet_twsk(sk));
1952		goto discard_it;
1953	case TCP_TW_SUCCESS:
1954		;
1955	}
1956	goto discard_it;
1957}
1958
1959void tcp_v6_early_demux(struct sk_buff *skb)
1960{
1961	struct net *net = dev_net_rcu(skb->dev);
1962	const struct ipv6hdr *hdr;
1963	const struct tcphdr *th;
1964	struct sock *sk;
1965
1966	if (skb->pkt_type != PACKET_HOST)
1967		return;
1968
1969	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1970		return;
1971
1972	hdr = ipv6_hdr(skb);
1973	th = tcp_hdr(skb);
1974
1975	if (th->doff < sizeof(struct tcphdr) / 4)
1976		return;
1977
1978	/* Note : We use inet6_iif() here, not tcp_v6_iif() */
1979	sk = __inet6_lookup_established(net, &hdr->saddr, th->source,
1980					&hdr->daddr, ntohs(th->dest),
1981					inet6_iif(skb), inet6_sdif(skb));
1982	if (sk) {
1983		skb->sk = sk;
1984		skb->destructor = sock_edemux;
1985		if (sk_fullsock(sk)) {
1986			struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
1987
1988			if (dst)
1989				dst = dst_check(dst, sk->sk_rx_dst_cookie);
1990			if (dst &&
1991			    sk->sk_rx_dst_ifindex == skb->skb_iif)
1992				skb_dst_set_noref(skb, dst);
1993		}
1994	}
1995}
1996
1997static struct timewait_sock_ops tcp6_timewait_sock_ops = {
1998	.twsk_obj_size	= sizeof(struct tcp6_timewait_sock),
1999};
2000
2001INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
2002{
2003	__tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr);
2004}
2005
2006const struct inet_connection_sock_af_ops ipv6_specific = {
2007	.queue_xmit	   = inet6_csk_xmit,
2008	.send_check	   = tcp_v6_send_check,
2009	.rebuild_header	   = inet6_sk_rebuild_header,
2010	.sk_rx_dst_set	   = inet6_sk_rx_dst_set,
2011	.conn_request	   = tcp_v6_conn_request,
2012	.syn_recv_sock	   = tcp_v6_syn_recv_sock,
2013	.net_header_len	   = sizeof(struct ipv6hdr),
2014	.setsockopt	   = ipv6_setsockopt,
2015	.getsockopt	   = ipv6_getsockopt,
2016	.mtu_reduced	   = tcp_v6_mtu_reduced,
2017};
2018
2019#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2020static const struct tcp_sock_af_ops tcp_sock_ipv6_specific = {
2021#ifdef CONFIG_TCP_MD5SIG
2022	.md5_lookup	=	tcp_v6_md5_lookup,
2023	.calc_md5_hash	=	tcp_v6_md5_hash_skb,
2024	.md5_parse	=	tcp_v6_parse_md5_keys,
2025#endif
2026#ifdef CONFIG_TCP_AO
2027	.ao_lookup	=	tcp_v6_ao_lookup,
2028	.calc_ao_hash	=	tcp_v6_ao_hash_skb,
2029	.ao_parse	=	tcp_v6_parse_ao,
2030	.ao_calc_key_sk	=	tcp_v6_ao_calc_key_sk,
2031#endif
2032};
2033#endif
2034
2035/*
2036 *	TCP over IPv4 via INET6 API
2037 */
2038static const struct inet_connection_sock_af_ops ipv6_mapped = {
2039	.queue_xmit	   = ip_queue_xmit,
2040	.send_check	   = tcp_v4_send_check,
2041	.rebuild_header	   = inet_sk_rebuild_header,
2042	.sk_rx_dst_set	   = inet_sk_rx_dst_set,
2043	.conn_request	   = tcp_v6_conn_request,
2044	.syn_recv_sock	   = tcp_v6_syn_recv_sock,
2045	.net_header_len	   = sizeof(struct iphdr),
2046	.setsockopt	   = ipv6_setsockopt,
2047	.getsockopt	   = ipv6_getsockopt,
2048	.mtu_reduced	   = tcp_v4_mtu_reduced,
2049};
2050
2051#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2052static const struct tcp_sock_af_ops tcp_sock_ipv6_mapped_specific = {
2053#ifdef CONFIG_TCP_MD5SIG
2054	.md5_lookup	=	tcp_v4_md5_lookup,
2055	.calc_md5_hash	=	tcp_v4_md5_hash_skb,
2056	.md5_parse	=	tcp_v6_parse_md5_keys,
2057#endif
2058#ifdef CONFIG_TCP_AO
2059	.ao_lookup	=	tcp_v6_ao_lookup,
2060	.calc_ao_hash	=	tcp_v4_ao_hash_skb,
2061	.ao_parse	=	tcp_v6_parse_ao,
2062	.ao_calc_key_sk	=	tcp_v4_ao_calc_key_sk,
2063#endif
2064};
2065
2066static void tcp6_destruct_sock(struct sock *sk)
2067{
2068	tcp_md5_destruct_sock(sk);
2069	tcp_ao_destroy_sock(sk, false);
2070	inet6_sock_destruct(sk);
2071}
2072#endif
2073
2074/* NOTE: A lot of things set to zero explicitly by call to
2075 *       sk_alloc() so need not be done here.
2076 */
2077static int tcp_v6_init_sock(struct sock *sk)
2078{
2079	struct inet_connection_sock *icsk = inet_csk(sk);
2080
2081	tcp_init_sock(sk);
2082
2083	icsk->icsk_af_ops = &ipv6_specific;
2084
2085#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
2086	tcp_sk(sk)->af_specific = &tcp_sock_ipv6_specific;
2087	sk->sk_destruct = tcp6_destruct_sock;
2088#endif
2089
2090	return 0;
2091}
2092
2093#ifdef CONFIG_PROC_FS
2094/* Proc filesystem TCPv6 sock list dumping. */
2095static void get_openreq6(struct seq_file *seq,
2096			 const struct request_sock *req, int i)
2097{
2098	long ttd = req->rsk_timer.expires - jiffies;
2099	const struct in6_addr *src = &inet_rsk(req)->ir_v6_loc_addr;
2100	const struct in6_addr *dest = &inet_rsk(req)->ir_v6_rmt_addr;
2101
2102	if (ttd < 0)
2103		ttd = 0;
2104
2105	seq_printf(seq,
2106		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2107		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
2108		   i,
2109		   src->s6_addr32[0], src->s6_addr32[1],
2110		   src->s6_addr32[2], src->s6_addr32[3],
2111		   inet_rsk(req)->ir_num,
2112		   dest->s6_addr32[0], dest->s6_addr32[1],
2113		   dest->s6_addr32[2], dest->s6_addr32[3],
2114		   ntohs(inet_rsk(req)->ir_rmt_port),
2115		   TCP_SYN_RECV,
2116		   0, 0, /* could print option size, but that is af dependent. */
2117		   1,   /* timers active (only the expire timer) */
2118		   jiffies_to_clock_t(ttd),
2119		   req->num_timeout,
2120		   from_kuid_munged(seq_user_ns(seq),
2121				    sk_uid(req->rsk_listener)),
2122		   0,  /* non standard timer */
2123		   0, /* open_requests have no inode */
2124		   0, req);
2125}
2126
2127static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
2128{
2129	const struct in6_addr *dest, *src;
2130	__u16 destp, srcp;
2131	int timer_active;
2132	unsigned long timer_expires;
2133	const struct inet_sock *inet = inet_sk(sp);
2134	const struct tcp_sock *tp = tcp_sk(sp);
2135	const struct inet_connection_sock *icsk = inet_csk(sp);
2136	const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2137	u8 icsk_pending;
2138	int rx_queue;
2139	int state;
2140
2141	dest  = &sp->sk_v6_daddr;
2142	src   = &sp->sk_v6_rcv_saddr;
2143	destp = ntohs(inet->inet_dport);
2144	srcp  = ntohs(inet->inet_sport);
2145
2146	icsk_pending = smp_load_acquire(&icsk->icsk_pending);
2147	if (icsk_pending == ICSK_TIME_RETRANS ||
2148	    icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2149	    icsk_pending == ICSK_TIME_LOSS_PROBE) {
2150		timer_active	= 1;
2151		timer_expires	= tcp_timeout_expires(sp);
2152	} else if (icsk_pending == ICSK_TIME_PROBE0) {
2153		timer_active	= 4;
2154		timer_expires	= tcp_timeout_expires(sp);
2155	} else if (timer_pending(&icsk->icsk_keepalive_timer)) {
2156		timer_active	= 2;
2157		timer_expires	= icsk->icsk_keepalive_timer.expires;
2158	} else {
2159		timer_active	= 0;
2160		timer_expires = jiffies;
2161	}
2162
2163	state = inet_sk_state_load(sp);
2164	if (state == TCP_LISTEN)
2165		rx_queue = READ_ONCE(sp->sk_ack_backlog);
2166	else
2167		/* Because we don't lock the socket,
2168		 * we might find a transient negative value.
2169		 */
2170		rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2171				      READ_ONCE(tp->copied_seq), 0);
2172
2173	seq_printf(seq,
2174		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2175		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %lu %lu %u %u %d\n",
2176		   i,
2177		   src->s6_addr32[0], src->s6_addr32[1],
2178		   src->s6_addr32[2], src->s6_addr32[3], srcp,
2179		   dest->s6_addr32[0], dest->s6_addr32[1],
2180		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
2181		   state,
2182		   READ_ONCE(tp->write_seq) - tp->snd_una,
2183		   rx_queue,
2184		   timer_active,
2185		   jiffies_delta_to_clock_t(timer_expires - jiffies),
2186		   READ_ONCE(icsk->icsk_retransmits),
2187		   from_kuid_munged(seq_user_ns(seq), sk_uid(sp)),
2188		   READ_ONCE(icsk->icsk_probes_out),
2189		   sock_i_ino(sp),
2190		   refcount_read(&sp->sk_refcnt), sp,
2191		   jiffies_to_clock_t(icsk->icsk_rto),
2192		   jiffies_to_clock_t(icsk->icsk_ack.ato),
2193		   (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
2194		   tcp_snd_cwnd(tp),
2195		   state == TCP_LISTEN ?
2196			fastopenq->max_qlen :
2197			(tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)
2198		   );
2199}
2200
2201static void get_timewait6_sock(struct seq_file *seq,
2202			       struct inet_timewait_sock *tw, int i)
2203{
2204	long delta = tw->tw_timer.expires - jiffies;
2205	const struct in6_addr *dest, *src;
2206	__u16 destp, srcp;
2207
2208	dest = &tw->tw_v6_daddr;
2209	src  = &tw->tw_v6_rcv_saddr;
2210	destp = ntohs(tw->tw_dport);
2211	srcp  = ntohs(tw->tw_sport);
2212
2213	seq_printf(seq,
2214		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
2215		   "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
2216		   i,
2217		   src->s6_addr32[0], src->s6_addr32[1],
2218		   src->s6_addr32[2], src->s6_addr32[3], srcp,
2219		   dest->s6_addr32[0], dest->s6_addr32[1],
2220		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
2221		   READ_ONCE(tw->tw_substate), 0, 0,
2222		   3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2223		   refcount_read(&tw->tw_refcnt), tw);
2224}
2225
2226static int tcp6_seq_show(struct seq_file *seq, void *v)
2227{
2228	struct tcp_iter_state *st;
2229	struct sock *sk = v;
2230
2231	if (v == SEQ_START_TOKEN) {
2232		seq_puts(seq,
2233			 "  sl  "
2234			 "local_address                         "
2235			 "remote_address                        "
2236			 "st tx_queue rx_queue tr tm->when retrnsmt"
2237			 "   uid  timeout inode\n");
2238		goto out;
2239	}
2240	st = seq->private;
2241
2242	if (sk->sk_state == TCP_TIME_WAIT)
2243		get_timewait6_sock(seq, v, st->num);
2244	else if (sk->sk_state == TCP_NEW_SYN_RECV)
2245		get_openreq6(seq, v, st->num);
2246	else
2247		get_tcp6_sock(seq, v, st->num);
2248out:
2249	return 0;
2250}
2251
2252static const struct seq_operations tcp6_seq_ops = {
2253	.show		= tcp6_seq_show,
2254	.start		= tcp_seq_start,
2255	.next		= tcp_seq_next,
2256	.stop		= tcp_seq_stop,
2257};
2258
2259static struct tcp_seq_afinfo tcp6_seq_afinfo = {
2260	.family		= AF_INET6,
2261};
2262
2263int __net_init tcp6_proc_init(struct net *net)
2264{
2265	if (!proc_create_net_data("tcp6", 0444, net->proc_net, &tcp6_seq_ops,
2266			sizeof(struct tcp_iter_state), &tcp6_seq_afinfo))
2267		return -ENOMEM;
2268	return 0;
2269}
2270
2271void tcp6_proc_exit(struct net *net)
2272{
2273	remove_proc_entry("tcp6", net->proc_net);
2274}
2275#endif
2276
2277struct proto tcpv6_prot = {
2278	.name			= "TCPv6",
2279	.owner			= THIS_MODULE,
2280	.close			= tcp_close,
2281	.pre_connect		= tcp_v6_pre_connect,
2282	.connect		= tcp_v6_connect,
2283	.disconnect		= tcp_disconnect,
2284	.accept			= inet_csk_accept,
2285	.ioctl			= tcp_ioctl,
2286	.init			= tcp_v6_init_sock,
2287	.destroy		= tcp_v4_destroy_sock,
2288	.shutdown		= tcp_shutdown,
2289	.setsockopt		= tcp_setsockopt,
2290	.getsockopt		= tcp_getsockopt,
2291	.bpf_bypass_getsockopt	= tcp_bpf_bypass_getsockopt,
2292	.keepalive		= tcp_set_keepalive,
2293	.recvmsg		= tcp_recvmsg,
2294	.sendmsg		= tcp_sendmsg,
2295	.splice_eof		= tcp_splice_eof,
2296	.backlog_rcv		= tcp_v6_do_rcv,
2297	.release_cb		= tcp_release_cb,
2298	.hash			= inet_hash,
2299	.unhash			= inet_unhash,
2300	.get_port		= inet_csk_get_port,
2301	.put_port		= inet_put_port,
2302#ifdef CONFIG_BPF_SYSCALL
2303	.psock_update_sk_prot	= tcp_bpf_update_proto,
2304#endif
2305	.enter_memory_pressure	= tcp_enter_memory_pressure,
2306	.leave_memory_pressure	= tcp_leave_memory_pressure,
2307	.stream_memory_free	= tcp_stream_memory_free,
2308	.sockets_allocated	= &tcp_sockets_allocated,
2309
2310	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,
2311	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
2312
2313	.memory_pressure	= &tcp_memory_pressure,
2314	.sysctl_mem		= sysctl_tcp_mem,
2315	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_wmem),
2316	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_tcp_rmem),
2317	.max_header		= MAX_TCP_HEADER,
2318	.obj_size		= sizeof(struct tcp6_sock),
2319	.ipv6_pinfo_offset = offsetof(struct tcp6_sock, inet6),
2320	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
2321	.twsk_prot		= &tcp6_timewait_sock_ops,
2322	.rsk_prot		= &tcp6_request_sock_ops,
2323	.h.hashinfo		= NULL,
2324	.no_autobind		= true,
2325	.diag_destroy		= tcp_abort,
2326};
2327EXPORT_SYMBOL_GPL(tcpv6_prot);
2328
2329
2330static struct inet_protosw tcpv6_protosw = {
2331	.type		=	SOCK_STREAM,
2332	.protocol	=	IPPROTO_TCP,
2333	.prot		=	&tcpv6_prot,
2334	.ops		=	&inet6_stream_ops,
2335	.flags		=	INET_PROTOSW_PERMANENT |
2336				INET_PROTOSW_ICSK,
2337};
2338
2339static int __net_init tcpv6_net_init(struct net *net)
2340{
2341	int res;
2342
2343	res = inet_ctl_sock_create(&net->ipv6.tcp_sk, PF_INET6,
2344				   SOCK_RAW, IPPROTO_TCP, net);
2345	if (!res)
2346		net->ipv6.tcp_sk->sk_clockid = CLOCK_MONOTONIC;
2347
2348	return res;
2349}
2350
2351static void __net_exit tcpv6_net_exit(struct net *net)
2352{
2353	inet_ctl_sock_destroy(net->ipv6.tcp_sk);
2354}
2355
2356static struct pernet_operations tcpv6_net_ops = {
2357	.init	    = tcpv6_net_init,
2358	.exit	    = tcpv6_net_exit,
2359};
2360
2361int __init tcpv6_init(void)
2362{
2363	int ret;
2364
2365	net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2366		.handler     = tcp_v6_rcv,
2367		.err_handler = tcp_v6_err,
2368		.flags	     = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2369	};
2370	ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2371	if (ret)
2372		goto out;
2373
2374	/* register inet6 protocol */
2375	ret = inet6_register_protosw(&tcpv6_protosw);
2376	if (ret)
2377		goto out_tcpv6_protocol;
2378
2379	ret = register_pernet_subsys(&tcpv6_net_ops);
2380	if (ret)
2381		goto out_tcpv6_protosw;
2382
2383	ret = mptcpv6_init();
2384	if (ret)
2385		goto out_tcpv6_pernet_subsys;
2386
2387out:
2388	return ret;
2389
2390out_tcpv6_pernet_subsys:
2391	unregister_pernet_subsys(&tcpv6_net_ops);
2392out_tcpv6_protosw:
2393	inet6_unregister_protosw(&tcpv6_protosw);
2394out_tcpv6_protocol:
2395	inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2396	goto out;
2397}
2398
2399void tcpv6_exit(void)
2400{
2401	unregister_pernet_subsys(&tcpv6_net_ops);
2402	inet6_unregister_protosw(&tcpv6_protosw);
2403	inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2404}