options.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Multipath TCP
   3 *
   4 * Copyright (c) 2017 - 2019, Intel Corporation.
   5 */
   6
   7#define pr_fmt(fmt) "MPTCP: " fmt
   8
   9#include <linux/kernel.h>
  10#include <crypto/sha2.h>
  11#include <net/tcp.h>
  12#include <net/mptcp.h>
  13#include "protocol.h"
  14#include "mib.h"
  15
  16#include <trace/events/mptcp.h>
  17
  18static bool mptcp_cap_flag_sha256(u8 flags)
  19{
  20	return (flags & MPTCP_CAP_FLAG_MASK) == MPTCP_CAP_HMAC_SHA256;
  21}
  22
  23static void mptcp_parse_option(const struct sk_buff *skb,
  24			       const unsigned char *ptr, int opsize,
  25			       struct mptcp_options_received *mp_opt)
  26{
  27	u8 subtype = *ptr >> 4;
  28	int expected_opsize;
  29	u16 subopt;
  30	u8 version;
  31	u8 flags;
  32	u8 i;
  33
  34	switch (subtype) {
  35	case MPTCPOPT_MP_CAPABLE:
  36		/* strict size checking */
  37		if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
  38			if (skb->len > tcp_hdr(skb)->doff << 2)
  39				expected_opsize = TCPOLEN_MPTCP_MPC_ACK_DATA;
  40			else
  41				expected_opsize = TCPOLEN_MPTCP_MPC_ACK;
  42			subopt = OPTION_MPTCP_MPC_ACK;
  43		} else {
  44			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK) {
  45				expected_opsize = TCPOLEN_MPTCP_MPC_SYNACK;
  46				subopt = OPTION_MPTCP_MPC_SYNACK;
  47			} else {
  48				expected_opsize = TCPOLEN_MPTCP_MPC_SYN;
  49				subopt = OPTION_MPTCP_MPC_SYN;
  50			}
  51		}
  52
  53		/* Cfr RFC 8684 Section 3.3.0:
  54		 * If a checksum is present but its use had
  55		 * not been negotiated in the MP_CAPABLE handshake, the receiver MUST
  56		 * close the subflow with a RST, as it is not behaving as negotiated.
  57		 * If a checksum is not present when its use has been negotiated, the
  58		 * receiver MUST close the subflow with a RST, as it is considered
  59		 * broken
  60		 * We parse even option with mismatching csum presence, so that
  61		 * later in subflow_data_ready we can trigger the reset.
  62		 */
  63		if (opsize != expected_opsize &&
  64		    (expected_opsize != TCPOLEN_MPTCP_MPC_ACK_DATA ||
  65		     opsize != TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM))
  66			break;
  67
  68		/* try to be gentle vs future versions on the initial syn */
  69		version = *ptr++ & MPTCP_VERSION_MASK;
  70		if (opsize != TCPOLEN_MPTCP_MPC_SYN) {
  71			if (version != MPTCP_SUPPORTED_VERSION)
  72				break;
  73		} else if (version < MPTCP_SUPPORTED_VERSION) {
  74			break;
  75		}
  76
  77		flags = *ptr++;
  78		if (!mptcp_cap_flag_sha256(flags) ||
  79		    (flags & MPTCP_CAP_EXTENSIBILITY))
  80			break;
  81
  82		/* RFC 6824, Section 3.1:
  83		 * "For the Checksum Required bit (labeled "A"), if either
  84		 * host requires the use of checksums, checksums MUST be used.
  85		 * In other words, the only way for checksums not to be used
  86		 * is if both hosts in their SYNs set A=0."
  87		 */
  88		if (flags & MPTCP_CAP_CHECKSUM_REQD)
  89			mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
  90
  91		mp_opt->deny_join_id0 = !!(flags & MPTCP_CAP_DENY_JOIN_ID0);
  92
  93		mp_opt->suboptions |= subopt;
  94		if (opsize >= TCPOLEN_MPTCP_MPC_SYNACK) {
  95			mp_opt->sndr_key = get_unaligned_be64(ptr);
  96			ptr += 8;
  97		}
  98		if (opsize >= TCPOLEN_MPTCP_MPC_ACK) {
  99			mp_opt->rcvr_key = get_unaligned_be64(ptr);
 100			ptr += 8;
 101		}
 102		if (opsize >= TCPOLEN_MPTCP_MPC_ACK_DATA) {
 103			/* Section 3.1.:
 104			 * "the data parameters in a MP_CAPABLE are semantically
 105			 * equivalent to those in a DSS option and can be used
 106			 * interchangeably."
 107			 */
 108			mp_opt->suboptions |= OPTION_MPTCP_DSS;
 109			mp_opt->use_map = 1;
 110			mp_opt->mpc_map = 1;
 111			mp_opt->data_len = get_unaligned_be16(ptr);
 112			ptr += 2;
 113		}
 114		if (opsize == TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM) {
 115			mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
 116			mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
 117			ptr += 2;
 118		}
 119		pr_debug("MP_CAPABLE version=%x, flags=%x, optlen=%d sndr=%llu, rcvr=%llu len=%d csum=%u\n",
 120			 version, flags, opsize, mp_opt->sndr_key,
 121			 mp_opt->rcvr_key, mp_opt->data_len, mp_opt->csum);
 122		break;
 123
 124	case MPTCPOPT_MP_JOIN:
 125		if (opsize == TCPOLEN_MPTCP_MPJ_SYN) {
 126			mp_opt->suboptions |= OPTION_MPTCP_MPJ_SYN;
 127			mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP;
 128			mp_opt->join_id = *ptr++;
 129			mp_opt->token = get_unaligned_be32(ptr);
 130			ptr += 4;
 131			mp_opt->nonce = get_unaligned_be32(ptr);
 132			ptr += 4;
 133			pr_debug("MP_JOIN bkup=%u, id=%u, token=%u, nonce=%u\n",
 134				 mp_opt->backup, mp_opt->join_id,
 135				 mp_opt->token, mp_opt->nonce);
 136		} else if (opsize == TCPOLEN_MPTCP_MPJ_SYNACK) {
 137			mp_opt->suboptions |= OPTION_MPTCP_MPJ_SYNACK;
 138			mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP;
 139			mp_opt->join_id = *ptr++;
 140			mp_opt->thmac = get_unaligned_be64(ptr);
 141			ptr += 8;
 142			mp_opt->nonce = get_unaligned_be32(ptr);
 143			ptr += 4;
 144			pr_debug("MP_JOIN bkup=%u, id=%u, thmac=%llu, nonce=%u\n",
 145				 mp_opt->backup, mp_opt->join_id,
 146				 mp_opt->thmac, mp_opt->nonce);
 147		} else if (opsize == TCPOLEN_MPTCP_MPJ_ACK) {
 148			mp_opt->suboptions |= OPTION_MPTCP_MPJ_ACK;
 149			ptr += 2;
 150			memcpy(mp_opt->hmac, ptr, MPTCPOPT_HMAC_LEN);
 151			pr_debug("MP_JOIN hmac\n");
 152		}
 153		break;
 154
 155	case MPTCPOPT_DSS:
 156		pr_debug("DSS\n");
 157		ptr++;
 158
 159		flags = (*ptr++) & MPTCP_DSS_FLAG_MASK;
 160		mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0;
 161		mp_opt->dsn64 = (flags & MPTCP_DSS_DSN64) != 0;
 162		mp_opt->use_map = (flags & MPTCP_DSS_HAS_MAP) != 0;
 163		mp_opt->ack64 = (flags & MPTCP_DSS_ACK64) != 0;
 164		mp_opt->use_ack = (flags & MPTCP_DSS_HAS_ACK);
 165
 166		pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d\n",
 167			 mp_opt->data_fin, mp_opt->dsn64,
 168			 mp_opt->use_map, mp_opt->ack64,
 169			 mp_opt->use_ack);
 170
 171		expected_opsize = TCPOLEN_MPTCP_DSS_BASE;
 172
 173		if (mp_opt->use_ack) {
 174			if (mp_opt->ack64)
 175				expected_opsize += TCPOLEN_MPTCP_DSS_ACK64;
 176			else
 177				expected_opsize += TCPOLEN_MPTCP_DSS_ACK32;
 178		}
 179
 180		if (mp_opt->use_map) {
 181			if (mp_opt->dsn64)
 182				expected_opsize += TCPOLEN_MPTCP_DSS_MAP64;
 183			else
 184				expected_opsize += TCPOLEN_MPTCP_DSS_MAP32;
 185		}
 186
 187		/* Always parse any csum presence combination, we will enforce
 188		 * RFC 8684 Section 3.3.0 checks later in subflow_data_ready
 189		 */
 190		if (opsize != expected_opsize &&
 191		    opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM)
 192			break;
 193
 194		mp_opt->suboptions |= OPTION_MPTCP_DSS;
 195		if (mp_opt->use_ack) {
 196			if (mp_opt->ack64) {
 197				mp_opt->data_ack = get_unaligned_be64(ptr);
 198				ptr += 8;
 199			} else {
 200				mp_opt->data_ack = get_unaligned_be32(ptr);
 201				ptr += 4;
 202			}
 203
 204			pr_debug("data_ack=%llu\n", mp_opt->data_ack);
 205		}
 206
 207		if (mp_opt->use_map) {
 208			if (mp_opt->dsn64) {
 209				mp_opt->data_seq = get_unaligned_be64(ptr);
 210				ptr += 8;
 211			} else {
 212				mp_opt->data_seq = get_unaligned_be32(ptr);
 213				ptr += 4;
 214			}
 215
 216			mp_opt->subflow_seq = get_unaligned_be32(ptr);
 217			ptr += 4;
 218
 219			mp_opt->data_len = get_unaligned_be16(ptr);
 220			ptr += 2;
 221
 222			if (opsize == expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) {
 223				mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
 224				mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
 225				ptr += 2;
 226			}
 227
 228			pr_debug("data_seq=%llu subflow_seq=%u data_len=%u csum=%d:%u\n",
 229				 mp_opt->data_seq, mp_opt->subflow_seq,
 230				 mp_opt->data_len, !!(mp_opt->suboptions & OPTION_MPTCP_CSUMREQD),
 231				 mp_opt->csum);
 232		}
 233
 234		break;
 235
 236	case MPTCPOPT_ADD_ADDR:
 237		mp_opt->echo = (*ptr++) & MPTCP_ADDR_ECHO;
 238		if (!mp_opt->echo) {
 239			if (opsize == TCPOLEN_MPTCP_ADD_ADDR ||
 240			    opsize == TCPOLEN_MPTCP_ADD_ADDR_PORT)
 241				mp_opt->addr.family = AF_INET;
 242#if IS_ENABLED(CONFIG_MPTCP_IPV6)
 243			else if (opsize == TCPOLEN_MPTCP_ADD_ADDR6 ||
 244				 opsize == TCPOLEN_MPTCP_ADD_ADDR6_PORT)
 245				mp_opt->addr.family = AF_INET6;
 246#endif
 247			else
 248				break;
 249		} else {
 250			if (opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE ||
 251			    opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT)
 252				mp_opt->addr.family = AF_INET;
 253#if IS_ENABLED(CONFIG_MPTCP_IPV6)
 254			else if (opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE ||
 255				 opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT)
 256				mp_opt->addr.family = AF_INET6;
 257#endif
 258			else
 259				break;
 260		}
 261
 262		mp_opt->suboptions |= OPTION_MPTCP_ADD_ADDR;
 263		mp_opt->addr.id = *ptr++;
 264		mp_opt->addr.port = 0;
 265		mp_opt->ahmac = 0;
 266		if (mp_opt->addr.family == AF_INET) {
 267			memcpy((u8 *)&mp_opt->addr.addr.s_addr, (u8 *)ptr, 4);
 268			ptr += 4;
 269			if (opsize == TCPOLEN_MPTCP_ADD_ADDR_PORT ||
 270			    opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT) {
 271				mp_opt->addr.port = htons(get_unaligned_be16(ptr));
 272				ptr += 2;
 273			}
 274		}
 275#if IS_ENABLED(CONFIG_MPTCP_IPV6)
 276		else {
 277			memcpy(mp_opt->addr.addr6.s6_addr, (u8 *)ptr, 16);
 278			ptr += 16;
 279			if (opsize == TCPOLEN_MPTCP_ADD_ADDR6_PORT ||
 280			    opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT) {
 281				mp_opt->addr.port = htons(get_unaligned_be16(ptr));
 282				ptr += 2;
 283			}
 284		}
 285#endif
 286		if (!mp_opt->echo) {
 287			mp_opt->ahmac = get_unaligned_be64(ptr);
 288			ptr += 8;
 289		}
 290		pr_debug("ADD_ADDR%s: id=%d, ahmac=%llu, echo=%d, port=%d\n",
 291			 (mp_opt->addr.family == AF_INET6) ? "6" : "",
 292			 mp_opt->addr.id, mp_opt->ahmac, mp_opt->echo, ntohs(mp_opt->addr.port));
 293		break;
 294
 295	case MPTCPOPT_RM_ADDR:
 296		if (opsize < TCPOLEN_MPTCP_RM_ADDR_BASE + 1 ||
 297		    opsize > TCPOLEN_MPTCP_RM_ADDR_BASE + MPTCP_RM_IDS_MAX)
 298			break;
 299
 300		ptr++;
 301
 302		mp_opt->suboptions |= OPTION_MPTCP_RM_ADDR;
 303		mp_opt->rm_list.nr = opsize - TCPOLEN_MPTCP_RM_ADDR_BASE;
 304		for (i = 0; i < mp_opt->rm_list.nr; i++)
 305			mp_opt->rm_list.ids[i] = *ptr++;
 306		pr_debug("RM_ADDR: rm_list_nr=%d\n", mp_opt->rm_list.nr);
 307		break;
 308
 309	case MPTCPOPT_MP_PRIO:
 310		if (opsize != TCPOLEN_MPTCP_PRIO)
 311			break;
 312
 313		mp_opt->suboptions |= OPTION_MPTCP_PRIO;
 314		mp_opt->backup = *ptr++ & MPTCP_PRIO_BKUP;
 315		pr_debug("MP_PRIO: prio=%d\n", mp_opt->backup);
 316		break;
 317
 318	case MPTCPOPT_MP_FASTCLOSE:
 319		if (opsize != TCPOLEN_MPTCP_FASTCLOSE)
 320			break;
 321
 322		ptr += 2;
 323		mp_opt->rcvr_key = get_unaligned_be64(ptr);
 324		ptr += 8;
 325		mp_opt->suboptions |= OPTION_MPTCP_FASTCLOSE;
 326		pr_debug("MP_FASTCLOSE: recv_key=%llu\n", mp_opt->rcvr_key);
 327		break;
 328
 329	case MPTCPOPT_RST:
 330		if (opsize != TCPOLEN_MPTCP_RST)
 331			break;
 332
 333		if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST))
 334			break;
 335
 336		mp_opt->suboptions |= OPTION_MPTCP_RST;
 337		flags = *ptr++;
 338		mp_opt->reset_transient = flags & MPTCP_RST_TRANSIENT;
 339		mp_opt->reset_reason = *ptr;
 340		pr_debug("MP_RST: transient=%u reason=%u\n",
 341			 mp_opt->reset_transient, mp_opt->reset_reason);
 342		break;
 343
 344	case MPTCPOPT_MP_FAIL:
 345		if (opsize != TCPOLEN_MPTCP_FAIL)
 346			break;
 347
 348		ptr += 2;
 349		mp_opt->suboptions |= OPTION_MPTCP_FAIL;
 350		mp_opt->fail_seq = get_unaligned_be64(ptr);
 351		pr_debug("MP_FAIL: data_seq=%llu\n", mp_opt->fail_seq);
 352		break;
 353
 354	default:
 355		break;
 356	}
 357}
 358
 359void mptcp_get_options(const struct sk_buff *skb,
 360		       struct mptcp_options_received *mp_opt)
 361{
 362	const struct tcphdr *th = tcp_hdr(skb);
 363	const unsigned char *ptr;
 364	int length;
 365
 366	/* Ensure that casting the whole status to u32 is efficient and safe */
 367	BUILD_BUG_ON(sizeof_field(struct mptcp_options_received, status) != sizeof(u32));
 368	BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct mptcp_options_received, status),
 369				 sizeof(u32)));
 370	*(u32 *)&mp_opt->status = 0;
 371
 372	length = (th->doff * 4) - sizeof(struct tcphdr);
 373	ptr = (const unsigned char *)(th + 1);
 374
 375	while (length > 0) {
 376		int opcode = *ptr++;
 377		int opsize;
 378
 379		switch (opcode) {
 380		case TCPOPT_EOL:
 381			return;
 382		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
 383			length--;
 384			continue;
 385		default:
 386			if (length < 2)
 387				return;
 388			opsize = *ptr++;
 389			if (opsize < 2) /* "silly options" */
 390				return;
 391			if (opsize > length)
 392				return;	/* don't parse partial options */
 393			if (opcode == TCPOPT_MPTCP)
 394				mptcp_parse_option(skb, ptr, opsize, mp_opt);
 395			ptr += opsize - 2;
 396			length -= opsize;
 397		}
 398	}
 399}
 400
 401bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb,
 402		       unsigned int *size, struct mptcp_out_options *opts)
 403{
 404	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 405
 406	/* we will use snd_isn to detect first pkt [re]transmission
 407	 * in mptcp_established_options_mp()
 408	 */
 409	subflow->snd_isn = TCP_SKB_CB(skb)->end_seq;
 410	if (subflow->request_mptcp) {
 411		if (unlikely(subflow_simultaneous_connect(sk))) {
 412			WARN_ON_ONCE(!mptcp_try_fallback(sk, MPTCP_MIB_SIMULTCONNFALLBACK));
 413
 414			/* Ensure mptcp_finish_connect() will not process the
 415			 * MPC handshake.
 416			 */
 417			subflow->request_mptcp = 0;
 418			return false;
 419		}
 420
 421		opts->suboptions = OPTION_MPTCP_MPC_SYN;
 422		opts->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk));
 423		opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));
 424		*size = TCPOLEN_MPTCP_MPC_SYN;
 425		return true;
 426	} else if (subflow->request_join) {
 427		pr_debug("remote_token=%u, nonce=%u\n", subflow->remote_token,
 428			 subflow->local_nonce);
 429		opts->suboptions = OPTION_MPTCP_MPJ_SYN;
 430		opts->join_id = subflow->local_id;
 431		opts->token = subflow->remote_token;
 432		opts->nonce = subflow->local_nonce;
 433		opts->backup = subflow->request_bkup;
 434		*size = TCPOLEN_MPTCP_MPJ_SYN;
 435		return true;
 436	}
 437	return false;
 438}
 439
 440static void clear_3rdack_retransmission(struct sock *sk)
 441{
 442	struct inet_connection_sock *icsk = inet_csk(sk);
 443
 444	sk_stop_timer(sk, &icsk->icsk_delack_timer);
 445	icsk->icsk_ack.ato = 0;
 446	icsk->icsk_ack.pending &= ~(ICSK_ACK_SCHED | ICSK_ACK_TIMER);
 447}
 448
 449static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb,
 450					 bool snd_data_fin_enable,
 451					 unsigned int *size,
 452					 struct mptcp_out_options *opts)
 453{
 454	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 455	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 456	struct mptcp_ext *mpext;
 457	unsigned int data_len;
 458	u8 len;
 459
 460	/* When skb is not available, we better over-estimate the emitted
 461	 * options len. A full DSS option (28 bytes) is longer than
 462	 * TCPOLEN_MPTCP_MPC_ACK_DATA(22) or TCPOLEN_MPTCP_MPJ_ACK(24), so
 463	 * tell the caller to defer the estimate to
 464	 * mptcp_established_options_dss(), which will reserve enough space.
 465	 */
 466	if (!skb)
 467		return false;
 468
 469	/* MPC/MPJ needed only on 3rd ack packet, DATA_FIN and TCP shutdown take precedence */
 470	if (READ_ONCE(subflow->fully_established) || snd_data_fin_enable ||
 471	    subflow->snd_isn != TCP_SKB_CB(skb)->seq ||
 472	    sk->sk_state != TCP_ESTABLISHED)
 473		return false;
 474
 475	if (subflow->mp_capable) {
 476		mpext = mptcp_get_ext(skb);
 477		data_len = mpext ? mpext->data_len : 0;
 478
 479		/* we will check ops->data_len in mptcp_write_options() to
 480		 * discriminate between TCPOLEN_MPTCP_MPC_ACK_DATA and
 481		 * TCPOLEN_MPTCP_MPC_ACK
 482		 */
 483		opts->data_len = data_len;
 484		opts->suboptions = OPTION_MPTCP_MPC_ACK;
 485		opts->sndr_key = subflow->local_key;
 486		opts->rcvr_key = subflow->remote_key;
 487		opts->csum_reqd = READ_ONCE(msk->csum_enabled);
 488		opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));
 489
 490		/* Section 3.1.
 491		 * The MP_CAPABLE option is carried on the SYN, SYN/ACK, and ACK
 492		 * packets that start the first subflow of an MPTCP connection,
 493		 * as well as the first packet that carries data
 494		 */
 495		if (data_len > 0) {
 496			len = TCPOLEN_MPTCP_MPC_ACK_DATA;
 497			if (opts->csum_reqd) {
 498				/* we need to propagate more info to csum the pseudo hdr */
 499				opts->data_seq = mpext->data_seq;
 500				opts->subflow_seq = mpext->subflow_seq;
 501				opts->csum = mpext->csum;
 502				len += TCPOLEN_MPTCP_DSS_CHECKSUM;
 503			}
 504			*size = ALIGN(len, 4);
 505		} else {
 506			*size = TCPOLEN_MPTCP_MPC_ACK;
 507		}
 508
 509		pr_debug("subflow=%p, local_key=%llu, remote_key=%llu map_len=%d\n",
 510			 subflow, subflow->local_key, subflow->remote_key,
 511			 data_len);
 512
 513		return true;
 514	} else if (subflow->mp_join) {
 515		opts->suboptions = OPTION_MPTCP_MPJ_ACK;
 516		memcpy(opts->hmac, subflow->hmac, MPTCPOPT_HMAC_LEN);
 517		*size = TCPOLEN_MPTCP_MPJ_ACK;
 518		pr_debug("subflow=%p\n", subflow);
 519
 520		/* we can use the full delegate action helper only from BH context
 521		 * If we are in process context - sk is flushing the backlog at
 522		 * socket lock release time - just set the appropriate flag, will
 523		 * be handled by the release callback
 524		 */
 525		if (sock_owned_by_user(sk))
 526			set_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status);
 527		else
 528			mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_ACK);
 529		return true;
 530	}
 531	return false;
 532}
 533
 534static void mptcp_write_data_fin(struct mptcp_subflow_context *subflow,
 535				 struct sk_buff *skb, struct mptcp_ext *ext)
 536{
 537	/* The write_seq value has already been incremented, so the actual
 538	 * sequence number for the DATA_FIN is one less.
 539	 */
 540	u64 data_fin_tx_seq = READ_ONCE(mptcp_sk(subflow->conn)->write_seq) - 1;
 541
 542	if (!ext->use_map || !skb->len) {
 543		/* RFC6824 requires a DSS mapping with specific values
 544		 * if DATA_FIN is set but no data payload is mapped
 545		 */
 546		ext->data_fin = 1;
 547		ext->use_map = 1;
 548		ext->dsn64 = 1;
 549		ext->data_seq = data_fin_tx_seq;
 550		ext->subflow_seq = 0;
 551		ext->data_len = 1;
 552	} else if (ext->data_seq + ext->data_len == data_fin_tx_seq) {
 553		/* If there's an existing DSS mapping and it is the
 554		 * final mapping, DATA_FIN consumes 1 additional byte of
 555		 * mapping space.
 556		 */
 557		ext->data_fin = 1;
 558		ext->data_len++;
 559	}
 560}
 561
 562static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
 563					  bool snd_data_fin_enable,
 564					  unsigned int *size,
 565					  struct mptcp_out_options *opts)
 566{
 567	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 568	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 569	unsigned int dss_size = 0;
 570	struct mptcp_ext *mpext;
 571	unsigned int ack_size;
 572	bool ret = false;
 573	u64 ack_seq;
 574
 575	opts->csum_reqd = READ_ONCE(msk->csum_enabled);
 576	mpext = skb ? mptcp_get_ext(skb) : NULL;
 577
 578	if (!skb || (mpext && mpext->use_map) || snd_data_fin_enable) {
 579		unsigned int map_size = TCPOLEN_MPTCP_DSS_BASE + TCPOLEN_MPTCP_DSS_MAP64;
 580
 581		if (mpext) {
 582			if (opts->csum_reqd)
 583				map_size += TCPOLEN_MPTCP_DSS_CHECKSUM;
 584
 585			opts->ext_copy = *mpext;
 586		}
 587
 588		dss_size = map_size;
 589		if (skb && snd_data_fin_enable)
 590			mptcp_write_data_fin(subflow, skb, &opts->ext_copy);
 591		opts->suboptions = OPTION_MPTCP_DSS;
 592		ret = true;
 593	}
 594
 595	/* passive sockets msk will set the 'can_ack' after accept(), even
 596	 * if the first subflow may have the already the remote key handy
 597	 */
 598	opts->ext_copy.use_ack = 0;
 599	if (!READ_ONCE(msk->can_ack)) {
 600		*size = ALIGN(dss_size, 4);
 601		return ret;
 602	}
 603
 604	ack_seq = READ_ONCE(msk->ack_seq);
 605	if (READ_ONCE(msk->use_64bit_ack)) {
 606		ack_size = TCPOLEN_MPTCP_DSS_ACK64;
 607		opts->ext_copy.data_ack = ack_seq;
 608		opts->ext_copy.ack64 = 1;
 609	} else {
 610		ack_size = TCPOLEN_MPTCP_DSS_ACK32;
 611		opts->ext_copy.data_ack32 = (uint32_t)ack_seq;
 612		opts->ext_copy.ack64 = 0;
 613	}
 614	opts->ext_copy.use_ack = 1;
 615	opts->suboptions = OPTION_MPTCP_DSS;
 616
 617	/* Add kind/length/subtype/flag overhead if mapping is not populated */
 618	if (dss_size == 0)
 619		ack_size += TCPOLEN_MPTCP_DSS_BASE;
 620
 621	dss_size += ack_size;
 622
 623	*size = ALIGN(dss_size, 4);
 624	return true;
 625}
 626
 627static u64 add_addr_generate_hmac(u64 key1, u64 key2,
 628				  struct mptcp_addr_info *addr)
 629{
 630	u16 port = ntohs(addr->port);
 631	u8 hmac[SHA256_DIGEST_SIZE];
 632	u8 msg[19];
 633	int i = 0;
 634
 635	msg[i++] = addr->id;
 636	if (addr->family == AF_INET) {
 637		memcpy(&msg[i], &addr->addr.s_addr, 4);
 638		i += 4;
 639	}
 640#if IS_ENABLED(CONFIG_MPTCP_IPV6)
 641	else if (addr->family == AF_INET6) {
 642		memcpy(&msg[i], &addr->addr6.s6_addr, 16);
 643		i += 16;
 644	}
 645#endif
 646	msg[i++] = port >> 8;
 647	msg[i++] = port & 0xFF;
 648
 649	mptcp_crypto_hmac_sha(key1, key2, msg, i, hmac);
 650
 651	return get_unaligned_be64(&hmac[SHA256_DIGEST_SIZE - sizeof(u64)]);
 652}
 653
 654static bool mptcp_established_options_add_addr(struct sock *sk, struct sk_buff *skb,
 655					       unsigned int *size,
 656					       unsigned int remaining,
 657					       struct mptcp_out_options *opts)
 658{
 659	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 660	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 661	bool drop_other_suboptions = false;
 662	unsigned int opt_size = *size;
 663	struct mptcp_addr_info addr;
 664	bool echo;
 665	int len;
 666
 667	/* add addr will strip the existing options, be sure to avoid breaking
 668	 * MPC/MPJ handshakes
 669	 */
 670	if (!mptcp_pm_should_add_signal(msk) ||
 671	    (opts->suboptions & (OPTION_MPTCP_MPJ_ACK | OPTION_MPTCP_MPC_ACK)) ||
 672	    !mptcp_pm_add_addr_signal(msk, skb, opt_size, remaining, &addr,
 673		    &echo, &drop_other_suboptions))
 674		return false;
 675
 676	/*
 677	 * Later on, mptcp_write_options() will enforce mutually exclusion with
 678	 * DSS, bail out if such option is set and we can't drop it.
 679	 */
 680	if (drop_other_suboptions)
 681		remaining += opt_size;
 682	else if (opts->suboptions & OPTION_MPTCP_DSS)
 683		return false;
 684
 685	len = mptcp_add_addr_len(addr.family, echo, !!addr.port);
 686	if (remaining < len)
 687		return false;
 688
 689	*size = len;
 690	if (drop_other_suboptions) {
 691		pr_debug("drop other suboptions\n");
 692		opts->suboptions = 0;
 693
 694		/* note that e.g. DSS could have written into the memory
 695		 * aliased by ahmac, we must reset the field here
 696		 * to avoid appending the hmac even for ADD_ADDR echo
 697		 * options
 698		 */
 699		opts->ahmac = 0;
 700		*size -= opt_size;
 701	}
 702	opts->addr = addr;
 703	opts->suboptions |= OPTION_MPTCP_ADD_ADDR;
 704	if (!echo) {
 705		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ADDADDRTX);
 706		opts->ahmac = add_addr_generate_hmac(READ_ONCE(msk->local_key),
 707						     READ_ONCE(msk->remote_key),
 708						     &opts->addr);
 709	} else {
 710		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADDTX);
 711	}
 712	pr_debug("addr_id=%d, ahmac=%llu, echo=%d, port=%d\n",
 713		 opts->addr.id, opts->ahmac, echo, ntohs(opts->addr.port));
 714
 715	return true;
 716}
 717
 718static bool mptcp_established_options_rm_addr(struct sock *sk,
 719					      unsigned int *size,
 720					      unsigned int remaining,
 721					      struct mptcp_out_options *opts)
 722{
 723	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 724	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 725	struct mptcp_rm_list rm_list;
 726	int i, len;
 727
 728	if (!mptcp_pm_should_rm_signal(msk) ||
 729	    !(mptcp_pm_rm_addr_signal(msk, remaining, &rm_list)))
 730		return false;
 731
 732	len = mptcp_rm_addr_len(&rm_list);
 733	if (len < 0)
 734		return false;
 735	if (remaining < len)
 736		return false;
 737
 738	*size = len;
 739	opts->suboptions |= OPTION_MPTCP_RM_ADDR;
 740	opts->rm_list = rm_list;
 741
 742	for (i = 0; i < opts->rm_list.nr; i++)
 743		pr_debug("rm_list_ids[%d]=%d\n", i, opts->rm_list.ids[i]);
 744	MPTCP_ADD_STATS(sock_net(sk), MPTCP_MIB_RMADDRTX, opts->rm_list.nr);
 745	return true;
 746}
 747
 748static bool mptcp_established_options_mp_prio(struct sock *sk,
 749					      unsigned int *size,
 750					      unsigned int remaining,
 751					      struct mptcp_out_options *opts)
 752{
 753	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 754
 755	/* can't send MP_PRIO with MPC, as they share the same option space:
 756	 * 'backup'. Also it makes no sense at all
 757	 */
 758	if (!subflow->send_mp_prio || (opts->suboptions & OPTIONS_MPTCP_MPC))
 759		return false;
 760
 761	/* account for the trailing 'nop' option */
 762	if (remaining < TCPOLEN_MPTCP_PRIO_ALIGN)
 763		return false;
 764
 765	*size = TCPOLEN_MPTCP_PRIO_ALIGN;
 766	opts->suboptions |= OPTION_MPTCP_PRIO;
 767	opts->backup = subflow->request_bkup;
 768
 769	pr_debug("prio=%d\n", opts->backup);
 770
 771	return true;
 772}
 773
 774static noinline bool mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
 775						   unsigned int *size,
 776						   unsigned int remaining,
 777						   struct mptcp_out_options *opts)
 778{
 779	const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 780
 781	if (remaining < TCPOLEN_MPTCP_RST)
 782		return false;
 783
 784	*size = TCPOLEN_MPTCP_RST;
 785	opts->suboptions |= OPTION_MPTCP_RST;
 786	opts->reset_transient = subflow->reset_transient;
 787	opts->reset_reason = subflow->reset_reason;
 788	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPRSTTX);
 789
 790	return true;
 791}
 792
 793static bool mptcp_established_options_fastclose(struct sock *sk,
 794						unsigned int *size,
 795						unsigned int remaining,
 796						struct mptcp_out_options *opts)
 797{
 798	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 799	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 800
 801	if (likely(!subflow->send_fastclose))
 802		return false;
 803
 804	if (remaining < TCPOLEN_MPTCP_FASTCLOSE)
 805		return false;
 806
 807	*size = TCPOLEN_MPTCP_FASTCLOSE;
 808	opts->suboptions |= OPTION_MPTCP_FASTCLOSE;
 809	opts->rcvr_key = READ_ONCE(msk->remote_key);
 810
 811	pr_debug("FASTCLOSE key=%llu\n", opts->rcvr_key);
 812	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSETX);
 813	return true;
 814}
 815
 816static bool mptcp_established_options_mp_fail(struct sock *sk,
 817					      unsigned int *size,
 818					      unsigned int remaining,
 819					      struct mptcp_out_options *opts)
 820{
 821	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 822
 823	if (likely(!subflow->send_mp_fail))
 824		return false;
 825
 826	if (remaining < TCPOLEN_MPTCP_FAIL)
 827		return false;
 828
 829	*size = TCPOLEN_MPTCP_FAIL;
 830	opts->suboptions |= OPTION_MPTCP_FAIL;
 831	opts->fail_seq = subflow->map_seq;
 832
 833	pr_debug("MP_FAIL fail_seq=%llu\n", opts->fail_seq);
 834	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILTX);
 835
 836	return true;
 837}
 838
 839bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
 840			       unsigned int *size, unsigned int remaining,
 841			       struct mptcp_out_options *opts)
 842{
 843	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
 844	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
 845	unsigned int opt_size = 0;
 846	bool snd_data_fin;
 847	bool ret = false;
 848
 849	opts->suboptions = 0;
 850
 851	/* Force later mptcp_write_options(), but do not use any actual
 852	 * option space.
 853	 */
 854	if (unlikely(__mptcp_check_fallback(msk) && !mptcp_check_infinite_map(skb)))
 855		return true;
 856
 857	if (unlikely(skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST)) {
 858		if (mptcp_established_options_fastclose(sk, &opt_size, remaining, opts) ||
 859		    mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
 860			*size += opt_size;
 861			remaining -= opt_size;
 862		}
 863		/* MP_RST can be used with MP_FASTCLOSE and MP_FAIL if there is room */
 864		if (mptcp_established_options_rst(sk, skb, &opt_size, remaining, opts)) {
 865			*size += opt_size;
 866			remaining -= opt_size;
 867		}
 868		return true;
 869	}
 870
 871	snd_data_fin = mptcp_data_fin_enabled(msk);
 872	if (mptcp_established_options_mp(sk, skb, snd_data_fin, &opt_size, opts))
 873		ret = true;
 874	else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, opts)) {
 875		unsigned int mp_fail_size;
 876
 877		ret = true;
 878		if (mptcp_established_options_mp_fail(sk, &mp_fail_size,
 879						      remaining - opt_size, opts)) {
 880			*size += opt_size + mp_fail_size;
 881			remaining -= opt_size - mp_fail_size;
 882			return true;
 883		}
 884	}
 885
 886	/* we reserved enough space for the above options, and exceeding the
 887	 * TCP option space would be fatal
 888	 */
 889	if (WARN_ON_ONCE(opt_size > remaining))
 890		return false;
 891
 892	*size += opt_size;
 893	remaining -= opt_size;
 894	if (mptcp_established_options_add_addr(sk, skb, &opt_size, remaining, opts)) {
 895		*size += opt_size;
 896		remaining -= opt_size;
 897		ret = true;
 898	} else if (mptcp_established_options_rm_addr(sk, &opt_size, remaining, opts)) {
 899		*size += opt_size;
 900		remaining -= opt_size;
 901		ret = true;
 902	}
 903
 904	if (mptcp_established_options_mp_prio(sk, &opt_size, remaining, opts)) {
 905		*size += opt_size;
 906		remaining -= opt_size;
 907		ret = true;
 908	}
 909
 910	return ret;
 911}
 912
 913bool mptcp_synack_options(const struct request_sock *req, unsigned int *size,
 914			  struct mptcp_out_options *opts)
 915{
 916	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
 917
 918	if (subflow_req->mp_capable) {
 919		opts->suboptions = OPTION_MPTCP_MPC_SYNACK;
 920		opts->sndr_key = subflow_req->local_key;
 921		opts->csum_reqd = subflow_req->csum_reqd;
 922		opts->allow_join_id0 = subflow_req->allow_join_id0;
 923		*size = TCPOLEN_MPTCP_MPC_SYNACK;
 924		pr_debug("subflow_req=%p, local_key=%llu\n",
 925			 subflow_req, subflow_req->local_key);
 926		return true;
 927	} else if (subflow_req->mp_join) {
 928		opts->suboptions = OPTION_MPTCP_MPJ_SYNACK;
 929		opts->backup = subflow_req->request_bkup;
 930		opts->join_id = subflow_req->local_id;
 931		opts->thmac = subflow_req->thmac;
 932		opts->nonce = subflow_req->local_nonce;
 933		pr_debug("req=%p, bkup=%u, id=%u, thmac=%llu, nonce=%u\n",
 934			 subflow_req, opts->backup, opts->join_id,
 935			 opts->thmac, opts->nonce);
 936		*size = TCPOLEN_MPTCP_MPJ_SYNACK;
 937		return true;
 938	}
 939	return false;
 940}
 941
 942static bool check_fully_established(struct mptcp_sock *msk, struct sock *ssk,
 943				    struct mptcp_subflow_context *subflow,
 944				    struct sk_buff *skb,
 945				    struct mptcp_options_received *mp_opt)
 946{
 947	/* here we can process OoO, in-window pkts, only in-sequence 4th ack
 948	 * will make the subflow fully established
 949	 */
 950	if (likely(READ_ONCE(subflow->fully_established))) {
 951		/* on passive sockets, check for 3rd ack retransmission
 952		 * note that msk is always set by subflow_syn_recv_sock()
 953		 * for mp_join subflows
 954		 */
 955		if (TCP_SKB_CB(skb)->seq == subflow->ssn_offset + 1 &&
 956		    TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq &&
 957		    subflow->mp_join && (mp_opt->suboptions & OPTIONS_MPTCP_MPJ) &&
 958		    !subflow->request_join)
 959			tcp_send_ack(ssk);
 960		goto check_notify;
 961	}
 962
 963	/* we must process OoO packets before the first subflow is fully
 964	 * established. OoO packets are instead a protocol violation
 965	 * for MP_JOIN subflows as the peer must not send any data
 966	 * before receiving the forth ack - cfr. RFC 8684 section 3.2.
 967	 */
 968	if (TCP_SKB_CB(skb)->seq != subflow->ssn_offset + 1) {
 969		if (subflow->mp_join)
 970			goto reset;
 971		if (subflow->is_mptfo && mp_opt->suboptions & OPTION_MPTCP_MPC_ACK)
 972			goto set_fully_established;
 973		return subflow->mp_capable;
 974	}
 975
 976	if (subflow->remote_key_valid &&
 977	    (((mp_opt->suboptions & OPTION_MPTCP_DSS) && mp_opt->use_ack) ||
 978	     ((mp_opt->suboptions & OPTION_MPTCP_ADD_ADDR) &&
 979	      (!mp_opt->echo || subflow->mp_join)))) {
 980		/* subflows are fully established as soon as we get any
 981		 * additional ack, including ADD_ADDR.
 982		 */
 983		goto set_fully_established;
 984	}
 985
 986	/* If the first established packet does not contain MP_CAPABLE + data
 987	 * then fallback to TCP. Fallback scenarios requires a reset for
 988	 * MP_JOIN subflows.
 989	 */
 990	if (!(mp_opt->suboptions & OPTIONS_MPTCP_MPC)) {
 991		if (subflow->mp_join)
 992			goto reset;
 993		subflow->mp_capable = 0;
 994		if (!mptcp_try_fallback(ssk, MPTCP_MIB_MPCAPABLEDATAFALLBACK)) {
 995			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_FALLBACKFAILED);
 996			goto reset;
 997		}
 998		return false;
 999	}
1000
1001	if (unlikely(!READ_ONCE(msk->pm.server_side)))
1002		pr_warn_once("bogus mpc option on established client sk");
1003
1004set_fully_established:
1005	if (mp_opt->deny_join_id0)
1006		WRITE_ONCE(msk->pm.remote_deny_join_id0, true);
1007
1008	mptcp_data_lock((struct sock *)msk);
1009	__mptcp_subflow_fully_established(msk, subflow, mp_opt);
1010	mptcp_data_unlock((struct sock *)msk);
1011
1012check_notify:
1013	/* if the subflow is not already linked into the conn_list, we can't
1014	 * notify the PM: this subflow is still on the listener queue
1015	 * and the PM possibly acquiring the subflow lock could race with
1016	 * the listener close
1017	 */
1018	if (likely(subflow->pm_notified) || list_empty(&subflow->node))
1019		return true;
1020
1021	subflow->pm_notified = 1;
1022	if (subflow->mp_join) {
1023		clear_3rdack_retransmission(ssk);
1024		mptcp_pm_subflow_established(msk);
1025	} else {
1026		mptcp_pm_fully_established(msk, ssk);
1027	}
1028	return true;
1029
1030reset:
1031	mptcp_subflow_reset(ssk);
1032	return false;
1033}
1034
1035u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq)
1036{
1037	u32 old_seq32, cur_seq32;
1038
1039	old_seq32 = (u32)old_seq;
1040	cur_seq32 = (u32)cur_seq;
1041	cur_seq = (old_seq & GENMASK_ULL(63, 32)) + cur_seq32;
1042	if (unlikely(cur_seq32 < old_seq32 && before(old_seq32, cur_seq32)))
1043		return cur_seq + (1LL << 32);
1044
1045	/* reverse wrap could happen, too */
1046	if (unlikely(cur_seq32 > old_seq32 && after(old_seq32, cur_seq32)))
1047		return cur_seq - (1LL << 32);
1048	return cur_seq;
1049}
1050
1051static void __mptcp_snd_una_update(struct mptcp_sock *msk, u64 new_snd_una)
1052{
1053	msk->bytes_acked += new_snd_una - msk->snd_una;
1054	WRITE_ONCE(msk->snd_una, new_snd_una);
1055}
1056
1057static void rwin_update(struct mptcp_sock *msk, struct sock *ssk,
1058			struct sk_buff *skb)
1059{
1060	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1061	struct tcp_sock *tp = tcp_sk(ssk);
1062	u64 mptcp_rcv_wnd;
1063
1064	/* Avoid touching extra cachelines if TCP is going to accept this
1065	 * skb without filling the TCP-level window even with a possibly
1066	 * outdated mptcp-level rwin.
1067	 */
1068	if (!skb->len || skb->len < tcp_receive_window(tp))
1069		return;
1070
1071	mptcp_rcv_wnd = atomic64_read(&msk->rcv_wnd_sent);
1072	if (!after64(mptcp_rcv_wnd, subflow->rcv_wnd_sent))
1073		return;
1074
1075	/* Some other subflow grew the mptcp-level rwin since rcv_wup,
1076	 * resync.
1077	 */
1078	tp->rcv_wnd += mptcp_rcv_wnd - subflow->rcv_wnd_sent;
1079	subflow->rcv_wnd_sent = mptcp_rcv_wnd;
1080}
1081
1082static void ack_update_msk(struct mptcp_sock *msk,
1083			   struct sock *ssk,
1084			   struct mptcp_options_received *mp_opt)
1085{
1086	u64 new_wnd_end, new_snd_una, snd_nxt = READ_ONCE(msk->snd_nxt);
1087	struct sock *sk = (struct sock *)msk;
1088	u64 old_snd_una;
1089
1090	mptcp_data_lock(sk);
1091
1092	/* avoid ack expansion on update conflict, to reduce the risk of
1093	 * wrongly expanding to a future ack sequence number, which is way
1094	 * more dangerous than missing an ack
1095	 */
1096	old_snd_una = msk->snd_una;
1097	new_snd_una = mptcp_expand_seq(old_snd_una, mp_opt->data_ack, mp_opt->ack64);
1098
1099	/* ACK for data not even sent yet? Ignore.*/
1100	if (unlikely(after64(new_snd_una, snd_nxt)))
1101		new_snd_una = old_snd_una;
1102
1103	new_wnd_end = new_snd_una + tcp_sk(ssk)->snd_wnd;
1104
1105	if (after64(new_wnd_end, msk->wnd_end))
1106		WRITE_ONCE(msk->wnd_end, new_wnd_end);
1107
1108	/* this assumes mptcp_incoming_options() is invoked after tcp_ack() */
1109	if (after64(msk->wnd_end, snd_nxt))
1110		__mptcp_check_push(sk, ssk);
1111
1112	if (after64(new_snd_una, old_snd_una)) {
1113		__mptcp_snd_una_update(msk, new_snd_una);
1114		__mptcp_data_acked(sk);
1115	}
1116	msk->last_ack_recv = tcp_jiffies32;
1117	mptcp_data_unlock(sk);
1118
1119	trace_ack_update_msk(mp_opt->data_ack,
1120			     old_snd_una, new_snd_una,
1121			     new_wnd_end, READ_ONCE(msk->wnd_end));
1122}
1123
1124bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit)
1125{
1126	/* Skip if DATA_FIN was already received.
1127	 * If updating simultaneously with the recvmsg loop, values
1128	 * should match. If they mismatch, the peer is misbehaving and
1129	 * we will prefer the most recent information.
1130	 */
1131	if (READ_ONCE(msk->rcv_data_fin))
1132		return false;
1133
1134	WRITE_ONCE(msk->rcv_data_fin_seq,
1135		   mptcp_expand_seq(READ_ONCE(msk->ack_seq), data_fin_seq, use_64bit));
1136	WRITE_ONCE(msk->rcv_data_fin, 1);
1137
1138	return true;
1139}
1140
1141static bool add_addr_hmac_valid(struct mptcp_sock *msk,
1142				struct mptcp_options_received *mp_opt)
1143{
1144	u64 hmac = 0;
1145
1146	if (mp_opt->echo)
1147		return true;
1148
1149	hmac = add_addr_generate_hmac(READ_ONCE(msk->remote_key),
1150				      READ_ONCE(msk->local_key),
1151				      &mp_opt->addr);
1152
1153	pr_debug("msk=%p, ahmac=%llu, mp_opt->ahmac=%llu\n",
1154		 msk, hmac, mp_opt->ahmac);
1155
1156	return hmac == mp_opt->ahmac;
1157}
1158
1159/* Return false in case of error (or subflow has been reset),
1160 * else return true.
1161 */
1162bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
1163{
1164	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1165	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
1166	struct mptcp_options_received mp_opt;
1167	struct mptcp_ext *mpext;
1168
1169	if (__mptcp_check_fallback(msk)) {
1170		/* Keep it simple and unconditionally trigger send data cleanup and
1171		 * pending queue spooling. We will need to acquire the data lock
1172		 * for more accurate checks, and once the lock is acquired, such
1173		 * helpers are cheap.
1174		 */
1175		mptcp_data_lock(subflow->conn);
1176		if (sk_stream_memory_free(sk))
1177			__mptcp_check_push(subflow->conn, sk);
1178
1179		/* on fallback we just need to ignore the msk-level snd_una, as
1180		 * this is really plain TCP
1181		 */
1182		__mptcp_snd_una_update(msk, READ_ONCE(msk->snd_nxt));
1183
1184		__mptcp_data_acked(subflow->conn);
1185		mptcp_data_unlock(subflow->conn);
1186		return true;
1187	}
1188
1189	mptcp_get_options(skb, &mp_opt);
1190
1191	/* The subflow can be in close state only if check_fully_established()
1192	 * just sent a reset. If so, tell the caller to ignore the current packet.
1193	 */
1194	if (!check_fully_established(msk, sk, subflow, skb, &mp_opt))
1195		return sk->sk_state != TCP_CLOSE;
1196
1197	if (unlikely(mp_opt.suboptions != OPTION_MPTCP_DSS)) {
1198		if ((mp_opt.suboptions & OPTION_MPTCP_FASTCLOSE) &&
1199		    READ_ONCE(msk->local_key) == mp_opt.rcvr_key) {
1200			WRITE_ONCE(msk->rcv_fastclose, true);
1201			mptcp_schedule_work((struct sock *)msk);
1202			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSERX);
1203		}
1204
1205		if ((mp_opt.suboptions & OPTION_MPTCP_ADD_ADDR) &&
1206		    add_addr_hmac_valid(msk, &mp_opt)) {
1207			if (!mp_opt.echo) {
1208				mptcp_pm_add_addr_received(sk, &mp_opt.addr);
1209				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ADDADDR);
1210			} else {
1211				mptcp_pm_add_addr_echoed(msk, &mp_opt.addr);
1212				mptcp_pm_del_add_timer(msk, &mp_opt.addr, true);
1213				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADD);
1214			}
1215
1216			if (mp_opt.addr.port)
1217				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_PORTADD);
1218		}
1219
1220		if (mp_opt.suboptions & OPTION_MPTCP_RM_ADDR)
1221			mptcp_pm_rm_addr_received(msk, &mp_opt.rm_list);
1222
1223		if (mp_opt.suboptions & OPTION_MPTCP_PRIO) {
1224			mptcp_pm_mp_prio_received(sk, mp_opt.backup);
1225			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPPRIORX);
1226		}
1227
1228		if (mp_opt.suboptions & OPTION_MPTCP_FAIL) {
1229			mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
1230			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILRX);
1231		}
1232
1233		if (mp_opt.suboptions & OPTION_MPTCP_RST) {
1234			subflow->reset_seen = 1;
1235			subflow->reset_reason = mp_opt.reset_reason;
1236			subflow->reset_transient = mp_opt.reset_transient;
1237			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPRSTRX);
1238		}
1239
1240		if (!(mp_opt.suboptions & OPTION_MPTCP_DSS))
1241			return true;
1242	}
1243
1244	/* we can't wait for recvmsg() to update the ack_seq, otherwise
1245	 * monodirectional flows will stuck
1246	 */
1247	if (mp_opt.use_ack)
1248		ack_update_msk(msk, sk, &mp_opt);
1249	rwin_update(msk, sk, skb);
1250
1251	/* Zero-data-length packets are dropped by the caller and not
1252	 * propagated to the MPTCP layer, so the skb extension does not
1253	 * need to be allocated or populated. DATA_FIN information, if
1254	 * present, needs to be updated here before the skb is freed.
1255	 */
1256	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
1257		if (mp_opt.data_fin && mp_opt.data_len == 1 &&
1258		    mptcp_update_rcv_data_fin(msk, mp_opt.data_seq, mp_opt.dsn64))
1259			mptcp_schedule_work((struct sock *)msk);
1260
1261		return true;
1262	}
1263
1264	mpext = skb_ext_add(skb, SKB_EXT_MPTCP);
1265	if (!mpext)
1266		return false;
1267
1268	memset(mpext, 0, sizeof(*mpext));
1269
1270	if (likely(mp_opt.use_map)) {
1271		if (mp_opt.mpc_map) {
1272			/* this is an MP_CAPABLE carrying MPTCP data
1273			 * we know this map the first chunk of data
1274			 */
1275			mptcp_crypto_key_sha(subflow->remote_key, NULL,
1276					     &mpext->data_seq);
1277			mpext->data_seq++;
1278			mpext->subflow_seq = 1;
1279			mpext->dsn64 = 1;
1280			mpext->mpc_map = 1;
1281			mpext->data_fin = 0;
1282		} else {
1283			mpext->data_seq = mp_opt.data_seq;
1284			mpext->subflow_seq = mp_opt.subflow_seq;
1285			mpext->dsn64 = mp_opt.dsn64;
1286			mpext->data_fin = mp_opt.data_fin;
1287		}
1288		mpext->data_len = mp_opt.data_len;
1289		mpext->use_map = 1;
1290		mpext->csum_reqd = !!(mp_opt.suboptions & OPTION_MPTCP_CSUMREQD);
1291
1292		if (mpext->csum_reqd)
1293			mpext->csum = mp_opt.csum;
1294	}
1295
1296	return true;
1297}
1298
1299static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th)
1300{
1301	const struct sock *ssk = (const struct sock *)tp;
1302	struct mptcp_subflow_context *subflow;
1303	u64 ack_seq, rcv_wnd_old, rcv_wnd_new;
1304	struct mptcp_sock *msk;
1305	u32 new_win;
1306	u64 win;
1307
1308	subflow = mptcp_subflow_ctx(ssk);
1309	msk = mptcp_sk(subflow->conn);
1310
1311	ack_seq = READ_ONCE(msk->ack_seq);
1312	rcv_wnd_new = ack_seq + tp->rcv_wnd;
1313
1314	rcv_wnd_old = atomic64_read(&msk->rcv_wnd_sent);
1315	if (after64(rcv_wnd_new, rcv_wnd_old)) {
1316		u64 rcv_wnd;
1317
1318		for (;;) {
1319			rcv_wnd = atomic64_cmpxchg(&msk->rcv_wnd_sent, rcv_wnd_old, rcv_wnd_new);
1320
1321			if (rcv_wnd == rcv_wnd_old)
1322				break;
1323
1324			rcv_wnd_old = rcv_wnd;
1325			if (before64(rcv_wnd_new, rcv_wnd_old)) {
1326				MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDCONFLICTUPDATE);
1327				goto raise_win;
1328			}
1329			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDCONFLICT);
1330		}
1331		goto update_wspace;
1332	}
1333
1334	if (rcv_wnd_new != rcv_wnd_old) {
1335raise_win:
1336		/* The msk-level rcv wnd is after the tcp level one,
1337		 * sync the latter.
1338		 */
1339		rcv_wnd_new = rcv_wnd_old;
1340		win = rcv_wnd_old - ack_seq;
1341		tp->rcv_wnd = min_t(u64, win, U32_MAX);
1342		new_win = tp->rcv_wnd;
1343
1344		/* Make sure we do not exceed the maximum possible
1345		 * scaled window.
1346		 */
1347		if (unlikely(th->syn))
1348			new_win = min(new_win, 65535U) << tp->rx_opt.rcv_wscale;
1349		if (!tp->rx_opt.rcv_wscale &&
1350		    READ_ONCE(sock_net(ssk)->ipv4.sysctl_tcp_workaround_signed_windows))
1351			new_win = min(new_win, MAX_TCP_WINDOW);
1352		else
1353			new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
1354
1355		/* RFC1323 scaling applied */
1356		new_win >>= tp->rx_opt.rcv_wscale;
1357		th->window = htons(new_win);
1358		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDSHARED);
1359	}
1360
1361update_wspace:
1362	WRITE_ONCE(msk->old_wspace, tp->rcv_wnd);
1363	subflow->rcv_wnd_sent = rcv_wnd_new;
1364}
1365
1366static void mptcp_track_rwin(struct tcp_sock *tp)
1367{
1368	const struct sock *ssk = (const struct sock *)tp;
1369	struct mptcp_subflow_context *subflow;
1370	struct mptcp_sock *msk;
1371
1372	if (!ssk)
1373		return;
1374
1375	subflow = mptcp_subflow_ctx(ssk);
1376	msk = mptcp_sk(subflow->conn);
1377	WRITE_ONCE(msk->old_wspace, tp->rcv_wnd);
1378}
1379
1380__sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum)
1381{
1382	struct csum_pseudo_header header;
1383	__wsum csum;
1384
1385	/* cfr RFC 8684 3.3.1.:
1386	 * the data sequence number used in the pseudo-header is
1387	 * always the 64-bit value, irrespective of what length is used in the
1388	 * DSS option itself.
1389	 */
1390	header.data_seq = cpu_to_be64(data_seq);
1391	header.subflow_seq = htonl(subflow_seq);
1392	header.data_len = htons(data_len);
1393	header.csum = 0;
1394
1395	csum = csum_partial(&header, sizeof(header), sum);
1396	return csum_fold(csum);
1397}
1398
1399static __sum16 mptcp_make_csum(const struct mptcp_ext *mpext)
1400{
1401	return __mptcp_make_csum(mpext->data_seq, mpext->subflow_seq, mpext->data_len,
1402				 ~csum_unfold(mpext->csum));
1403}
1404
1405static void put_len_csum(u16 len, __sum16 csum, void *data)
1406{
1407	__sum16 *sumptr = data + 2;
1408	__be16 *ptr = data;
1409
1410	put_unaligned_be16(len, ptr);
1411
1412	put_unaligned(csum, sumptr);
1413}
1414
1415void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp,
1416			 struct mptcp_out_options *opts)
1417{
1418	const struct sock *ssk = (const struct sock *)tp;
1419	struct mptcp_subflow_context *subflow;
1420
1421	/* Which options can be used together?
1422	 *
1423	 * X: mutually exclusive
1424	 * O: often used together
1425	 * C: can be used together in some cases
1426	 * P: could be used together but we prefer not to (optimisations)
1427	 *
1428	 *  Opt: | MPC  | MPJ  | DSS  | ADD  |  RM  | PRIO | FAIL |  FC  |
1429	 * ------|------|------|------|------|------|------|------|------|
1430	 *  MPC  |------|------|------|------|------|------|------|------|
1431	 *  MPJ  |  X   |------|------|------|------|------|------|------|
1432	 *  DSS  |  X   |  X   |------|------|------|------|------|------|
1433	 *  ADD  |  X   |  X   |  P   |------|------|------|------|------|
1434	 *  RM   |  C   |  C   |  C   |  P   |------|------|------|------|
1435	 *  PRIO |  X   |  C   |  C   |  C   |  C   |------|------|------|
1436	 *  FAIL |  X   |  X   |  C   |  X   |  X   |  X   |------|------|
1437	 *  FC   |  X   |  X   |  X   |  X   |  X   |  X   |  X   |------|
1438	 *  RST  |  X   |  X   |  X   |  X   |  X   |  X   |  O   |  O   |
1439	 * ------|------|------|------|------|------|------|------|------|
1440	 *
1441	 * The same applies in mptcp_established_options() function.
1442	 */
1443	if (likely(OPTION_MPTCP_DSS & opts->suboptions)) {
1444		struct mptcp_ext *mpext = &opts->ext_copy;
1445		u8 len = TCPOLEN_MPTCP_DSS_BASE;
1446		u8 flags = 0;
1447
1448		if (mpext->use_ack) {
1449			flags = MPTCP_DSS_HAS_ACK;
1450			if (mpext->ack64) {
1451				len += TCPOLEN_MPTCP_DSS_ACK64;
1452				flags |= MPTCP_DSS_ACK64;
1453			} else {
1454				len += TCPOLEN_MPTCP_DSS_ACK32;
1455			}
1456		}
1457
1458		if (mpext->use_map) {
1459			len += TCPOLEN_MPTCP_DSS_MAP64;
1460
1461			/* Use only 64-bit mapping flags for now, add
1462			 * support for optional 32-bit mappings later.
1463			 */
1464			flags |= MPTCP_DSS_HAS_MAP | MPTCP_DSS_DSN64;
1465			if (mpext->data_fin)
1466				flags |= MPTCP_DSS_DATA_FIN;
1467
1468			if (opts->csum_reqd)
1469				len += TCPOLEN_MPTCP_DSS_CHECKSUM;
1470		}
1471
1472		*ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);
1473
1474		if (mpext->use_ack) {
1475			if (mpext->ack64) {
1476				put_unaligned_be64(mpext->data_ack, ptr);
1477				ptr += 2;
1478			} else {
1479				put_unaligned_be32(mpext->data_ack32, ptr);
1480				ptr += 1;
1481			}
1482		}
1483
1484		if (mpext->use_map) {
1485			put_unaligned_be64(mpext->data_seq, ptr);
1486			ptr += 2;
1487			put_unaligned_be32(mpext->subflow_seq, ptr);
1488			ptr += 1;
1489			if (opts->csum_reqd) {
1490				/* data_len == 0 is reserved for the infinite mapping,
1491				 * the checksum will also be set to 0.
1492				 */
1493				put_len_csum(mpext->data_len,
1494					     (mpext->data_len ? mptcp_make_csum(mpext) : 0),
1495					     ptr);
1496			} else {
1497				put_unaligned_be32(mpext->data_len << 16 |
1498						   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
1499			}
1500			ptr += 1;
1501		}
1502
1503		/* We might need to add MP_FAIL options in rare cases */
1504		if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions))
1505			goto mp_fail;
1506	} else if (OPTIONS_MPTCP_MPC & opts->suboptions) {
1507		u8 len, flag = MPTCP_CAP_HMAC_SHA256;
1508
1509		if (OPTION_MPTCP_MPC_SYN & opts->suboptions) {
1510			len = TCPOLEN_MPTCP_MPC_SYN;
1511		} else if (OPTION_MPTCP_MPC_SYNACK & opts->suboptions) {
1512			len = TCPOLEN_MPTCP_MPC_SYNACK;
1513		} else if (opts->data_len) {
1514			len = TCPOLEN_MPTCP_MPC_ACK_DATA;
1515			if (opts->csum_reqd)
1516				len += TCPOLEN_MPTCP_DSS_CHECKSUM;
1517		} else {
1518			len = TCPOLEN_MPTCP_MPC_ACK;
1519		}
1520
1521		if (opts->csum_reqd)
1522			flag |= MPTCP_CAP_CHECKSUM_REQD;
1523
1524		if (!opts->allow_join_id0)
1525			flag |= MPTCP_CAP_DENY_JOIN_ID0;
1526
1527		*ptr++ = mptcp_option(MPTCPOPT_MP_CAPABLE, len,
1528				      MPTCP_SUPPORTED_VERSION,
1529				      flag);
1530
1531		if (!((OPTION_MPTCP_MPC_SYNACK | OPTION_MPTCP_MPC_ACK) &
1532		    opts->suboptions))
1533			goto mp_capable_done;
1534
1535		put_unaligned_be64(opts->sndr_key, ptr);
1536		ptr += 2;
1537		if (!((OPTION_MPTCP_MPC_ACK) & opts->suboptions))
1538			goto mp_capable_done;
1539
1540		put_unaligned_be64(opts->rcvr_key, ptr);
1541		ptr += 2;
1542		if (!opts->data_len)
1543			goto mp_capable_done;
1544
1545		if (opts->csum_reqd) {
1546			put_len_csum(opts->data_len,
1547				     __mptcp_make_csum(opts->data_seq,
1548						       opts->subflow_seq,
1549						       opts->data_len,
1550						       ~csum_unfold(opts->csum)),
1551				     ptr);
1552		} else {
1553			put_unaligned_be32(opts->data_len << 16 |
1554					   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
1555		}
1556		ptr += 1;
1557
1558		/* MPC is additionally mutually exclusive with MP_PRIO */
1559		goto mp_capable_done;
1560	} else if (OPTIONS_MPTCP_MPJ & opts->suboptions) {
1561		if (OPTION_MPTCP_MPJ_SYN & opts->suboptions) {
1562			*ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
1563					      TCPOLEN_MPTCP_MPJ_SYN,
1564					      opts->backup, opts->join_id);
1565			put_unaligned_be32(opts->token, ptr);
1566			ptr += 1;
1567			put_unaligned_be32(opts->nonce, ptr);
1568			ptr += 1;
1569		} else if (OPTION_MPTCP_MPJ_SYNACK & opts->suboptions) {
1570			*ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
1571					      TCPOLEN_MPTCP_MPJ_SYNACK,
1572					      opts->backup, opts->join_id);
1573			put_unaligned_be64(opts->thmac, ptr);
1574			ptr += 2;
1575			put_unaligned_be32(opts->nonce, ptr);
1576			ptr += 1;
1577		} else {
1578			*ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
1579					      TCPOLEN_MPTCP_MPJ_ACK, 0, 0);
1580			memcpy(ptr, opts->hmac, MPTCPOPT_HMAC_LEN);
1581			ptr += 5;
1582		}
1583	} else if (OPTION_MPTCP_ADD_ADDR & opts->suboptions) {
1584		u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE;
1585		u8 echo = MPTCP_ADDR_ECHO;
1586
1587#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1588		if (opts->addr.family == AF_INET6)
1589			len = TCPOLEN_MPTCP_ADD_ADDR6_BASE;
1590#endif
1591
1592		if (opts->addr.port)
1593			len += TCPOLEN_MPTCP_PORT_LEN;
1594
1595		if (opts->ahmac) {
1596			len += sizeof(opts->ahmac);
1597			echo = 0;
1598		}
1599
1600		*ptr++ = mptcp_option(MPTCPOPT_ADD_ADDR,
1601				      len, echo, opts->addr.id);
1602		if (opts->addr.family == AF_INET) {
1603			memcpy((u8 *)ptr, (u8 *)&opts->addr.addr.s_addr, 4);
1604			ptr += 1;
1605		}
1606#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1607		else if (opts->addr.family == AF_INET6) {
1608			memcpy((u8 *)ptr, opts->addr.addr6.s6_addr, 16);
1609			ptr += 4;
1610		}
1611#endif
1612
1613		if (!opts->addr.port) {
1614			if (opts->ahmac) {
1615				put_unaligned_be64(opts->ahmac, ptr);
1616				ptr += 2;
1617			}
1618		} else {
1619			u16 port = ntohs(opts->addr.port);
1620
1621			if (opts->ahmac) {
1622				u8 *bptr = (u8 *)ptr;
1623
1624				put_unaligned_be16(port, bptr);
1625				bptr += 2;
1626				put_unaligned_be64(opts->ahmac, bptr);
1627				bptr += 8;
1628				put_unaligned_be16(TCPOPT_NOP << 8 |
1629						   TCPOPT_NOP, bptr);
1630
1631				ptr += 3;
1632			} else {
1633				put_unaligned_be32(port << 16 |
1634						   TCPOPT_NOP << 8 |
1635						   TCPOPT_NOP, ptr);
1636				ptr += 1;
1637			}
1638		}
1639	} else if (unlikely(OPTION_MPTCP_FASTCLOSE & opts->suboptions)) {
1640		/* FASTCLOSE is mutually exclusive with others except RST */
1641		*ptr++ = mptcp_option(MPTCPOPT_MP_FASTCLOSE,
1642				      TCPOLEN_MPTCP_FASTCLOSE,
1643				      0, 0);
1644		put_unaligned_be64(opts->rcvr_key, ptr);
1645		ptr += 2;
1646
1647		if (OPTION_MPTCP_RST & opts->suboptions)
1648			goto mp_rst;
1649		return;
1650	} else if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions)) {
1651mp_fail:
1652		/* MP_FAIL is mutually exclusive with others except RST */
1653		subflow = mptcp_subflow_ctx(ssk);
1654		subflow->send_mp_fail = 0;
1655
1656		*ptr++ = mptcp_option(MPTCPOPT_MP_FAIL,
1657				      TCPOLEN_MPTCP_FAIL,
1658				      0, 0);
1659		put_unaligned_be64(opts->fail_seq, ptr);
1660		ptr += 2;
1661
1662		if (OPTION_MPTCP_RST & opts->suboptions)
1663			goto mp_rst;
1664		return;
1665	} else if (unlikely(OPTION_MPTCP_RST & opts->suboptions)) {
1666mp_rst:
1667		*ptr++ = mptcp_option(MPTCPOPT_RST,
1668				      TCPOLEN_MPTCP_RST,
1669				      opts->reset_transient,
1670				      opts->reset_reason);
1671		return;
1672	} else if (unlikely(!opts->suboptions)) {
1673		/* Fallback to TCP */
1674		mptcp_track_rwin(tp);
1675		return;
1676	}
1677
1678	if (OPTION_MPTCP_PRIO & opts->suboptions) {
1679		subflow = mptcp_subflow_ctx(ssk);
1680		subflow->send_mp_prio = 0;
1681
1682		*ptr++ = mptcp_option(MPTCPOPT_MP_PRIO,
1683				      TCPOLEN_MPTCP_PRIO,
1684				      opts->backup, TCPOPT_NOP);
1685
1686		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPPRIOTX);
1687	}
1688
1689mp_capable_done:
1690	if (OPTION_MPTCP_RM_ADDR & opts->suboptions) {
1691		u8 i = 1;
1692
1693		*ptr++ = mptcp_option(MPTCPOPT_RM_ADDR,
1694				      TCPOLEN_MPTCP_RM_ADDR_BASE + opts->rm_list.nr,
1695				      0, opts->rm_list.ids[0]);
1696
1697		while (i < opts->rm_list.nr) {
1698			u8 id1, id2, id3, id4;
1699
1700			id1 = opts->rm_list.ids[i];
1701			id2 = i + 1 < opts->rm_list.nr ? opts->rm_list.ids[i + 1] : TCPOPT_NOP;
1702			id3 = i + 2 < opts->rm_list.nr ? opts->rm_list.ids[i + 2] : TCPOPT_NOP;
1703			id4 = i + 3 < opts->rm_list.nr ? opts->rm_list.ids[i + 3] : TCPOPT_NOP;
1704			put_unaligned_be32(id1 << 24 | id2 << 16 | id3 << 8 | id4, ptr);
1705			ptr += 1;
1706			i += 4;
1707		}
1708	}
1709
1710	if (tp)
1711		mptcp_set_rwin(tp, th);
1712}
1713
1714__be32 mptcp_get_reset_option(const struct sk_buff *skb)
1715{
1716	const struct mptcp_ext *ext = mptcp_get_ext(skb);
1717	u8 flags, reason;
1718
1719	if (ext) {
1720		flags = ext->reset_transient;
1721		reason = ext->reset_reason;
1722
1723		return mptcp_option(MPTCPOPT_RST, TCPOLEN_MPTCP_RST,
1724				    flags, reason);
1725	}
1726
1727	return htonl(0u);
1728}
1729EXPORT_SYMBOL_GPL(mptcp_get_reset_option);