tcp_ecn.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2#ifndef _TCP_ECN_H
  3#define _TCP_ECN_H
  4
  5#include <linux/tcp.h>
  6#include <linux/skbuff.h>
  7#include <linux/bitfield.h>
  8
  9#include <net/inet_connection_sock.h>
 10#include <net/sock.h>
 11#include <net/tcp.h>
 12#include <net/inet_ecn.h>
 13
 14/* The highest ECN variant (Accurate ECN, ECN, or no ECN) that is
 15 * attemped to be negotiated and requested for incoming connection
 16 * and outgoing connection, respectively.
 17 */
 18enum tcp_ecn_mode {
 19	TCP_ECN_IN_NOECN_OUT_NOECN = 0,
 20	TCP_ECN_IN_ECN_OUT_ECN = 1,
 21	TCP_ECN_IN_ECN_OUT_NOECN = 2,
 22	TCP_ECN_IN_ACCECN_OUT_ACCECN = 3,
 23	TCP_ECN_IN_ACCECN_OUT_ECN = 4,
 24	TCP_ECN_IN_ACCECN_OUT_NOECN = 5,
 25};
 26
 27/* AccECN option sending when AccECN has been successfully negotiated */
 28enum tcp_accecn_option {
 29	TCP_ACCECN_OPTION_DISABLED = 0,
 30	TCP_ACCECN_OPTION_MINIMUM = 1,
 31	TCP_ACCECN_OPTION_FULL = 2,
 32};
 33
 34/* Apply either ECT(0) or ECT(1) based on TCP_CONG_ECT_1_NEGOTIATION flag */
 35static inline void INET_ECN_xmit_ect_1_negotiation(struct sock *sk)
 36{
 37	__INET_ECN_xmit(sk, tcp_ca_ect_1_negotiation(sk));
 38}
 39
 40static inline void tcp_ecn_queue_cwr(struct tcp_sock *tp)
 41{
 42	/* Do not set CWR if in AccECN mode! */
 43	if (tcp_ecn_mode_rfc3168(tp))
 44		tp->ecn_flags |= TCP_ECN_QUEUE_CWR;
 45}
 46
 47static inline void tcp_ecn_accept_cwr(struct sock *sk,
 48				      const struct sk_buff *skb)
 49{
 50	struct tcp_sock *tp = tcp_sk(sk);
 51
 52	if (tcp_ecn_mode_rfc3168(tp) && tcp_hdr(skb)->cwr) {
 53		tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
 54
 55		/* If the sender is telling us it has entered CWR, then its
 56		 * cwnd may be very low (even just 1 packet), so we should ACK
 57		 * immediately.
 58		 */
 59		if (TCP_SKB_CB(skb)->seq != TCP_SKB_CB(skb)->end_seq)
 60			inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
 61	}
 62}
 63
 64static inline void tcp_ecn_withdraw_cwr(struct tcp_sock *tp)
 65{
 66	tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
 67}
 68
 69/* tp->accecn_fail_mode */
 70#define TCP_ACCECN_ACE_FAIL_SEND	BIT(0)
 71#define TCP_ACCECN_ACE_FAIL_RECV	BIT(1)
 72#define TCP_ACCECN_OPT_FAIL_SEND	BIT(2)
 73#define TCP_ACCECN_OPT_FAIL_RECV	BIT(3)
 74
 75static inline bool tcp_accecn_ace_fail_send(const struct tcp_sock *tp)
 76{
 77	return tp->accecn_fail_mode & TCP_ACCECN_ACE_FAIL_SEND;
 78}
 79
 80static inline bool tcp_accecn_ace_fail_recv(const struct tcp_sock *tp)
 81{
 82	return tp->accecn_fail_mode & TCP_ACCECN_ACE_FAIL_RECV;
 83}
 84
 85static inline bool tcp_accecn_opt_fail_send(const struct tcp_sock *tp)
 86{
 87	return tp->accecn_fail_mode & TCP_ACCECN_OPT_FAIL_SEND;
 88}
 89
 90static inline bool tcp_accecn_opt_fail_recv(const struct tcp_sock *tp)
 91{
 92	return tp->accecn_fail_mode & TCP_ACCECN_OPT_FAIL_RECV;
 93}
 94
 95static inline void tcp_accecn_fail_mode_set(struct tcp_sock *tp, u8 mode)
 96{
 97	tp->accecn_fail_mode |= mode;
 98}
 99
100#define TCP_ACCECN_OPT_NOT_SEEN		0x0
101#define TCP_ACCECN_OPT_EMPTY_SEEN	0x1
102#define TCP_ACCECN_OPT_COUNTER_SEEN	0x2
103#define TCP_ACCECN_OPT_FAIL_SEEN	0x3
104
105static inline u8 tcp_accecn_ace(const struct tcphdr *th)
106{
107	return (th->ae << 2) | (th->cwr << 1) | th->ece;
108}
109
110/* Infer the ECT value our SYN arrived with from the echoed ACE field */
111static inline int tcp_accecn_extract_syn_ect(u8 ace)
112{
113	/* Below is an excerpt from the 1st block of Table 2 of AccECN spec */
114	static const int ace_to_ecn[8] = {
115		INET_ECN_ECT_0,		/* 0b000 (Undefined) */
116		INET_ECN_ECT_1,		/* 0b001 (Undefined) */
117		INET_ECN_NOT_ECT,	/* 0b010 (Not-ECT is received) */
118		INET_ECN_ECT_1,		/* 0b011 (ECT-1 is received) */
119		INET_ECN_ECT_0,		/* 0b100 (ECT-0 is received) */
120		INET_ECN_ECT_1,		/* 0b101 (Reserved) */
121		INET_ECN_CE,		/* 0b110 (CE is received) */
122		INET_ECN_ECT_1		/* 0b111 (Undefined) */
123	};
124
125	return ace_to_ecn[ace & 0x7];
126}
127
128/* Check ECN field transition to detect invalid transitions */
129static inline bool tcp_ect_transition_valid(u8 snt, u8 rcv)
130{
131	if (rcv == snt)
132		return true;
133
134	/* Non-ECT altered to something or something became non-ECT */
135	if (snt == INET_ECN_NOT_ECT || rcv == INET_ECN_NOT_ECT)
136		return false;
137	/* CE -> ECT(0/1)? */
138	if (snt == INET_ECN_CE)
139		return false;
140	return true;
141}
142
143static inline bool tcp_accecn_validate_syn_feedback(struct sock *sk, u8 ace,
144						    u8 sent_ect)
145{
146	u8 ect = tcp_accecn_extract_syn_ect(ace);
147	struct tcp_sock *tp = tcp_sk(sk);
148
149	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback))
150		return true;
151
152	if (!tcp_ect_transition_valid(sent_ect, ect)) {
153		tcp_accecn_fail_mode_set(tp, TCP_ACCECN_ACE_FAIL_RECV);
154		return false;
155	}
156
157	return true;
158}
159
160static inline void tcp_accecn_saw_opt_fail_recv(struct tcp_sock *tp,
161						u8 saw_opt)
162{
163	tp->saw_accecn_opt = saw_opt;
164	if (tp->saw_accecn_opt == TCP_ACCECN_OPT_FAIL_SEEN)
165		tcp_accecn_fail_mode_set(tp, TCP_ACCECN_OPT_FAIL_RECV);
166}
167
168/* Validate the 3rd ACK based on the ACE field, see Table 4 of AccECN spec */
169static inline void tcp_accecn_third_ack(struct sock *sk,
170					const struct sk_buff *skb, u8 sent_ect)
171{
172	u8 ace = tcp_accecn_ace(tcp_hdr(skb));
173	struct tcp_sock *tp = tcp_sk(sk);
174
175	switch (ace) {
176	case 0x0:
177		/* Invalid value */
178		tcp_accecn_fail_mode_set(tp, TCP_ACCECN_ACE_FAIL_RECV);
179		break;
180	case 0x7:
181	case 0x5:
182	case 0x1:
183		/* Unused but legal values */
184		break;
185	default:
186		/* Validation only applies to first non-data packet */
187		if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq &&
188		    !TCP_SKB_CB(skb)->sacked &&
189		    tcp_accecn_validate_syn_feedback(sk, ace, sent_ect)) {
190			if ((tcp_accecn_extract_syn_ect(ace) == INET_ECN_CE) &&
191			    !tp->delivered_ce)
192				tp->delivered_ce++;
193		}
194		break;
195	}
196}
197
198/* Demand the minimum # to send AccECN optnio */
199static inline void tcp_accecn_opt_demand_min(struct sock *sk,
200					     u8 opt_demand_min)
201{
202	struct tcp_sock *tp = tcp_sk(sk);
203	u8 opt_demand;
204
205	opt_demand = max_t(u8, opt_demand_min, tp->accecn_opt_demand);
206	tp->accecn_opt_demand = opt_demand;
207}
208
209/* Maps IP ECN field ECT/CE code point to AccECN option field number, given
210 * we are sending fields with Accurate ECN Order 1: ECT(1), CE, ECT(0).
211 */
212static inline u8 tcp_ecnfield_to_accecn_optfield(u8 ecnfield)
213{
214	switch (ecnfield & INET_ECN_MASK) {
215	case INET_ECN_NOT_ECT:
216		return 0;	/* AccECN does not send counts of NOT_ECT */
217	case INET_ECN_ECT_1:
218		return 1;
219	case INET_ECN_CE:
220		return 2;
221	case INET_ECN_ECT_0:
222		return 3;
223	}
224	return 0;
225}
226
227/* Maps IP ECN field ECT/CE code point to AccECN option field value offset.
228 * Some fields do not start from zero, to detect zeroing by middleboxes.
229 */
230static inline u32 tcp_accecn_field_init_offset(u8 ecnfield)
231{
232	switch (ecnfield & INET_ECN_MASK) {
233	case INET_ECN_NOT_ECT:
234		return 0;	/* AccECN does not send counts of NOT_ECT */
235	case INET_ECN_ECT_1:
236		return TCP_ACCECN_E1B_INIT_OFFSET;
237	case INET_ECN_CE:
238		return TCP_ACCECN_CEB_INIT_OFFSET;
239	case INET_ECN_ECT_0:
240		return TCP_ACCECN_E0B_INIT_OFFSET;
241	}
242	return 0;
243}
244
245/* Maps AccECN option field #nr to IP ECN field ECT/CE bits */
246static inline unsigned int tcp_accecn_optfield_to_ecnfield(unsigned int option,
247							   bool order)
248{
249	/* Based on Table 5 of the AccECN spec to map (option, order) to
250	 * the corresponding ECN conuters (ECT-1, ECT-0, or CE).
251	 */
252	static const u8 optfield_lookup[2][3] = {
253		/* order = 0: 1st field ECT-0, 2nd field CE, 3rd field ECT-1 */
254		{ INET_ECN_ECT_0, INET_ECN_CE, INET_ECN_ECT_1 },
255		/* order = 1: 1st field ECT-1, 2nd field CE, 3rd field ECT-0 */
256		{ INET_ECN_ECT_1, INET_ECN_CE, INET_ECN_ECT_0 }
257	};
258
259	return optfield_lookup[order][option % 3];
260}
261
262/* Handles AccECN option ECT and CE 24-bit byte counters update into
263 * the u32 value in tcp_sock. As we're processing TCP options, it is
264 * safe to access from - 1.
265 */
266static inline s32 tcp_update_ecn_bytes(u32 *cnt, const char *from,
267				       u32 init_offset)
268{
269	u32 truncated = (get_unaligned_be32(from - 1) - init_offset) &
270			0xFFFFFFU;
271	u32 delta = (truncated - *cnt) & 0xFFFFFFU;
272
273	/* If delta has the highest bit set (24th bit) indicating
274	 * negative, sign extend to correct an estimation using
275	 * sign_extend32(delta, 24 - 1)
276	 */
277	delta = sign_extend32(delta, 23);
278	*cnt += delta;
279	return (s32)delta;
280}
281
282/* Updates Accurate ECN received counters from the received IP ECN field */
283static inline void tcp_ecn_received_counters(struct sock *sk,
284					     const struct sk_buff *skb, u32 len)
285{
286	u8 ecnfield = TCP_SKB_CB(skb)->ip_dsfield & INET_ECN_MASK;
287	u8 is_ce = INET_ECN_is_ce(ecnfield);
288	struct tcp_sock *tp = tcp_sk(sk);
289	bool ecn_edge;
290
291	if (!INET_ECN_is_not_ect(ecnfield)) {
292		u32 pcount = is_ce * max_t(u16, 1, skb_shinfo(skb)->gso_segs);
293
294		/* As for accurate ECN, the TCP_ECN_SEEN flag is set by
295		 * tcp_ecn_received_counters() when the ECN codepoint of
296		 * received TCP data or ACK contains ECT(0), ECT(1), or CE.
297		 */
298		if (!tcp_ecn_mode_rfc3168(tp))
299			tp->ecn_flags |= TCP_ECN_SEEN;
300
301		/* ACE counter tracks *all* segments including pure ACKs */
302		tp->received_ce += pcount;
303		tp->received_ce_pending = min(tp->received_ce_pending + pcount,
304					      0xfU);
305
306		if (len > 0) {
307			u8 minlen = tcp_ecnfield_to_accecn_optfield(ecnfield);
308			u32 oldbytes = tp->received_ecn_bytes[ecnfield - 1];
309			u32 bytes_mask = GENMASK_U32(31, 22);
310
311			tp->received_ecn_bytes[ecnfield - 1] += len;
312			tp->accecn_minlen = max_t(u8, tp->accecn_minlen,
313						  minlen);
314
315			/* Send AccECN option at least once per 2^22-byte
316			 * increase in any ECN byte counter.
317			 */
318			if ((tp->received_ecn_bytes[ecnfield - 1] ^ oldbytes) &
319			    bytes_mask) {
320				tcp_accecn_opt_demand_min(sk, 1);
321			}
322		}
323	}
324
325	ecn_edge = tp->prev_ecnfield != ecnfield;
326	if (ecn_edge || is_ce) {
327		tp->prev_ecnfield = ecnfield;
328		/* Demand Accurate ECN change-triggered ACKs. Two ACK are
329		 * demanded to indicate unambiguously the ecnfield value
330		 * in the latter ACK.
331		 */
332		if (tcp_ecn_mode_accecn(tp)) {
333			if (ecn_edge)
334				inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
335			tp->accecn_opt_demand = 2;
336		}
337	}
338}
339
340/* AccECN specification, 2.2: [...] A Data Receiver maintains four counters
341 * initialized at the start of	the half-connection. [...] These byte counters
342 * reflect only the TCP payload length, excluding TCP header and TCP options.
343 */
344static inline void tcp_ecn_received_counters_payload(struct sock *sk,
345						     const struct sk_buff *skb)
346{
347	const struct tcphdr *th = (const struct tcphdr *)skb->data;
348
349	tcp_ecn_received_counters(sk, skb, skb->len - th->doff * 4);
350}
351
352/* AccECN specification, 5.1: [...] a server can determine that it
353 * negotiated AccECN as [...] if the ACK contains an ACE field with
354 * the value 0b010 to 0b111 (decimal 2 to 7).
355 */
356static inline bool cookie_accecn_ok(const struct tcphdr *th)
357{
358	return tcp_accecn_ace(th) > 0x1;
359}
360
361/* Used to form the ACE flags for SYN/ACK */
362static inline u16 tcp_accecn_reflector_flags(u8 ect)
363{
364	/* TCP ACE flags of SYN/ACK are set based on IP-ECN received from SYN.
365	 * Below is an excerpt from the 1st block of Table 2 of AccECN spec,
366	 * in which TCP ACE flags are encoded as: (AE << 2) | (CWR << 1) | ECE
367	 */
368	static const u8 ecn_to_ace_flags[4] = {
369		0b010,	/* Not-ECT is received */
370		0b011,	/* ECT(1) is received */
371		0b100,	/* ECT(0) is received */
372		0b110	/* CE is received */
373	};
374
375	return FIELD_PREP(TCPHDR_ACE, ecn_to_ace_flags[ect & 0x3]);
376}
377
378/* AccECN specification, 3.1.2: If a TCP server that implements AccECN
379 * receives a SYN with the three TCP header flags (AE, CWR and ECE) set
380 * to any combination other than 000, 011 or 111, it MUST negotiate the
381 * use of AccECN as if they had been set to 111.
382 */
383static inline bool tcp_accecn_syn_requested(const struct tcphdr *th)
384{
385	u8 ace = tcp_accecn_ace(th);
386
387	return ace && ace != 0x3;
388}
389
390static inline void __tcp_accecn_init_bytes_counters(int *counter_array)
391{
392	BUILD_BUG_ON(INET_ECN_ECT_1 != 0x1);
393	BUILD_BUG_ON(INET_ECN_ECT_0 != 0x2);
394	BUILD_BUG_ON(INET_ECN_CE != 0x3);
395
396	counter_array[INET_ECN_ECT_1 - 1] = 0;
397	counter_array[INET_ECN_ECT_0 - 1] = 0;
398	counter_array[INET_ECN_CE - 1] = 0;
399}
400
401static inline void tcp_accecn_init_counters(struct tcp_sock *tp)
402{
403	tp->received_ce = 0;
404	tp->received_ce_pending = 0;
405	__tcp_accecn_init_bytes_counters(tp->received_ecn_bytes);
406	__tcp_accecn_init_bytes_counters(tp->delivered_ecn_bytes);
407	tp->accecn_minlen = 0;
408	tp->accecn_opt_demand = 0;
409	tp->est_ecnfield = 0;
410}
411
412/* Used for make_synack to form the ACE flags */
413static inline void tcp_accecn_echo_syn_ect(struct tcphdr *th, u8 ect)
414{
415	/* TCP ACE flags of SYN/ACK are set based on IP-ECN codepoint received
416	 * from SYN. Below is an excerpt from Table 2 of the AccECN spec:
417	 * +====================+====================================+
418	 * |  IP-ECN codepoint  |  Respective ACE falgs on SYN/ACK   |
419	 * |   received on SYN  |       AE       CWR       ECE       |
420	 * +====================+====================================+
421	 * |      Not-ECT       |       0         1         0        |
422	 * |      ECT(1)        |       0         1         1        |
423	 * |      ECT(0)        |       1         0         0        |
424	 * |        CE          |       1         1         0        |
425	 * +====================+====================================+
426	 */
427	th->ae = !!(ect & INET_ECN_ECT_0);
428	th->cwr = ect != INET_ECN_ECT_0;
429	th->ece = ect == INET_ECN_ECT_1;
430}
431
432static inline void tcp_accecn_set_ace(struct tcp_sock *tp, struct sk_buff *skb,
433				      struct tcphdr *th)
434{
435	u32 wire_ace;
436
437	/* The final packet of the 3WHS or anything like it must reflect
438	 * the SYN/ACK ECT instead of putting CEP into ACE field, such
439	 * case show up in tcp_flags.
440	 */
441	if (likely(!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACE))) {
442		wire_ace = tp->received_ce + TCP_ACCECN_CEP_INIT_OFFSET;
443		th->ece = !!(wire_ace & 0x1);
444		th->cwr = !!(wire_ace & 0x2);
445		th->ae = !!(wire_ace & 0x4);
446		tp->received_ce_pending = 0;
447	}
448}
449
450static inline u8 tcp_accecn_option_init(const struct sk_buff *skb,
451					u8 opt_offset)
452{
453	u8 *ptr = skb_transport_header(skb) + opt_offset;
454	unsigned int optlen = ptr[1] - 2;
455
456	if (WARN_ON_ONCE(ptr[0] != TCPOPT_ACCECN0 && ptr[0] != TCPOPT_ACCECN1))
457		return TCP_ACCECN_OPT_FAIL_SEEN;
458	ptr += 2;
459
460	/* Detect option zeroing: an AccECN connection "MAY check that the
461	 * initial value of the EE0B field or the EE1B field is non-zero"
462	 */
463	if (optlen < TCPOLEN_ACCECN_PERFIELD)
464		return TCP_ACCECN_OPT_EMPTY_SEEN;
465	if (get_unaligned_be24(ptr) == 0)
466		return TCP_ACCECN_OPT_FAIL_SEEN;
467	if (optlen < TCPOLEN_ACCECN_PERFIELD * 3)
468		return TCP_ACCECN_OPT_COUNTER_SEEN;
469	ptr += TCPOLEN_ACCECN_PERFIELD * 2;
470	if (get_unaligned_be24(ptr) == 0)
471		return TCP_ACCECN_OPT_FAIL_SEEN;
472
473	return TCP_ACCECN_OPT_COUNTER_SEEN;
474}
475
476static inline void tcp_ecn_rcv_synack_accecn(struct sock *sk,
477					     const struct sk_buff *skb, u8 dsf)
478{
479	struct tcp_sock *tp = tcp_sk(sk);
480
481	tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN);
482	tp->syn_ect_rcv = dsf & INET_ECN_MASK;
483	/* Demand Accurate ECN option in response to the SYN on the SYN/ACK
484	 * and the TCP server will try to send one more packet with an AccECN
485	 * Option at a later point during the connection.
486	 */
487	if (tp->rx_opt.accecn &&
488	    tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) {
489		u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn);
490
491		tcp_accecn_saw_opt_fail_recv(tp, saw_opt);
492		tp->accecn_opt_demand = 2;
493	}
494}
495
496/* See Table 2 of the AccECN draft */
497static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb,
498				      const struct tcphdr *th, u8 ip_dsfield)
499{
500	struct tcp_sock *tp = tcp_sk(sk);
501	u8 ace = tcp_accecn_ace(th);
502
503	switch (ace) {
504	case 0x0:
505	case 0x7:
506		/* +========+========+============+=============+
507		 * | A      | B      |  SYN/ACK   |  Feedback   |
508		 * |        |        |    B->A    |  Mode of A  |
509		 * |        |        | AE CWR ECE |             |
510		 * +========+========+============+=============+
511		 * | AccECN | No ECN | 0   0   0  |   Not ECN   |
512		 * | AccECN | Broken | 1   1   1  |   Not ECN   |
513		 * +========+========+============+=============+
514		 */
515		tcp_ecn_mode_set(tp, TCP_ECN_DISABLED);
516		break;
517	case 0x1:
518		/* +========+========+============+=============+
519		 * | A      | B      |  SYN/ACK   |  Feedback   |
520		 * |        |        |    B->A    |  Mode of A  |
521		 * |        |        | AE CWR ECE |             |
522		 * +========+========+============+=============+
523		 * | AccECN | ECN    | 0   0   1  | Classic ECN |
524		 * | Nonce  | AccECN | 0   0   1  | Classic ECN |
525		 * | ECN    | AccECN | 0   0   1  | Classic ECN |
526		 * +========+========+============+=============+
527		 */
528		if (tcp_ca_no_fallback_rfc3168(sk))
529			tcp_ecn_mode_set(tp, TCP_ECN_DISABLED);
530		else
531			tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168);
532		break;
533	case 0x5:
534		if (tcp_ecn_mode_pending(tp)) {
535			tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield);
536			if (INET_ECN_is_ce(ip_dsfield)) {
537				tp->received_ce++;
538				tp->received_ce_pending++;
539			}
540		}
541		break;
542	default:
543		tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield);
544		if (INET_ECN_is_ce(ip_dsfield) &&
545		    tcp_accecn_validate_syn_feedback(sk, ace,
546						     tp->syn_ect_snt)) {
547			tp->received_ce++;
548			tp->received_ce_pending++;
549		}
550		break;
551	}
552}
553
554static inline void tcp_ecn_rcv_syn(struct sock *sk, const struct tcphdr *th,
555				   const struct sk_buff *skb)
556{
557	struct tcp_sock *tp = tcp_sk(sk);
558
559	if (tcp_ecn_mode_pending(tp)) {
560		if (!tcp_accecn_syn_requested(th)) {
561			/* Downgrade to classic ECN feedback */
562			tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168);
563		} else {
564			tp->syn_ect_rcv = TCP_SKB_CB(skb)->ip_dsfield &
565					  INET_ECN_MASK;
566			tp->prev_ecnfield = tp->syn_ect_rcv;
567			tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN);
568		}
569	}
570	if (tcp_ecn_mode_rfc3168(tp) &&
571	    (!th->ece || !th->cwr || tcp_ca_no_fallback_rfc3168(sk)))
572		tcp_ecn_mode_set(tp, TCP_ECN_DISABLED);
573}
574
575static inline bool tcp_ecn_rcv_ecn_echo(const struct tcp_sock *tp,
576					const struct tcphdr *th)
577{
578	if (th->ece && !th->syn && tcp_ecn_mode_rfc3168(tp))
579		return true;
580	return false;
581}
582
583/* Packet ECN state for a SYN-ACK */
584static inline void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
585{
586	struct tcp_sock *tp = tcp_sk(sk);
587
588	TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
589	if (tcp_ecn_disabled(tp))
590		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
591	else if (tcp_ca_needs_ecn(sk) ||
592		 tcp_bpf_ca_needs_ecn(sk))
593		INET_ECN_xmit_ect_1_negotiation(sk);
594
595	if (tp->ecn_flags & TCP_ECN_MODE_ACCECN) {
596		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE;
597		TCP_SKB_CB(skb)->tcp_flags |=
598			tcp_accecn_reflector_flags(tp->syn_ect_rcv);
599		tp->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK;
600	}
601}
602
603/* Packet ECN state for a SYN.  */
604static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
605{
606	struct tcp_sock *tp = tcp_sk(sk);
607	bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
608	bool use_ecn, use_accecn;
609	u8 tcp_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn);
610
611	use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN ||
612		     tcp_ca_needs_accecn(sk);
613	use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN ||
614		  tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN ||
615		  tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn;
616
617	if (!use_ecn) {
618		const struct dst_entry *dst = __sk_dst_get(sk);
619
620		if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
621			use_ecn = true;
622	}
623
624	tp->ecn_flags = 0;
625
626	if (use_ecn) {
627		if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
628			INET_ECN_xmit_ect_1_negotiation(sk);
629
630		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
631		if (use_accecn) {
632			TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_AE;
633			tcp_ecn_mode_set(tp, TCP_ECN_MODE_PENDING);
634			tp->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK;
635		} else {
636			tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168);
637		}
638	}
639}
640
641static inline void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
642{
643	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback)) {
644		/* tp->ecn_flags are cleared at a later point in time when
645		 * SYN ACK is ultimatively being received.
646		 */
647		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE;
648	}
649}
650
651static inline void
652tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
653{
654	if (tcp_rsk(req)->accecn_ok)
655		tcp_accecn_echo_syn_ect(th, tcp_rsk(req)->syn_ect_rcv);
656	else if (inet_rsk(req)->ecn_ok)
657		th->ece = 1;
658}
659
660static inline bool tcp_accecn_option_beacon_check(const struct sock *sk)
661{
662	u32 ecn_beacon = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_option_beacon);
663	const struct tcp_sock *tp = tcp_sk(sk);
664
665	if (!ecn_beacon)
666		return false;
667
668	return tcp_stamp_us_delta(tp->tcp_mstamp, tp->accecn_opt_tstamp) * ecn_beacon >=
669	       (tp->srtt_us >> 3);
670}
671
672#endif /* _LINUX_TCP_ECN_H */