bpf_cc_cubic.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2
  3/* Highlights:
  4 * 1. The major difference between this bpf program and tcp_cubic.c
  5 *    is that this bpf program relies on `cong_control` rather than
  6 *    `cong_avoid` in the struct tcp_congestion_ops.
  7 * 2. Logic such as tcp_cwnd_reduction, tcp_cong_avoid, and
  8 *    tcp_update_pacing_rate is bypassed when `cong_control` is
  9 *    defined, so moving these logic to `cong_control`.
 10 * 3. WARNING: This bpf program is NOT the same as tcp_cubic.c.
 11 *    The main purpose is to show use cases of the arguments in
 12 *    `cong_control`. For simplicity's sake, it reuses tcp cubic's
 13 *    kernel functions.
 14 */
 15
 16#include "bpf_tracing_net.h"
 17#include <bpf/bpf_helpers.h>
 18#include <bpf/bpf_tracing.h>
 19
 20#define USEC_PER_SEC 1000000UL
 21#define TCP_PACING_SS_RATIO (200)
 22#define TCP_PACING_CA_RATIO (120)
 23#define TCP_REORDERING (12)
 24
 25extern void cubictcp_init(struct sock *sk) __ksym;
 26extern void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event) __ksym;
 27extern __u32 cubictcp_recalc_ssthresh(struct sock *sk) __ksym;
 28extern void cubictcp_state(struct sock *sk, __u8 new_state) __ksym;
 29extern __u32 tcp_reno_undo_cwnd(struct sock *sk) __ksym;
 30extern void cubictcp_acked(struct sock *sk, const struct ack_sample *sample) __ksym;
 31extern void cubictcp_cong_avoid(struct sock *sk, __u32 ack, __u32 acked) __ksym;
 32
 33static __u64 div64_u64(__u64 dividend, __u64 divisor)
 34{
 35	return dividend / divisor;
 36}
 37
 38static void tcp_update_pacing_rate(struct sock *sk)
 39{
 40	const struct tcp_sock *tp = tcp_sk(sk);
 41	__u64 rate;
 42
 43	/* set sk_pacing_rate to 200 % of current rate (mss * cwnd / srtt) */
 44	rate = (__u64)tp->mss_cache * ((USEC_PER_SEC / 100) << 3);
 45
 46	/* current rate is (cwnd * mss) / srtt
 47	 * In Slow Start [1], set sk_pacing_rate to 200 % the current rate.
 48	 * In Congestion Avoidance phase, set it to 120 % the current rate.
 49	 *
 50	 * [1] : Normal Slow Start condition is (tp->snd_cwnd < tp->snd_ssthresh)
 51	 *	 If snd_cwnd >= (tp->snd_ssthresh / 2), we are approaching
 52	 *	 end of slow start and should slow down.
 53	 */
 54	if (tp->snd_cwnd < tp->snd_ssthresh / 2)
 55		rate *= TCP_PACING_SS_RATIO;
 56	else
 57		rate *= TCP_PACING_CA_RATIO;
 58
 59	rate *= max(tp->snd_cwnd, tp->packets_out);
 60
 61	if (tp->srtt_us)
 62		rate = div64_u64(rate, (__u64)tp->srtt_us);
 63
 64	sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate);
 65}
 66
 67static void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked,
 68			       int newly_lost, int flag)
 69{
 70	struct tcp_sock *tp = tcp_sk(sk);
 71	int sndcnt = 0;
 72	__u32 pkts_in_flight = tp->packets_out - (tp->sacked_out + tp->lost_out) + tp->retrans_out;
 73	int delta = tp->snd_ssthresh - pkts_in_flight;
 74
 75	if (newly_acked_sacked <= 0 || !tp->prior_cwnd)
 76		return;
 77
 78	__u32 prr_delivered = tp->prr_delivered + newly_acked_sacked;
 79
 80	if (delta < 0) {
 81		__u64 dividend =
 82			(__u64)tp->snd_ssthresh * prr_delivered + tp->prior_cwnd - 1;
 83		sndcnt = (__u32)div64_u64(dividend, (__u64)tp->prior_cwnd) - tp->prr_out;
 84	} else {
 85		sndcnt = max(prr_delivered - tp->prr_out, newly_acked_sacked);
 86		if (flag & FLAG_SND_UNA_ADVANCED && !newly_lost)
 87			sndcnt++;
 88		sndcnt = min(delta, sndcnt);
 89	}
 90	/* Force a fast retransmit upon entering fast recovery */
 91	sndcnt = max(sndcnt, (tp->prr_out ? 0 : 1));
 92	tp->snd_cwnd = pkts_in_flight + sndcnt;
 93}
 94
 95/* Decide whether to run the increase function of congestion control. */
 96static bool tcp_may_raise_cwnd(const struct sock *sk, const int flag)
 97{
 98	if (tcp_sk(sk)->reordering > TCP_REORDERING)
 99		return flag & FLAG_FORWARD_PROGRESS;
100
101	return flag & FLAG_DATA_ACKED;
102}
103
104SEC("struct_ops")
105void BPF_PROG(bpf_cubic_init, struct sock *sk)
106{
107	cubictcp_init(sk);
108}
109
110SEC("struct_ops")
111void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
112{
113	cubictcp_cwnd_event(sk, event);
114}
115
116SEC("struct_ops")
117void BPF_PROG(bpf_cubic_cong_control, struct sock *sk, __u32 ack, int flag,
118	      const struct rate_sample *rs)
119{
120	struct tcp_sock *tp = tcp_sk(sk);
121
122	if (((1<<TCP_CA_CWR) | (1<<TCP_CA_Recovery)) &
123			(1 << inet_csk(sk)->icsk_ca_state)) {
124		/* Reduce cwnd if state mandates */
125		tcp_cwnd_reduction(sk, rs->acked_sacked, rs->losses, flag);
126
127		if (!before(tp->snd_una, tp->high_seq)) {
128			/* Reset cwnd to ssthresh in CWR or Recovery (unless it's undone) */
129			if (tp->snd_ssthresh < TCP_INFINITE_SSTHRESH &&
130					inet_csk(sk)->icsk_ca_state == TCP_CA_CWR) {
131				tp->snd_cwnd = tp->snd_ssthresh;
132				tp->snd_cwnd_stamp = tcp_jiffies32;
133			}
134		}
135	} else if (tcp_may_raise_cwnd(sk, flag)) {
136		/* Advance cwnd if state allows */
137		cubictcp_cong_avoid(sk, ack, rs->acked_sacked);
138		tp->snd_cwnd_stamp = tcp_jiffies32;
139	}
140
141	tcp_update_pacing_rate(sk);
142}
143
144SEC("struct_ops")
145__u32 BPF_PROG(bpf_cubic_recalc_ssthresh, struct sock *sk)
146{
147	return cubictcp_recalc_ssthresh(sk);
148}
149
150SEC("struct_ops")
151void BPF_PROG(bpf_cubic_state, struct sock *sk, __u8 new_state)
152{
153	cubictcp_state(sk, new_state);
154}
155
156SEC("struct_ops")
157void BPF_PROG(bpf_cubic_acked, struct sock *sk, const struct ack_sample *sample)
158{
159	cubictcp_acked(sk, sample);
160}
161
162SEC("struct_ops")
163__u32 BPF_PROG(bpf_cubic_undo_cwnd, struct sock *sk)
164{
165	return tcp_reno_undo_cwnd(sk);
166}
167
168SEC(".struct_ops")
169struct tcp_congestion_ops cc_cubic = {
170	.init		= (void *)bpf_cubic_init,
171	.ssthresh	= (void *)bpf_cubic_recalc_ssthresh,
172	.cong_control	= (void *)bpf_cubic_cong_control,
173	.set_state	= (void *)bpf_cubic_state,
174	.undo_cwnd	= (void *)bpf_cubic_undo_cwnd,
175	.cwnd_event	= (void *)bpf_cubic_cwnd_event,
176	.pkts_acked     = (void *)bpf_cubic_acked,
177	.name		= "bpf_cc_cubic",
178};
179
180char _license[] SEC("license") = "GPL";