gen_estimator.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * net/sched/gen_estimator.c	Simple rate estimator.
  4 *
  5 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  6 *		Eric Dumazet <edumazet@google.com>
  7 *
  8 * Changes:
  9 *              Jamal Hadi Salim - moved it to net/core and reshulfed
 10 *              names to make it usable in general net subsystem.
 11 */
 12
 13#include <linux/uaccess.h>
 14#include <linux/bitops.h>
 15#include <linux/module.h>
 16#include <linux/types.h>
 17#include <linux/kernel.h>
 18#include <linux/jiffies.h>
 19#include <linux/string.h>
 20#include <linux/mm.h>
 21#include <linux/socket.h>
 22#include <linux/sockios.h>
 23#include <linux/in.h>
 24#include <linux/errno.h>
 25#include <linux/interrupt.h>
 26#include <linux/netdevice.h>
 27#include <linux/skbuff.h>
 28#include <linux/rtnetlink.h>
 29#include <linux/init.h>
 30#include <linux/slab.h>
 31#include <linux/seqlock.h>
 32#include <net/sock.h>
 33#include <net/gen_stats.h>
 34
 35/* This code is NOT intended to be used for statistics collection,
 36 * its purpose is to provide a base for statistical multiplexing
 37 * for controlled load service.
 38 * If you need only statistics, run a user level daemon which
 39 * periodically reads byte counters.
 40 */
 41
 42struct net_rate_estimator {
 43	struct gnet_stats_basic_sync	*bstats;
 44	spinlock_t		*stats_lock;
 45	bool			running;
 46	struct gnet_stats_basic_sync __percpu *cpu_bstats;
 47	u8			ewma_log;
 48	u8			intvl_log; /* period : (250ms << intvl_log) */
 49
 50	seqcount_t		seq;
 51	u64			last_packets;
 52	u64			last_bytes;
 53
 54	u64			avpps;
 55	u64			avbps;
 56
 57	unsigned long           next_jiffies;
 58	struct timer_list       timer;
 59	struct rcu_head		rcu;
 60};
 61
 62static void est_fetch_counters(struct net_rate_estimator *e,
 63			       struct gnet_stats_basic_sync *b)
 64{
 65	gnet_stats_basic_sync_init(b);
 66	if (e->stats_lock)
 67		spin_lock(e->stats_lock);
 68
 69	gnet_stats_add_basic(b, e->cpu_bstats, e->bstats, e->running);
 70
 71	if (e->stats_lock)
 72		spin_unlock(e->stats_lock);
 73
 74}
 75
 76static void est_timer(struct timer_list *t)
 77{
 78	struct net_rate_estimator *est = timer_container_of(est, t, timer);
 79	struct gnet_stats_basic_sync b;
 80	u64 b_bytes, b_packets;
 81	u64 rate, brate;
 82
 83	est_fetch_counters(est, &b);
 84	b_bytes = u64_stats_read(&b.bytes);
 85	b_packets = u64_stats_read(&b.packets);
 86
 87	brate = (b_bytes - est->last_bytes) << (10 - est->intvl_log);
 88	brate = (brate >> est->ewma_log) - (est->avbps >> est->ewma_log);
 89
 90	rate = (b_packets - est->last_packets) << (10 - est->intvl_log);
 91	rate = (rate >> est->ewma_log) - (est->avpps >> est->ewma_log);
 92
 93	preempt_disable_nested();
 94	write_seqcount_begin(&est->seq);
 95	est->avbps += brate;
 96	est->avpps += rate;
 97	write_seqcount_end(&est->seq);
 98	preempt_enable_nested();
 99
100	est->last_bytes = b_bytes;
101	est->last_packets = b_packets;
102
103	est->next_jiffies += ((HZ/4) << est->intvl_log);
104
105	if (unlikely(time_after_eq(jiffies, est->next_jiffies))) {
106		/* Ouch... timer was delayed. */
107		est->next_jiffies = jiffies + 1;
108	}
109	mod_timer(&est->timer, est->next_jiffies);
110}
111
112/**
113 * gen_new_estimator - create a new rate estimator
114 * @bstats: basic statistics
115 * @cpu_bstats: bstats per cpu
116 * @rate_est: rate estimator statistics
117 * @lock: lock for statistics and control path
118 * @running: true if @bstats represents a running qdisc, thus @bstats'
119 *           internal values might change during basic reads. Only used
120 *           if @bstats_cpu is NULL
121 * @opt: rate estimator configuration TLV
122 *
123 * Creates a new rate estimator with &bstats as source and &rate_est
124 * as destination. A new timer with the interval specified in the
125 * configuration TLV is created. Upon each interval, the latest statistics
126 * will be read from &bstats and the estimated rate will be stored in
127 * &rate_est with the statistics lock grabbed during this period.
128 *
129 * Returns 0 on success or a negative error code.
130 *
131 */
132int gen_new_estimator(struct gnet_stats_basic_sync *bstats,
133		      struct gnet_stats_basic_sync __percpu *cpu_bstats,
134		      struct net_rate_estimator __rcu **rate_est,
135		      spinlock_t *lock,
136		      bool running,
137		      struct nlattr *opt)
138{
139	struct gnet_estimator *parm = nla_data(opt);
140	struct net_rate_estimator *old, *est;
141	struct gnet_stats_basic_sync b;
142	int intvl_log;
143
144	if (nla_len(opt) < sizeof(*parm))
145		return -EINVAL;
146
147	/* allowed timer periods are :
148	 * -2 : 250ms,   -1 : 500ms,    0 : 1 sec
149	 *  1 : 2 sec,    2 : 4 sec,    3 : 8 sec
150	 */
151	if (parm->interval < -2 || parm->interval > 3)
152		return -EINVAL;
153
154	if (parm->ewma_log == 0 || parm->ewma_log >= 31)
155		return -EINVAL;
156
157	est = kzalloc(sizeof(*est), GFP_KERNEL);
158	if (!est)
159		return -ENOBUFS;
160
161	seqcount_init(&est->seq);
162	intvl_log = parm->interval + 2;
163	est->bstats = bstats;
164	est->stats_lock = lock;
165	est->running  = running;
166	est->ewma_log = parm->ewma_log;
167	est->intvl_log = intvl_log;
168	est->cpu_bstats = cpu_bstats;
169
170	if (lock)
171		local_bh_disable();
172	est_fetch_counters(est, &b);
173	if (lock)
174		local_bh_enable();
175	est->last_bytes = u64_stats_read(&b.bytes);
176	est->last_packets = u64_stats_read(&b.packets);
177
178	if (lock)
179		spin_lock_bh(lock);
180	old = rcu_dereference_protected(*rate_est, 1);
181	if (old) {
182		timer_delete_sync(&old->timer);
183		est->avbps = old->avbps;
184		est->avpps = old->avpps;
185	}
186
187	est->next_jiffies = jiffies + ((HZ/4) << intvl_log);
188	timer_setup(&est->timer, est_timer, 0);
189	mod_timer(&est->timer, est->next_jiffies);
190
191	rcu_assign_pointer(*rate_est, est);
192	if (lock)
193		spin_unlock_bh(lock);
194	if (old)
195		kfree_rcu(old, rcu);
196	return 0;
197}
198EXPORT_SYMBOL(gen_new_estimator);
199
200/**
201 * gen_kill_estimator - remove a rate estimator
202 * @rate_est: rate estimator
203 *
204 * Removes the rate estimator.
205 *
206 */
207void gen_kill_estimator(struct net_rate_estimator __rcu **rate_est)
208{
209	struct net_rate_estimator *est;
210
211	est = unrcu_pointer(xchg(rate_est, NULL));
212	if (est) {
213		timer_shutdown_sync(&est->timer);
214		kfree_rcu(est, rcu);
215	}
216}
217EXPORT_SYMBOL(gen_kill_estimator);
218
219/**
220 * gen_replace_estimator - replace rate estimator configuration
221 * @bstats: basic statistics
222 * @cpu_bstats: bstats per cpu
223 * @rate_est: rate estimator statistics
224 * @lock: lock for statistics and control path
225 * @running: true if @bstats represents a running qdisc, thus @bstats'
226 *           internal values might change during basic reads. Only used
227 *           if @cpu_bstats is NULL
228 * @opt: rate estimator configuration TLV
229 *
230 * Replaces the configuration of a rate estimator by calling
231 * gen_kill_estimator() and gen_new_estimator().
232 *
233 * Returns 0 on success or a negative error code.
234 */
235int gen_replace_estimator(struct gnet_stats_basic_sync *bstats,
236			  struct gnet_stats_basic_sync __percpu *cpu_bstats,
237			  struct net_rate_estimator __rcu **rate_est,
238			  spinlock_t *lock,
239			  bool running, struct nlattr *opt)
240{
241	return gen_new_estimator(bstats, cpu_bstats, rate_est,
242				 lock, running, opt);
243}
244EXPORT_SYMBOL(gen_replace_estimator);
245
246/**
247 * gen_estimator_active - test if estimator is currently in use
248 * @rate_est: rate estimator
249 *
250 * Returns true if estimator is active, and false if not.
251 */
252bool gen_estimator_active(struct net_rate_estimator __rcu **rate_est)
253{
254	return !!rcu_access_pointer(*rate_est);
255}
256EXPORT_SYMBOL(gen_estimator_active);
257
258bool gen_estimator_read(struct net_rate_estimator __rcu **rate_est,
259			struct gnet_stats_rate_est64 *sample)
260{
261	struct net_rate_estimator *est;
262	unsigned seq;
263
264	rcu_read_lock();
265	est = rcu_dereference(*rate_est);
266	if (!est) {
267		rcu_read_unlock();
268		return false;
269	}
270
271	do {
272		seq = read_seqcount_begin(&est->seq);
273		sample->bps = est->avbps >> 8;
274		sample->pps = est->avpps >> 8;
275	} while (read_seqcount_retry(&est->seq, seq));
276
277	rcu_read_unlock();
278	return true;
279}
280EXPORT_SYMBOL(gen_estimator_read);