nft_counter.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
  4 *
  5 * Development of this code funded by Astaro AG (http://www.astaro.com/)
  6 */
  7
  8#include <linux/kernel.h>
  9#include <linux/init.h>
 10#include <linux/module.h>
 11#include <linux/u64_stats_sync.h>
 12#include <linux/netlink.h>
 13#include <linux/netfilter.h>
 14#include <linux/netfilter/nf_tables.h>
 15#include <net/netfilter/nf_tables.h>
 16#include <net/netfilter/nf_tables_core.h>
 17#include <net/netfilter/nf_tables_offload.h>
 18
 19struct nft_counter {
 20	u64_stats_t	bytes;
 21	u64_stats_t	packets;
 22};
 23
 24struct nft_counter_tot {
 25	s64		bytes;
 26	s64		packets;
 27};
 28
 29struct nft_counter_percpu_priv {
 30	struct nft_counter __percpu *counter;
 31};
 32
 33static DEFINE_PER_CPU(struct u64_stats_sync, nft_counter_sync);
 34
 35/* control plane only: sync fetch+reset */
 36static DEFINE_SPINLOCK(nft_counter_lock);
 37
 38static inline void nft_counter_do_eval(struct nft_counter_percpu_priv *priv,
 39				       struct nft_regs *regs,
 40				       const struct nft_pktinfo *pkt)
 41{
 42	struct u64_stats_sync *nft_sync;
 43	struct nft_counter *this_cpu;
 44
 45	local_bh_disable();
 46	this_cpu = this_cpu_ptr(priv->counter);
 47	nft_sync = this_cpu_ptr(&nft_counter_sync);
 48
 49	u64_stats_update_begin(nft_sync);
 50	u64_stats_add(&this_cpu->bytes, pkt->skb->len);
 51	u64_stats_inc(&this_cpu->packets);
 52	u64_stats_update_end(nft_sync);
 53
 54	local_bh_enable();
 55}
 56
 57static inline void nft_counter_obj_eval(struct nft_object *obj,
 58					struct nft_regs *regs,
 59					const struct nft_pktinfo *pkt)
 60{
 61	struct nft_counter_percpu_priv *priv = nft_obj_data(obj);
 62
 63	nft_counter_do_eval(priv, regs, pkt);
 64}
 65
 66static int nft_counter_do_init(const struct nlattr * const tb[],
 67			       struct nft_counter_percpu_priv *priv)
 68{
 69	struct nft_counter __percpu *cpu_stats;
 70	struct nft_counter *this_cpu;
 71
 72	cpu_stats = alloc_percpu_gfp(struct nft_counter, GFP_KERNEL_ACCOUNT);
 73	if (cpu_stats == NULL)
 74		return -ENOMEM;
 75
 76	this_cpu = raw_cpu_ptr(cpu_stats);
 77	if (tb[NFTA_COUNTER_PACKETS]) {
 78		u64_stats_set(&this_cpu->packets,
 79			      be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS])));
 80	}
 81	if (tb[NFTA_COUNTER_BYTES]) {
 82		u64_stats_set(&this_cpu->bytes,
 83			      be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES])));
 84	}
 85
 86	priv->counter = cpu_stats;
 87	return 0;
 88}
 89
 90static int nft_counter_obj_init(const struct nft_ctx *ctx,
 91				const struct nlattr * const tb[],
 92				struct nft_object *obj)
 93{
 94	struct nft_counter_percpu_priv *priv = nft_obj_data(obj);
 95
 96	return nft_counter_do_init(tb, priv);
 97}
 98
 99static void nft_counter_do_destroy(struct nft_counter_percpu_priv *priv)
100{
101	free_percpu(priv->counter);
102}
103
104static void nft_counter_obj_destroy(const struct nft_ctx *ctx,
105				    struct nft_object *obj)
106{
107	struct nft_counter_percpu_priv *priv = nft_obj_data(obj);
108
109	nft_counter_do_destroy(priv);
110}
111
112static void nft_counter_reset(struct nft_counter_percpu_priv *priv,
113			      struct nft_counter_tot *total)
114{
115	struct u64_stats_sync *nft_sync;
116	struct nft_counter *this_cpu;
117
118	local_bh_disable();
119	this_cpu = this_cpu_ptr(priv->counter);
120	nft_sync = this_cpu_ptr(&nft_counter_sync);
121
122	u64_stats_update_begin(nft_sync);
123	u64_stats_sub(&this_cpu->packets, total->packets);
124	u64_stats_sub(&this_cpu->bytes, total->bytes);
125	u64_stats_update_end(nft_sync);
126
127	local_bh_enable();
128}
129
130static void nft_counter_fetch(struct nft_counter_percpu_priv *priv,
131			      struct nft_counter_tot *total)
132{
133	struct nft_counter *this_cpu;
134	u64 bytes, packets;
135	unsigned int seq;
136	int cpu;
137
138	memset(total, 0, sizeof(*total));
139	for_each_possible_cpu(cpu) {
140		struct u64_stats_sync *nft_sync = per_cpu_ptr(&nft_counter_sync, cpu);
141
142		this_cpu = per_cpu_ptr(priv->counter, cpu);
143		do {
144			seq	= u64_stats_fetch_begin(nft_sync);
145			bytes	= u64_stats_read(&this_cpu->bytes);
146			packets	= u64_stats_read(&this_cpu->packets);
147		} while (u64_stats_fetch_retry(nft_sync, seq));
148
149		total->bytes	+= bytes;
150		total->packets	+= packets;
151	}
152}
153
154static void nft_counter_fetch_and_reset(struct nft_counter_percpu_priv *priv,
155					struct nft_counter_tot *total)
156{
157	spin_lock(&nft_counter_lock);
158	nft_counter_fetch(priv, total);
159	nft_counter_reset(priv, total);
160	spin_unlock(&nft_counter_lock);
161}
162
163static int nft_counter_do_dump(struct sk_buff *skb,
164			       struct nft_counter_percpu_priv *priv,
165			       bool reset)
166{
167	struct nft_counter_tot total;
168
169	if (unlikely(reset))
170		nft_counter_fetch_and_reset(priv, &total);
171	else
172		nft_counter_fetch(priv, &total);
173
174	if (nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
175			 NFTA_COUNTER_PAD) ||
176	    nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.packets),
177			 NFTA_COUNTER_PAD))
178		goto nla_put_failure;
179
180	return 0;
181
182nla_put_failure:
183	return -1;
184}
185
186static int nft_counter_obj_dump(struct sk_buff *skb,
187				struct nft_object *obj, bool reset)
188{
189	struct nft_counter_percpu_priv *priv = nft_obj_data(obj);
190
191	return nft_counter_do_dump(skb, priv, reset);
192}
193
194static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
195	[NFTA_COUNTER_PACKETS]	= { .type = NLA_U64 },
196	[NFTA_COUNTER_BYTES]	= { .type = NLA_U64 },
197};
198
199struct nft_object_type nft_counter_obj_type;
200static const struct nft_object_ops nft_counter_obj_ops = {
201	.type		= &nft_counter_obj_type,
202	.size		= sizeof(struct nft_counter_percpu_priv),
203	.eval		= nft_counter_obj_eval,
204	.init		= nft_counter_obj_init,
205	.destroy	= nft_counter_obj_destroy,
206	.dump		= nft_counter_obj_dump,
207};
208
209struct nft_object_type nft_counter_obj_type __read_mostly = {
210	.type		= NFT_OBJECT_COUNTER,
211	.ops		= &nft_counter_obj_ops,
212	.maxattr	= NFTA_COUNTER_MAX,
213	.policy		= nft_counter_policy,
214	.owner		= THIS_MODULE,
215};
216
217void nft_counter_eval(const struct nft_expr *expr, struct nft_regs *regs,
218		      const struct nft_pktinfo *pkt)
219{
220	struct nft_counter_percpu_priv *priv = nft_expr_priv(expr);
221
222	nft_counter_do_eval(priv, regs, pkt);
223}
224
225static int nft_counter_dump(struct sk_buff *skb,
226			    const struct nft_expr *expr, bool reset)
227{
228	struct nft_counter_percpu_priv *priv = nft_expr_priv(expr);
229
230	return nft_counter_do_dump(skb, priv, reset);
231}
232
233static int nft_counter_init(const struct nft_ctx *ctx,
234			    const struct nft_expr *expr,
235			    const struct nlattr * const tb[])
236{
237	struct nft_counter_percpu_priv *priv = nft_expr_priv(expr);
238
239	return nft_counter_do_init(tb, priv);
240}
241
242static void nft_counter_destroy(const struct nft_ctx *ctx,
243				const struct nft_expr *expr)
244{
245	struct nft_counter_percpu_priv *priv = nft_expr_priv(expr);
246
247	nft_counter_do_destroy(priv);
248}
249
250static int nft_counter_clone(struct nft_expr *dst, const struct nft_expr *src, gfp_t gfp)
251{
252	struct nft_counter_percpu_priv *priv = nft_expr_priv(src);
253	struct nft_counter_percpu_priv *priv_clone = nft_expr_priv(dst);
254	struct nft_counter __percpu *cpu_stats;
255	struct nft_counter *this_cpu;
256	struct nft_counter_tot total;
257
258	nft_counter_fetch(priv, &total);
259
260	cpu_stats = alloc_percpu_gfp(struct nft_counter, gfp);
261	if (cpu_stats == NULL)
262		return -ENOMEM;
263
264	this_cpu = raw_cpu_ptr(cpu_stats);
265	u64_stats_set(&this_cpu->packets, total.packets);
266	u64_stats_set(&this_cpu->bytes, total.bytes);
267
268	priv_clone->counter = cpu_stats;
269	return 0;
270}
271
272static int nft_counter_offload(struct nft_offload_ctx *ctx,
273			       struct nft_flow_rule *flow,
274			       const struct nft_expr *expr)
275{
276	/* No specific offload action is needed, but report success. */
277	return 0;
278}
279
280static void nft_counter_offload_stats(struct nft_expr *expr,
281				      const struct flow_stats *stats)
282{
283	struct nft_counter_percpu_priv *priv = nft_expr_priv(expr);
284	struct u64_stats_sync *nft_sync;
285	struct nft_counter *this_cpu;
286
287	local_bh_disable();
288	this_cpu = this_cpu_ptr(priv->counter);
289	nft_sync = this_cpu_ptr(&nft_counter_sync);
290
291	u64_stats_update_begin(nft_sync);
292	u64_stats_add(&this_cpu->packets, stats->pkts);
293	u64_stats_add(&this_cpu->bytes, stats->bytes);
294	u64_stats_update_end(nft_sync);
295	local_bh_enable();
296}
297
298void nft_counter_init_seqcount(void)
299{
300	int cpu;
301
302	for_each_possible_cpu(cpu)
303		u64_stats_init(per_cpu_ptr(&nft_counter_sync, cpu));
304}
305
306struct nft_expr_type nft_counter_type;
307static const struct nft_expr_ops nft_counter_ops = {
308	.type		= &nft_counter_type,
309	.size		= NFT_EXPR_SIZE(sizeof(struct nft_counter_percpu_priv)),
310	.eval		= nft_counter_eval,
311	.init		= nft_counter_init,
312	.destroy	= nft_counter_destroy,
313	.destroy_clone	= nft_counter_destroy,
314	.dump		= nft_counter_dump,
315	.clone		= nft_counter_clone,
316	.reduce		= NFT_REDUCE_READONLY,
317	.offload	= nft_counter_offload,
318	.offload_stats	= nft_counter_offload_stats,
319};
320
321struct nft_expr_type nft_counter_type __read_mostly = {
322	.name		= "counter",
323	.ops		= &nft_counter_ops,
324	.policy		= nft_counter_policy,
325	.maxattr	= NFTA_COUNTER_MAX,
326	.flags		= NFT_EXPR_STATEFUL,
327	.owner		= THIS_MODULE,
328};