act_police.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * net/sched/act_police.c	Input police filter
  4 *
  5 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  6 * 		J Hadi Salim (action changes)
  7 */
  8
  9#include <linux/module.h>
 10#include <linux/types.h>
 11#include <linux/kernel.h>
 12#include <linux/string.h>
 13#include <linux/errno.h>
 14#include <linux/skbuff.h>
 15#include <linux/rtnetlink.h>
 16#include <linux/init.h>
 17#include <linux/slab.h>
 18#include <net/act_api.h>
 19#include <net/gso.h>
 20#include <net/netlink.h>
 21#include <net/pkt_cls.h>
 22#include <net/tc_act/tc_police.h>
 23#include <net/tc_wrapper.h>
 24
 25/* Each policer is serialized by its individual spinlock */
 26
 27static struct tc_action_ops act_police_ops;
 28
 29static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = {
 30	[TCA_POLICE_RATE]	= { .len = TC_RTAB_SIZE },
 31	[TCA_POLICE_PEAKRATE]	= { .len = TC_RTAB_SIZE },
 32	[TCA_POLICE_AVRATE]	= { .type = NLA_U32 },
 33	[TCA_POLICE_RESULT]	= { .type = NLA_U32 },
 34	[TCA_POLICE_RATE64]     = { .type = NLA_U64 },
 35	[TCA_POLICE_PEAKRATE64] = { .type = NLA_U64 },
 36	[TCA_POLICE_PKTRATE64]  = { .type = NLA_U64, .min = 1 },
 37	[TCA_POLICE_PKTBURST64] = { .type = NLA_U64, .min = 1 },
 38};
 39
 40static int tcf_police_init(struct net *net, struct nlattr *nla,
 41			       struct nlattr *est, struct tc_action **a,
 42			       struct tcf_proto *tp, u32 flags,
 43			       struct netlink_ext_ack *extack)
 44{
 45	int ret = 0, tcfp_result = TC_ACT_OK, err, size;
 46	bool bind = flags & TCA_ACT_FLAGS_BIND;
 47	struct nlattr *tb[TCA_POLICE_MAX + 1];
 48	struct tcf_chain *goto_ch = NULL;
 49	struct tc_police *parm;
 50	struct tcf_police *police;
 51	struct qdisc_rate_table *R_tab = NULL, *P_tab = NULL;
 52	struct tc_action_net *tn = net_generic(net, act_police_ops.net_id);
 53	struct tcf_police_params *new;
 54	bool exists = false;
 55	u32 index;
 56	u64 rate64, prate64;
 57	u64 pps, ppsburst;
 58
 59	if (nla == NULL)
 60		return -EINVAL;
 61
 62	err = nla_parse_nested_deprecated(tb, TCA_POLICE_MAX, nla,
 63					  police_policy, NULL);
 64	if (err < 0)
 65		return err;
 66
 67	if (tb[TCA_POLICE_TBF] == NULL)
 68		return -EINVAL;
 69	size = nla_len(tb[TCA_POLICE_TBF]);
 70	if (size != sizeof(*parm) && size != sizeof(struct tc_police_compat))
 71		return -EINVAL;
 72
 73	parm = nla_data(tb[TCA_POLICE_TBF]);
 74	index = parm->index;
 75	err = tcf_idr_check_alloc(tn, &index, a, bind);
 76	if (err < 0)
 77		return err;
 78	exists = err;
 79	if (exists && bind)
 80		return ACT_P_BOUND;
 81
 82	if (!exists) {
 83		ret = tcf_idr_create(tn, index, NULL, a,
 84				     &act_police_ops, bind, true, flags);
 85		if (ret) {
 86			tcf_idr_cleanup(tn, index);
 87			return ret;
 88		}
 89		ret = ACT_P_CREATED;
 90		spin_lock_init(&(to_police(*a)->tcfp_lock));
 91	} else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
 92		tcf_idr_release(*a, bind);
 93		return -EEXIST;
 94	}
 95	err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
 96	if (err < 0)
 97		goto release_idr;
 98
 99	police = to_police(*a);
100	if (parm->rate.rate) {
101		err = -ENOMEM;
102		R_tab = qdisc_get_rtab(&parm->rate, tb[TCA_POLICE_RATE], NULL);
103		if (R_tab == NULL)
104			goto failure;
105
106		if (parm->peakrate.rate) {
107			P_tab = qdisc_get_rtab(&parm->peakrate,
108					       tb[TCA_POLICE_PEAKRATE], NULL);
109			if (P_tab == NULL)
110				goto failure;
111		}
112	}
113
114	if (est) {
115		err = gen_replace_estimator(&police->tcf_bstats,
116					    police->common.cpu_bstats,
117					    &police->tcf_rate_est,
118					    &police->tcf_lock,
119					    false, est);
120		if (err)
121			goto failure;
122	} else if (tb[TCA_POLICE_AVRATE] &&
123		   (ret == ACT_P_CREATED ||
124		    !gen_estimator_active(&police->tcf_rate_est))) {
125		err = -EINVAL;
126		goto failure;
127	}
128
129	if (tb[TCA_POLICE_RESULT]) {
130		tcfp_result = nla_get_u32(tb[TCA_POLICE_RESULT]);
131		if (TC_ACT_EXT_CMP(tcfp_result, TC_ACT_GOTO_CHAIN)) {
132			NL_SET_ERR_MSG(extack,
133				       "goto chain not allowed on fallback");
134			err = -EINVAL;
135			goto failure;
136		}
137	}
138
139	if ((tb[TCA_POLICE_PKTRATE64] && !tb[TCA_POLICE_PKTBURST64]) ||
140	    (!tb[TCA_POLICE_PKTRATE64] && tb[TCA_POLICE_PKTBURST64])) {
141		NL_SET_ERR_MSG(extack,
142			       "Both or neither packet-per-second burst and rate must be provided");
143		err = -EINVAL;
144		goto failure;
145	}
146
147	if (tb[TCA_POLICE_PKTRATE64] && R_tab) {
148		NL_SET_ERR_MSG(extack,
149			       "packet-per-second and byte-per-second rate limits not allowed in same action");
150		err = -EINVAL;
151		goto failure;
152	}
153
154	new = kzalloc(sizeof(*new), GFP_KERNEL);
155	if (unlikely(!new)) {
156		err = -ENOMEM;
157		goto failure;
158	}
159
160	/* No failure allowed after this point */
161	new->tcfp_result = tcfp_result;
162	new->tcfp_mtu = parm->mtu;
163	if (!new->tcfp_mtu) {
164		new->tcfp_mtu = ~0;
165		if (R_tab)
166			new->tcfp_mtu = 255 << R_tab->rate.cell_log;
167	}
168	if (R_tab) {
169		new->rate_present = true;
170		rate64 = nla_get_u64_default(tb[TCA_POLICE_RATE64], 0);
171		psched_ratecfg_precompute(&new->rate, &R_tab->rate, rate64);
172		qdisc_put_rtab(R_tab);
173	} else {
174		new->rate_present = false;
175	}
176	if (P_tab) {
177		new->peak_present = true;
178		prate64 = nla_get_u64_default(tb[TCA_POLICE_PEAKRATE64], 0);
179		psched_ratecfg_precompute(&new->peak, &P_tab->rate, prate64);
180		qdisc_put_rtab(P_tab);
181	} else {
182		new->peak_present = false;
183	}
184
185	new->tcfp_burst = PSCHED_TICKS2NS(parm->burst);
186	if (new->peak_present)
187		new->tcfp_mtu_ptoks = (s64)psched_l2t_ns(&new->peak,
188							 new->tcfp_mtu);
189
190	if (tb[TCA_POLICE_AVRATE])
191		new->tcfp_ewma_rate = nla_get_u32(tb[TCA_POLICE_AVRATE]);
192
193	if (tb[TCA_POLICE_PKTRATE64]) {
194		pps = nla_get_u64(tb[TCA_POLICE_PKTRATE64]);
195		ppsburst = nla_get_u64(tb[TCA_POLICE_PKTBURST64]);
196		new->pps_present = true;
197		new->tcfp_pkt_burst = PSCHED_TICKS2NS(ppsburst);
198		psched_ppscfg_precompute(&new->ppsrate, pps);
199	}
200
201	new->action = parm->action;
202	spin_lock_bh(&police->tcf_lock);
203	spin_lock_bh(&police->tcfp_lock);
204	police->tcfp_t_c = ktime_get_ns();
205	police->tcfp_toks = new->tcfp_burst;
206	if (new->peak_present)
207		police->tcfp_ptoks = new->tcfp_mtu_ptoks;
208	spin_unlock_bh(&police->tcfp_lock);
209	goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
210	new = rcu_replace_pointer(police->params,
211				  new,
212				  lockdep_is_held(&police->tcf_lock));
213	spin_unlock_bh(&police->tcf_lock);
214
215	if (goto_ch)
216		tcf_chain_put_by_act(goto_ch);
217	if (new)
218		kfree_rcu(new, rcu);
219
220	return ret;
221
222failure:
223	qdisc_put_rtab(P_tab);
224	qdisc_put_rtab(R_tab);
225	if (goto_ch)
226		tcf_chain_put_by_act(goto_ch);
227release_idr:
228	tcf_idr_release(*a, bind);
229	return err;
230}
231
232static bool tcf_police_mtu_check(struct sk_buff *skb, u32 limit)
233{
234	u32 len;
235
236	if (skb_is_gso(skb))
237		return skb_gso_validate_mac_len(skb, limit);
238
239	len = qdisc_pkt_len(skb);
240	if (skb_at_tc_ingress(skb))
241		len += skb->mac_len;
242
243	return len <= limit;
244}
245
246TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb,
247				     const struct tc_action *a,
248				     struct tcf_result *res)
249{
250	struct tcf_police *police = to_police(a);
251	s64 now, toks, ppstoks = 0, ptoks = 0;
252	struct tcf_police_params *p;
253	int ret;
254
255	tcf_lastuse_update(&police->tcf_tm);
256	bstats_update(this_cpu_ptr(police->common.cpu_bstats), skb);
257
258	p = rcu_dereference_bh(police->params);
259	ret = p->action;
260
261	if (p->tcfp_ewma_rate) {
262		struct gnet_stats_rate_est64 sample;
263
264		if (!gen_estimator_read(&police->tcf_rate_est, &sample) ||
265		    sample.bps >= p->tcfp_ewma_rate)
266			goto inc_overlimits;
267	}
268
269	if (tcf_police_mtu_check(skb, p->tcfp_mtu)) {
270		if (!p->rate_present && !p->pps_present) {
271			ret = p->tcfp_result;
272			goto end;
273		}
274
275		now = ktime_get_ns();
276		spin_lock_bh(&police->tcfp_lock);
277		toks = min_t(s64, now - police->tcfp_t_c, p->tcfp_burst);
278		if (p->peak_present) {
279			ptoks = toks + police->tcfp_ptoks;
280			if (ptoks > p->tcfp_mtu_ptoks)
281				ptoks = p->tcfp_mtu_ptoks;
282			ptoks -= (s64)psched_l2t_ns(&p->peak,
283						    qdisc_pkt_len(skb));
284		}
285		if (p->rate_present) {
286			toks += police->tcfp_toks;
287			if (toks > p->tcfp_burst)
288				toks = p->tcfp_burst;
289			toks -= (s64)psched_l2t_ns(&p->rate, qdisc_pkt_len(skb));
290		} else if (p->pps_present) {
291			ppstoks = min_t(s64, now - police->tcfp_t_c, p->tcfp_pkt_burst);
292			ppstoks += police->tcfp_pkttoks;
293			if (ppstoks > p->tcfp_pkt_burst)
294				ppstoks = p->tcfp_pkt_burst;
295			ppstoks -= (s64)psched_pkt2t_ns(&p->ppsrate, 1);
296		}
297		if ((toks | ptoks | ppstoks) >= 0) {
298			police->tcfp_t_c = now;
299			police->tcfp_toks = toks;
300			police->tcfp_ptoks = ptoks;
301			police->tcfp_pkttoks = ppstoks;
302			spin_unlock_bh(&police->tcfp_lock);
303			ret = p->tcfp_result;
304			goto inc_drops;
305		}
306		spin_unlock_bh(&police->tcfp_lock);
307	}
308
309inc_overlimits:
310	qstats_overlimit_inc(this_cpu_ptr(police->common.cpu_qstats));
311inc_drops:
312	if (ret == TC_ACT_SHOT)
313		qstats_drop_inc(this_cpu_ptr(police->common.cpu_qstats));
314end:
315	return ret;
316}
317
318static void tcf_police_cleanup(struct tc_action *a)
319{
320	struct tcf_police *police = to_police(a);
321	struct tcf_police_params *p;
322
323	p = rcu_dereference_protected(police->params, 1);
324	if (p)
325		kfree_rcu(p, rcu);
326}
327
328static void tcf_police_stats_update(struct tc_action *a,
329				    u64 bytes, u64 packets, u64 drops,
330				    u64 lastuse, bool hw)
331{
332	struct tcf_police *police = to_police(a);
333	struct tcf_t *tm = &police->tcf_tm;
334
335	tcf_action_update_stats(a, bytes, packets, drops, hw);
336	tm->lastuse = max_t(u64, tm->lastuse, lastuse);
337}
338
339static int tcf_police_dump(struct sk_buff *skb, struct tc_action *a,
340			       int bind, int ref)
341{
342	const struct tcf_police *police = to_police(a);
343	unsigned char *b = skb_tail_pointer(skb);
344	const struct tcf_police_params *p;
345	struct tc_police opt = {
346		.index = police->tcf_index,
347		.refcnt = refcount_read(&police->tcf_refcnt) - ref,
348		.bindcnt = atomic_read(&police->tcf_bindcnt) - bind,
349	};
350	struct tcf_t t;
351
352	rcu_read_lock();
353	p = rcu_dereference(police->params);
354	opt.action = p->action;
355	opt.mtu = p->tcfp_mtu;
356	opt.burst = PSCHED_NS2TICKS(p->tcfp_burst);
357	if (p->rate_present) {
358		psched_ratecfg_getrate(&opt.rate, &p->rate);
359		if ((p->rate.rate_bytes_ps >= (1ULL << 32)) &&
360		    nla_put_u64_64bit(skb, TCA_POLICE_RATE64,
361				      p->rate.rate_bytes_ps,
362				      TCA_POLICE_PAD))
363			goto nla_put_failure;
364	}
365	if (p->peak_present) {
366		psched_ratecfg_getrate(&opt.peakrate, &p->peak);
367		if ((p->peak.rate_bytes_ps >= (1ULL << 32)) &&
368		    nla_put_u64_64bit(skb, TCA_POLICE_PEAKRATE64,
369				      p->peak.rate_bytes_ps,
370				      TCA_POLICE_PAD))
371			goto nla_put_failure;
372	}
373	if (p->pps_present) {
374		if (nla_put_u64_64bit(skb, TCA_POLICE_PKTRATE64,
375				      p->ppsrate.rate_pkts_ps,
376				      TCA_POLICE_PAD))
377			goto nla_put_failure;
378		if (nla_put_u64_64bit(skb, TCA_POLICE_PKTBURST64,
379				      PSCHED_NS2TICKS(p->tcfp_pkt_burst),
380				      TCA_POLICE_PAD))
381			goto nla_put_failure;
382	}
383	if (nla_put(skb, TCA_POLICE_TBF, sizeof(opt), &opt))
384		goto nla_put_failure;
385	if (p->tcfp_result &&
386	    nla_put_u32(skb, TCA_POLICE_RESULT, p->tcfp_result))
387		goto nla_put_failure;
388	if (p->tcfp_ewma_rate &&
389	    nla_put_u32(skb, TCA_POLICE_AVRATE, p->tcfp_ewma_rate))
390		goto nla_put_failure;
391
392	tcf_tm_dump(&t, &police->tcf_tm);
393	if (nla_put_64bit(skb, TCA_POLICE_TM, sizeof(t), &t, TCA_POLICE_PAD))
394		goto nla_put_failure;
395	rcu_read_unlock();
396
397	return skb->len;
398
399nla_put_failure:
400	rcu_read_unlock();
401	nlmsg_trim(skb, b);
402	return -1;
403}
404
405static int tcf_police_act_to_flow_act(int tc_act, u32 *extval,
406				      struct netlink_ext_ack *extack)
407{
408	int act_id = -EOPNOTSUPP;
409
410	if (!TC_ACT_EXT_OPCODE(tc_act)) {
411		if (tc_act == TC_ACT_OK)
412			act_id = FLOW_ACTION_ACCEPT;
413		else if (tc_act ==  TC_ACT_SHOT)
414			act_id = FLOW_ACTION_DROP;
415		else if (tc_act == TC_ACT_PIPE)
416			act_id = FLOW_ACTION_PIPE;
417		else if (tc_act == TC_ACT_RECLASSIFY)
418			NL_SET_ERR_MSG_MOD(extack, "Offload not supported when conform/exceed action is \"reclassify\"");
419		else
420			NL_SET_ERR_MSG_MOD(extack, "Unsupported conform/exceed action offload");
421	} else if (TC_ACT_EXT_CMP(tc_act, TC_ACT_GOTO_CHAIN)) {
422		act_id = FLOW_ACTION_GOTO;
423		*extval = tc_act & TC_ACT_EXT_VAL_MASK;
424	} else if (TC_ACT_EXT_CMP(tc_act, TC_ACT_JUMP)) {
425		act_id = FLOW_ACTION_JUMP;
426		*extval = tc_act & TC_ACT_EXT_VAL_MASK;
427	} else if (tc_act == TC_ACT_UNSPEC) {
428		act_id = FLOW_ACTION_CONTINUE;
429	} else {
430		NL_SET_ERR_MSG_MOD(extack, "Unsupported conform/exceed action offload");
431	}
432
433	return act_id;
434}
435
436static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data,
437					u32 *index_inc, bool bind,
438					struct netlink_ext_ack *extack)
439{
440	if (bind) {
441		struct flow_action_entry *entry = entry_data;
442		struct tcf_police *police = to_police(act);
443		struct tcf_police_params *p;
444		int act_id;
445
446		p = rcu_dereference_protected(police->params,
447					      lockdep_is_held(&police->tcf_lock));
448
449		entry->id = FLOW_ACTION_POLICE;
450		entry->police.burst = tcf_police_burst(act);
451		entry->police.rate_bytes_ps =
452			tcf_police_rate_bytes_ps(act);
453		entry->police.peakrate_bytes_ps = tcf_police_peakrate_bytes_ps(act);
454		entry->police.avrate = tcf_police_tcfp_ewma_rate(act);
455		entry->police.overhead = tcf_police_rate_overhead(act);
456		entry->police.burst_pkt = tcf_police_burst_pkt(act);
457		entry->police.rate_pkt_ps =
458			tcf_police_rate_pkt_ps(act);
459		entry->police.mtu = tcf_police_tcfp_mtu(act);
460
461		act_id = tcf_police_act_to_flow_act(police->tcf_action,
462						    &entry->police.exceed.extval,
463						    extack);
464		if (act_id < 0)
465			return act_id;
466
467		entry->police.exceed.act_id = act_id;
468
469		act_id = tcf_police_act_to_flow_act(p->tcfp_result,
470						    &entry->police.notexceed.extval,
471						    extack);
472		if (act_id < 0)
473			return act_id;
474
475		entry->police.notexceed.act_id = act_id;
476
477		*index_inc = 1;
478	} else {
479		struct flow_offload_action *fl_action = entry_data;
480
481		fl_action->id = FLOW_ACTION_POLICE;
482	}
483
484	return 0;
485}
486
487MODULE_AUTHOR("Alexey Kuznetsov");
488MODULE_DESCRIPTION("Policing actions");
489MODULE_LICENSE("GPL");
490
491static struct tc_action_ops act_police_ops = {
492	.kind		=	"police",
493	.id		=	TCA_ID_POLICE,
494	.owner		=	THIS_MODULE,
495	.stats_update	=	tcf_police_stats_update,
496	.act		=	tcf_police_act,
497	.dump		=	tcf_police_dump,
498	.init		=	tcf_police_init,
499	.cleanup	=	tcf_police_cleanup,
500	.offload_act_setup =	tcf_police_offload_act_setup,
501	.size		=	sizeof(struct tcf_police),
502};
503MODULE_ALIAS_NET_ACT("police");
504
505static __net_init int police_init_net(struct net *net)
506{
507	struct tc_action_net *tn = net_generic(net, act_police_ops.net_id);
508
509	return tc_action_net_init(net, tn, &act_police_ops);
510}
511
512static void __net_exit police_exit_net(struct list_head *net_list)
513{
514	tc_action_net_exit(net_list, act_police_ops.net_id);
515}
516
517static struct pernet_operations police_net_ops = {
518	.init = police_init_net,
519	.exit_batch = police_exit_net,
520	.id   = &act_police_ops.net_id,
521	.size = sizeof(struct tc_action_net),
522};
523
524static int __init police_init_module(void)
525{
526	return tcf_register_action(&act_police_ops, &police_net_ops);
527}
528
529static void __exit police_cleanup_module(void)
530{
531	tcf_unregister_action(&act_police_ops, &police_net_ops);
532}
533
534module_init(police_init_module);
535module_exit(police_cleanup_module);