nf_tables_offload.c

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#include <linux/init.h>
  3#include <linux/module.h>
  4#include <linux/netfilter.h>
  5#include <net/flow_offload.h>
  6#include <net/netfilter/nf_tables.h>
  7#include <net/netfilter/nf_tables_offload.h>
  8#include <net/pkt_cls.h>
  9
 10static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
 11{
 12	struct nft_flow_rule *flow;
 13
 14	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
 15	if (!flow)
 16		return NULL;
 17
 18	flow->rule = flow_rule_alloc(num_actions);
 19	if (!flow->rule) {
 20		kfree(flow);
 21		return NULL;
 22	}
 23
 24	flow->rule->match.dissector	= &flow->match.dissector;
 25	flow->rule->match.mask		= &flow->match.mask;
 26	flow->rule->match.key		= &flow->match.key;
 27
 28	return flow;
 29}
 30
 31void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
 32				 enum flow_dissector_key_id addr_type)
 33{
 34	struct nft_flow_match *match = &flow->match;
 35	struct nft_flow_key *mask = &match->mask;
 36	struct nft_flow_key *key = &match->key;
 37
 38	if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL))
 39		return;
 40
 41	key->control.addr_type = addr_type;
 42	mask->control.addr_type = 0xffff;
 43	match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL);
 44	match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
 45		offsetof(struct nft_flow_key, control);
 46}
 47
 48struct nft_offload_ethertype {
 49	__be16 value;
 50	__be16 mask;
 51};
 52
 53static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx,
 54					struct nft_flow_rule *flow)
 55{
 56	struct nft_flow_match *match = &flow->match;
 57	struct nft_offload_ethertype ethertype = {
 58		.value	= match->key.basic.n_proto,
 59		.mask	= match->mask.basic.n_proto,
 60	};
 61
 62	if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) &&
 63	    (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) ||
 64	     match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) {
 65		match->key.basic.n_proto = match->key.cvlan.vlan_tpid;
 66		match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid;
 67		match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid;
 68		match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid;
 69		match->key.vlan.vlan_tpid = ethertype.value;
 70		match->mask.vlan.vlan_tpid = ethertype.mask;
 71		match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] =
 72			offsetof(struct nft_flow_key, cvlan);
 73		match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN);
 74	} else if (match->dissector.used_keys &
 75		   BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) &&
 76		   (match->key.basic.n_proto == htons(ETH_P_8021Q) ||
 77		    match->key.basic.n_proto == htons(ETH_P_8021AD))) {
 78		match->key.basic.n_proto = match->key.vlan.vlan_tpid;
 79		match->mask.basic.n_proto = match->mask.vlan.vlan_tpid;
 80		match->key.vlan.vlan_tpid = ethertype.value;
 81		match->mask.vlan.vlan_tpid = ethertype.mask;
 82		match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
 83			offsetof(struct nft_flow_key, vlan);
 84		match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_VLAN);
 85	}
 86}
 87
 88struct nft_flow_rule *nft_flow_rule_create(struct net *net,
 89					   const struct nft_rule *rule)
 90{
 91	struct nft_offload_ctx *ctx;
 92	struct nft_flow_rule *flow;
 93	int num_actions = 0, err;
 94	struct nft_expr *expr;
 95
 96	expr = nft_expr_first(rule);
 97	while (nft_expr_more(rule, expr)) {
 98		if (expr->ops->offload_action &&
 99		    expr->ops->offload_action(expr))
100			num_actions++;
101
102		expr = nft_expr_next(expr);
103	}
104
105	if (num_actions == 0)
106		return ERR_PTR(-EOPNOTSUPP);
107
108	flow = nft_flow_rule_alloc(num_actions);
109	if (!flow)
110		return ERR_PTR(-ENOMEM);
111
112	expr = nft_expr_first(rule);
113
114	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
115	if (!ctx) {
116		err = -ENOMEM;
117		goto err_out;
118	}
119	ctx->net = net;
120	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
121
122	while (nft_expr_more(rule, expr)) {
123		if (!expr->ops->offload) {
124			err = -EOPNOTSUPP;
125			goto err_out;
126		}
127		err = expr->ops->offload(ctx, flow, expr);
128		if (err < 0)
129			goto err_out;
130
131		expr = nft_expr_next(expr);
132	}
133	nft_flow_rule_transfer_vlan(ctx, flow);
134
135	flow->proto = ctx->dep.l3num;
136	kfree(ctx);
137
138	return flow;
139err_out:
140	kfree(ctx);
141	nft_flow_rule_destroy(flow);
142
143	return ERR_PTR(err);
144}
145
146void nft_flow_rule_destroy(struct nft_flow_rule *flow)
147{
148	struct flow_action_entry *entry;
149	int i;
150
151	flow_action_for_each(i, entry, &flow->rule->action) {
152		switch (entry->id) {
153		case FLOW_ACTION_REDIRECT:
154		case FLOW_ACTION_MIRRED:
155			dev_put(entry->dev);
156			break;
157		default:
158			break;
159		}
160	}
161	kfree(flow->rule);
162	kfree(flow);
163}
164
165void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
166				enum nft_offload_dep_type type)
167{
168	ctx->dep.type = type;
169}
170
171void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
172				   const void *data, u32 len)
173{
174	switch (ctx->dep.type) {
175	case NFT_OFFLOAD_DEP_NETWORK:
176		WARN_ON(len != sizeof(__u16));
177		memcpy(&ctx->dep.l3num, data, sizeof(__u16));
178		break;
179	case NFT_OFFLOAD_DEP_TRANSPORT:
180		WARN_ON(len != sizeof(__u8));
181		memcpy(&ctx->dep.protonum, data, sizeof(__u8));
182		break;
183	default:
184		break;
185	}
186	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
187}
188
189static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
190					 __be16 proto, int priority,
191					 struct netlink_ext_ack *extack)
192{
193	common->protocol = proto;
194	common->prio = priority;
195	common->extack = extack;
196}
197
198static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
199			     struct list_head *cb_list)
200{
201	struct flow_block_cb *block_cb;
202	int err;
203
204	list_for_each_entry(block_cb, cb_list, list) {
205		err = block_cb->cb(type, type_data, block_cb->cb_priv);
206		if (err < 0)
207			return err;
208	}
209	return 0;
210}
211
212static int nft_chain_offload_priority(const struct nft_base_chain *basechain)
213{
214	if (basechain->ops.priority <= 0 ||
215	    basechain->ops.priority > USHRT_MAX)
216		return -1;
217
218	return 0;
219}
220
221bool nft_chain_offload_support(const struct nft_base_chain *basechain)
222{
223	struct nf_hook_ops *ops;
224	struct net_device *dev;
225	struct nft_hook *hook;
226
227	if (nft_chain_offload_priority(basechain) < 0)
228		return false;
229
230	list_for_each_entry(hook, &basechain->hook_list, list) {
231		list_for_each_entry(ops, &hook->ops_list, list) {
232			if (ops->pf != NFPROTO_NETDEV ||
233			    ops->hooknum != NF_NETDEV_INGRESS)
234				return false;
235
236			dev = ops->dev;
237			if (!dev->netdev_ops->ndo_setup_tc &&
238			    !flow_indr_dev_exists())
239				return false;
240		}
241	}
242
243	return true;
244}
245
246static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
247				       const struct nft_base_chain *basechain,
248				       const struct nft_rule *rule,
249				       const struct nft_flow_rule *flow,
250				       struct netlink_ext_ack *extack,
251				       enum flow_cls_command command)
252{
253	__be16 proto = ETH_P_ALL;
254
255	memset(cls_flow, 0, sizeof(*cls_flow));
256
257	if (flow)
258		proto = flow->proto;
259
260	nft_flow_offload_common_init(&cls_flow->common, proto,
261				     basechain->ops.priority, extack);
262	cls_flow->command = command;
263	cls_flow->cookie = (unsigned long) rule;
264	if (flow)
265		cls_flow->rule = flow->rule;
266}
267
268static int nft_flow_offload_cmd(const struct nft_chain *chain,
269				const struct nft_rule *rule,
270				struct nft_flow_rule *flow,
271				enum flow_cls_command command,
272				struct flow_cls_offload *cls_flow)
273{
274	struct netlink_ext_ack extack = {};
275	struct nft_base_chain *basechain;
276
277	if (!nft_is_base_chain(chain))
278		return -EOPNOTSUPP;
279
280	basechain = nft_base_chain(chain);
281	nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, &extack,
282				   command);
283
284	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, cls_flow,
285				 &basechain->flow_block.cb_list);
286}
287
288static int nft_flow_offload_rule(const struct nft_chain *chain,
289				 struct nft_rule *rule,
290				 struct nft_flow_rule *flow,
291				 enum flow_cls_command command)
292{
293	struct flow_cls_offload cls_flow;
294
295	return nft_flow_offload_cmd(chain, rule, flow, command, &cls_flow);
296}
297
298int nft_flow_rule_stats(const struct nft_chain *chain,
299			const struct nft_rule *rule)
300{
301	struct flow_cls_offload cls_flow = {};
302	struct nft_expr *expr, *next;
303	int err;
304
305	err = nft_flow_offload_cmd(chain, rule, NULL, FLOW_CLS_STATS,
306				   &cls_flow);
307	if (err < 0)
308		return err;
309
310	nft_rule_for_each_expr(expr, next, rule) {
311		if (expr->ops->offload_stats)
312			expr->ops->offload_stats(expr, &cls_flow.stats);
313	}
314
315	return 0;
316}
317
318static int nft_flow_offload_bind(struct flow_block_offload *bo,
319				 struct nft_base_chain *basechain)
320{
321	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
322	return 0;
323}
324
325static int nft_flow_offload_unbind(struct flow_block_offload *bo,
326				   struct nft_base_chain *basechain)
327{
328	struct flow_block_cb *block_cb, *next;
329	struct flow_cls_offload cls_flow;
330	struct netlink_ext_ack extack;
331	struct nft_chain *chain;
332	struct nft_rule *rule;
333
334	chain = &basechain->chain;
335	list_for_each_entry(rule, &chain->rules, list) {
336		memset(&extack, 0, sizeof(extack));
337		nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
338					   &extack, FLOW_CLS_DESTROY);
339		nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
340	}
341
342	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
343		list_del(&block_cb->list);
344		flow_block_cb_free(block_cb);
345	}
346
347	return 0;
348}
349
350static int nft_block_setup(struct nft_base_chain *basechain,
351			   struct flow_block_offload *bo,
352			   enum flow_block_command cmd)
353{
354	int err;
355
356	switch (cmd) {
357	case FLOW_BLOCK_BIND:
358		err = nft_flow_offload_bind(bo, basechain);
359		break;
360	case FLOW_BLOCK_UNBIND:
361		err = nft_flow_offload_unbind(bo, basechain);
362		break;
363	default:
364		WARN_ON_ONCE(1);
365		err = -EOPNOTSUPP;
366	}
367
368	return err;
369}
370
371static void nft_flow_block_offload_init(struct flow_block_offload *bo,
372					struct net *net,
373					enum flow_block_command cmd,
374					struct nft_base_chain *basechain,
375					struct netlink_ext_ack *extack)
376{
377	memset(bo, 0, sizeof(*bo));
378	bo->net		= net;
379	bo->block	= &basechain->flow_block;
380	bo->command	= cmd;
381	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
382	bo->extack	= extack;
383	bo->cb_list_head = &basechain->flow_block.cb_list;
384	INIT_LIST_HEAD(&bo->cb_list);
385}
386
387static int nft_block_offload_cmd(struct nft_base_chain *chain,
388				 struct net_device *dev,
389				 enum flow_block_command cmd)
390{
391	struct netlink_ext_ack extack = {};
392	struct flow_block_offload bo;
393	int err;
394
395	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
396
397	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
398	if (err < 0)
399		return err;
400
401	return nft_block_setup(chain, &bo, cmd);
402}
403
404static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
405{
406	struct nft_base_chain *basechain = block_cb->indr.data;
407	struct net_device *dev = block_cb->indr.dev;
408	struct netlink_ext_ack extack = {};
409	struct nftables_pernet *nft_net;
410	struct net *net = dev_net(dev);
411	struct flow_block_offload bo;
412
413	nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
414				    basechain, &extack);
415	nft_net = nft_pernet(net);
416	mutex_lock(&nft_net->commit_mutex);
417	list_del(&block_cb->driver_list);
418	list_move(&block_cb->list, &bo.cb_list);
419	nft_flow_offload_unbind(&bo, basechain);
420	mutex_unlock(&nft_net->commit_mutex);
421}
422
423static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
424				      struct net_device *dev,
425				      enum flow_block_command cmd)
426{
427	struct netlink_ext_ack extack = {};
428	struct flow_block_offload bo;
429	int err;
430
431	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
432
433	err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
434					  nft_indr_block_cleanup);
435	if (err < 0)
436		return err;
437
438	if (list_empty(&bo.cb_list))
439		return -EOPNOTSUPP;
440
441	return nft_block_setup(basechain, &bo, cmd);
442}
443
444static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
445				 struct net_device *dev,
446				 enum flow_block_command cmd)
447{
448	int err;
449
450	if (dev->netdev_ops->ndo_setup_tc)
451		err = nft_block_offload_cmd(basechain, dev, cmd);
452	else
453		err = nft_indr_block_offload_cmd(basechain, dev, cmd);
454
455	return err;
456}
457
458static int nft_flow_block_chain(struct nft_base_chain *basechain,
459				const struct net_device *this_dev,
460				enum flow_block_command cmd)
461{
462	struct nf_hook_ops *ops;
463	struct nft_hook *hook;
464	int err, i = 0;
465
466	list_for_each_entry(hook, &basechain->hook_list, list) {
467		list_for_each_entry(ops, &hook->ops_list, list) {
468			if (this_dev && this_dev != ops->dev)
469				continue;
470
471			err = nft_chain_offload_cmd(basechain, ops->dev, cmd);
472			if (err < 0 && cmd == FLOW_BLOCK_BIND) {
473				if (!this_dev)
474					goto err_flow_block;
475
476				return err;
477			}
478			i++;
479		}
480	}
481
482	return 0;
483
484err_flow_block:
485	list_for_each_entry(hook, &basechain->hook_list, list) {
486		list_for_each_entry(ops, &hook->ops_list, list) {
487			if (i-- <= 0)
488				break;
489
490			nft_chain_offload_cmd(basechain, ops->dev,
491					      FLOW_BLOCK_UNBIND);
492		}
493	}
494	return err;
495}
496
497static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
498				  enum flow_block_command cmd)
499{
500	struct nft_base_chain *basechain;
501	u8 policy;
502
503	if (!nft_is_base_chain(chain))
504		return -EOPNOTSUPP;
505
506	basechain = nft_base_chain(chain);
507	policy = ppolicy ? *ppolicy : basechain->policy;
508
509	/* Only default policy to accept is supported for now. */
510	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
511		return -EOPNOTSUPP;
512
513	return nft_flow_block_chain(basechain, NULL, cmd);
514}
515
516static void nft_flow_rule_offload_abort(struct net *net,
517					struct nft_trans *trans)
518{
519	struct nftables_pernet *nft_net = nft_pernet(net);
520	int err = 0;
521
522	list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) {
523		if (trans->table->family != NFPROTO_NETDEV)
524			continue;
525
526		switch (trans->msg_type) {
527		case NFT_MSG_NEWCHAIN:
528			if (!(nft_trans_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD) ||
529			    nft_trans_chain_update(trans))
530				continue;
531
532			err = nft_flow_offload_chain(nft_trans_chain(trans), NULL,
533						     FLOW_BLOCK_UNBIND);
534			break;
535		case NFT_MSG_DELCHAIN:
536			if (!(nft_trans_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD))
537				continue;
538
539			err = nft_flow_offload_chain(nft_trans_chain(trans), NULL,
540						     FLOW_BLOCK_BIND);
541			break;
542		case NFT_MSG_NEWRULE:
543			if (!(nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD))
544				continue;
545
546			err = nft_flow_offload_rule(nft_trans_rule_chain(trans),
547						    nft_trans_rule(trans),
548						    NULL, FLOW_CLS_DESTROY);
549			break;
550		case NFT_MSG_DELRULE:
551			if (!(nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD))
552				continue;
553
554			err = nft_flow_offload_rule(nft_trans_rule_chain(trans),
555						    nft_trans_rule(trans),
556						    nft_trans_flow_rule(trans),
557						    FLOW_CLS_REPLACE);
558			break;
559		}
560
561		if (WARN_ON_ONCE(err))
562			break;
563	}
564}
565
566int nft_flow_rule_offload_commit(struct net *net)
567{
568	struct nftables_pernet *nft_net = nft_pernet(net);
569	struct nft_trans *trans;
570	int err = 0;
571	u8 policy;
572
573	list_for_each_entry(trans, &nft_net->commit_list, list) {
574		if (trans->table->family != NFPROTO_NETDEV)
575			continue;
576
577		switch (trans->msg_type) {
578		case NFT_MSG_NEWCHAIN:
579			if (!(nft_trans_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD) ||
580			    nft_trans_chain_update(trans))
581				continue;
582
583			policy = nft_trans_chain_policy(trans);
584			err = nft_flow_offload_chain(nft_trans_chain(trans), &policy,
585						     FLOW_BLOCK_BIND);
586			break;
587		case NFT_MSG_DELCHAIN:
588			if (!(nft_trans_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD))
589				continue;
590
591			policy = nft_trans_chain_policy(trans);
592			err = nft_flow_offload_chain(nft_trans_chain(trans), &policy,
593						     FLOW_BLOCK_UNBIND);
594			break;
595		case NFT_MSG_NEWRULE:
596			if (!(nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD))
597				continue;
598
599			if (trans->flags & NLM_F_REPLACE ||
600			    !(trans->flags & NLM_F_APPEND)) {
601				err = -EOPNOTSUPP;
602				break;
603			}
604			err = nft_flow_offload_rule(nft_trans_rule_chain(trans),
605						    nft_trans_rule(trans),
606						    nft_trans_flow_rule(trans),
607						    FLOW_CLS_REPLACE);
608			break;
609		case NFT_MSG_DELRULE:
610			if (!(nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD))
611				continue;
612
613			err = nft_flow_offload_rule(nft_trans_rule_chain(trans),
614						    nft_trans_rule(trans),
615						    NULL, FLOW_CLS_DESTROY);
616			break;
617		}
618
619		if (err) {
620			nft_flow_rule_offload_abort(net, trans);
621			break;
622		}
623	}
624
625	return err;
626}
627
628static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net,
629						 struct net_device *dev)
630{
631	struct nft_base_chain *basechain;
632	struct nft_hook *hook, *found;
633	const struct nft_table *table;
634	struct nft_chain *chain;
635
636	list_for_each_entry(table, &nft_net->tables, list) {
637		if (table->family != NFPROTO_NETDEV)
638			continue;
639
640		list_for_each_entry(chain, &table->chains, list) {
641			if (!nft_is_base_chain(chain) ||
642			    !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
643				continue;
644
645			found = NULL;
646			basechain = nft_base_chain(chain);
647			list_for_each_entry(hook, &basechain->hook_list, list) {
648				if (!nft_hook_find_ops(hook, dev))
649					continue;
650
651				found = hook;
652				break;
653			}
654			if (!found)
655				continue;
656
657			return chain;
658		}
659	}
660
661	return NULL;
662}
663
664static int nft_offload_netdev_event(struct notifier_block *this,
665				    unsigned long event, void *ptr)
666{
667	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
668	struct nftables_pernet *nft_net;
669	struct net *net = dev_net(dev);
670	struct nft_chain *chain;
671
672	if (event != NETDEV_UNREGISTER)
673		return NOTIFY_DONE;
674
675	nft_net = nft_pernet(net);
676	mutex_lock(&nft_net->commit_mutex);
677	chain = __nft_offload_get_chain(nft_net, dev);
678	if (chain)
679		nft_flow_block_chain(nft_base_chain(chain), dev,
680				     FLOW_BLOCK_UNBIND);
681
682	mutex_unlock(&nft_net->commit_mutex);
683
684	return NOTIFY_DONE;
685}
686
687static struct notifier_block nft_offload_netdev_notifier = {
688	.notifier_call	= nft_offload_netdev_event,
689};
690
691int nft_offload_init(void)
692{
693	return register_netdevice_notifier(&nft_offload_netdev_notifier);
694}
695
696void nft_offload_exit(void)
697{
698	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
699}