nft_flow_offload.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2#include <linux/kernel.h>
  3#include <linux/module.h>
  4#include <linux/init.h>
  5#include <linux/netlink.h>
  6#include <linux/netfilter.h>
  7#include <linux/workqueue.h>
  8#include <linux/spinlock.h>
  9#include <linux/netfilter/nf_conntrack_common.h>
 10#include <linux/netfilter/nf_tables.h>
 11#include <net/ip.h>
 12#include <net/flow.h>
 13#include <net/netfilter/nf_tables.h>
 14#include <net/netfilter/nf_tables_core.h>
 15#include <net/netfilter/nf_conntrack_core.h>
 16#include <net/netfilter/nf_conntrack_extend.h>
 17#include <net/netfilter/nf_flow_table.h>
 18
 19struct nft_flow_offload {
 20	struct nft_flowtable	*flowtable;
 21};
 22
 23static bool nft_flow_offload_skip(struct sk_buff *skb, int family)
 24{
 25	if (skb_sec_path(skb))
 26		return true;
 27
 28	if (family == NFPROTO_IPV4) {
 29		const struct ip_options *opt;
 30
 31		opt = &(IPCB(skb)->opt);
 32
 33		if (unlikely(opt->optlen))
 34			return true;
 35	}
 36
 37	return false;
 38}
 39
 40static void flow_offload_ct_tcp(struct nf_conn *ct)
 41{
 42	/* conntrack will not see all packets, disable tcp window validation. */
 43	spin_lock_bh(&ct->lock);
 44	ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
 45	ct->proto.tcp.seen[1].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
 46	spin_unlock_bh(&ct->lock);
 47}
 48
 49static void nft_flow_offload_eval(const struct nft_expr *expr,
 50				  struct nft_regs *regs,
 51				  const struct nft_pktinfo *pkt)
 52{
 53	struct nft_flow_offload *priv = nft_expr_priv(expr);
 54	struct nf_flowtable *flowtable = &priv->flowtable->data;
 55	struct tcphdr _tcph, *tcph = NULL;
 56	struct nf_flow_route route = {};
 57	enum ip_conntrack_info ctinfo;
 58	struct flow_offload *flow;
 59	enum ip_conntrack_dir dir;
 60	struct nf_conn *ct;
 61	int ret;
 62
 63	if (nft_flow_offload_skip(pkt->skb, nft_pf(pkt)))
 64		goto out;
 65
 66	ct = nf_ct_get(pkt->skb, &ctinfo);
 67	if (!ct)
 68		goto out;
 69
 70	switch (ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.protonum) {
 71	case IPPROTO_TCP:
 72		tcph = skb_header_pointer(pkt->skb, nft_thoff(pkt),
 73					  sizeof(_tcph), &_tcph);
 74		if (unlikely(!tcph || tcph->fin || tcph->rst ||
 75			     !nf_conntrack_tcp_established(ct)))
 76			goto out;
 77		break;
 78	case IPPROTO_UDP:
 79		break;
 80#ifdef CONFIG_NF_CT_PROTO_GRE
 81	case IPPROTO_GRE: {
 82		struct nf_conntrack_tuple *tuple;
 83
 84		if (ct->status & IPS_NAT_MASK)
 85			goto out;
 86		tuple = &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple;
 87		/* No support for GRE v1 */
 88		if (tuple->src.u.gre.key || tuple->dst.u.gre.key)
 89			goto out;
 90		break;
 91	}
 92#endif
 93	default:
 94		goto out;
 95	}
 96
 97	if (nf_ct_ext_exist(ct, NF_CT_EXT_HELPER) ||
 98	    ct->status & (IPS_SEQ_ADJUST | IPS_NAT_CLASH))
 99		goto out;
100
101	if (!nf_ct_is_confirmed(ct))
102		goto out;
103
104	if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status))
105		goto out;
106
107	dir = CTINFO2DIR(ctinfo);
108	if (nft_flow_route(pkt, ct, &route, dir, priv->flowtable) < 0)
109		goto err_flow_route;
110
111	flow = flow_offload_alloc(ct);
112	if (!flow)
113		goto err_flow_alloc;
114
115	flow_offload_route_init(flow, &route);
116	if (tcph)
117		flow_offload_ct_tcp(ct);
118
119	__set_bit(NF_FLOW_HW_BIDIRECTIONAL, &flow->flags);
120	ret = flow_offload_add(flowtable, flow);
121	if (ret < 0)
122		goto err_flow_add;
123
124	return;
125
126err_flow_add:
127	flow_offload_free(flow);
128err_flow_alloc:
129	dst_release(route.tuple[dir].dst);
130	dst_release(route.tuple[!dir].dst);
131err_flow_route:
132	clear_bit(IPS_OFFLOAD_BIT, &ct->status);
133out:
134	regs->verdict.code = NFT_BREAK;
135}
136
137static int nft_flow_offload_validate(const struct nft_ctx *ctx,
138				     const struct nft_expr *expr)
139{
140	unsigned int hook_mask = (1 << NF_INET_FORWARD);
141
142	if (ctx->family != NFPROTO_IPV4 &&
143	    ctx->family != NFPROTO_IPV6 &&
144	    ctx->family != NFPROTO_INET)
145		return -EOPNOTSUPP;
146
147	return nft_chain_validate_hooks(ctx->chain, hook_mask);
148}
149
150static const struct nla_policy nft_flow_offload_policy[NFTA_FLOW_MAX + 1] = {
151	[NFTA_FLOW_TABLE_NAME]	= { .type = NLA_STRING,
152				    .len = NFT_NAME_MAXLEN - 1 },
153};
154
155static int nft_flow_offload_init(const struct nft_ctx *ctx,
156				 const struct nft_expr *expr,
157				 const struct nlattr * const tb[])
158{
159	struct nft_flow_offload *priv = nft_expr_priv(expr);
160	u8 genmask = nft_genmask_next(ctx->net);
161	struct nft_flowtable *flowtable;
162
163	if (!tb[NFTA_FLOW_TABLE_NAME])
164		return -EINVAL;
165
166	flowtable = nft_flowtable_lookup(ctx->net, ctx->table,
167					 tb[NFTA_FLOW_TABLE_NAME], genmask);
168	if (IS_ERR(flowtable))
169		return PTR_ERR(flowtable);
170
171	if (!nft_use_inc(&flowtable->use))
172		return -EMFILE;
173
174	priv->flowtable = flowtable;
175
176	return nf_ct_netns_get(ctx->net, ctx->family);
177}
178
179static void nft_flow_offload_deactivate(const struct nft_ctx *ctx,
180					const struct nft_expr *expr,
181					enum nft_trans_phase phase)
182{
183	struct nft_flow_offload *priv = nft_expr_priv(expr);
184
185	nf_tables_deactivate_flowtable(ctx, priv->flowtable, phase);
186}
187
188static void nft_flow_offload_activate(const struct nft_ctx *ctx,
189				      const struct nft_expr *expr)
190{
191	struct nft_flow_offload *priv = nft_expr_priv(expr);
192
193	nft_use_inc_restore(&priv->flowtable->use);
194}
195
196static void nft_flow_offload_destroy(const struct nft_ctx *ctx,
197				     const struct nft_expr *expr)
198{
199	nf_ct_netns_put(ctx->net, ctx->family);
200}
201
202static int nft_flow_offload_dump(struct sk_buff *skb,
203				 const struct nft_expr *expr, bool reset)
204{
205	struct nft_flow_offload *priv = nft_expr_priv(expr);
206
207	if (nla_put_string(skb, NFTA_FLOW_TABLE_NAME, priv->flowtable->name))
208		goto nla_put_failure;
209
210	return 0;
211
212nla_put_failure:
213	return -1;
214}
215
216static struct nft_expr_type nft_flow_offload_type;
217static const struct nft_expr_ops nft_flow_offload_ops = {
218	.type		= &nft_flow_offload_type,
219	.size		= NFT_EXPR_SIZE(sizeof(struct nft_flow_offload)),
220	.eval		= nft_flow_offload_eval,
221	.init		= nft_flow_offload_init,
222	.activate	= nft_flow_offload_activate,
223	.deactivate	= nft_flow_offload_deactivate,
224	.destroy	= nft_flow_offload_destroy,
225	.validate	= nft_flow_offload_validate,
226	.dump		= nft_flow_offload_dump,
227	.reduce		= NFT_REDUCE_READONLY,
228};
229
230static struct nft_expr_type nft_flow_offload_type __read_mostly = {
231	.name		= "flow_offload",
232	.ops		= &nft_flow_offload_ops,
233	.policy		= nft_flow_offload_policy,
234	.maxattr	= NFTA_FLOW_MAX,
235	.owner		= THIS_MODULE,
236};
237
238static int flow_offload_netdev_event(struct notifier_block *this,
239				     unsigned long event, void *ptr)
240{
241	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
242
243	if (event != NETDEV_DOWN)
244		return NOTIFY_DONE;
245
246	nf_flow_table_cleanup(dev);
247
248	return NOTIFY_DONE;
249}
250
251static struct notifier_block flow_offload_netdev_notifier = {
252	.notifier_call	= flow_offload_netdev_event,
253};
254
255static int __init nft_flow_offload_module_init(void)
256{
257	int err;
258
259	err = register_netdevice_notifier(&flow_offload_netdev_notifier);
260	if (err)
261		goto err;
262
263	err = nft_register_expr(&nft_flow_offload_type);
264	if (err < 0)
265		goto register_expr;
266
267	return 0;
268
269register_expr:
270	unregister_netdevice_notifier(&flow_offload_netdev_notifier);
271err:
272	return err;
273}
274
275static void __exit nft_flow_offload_module_exit(void)
276{
277	nft_unregister_expr(&nft_flow_offload_type);
278	unregister_netdevice_notifier(&flow_offload_netdev_notifier);
279}
280
281module_init(nft_flow_offload_module_init);
282module_exit(nft_flow_offload_module_exit);
283
284MODULE_LICENSE("GPL");
285MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>");
286MODULE_ALIAS_NFT_EXPR("flow_offload");
287MODULE_DESCRIPTION("nftables hardware flow offload module");