nf_flow_table_offload.c

   1#include <linux/kernel.h>
   2#include <linux/init.h>
   3#include <linux/module.h>
   4#include <linux/netfilter.h>
   5#include <linux/rhashtable.h>
   6#include <linux/netdevice.h>
   7#include <linux/tc_act/tc_csum.h>
   8#include <net/flow_offload.h>
   9#include <net/netfilter/nf_flow_table.h>
  10#include <net/netfilter/nf_tables.h>
  11#include <net/netfilter/nf_conntrack.h>
  12#include <net/netfilter/nf_conntrack_acct.h>
  13#include <net/netfilter/nf_conntrack_core.h>
  14#include <net/netfilter/nf_conntrack_tuple.h>
  15
  16#define NF_FLOW_RULE_ACTION_MAX	24
  17
  18static struct workqueue_struct *nf_flow_offload_add_wq;
  19static struct workqueue_struct *nf_flow_offload_del_wq;
  20static struct workqueue_struct *nf_flow_offload_stats_wq;
  21
  22struct flow_offload_work {
  23	struct list_head	list;
  24	enum flow_cls_command	cmd;
  25	struct nf_flowtable	*flowtable;
  26	struct flow_offload	*flow;
  27	struct work_struct	work;
  28};
  29
  30#define NF_FLOW_DISSECTOR(__match, __type, __field)	\
  31	(__match)->dissector.offset[__type] =		\
  32		offsetof(struct nf_flow_key, __field)
  33
  34static void nf_flow_rule_lwt_match(struct nf_flow_match *match,
  35				   struct ip_tunnel_info *tun_info)
  36{
  37	struct nf_flow_key *mask = &match->mask;
  38	struct nf_flow_key *key = &match->key;
  39	unsigned long long enc_keys;
  40
  41	if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX))
  42		return;
  43
  44	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control);
  45	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
  46	key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id);
  47	mask->enc_key_id.keyid = 0xffffffff;
  48	enc_keys = BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
  49		   BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL);
  50
  51	if (ip_tunnel_info_af(tun_info) == AF_INET) {
  52		NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
  53				  enc_ipv4);
  54		key->enc_ipv4.src = tun_info->key.u.ipv4.dst;
  55		key->enc_ipv4.dst = tun_info->key.u.ipv4.src;
  56		if (key->enc_ipv4.src)
  57			mask->enc_ipv4.src = 0xffffffff;
  58		if (key->enc_ipv4.dst)
  59			mask->enc_ipv4.dst = 0xffffffff;
  60		enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
  61		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
  62	} else {
  63		memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst,
  64		       sizeof(struct in6_addr));
  65		memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src,
  66		       sizeof(struct in6_addr));
  67		if (memcmp(&key->enc_ipv6.src, &in6addr_any,
  68			   sizeof(struct in6_addr)))
  69			memset(&mask->enc_ipv6.src, 0xff,
  70			       sizeof(struct in6_addr));
  71		if (memcmp(&key->enc_ipv6.dst, &in6addr_any,
  72			   sizeof(struct in6_addr)))
  73			memset(&mask->enc_ipv6.dst, 0xff,
  74			       sizeof(struct in6_addr));
  75		enc_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS);
  76		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
  77	}
  78
  79	match->dissector.used_keys |= enc_keys;
  80}
  81
  82static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key,
  83				    struct flow_dissector_key_vlan *mask,
  84				    u16 vlan_id, __be16 proto)
  85{
  86	key->vlan_id = vlan_id;
  87	mask->vlan_id = VLAN_VID_MASK;
  88	key->vlan_tpid = proto;
  89	mask->vlan_tpid = 0xffff;
  90}
  91
  92static int nf_flow_rule_match(struct nf_flow_match *match,
  93			      const struct flow_offload_tuple *tuple,
  94			      struct dst_entry *other_dst)
  95{
  96	struct nf_flow_key *mask = &match->mask;
  97	struct nf_flow_key *key = &match->key;
  98	struct ip_tunnel_info *tun_info;
  99	bool vlan_encap = false;
 100
 101	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta);
 102	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control);
 103	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic);
 104	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
 105	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
 106	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp);
 107	NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp);
 108
 109	if (other_dst && other_dst->lwtstate) {
 110		tun_info = lwt_tun_info(other_dst->lwtstate);
 111		nf_flow_rule_lwt_match(match, tun_info);
 112	}
 113
 114	if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC)
 115		key->meta.ingress_ifindex = tuple->tc.iifidx;
 116	else
 117		key->meta.ingress_ifindex = tuple->iifidx;
 118
 119	mask->meta.ingress_ifindex = 0xffffffff;
 120
 121	if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) &&
 122	    tuple->encap[0].proto == htons(ETH_P_8021Q)) {
 123		NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan);
 124		nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
 125					tuple->encap[0].id,
 126					tuple->encap[0].proto);
 127		vlan_encap = true;
 128	}
 129
 130	if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) &&
 131	    tuple->encap[1].proto == htons(ETH_P_8021Q)) {
 132		if (vlan_encap) {
 133			NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN,
 134					  cvlan);
 135			nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan,
 136						tuple->encap[1].id,
 137						tuple->encap[1].proto);
 138		} else {
 139			NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN,
 140					  vlan);
 141			nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
 142						tuple->encap[1].id,
 143						tuple->encap[1].proto);
 144		}
 145	}
 146
 147	switch (tuple->l3proto) {
 148	case AF_INET:
 149		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
 150		key->basic.n_proto = htons(ETH_P_IP);
 151		key->ipv4.src = tuple->src_v4.s_addr;
 152		mask->ipv4.src = 0xffffffff;
 153		key->ipv4.dst = tuple->dst_v4.s_addr;
 154		mask->ipv4.dst = 0xffffffff;
 155		break;
 156       case AF_INET6:
 157		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
 158		key->basic.n_proto = htons(ETH_P_IPV6);
 159		key->ipv6.src = tuple->src_v6;
 160		memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src));
 161		key->ipv6.dst = tuple->dst_v6;
 162		memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst));
 163		break;
 164	default:
 165		return -EOPNOTSUPP;
 166	}
 167	mask->control.addr_type = 0xffff;
 168	match->dissector.used_keys |= BIT_ULL(key->control.addr_type);
 169	mask->basic.n_proto = 0xffff;
 170
 171	switch (tuple->l4proto) {
 172	case IPPROTO_TCP:
 173		key->tcp.flags = 0;
 174		mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16);
 175		match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_TCP);
 176		break;
 177	case IPPROTO_UDP:
 178	case IPPROTO_GRE:
 179		break;
 180	default:
 181		return -EOPNOTSUPP;
 182	}
 183
 184	key->basic.ip_proto = tuple->l4proto;
 185	mask->basic.ip_proto = 0xff;
 186
 187	match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_META) |
 188				      BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
 189				      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC);
 190
 191	switch (tuple->l4proto) {
 192	case IPPROTO_TCP:
 193	case IPPROTO_UDP:
 194		key->tp.src = tuple->src_port;
 195		mask->tp.src = 0xffff;
 196		key->tp.dst = tuple->dst_port;
 197		mask->tp.dst = 0xffff;
 198
 199		match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_PORTS);
 200		break;
 201	}
 202
 203	return 0;
 204}
 205
 206static void flow_offload_mangle(struct flow_action_entry *entry,
 207				enum flow_action_mangle_base htype, u32 offset,
 208				const __be32 *value, const __be32 *mask)
 209{
 210	entry->id = FLOW_ACTION_MANGLE;
 211	entry->mangle.htype = htype;
 212	entry->mangle.offset = offset;
 213	memcpy(&entry->mangle.mask, mask, sizeof(u32));
 214	memcpy(&entry->mangle.val, value, sizeof(u32));
 215}
 216
 217static inline struct flow_action_entry *
 218flow_action_entry_next(struct nf_flow_rule *flow_rule)
 219{
 220	int i;
 221
 222	if (unlikely(flow_rule->rule->action.num_entries >= NF_FLOW_RULE_ACTION_MAX))
 223		return NULL;
 224
 225	i = flow_rule->rule->action.num_entries++;
 226
 227	return &flow_rule->rule->action.entries[i];
 228}
 229
 230static int flow_offload_eth_src(struct net *net,
 231				const struct flow_offload *flow,
 232				enum flow_offload_tuple_dir dir,
 233				struct nf_flow_rule *flow_rule)
 234{
 235	struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
 236	struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
 237	const struct flow_offload_tuple *other_tuple, *this_tuple;
 238	struct net_device *dev = NULL;
 239	const unsigned char *addr;
 240	u32 mask, val;
 241	u16 val16;
 242
 243	if (!entry0 || !entry1)
 244		return -E2BIG;
 245
 246	this_tuple = &flow->tuplehash[dir].tuple;
 247
 248	switch (this_tuple->xmit_type) {
 249	case FLOW_OFFLOAD_XMIT_DIRECT:
 250		addr = this_tuple->out.h_source;
 251		break;
 252	case FLOW_OFFLOAD_XMIT_NEIGH:
 253		other_tuple = &flow->tuplehash[!dir].tuple;
 254		dev = dev_get_by_index(net, other_tuple->iifidx);
 255		if (!dev)
 256			return -ENOENT;
 257
 258		addr = dev->dev_addr;
 259		break;
 260	default:
 261		return -EOPNOTSUPP;
 262	}
 263
 264	mask = ~0xffff0000;
 265	memcpy(&val16, addr, 2);
 266	val = val16 << 16;
 267	flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
 268			    &val, &mask);
 269
 270	mask = ~0xffffffff;
 271	memcpy(&val, addr + 2, 4);
 272	flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8,
 273			    &val, &mask);
 274
 275	dev_put(dev);
 276
 277	return 0;
 278}
 279
 280static int flow_offload_eth_dst(struct net *net,
 281				const struct flow_offload *flow,
 282				enum flow_offload_tuple_dir dir,
 283				struct nf_flow_rule *flow_rule)
 284{
 285	struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
 286	struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
 287	const struct flow_offload_tuple *other_tuple, *this_tuple;
 288	const struct dst_entry *dst_cache;
 289	unsigned char ha[ETH_ALEN];
 290	struct neighbour *n;
 291	const void *daddr;
 292	u32 mask, val;
 293	u8 nud_state;
 294	u16 val16;
 295
 296	if (!entry0 || !entry1)
 297		return -E2BIG;
 298
 299	this_tuple = &flow->tuplehash[dir].tuple;
 300
 301	switch (this_tuple->xmit_type) {
 302	case FLOW_OFFLOAD_XMIT_DIRECT:
 303		ether_addr_copy(ha, this_tuple->out.h_dest);
 304		break;
 305	case FLOW_OFFLOAD_XMIT_NEIGH:
 306		other_tuple = &flow->tuplehash[!dir].tuple;
 307		daddr = &other_tuple->src_v4;
 308		dst_cache = this_tuple->dst_cache;
 309		n = dst_neigh_lookup(dst_cache, daddr);
 310		if (!n)
 311			return -ENOENT;
 312
 313		read_lock_bh(&n->lock);
 314		nud_state = n->nud_state;
 315		ether_addr_copy(ha, n->ha);
 316		read_unlock_bh(&n->lock);
 317		neigh_release(n);
 318
 319		if (!(nud_state & NUD_VALID))
 320			return -ENOENT;
 321		break;
 322	default:
 323		return -EOPNOTSUPP;
 324	}
 325
 326	mask = ~0xffffffff;
 327	memcpy(&val, ha, 4);
 328	flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0,
 329			    &val, &mask);
 330
 331	mask = ~0x0000ffff;
 332	memcpy(&val16, ha + 4, 2);
 333	val = val16;
 334	flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
 335			    &val, &mask);
 336
 337	return 0;
 338}
 339
 340static int flow_offload_ipv4_snat(struct net *net,
 341				  const struct flow_offload *flow,
 342				  enum flow_offload_tuple_dir dir,
 343				  struct nf_flow_rule *flow_rule)
 344{
 345	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 346	u32 mask = ~htonl(0xffffffff);
 347	__be32 addr;
 348	u32 offset;
 349
 350	if (!entry)
 351		return -E2BIG;
 352
 353	switch (dir) {
 354	case FLOW_OFFLOAD_DIR_ORIGINAL:
 355		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
 356		offset = offsetof(struct iphdr, saddr);
 357		break;
 358	case FLOW_OFFLOAD_DIR_REPLY:
 359		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
 360		offset = offsetof(struct iphdr, daddr);
 361		break;
 362	default:
 363		return -EOPNOTSUPP;
 364	}
 365
 366	flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
 367			    &addr, &mask);
 368	return 0;
 369}
 370
 371static int flow_offload_ipv4_dnat(struct net *net,
 372				  const struct flow_offload *flow,
 373				  enum flow_offload_tuple_dir dir,
 374				  struct nf_flow_rule *flow_rule)
 375{
 376	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 377	u32 mask = ~htonl(0xffffffff);
 378	__be32 addr;
 379	u32 offset;
 380
 381	if (!entry)
 382		return -E2BIG;
 383
 384	switch (dir) {
 385	case FLOW_OFFLOAD_DIR_ORIGINAL:
 386		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
 387		offset = offsetof(struct iphdr, daddr);
 388		break;
 389	case FLOW_OFFLOAD_DIR_REPLY:
 390		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
 391		offset = offsetof(struct iphdr, saddr);
 392		break;
 393	default:
 394		return -EOPNOTSUPP;
 395	}
 396
 397	flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
 398			    &addr, &mask);
 399	return 0;
 400}
 401
 402static int flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule,
 403				     unsigned int offset,
 404				     const __be32 *addr, const __be32 *mask)
 405{
 406	struct flow_action_entry *entry;
 407	int i;
 408
 409	for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++) {
 410		entry = flow_action_entry_next(flow_rule);
 411		if (!entry)
 412			return -E2BIG;
 413
 414		flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
 415				    offset + i * sizeof(u32), &addr[i], mask);
 416	}
 417
 418	return 0;
 419}
 420
 421static int flow_offload_ipv6_snat(struct net *net,
 422				  const struct flow_offload *flow,
 423				  enum flow_offload_tuple_dir dir,
 424				  struct nf_flow_rule *flow_rule)
 425{
 426	u32 mask = ~htonl(0xffffffff);
 427	const __be32 *addr;
 428	u32 offset;
 429
 430	switch (dir) {
 431	case FLOW_OFFLOAD_DIR_ORIGINAL:
 432		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32;
 433		offset = offsetof(struct ipv6hdr, saddr);
 434		break;
 435	case FLOW_OFFLOAD_DIR_REPLY:
 436		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32;
 437		offset = offsetof(struct ipv6hdr, daddr);
 438		break;
 439	default:
 440		return -EOPNOTSUPP;
 441	}
 442
 443	return flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
 444}
 445
 446static int flow_offload_ipv6_dnat(struct net *net,
 447				  const struct flow_offload *flow,
 448				  enum flow_offload_tuple_dir dir,
 449				  struct nf_flow_rule *flow_rule)
 450{
 451	u32 mask = ~htonl(0xffffffff);
 452	const __be32 *addr;
 453	u32 offset;
 454
 455	switch (dir) {
 456	case FLOW_OFFLOAD_DIR_ORIGINAL:
 457		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32;
 458		offset = offsetof(struct ipv6hdr, daddr);
 459		break;
 460	case FLOW_OFFLOAD_DIR_REPLY:
 461		addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32;
 462		offset = offsetof(struct ipv6hdr, saddr);
 463		break;
 464	default:
 465		return -EOPNOTSUPP;
 466	}
 467
 468	return flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
 469}
 470
 471static int flow_offload_l4proto(const struct flow_offload *flow)
 472{
 473	u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
 474	u8 type = 0;
 475
 476	switch (protonum) {
 477	case IPPROTO_TCP:
 478		type = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
 479		break;
 480	case IPPROTO_UDP:
 481		type = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
 482		break;
 483	default:
 484		break;
 485	}
 486
 487	return type;
 488}
 489
 490static int flow_offload_port_snat(struct net *net,
 491				  const struct flow_offload *flow,
 492				  enum flow_offload_tuple_dir dir,
 493				  struct nf_flow_rule *flow_rule)
 494{
 495	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 496	u32 mask, port;
 497	u32 offset;
 498
 499	if (!entry)
 500		return -E2BIG;
 501
 502	switch (dir) {
 503	case FLOW_OFFLOAD_DIR_ORIGINAL:
 504		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port);
 505		offset = 0; /* offsetof(struct tcphdr, source); */
 506		port = htonl(port << 16);
 507		mask = ~htonl(0xffff0000);
 508		break;
 509	case FLOW_OFFLOAD_DIR_REPLY:
 510		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port);
 511		offset = 0; /* offsetof(struct tcphdr, dest); */
 512		port = htonl(port);
 513		mask = ~htonl(0xffff);
 514		break;
 515	default:
 516		return -EOPNOTSUPP;
 517	}
 518
 519	flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
 520			    &port, &mask);
 521	return 0;
 522}
 523
 524static int flow_offload_port_dnat(struct net *net,
 525				  const struct flow_offload *flow,
 526				  enum flow_offload_tuple_dir dir,
 527				  struct nf_flow_rule *flow_rule)
 528{
 529	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 530	u32 mask, port;
 531	u32 offset;
 532
 533	if (!entry)
 534		return -E2BIG;
 535
 536	switch (dir) {
 537	case FLOW_OFFLOAD_DIR_ORIGINAL:
 538		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port);
 539		offset = 0; /* offsetof(struct tcphdr, dest); */
 540		port = htonl(port);
 541		mask = ~htonl(0xffff);
 542		break;
 543	case FLOW_OFFLOAD_DIR_REPLY:
 544		port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port);
 545		offset = 0; /* offsetof(struct tcphdr, source); */
 546		port = htonl(port << 16);
 547		mask = ~htonl(0xffff0000);
 548		break;
 549	default:
 550		return -EOPNOTSUPP;
 551	}
 552
 553	flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
 554			    &port, &mask);
 555	return 0;
 556}
 557
 558static int flow_offload_ipv4_checksum(struct net *net,
 559				      const struct flow_offload *flow,
 560				      struct nf_flow_rule *flow_rule)
 561{
 562	u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
 563	struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 564
 565	if (!entry)
 566		return -E2BIG;
 567
 568	entry->id = FLOW_ACTION_CSUM;
 569	entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR;
 570
 571	switch (protonum) {
 572	case IPPROTO_TCP:
 573		entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP;
 574		break;
 575	case IPPROTO_UDP:
 576		entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP;
 577		break;
 578	}
 579
 580	return 0;
 581}
 582
 583static int flow_offload_redirect(struct net *net,
 584				 const struct flow_offload *flow,
 585				 enum flow_offload_tuple_dir dir,
 586				 struct nf_flow_rule *flow_rule)
 587{
 588	const struct flow_offload_tuple *this_tuple, *other_tuple;
 589	struct flow_action_entry *entry;
 590	struct net_device *dev;
 591	int ifindex;
 592
 593	this_tuple = &flow->tuplehash[dir].tuple;
 594	switch (this_tuple->xmit_type) {
 595	case FLOW_OFFLOAD_XMIT_DIRECT:
 596		this_tuple = &flow->tuplehash[dir].tuple;
 597		ifindex = this_tuple->out.ifidx;
 598		break;
 599	case FLOW_OFFLOAD_XMIT_NEIGH:
 600		other_tuple = &flow->tuplehash[!dir].tuple;
 601		ifindex = other_tuple->iifidx;
 602		break;
 603	default:
 604		return -EOPNOTSUPP;
 605	}
 606
 607	dev = dev_get_by_index(net, ifindex);
 608	if (!dev)
 609		return -ENODEV;
 610
 611	entry = flow_action_entry_next(flow_rule);
 612	if (!entry) {
 613		dev_put(dev);
 614		return -E2BIG;
 615	}
 616
 617	entry->id = FLOW_ACTION_REDIRECT;
 618	entry->dev = dev;
 619
 620	return 0;
 621}
 622
 623static int flow_offload_encap_tunnel(const struct flow_offload *flow,
 624				     enum flow_offload_tuple_dir dir,
 625				     struct nf_flow_rule *flow_rule)
 626{
 627	const struct flow_offload_tuple *this_tuple;
 628	struct flow_action_entry *entry;
 629	struct dst_entry *dst;
 630
 631	this_tuple = &flow->tuplehash[dir].tuple;
 632	if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
 633		return 0;
 634
 635	dst = this_tuple->dst_cache;
 636	if (dst && dst->lwtstate) {
 637		struct ip_tunnel_info *tun_info;
 638
 639		tun_info = lwt_tun_info(dst->lwtstate);
 640		if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
 641			entry = flow_action_entry_next(flow_rule);
 642			if (!entry)
 643				return -E2BIG;
 644			entry->id = FLOW_ACTION_TUNNEL_ENCAP;
 645			entry->tunnel = tun_info;
 646		}
 647	}
 648
 649	return 0;
 650}
 651
 652static int flow_offload_decap_tunnel(const struct flow_offload *flow,
 653				     enum flow_offload_tuple_dir dir,
 654				     struct nf_flow_rule *flow_rule)
 655{
 656	const struct flow_offload_tuple *other_tuple;
 657	struct flow_action_entry *entry;
 658	struct dst_entry *dst;
 659
 660	other_tuple = &flow->tuplehash[!dir].tuple;
 661	if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
 662		return 0;
 663
 664	dst = other_tuple->dst_cache;
 665	if (dst && dst->lwtstate) {
 666		struct ip_tunnel_info *tun_info;
 667
 668		tun_info = lwt_tun_info(dst->lwtstate);
 669		if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
 670			entry = flow_action_entry_next(flow_rule);
 671			if (!entry)
 672				return -E2BIG;
 673			entry->id = FLOW_ACTION_TUNNEL_DECAP;
 674		}
 675	}
 676
 677	return 0;
 678}
 679
 680static int
 681nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow,
 682			  enum flow_offload_tuple_dir dir,
 683			  struct nf_flow_rule *flow_rule)
 684{
 685	const struct flow_offload_tuple *other_tuple;
 686	const struct flow_offload_tuple *tuple;
 687	int i;
 688
 689	if (flow_offload_decap_tunnel(flow, dir, flow_rule) < 0 ||
 690	    flow_offload_encap_tunnel(flow, dir, flow_rule) < 0)
 691		return -1;
 692
 693	if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
 694	    flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
 695		return -1;
 696
 697	tuple = &flow->tuplehash[dir].tuple;
 698
 699	for (i = 0; i < tuple->encap_num; i++) {
 700		struct flow_action_entry *entry;
 701
 702		if (tuple->in_vlan_ingress & BIT(i))
 703			continue;
 704
 705		if (tuple->encap[i].proto == htons(ETH_P_8021Q)) {
 706			entry = flow_action_entry_next(flow_rule);
 707			if (!entry)
 708				return -1;
 709			entry->id = FLOW_ACTION_VLAN_POP;
 710		}
 711	}
 712
 713	other_tuple = &flow->tuplehash[!dir].tuple;
 714
 715	for (i = 0; i < other_tuple->encap_num; i++) {
 716		struct flow_action_entry *entry;
 717
 718		if (other_tuple->in_vlan_ingress & BIT(i))
 719			continue;
 720
 721		entry = flow_action_entry_next(flow_rule);
 722		if (!entry)
 723			return -1;
 724
 725		switch (other_tuple->encap[i].proto) {
 726		case htons(ETH_P_PPP_SES):
 727			entry->id = FLOW_ACTION_PPPOE_PUSH;
 728			entry->pppoe.sid = other_tuple->encap[i].id;
 729			break;
 730		case htons(ETH_P_8021Q):
 731			entry->id = FLOW_ACTION_VLAN_PUSH;
 732			entry->vlan.vid = other_tuple->encap[i].id;
 733			entry->vlan.proto = other_tuple->encap[i].proto;
 734			break;
 735		}
 736	}
 737
 738	return 0;
 739}
 740
 741int nf_flow_rule_route_ipv4(struct net *net, struct flow_offload *flow,
 742			    enum flow_offload_tuple_dir dir,
 743			    struct nf_flow_rule *flow_rule)
 744{
 745	if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
 746		return -1;
 747
 748	if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
 749		if (flow_offload_ipv4_snat(net, flow, dir, flow_rule) < 0 ||
 750		    flow_offload_port_snat(net, flow, dir, flow_rule) < 0)
 751			return -1;
 752	}
 753	if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
 754		if (flow_offload_ipv4_dnat(net, flow, dir, flow_rule) < 0 ||
 755		    flow_offload_port_dnat(net, flow, dir, flow_rule) < 0)
 756			return -1;
 757	}
 758	if (test_bit(NF_FLOW_SNAT, &flow->flags) ||
 759	    test_bit(NF_FLOW_DNAT, &flow->flags))
 760		if (flow_offload_ipv4_checksum(net, flow, flow_rule) < 0)
 761			return -1;
 762
 763	if (flow_offload_redirect(net, flow, dir, flow_rule) < 0)
 764		return -1;
 765
 766	return 0;
 767}
 768EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4);
 769
 770int nf_flow_rule_route_ipv6(struct net *net, struct flow_offload *flow,
 771			    enum flow_offload_tuple_dir dir,
 772			    struct nf_flow_rule *flow_rule)
 773{
 774	if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
 775		return -1;
 776
 777	if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
 778		if (flow_offload_ipv6_snat(net, flow, dir, flow_rule) < 0 ||
 779		    flow_offload_port_snat(net, flow, dir, flow_rule) < 0)
 780			return -1;
 781	}
 782	if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
 783		if (flow_offload_ipv6_dnat(net, flow, dir, flow_rule) < 0 ||
 784		    flow_offload_port_dnat(net, flow, dir, flow_rule) < 0)
 785			return -1;
 786	}
 787
 788	if (flow_offload_redirect(net, flow, dir, flow_rule) < 0)
 789		return -1;
 790
 791	return 0;
 792}
 793EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6);
 794
 795static struct nf_flow_rule *
 796nf_flow_offload_rule_alloc(struct net *net,
 797			   const struct flow_offload_work *offload,
 798			   enum flow_offload_tuple_dir dir)
 799{
 800	const struct nf_flowtable *flowtable = offload->flowtable;
 801	const struct flow_offload_tuple *tuple, *other_tuple;
 802	struct flow_offload *flow = offload->flow;
 803	struct dst_entry *other_dst = NULL;
 804	struct nf_flow_rule *flow_rule;
 805	int err = -ENOMEM;
 806
 807	flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL);
 808	if (!flow_rule)
 809		goto err_flow;
 810
 811	flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX);
 812	if (!flow_rule->rule)
 813		goto err_flow_rule;
 814
 815	flow_rule->rule->match.dissector = &flow_rule->match.dissector;
 816	flow_rule->rule->match.mask = &flow_rule->match.mask;
 817	flow_rule->rule->match.key = &flow_rule->match.key;
 818
 819	tuple = &flow->tuplehash[dir].tuple;
 820	other_tuple = &flow->tuplehash[!dir].tuple;
 821	if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH)
 822		other_dst = other_tuple->dst_cache;
 823
 824	err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst);
 825	if (err < 0)
 826		goto err_flow_match;
 827
 828	flow_rule->rule->action.num_entries = 0;
 829	if (flowtable->type->action(net, flow, dir, flow_rule) < 0)
 830		goto err_flow_match;
 831
 832	return flow_rule;
 833
 834err_flow_match:
 835	kfree(flow_rule->rule);
 836err_flow_rule:
 837	kfree(flow_rule);
 838err_flow:
 839	return NULL;
 840}
 841
 842static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule)
 843{
 844	struct flow_action_entry *entry;
 845	int i;
 846
 847	for (i = 0; i < flow_rule->rule->action.num_entries; i++) {
 848		entry = &flow_rule->rule->action.entries[i];
 849		if (entry->id != FLOW_ACTION_REDIRECT)
 850			continue;
 851
 852		dev_put(entry->dev);
 853	}
 854	kfree(flow_rule->rule);
 855	kfree(flow_rule);
 856}
 857
 858static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[])
 859{
 860	int i;
 861
 862	for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++)
 863		__nf_flow_offload_destroy(flow_rule[i]);
 864}
 865
 866static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
 867				 struct nf_flow_rule *flow_rule[])
 868{
 869	struct net *net = read_pnet(&offload->flowtable->net);
 870
 871	flow_rule[0] = nf_flow_offload_rule_alloc(net, offload,
 872						  FLOW_OFFLOAD_DIR_ORIGINAL);
 873	if (!flow_rule[0])
 874		return -ENOMEM;
 875
 876	flow_rule[1] = nf_flow_offload_rule_alloc(net, offload,
 877						  FLOW_OFFLOAD_DIR_REPLY);
 878	if (!flow_rule[1]) {
 879		__nf_flow_offload_destroy(flow_rule[0]);
 880		return -ENOMEM;
 881	}
 882
 883	return 0;
 884}
 885
 886static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
 887				 __be16 proto, int priority,
 888				 enum flow_cls_command cmd,
 889				 const struct flow_offload_tuple *tuple,
 890				 struct netlink_ext_ack *extack)
 891{
 892	cls_flow->common.protocol = proto;
 893	cls_flow->common.prio = priority;
 894	cls_flow->common.extack = extack;
 895	cls_flow->command = cmd;
 896	cls_flow->cookie = (unsigned long)tuple;
 897}
 898
 899static int nf_flow_offload_tuple(struct nf_flowtable *flowtable,
 900				 struct flow_offload *flow,
 901				 struct nf_flow_rule *flow_rule,
 902				 enum flow_offload_tuple_dir dir,
 903				 int priority, int cmd,
 904				 struct flow_stats *stats,
 905				 struct list_head *block_cb_list)
 906{
 907	struct flow_cls_offload cls_flow = {};
 908	struct netlink_ext_ack extack = {};
 909	struct flow_block_cb *block_cb;
 910	__be16 proto = ETH_P_ALL;
 911	int err, i = 0;
 912
 913	nf_flow_offload_init(&cls_flow, proto, priority, cmd,
 914			     &flow->tuplehash[dir].tuple, &extack);
 915	if (cmd == FLOW_CLS_REPLACE)
 916		cls_flow.rule = flow_rule->rule;
 917
 918	down_read(&flowtable->flow_block_lock);
 919	list_for_each_entry(block_cb, block_cb_list, list) {
 920		err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow,
 921				   block_cb->cb_priv);
 922		if (err < 0)
 923			continue;
 924
 925		i++;
 926	}
 927	up_read(&flowtable->flow_block_lock);
 928
 929	if (cmd == FLOW_CLS_STATS)
 930		memcpy(stats, &cls_flow.stats, sizeof(*stats));
 931
 932	return i;
 933}
 934
 935static int flow_offload_tuple_add(struct flow_offload_work *offload,
 936				  struct nf_flow_rule *flow_rule,
 937				  enum flow_offload_tuple_dir dir)
 938{
 939	return nf_flow_offload_tuple(offload->flowtable, offload->flow,
 940				     flow_rule, dir,
 941				     offload->flowtable->priority,
 942				     FLOW_CLS_REPLACE, NULL,
 943				     &offload->flowtable->flow_block.cb_list);
 944}
 945
 946static void flow_offload_tuple_del(struct flow_offload_work *offload,
 947				   enum flow_offload_tuple_dir dir)
 948{
 949	nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
 950			      offload->flowtable->priority,
 951			      FLOW_CLS_DESTROY, NULL,
 952			      &offload->flowtable->flow_block.cb_list);
 953}
 954
 955static int flow_offload_rule_add(struct flow_offload_work *offload,
 956				 struct nf_flow_rule *flow_rule[])
 957{
 958	int ok_count = 0;
 959
 960	ok_count += flow_offload_tuple_add(offload, flow_rule[0],
 961					   FLOW_OFFLOAD_DIR_ORIGINAL);
 962	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
 963		ok_count += flow_offload_tuple_add(offload, flow_rule[1],
 964						   FLOW_OFFLOAD_DIR_REPLY);
 965	if (ok_count == 0)
 966		return -ENOENT;
 967
 968	return 0;
 969}
 970
 971static void flow_offload_work_add(struct flow_offload_work *offload)
 972{
 973	struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX];
 974	int err;
 975
 976	err = nf_flow_offload_alloc(offload, flow_rule);
 977	if (err < 0)
 978		return;
 979
 980	err = flow_offload_rule_add(offload, flow_rule);
 981	if (err < 0)
 982		goto out;
 983
 984	set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
 985
 986out:
 987	nf_flow_offload_destroy(flow_rule);
 988}
 989
 990static void flow_offload_work_del(struct flow_offload_work *offload)
 991{
 992	clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
 993	flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
 994	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
 995		flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
 996	set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
 997}
 998
 999static void flow_offload_tuple_stats(struct flow_offload_work *offload,
1000				     enum flow_offload_tuple_dir dir,
1001				     struct flow_stats *stats)
1002{
1003	nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
1004			      offload->flowtable->priority,
1005			      FLOW_CLS_STATS, stats,
1006			      &offload->flowtable->flow_block.cb_list);
1007}
1008
1009static void flow_offload_work_stats(struct flow_offload_work *offload)
1010{
1011	struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {};
1012	u64 lastused;
1013
1014	flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]);
1015	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
1016		flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY,
1017					 &stats[1]);
1018
1019	lastused = max_t(u64, stats[0].lastused, stats[1].lastused);
1020	offload->flow->timeout = max_t(u64, offload->flow->timeout,
1021				       lastused + flow_offload_get_timeout(offload->flow));
1022
1023	if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) {
1024		if (stats[0].pkts)
1025			nf_ct_acct_add(offload->flow->ct,
1026				       FLOW_OFFLOAD_DIR_ORIGINAL,
1027				       stats[0].pkts, stats[0].bytes);
1028		if (stats[1].pkts)
1029			nf_ct_acct_add(offload->flow->ct,
1030				       FLOW_OFFLOAD_DIR_REPLY,
1031				       stats[1].pkts, stats[1].bytes);
1032	}
1033}
1034
1035static void flow_offload_work_handler(struct work_struct *work)
1036{
1037	struct flow_offload_work *offload;
1038	struct net *net;
1039
1040	offload = container_of(work, struct flow_offload_work, work);
1041	net = read_pnet(&offload->flowtable->net);
1042	switch (offload->cmd) {
1043		case FLOW_CLS_REPLACE:
1044			flow_offload_work_add(offload);
1045			NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add);
1046			break;
1047		case FLOW_CLS_DESTROY:
1048			flow_offload_work_del(offload);
1049			NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del);
1050			break;
1051		case FLOW_CLS_STATS:
1052			flow_offload_work_stats(offload);
1053			NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats);
1054			break;
1055		default:
1056			WARN_ON_ONCE(1);
1057	}
1058
1059	clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags);
1060	kfree(offload);
1061}
1062
1063static void flow_offload_queue_work(struct flow_offload_work *offload)
1064{
1065	struct net *net = read_pnet(&offload->flowtable->net);
1066
1067	if (offload->cmd == FLOW_CLS_REPLACE) {
1068		NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add);
1069		queue_work(nf_flow_offload_add_wq, &offload->work);
1070	} else if (offload->cmd == FLOW_CLS_DESTROY) {
1071		NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del);
1072		queue_work(nf_flow_offload_del_wq, &offload->work);
1073	} else {
1074		NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats);
1075		queue_work(nf_flow_offload_stats_wq, &offload->work);
1076	}
1077}
1078
1079static struct flow_offload_work *
1080nf_flow_offload_work_alloc(struct nf_flowtable *flowtable,
1081			   struct flow_offload *flow, unsigned int cmd)
1082{
1083	struct flow_offload_work *offload;
1084
1085	if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags))
1086		return NULL;
1087
1088	offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
1089	if (!offload) {
1090		clear_bit(NF_FLOW_HW_PENDING, &flow->flags);
1091		return NULL;
1092	}
1093
1094	offload->cmd = cmd;
1095	offload->flow = flow;
1096	offload->flowtable = flowtable;
1097	INIT_WORK(&offload->work, flow_offload_work_handler);
1098
1099	return offload;
1100}
1101
1102
1103void nf_flow_offload_add(struct nf_flowtable *flowtable,
1104			 struct flow_offload *flow)
1105{
1106	struct flow_offload_work *offload;
1107
1108	offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE);
1109	if (!offload)
1110		return;
1111
1112	flow_offload_queue_work(offload);
1113}
1114
1115void nf_flow_offload_del(struct nf_flowtable *flowtable,
1116			 struct flow_offload *flow)
1117{
1118	struct flow_offload_work *offload;
1119
1120	offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY);
1121	if (!offload)
1122		return;
1123
1124	set_bit(NF_FLOW_HW_DYING, &flow->flags);
1125	flow_offload_queue_work(offload);
1126}
1127
1128void nf_flow_offload_stats(struct nf_flowtable *flowtable,
1129			   struct flow_offload *flow)
1130{
1131	struct flow_offload_work *offload;
1132	__s32 delta;
1133
1134	delta = nf_flow_timeout_delta(flow->timeout);
1135	if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10))
1136		return;
1137
1138	offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS);
1139	if (!offload)
1140		return;
1141
1142	flow_offload_queue_work(offload);
1143}
1144
1145void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable)
1146{
1147	if (nf_flowtable_hw_offload(flowtable)) {
1148		flush_workqueue(nf_flow_offload_del_wq);
1149		nf_flow_table_gc_run(flowtable);
1150	}
1151}
1152
1153void nf_flow_table_offload_flush(struct nf_flowtable *flowtable)
1154{
1155	if (nf_flowtable_hw_offload(flowtable)) {
1156		flush_workqueue(nf_flow_offload_add_wq);
1157		flush_workqueue(nf_flow_offload_del_wq);
1158		flush_workqueue(nf_flow_offload_stats_wq);
1159	}
1160}
1161
1162static int nf_flow_table_block_setup(struct nf_flowtable *flowtable,
1163				     struct flow_block_offload *bo,
1164				     enum flow_block_command cmd)
1165{
1166	struct flow_block_cb *block_cb, *next;
1167	int err = 0;
1168
1169	down_write(&flowtable->flow_block_lock);
1170	switch (cmd) {
1171	case FLOW_BLOCK_BIND:
1172		list_splice(&bo->cb_list, &flowtable->flow_block.cb_list);
1173		break;
1174	case FLOW_BLOCK_UNBIND:
1175		list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1176			list_del(&block_cb->list);
1177			flow_block_cb_free(block_cb);
1178		}
1179		break;
1180	default:
1181		WARN_ON_ONCE(1);
1182		err = -EOPNOTSUPP;
1183	}
1184	up_write(&flowtable->flow_block_lock);
1185
1186	return err;
1187}
1188
1189static void nf_flow_table_block_offload_init(struct flow_block_offload *bo,
1190					     struct net *net,
1191					     enum flow_block_command cmd,
1192					     struct nf_flowtable *flowtable,
1193					     struct netlink_ext_ack *extack)
1194{
1195	memset(bo, 0, sizeof(*bo));
1196	bo->net		= net;
1197	bo->block	= &flowtable->flow_block;
1198	bo->command	= cmd;
1199	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
1200	bo->extack	= extack;
1201	bo->cb_list_head = &flowtable->flow_block.cb_list;
1202	INIT_LIST_HEAD(&bo->cb_list);
1203}
1204
1205static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb)
1206{
1207	struct nf_flowtable *flowtable = block_cb->indr.data;
1208	struct net_device *dev = block_cb->indr.dev;
1209
1210	nf_flow_table_gc_cleanup(flowtable, dev);
1211	down_write(&flowtable->flow_block_lock);
1212	list_del(&block_cb->list);
1213	list_del(&block_cb->driver_list);
1214	flow_block_cb_free(block_cb);
1215	up_write(&flowtable->flow_block_lock);
1216}
1217
1218static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo,
1219					  struct nf_flowtable *flowtable,
1220					  struct net_device *dev,
1221					  enum flow_block_command cmd,
1222					  struct netlink_ext_ack *extack)
1223{
1224	nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
1225					 extack);
1226
1227	return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo,
1228					   nf_flow_table_indr_cleanup);
1229}
1230
1231static int nf_flow_table_offload_cmd(struct flow_block_offload *bo,
1232				     struct nf_flowtable *flowtable,
1233				     struct net_device *dev,
1234				     enum flow_block_command cmd,
1235				     struct netlink_ext_ack *extack)
1236{
1237	int err;
1238
1239	nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
1240					 extack);
1241	down_write(&flowtable->flow_block_lock);
1242	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo);
1243	up_write(&flowtable->flow_block_lock);
1244	if (err < 0)
1245		return err;
1246
1247	return 0;
1248}
1249
1250int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
1251				struct net_device *dev,
1252				enum flow_block_command cmd)
1253{
1254	struct netlink_ext_ack extack = {};
1255	struct flow_block_offload bo;
1256	int err;
1257
1258	if (!nf_flowtable_hw_offload(flowtable))
1259		return nf_flow_offload_xdp_setup(flowtable, dev, cmd);
1260
1261	if (dev->netdev_ops->ndo_setup_tc)
1262		err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd,
1263						&extack);
1264	else
1265		err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd,
1266						     &extack);
1267	if (err < 0)
1268		return err;
1269
1270	return nf_flow_table_block_setup(flowtable, &bo, cmd);
1271}
1272EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup);
1273
1274int nf_flow_table_offload_init(void)
1275{
1276	nf_flow_offload_add_wq  = alloc_workqueue("nf_ft_offload_add",
1277						  WQ_UNBOUND | WQ_SYSFS, 0);
1278	if (!nf_flow_offload_add_wq)
1279		return -ENOMEM;
1280
1281	nf_flow_offload_del_wq  = alloc_workqueue("nf_ft_offload_del",
1282						  WQ_UNBOUND | WQ_SYSFS, 0);
1283	if (!nf_flow_offload_del_wq)
1284		goto err_del_wq;
1285
1286	nf_flow_offload_stats_wq  = alloc_workqueue("nf_ft_offload_stats",
1287						    WQ_UNBOUND | WQ_SYSFS, 0);
1288	if (!nf_flow_offload_stats_wq)
1289		goto err_stats_wq;
1290
1291	return 0;
1292
1293err_stats_wq:
1294	destroy_workqueue(nf_flow_offload_del_wq);
1295err_del_wq:
1296	destroy_workqueue(nf_flow_offload_add_wq);
1297	return -ENOMEM;
1298}
1299
1300void nf_flow_table_offload_exit(void)
1301{
1302	destroy_workqueue(nf_flow_offload_add_wq);
1303	destroy_workqueue(nf_flow_offload_del_wq);
1304	destroy_workqueue(nf_flow_offload_stats_wq);
1305}