nf_tables_api.c

    1// SPDX-License-Identifier: GPL-2.0-only
    2/*
    3 * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
    4 *
    5 * Development of this code funded by Astaro AG (http://www.astaro.com/)
    6 */
    7
    8#include <linux/module.h>
    9#include <linux/init.h>
   10#include <linux/list.h>
   11#include <linux/skbuff.h>
   12#include <linux/netlink.h>
   13#include <linux/vmalloc.h>
   14#include <linux/rhashtable.h>
   15#include <linux/audit.h>
   16#include <linux/netfilter.h>
   17#include <linux/netfilter/nfnetlink.h>
   18#include <linux/netfilter/nf_tables.h>
   19#include <net/netfilter/nf_flow_table.h>
   20#include <net/netfilter/nf_tables_core.h>
   21#include <net/netfilter/nf_tables.h>
   22#include <net/netfilter/nf_tables_offload.h>
   23#include <net/net_namespace.h>
   24#include <net/sock.h>
   25
   26#define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
   27#define NFT_SET_MAX_ANONLEN 16
   28
   29/* limit compaction to avoid huge kmalloc/krealloc sizes. */
   30#define NFT_MAX_SET_NELEMS ((2048 - sizeof(struct nft_trans_elem)) / sizeof(struct nft_trans_one_elem))
   31
   32unsigned int nf_tables_net_id __read_mostly;
   33
   34static LIST_HEAD(nf_tables_expressions);
   35static LIST_HEAD(nf_tables_objects);
   36static LIST_HEAD(nf_tables_flowtables);
   37static LIST_HEAD(nf_tables_gc_list);
   38static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
   39static DEFINE_SPINLOCK(nf_tables_gc_list_lock);
   40
   41enum {
   42	NFT_VALIDATE_SKIP	= 0,
   43	NFT_VALIDATE_NEED,
   44	NFT_VALIDATE_DO,
   45};
   46
   47static struct rhltable nft_objname_ht;
   48
   49static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
   50static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
   51static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
   52
   53static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
   54static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
   55static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
   56
   57static const struct rhashtable_params nft_chain_ht_params = {
   58	.head_offset		= offsetof(struct nft_chain, rhlhead),
   59	.key_offset		= offsetof(struct nft_chain, name),
   60	.hashfn			= nft_chain_hash,
   61	.obj_hashfn		= nft_chain_hash_obj,
   62	.obj_cmpfn		= nft_chain_hash_cmp,
   63	.automatic_shrinking	= true,
   64};
   65
   66static const struct rhashtable_params nft_objname_ht_params = {
   67	.head_offset		= offsetof(struct nft_object, rhlhead),
   68	.key_offset		= offsetof(struct nft_object, key),
   69	.hashfn			= nft_objname_hash,
   70	.obj_hashfn		= nft_objname_hash_obj,
   71	.obj_cmpfn		= nft_objname_hash_cmp,
   72	.automatic_shrinking	= true,
   73};
   74
   75struct nft_audit_data {
   76	struct nft_table *table;
   77	int entries;
   78	int op;
   79	struct list_head list;
   80};
   81
   82static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
   83	[NFT_MSG_NEWTABLE]	= AUDIT_NFT_OP_TABLE_REGISTER,
   84	[NFT_MSG_GETTABLE]	= AUDIT_NFT_OP_INVALID,
   85	[NFT_MSG_DELTABLE]	= AUDIT_NFT_OP_TABLE_UNREGISTER,
   86	[NFT_MSG_NEWCHAIN]	= AUDIT_NFT_OP_CHAIN_REGISTER,
   87	[NFT_MSG_GETCHAIN]	= AUDIT_NFT_OP_INVALID,
   88	[NFT_MSG_DELCHAIN]	= AUDIT_NFT_OP_CHAIN_UNREGISTER,
   89	[NFT_MSG_NEWRULE]	= AUDIT_NFT_OP_RULE_REGISTER,
   90	[NFT_MSG_GETRULE]	= AUDIT_NFT_OP_INVALID,
   91	[NFT_MSG_DELRULE]	= AUDIT_NFT_OP_RULE_UNREGISTER,
   92	[NFT_MSG_NEWSET]	= AUDIT_NFT_OP_SET_REGISTER,
   93	[NFT_MSG_GETSET]	= AUDIT_NFT_OP_INVALID,
   94	[NFT_MSG_DELSET]	= AUDIT_NFT_OP_SET_UNREGISTER,
   95	[NFT_MSG_NEWSETELEM]	= AUDIT_NFT_OP_SETELEM_REGISTER,
   96	[NFT_MSG_GETSETELEM]	= AUDIT_NFT_OP_INVALID,
   97	[NFT_MSG_DELSETELEM]	= AUDIT_NFT_OP_SETELEM_UNREGISTER,
   98	[NFT_MSG_NEWGEN]	= AUDIT_NFT_OP_GEN_REGISTER,
   99	[NFT_MSG_GETGEN]	= AUDIT_NFT_OP_INVALID,
  100	[NFT_MSG_TRACE]		= AUDIT_NFT_OP_INVALID,
  101	[NFT_MSG_NEWOBJ]	= AUDIT_NFT_OP_OBJ_REGISTER,
  102	[NFT_MSG_GETOBJ]	= AUDIT_NFT_OP_INVALID,
  103	[NFT_MSG_DELOBJ]	= AUDIT_NFT_OP_OBJ_UNREGISTER,
  104	[NFT_MSG_GETOBJ_RESET]	= AUDIT_NFT_OP_OBJ_RESET,
  105	[NFT_MSG_NEWFLOWTABLE]	= AUDIT_NFT_OP_FLOWTABLE_REGISTER,
  106	[NFT_MSG_GETFLOWTABLE]	= AUDIT_NFT_OP_INVALID,
  107	[NFT_MSG_DELFLOWTABLE]	= AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
  108	[NFT_MSG_GETSETELEM_RESET] = AUDIT_NFT_OP_SETELEM_RESET,
  109};
  110
  111static void nft_validate_state_update(struct nft_table *table, u8 new_validate_state)
  112{
  113	switch (table->validate_state) {
  114	case NFT_VALIDATE_SKIP:
  115		WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
  116		break;
  117	case NFT_VALIDATE_NEED:
  118		break;
  119	case NFT_VALIDATE_DO:
  120		if (new_validate_state == NFT_VALIDATE_NEED)
  121			return;
  122	}
  123
  124	table->validate_state = new_validate_state;
  125}
  126
  127static bool nft_chain_vstate_valid(const struct nft_ctx *ctx,
  128				   const struct nft_chain *chain)
  129{
  130	const struct nft_base_chain *base_chain;
  131	enum nft_chain_types type;
  132	u8 hooknum;
  133
  134	if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain)))
  135		return false;
  136
  137	base_chain = nft_base_chain(ctx->chain);
  138	hooknum = base_chain->ops.hooknum;
  139	type = base_chain->type->type;
  140
  141	/* chain is already validated for this call depth */
  142	if (chain->vstate.depth >= ctx->level &&
  143	    chain->vstate.hook_mask[type] & BIT(hooknum))
  144		return true;
  145
  146	return false;
  147}
  148
  149static void nf_tables_trans_destroy_work(struct work_struct *w);
  150
  151static void nft_trans_gc_work(struct work_struct *work);
  152static DECLARE_WORK(trans_gc_work, nft_trans_gc_work);
  153
  154static void nft_ctx_init(struct nft_ctx *ctx,
  155			 struct net *net,
  156			 const struct sk_buff *skb,
  157			 const struct nlmsghdr *nlh,
  158			 u8 family,
  159			 struct nft_table *table,
  160			 struct nft_chain *chain,
  161			 const struct nlattr * const *nla)
  162{
  163	ctx->net	= net;
  164	ctx->family	= family;
  165	ctx->level	= 0;
  166	ctx->table	= table;
  167	ctx->chain	= chain;
  168	ctx->nla   	= nla;
  169	ctx->portid	= NETLINK_CB(skb).portid;
  170	ctx->report	= nlmsg_report(nlh);
  171	ctx->flags	= nlh->nlmsg_flags;
  172	ctx->seq	= nlh->nlmsg_seq;
  173
  174	bitmap_zero(ctx->reg_inited, NFT_REG32_NUM);
  175}
  176
  177static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
  178					 int msg_type, u32 size)
  179{
  180	struct nft_trans *trans;
  181
  182	trans = kzalloc(size, GFP_KERNEL);
  183	if (trans == NULL)
  184		return NULL;
  185
  186	INIT_LIST_HEAD(&trans->list);
  187	trans->msg_type = msg_type;
  188
  189	trans->net = ctx->net;
  190	trans->table = ctx->table;
  191	trans->seq = ctx->seq;
  192	trans->flags = ctx->flags;
  193	trans->report = ctx->report;
  194
  195	return trans;
  196}
  197
  198static struct nft_trans_binding *nft_trans_get_binding(struct nft_trans *trans)
  199{
  200	switch (trans->msg_type) {
  201	case NFT_MSG_NEWCHAIN:
  202	case NFT_MSG_NEWSET:
  203		return container_of(trans, struct nft_trans_binding, nft_trans);
  204	}
  205
  206	return NULL;
  207}
  208
  209static void nft_trans_list_del(struct nft_trans *trans)
  210{
  211	struct nft_trans_binding *trans_binding;
  212
  213	list_del(&trans->list);
  214
  215	trans_binding = nft_trans_get_binding(trans);
  216	if (trans_binding)
  217		list_del(&trans_binding->binding_list);
  218}
  219
  220static void nft_trans_destroy(struct nft_trans *trans)
  221{
  222	nft_trans_list_del(trans);
  223	kfree(trans);
  224}
  225
  226static void __nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set,
  227				 bool bind)
  228{
  229	struct nftables_pernet *nft_net;
  230	struct net *net = ctx->net;
  231	struct nft_trans *trans;
  232
  233	if (!nft_set_is_anonymous(set))
  234		return;
  235
  236	nft_net = nft_pernet(net);
  237	list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
  238		switch (trans->msg_type) {
  239		case NFT_MSG_NEWSET:
  240			if (nft_trans_set(trans) == set)
  241				nft_trans_set_bound(trans) = bind;
  242			break;
  243		case NFT_MSG_NEWSETELEM:
  244			if (nft_trans_elem_set(trans) == set)
  245				nft_trans_elem_set_bound(trans) = bind;
  246			break;
  247		}
  248	}
  249}
  250
  251static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
  252{
  253	return __nft_set_trans_bind(ctx, set, true);
  254}
  255
  256static void nft_set_trans_unbind(const struct nft_ctx *ctx, struct nft_set *set)
  257{
  258	return __nft_set_trans_bind(ctx, set, false);
  259}
  260
  261static void __nft_chain_trans_bind(const struct nft_ctx *ctx,
  262				   struct nft_chain *chain, bool bind)
  263{
  264	struct nftables_pernet *nft_net;
  265	struct net *net = ctx->net;
  266	struct nft_trans *trans;
  267
  268	if (!nft_chain_binding(chain))
  269		return;
  270
  271	nft_net = nft_pernet(net);
  272	list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
  273		switch (trans->msg_type) {
  274		case NFT_MSG_NEWCHAIN:
  275			if (nft_trans_chain(trans) == chain)
  276				nft_trans_chain_bound(trans) = bind;
  277			break;
  278		case NFT_MSG_NEWRULE:
  279			if (nft_trans_rule_chain(trans) == chain)
  280				nft_trans_rule_bound(trans) = bind;
  281			break;
  282		}
  283	}
  284}
  285
  286static void nft_chain_trans_bind(const struct nft_ctx *ctx,
  287				 struct nft_chain *chain)
  288{
  289	__nft_chain_trans_bind(ctx, chain, true);
  290}
  291
  292int nf_tables_bind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
  293{
  294	if (!nft_chain_binding(chain))
  295		return 0;
  296
  297	if (nft_chain_binding(ctx->chain))
  298		return -EOPNOTSUPP;
  299
  300	if (chain->bound)
  301		return -EBUSY;
  302
  303	if (!nft_use_inc(&chain->use))
  304		return -EMFILE;
  305
  306	chain->bound = true;
  307	nft_chain_trans_bind(ctx, chain);
  308
  309	return 0;
  310}
  311
  312void nf_tables_unbind_chain(const struct nft_ctx *ctx, struct nft_chain *chain)
  313{
  314	__nft_chain_trans_bind(ctx, chain, false);
  315}
  316
  317static int nft_netdev_register_hooks(struct net *net,
  318				     struct list_head *hook_list)
  319{
  320	struct nf_hook_ops *ops;
  321	struct nft_hook *hook;
  322	int err, j;
  323
  324	j = 0;
  325	list_for_each_entry(hook, hook_list, list) {
  326		list_for_each_entry(ops, &hook->ops_list, list) {
  327			err = nf_register_net_hook(net, ops);
  328			if (err < 0)
  329				goto err_register;
  330
  331			j++;
  332		}
  333	}
  334	return 0;
  335
  336err_register:
  337	list_for_each_entry(hook, hook_list, list) {
  338		list_for_each_entry(ops, &hook->ops_list, list) {
  339			if (j-- <= 0)
  340				break;
  341
  342			nf_unregister_net_hook(net, ops);
  343		}
  344	}
  345	return err;
  346}
  347
  348static void nft_netdev_hook_free_ops(struct nft_hook *hook)
  349{
  350	struct nf_hook_ops *ops, *next;
  351
  352	list_for_each_entry_safe(ops, next, &hook->ops_list, list) {
  353		list_del(&ops->list);
  354		kfree(ops);
  355	}
  356}
  357
  358static void nft_netdev_hook_free(struct nft_hook *hook)
  359{
  360	nft_netdev_hook_free_ops(hook);
  361	kfree(hook);
  362}
  363
  364static void __nft_netdev_hook_free_rcu(struct rcu_head *rcu)
  365{
  366	struct nft_hook *hook = container_of(rcu, struct nft_hook, rcu);
  367
  368	nft_netdev_hook_free(hook);
  369}
  370
  371static void nft_netdev_hook_free_rcu(struct nft_hook *hook)
  372{
  373	call_rcu(&hook->rcu, __nft_netdev_hook_free_rcu);
  374}
  375
  376static void nft_netdev_unregister_hooks(struct net *net,
  377					struct list_head *hook_list,
  378					bool release_netdev)
  379{
  380	struct nft_hook *hook, *next;
  381	struct nf_hook_ops *ops;
  382
  383	list_for_each_entry_safe(hook, next, hook_list, list) {
  384		list_for_each_entry(ops, &hook->ops_list, list)
  385			nf_unregister_net_hook(net, ops);
  386		if (release_netdev) {
  387			list_del(&hook->list);
  388			nft_netdev_hook_free_rcu(hook);
  389		}
  390	}
  391}
  392
  393static int nf_tables_register_hook(struct net *net,
  394				   const struct nft_table *table,
  395				   struct nft_chain *chain)
  396{
  397	struct nft_base_chain *basechain;
  398	const struct nf_hook_ops *ops;
  399
  400	if (table->flags & NFT_TABLE_F_DORMANT ||
  401	    !nft_is_base_chain(chain))
  402		return 0;
  403
  404	basechain = nft_base_chain(chain);
  405	ops = &basechain->ops;
  406
  407	if (basechain->type->ops_register)
  408		return basechain->type->ops_register(net, ops);
  409
  410	if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
  411		return nft_netdev_register_hooks(net, &basechain->hook_list);
  412
  413	return nf_register_net_hook(net, &basechain->ops);
  414}
  415
  416static void __nf_tables_unregister_hook(struct net *net,
  417					const struct nft_table *table,
  418					struct nft_chain *chain,
  419					bool release_netdev)
  420{
  421	struct nft_base_chain *basechain;
  422	const struct nf_hook_ops *ops;
  423
  424	if (table->flags & NFT_TABLE_F_DORMANT ||
  425	    !nft_is_base_chain(chain))
  426		return;
  427	basechain = nft_base_chain(chain);
  428	ops = &basechain->ops;
  429
  430	if (basechain->type->ops_unregister)
  431		return basechain->type->ops_unregister(net, ops);
  432
  433	if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
  434		nft_netdev_unregister_hooks(net, &basechain->hook_list,
  435					    release_netdev);
  436	else
  437		nf_unregister_net_hook(net, &basechain->ops);
  438}
  439
  440static void nf_tables_unregister_hook(struct net *net,
  441				      const struct nft_table *table,
  442				      struct nft_chain *chain)
  443{
  444	return __nf_tables_unregister_hook(net, table, chain, false);
  445}
  446
  447static bool nft_trans_collapse_set_elem_allowed(const struct nft_trans_elem *a, const struct nft_trans_elem *b)
  448{
  449	/* NB: the ->bound equality check is defensive, at this time we only merge
  450	 * a new nft_trans_elem transaction request with the transaction tail
  451	 * element, but a->bound != b->bound would imply a NEWRULE transaction
  452	 * is queued in-between.
  453	 *
  454	 * The set check is mandatory, the NFT_MAX_SET_NELEMS check prevents
  455	 * huge krealloc() requests.
  456	 */
  457	return a->set == b->set && a->bound == b->bound && a->nelems < NFT_MAX_SET_NELEMS;
  458}
  459
  460static bool nft_trans_collapse_set_elem(struct nftables_pernet *nft_net,
  461					struct nft_trans_elem *tail,
  462					struct nft_trans_elem *trans)
  463{
  464	unsigned int nelems, old_nelems = tail->nelems;
  465	struct nft_trans_elem *new_trans;
  466
  467	if (!nft_trans_collapse_set_elem_allowed(tail, trans))
  468		return false;
  469
  470	/* "cannot happen", at this time userspace element add
  471	 * requests always allocate a new transaction element.
  472	 *
  473	 * This serves as a reminder to adjust the list_add_tail
  474	 * logic below in case this ever changes.
  475	 */
  476	if (WARN_ON_ONCE(trans->nelems != 1))
  477		return false;
  478
  479	if (check_add_overflow(old_nelems, trans->nelems, &nelems))
  480		return false;
  481
  482	/* krealloc might free tail which invalidates list pointers */
  483	list_del_init(&tail->nft_trans.list);
  484
  485	new_trans = krealloc(tail, struct_size(tail, elems, nelems),
  486			     GFP_KERNEL);
  487	if (!new_trans) {
  488		list_add_tail(&tail->nft_trans.list,
  489			      &nft_net->commit_list);
  490		return false;
  491	}
  492
  493	/*
  494	 * new_trans->nft_trans.list contains garbage, but
  495	 * list_add_tail() doesn't care.
  496	 */
  497	new_trans->nelems = nelems;
  498	new_trans->elems[old_nelems] = trans->elems[0];
  499	list_add_tail(&new_trans->nft_trans.list, &nft_net->commit_list);
  500
  501	return true;
  502}
  503
  504static bool nft_trans_try_collapse(struct nftables_pernet *nft_net,
  505				   struct nft_trans *trans)
  506{
  507	struct nft_trans *tail;
  508
  509	if (list_empty(&nft_net->commit_list))
  510		return false;
  511
  512	tail = list_last_entry(&nft_net->commit_list, struct nft_trans, list);
  513
  514	if (tail->msg_type != trans->msg_type)
  515		return false;
  516
  517	switch (trans->msg_type) {
  518	case NFT_MSG_NEWSETELEM:
  519	case NFT_MSG_DELSETELEM:
  520		return nft_trans_collapse_set_elem(nft_net,
  521						   nft_trans_container_elem(tail),
  522						   nft_trans_container_elem(trans));
  523	}
  524
  525	return false;
  526}
  527
  528static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
  529{
  530	struct nftables_pernet *nft_net = nft_pernet(net);
  531	struct nft_trans_binding *binding;
  532	struct nft_trans_set *trans_set;
  533
  534	list_add_tail(&trans->list, &nft_net->commit_list);
  535
  536	binding = nft_trans_get_binding(trans);
  537	if (!binding)
  538		return;
  539
  540	switch (trans->msg_type) {
  541	case NFT_MSG_NEWSET:
  542		trans_set = nft_trans_container_set(trans);
  543
  544		if (!nft_trans_set_update(trans) &&
  545		    nft_set_is_anonymous(nft_trans_set(trans)))
  546			list_add_tail(&binding->binding_list, &nft_net->binding_list);
  547
  548		list_add_tail(&trans_set->list_trans_newset, &nft_net->commit_set_list);
  549		break;
  550	case NFT_MSG_NEWCHAIN:
  551		if (!nft_trans_chain_update(trans) &&
  552		    nft_chain_binding(nft_trans_chain(trans)))
  553			list_add_tail(&binding->binding_list, &nft_net->binding_list);
  554		break;
  555	}
  556}
  557
  558static void nft_trans_commit_list_add_elem(struct net *net, struct nft_trans *trans)
  559{
  560	struct nftables_pernet *nft_net = nft_pernet(net);
  561
  562	WARN_ON_ONCE(trans->msg_type != NFT_MSG_NEWSETELEM &&
  563		     trans->msg_type != NFT_MSG_DELSETELEM);
  564
  565	if (nft_trans_try_collapse(nft_net, trans)) {
  566		kfree(trans);
  567		return;
  568	}
  569
  570	nft_trans_commit_list_add_tail(net, trans);
  571}
  572
  573static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
  574{
  575	struct nft_trans *trans;
  576
  577	trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
  578	if (trans == NULL)
  579		return -ENOMEM;
  580
  581	if (msg_type == NFT_MSG_NEWTABLE)
  582		nft_activate_next(ctx->net, ctx->table);
  583
  584	nft_trans_commit_list_add_tail(ctx->net, trans);
  585	return 0;
  586}
  587
  588static int nft_deltable(struct nft_ctx *ctx)
  589{
  590	int err;
  591
  592	err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
  593	if (err < 0)
  594		return err;
  595
  596	nft_deactivate_next(ctx->net, ctx->table);
  597	return err;
  598}
  599
  600static struct nft_trans *
  601nft_trans_alloc_chain(const struct nft_ctx *ctx, int msg_type)
  602{
  603	struct nft_trans_chain *trans_chain;
  604	struct nft_trans *trans;
  605
  606	trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
  607	if (!trans)
  608		return NULL;
  609
  610	trans_chain = nft_trans_container_chain(trans);
  611	INIT_LIST_HEAD(&trans_chain->nft_trans_binding.binding_list);
  612	trans_chain->chain = ctx->chain;
  613
  614	return trans;
  615}
  616
  617static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
  618{
  619	struct nft_trans *trans;
  620
  621	trans = nft_trans_alloc_chain(ctx, msg_type);
  622	if (trans == NULL)
  623		return ERR_PTR(-ENOMEM);
  624
  625	if (msg_type == NFT_MSG_NEWCHAIN) {
  626		nft_activate_next(ctx->net, ctx->chain);
  627
  628		if (ctx->nla[NFTA_CHAIN_ID]) {
  629			nft_trans_chain_id(trans) =
  630				ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
  631		}
  632	}
  633	nft_trans_commit_list_add_tail(ctx->net, trans);
  634
  635	return trans;
  636}
  637
  638static int nft_delchain(struct nft_ctx *ctx)
  639{
  640	struct nft_trans *trans;
  641
  642	trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
  643	if (IS_ERR(trans))
  644		return PTR_ERR(trans);
  645
  646	nft_use_dec(&ctx->table->use);
  647	nft_deactivate_next(ctx->net, ctx->chain);
  648
  649	return 0;
  650}
  651
  652void nft_rule_expr_activate(const struct nft_ctx *ctx, struct nft_rule *rule)
  653{
  654	struct nft_expr *expr;
  655
  656	expr = nft_expr_first(rule);
  657	while (nft_expr_more(rule, expr)) {
  658		if (expr->ops->activate)
  659			expr->ops->activate(ctx, expr);
  660
  661		expr = nft_expr_next(expr);
  662	}
  663}
  664
  665void nft_rule_expr_deactivate(const struct nft_ctx *ctx, struct nft_rule *rule,
  666			      enum nft_trans_phase phase)
  667{
  668	struct nft_expr *expr;
  669
  670	expr = nft_expr_first(rule);
  671	while (nft_expr_more(rule, expr)) {
  672		if (expr->ops->deactivate)
  673			expr->ops->deactivate(ctx, expr, phase);
  674
  675		expr = nft_expr_next(expr);
  676	}
  677}
  678
  679static int
  680nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
  681{
  682	/* You cannot delete the same rule twice */
  683	if (nft_is_active_next(ctx->net, rule)) {
  684		nft_deactivate_next(ctx->net, rule);
  685		nft_use_dec(&ctx->chain->use);
  686		return 0;
  687	}
  688	return -ENOENT;
  689}
  690
  691static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
  692					    struct nft_rule *rule)
  693{
  694	struct nft_trans *trans;
  695
  696	trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
  697	if (trans == NULL)
  698		return NULL;
  699
  700	if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
  701		nft_trans_rule_id(trans) =
  702			ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
  703	}
  704	nft_trans_rule(trans) = rule;
  705	nft_trans_rule_chain(trans) = ctx->chain;
  706	nft_trans_commit_list_add_tail(ctx->net, trans);
  707
  708	return trans;
  709}
  710
  711static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
  712{
  713	struct nft_flow_rule *flow;
  714	struct nft_trans *trans;
  715	int err;
  716
  717	trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
  718	if (trans == NULL)
  719		return -ENOMEM;
  720
  721	if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
  722		flow = nft_flow_rule_create(ctx->net, rule);
  723		if (IS_ERR(flow)) {
  724			nft_trans_destroy(trans);
  725			return PTR_ERR(flow);
  726		}
  727
  728		nft_trans_flow_rule(trans) = flow;
  729	}
  730
  731	err = nf_tables_delrule_deactivate(ctx, rule);
  732	if (err < 0) {
  733		nft_trans_destroy(trans);
  734		return err;
  735	}
  736	nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
  737
  738	return 0;
  739}
  740
  741static int nft_delrule_by_chain(struct nft_ctx *ctx)
  742{
  743	struct nft_rule *rule;
  744	int err;
  745
  746	list_for_each_entry(rule, &ctx->chain->rules, list) {
  747		if (!nft_is_active_next(ctx->net, rule))
  748			continue;
  749
  750		err = nft_delrule(ctx, rule);
  751		if (err < 0)
  752			return err;
  753	}
  754	return 0;
  755}
  756
  757static int __nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
  758			       struct nft_set *set,
  759			       const struct nft_set_desc *desc)
  760{
  761	struct nft_trans_set *trans_set;
  762	struct nft_trans *trans;
  763
  764	trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
  765	if (trans == NULL)
  766		return -ENOMEM;
  767
  768	trans_set = nft_trans_container_set(trans);
  769	INIT_LIST_HEAD(&trans_set->nft_trans_binding.binding_list);
  770	INIT_LIST_HEAD(&trans_set->list_trans_newset);
  771
  772	if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] && !desc) {
  773		nft_trans_set_id(trans) =
  774			ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
  775		nft_activate_next(ctx->net, set);
  776	}
  777	nft_trans_set(trans) = set;
  778	if (desc) {
  779		nft_trans_set_update(trans) = true;
  780		nft_trans_set_gc_int(trans) = desc->gc_int;
  781		nft_trans_set_timeout(trans) = desc->timeout;
  782		nft_trans_set_size(trans) = desc->size;
  783	}
  784	nft_trans_commit_list_add_tail(ctx->net, trans);
  785
  786	return 0;
  787}
  788
  789static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
  790			     struct nft_set *set)
  791{
  792	return __nft_trans_set_add(ctx, msg_type, set, NULL);
  793}
  794
  795static int nft_mapelem_deactivate(const struct nft_ctx *ctx,
  796				  struct nft_set *set,
  797				  const struct nft_set_iter *iter,
  798				  struct nft_elem_priv *elem_priv)
  799{
  800	struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
  801
  802	if (!nft_set_elem_active(ext, iter->genmask))
  803		return 0;
  804
  805	nft_set_elem_change_active(ctx->net, set, ext);
  806	nft_setelem_data_deactivate(ctx->net, set, elem_priv);
  807
  808	return 0;
  809}
  810
  811struct nft_set_elem_catchall {
  812	struct list_head	list;
  813	struct rcu_head		rcu;
  814	struct nft_elem_priv	*elem;
  815};
  816
  817static void nft_map_catchall_deactivate(const struct nft_ctx *ctx,
  818					struct nft_set *set)
  819{
  820	u8 genmask = nft_genmask_next(ctx->net);
  821	struct nft_set_elem_catchall *catchall;
  822	struct nft_set_ext *ext;
  823
  824	list_for_each_entry(catchall, &set->catchall_list, list) {
  825		ext = nft_set_elem_ext(set, catchall->elem);
  826		if (!nft_set_elem_active(ext, genmask))
  827			continue;
  828
  829		nft_set_elem_change_active(ctx->net, set, ext);
  830		nft_setelem_data_deactivate(ctx->net, set, catchall->elem);
  831	}
  832}
  833
  834/* Use NFT_ITER_UPDATE iterator even if this may be called from the preparation
  835 * phase, the set clone might already exist from a previous command, or it might
  836 * be a set that is going away and does not require a clone. The netns and
  837 * netlink release paths also need to work on the live set.
  838 */
  839static void nft_map_deactivate(const struct nft_ctx *ctx, struct nft_set *set)
  840{
  841	struct nft_set_iter iter = {
  842		.genmask	= nft_genmask_next(ctx->net),
  843		.type		= NFT_ITER_UPDATE,
  844		.fn		= nft_mapelem_deactivate,
  845	};
  846
  847	set->ops->walk(ctx, set, &iter);
  848	WARN_ON_ONCE(iter.err);
  849
  850	nft_map_catchall_deactivate(ctx, set);
  851}
  852
  853static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
  854{
  855	int err;
  856
  857	err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
  858	if (err < 0)
  859		return err;
  860
  861	if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
  862		nft_map_deactivate(ctx, set);
  863
  864	nft_deactivate_next(ctx->net, set);
  865	nft_use_dec(&ctx->table->use);
  866
  867	return err;
  868}
  869
  870static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
  871			     struct nft_object *obj)
  872{
  873	struct nft_trans *trans;
  874
  875	trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
  876	if (trans == NULL)
  877		return -ENOMEM;
  878
  879	if (msg_type == NFT_MSG_NEWOBJ)
  880		nft_activate_next(ctx->net, obj);
  881
  882	nft_trans_obj(trans) = obj;
  883	nft_trans_commit_list_add_tail(ctx->net, trans);
  884
  885	return 0;
  886}
  887
  888static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
  889{
  890	int err;
  891
  892	err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
  893	if (err < 0)
  894		return err;
  895
  896	nft_deactivate_next(ctx->net, obj);
  897	nft_use_dec(&ctx->table->use);
  898
  899	return err;
  900}
  901
  902static struct nft_trans *
  903nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
  904		        struct nft_flowtable *flowtable)
  905{
  906	struct nft_trans *trans;
  907
  908	trans = nft_trans_alloc(ctx, msg_type,
  909				sizeof(struct nft_trans_flowtable));
  910	if (trans == NULL)
  911		return ERR_PTR(-ENOMEM);
  912
  913	if (msg_type == NFT_MSG_NEWFLOWTABLE)
  914		nft_activate_next(ctx->net, flowtable);
  915
  916	INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
  917	nft_trans_flowtable(trans) = flowtable;
  918	nft_trans_commit_list_add_tail(ctx->net, trans);
  919
  920	return trans;
  921}
  922
  923static int nft_delflowtable(struct nft_ctx *ctx,
  924			    struct nft_flowtable *flowtable)
  925{
  926	struct nft_trans *trans;
  927
  928	trans = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
  929	if (IS_ERR(trans))
  930		return PTR_ERR(trans);
  931
  932	nft_deactivate_next(ctx->net, flowtable);
  933	nft_use_dec(&ctx->table->use);
  934
  935	return 0;
  936}
  937
  938static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
  939{
  940	int i;
  941
  942	for (i = track->regs[dreg].num_reg; i > 0; i--)
  943		__nft_reg_track_cancel(track, dreg - i);
  944}
  945
  946static void __nft_reg_track_update(struct nft_regs_track *track,
  947				   const struct nft_expr *expr,
  948				   u8 dreg, u8 num_reg)
  949{
  950	track->regs[dreg].selector = expr;
  951	track->regs[dreg].bitwise = NULL;
  952	track->regs[dreg].num_reg = num_reg;
  953}
  954
  955void nft_reg_track_update(struct nft_regs_track *track,
  956			  const struct nft_expr *expr, u8 dreg, u8 len)
  957{
  958	unsigned int regcount;
  959	int i;
  960
  961	__nft_reg_track_clobber(track, dreg);
  962
  963	regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
  964	for (i = 0; i < regcount; i++, dreg++)
  965		__nft_reg_track_update(track, expr, dreg, i);
  966}
  967EXPORT_SYMBOL_GPL(nft_reg_track_update);
  968
  969void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len)
  970{
  971	unsigned int regcount;
  972	int i;
  973
  974	__nft_reg_track_clobber(track, dreg);
  975
  976	regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE);
  977	for (i = 0; i < regcount; i++, dreg++)
  978		__nft_reg_track_cancel(track, dreg);
  979}
  980EXPORT_SYMBOL_GPL(nft_reg_track_cancel);
  981
  982void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg)
  983{
  984	track->regs[dreg].selector = NULL;
  985	track->regs[dreg].bitwise = NULL;
  986	track->regs[dreg].num_reg = 0;
  987}
  988EXPORT_SYMBOL_GPL(__nft_reg_track_cancel);
  989
  990/*
  991 * Tables
  992 */
  993
  994static struct nft_table *nft_table_lookup(const struct net *net,
  995					  const struct nlattr *nla,
  996					  u8 family, u8 genmask, u32 nlpid)
  997{
  998	struct nftables_pernet *nft_net;
  999	struct nft_table *table;
 1000
 1001	if (nla == NULL)
 1002		return ERR_PTR(-EINVAL);
 1003
 1004	nft_net = nft_pernet(net);
 1005	list_for_each_entry_rcu(table, &nft_net->tables, list,
 1006				lockdep_is_held(&nft_net->commit_mutex)) {
 1007		if (!nla_strcmp(nla, table->name) &&
 1008		    table->family == family &&
 1009		    nft_active_genmask(table, genmask)) {
 1010			if (nft_table_has_owner(table) &&
 1011			    nlpid && table->nlpid != nlpid)
 1012				return ERR_PTR(-EPERM);
 1013
 1014			return table;
 1015		}
 1016	}
 1017
 1018	return ERR_PTR(-ENOENT);
 1019}
 1020
 1021static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 1022						   const struct nlattr *nla,
 1023						   int family, u8 genmask, u32 nlpid)
 1024{
 1025	struct nftables_pernet *nft_net;
 1026	struct nft_table *table;
 1027
 1028	nft_net = nft_pernet(net);
 1029	list_for_each_entry(table, &nft_net->tables, list) {
 1030		if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 1031		    table->family == family &&
 1032		    nft_active_genmask(table, genmask)) {
 1033			if (nft_table_has_owner(table) &&
 1034			    nlpid && table->nlpid != nlpid)
 1035				return ERR_PTR(-EPERM);
 1036
 1037			return table;
 1038		}
 1039	}
 1040
 1041	return ERR_PTR(-ENOENT);
 1042}
 1043
 1044static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 1045{
 1046	return ++table->hgenerator;
 1047}
 1048
 1049static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 1050
 1051static const struct nft_chain_type *
 1052__nft_chain_type_get(u8 family, enum nft_chain_types type)
 1053{
 1054	if (family >= NFPROTO_NUMPROTO ||
 1055	    type >= NFT_CHAIN_T_MAX)
 1056		return NULL;
 1057
 1058	return chain_type[family][type];
 1059}
 1060
 1061static const struct nft_chain_type *
 1062__nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 1063{
 1064	const struct nft_chain_type *type;
 1065	int i;
 1066
 1067	for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 1068		type = __nft_chain_type_get(family, i);
 1069		if (!type)
 1070			continue;
 1071		if (!nla_strcmp(nla, type->name))
 1072			return type;
 1073	}
 1074	return NULL;
 1075}
 1076
 1077struct nft_module_request {
 1078	struct list_head	list;
 1079	char			module[MODULE_NAME_LEN];
 1080	bool			done;
 1081};
 1082
 1083#ifdef CONFIG_MODULES
 1084__printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 1085				      ...)
 1086{
 1087	char module_name[MODULE_NAME_LEN];
 1088	struct nftables_pernet *nft_net;
 1089	struct nft_module_request *req;
 1090	va_list args;
 1091	int ret;
 1092
 1093	va_start(args, fmt);
 1094	ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 1095	va_end(args);
 1096	if (ret >= MODULE_NAME_LEN)
 1097		return 0;
 1098
 1099	nft_net = nft_pernet(net);
 1100	list_for_each_entry(req, &nft_net->module_list, list) {
 1101		if (!strcmp(req->module, module_name)) {
 1102			if (req->done)
 1103				return 0;
 1104
 1105			/* A request to load this module already exists. */
 1106			return -EAGAIN;
 1107		}
 1108	}
 1109
 1110	req = kmalloc(sizeof(*req), GFP_KERNEL);
 1111	if (!req)
 1112		return -ENOMEM;
 1113
 1114	req->done = false;
 1115	strscpy(req->module, module_name, MODULE_NAME_LEN);
 1116	list_add_tail(&req->list, &nft_net->module_list);
 1117
 1118	return -EAGAIN;
 1119}
 1120EXPORT_SYMBOL_GPL(nft_request_module);
 1121#endif
 1122
 1123static void lockdep_nfnl_nft_mutex_not_held(void)
 1124{
 1125#ifdef CONFIG_PROVE_LOCKING
 1126	if (debug_locks)
 1127		WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 1128#endif
 1129}
 1130
 1131static const struct nft_chain_type *
 1132nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 1133			    u8 family, bool autoload)
 1134{
 1135	const struct nft_chain_type *type;
 1136
 1137	type = __nf_tables_chain_type_lookup(nla, family);
 1138	if (type != NULL)
 1139		return type;
 1140
 1141	lockdep_nfnl_nft_mutex_not_held();
 1142#ifdef CONFIG_MODULES
 1143	if (autoload) {
 1144		if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 1145				       nla_len(nla),
 1146				       (const char *)nla_data(nla)) == -EAGAIN)
 1147			return ERR_PTR(-EAGAIN);
 1148	}
 1149#endif
 1150	return ERR_PTR(-ENOENT);
 1151}
 1152
 1153static unsigned int nft_base_seq(const struct net *net)
 1154{
 1155	return READ_ONCE(net->nft.base_seq);
 1156}
 1157
 1158static __be16 nft_base_seq_be16(const struct net *net)
 1159{
 1160	return htons(nft_base_seq(net) & 0xffff);
 1161}
 1162
 1163static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 1164	[NFTA_TABLE_NAME]	= { .type = NLA_STRING,
 1165				    .len = NFT_TABLE_MAXNAMELEN - 1 },
 1166	[NFTA_TABLE_FLAGS]	= { .type = NLA_U32 },
 1167	[NFTA_TABLE_HANDLE]	= { .type = NLA_U64 },
 1168	[NFTA_TABLE_USERDATA]	= { .type = NLA_BINARY,
 1169				    .len = NFT_USERDATA_MAXLEN }
 1170};
 1171
 1172static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 1173				     u32 portid, u32 seq, int event, u32 flags,
 1174				     int family, const struct nft_table *table)
 1175{
 1176	struct nlmsghdr *nlh;
 1177
 1178	nlh = nfnl_msg_put(skb, portid, seq,
 1179			   nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event),
 1180			   flags, family, NFNETLINK_V0, nft_base_seq_be16(net));
 1181	if (!nlh)
 1182		goto nla_put_failure;
 1183
 1184	if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 1185	    nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 1186	    nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 1187			 NFTA_TABLE_PAD))
 1188		goto nla_put_failure;
 1189
 1190	if (event == NFT_MSG_DELTABLE ||
 1191	    event == NFT_MSG_DESTROYTABLE) {
 1192		nlmsg_end(skb, nlh);
 1193		return 0;
 1194	}
 1195
 1196	if (nla_put_be32(skb, NFTA_TABLE_FLAGS,
 1197			 htonl(table->flags & NFT_TABLE_F_MASK)))
 1198		goto nla_put_failure;
 1199
 1200	if (nft_table_has_owner(table) &&
 1201	    nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 1202		goto nla_put_failure;
 1203
 1204	if (table->udata) {
 1205		if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 1206			goto nla_put_failure;
 1207	}
 1208
 1209	nlmsg_end(skb, nlh);
 1210	return 0;
 1211
 1212nla_put_failure:
 1213	nlmsg_trim(skb, nlh);
 1214	return -1;
 1215}
 1216
 1217struct nftnl_skb_parms {
 1218	bool report;
 1219};
 1220#define NFT_CB(skb)	(*(struct nftnl_skb_parms*)&((skb)->cb))
 1221
 1222static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 1223			       struct list_head *notify_list)
 1224{
 1225	NFT_CB(skb).report = report;
 1226	list_add_tail(&skb->list, notify_list);
 1227}
 1228
 1229static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 1230{
 1231	struct nftables_pernet *nft_net;
 1232	struct sk_buff *skb;
 1233	u16 flags = 0;
 1234	int err;
 1235
 1236	if (!ctx->report &&
 1237	    !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 1238		return;
 1239
 1240	skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 1241	if (skb == NULL)
 1242		goto err;
 1243
 1244	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 1245		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 1246
 1247	err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 1248					event, flags, ctx->family, ctx->table);
 1249	if (err < 0) {
 1250		kfree_skb(skb);
 1251		goto err;
 1252	}
 1253
 1254	nft_net = nft_pernet(ctx->net);
 1255	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 1256	return;
 1257err:
 1258	nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 1259}
 1260
 1261static int nf_tables_dump_tables(struct sk_buff *skb,
 1262				 struct netlink_callback *cb)
 1263{
 1264	const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 1265	struct nftables_pernet *nft_net;
 1266	const struct nft_table *table;
 1267	unsigned int idx = 0, s_idx = cb->args[0];
 1268	struct net *net = sock_net(skb->sk);
 1269	int family = nfmsg->nfgen_family;
 1270
 1271	rcu_read_lock();
 1272	nft_net = nft_pernet(net);
 1273	cb->seq = nft_base_seq(net);
 1274
 1275	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 1276		if (family != NFPROTO_UNSPEC && family != table->family)
 1277			continue;
 1278
 1279		if (idx < s_idx)
 1280			goto cont;
 1281		if (idx > s_idx)
 1282			memset(&cb->args[1], 0,
 1283			       sizeof(cb->args) - sizeof(cb->args[0]));
 1284		if (!nft_is_active(net, table))
 1285			continue;
 1286		if (nf_tables_fill_table_info(skb, net,
 1287					      NETLINK_CB(cb->skb).portid,
 1288					      cb->nlh->nlmsg_seq,
 1289					      NFT_MSG_NEWTABLE, NLM_F_MULTI,
 1290					      table->family, table) < 0)
 1291			goto done;
 1292
 1293		nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 1294cont:
 1295		idx++;
 1296	}
 1297done:
 1298	rcu_read_unlock();
 1299	cb->args[0] = idx;
 1300	return skb->len;
 1301}
 1302
 1303static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 1304				      const struct nlmsghdr *nlh,
 1305				      struct netlink_dump_control *c)
 1306{
 1307	int err;
 1308
 1309	if (!try_module_get(THIS_MODULE))
 1310		return -EINVAL;
 1311
 1312	rcu_read_unlock();
 1313	err = netlink_dump_start(nlsk, skb, nlh, c);
 1314	rcu_read_lock();
 1315	module_put(THIS_MODULE);
 1316
 1317	return err;
 1318}
 1319
 1320/* called with rcu_read_lock held */
 1321static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
 1322			      const struct nlattr * const nla[])
 1323{
 1324	struct netlink_ext_ack *extack = info->extack;
 1325	u8 genmask = nft_genmask_cur(info->net);
 1326	u8 family = info->nfmsg->nfgen_family;
 1327	const struct nft_table *table;
 1328	struct net *net = info->net;
 1329	struct sk_buff *skb2;
 1330	int err;
 1331
 1332	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 1333		struct netlink_dump_control c = {
 1334			.dump = nf_tables_dump_tables,
 1335			.module = THIS_MODULE,
 1336		};
 1337
 1338		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 1339	}
 1340
 1341	table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
 1342	if (IS_ERR(table)) {
 1343		NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
 1344		return PTR_ERR(table);
 1345	}
 1346
 1347	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 1348	if (!skb2)
 1349		return -ENOMEM;
 1350
 1351	err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
 1352					info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
 1353					0, family, table);
 1354	if (err < 0)
 1355		goto err_fill_table_info;
 1356
 1357	return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 1358
 1359err_fill_table_info:
 1360	kfree_skb(skb2);
 1361	return err;
 1362}
 1363
 1364static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
 1365{
 1366	struct nft_chain *chain;
 1367	u32 i = 0;
 1368
 1369	list_for_each_entry(chain, &table->chains, list) {
 1370		if (!nft_is_active_next(net, chain))
 1371			continue;
 1372		if (!nft_is_base_chain(chain))
 1373			continue;
 1374
 1375		if (cnt && i++ == cnt)
 1376			break;
 1377
 1378		nf_tables_unregister_hook(net, table, chain);
 1379	}
 1380}
 1381
 1382static int nf_tables_table_enable(struct net *net, struct nft_table *table)
 1383{
 1384	struct nft_chain *chain;
 1385	int err, i = 0;
 1386
 1387	list_for_each_entry(chain, &table->chains, list) {
 1388		if (!nft_is_active_next(net, chain))
 1389			continue;
 1390		if (!nft_is_base_chain(chain))
 1391			continue;
 1392
 1393		err = nf_tables_register_hook(net, table, chain);
 1394		if (err < 0)
 1395			goto err_register_hooks;
 1396
 1397		i++;
 1398	}
 1399	return 0;
 1400
 1401err_register_hooks:
 1402	if (i)
 1403		nft_table_disable(net, table, i);
 1404	return err;
 1405}
 1406
 1407static void nf_tables_table_disable(struct net *net, struct nft_table *table)
 1408{
 1409	table->flags &= ~NFT_TABLE_F_DORMANT;
 1410	nft_table_disable(net, table, 0);
 1411	table->flags |= NFT_TABLE_F_DORMANT;
 1412}
 1413
 1414#define __NFT_TABLE_F_INTERNAL		(NFT_TABLE_F_MASK + 1)
 1415#define __NFT_TABLE_F_WAS_DORMANT	(__NFT_TABLE_F_INTERNAL << 0)
 1416#define __NFT_TABLE_F_WAS_AWAKEN	(__NFT_TABLE_F_INTERNAL << 1)
 1417#define __NFT_TABLE_F_WAS_ORPHAN	(__NFT_TABLE_F_INTERNAL << 2)
 1418#define __NFT_TABLE_F_UPDATE		(__NFT_TABLE_F_WAS_DORMANT | \
 1419					 __NFT_TABLE_F_WAS_AWAKEN | \
 1420					 __NFT_TABLE_F_WAS_ORPHAN)
 1421
 1422static bool nft_table_pending_update(const struct nft_ctx *ctx)
 1423{
 1424	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 1425	struct nft_trans *trans;
 1426
 1427	if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
 1428		return true;
 1429
 1430	list_for_each_entry(trans, &nft_net->commit_list, list) {
 1431		if (trans->table == ctx->table &&
 1432		    ((trans->msg_type == NFT_MSG_NEWCHAIN &&
 1433		      nft_trans_chain_update(trans)) ||
 1434		     (trans->msg_type == NFT_MSG_DELCHAIN &&
 1435		      nft_is_base_chain(nft_trans_chain(trans)))))
 1436			return true;
 1437	}
 1438
 1439	return false;
 1440}
 1441
 1442static int nf_tables_updtable(struct nft_ctx *ctx)
 1443{
 1444	struct nft_trans *trans;
 1445	u32 flags;
 1446	int ret;
 1447
 1448	if (!ctx->nla[NFTA_TABLE_FLAGS])
 1449		return 0;
 1450
 1451	flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
 1452	if (flags & ~NFT_TABLE_F_MASK)
 1453		return -EOPNOTSUPP;
 1454
 1455	if (flags == (ctx->table->flags & NFT_TABLE_F_MASK))
 1456		return 0;
 1457
 1458	if ((nft_table_has_owner(ctx->table) &&
 1459	     !(flags & NFT_TABLE_F_OWNER)) ||
 1460	    (flags & NFT_TABLE_F_OWNER &&
 1461	     !nft_table_is_orphan(ctx->table)))
 1462		return -EOPNOTSUPP;
 1463
 1464	if ((flags ^ ctx->table->flags) & NFT_TABLE_F_PERSIST)
 1465		return -EOPNOTSUPP;
 1466
 1467	/* No dormant off/on/off/on games in single transaction */
 1468	if (nft_table_pending_update(ctx))
 1469		return -EINVAL;
 1470
 1471	trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
 1472				sizeof(struct nft_trans_table));
 1473	if (trans == NULL)
 1474		return -ENOMEM;
 1475
 1476	if ((flags & NFT_TABLE_F_DORMANT) &&
 1477	    !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
 1478		ctx->table->flags |= NFT_TABLE_F_DORMANT;
 1479		if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
 1480			ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
 1481	} else if (!(flags & NFT_TABLE_F_DORMANT) &&
 1482		   ctx->table->flags & NFT_TABLE_F_DORMANT) {
 1483		ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
 1484		if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
 1485			ret = nf_tables_table_enable(ctx->net, ctx->table);
 1486			if (ret < 0)
 1487				goto err_register_hooks;
 1488
 1489			ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
 1490		}
 1491	}
 1492
 1493	if ((flags & NFT_TABLE_F_OWNER) &&
 1494	    !nft_table_has_owner(ctx->table)) {
 1495		ctx->table->nlpid = ctx->portid;
 1496		ctx->table->flags |= NFT_TABLE_F_OWNER |
 1497				     __NFT_TABLE_F_WAS_ORPHAN;
 1498	}
 1499
 1500	nft_trans_table_update(trans) = true;
 1501	nft_trans_commit_list_add_tail(ctx->net, trans);
 1502
 1503	return 0;
 1504
 1505err_register_hooks:
 1506	ctx->table->flags |= NFT_TABLE_F_DORMANT;
 1507	nft_trans_destroy(trans);
 1508	return ret;
 1509}
 1510
 1511static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
 1512{
 1513	const char *name = data;
 1514
 1515	return jhash(name, strlen(name), seed);
 1516}
 1517
 1518static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
 1519{
 1520	const struct nft_chain *chain = data;
 1521
 1522	return nft_chain_hash(chain->name, 0, seed);
 1523}
 1524
 1525static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
 1526			      const void *ptr)
 1527{
 1528	const struct nft_chain *chain = ptr;
 1529	const char *name = arg->key;
 1530
 1531	return strcmp(chain->name, name);
 1532}
 1533
 1534static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
 1535{
 1536	const struct nft_object_hash_key *k = data;
 1537
 1538	seed ^= hash_ptr(k->table, 32);
 1539
 1540	return jhash(k->name, strlen(k->name), seed);
 1541}
 1542
 1543static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
 1544{
 1545	const struct nft_object *obj = data;
 1546
 1547	return nft_objname_hash(&obj->key, 0, seed);
 1548}
 1549
 1550static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
 1551				const void *ptr)
 1552{
 1553	const struct nft_object_hash_key *k = arg->key;
 1554	const struct nft_object *obj = ptr;
 1555
 1556	if (obj->key.table != k->table)
 1557		return -1;
 1558
 1559	return strcmp(obj->key.name, k->name);
 1560}
 1561
 1562static bool nft_supported_family(u8 family)
 1563{
 1564	return false
 1565#ifdef CONFIG_NF_TABLES_INET
 1566		|| family == NFPROTO_INET
 1567#endif
 1568#ifdef CONFIG_NF_TABLES_IPV4
 1569		|| family == NFPROTO_IPV4
 1570#endif
 1571#ifdef CONFIG_NF_TABLES_ARP
 1572		|| family == NFPROTO_ARP
 1573#endif
 1574#ifdef CONFIG_NF_TABLES_NETDEV
 1575		|| family == NFPROTO_NETDEV
 1576#endif
 1577#if IS_ENABLED(CONFIG_NF_TABLES_BRIDGE)
 1578		|| family == NFPROTO_BRIDGE
 1579#endif
 1580#ifdef CONFIG_NF_TABLES_IPV6
 1581		|| family == NFPROTO_IPV6
 1582#endif
 1583		;
 1584}
 1585
 1586static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
 1587			      const struct nlattr * const nla[])
 1588{
 1589	struct nftables_pernet *nft_net = nft_pernet(info->net);
 1590	struct netlink_ext_ack *extack = info->extack;
 1591	u8 genmask = nft_genmask_next(info->net);
 1592	u8 family = info->nfmsg->nfgen_family;
 1593	struct net *net = info->net;
 1594	const struct nlattr *attr;
 1595	struct nft_table *table;
 1596	struct nft_ctx ctx;
 1597	u32 flags = 0;
 1598	int err;
 1599
 1600	if (!nft_supported_family(family))
 1601		return -EOPNOTSUPP;
 1602
 1603	lockdep_assert_held(&nft_net->commit_mutex);
 1604	attr = nla[NFTA_TABLE_NAME];
 1605	table = nft_table_lookup(net, attr, family, genmask,
 1606				 NETLINK_CB(skb).portid);
 1607	if (IS_ERR(table)) {
 1608		if (PTR_ERR(table) != -ENOENT)
 1609			return PTR_ERR(table);
 1610	} else {
 1611		if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
 1612			NL_SET_BAD_ATTR(extack, attr);
 1613			return -EEXIST;
 1614		}
 1615		if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
 1616			return -EOPNOTSUPP;
 1617
 1618		nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 1619
 1620		return nf_tables_updtable(&ctx);
 1621	}
 1622
 1623	if (nla[NFTA_TABLE_FLAGS]) {
 1624		flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
 1625		if (flags & ~NFT_TABLE_F_MASK)
 1626			return -EOPNOTSUPP;
 1627	}
 1628
 1629	err = -ENOMEM;
 1630	table = kzalloc(sizeof(*table), GFP_KERNEL_ACCOUNT);
 1631	if (table == NULL)
 1632		goto err_kzalloc;
 1633
 1634	table->validate_state = nft_net->validate_state;
 1635	table->name = nla_strdup(attr, GFP_KERNEL_ACCOUNT);
 1636	if (table->name == NULL)
 1637		goto err_strdup;
 1638
 1639	if (nla[NFTA_TABLE_USERDATA]) {
 1640		table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL_ACCOUNT);
 1641		if (table->udata == NULL)
 1642			goto err_table_udata;
 1643
 1644		table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
 1645	}
 1646
 1647	err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
 1648	if (err)
 1649		goto err_chain_ht;
 1650
 1651	INIT_LIST_HEAD(&table->chains);
 1652	INIT_LIST_HEAD(&table->sets);
 1653	INIT_LIST_HEAD(&table->objects);
 1654	INIT_LIST_HEAD(&table->flowtables);
 1655	table->family = family;
 1656	table->flags = flags;
 1657	table->handle = ++nft_net->table_handle;
 1658	if (table->flags & NFT_TABLE_F_OWNER)
 1659		table->nlpid = NETLINK_CB(skb).portid;
 1660
 1661	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 1662	err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
 1663	if (err < 0)
 1664		goto err_trans;
 1665
 1666	list_add_tail_rcu(&table->list, &nft_net->tables);
 1667	return 0;
 1668err_trans:
 1669	rhltable_destroy(&table->chains_ht);
 1670err_chain_ht:
 1671	kfree(table->udata);
 1672err_table_udata:
 1673	kfree(table->name);
 1674err_strdup:
 1675	kfree(table);
 1676err_kzalloc:
 1677	return err;
 1678}
 1679
 1680static int nft_flush_table(struct nft_ctx *ctx)
 1681{
 1682	struct nft_flowtable *flowtable, *nft;
 1683	struct nft_chain *chain, *nc;
 1684	struct nft_object *obj, *ne;
 1685	struct nft_set *set, *ns;
 1686	int err;
 1687
 1688	list_for_each_entry(chain, &ctx->table->chains, list) {
 1689		if (!nft_is_active_next(ctx->net, chain))
 1690			continue;
 1691
 1692		if (nft_chain_binding(chain))
 1693			continue;
 1694
 1695		ctx->chain = chain;
 1696
 1697		err = nft_delrule_by_chain(ctx);
 1698		if (err < 0)
 1699			goto out;
 1700	}
 1701
 1702	list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
 1703		if (!nft_is_active_next(ctx->net, set))
 1704			continue;
 1705
 1706		if (nft_set_is_anonymous(set))
 1707			continue;
 1708
 1709		err = nft_delset(ctx, set);
 1710		if (err < 0)
 1711			goto out;
 1712	}
 1713
 1714	list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
 1715		if (!nft_is_active_next(ctx->net, flowtable))
 1716			continue;
 1717
 1718		err = nft_delflowtable(ctx, flowtable);
 1719		if (err < 0)
 1720			goto out;
 1721	}
 1722
 1723	list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
 1724		if (!nft_is_active_next(ctx->net, obj))
 1725			continue;
 1726
 1727		err = nft_delobj(ctx, obj);
 1728		if (err < 0)
 1729			goto out;
 1730	}
 1731
 1732	list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
 1733		if (!nft_is_active_next(ctx->net, chain))
 1734			continue;
 1735
 1736		if (nft_chain_binding(chain))
 1737			continue;
 1738
 1739		ctx->chain = chain;
 1740
 1741		err = nft_delchain(ctx);
 1742		if (err < 0)
 1743			goto out;
 1744	}
 1745
 1746	err = nft_deltable(ctx);
 1747out:
 1748	return err;
 1749}
 1750
 1751static int nft_flush(struct nft_ctx *ctx, int family)
 1752{
 1753	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 1754	const struct nlattr * const *nla = ctx->nla;
 1755	struct nft_table *table, *nt;
 1756	int err = 0;
 1757
 1758	list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
 1759		if (family != AF_UNSPEC && table->family != family)
 1760			continue;
 1761
 1762		ctx->family = table->family;
 1763
 1764		if (!nft_is_active_next(ctx->net, table))
 1765			continue;
 1766
 1767		if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
 1768			continue;
 1769
 1770		if (nla[NFTA_TABLE_NAME] &&
 1771		    nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
 1772			continue;
 1773
 1774		ctx->table = table;
 1775
 1776		err = nft_flush_table(ctx);
 1777		if (err < 0)
 1778			goto out;
 1779	}
 1780out:
 1781	return err;
 1782}
 1783
 1784static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
 1785			      const struct nlattr * const nla[])
 1786{
 1787	struct netlink_ext_ack *extack = info->extack;
 1788	u8 genmask = nft_genmask_next(info->net);
 1789	u8 family = info->nfmsg->nfgen_family;
 1790	struct net *net = info->net;
 1791	const struct nlattr *attr;
 1792	struct nft_table *table;
 1793	struct nft_ctx ctx;
 1794
 1795	nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
 1796	if (family == AF_UNSPEC ||
 1797	    (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
 1798		return nft_flush(&ctx, family);
 1799
 1800	if (nla[NFTA_TABLE_HANDLE]) {
 1801		attr = nla[NFTA_TABLE_HANDLE];
 1802		table = nft_table_lookup_byhandle(net, attr, family, genmask,
 1803						  NETLINK_CB(skb).portid);
 1804	} else {
 1805		attr = nla[NFTA_TABLE_NAME];
 1806		table = nft_table_lookup(net, attr, family, genmask,
 1807					 NETLINK_CB(skb).portid);
 1808	}
 1809
 1810	if (IS_ERR(table)) {
 1811		if (PTR_ERR(table) == -ENOENT &&
 1812		    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYTABLE)
 1813			return 0;
 1814
 1815		NL_SET_BAD_ATTR(extack, attr);
 1816		return PTR_ERR(table);
 1817	}
 1818
 1819	if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
 1820	    table->use > 0)
 1821		return -EBUSY;
 1822
 1823	ctx.family = family;
 1824	ctx.table = table;
 1825
 1826	return nft_flush_table(&ctx);
 1827}
 1828
 1829static void nf_tables_table_destroy(struct nft_table *table)
 1830{
 1831	if (WARN_ON(table->use > 0))
 1832		return;
 1833
 1834	rhltable_destroy(&table->chains_ht);
 1835	kfree(table->name);
 1836	kfree(table->udata);
 1837	kfree(table);
 1838}
 1839
 1840void nft_register_chain_type(const struct nft_chain_type *ctype)
 1841{
 1842	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 1843	if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
 1844		nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 1845		return;
 1846	}
 1847	chain_type[ctype->family][ctype->type] = ctype;
 1848	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 1849}
 1850EXPORT_SYMBOL_GPL(nft_register_chain_type);
 1851
 1852void nft_unregister_chain_type(const struct nft_chain_type *ctype)
 1853{
 1854	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 1855	chain_type[ctype->family][ctype->type] = NULL;
 1856	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 1857}
 1858EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
 1859
 1860/*
 1861 * Chains
 1862 */
 1863
 1864static struct nft_chain *
 1865nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
 1866{
 1867	struct nft_chain *chain;
 1868
 1869	list_for_each_entry(chain, &table->chains, list) {
 1870		if (chain->handle == handle &&
 1871		    nft_active_genmask(chain, genmask))
 1872			return chain;
 1873	}
 1874
 1875	return ERR_PTR(-ENOENT);
 1876}
 1877
 1878static bool lockdep_commit_lock_is_held(const struct net *net)
 1879{
 1880#ifdef CONFIG_PROVE_LOCKING
 1881	struct nftables_pernet *nft_net = nft_pernet(net);
 1882
 1883	return lockdep_is_held(&nft_net->commit_mutex);
 1884#else
 1885	return true;
 1886#endif
 1887}
 1888
 1889static struct nft_chain *nft_chain_lookup(struct net *net,
 1890					  struct nft_table *table,
 1891					  const struct nlattr *nla, u8 genmask)
 1892{
 1893	char search[NFT_CHAIN_MAXNAMELEN + 1];
 1894	struct rhlist_head *tmp, *list;
 1895	struct nft_chain *chain;
 1896
 1897	if (nla == NULL)
 1898		return ERR_PTR(-EINVAL);
 1899
 1900	nla_strscpy(search, nla, sizeof(search));
 1901
 1902	WARN_ON(!rcu_read_lock_held() &&
 1903		!lockdep_commit_lock_is_held(net));
 1904
 1905	chain = ERR_PTR(-ENOENT);
 1906	rcu_read_lock();
 1907	list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
 1908	if (!list)
 1909		goto out_unlock;
 1910
 1911	rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
 1912		if (nft_active_genmask(chain, genmask))
 1913			goto out_unlock;
 1914	}
 1915	chain = ERR_PTR(-ENOENT);
 1916out_unlock:
 1917	rcu_read_unlock();
 1918	return chain;
 1919}
 1920
 1921static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
 1922	[NFTA_CHAIN_TABLE]	= { .type = NLA_STRING,
 1923				    .len = NFT_TABLE_MAXNAMELEN - 1 },
 1924	[NFTA_CHAIN_HANDLE]	= { .type = NLA_U64 },
 1925	[NFTA_CHAIN_NAME]	= { .type = NLA_STRING,
 1926				    .len = NFT_CHAIN_MAXNAMELEN - 1 },
 1927	[NFTA_CHAIN_HOOK]	= { .type = NLA_NESTED },
 1928	[NFTA_CHAIN_POLICY]	= { .type = NLA_U32 },
 1929	[NFTA_CHAIN_TYPE]	= { .type = NLA_STRING,
 1930				    .len = NFT_MODULE_AUTOLOAD_LIMIT },
 1931	[NFTA_CHAIN_COUNTERS]	= { .type = NLA_NESTED },
 1932	[NFTA_CHAIN_FLAGS]	= { .type = NLA_U32 },
 1933	[NFTA_CHAIN_ID]		= { .type = NLA_U32 },
 1934	[NFTA_CHAIN_USERDATA]	= { .type = NLA_BINARY,
 1935				    .len = NFT_USERDATA_MAXLEN },
 1936};
 1937
 1938static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
 1939	[NFTA_HOOK_HOOKNUM]	= { .type = NLA_U32 },
 1940	[NFTA_HOOK_PRIORITY]	= { .type = NLA_U32 },
 1941	[NFTA_HOOK_DEV]		= { .type = NLA_STRING,
 1942				    .len = IFNAMSIZ - 1 },
 1943};
 1944
 1945static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
 1946{
 1947	struct nft_stats *cpu_stats, total;
 1948	struct nlattr *nest;
 1949	unsigned int seq;
 1950	u64 pkts, bytes;
 1951	int cpu;
 1952
 1953	if (!stats)
 1954		return 0;
 1955
 1956	memset(&total, 0, sizeof(total));
 1957	for_each_possible_cpu(cpu) {
 1958		cpu_stats = per_cpu_ptr(stats, cpu);
 1959		do {
 1960			seq = u64_stats_fetch_begin(&cpu_stats->syncp);
 1961			pkts = cpu_stats->pkts;
 1962			bytes = cpu_stats->bytes;
 1963		} while (u64_stats_fetch_retry(&cpu_stats->syncp, seq));
 1964		total.pkts += pkts;
 1965		total.bytes += bytes;
 1966	}
 1967	nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
 1968	if (nest == NULL)
 1969		goto nla_put_failure;
 1970
 1971	if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
 1972			 NFTA_COUNTER_PAD) ||
 1973	    nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
 1974			 NFTA_COUNTER_PAD))
 1975		goto nla_put_failure;
 1976
 1977	nla_nest_end(skb, nest);
 1978	return 0;
 1979
 1980nla_put_failure:
 1981	return -ENOSPC;
 1982}
 1983
 1984static bool hook_is_prefix(struct nft_hook *hook)
 1985{
 1986	return strlen(hook->ifname) >= hook->ifnamelen;
 1987}
 1988
 1989static int nft_nla_put_hook_dev(struct sk_buff *skb, struct nft_hook *hook)
 1990{
 1991	int attr = hook_is_prefix(hook) ? NFTA_DEVICE_PREFIX : NFTA_DEVICE_NAME;
 1992
 1993	return nla_put_string(skb, attr, hook->ifname);
 1994}
 1995
 1996static int nft_dump_basechain_hook(struct sk_buff *skb,
 1997				   const struct net *net, int family,
 1998				   const struct nft_base_chain *basechain,
 1999				   const struct list_head *hook_list)
 2000{
 2001	const struct nf_hook_ops *ops = &basechain->ops;
 2002	struct nft_hook *hook, *first = NULL;
 2003	struct nlattr *nest, *nest_devs;
 2004	int n = 0;
 2005
 2006	nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
 2007	if (nest == NULL)
 2008		goto nla_put_failure;
 2009	if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
 2010		goto nla_put_failure;
 2011	if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
 2012		goto nla_put_failure;
 2013
 2014	if (nft_base_chain_netdev(family, ops->hooknum)) {
 2015		nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
 2016		if (!nest_devs)
 2017			goto nla_put_failure;
 2018
 2019		if (!hook_list)
 2020			hook_list = &basechain->hook_list;
 2021
 2022		list_for_each_entry_rcu(hook, hook_list, list,
 2023					lockdep_commit_lock_is_held(net)) {
 2024			if (!first)
 2025				first = hook;
 2026
 2027			if (nft_nla_put_hook_dev(skb, hook))
 2028				goto nla_put_failure;
 2029			n++;
 2030		}
 2031		nla_nest_end(skb, nest_devs);
 2032
 2033		if (n == 1 &&
 2034		    !hook_is_prefix(first) &&
 2035		    nla_put_string(skb, NFTA_HOOK_DEV, first->ifname))
 2036			goto nla_put_failure;
 2037	}
 2038	nla_nest_end(skb, nest);
 2039
 2040	return 0;
 2041nla_put_failure:
 2042	return -1;
 2043}
 2044
 2045static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
 2046				     u32 portid, u32 seq, int event, u32 flags,
 2047				     int family, const struct nft_table *table,
 2048				     const struct nft_chain *chain,
 2049				     const struct list_head *hook_list)
 2050{
 2051	struct nlmsghdr *nlh;
 2052
 2053	nlh = nfnl_msg_put(skb, portid, seq,
 2054			   nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event),
 2055			   flags, family, NFNETLINK_V0, nft_base_seq_be16(net));
 2056	if (!nlh)
 2057		goto nla_put_failure;
 2058
 2059	if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name) ||
 2060	    nla_put_string(skb, NFTA_CHAIN_NAME, chain->name) ||
 2061	    nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
 2062			 NFTA_CHAIN_PAD))
 2063		goto nla_put_failure;
 2064
 2065	if (!hook_list &&
 2066	    (event == NFT_MSG_DELCHAIN ||
 2067	     event == NFT_MSG_DESTROYCHAIN)) {
 2068		nlmsg_end(skb, nlh);
 2069		return 0;
 2070	}
 2071
 2072	if (nft_is_base_chain(chain)) {
 2073		const struct nft_base_chain *basechain = nft_base_chain(chain);
 2074		struct nft_stats __percpu *stats;
 2075
 2076		if (nft_dump_basechain_hook(skb, net, family, basechain, hook_list))
 2077			goto nla_put_failure;
 2078
 2079		if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
 2080				 htonl(basechain->policy)))
 2081			goto nla_put_failure;
 2082
 2083		if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
 2084			goto nla_put_failure;
 2085
 2086		stats = rcu_dereference_check(basechain->stats,
 2087					      lockdep_commit_lock_is_held(net));
 2088		if (nft_dump_stats(skb, stats))
 2089			goto nla_put_failure;
 2090	}
 2091
 2092	if (chain->flags &&
 2093	    nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
 2094		goto nla_put_failure;
 2095
 2096	if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
 2097		goto nla_put_failure;
 2098
 2099	if (chain->udata &&
 2100	    nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
 2101		goto nla_put_failure;
 2102
 2103	nlmsg_end(skb, nlh);
 2104	return 0;
 2105
 2106nla_put_failure:
 2107	nlmsg_trim(skb, nlh);
 2108	return -1;
 2109}
 2110
 2111static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event,
 2112				   const struct list_head *hook_list)
 2113{
 2114	struct nftables_pernet *nft_net;
 2115	struct sk_buff *skb;
 2116	u16 flags = 0;
 2117	int err;
 2118
 2119	if (!ctx->report &&
 2120	    !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 2121		return;
 2122
 2123	skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 2124	if (skb == NULL)
 2125		goto err;
 2126
 2127	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 2128		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 2129
 2130	err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
 2131					event, flags, ctx->family, ctx->table,
 2132					ctx->chain, hook_list);
 2133	if (err < 0) {
 2134		kfree_skb(skb);
 2135		goto err;
 2136	}
 2137
 2138	nft_net = nft_pernet(ctx->net);
 2139	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 2140	return;
 2141err:
 2142	nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 2143}
 2144
 2145static int nf_tables_dump_chains(struct sk_buff *skb,
 2146				 struct netlink_callback *cb)
 2147{
 2148	const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 2149	unsigned int idx = 0, s_idx = cb->args[0];
 2150	struct net *net = sock_net(skb->sk);
 2151	int family = nfmsg->nfgen_family;
 2152	struct nftables_pernet *nft_net;
 2153	const struct nft_table *table;
 2154	const struct nft_chain *chain;
 2155
 2156	rcu_read_lock();
 2157	nft_net = nft_pernet(net);
 2158	cb->seq = nft_base_seq(net);
 2159
 2160	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 2161		if (family != NFPROTO_UNSPEC && family != table->family)
 2162			continue;
 2163
 2164		list_for_each_entry_rcu(chain, &table->chains, list) {
 2165			if (idx < s_idx)
 2166				goto cont;
 2167			if (idx > s_idx)
 2168				memset(&cb->args[1], 0,
 2169				       sizeof(cb->args) - sizeof(cb->args[0]));
 2170			if (!nft_is_active(net, chain))
 2171				continue;
 2172			if (nf_tables_fill_chain_info(skb, net,
 2173						      NETLINK_CB(cb->skb).portid,
 2174						      cb->nlh->nlmsg_seq,
 2175						      NFT_MSG_NEWCHAIN,
 2176						      NLM_F_MULTI,
 2177						      table->family, table,
 2178						      chain, NULL) < 0)
 2179				goto done;
 2180
 2181			nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 2182cont:
 2183			idx++;
 2184		}
 2185	}
 2186done:
 2187	rcu_read_unlock();
 2188	cb->args[0] = idx;
 2189	return skb->len;
 2190}
 2191
 2192/* called with rcu_read_lock held */
 2193static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
 2194			      const struct nlattr * const nla[])
 2195{
 2196	struct netlink_ext_ack *extack = info->extack;
 2197	u8 genmask = nft_genmask_cur(info->net);
 2198	u8 family = info->nfmsg->nfgen_family;
 2199	const struct nft_chain *chain;
 2200	struct net *net = info->net;
 2201	struct nft_table *table;
 2202	struct sk_buff *skb2;
 2203	int err;
 2204
 2205	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 2206		struct netlink_dump_control c = {
 2207			.dump = nf_tables_dump_chains,
 2208			.module = THIS_MODULE,
 2209		};
 2210
 2211		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 2212	}
 2213
 2214	table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
 2215	if (IS_ERR(table)) {
 2216		NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
 2217		return PTR_ERR(table);
 2218	}
 2219
 2220	chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
 2221	if (IS_ERR(chain)) {
 2222		NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
 2223		return PTR_ERR(chain);
 2224	}
 2225
 2226	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 2227	if (!skb2)
 2228		return -ENOMEM;
 2229
 2230	err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
 2231					info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
 2232					0, family, table, chain, NULL);
 2233	if (err < 0)
 2234		goto err_fill_chain_info;
 2235
 2236	return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 2237
 2238err_fill_chain_info:
 2239	kfree_skb(skb2);
 2240	return err;
 2241}
 2242
 2243static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
 2244	[NFTA_COUNTER_PACKETS]	= { .type = NLA_U64 },
 2245	[NFTA_COUNTER_BYTES]	= { .type = NLA_U64 },
 2246};
 2247
 2248static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
 2249{
 2250	struct nlattr *tb[NFTA_COUNTER_MAX+1];
 2251	struct nft_stats __percpu *newstats;
 2252	struct nft_stats *stats;
 2253	int err;
 2254
 2255	err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
 2256					  nft_counter_policy, NULL);
 2257	if (err < 0)
 2258		return ERR_PTR_PCPU(err);
 2259
 2260	if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
 2261		return ERR_PTR_PCPU(-EINVAL);
 2262
 2263	newstats = netdev_alloc_pcpu_stats(struct nft_stats);
 2264	if (newstats == NULL)
 2265		return ERR_PTR_PCPU(-ENOMEM);
 2266
 2267	/* Restore old counters on this cpu, no problem. Per-cpu statistics
 2268	 * are not exposed to userspace.
 2269	 */
 2270	preempt_disable();
 2271	stats = this_cpu_ptr(newstats);
 2272	stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
 2273	stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
 2274	preempt_enable();
 2275
 2276	return newstats;
 2277}
 2278
 2279static void nft_chain_stats_replace(struct nft_trans_chain *trans)
 2280{
 2281	const struct nft_trans *t = &trans->nft_trans_binding.nft_trans;
 2282	struct nft_base_chain *chain = nft_base_chain(trans->chain);
 2283
 2284	if (!trans->stats)
 2285		return;
 2286
 2287	trans->stats =
 2288		rcu_replace_pointer(chain->stats, trans->stats,
 2289				    lockdep_commit_lock_is_held(t->net));
 2290
 2291	if (!trans->stats)
 2292		static_branch_inc(&nft_counters_enabled);
 2293}
 2294
 2295static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
 2296{
 2297	struct nft_rule_blob *g0 = rcu_dereference_raw(chain->blob_gen_0);
 2298	struct nft_rule_blob *g1 = rcu_dereference_raw(chain->blob_gen_1);
 2299
 2300	if (g0 != g1)
 2301		kvfree(g1);
 2302	kvfree(g0);
 2303
 2304	/* should be NULL either via abort or via successful commit */
 2305	WARN_ON_ONCE(chain->blob_next);
 2306	kvfree(chain->blob_next);
 2307}
 2308
 2309void nf_tables_chain_destroy(struct nft_chain *chain)
 2310{
 2311	const struct nft_table *table = chain->table;
 2312	struct nft_hook *hook, *next;
 2313
 2314	if (WARN_ON(chain->use > 0))
 2315		return;
 2316
 2317	/* no concurrent access possible anymore */
 2318	nf_tables_chain_free_chain_rules(chain);
 2319
 2320	if (nft_is_base_chain(chain)) {
 2321		struct nft_base_chain *basechain = nft_base_chain(chain);
 2322
 2323		if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
 2324			list_for_each_entry_safe(hook, next,
 2325						 &basechain->hook_list, list) {
 2326				list_del_rcu(&hook->list);
 2327				nft_netdev_hook_free_rcu(hook);
 2328			}
 2329		}
 2330		module_put(basechain->type->owner);
 2331		if (rcu_access_pointer(basechain->stats)) {
 2332			static_branch_dec(&nft_counters_enabled);
 2333			free_percpu(rcu_dereference_raw(basechain->stats));
 2334		}
 2335		kfree(chain->name);
 2336		kfree(chain->udata);
 2337		kfree(basechain);
 2338	} else {
 2339		kfree(chain->name);
 2340		kfree(chain->udata);
 2341		kfree(chain);
 2342	}
 2343}
 2344
 2345static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
 2346					      const struct nlattr *attr,
 2347					      bool prefix)
 2348{
 2349	struct nf_hook_ops *ops;
 2350	struct net_device *dev;
 2351	struct nft_hook *hook;
 2352	int err;
 2353
 2354	hook = kzalloc(sizeof(struct nft_hook), GFP_KERNEL_ACCOUNT);
 2355	if (!hook)
 2356		return ERR_PTR(-ENOMEM);
 2357
 2358	INIT_LIST_HEAD(&hook->ops_list);
 2359
 2360	err = nla_strscpy(hook->ifname, attr, IFNAMSIZ);
 2361	if (err < 0)
 2362		goto err_hook_free;
 2363
 2364	/* include the terminating NUL-char when comparing non-prefixes */
 2365	hook->ifnamelen = strlen(hook->ifname) + !prefix;
 2366
 2367	/* nf_tables_netdev_event() is called under rtnl_mutex, this is
 2368	 * indirectly serializing all the other holders of the commit_mutex with
 2369	 * the rtnl_mutex.
 2370	 */
 2371	for_each_netdev(net, dev) {
 2372		if (strncmp(dev->name, hook->ifname, hook->ifnamelen))
 2373			continue;
 2374
 2375		ops = kzalloc(sizeof(struct nf_hook_ops), GFP_KERNEL_ACCOUNT);
 2376		if (!ops) {
 2377			err = -ENOMEM;
 2378			goto err_hook_free;
 2379		}
 2380		ops->dev = dev;
 2381		list_add_tail(&ops->list, &hook->ops_list);
 2382	}
 2383	return hook;
 2384
 2385err_hook_free:
 2386	nft_netdev_hook_free(hook);
 2387	return ERR_PTR(err);
 2388}
 2389
 2390static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
 2391					   const struct nft_hook *this)
 2392{
 2393	struct nft_hook *hook;
 2394
 2395	list_for_each_entry(hook, hook_list, list) {
 2396		if (!strncmp(hook->ifname, this->ifname,
 2397			     min(hook->ifnamelen, this->ifnamelen)))
 2398			return hook;
 2399	}
 2400
 2401	return NULL;
 2402}
 2403
 2404static int nf_tables_parse_netdev_hooks(struct net *net,
 2405					const struct nlattr *attr,
 2406					struct list_head *hook_list,
 2407					struct netlink_ext_ack *extack)
 2408{
 2409	struct nft_hook *hook, *next;
 2410	const struct nlattr *tmp;
 2411	int rem, n = 0, err;
 2412	bool prefix;
 2413
 2414	nla_for_each_nested(tmp, attr, rem) {
 2415		switch (nla_type(tmp)) {
 2416		case NFTA_DEVICE_NAME:
 2417			prefix = false;
 2418			break;
 2419		case NFTA_DEVICE_PREFIX:
 2420			prefix = true;
 2421			break;
 2422		default:
 2423			err = -EINVAL;
 2424			goto err_hook;
 2425		}
 2426
 2427		hook = nft_netdev_hook_alloc(net, tmp, prefix);
 2428		if (IS_ERR(hook)) {
 2429			NL_SET_BAD_ATTR(extack, tmp);
 2430			err = PTR_ERR(hook);
 2431			goto err_hook;
 2432		}
 2433		if (nft_hook_list_find(hook_list, hook)) {
 2434			NL_SET_BAD_ATTR(extack, tmp);
 2435			nft_netdev_hook_free(hook);
 2436			err = -EEXIST;
 2437			goto err_hook;
 2438		}
 2439		list_add_tail(&hook->list, hook_list);
 2440		n++;
 2441
 2442		if (n == NFT_NETDEVICE_MAX) {
 2443			err = -EFBIG;
 2444			goto err_hook;
 2445		}
 2446	}
 2447
 2448	return 0;
 2449
 2450err_hook:
 2451	list_for_each_entry_safe(hook, next, hook_list, list) {
 2452		list_del(&hook->list);
 2453		nft_netdev_hook_free(hook);
 2454	}
 2455	return err;
 2456}
 2457
 2458struct nft_chain_hook {
 2459	u32				num;
 2460	s32				priority;
 2461	const struct nft_chain_type	*type;
 2462	struct list_head		list;
 2463};
 2464
 2465static int nft_chain_parse_netdev(struct net *net, struct nlattr *tb[],
 2466				  struct list_head *hook_list,
 2467				  struct netlink_ext_ack *extack, u32 flags)
 2468{
 2469	struct nft_hook *hook;
 2470	int err;
 2471
 2472	if (tb[NFTA_HOOK_DEV]) {
 2473		hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV], false);
 2474		if (IS_ERR(hook)) {
 2475			NL_SET_BAD_ATTR(extack, tb[NFTA_HOOK_DEV]);
 2476			return PTR_ERR(hook);
 2477		}
 2478
 2479		list_add_tail(&hook->list, hook_list);
 2480	} else if (tb[NFTA_HOOK_DEVS]) {
 2481		err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
 2482						   hook_list, extack);
 2483		if (err < 0)
 2484			return err;
 2485
 2486	}
 2487
 2488	if (flags & NFT_CHAIN_HW_OFFLOAD &&
 2489	    list_empty(hook_list))
 2490		return -EINVAL;
 2491
 2492	return 0;
 2493}
 2494
 2495static int nft_chain_parse_hook(struct net *net,
 2496				struct nft_base_chain *basechain,
 2497				const struct nlattr * const nla[],
 2498				struct nft_chain_hook *hook, u8 family,
 2499				u32 flags, struct netlink_ext_ack *extack)
 2500{
 2501	struct nftables_pernet *nft_net = nft_pernet(net);
 2502	struct nlattr *ha[NFTA_HOOK_MAX + 1];
 2503	const struct nft_chain_type *type;
 2504	int err;
 2505
 2506	lockdep_assert_held(&nft_net->commit_mutex);
 2507	lockdep_nfnl_nft_mutex_not_held();
 2508
 2509	err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
 2510					  nla[NFTA_CHAIN_HOOK],
 2511					  nft_hook_policy, NULL);
 2512	if (err < 0)
 2513		return err;
 2514
 2515	if (!basechain) {
 2516		if (!ha[NFTA_HOOK_HOOKNUM] ||
 2517		    !ha[NFTA_HOOK_PRIORITY]) {
 2518			NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
 2519			return -ENOENT;
 2520		}
 2521
 2522		hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
 2523		hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
 2524
 2525		type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
 2526		if (!type)
 2527			return -EOPNOTSUPP;
 2528
 2529		if (nla[NFTA_CHAIN_TYPE]) {
 2530			type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
 2531							   family, true);
 2532			if (IS_ERR(type)) {
 2533				NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
 2534				return PTR_ERR(type);
 2535			}
 2536		}
 2537		if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
 2538			return -EOPNOTSUPP;
 2539
 2540		if (type->type == NFT_CHAIN_T_NAT &&
 2541		    hook->priority <= NF_IP_PRI_CONNTRACK)
 2542			return -EOPNOTSUPP;
 2543	} else {
 2544		if (ha[NFTA_HOOK_HOOKNUM]) {
 2545			hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
 2546			if (hook->num != basechain->ops.hooknum)
 2547				return -EOPNOTSUPP;
 2548		}
 2549		if (ha[NFTA_HOOK_PRIORITY]) {
 2550			hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
 2551			if (hook->priority != basechain->ops.priority)
 2552				return -EOPNOTSUPP;
 2553		}
 2554
 2555		if (nla[NFTA_CHAIN_TYPE]) {
 2556			type = __nf_tables_chain_type_lookup(nla[NFTA_CHAIN_TYPE],
 2557							     family);
 2558			if (!type) {
 2559				NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
 2560				return -ENOENT;
 2561			}
 2562		} else {
 2563			type = basechain->type;
 2564		}
 2565	}
 2566
 2567	if (!try_module_get(type->owner)) {
 2568		if (nla[NFTA_CHAIN_TYPE])
 2569			NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
 2570		return -ENOENT;
 2571	}
 2572
 2573	hook->type = type;
 2574
 2575	INIT_LIST_HEAD(&hook->list);
 2576	if (nft_base_chain_netdev(family, hook->num)) {
 2577		err = nft_chain_parse_netdev(net, ha, &hook->list, extack, flags);
 2578		if (err < 0) {
 2579			module_put(type->owner);
 2580			return err;
 2581		}
 2582	} else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
 2583		module_put(type->owner);
 2584		return -EOPNOTSUPP;
 2585	}
 2586
 2587	return 0;
 2588}
 2589
 2590static void nft_chain_release_hook(struct nft_chain_hook *hook)
 2591{
 2592	struct nft_hook *h, *next;
 2593
 2594	list_for_each_entry_safe(h, next, &hook->list, list) {
 2595		list_del(&h->list);
 2596		nft_netdev_hook_free(h);
 2597	}
 2598	module_put(hook->type->owner);
 2599}
 2600
 2601static void nft_last_rule(const struct nft_chain *chain, const void *ptr)
 2602{
 2603	struct nft_rule_dp_last *lrule;
 2604
 2605	BUILD_BUG_ON(offsetof(struct nft_rule_dp_last, end) != 0);
 2606
 2607	lrule = (struct nft_rule_dp_last *)ptr;
 2608	lrule->end.is_last = 1;
 2609	lrule->chain = chain;
 2610	/* blob size does not include the trailer rule */
 2611}
 2612
 2613static struct nft_rule_blob *nf_tables_chain_alloc_rules(const struct nft_chain *chain,
 2614							 unsigned int size)
 2615{
 2616	struct nft_rule_blob *blob;
 2617
 2618	if (size > INT_MAX)
 2619		return NULL;
 2620
 2621	size += sizeof(struct nft_rule_blob) + sizeof(struct nft_rule_dp_last);
 2622
 2623	blob = kvmalloc(size, GFP_KERNEL_ACCOUNT);
 2624	if (!blob)
 2625		return NULL;
 2626
 2627	blob->size = 0;
 2628	nft_last_rule(chain, blob->data);
 2629
 2630	return blob;
 2631}
 2632
 2633static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
 2634				    const struct nft_chain_hook *hook,
 2635				    struct nft_chain *chain)
 2636{
 2637	ops->pf			= family;
 2638	ops->hooknum		= hook->num;
 2639	ops->priority		= hook->priority;
 2640	ops->priv		= chain;
 2641	ops->hook		= hook->type->hooks[ops->hooknum];
 2642	ops->hook_ops_type	= NF_HOOK_OP_NF_TABLES;
 2643}
 2644
 2645static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
 2646			      struct nft_chain_hook *hook, u32 flags)
 2647{
 2648	struct nft_chain *chain;
 2649	struct nf_hook_ops *ops;
 2650	struct nft_hook *h;
 2651
 2652	basechain->type = hook->type;
 2653	INIT_LIST_HEAD(&basechain->hook_list);
 2654	chain = &basechain->chain;
 2655
 2656	if (nft_base_chain_netdev(family, hook->num)) {
 2657		list_splice_init(&hook->list, &basechain->hook_list);
 2658		list_for_each_entry(h, &basechain->hook_list, list) {
 2659			list_for_each_entry(ops, &h->ops_list, list)
 2660				nft_basechain_hook_init(ops, family, hook, chain);
 2661		}
 2662	}
 2663	nft_basechain_hook_init(&basechain->ops, family, hook, chain);
 2664
 2665	chain->flags |= NFT_CHAIN_BASE | flags;
 2666	basechain->policy = NF_ACCEPT;
 2667	if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
 2668	    !nft_chain_offload_support(basechain)) {
 2669		list_splice_init(&basechain->hook_list, &hook->list);
 2670		return -EOPNOTSUPP;
 2671	}
 2672
 2673	flow_block_init(&basechain->flow_block);
 2674
 2675	return 0;
 2676}
 2677
 2678int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
 2679{
 2680	int err;
 2681
 2682	err = rhltable_insert_key(&table->chains_ht, chain->name,
 2683				  &chain->rhlhead, nft_chain_ht_params);
 2684	if (err)
 2685		return err;
 2686
 2687	list_add_tail_rcu(&chain->list, &table->chains);
 2688
 2689	return 0;
 2690}
 2691
 2692static u64 chain_id;
 2693
 2694static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 policy,
 2695			      u32 flags, struct netlink_ext_ack *extack)
 2696{
 2697	const struct nlattr * const *nla = ctx->nla;
 2698	struct nft_table *table = ctx->table;
 2699	struct nft_base_chain *basechain;
 2700	struct net *net = ctx->net;
 2701	char name[NFT_NAME_MAXLEN];
 2702	struct nft_rule_blob *blob;
 2703	struct nft_trans *trans;
 2704	struct nft_chain *chain;
 2705	int err;
 2706
 2707	if (nla[NFTA_CHAIN_HOOK]) {
 2708		struct nft_stats __percpu *stats = NULL;
 2709		struct nft_chain_hook hook = {};
 2710
 2711		if (table->flags & __NFT_TABLE_F_UPDATE)
 2712			return -EINVAL;
 2713
 2714		if (flags & NFT_CHAIN_BINDING)
 2715			return -EOPNOTSUPP;
 2716
 2717		err = nft_chain_parse_hook(net, NULL, nla, &hook, family, flags,
 2718					   extack);
 2719		if (err < 0)
 2720			return err;
 2721
 2722		basechain = kzalloc(sizeof(*basechain), GFP_KERNEL_ACCOUNT);
 2723		if (basechain == NULL) {
 2724			nft_chain_release_hook(&hook);
 2725			return -ENOMEM;
 2726		}
 2727		chain = &basechain->chain;
 2728
 2729		if (nla[NFTA_CHAIN_COUNTERS]) {
 2730			stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
 2731			if (IS_ERR_PCPU(stats)) {
 2732				nft_chain_release_hook(&hook);
 2733				kfree(basechain);
 2734				return PTR_ERR_PCPU(stats);
 2735			}
 2736			rcu_assign_pointer(basechain->stats, stats);
 2737		}
 2738
 2739		err = nft_basechain_init(basechain, family, &hook, flags);
 2740		if (err < 0) {
 2741			nft_chain_release_hook(&hook);
 2742			kfree(basechain);
 2743			free_percpu(stats);
 2744			return err;
 2745		}
 2746		if (stats)
 2747			static_branch_inc(&nft_counters_enabled);
 2748	} else {
 2749		if (flags & NFT_CHAIN_BASE)
 2750			return -EINVAL;
 2751		if (flags & NFT_CHAIN_HW_OFFLOAD)
 2752			return -EOPNOTSUPP;
 2753
 2754		chain = kzalloc(sizeof(*chain), GFP_KERNEL_ACCOUNT);
 2755		if (chain == NULL)
 2756			return -ENOMEM;
 2757
 2758		chain->flags = flags;
 2759	}
 2760	ctx->chain = chain;
 2761
 2762	INIT_LIST_HEAD(&chain->rules);
 2763	chain->handle = nf_tables_alloc_handle(table);
 2764	chain->table = table;
 2765
 2766	if (nla[NFTA_CHAIN_NAME]) {
 2767		chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
 2768	} else {
 2769		if (!(flags & NFT_CHAIN_BINDING)) {
 2770			err = -EINVAL;
 2771			goto err_destroy_chain;
 2772		}
 2773
 2774		snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
 2775		chain->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
 2776	}
 2777
 2778	if (!chain->name) {
 2779		err = -ENOMEM;
 2780		goto err_destroy_chain;
 2781	}
 2782
 2783	if (nla[NFTA_CHAIN_USERDATA]) {
 2784		chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL_ACCOUNT);
 2785		if (chain->udata == NULL) {
 2786			err = -ENOMEM;
 2787			goto err_destroy_chain;
 2788		}
 2789		chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
 2790	}
 2791
 2792	blob = nf_tables_chain_alloc_rules(chain, 0);
 2793	if (!blob) {
 2794		err = -ENOMEM;
 2795		goto err_destroy_chain;
 2796	}
 2797
 2798	RCU_INIT_POINTER(chain->blob_gen_0, blob);
 2799	RCU_INIT_POINTER(chain->blob_gen_1, blob);
 2800
 2801	if (!nft_use_inc(&table->use)) {
 2802		err = -EMFILE;
 2803		goto err_destroy_chain;
 2804	}
 2805
 2806	trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
 2807	if (IS_ERR(trans)) {
 2808		err = PTR_ERR(trans);
 2809		goto err_trans;
 2810	}
 2811
 2812	nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
 2813	if (nft_is_base_chain(chain))
 2814		nft_trans_chain_policy(trans) = policy;
 2815
 2816	err = nft_chain_add(table, chain);
 2817	if (err < 0)
 2818		goto err_chain_add;
 2819
 2820	/* This must be LAST to ensure no packets are walking over this chain. */
 2821	err = nf_tables_register_hook(net, table, chain);
 2822	if (err < 0)
 2823		goto err_register_hook;
 2824
 2825	return 0;
 2826
 2827err_register_hook:
 2828	nft_chain_del(chain);
 2829	synchronize_rcu();
 2830err_chain_add:
 2831	nft_trans_destroy(trans);
 2832err_trans:
 2833	nft_use_dec_restore(&table->use);
 2834err_destroy_chain:
 2835	nf_tables_chain_destroy(chain);
 2836
 2837	return err;
 2838}
 2839
 2840static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
 2841			      u32 flags, const struct nlattr *attr,
 2842			      struct netlink_ext_ack *extack)
 2843{
 2844	const struct nlattr * const *nla = ctx->nla;
 2845	struct nft_base_chain *basechain = NULL;
 2846	struct nft_table *table = ctx->table;
 2847	struct nft_chain *chain = ctx->chain;
 2848	struct nft_chain_hook hook = {};
 2849	struct nft_stats __percpu *stats = NULL;
 2850	struct nftables_pernet *nft_net;
 2851	struct nft_hook *h, *next;
 2852	struct nf_hook_ops *ops;
 2853	struct nft_trans *trans;
 2854	bool unregister = false;
 2855	int err;
 2856
 2857	if (chain->flags ^ flags)
 2858		return -EOPNOTSUPP;
 2859
 2860	INIT_LIST_HEAD(&hook.list);
 2861
 2862	if (nla[NFTA_CHAIN_HOOK]) {
 2863		if (!nft_is_base_chain(chain)) {
 2864			NL_SET_BAD_ATTR(extack, attr);
 2865			return -EEXIST;
 2866		}
 2867
 2868		basechain = nft_base_chain(chain);
 2869		err = nft_chain_parse_hook(ctx->net, basechain, nla, &hook,
 2870					   ctx->family, flags, extack);
 2871		if (err < 0)
 2872			return err;
 2873
 2874		if (basechain->type != hook.type) {
 2875			nft_chain_release_hook(&hook);
 2876			NL_SET_BAD_ATTR(extack, attr);
 2877			return -EEXIST;
 2878		}
 2879
 2880		if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
 2881			list_for_each_entry_safe(h, next, &hook.list, list) {
 2882				list_for_each_entry(ops, &h->ops_list, list) {
 2883					ops->pf		= basechain->ops.pf;
 2884					ops->hooknum	= basechain->ops.hooknum;
 2885					ops->priority	= basechain->ops.priority;
 2886					ops->priv	= basechain->ops.priv;
 2887					ops->hook	= basechain->ops.hook;
 2888				}
 2889
 2890				if (nft_hook_list_find(&basechain->hook_list, h)) {
 2891					list_del(&h->list);
 2892					nft_netdev_hook_free(h);
 2893					continue;
 2894				}
 2895
 2896				nft_net = nft_pernet(ctx->net);
 2897				list_for_each_entry(trans, &nft_net->commit_list, list) {
 2898					if (trans->msg_type != NFT_MSG_NEWCHAIN ||
 2899					    trans->table != ctx->table ||
 2900					    !nft_trans_chain_update(trans))
 2901						continue;
 2902
 2903					if (nft_hook_list_find(&nft_trans_chain_hooks(trans), h)) {
 2904						nft_chain_release_hook(&hook);
 2905						return -EEXIST;
 2906					}
 2907				}
 2908			}
 2909		} else {
 2910			ops = &basechain->ops;
 2911			if (ops->hooknum != hook.num ||
 2912			    ops->priority != hook.priority) {
 2913				nft_chain_release_hook(&hook);
 2914				NL_SET_BAD_ATTR(extack, attr);
 2915				return -EEXIST;
 2916			}
 2917		}
 2918	}
 2919
 2920	if (nla[NFTA_CHAIN_HANDLE] &&
 2921	    nla[NFTA_CHAIN_NAME]) {
 2922		struct nft_chain *chain2;
 2923
 2924		chain2 = nft_chain_lookup(ctx->net, table,
 2925					  nla[NFTA_CHAIN_NAME], genmask);
 2926		if (!IS_ERR(chain2)) {
 2927			NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
 2928			err = -EEXIST;
 2929			goto err_hooks;
 2930		}
 2931	}
 2932
 2933	if (table->flags & __NFT_TABLE_F_UPDATE &&
 2934	    !list_empty(&hook.list)) {
 2935		NL_SET_BAD_ATTR(extack, attr);
 2936		err = -EOPNOTSUPP;
 2937		goto err_hooks;
 2938	}
 2939
 2940	if (!(table->flags & NFT_TABLE_F_DORMANT) &&
 2941	    nft_is_base_chain(chain) &&
 2942	    !list_empty(&hook.list)) {
 2943		basechain = nft_base_chain(chain);
 2944		ops = &basechain->ops;
 2945
 2946		if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) {
 2947			err = nft_netdev_register_hooks(ctx->net, &hook.list);
 2948			if (err < 0)
 2949				goto err_hooks;
 2950
 2951			unregister = true;
 2952		}
 2953	}
 2954
 2955	if (nla[NFTA_CHAIN_COUNTERS]) {
 2956		if (!nft_is_base_chain(chain)) {
 2957			err = -EOPNOTSUPP;
 2958			goto err_hooks;
 2959		}
 2960
 2961		stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
 2962		if (IS_ERR_PCPU(stats)) {
 2963			err = PTR_ERR_PCPU(stats);
 2964			goto err_hooks;
 2965		}
 2966	}
 2967
 2968	err = -ENOMEM;
 2969	trans = nft_trans_alloc_chain(ctx, NFT_MSG_NEWCHAIN);
 2970	if (trans == NULL)
 2971		goto err_trans;
 2972
 2973	nft_trans_chain_stats(trans) = stats;
 2974	nft_trans_chain_update(trans) = true;
 2975
 2976	if (nla[NFTA_CHAIN_POLICY])
 2977		nft_trans_chain_policy(trans) = policy;
 2978	else
 2979		nft_trans_chain_policy(trans) = -1;
 2980
 2981	if (nla[NFTA_CHAIN_HANDLE] &&
 2982	    nla[NFTA_CHAIN_NAME]) {
 2983		struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 2984		struct nft_trans *tmp;
 2985		char *name;
 2986
 2987		err = -ENOMEM;
 2988		name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL_ACCOUNT);
 2989		if (!name)
 2990			goto err_trans;
 2991
 2992		err = -EEXIST;
 2993		list_for_each_entry(tmp, &nft_net->commit_list, list) {
 2994			if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
 2995			    tmp->table == table &&
 2996			    nft_trans_chain_update(tmp) &&
 2997			    nft_trans_chain_name(tmp) &&
 2998			    strcmp(name, nft_trans_chain_name(tmp)) == 0) {
 2999				NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
 3000				kfree(name);
 3001				goto err_trans;
 3002			}
 3003		}
 3004
 3005		nft_trans_chain_name(trans) = name;
 3006	}
 3007
 3008	nft_trans_basechain(trans) = basechain;
 3009	INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
 3010	list_splice(&hook.list, &nft_trans_chain_hooks(trans));
 3011	if (nla[NFTA_CHAIN_HOOK])
 3012		module_put(hook.type->owner);
 3013
 3014	nft_trans_commit_list_add_tail(ctx->net, trans);
 3015
 3016	return 0;
 3017
 3018err_trans:
 3019	free_percpu(stats);
 3020	kfree(trans);
 3021err_hooks:
 3022	if (nla[NFTA_CHAIN_HOOK]) {
 3023		list_for_each_entry_safe(h, next, &hook.list, list) {
 3024			if (unregister) {
 3025				list_for_each_entry(ops, &h->ops_list, list)
 3026					nf_unregister_net_hook(ctx->net, ops);
 3027			}
 3028			list_del(&h->list);
 3029			nft_netdev_hook_free_rcu(h);
 3030		}
 3031		module_put(hook.type->owner);
 3032	}
 3033
 3034	return err;
 3035}
 3036
 3037static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
 3038					       const struct nft_table *table,
 3039					       const struct nlattr *nla, u8 genmask)
 3040{
 3041	struct nftables_pernet *nft_net = nft_pernet(net);
 3042	u32 id = ntohl(nla_get_be32(nla));
 3043	struct nft_trans *trans;
 3044
 3045	list_for_each_entry(trans, &nft_net->commit_list, list) {
 3046		if (trans->msg_type == NFT_MSG_NEWCHAIN &&
 3047		    nft_trans_chain(trans)->table == table &&
 3048		    id == nft_trans_chain_id(trans) &&
 3049		    nft_active_genmask(nft_trans_chain(trans), genmask))
 3050			return nft_trans_chain(trans);
 3051	}
 3052	return ERR_PTR(-ENOENT);
 3053}
 3054
 3055static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
 3056			      const struct nlattr * const nla[])
 3057{
 3058	struct nftables_pernet *nft_net = nft_pernet(info->net);
 3059	struct netlink_ext_ack *extack = info->extack;
 3060	u8 genmask = nft_genmask_next(info->net);
 3061	u8 family = info->nfmsg->nfgen_family;
 3062	struct nft_chain *chain = NULL;
 3063	struct net *net = info->net;
 3064	const struct nlattr *attr;
 3065	struct nft_table *table;
 3066	u8 policy = NF_ACCEPT;
 3067	struct nft_ctx ctx;
 3068	u64 handle = 0;
 3069	u32 flags = 0;
 3070
 3071	lockdep_assert_held(&nft_net->commit_mutex);
 3072
 3073	table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
 3074				 NETLINK_CB(skb).portid);
 3075	if (IS_ERR(table)) {
 3076		NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
 3077		return PTR_ERR(table);
 3078	}
 3079
 3080	chain = NULL;
 3081	attr = nla[NFTA_CHAIN_NAME];
 3082
 3083	if (nla[NFTA_CHAIN_HANDLE]) {
 3084		handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
 3085		chain = nft_chain_lookup_byhandle(table, handle, genmask);
 3086		if (IS_ERR(chain)) {
 3087			NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
 3088			return PTR_ERR(chain);
 3089		}
 3090		attr = nla[NFTA_CHAIN_HANDLE];
 3091	} else if (nla[NFTA_CHAIN_NAME]) {
 3092		chain = nft_chain_lookup(net, table, attr, genmask);
 3093		if (IS_ERR(chain)) {
 3094			if (PTR_ERR(chain) != -ENOENT) {
 3095				NL_SET_BAD_ATTR(extack, attr);
 3096				return PTR_ERR(chain);
 3097			}
 3098			chain = NULL;
 3099		}
 3100	} else if (!nla[NFTA_CHAIN_ID]) {
 3101		return -EINVAL;
 3102	}
 3103
 3104	if (nla[NFTA_CHAIN_POLICY]) {
 3105		if (chain != NULL &&
 3106		    !nft_is_base_chain(chain)) {
 3107			NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
 3108			return -EOPNOTSUPP;
 3109		}
 3110
 3111		if (chain == NULL &&
 3112		    nla[NFTA_CHAIN_HOOK] == NULL) {
 3113			NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
 3114			return -EOPNOTSUPP;
 3115		}
 3116
 3117		policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
 3118		switch (policy) {
 3119		case NF_DROP:
 3120		case NF_ACCEPT:
 3121			break;
 3122		default:
 3123			return -EINVAL;
 3124		}
 3125	}
 3126
 3127	if (nla[NFTA_CHAIN_FLAGS])
 3128		flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
 3129	else if (chain)
 3130		flags = chain->flags;
 3131
 3132	if (flags & ~NFT_CHAIN_FLAGS)
 3133		return -EOPNOTSUPP;
 3134
 3135	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
 3136
 3137	if (chain != NULL) {
 3138		if (chain->flags & NFT_CHAIN_BINDING)
 3139			return -EINVAL;
 3140
 3141		if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
 3142			NL_SET_BAD_ATTR(extack, attr);
 3143			return -EEXIST;
 3144		}
 3145		if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
 3146			return -EOPNOTSUPP;
 3147
 3148		flags |= chain->flags & NFT_CHAIN_BASE;
 3149		return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
 3150					  extack);
 3151	}
 3152
 3153	return nf_tables_addchain(&ctx, family, policy, flags, extack);
 3154}
 3155
 3156static int nft_delchain_hook(struct nft_ctx *ctx,
 3157			     struct nft_base_chain *basechain,
 3158			     struct netlink_ext_ack *extack)
 3159{
 3160	const struct nft_chain *chain = &basechain->chain;
 3161	const struct nlattr * const *nla = ctx->nla;
 3162	struct nft_chain_hook chain_hook = {};
 3163	struct nft_hook *this, *hook;
 3164	LIST_HEAD(chain_del_list);
 3165	struct nft_trans *trans;
 3166	int err;
 3167
 3168	if (ctx->table->flags & __NFT_TABLE_F_UPDATE)
 3169		return -EOPNOTSUPP;
 3170
 3171	err = nft_chain_parse_hook(ctx->net, basechain, nla, &chain_hook,
 3172				   ctx->family, chain->flags, extack);
 3173	if (err < 0)
 3174		return err;
 3175
 3176	list_for_each_entry(this, &chain_hook.list, list) {
 3177		hook = nft_hook_list_find(&basechain->hook_list, this);
 3178		if (!hook) {
 3179			err = -ENOENT;
 3180			goto err_chain_del_hook;
 3181		}
 3182		list_move(&hook->list, &chain_del_list);
 3183	}
 3184
 3185	trans = nft_trans_alloc_chain(ctx, NFT_MSG_DELCHAIN);
 3186	if (!trans) {
 3187		err = -ENOMEM;
 3188		goto err_chain_del_hook;
 3189	}
 3190
 3191	nft_trans_basechain(trans) = basechain;
 3192	nft_trans_chain_update(trans) = true;
 3193	INIT_LIST_HEAD(&nft_trans_chain_hooks(trans));
 3194	list_splice(&chain_del_list, &nft_trans_chain_hooks(trans));
 3195	nft_chain_release_hook(&chain_hook);
 3196
 3197	nft_trans_commit_list_add_tail(ctx->net, trans);
 3198
 3199	return 0;
 3200
 3201err_chain_del_hook:
 3202	list_splice(&chain_del_list, &basechain->hook_list);
 3203	nft_chain_release_hook(&chain_hook);
 3204
 3205	return err;
 3206}
 3207
 3208static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
 3209			      const struct nlattr * const nla[])
 3210{
 3211	struct netlink_ext_ack *extack = info->extack;
 3212	u8 genmask = nft_genmask_next(info->net);
 3213	u8 family = info->nfmsg->nfgen_family;
 3214	struct net *net = info->net;
 3215	const struct nlattr *attr;
 3216	struct nft_table *table;
 3217	struct nft_chain *chain;
 3218	struct nft_rule *rule;
 3219	struct nft_ctx ctx;
 3220	u64 handle;
 3221	u32 use;
 3222	int err;
 3223
 3224	table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
 3225				 NETLINK_CB(skb).portid);
 3226	if (IS_ERR(table)) {
 3227		NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
 3228		return PTR_ERR(table);
 3229	}
 3230
 3231	if (nla[NFTA_CHAIN_HANDLE]) {
 3232		attr = nla[NFTA_CHAIN_HANDLE];
 3233		handle = be64_to_cpu(nla_get_be64(attr));
 3234		chain = nft_chain_lookup_byhandle(table, handle, genmask);
 3235	} else {
 3236		attr = nla[NFTA_CHAIN_NAME];
 3237		chain = nft_chain_lookup(net, table, attr, genmask);
 3238	}
 3239	if (IS_ERR(chain)) {
 3240		if (PTR_ERR(chain) == -ENOENT &&
 3241		    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN)
 3242			return 0;
 3243
 3244		NL_SET_BAD_ATTR(extack, attr);
 3245		return PTR_ERR(chain);
 3246	}
 3247
 3248	if (nft_chain_binding(chain))
 3249		return -EOPNOTSUPP;
 3250
 3251	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
 3252
 3253	if (nla[NFTA_CHAIN_HOOK]) {
 3254		if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYCHAIN ||
 3255		    chain->flags & NFT_CHAIN_HW_OFFLOAD)
 3256			return -EOPNOTSUPP;
 3257
 3258		if (nft_is_base_chain(chain)) {
 3259			struct nft_base_chain *basechain = nft_base_chain(chain);
 3260
 3261			if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 3262				return nft_delchain_hook(&ctx, basechain, extack);
 3263		}
 3264	}
 3265
 3266	if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
 3267	    chain->use > 0)
 3268		return -EBUSY;
 3269
 3270	use = chain->use;
 3271	list_for_each_entry(rule, &chain->rules, list) {
 3272		if (!nft_is_active_next(net, rule))
 3273			continue;
 3274		use--;
 3275
 3276		err = nft_delrule(&ctx, rule);
 3277		if (err < 0)
 3278			return err;
 3279	}
 3280
 3281	/* There are rules and elements that are still holding references to us,
 3282	 * we cannot do a recursive removal in this case.
 3283	 */
 3284	if (use > 0) {
 3285		NL_SET_BAD_ATTR(extack, attr);
 3286		return -EBUSY;
 3287	}
 3288
 3289	return nft_delchain(&ctx);
 3290}
 3291
 3292/*
 3293 * Expressions
 3294 */
 3295
 3296/**
 3297 *	nft_register_expr - register nf_tables expr type
 3298 *	@type: expr type
 3299 *
 3300 *	Registers the expr type for use with nf_tables. Returns zero on
 3301 *	success or a negative errno code otherwise.
 3302 */
 3303int nft_register_expr(struct nft_expr_type *type)
 3304{
 3305	if (WARN_ON_ONCE(type->maxattr > NFT_EXPR_MAXATTR))
 3306		return -ENOMEM;
 3307
 3308	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 3309	if (type->family == NFPROTO_UNSPEC)
 3310		list_add_tail_rcu(&type->list, &nf_tables_expressions);
 3311	else
 3312		list_add_rcu(&type->list, &nf_tables_expressions);
 3313	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 3314	return 0;
 3315}
 3316EXPORT_SYMBOL_GPL(nft_register_expr);
 3317
 3318/**
 3319 *	nft_unregister_expr - unregister nf_tables expr type
 3320 *	@type: expr type
 3321 *
 3322 * 	Unregisters the expr typefor use with nf_tables.
 3323 */
 3324void nft_unregister_expr(struct nft_expr_type *type)
 3325{
 3326	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 3327	list_del_rcu(&type->list);
 3328	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 3329}
 3330EXPORT_SYMBOL_GPL(nft_unregister_expr);
 3331
 3332static const struct nft_expr_type *__nft_expr_type_get(u8 family,
 3333						       struct nlattr *nla)
 3334{
 3335	const struct nft_expr_type *type, *candidate = NULL;
 3336
 3337	list_for_each_entry_rcu(type, &nf_tables_expressions, list) {
 3338		if (!nla_strcmp(nla, type->name)) {
 3339			if (!type->family && !candidate)
 3340				candidate = type;
 3341			else if (type->family == family)
 3342				candidate = type;
 3343		}
 3344	}
 3345	return candidate;
 3346}
 3347
 3348#ifdef CONFIG_MODULES
 3349static int nft_expr_type_request_module(struct net *net, u8 family,
 3350					struct nlattr *nla)
 3351{
 3352	if (nft_request_module(net, "nft-expr-%u-%.*s", family,
 3353			       nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
 3354		return -EAGAIN;
 3355
 3356	return 0;
 3357}
 3358#endif
 3359
 3360static const struct nft_expr_type *nft_expr_type_get(struct net *net,
 3361						     u8 family,
 3362						     struct nlattr *nla)
 3363{
 3364	const struct nft_expr_type *type;
 3365
 3366	if (nla == NULL)
 3367		return ERR_PTR(-EINVAL);
 3368
 3369	rcu_read_lock();
 3370	type = __nft_expr_type_get(family, nla);
 3371	if (type != NULL && try_module_get(type->owner)) {
 3372		rcu_read_unlock();
 3373		return type;
 3374	}
 3375	rcu_read_unlock();
 3376
 3377	lockdep_nfnl_nft_mutex_not_held();
 3378#ifdef CONFIG_MODULES
 3379	if (type == NULL) {
 3380		if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
 3381			return ERR_PTR(-EAGAIN);
 3382
 3383		if (nft_request_module(net, "nft-expr-%.*s",
 3384				       nla_len(nla),
 3385				       (char *)nla_data(nla)) == -EAGAIN)
 3386			return ERR_PTR(-EAGAIN);
 3387	}
 3388#endif
 3389	return ERR_PTR(-ENOENT);
 3390}
 3391
 3392static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
 3393	[NFTA_EXPR_NAME]	= { .type = NLA_STRING,
 3394				    .len = NFT_MODULE_AUTOLOAD_LIMIT },
 3395	[NFTA_EXPR_DATA]	= { .type = NLA_NESTED },
 3396};
 3397
 3398static int nf_tables_fill_expr_info(struct sk_buff *skb,
 3399				    const struct nft_expr *expr, bool reset)
 3400{
 3401	if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
 3402		goto nla_put_failure;
 3403
 3404	if (expr->ops->dump) {
 3405		struct nlattr *data = nla_nest_start_noflag(skb,
 3406							    NFTA_EXPR_DATA);
 3407		if (data == NULL)
 3408			goto nla_put_failure;
 3409		if (expr->ops->dump(skb, expr, reset) < 0)
 3410			goto nla_put_failure;
 3411		nla_nest_end(skb, data);
 3412	}
 3413
 3414	return skb->len;
 3415
 3416nla_put_failure:
 3417	return -1;
 3418};
 3419
 3420int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
 3421		  const struct nft_expr *expr, bool reset)
 3422{
 3423	struct nlattr *nest;
 3424
 3425	nest = nla_nest_start_noflag(skb, attr);
 3426	if (!nest)
 3427		goto nla_put_failure;
 3428	if (nf_tables_fill_expr_info(skb, expr, reset) < 0)
 3429		goto nla_put_failure;
 3430	nla_nest_end(skb, nest);
 3431	return 0;
 3432
 3433nla_put_failure:
 3434	return -1;
 3435}
 3436
 3437struct nft_expr_info {
 3438	const struct nft_expr_ops	*ops;
 3439	const struct nlattr		*attr;
 3440	struct nlattr			*tb[NFT_EXPR_MAXATTR + 1];
 3441};
 3442
 3443static int nf_tables_expr_parse(const struct nft_ctx *ctx,
 3444				const struct nlattr *nla,
 3445				struct nft_expr_info *info)
 3446{
 3447	const struct nft_expr_type *type;
 3448	const struct nft_expr_ops *ops;
 3449	struct nlattr *tb[NFTA_EXPR_MAX + 1];
 3450	int err;
 3451
 3452	err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
 3453					  nft_expr_policy, NULL);
 3454	if (err < 0)
 3455		return err;
 3456
 3457	type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
 3458	if (IS_ERR(type))
 3459		return PTR_ERR(type);
 3460
 3461	if (tb[NFTA_EXPR_DATA]) {
 3462		err = nla_parse_nested_deprecated(info->tb, type->maxattr,
 3463						  tb[NFTA_EXPR_DATA],
 3464						  type->policy, NULL);
 3465		if (err < 0)
 3466			goto err1;
 3467	} else
 3468		memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
 3469
 3470	if (type->select_ops != NULL) {
 3471		ops = type->select_ops(ctx,
 3472				       (const struct nlattr * const *)info->tb);
 3473		if (IS_ERR(ops)) {
 3474			err = PTR_ERR(ops);
 3475#ifdef CONFIG_MODULES
 3476			if (err == -EAGAIN)
 3477				if (nft_expr_type_request_module(ctx->net,
 3478								 ctx->family,
 3479								 tb[NFTA_EXPR_NAME]) != -EAGAIN)
 3480					err = -ENOENT;
 3481#endif
 3482			goto err1;
 3483		}
 3484	} else
 3485		ops = type->ops;
 3486
 3487	info->attr = nla;
 3488	info->ops = ops;
 3489
 3490	return 0;
 3491
 3492err1:
 3493	module_put(type->owner);
 3494	return err;
 3495}
 3496
 3497int nft_expr_inner_parse(const struct nft_ctx *ctx, const struct nlattr *nla,
 3498			 struct nft_expr_info *info)
 3499{
 3500	struct nlattr *tb[NFTA_EXPR_MAX + 1];
 3501	const struct nft_expr_type *type;
 3502	int err;
 3503
 3504	err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
 3505					  nft_expr_policy, NULL);
 3506	if (err < 0)
 3507		return err;
 3508
 3509	if (!tb[NFTA_EXPR_DATA] || !tb[NFTA_EXPR_NAME])
 3510		return -EINVAL;
 3511
 3512	rcu_read_lock();
 3513
 3514	type = __nft_expr_type_get(ctx->family, tb[NFTA_EXPR_NAME]);
 3515	if (!type) {
 3516		err = -ENOENT;
 3517		goto out_unlock;
 3518	}
 3519
 3520	if (!type->inner_ops) {
 3521		err = -EOPNOTSUPP;
 3522		goto out_unlock;
 3523	}
 3524
 3525	err = nla_parse_nested_deprecated(info->tb, type->maxattr,
 3526					  tb[NFTA_EXPR_DATA],
 3527					  type->policy, NULL);
 3528	if (err < 0)
 3529		goto out_unlock;
 3530
 3531	info->attr = nla;
 3532	info->ops = type->inner_ops;
 3533
 3534	/* No module reference will be taken on type->owner.
 3535	 * Presence of type->inner_ops implies that the expression
 3536	 * is builtin, so it cannot go away.
 3537	 */
 3538	rcu_read_unlock();
 3539	return 0;
 3540
 3541out_unlock:
 3542	rcu_read_unlock();
 3543	return err;
 3544}
 3545
 3546static int nf_tables_newexpr(const struct nft_ctx *ctx,
 3547			     const struct nft_expr_info *expr_info,
 3548			     struct nft_expr *expr)
 3549{
 3550	const struct nft_expr_ops *ops = expr_info->ops;
 3551	int err;
 3552
 3553	expr->ops = ops;
 3554	if (ops->init) {
 3555		err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
 3556		if (err < 0)
 3557			goto err1;
 3558	}
 3559
 3560	return 0;
 3561err1:
 3562	expr->ops = NULL;
 3563	return err;
 3564}
 3565
 3566static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
 3567				   struct nft_expr *expr)
 3568{
 3569	const struct nft_expr_type *type = expr->ops->type;
 3570
 3571	if (expr->ops->destroy)
 3572		expr->ops->destroy(ctx, expr);
 3573	module_put(type->owner);
 3574}
 3575
 3576static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
 3577				      const struct nlattr *nla)
 3578{
 3579	struct nft_expr_info expr_info;
 3580	struct nft_expr *expr;
 3581	struct module *owner;
 3582	int err;
 3583
 3584	err = nf_tables_expr_parse(ctx, nla, &expr_info);
 3585	if (err < 0)
 3586		goto err_expr_parse;
 3587
 3588	err = -EOPNOTSUPP;
 3589	if (!(expr_info.ops->type->flags & NFT_EXPR_STATEFUL))
 3590		goto err_expr_stateful;
 3591
 3592	err = -ENOMEM;
 3593	expr = kzalloc(expr_info.ops->size, GFP_KERNEL_ACCOUNT);
 3594	if (expr == NULL)
 3595		goto err_expr_stateful;
 3596
 3597	err = nf_tables_newexpr(ctx, &expr_info, expr);
 3598	if (err < 0)
 3599		goto err_expr_new;
 3600
 3601	return expr;
 3602err_expr_new:
 3603	kfree(expr);
 3604err_expr_stateful:
 3605	owner = expr_info.ops->type->owner;
 3606	if (expr_info.ops->type->release_ops)
 3607		expr_info.ops->type->release_ops(expr_info.ops);
 3608
 3609	module_put(owner);
 3610err_expr_parse:
 3611	return ERR_PTR(err);
 3612}
 3613
 3614int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp)
 3615{
 3616	int err;
 3617
 3618	if (WARN_ON_ONCE(!src->ops->clone))
 3619		return -EINVAL;
 3620
 3621	dst->ops = src->ops;
 3622	err = src->ops->clone(dst, src, gfp);
 3623	if (err < 0)
 3624		return err;
 3625
 3626	__module_get(src->ops->type->owner);
 3627
 3628	return 0;
 3629}
 3630
 3631void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
 3632{
 3633	nf_tables_expr_destroy(ctx, expr);
 3634	kfree(expr);
 3635}
 3636
 3637/*
 3638 * Rules
 3639 */
 3640
 3641static struct nft_rule *__nft_rule_lookup(const struct net *net,
 3642					  const struct nft_chain *chain,
 3643					  u64 handle)
 3644{
 3645	struct nft_rule *rule;
 3646
 3647	// FIXME: this sucks
 3648	list_for_each_entry_rcu(rule, &chain->rules, list,
 3649				lockdep_commit_lock_is_held(net)) {
 3650		if (handle == rule->handle)
 3651			return rule;
 3652	}
 3653
 3654	return ERR_PTR(-ENOENT);
 3655}
 3656
 3657static struct nft_rule *nft_rule_lookup(const struct net *net,
 3658					const struct nft_chain *chain,
 3659					const struct nlattr *nla)
 3660{
 3661	if (nla == NULL)
 3662		return ERR_PTR(-EINVAL);
 3663
 3664	return __nft_rule_lookup(net, chain, be64_to_cpu(nla_get_be64(nla)));
 3665}
 3666
 3667static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
 3668	[NFTA_RULE_TABLE]	= { .type = NLA_STRING,
 3669				    .len = NFT_TABLE_MAXNAMELEN - 1 },
 3670	[NFTA_RULE_CHAIN]	= { .type = NLA_STRING,
 3671				    .len = NFT_CHAIN_MAXNAMELEN - 1 },
 3672	[NFTA_RULE_HANDLE]	= { .type = NLA_U64 },
 3673	[NFTA_RULE_EXPRESSIONS]	= NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
 3674	[NFTA_RULE_COMPAT]	= { .type = NLA_NESTED },
 3675	[NFTA_RULE_POSITION]	= { .type = NLA_U64 },
 3676	[NFTA_RULE_USERDATA]	= { .type = NLA_BINARY,
 3677				    .len = NFT_USERDATA_MAXLEN },
 3678	[NFTA_RULE_ID]		= { .type = NLA_U32 },
 3679	[NFTA_RULE_POSITION_ID]	= { .type = NLA_U32 },
 3680	[NFTA_RULE_CHAIN_ID]	= { .type = NLA_U32 },
 3681};
 3682
 3683static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
 3684				    u32 portid, u32 seq, int event,
 3685				    u32 flags, int family,
 3686				    const struct nft_table *table,
 3687				    const struct nft_chain *chain,
 3688				    const struct nft_rule *rule, u64 handle,
 3689				    bool reset)
 3690{
 3691	struct nlmsghdr *nlh;
 3692	const struct nft_expr *expr, *next;
 3693	struct nlattr *list;
 3694	u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 3695
 3696	nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
 3697			   nft_base_seq_be16(net));
 3698	if (!nlh)
 3699		goto nla_put_failure;
 3700
 3701	if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
 3702		goto nla_put_failure;
 3703	if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
 3704		goto nla_put_failure;
 3705	if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
 3706			 NFTA_RULE_PAD))
 3707		goto nla_put_failure;
 3708
 3709	if (event != NFT_MSG_DELRULE && handle) {
 3710		if (nla_put_be64(skb, NFTA_RULE_POSITION, cpu_to_be64(handle),
 3711				 NFTA_RULE_PAD))
 3712			goto nla_put_failure;
 3713	}
 3714
 3715	if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
 3716		nft_flow_rule_stats(chain, rule);
 3717
 3718	list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
 3719	if (list == NULL)
 3720		goto nla_put_failure;
 3721	nft_rule_for_each_expr(expr, next, rule) {
 3722		if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
 3723			goto nla_put_failure;
 3724	}
 3725	nla_nest_end(skb, list);
 3726
 3727	if (rule->udata) {
 3728		struct nft_userdata *udata = nft_userdata(rule);
 3729		if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
 3730			    udata->data) < 0)
 3731			goto nla_put_failure;
 3732	}
 3733
 3734	nlmsg_end(skb, nlh);
 3735	return 0;
 3736
 3737nla_put_failure:
 3738	nlmsg_trim(skb, nlh);
 3739	return -1;
 3740}
 3741
 3742static void nf_tables_rule_notify(const struct nft_ctx *ctx,
 3743				  const struct nft_rule *rule, int event)
 3744{
 3745	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 3746	const struct nft_rule *prule;
 3747	struct sk_buff *skb;
 3748	u64 handle = 0;
 3749	u16 flags = 0;
 3750	int err;
 3751
 3752	if (!ctx->report &&
 3753	    !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 3754		return;
 3755
 3756	skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 3757	if (skb == NULL)
 3758		goto err;
 3759
 3760	if (event == NFT_MSG_NEWRULE &&
 3761	    !list_is_first(&rule->list, &ctx->chain->rules) &&
 3762	    !list_is_last(&rule->list, &ctx->chain->rules)) {
 3763		prule = list_prev_entry(rule, list);
 3764		handle = prule->handle;
 3765	}
 3766	if (ctx->flags & (NLM_F_APPEND | NLM_F_REPLACE))
 3767		flags |= NLM_F_APPEND;
 3768	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 3769		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 3770
 3771	err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
 3772				       event, flags, ctx->family, ctx->table,
 3773				       ctx->chain, rule, handle, false);
 3774	if (err < 0) {
 3775		kfree_skb(skb);
 3776		goto err;
 3777	}
 3778
 3779	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 3780	return;
 3781err:
 3782	nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 3783}
 3784
 3785static void audit_log_rule_reset(const struct nft_table *table,
 3786				 unsigned int base_seq,
 3787				 unsigned int nentries)
 3788{
 3789	char *buf = kasprintf(GFP_ATOMIC, "%s:%u",
 3790			      table->name, base_seq);
 3791
 3792	audit_log_nfcfg(buf, table->family, nentries,
 3793			AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
 3794	kfree(buf);
 3795}
 3796
 3797struct nft_rule_dump_ctx {
 3798	unsigned int s_idx;
 3799	char *table;
 3800	char *chain;
 3801	bool reset;
 3802};
 3803
 3804static int __nf_tables_dump_rules(struct sk_buff *skb,
 3805				  unsigned int *idx,
 3806				  struct netlink_callback *cb,
 3807				  const struct nft_table *table,
 3808				  const struct nft_chain *chain)
 3809{
 3810	struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
 3811	struct net *net = sock_net(skb->sk);
 3812	const struct nft_rule *rule, *prule;
 3813	unsigned int entries = 0;
 3814	int ret = 0;
 3815	u64 handle;
 3816
 3817	prule = NULL;
 3818	list_for_each_entry_rcu(rule, &chain->rules, list) {
 3819		if (!nft_is_active(net, rule))
 3820			goto cont_skip;
 3821		if (*idx < ctx->s_idx)
 3822			goto cont;
 3823		if (prule)
 3824			handle = prule->handle;
 3825		else
 3826			handle = 0;
 3827
 3828		if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
 3829					cb->nlh->nlmsg_seq,
 3830					NFT_MSG_NEWRULE,
 3831					NLM_F_MULTI | NLM_F_APPEND,
 3832					table->family,
 3833					table, chain, rule, handle, ctx->reset) < 0) {
 3834			ret = 1;
 3835			break;
 3836		}
 3837		entries++;
 3838		nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 3839cont:
 3840		prule = rule;
 3841cont_skip:
 3842		(*idx)++;
 3843	}
 3844
 3845	if (ctx->reset && entries)
 3846		audit_log_rule_reset(table, cb->seq, entries);
 3847
 3848	return ret;
 3849}
 3850
 3851static int nf_tables_dump_rules(struct sk_buff *skb,
 3852				struct netlink_callback *cb)
 3853{
 3854	const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 3855	struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
 3856	struct nft_table *table;
 3857	const struct nft_chain *chain;
 3858	unsigned int idx = 0;
 3859	struct net *net = sock_net(skb->sk);
 3860	int family = nfmsg->nfgen_family;
 3861	struct nftables_pernet *nft_net;
 3862
 3863	rcu_read_lock();
 3864	nft_net = nft_pernet(net);
 3865	cb->seq = nft_base_seq(net);
 3866
 3867	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 3868		if (family != NFPROTO_UNSPEC && family != table->family)
 3869			continue;
 3870
 3871		if (ctx->table && strcmp(ctx->table, table->name) != 0)
 3872			continue;
 3873
 3874		if (ctx->table && ctx->chain) {
 3875			struct rhlist_head *list, *tmp;
 3876
 3877			list = rhltable_lookup(&table->chains_ht, ctx->chain,
 3878					       nft_chain_ht_params);
 3879			if (!list)
 3880				goto done;
 3881
 3882			rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
 3883				if (!nft_is_active(net, chain))
 3884					continue;
 3885				__nf_tables_dump_rules(skb, &idx,
 3886						       cb, table, chain);
 3887				break;
 3888			}
 3889			goto done;
 3890		}
 3891
 3892		list_for_each_entry_rcu(chain, &table->chains, list) {
 3893			if (__nf_tables_dump_rules(skb, &idx,
 3894						   cb, table, chain))
 3895				goto done;
 3896		}
 3897
 3898		if (ctx->table)
 3899			break;
 3900	}
 3901done:
 3902	rcu_read_unlock();
 3903
 3904	ctx->s_idx = idx;
 3905	return skb->len;
 3906}
 3907
 3908static int nf_tables_dump_rules_start(struct netlink_callback *cb)
 3909{
 3910	struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
 3911	const struct nlattr * const *nla = cb->data;
 3912
 3913	BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
 3914
 3915	if (nla[NFTA_RULE_TABLE]) {
 3916		ctx->table = nla_strdup(nla[NFTA_RULE_TABLE], GFP_ATOMIC);
 3917		if (!ctx->table)
 3918			return -ENOMEM;
 3919	}
 3920	if (nla[NFTA_RULE_CHAIN]) {
 3921		ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN], GFP_ATOMIC);
 3922		if (!ctx->chain) {
 3923			kfree(ctx->table);
 3924			return -ENOMEM;
 3925		}
 3926	}
 3927	if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
 3928		ctx->reset = true;
 3929
 3930	return 0;
 3931}
 3932
 3933static int nf_tables_dump_rules_done(struct netlink_callback *cb)
 3934{
 3935	struct nft_rule_dump_ctx *ctx = (void *)cb->ctx;
 3936
 3937	kfree(ctx->table);
 3938	kfree(ctx->chain);
 3939	return 0;
 3940}
 3941
 3942/* Caller must hold rcu read lock or transaction mutex */
 3943static struct sk_buff *
 3944nf_tables_getrule_single(u32 portid, const struct nfnl_info *info,
 3945			 const struct nlattr * const nla[], bool reset)
 3946{
 3947	struct netlink_ext_ack *extack = info->extack;
 3948	u8 genmask = nft_genmask_cur(info->net);
 3949	u8 family = info->nfmsg->nfgen_family;
 3950	const struct nft_chain *chain;
 3951	const struct nft_rule *rule;
 3952	struct net *net = info->net;
 3953	struct nft_table *table;
 3954	struct sk_buff *skb2;
 3955	int err;
 3956
 3957	table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
 3958	if (IS_ERR(table)) {
 3959		NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
 3960		return ERR_CAST(table);
 3961	}
 3962
 3963	chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
 3964	if (IS_ERR(chain)) {
 3965		NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
 3966		return ERR_CAST(chain);
 3967	}
 3968
 3969	rule = nft_rule_lookup(net, chain, nla[NFTA_RULE_HANDLE]);
 3970	if (IS_ERR(rule)) {
 3971		NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
 3972		return ERR_CAST(rule);
 3973	}
 3974
 3975	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 3976	if (!skb2)
 3977		return ERR_PTR(-ENOMEM);
 3978
 3979	err = nf_tables_fill_rule_info(skb2, net, portid,
 3980				       info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
 3981				       family, table, chain, rule, 0, reset);
 3982	if (err < 0) {
 3983		kfree_skb(skb2);
 3984		return ERR_PTR(err);
 3985	}
 3986
 3987	return skb2;
 3988}
 3989
 3990static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
 3991			     const struct nlattr * const nla[])
 3992{
 3993	u32 portid = NETLINK_CB(skb).portid;
 3994	struct net *net = info->net;
 3995	struct sk_buff *skb2;
 3996	bool reset = false;
 3997	char *buf;
 3998
 3999	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 4000		struct netlink_dump_control c = {
 4001			.start= nf_tables_dump_rules_start,
 4002			.dump = nf_tables_dump_rules,
 4003			.done = nf_tables_dump_rules_done,
 4004			.module = THIS_MODULE,
 4005			.data = (void *)nla,
 4006		};
 4007
 4008		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 4009	}
 4010
 4011	if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET)
 4012		reset = true;
 4013
 4014	skb2 = nf_tables_getrule_single(portid, info, nla, reset);
 4015	if (IS_ERR(skb2))
 4016		return PTR_ERR(skb2);
 4017
 4018	if (!reset)
 4019		return nfnetlink_unicast(skb2, net, portid);
 4020
 4021	buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
 4022			nla_len(nla[NFTA_RULE_TABLE]),
 4023			(char *)nla_data(nla[NFTA_RULE_TABLE]),
 4024			nft_base_seq(net));
 4025	audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
 4026			AUDIT_NFT_OP_RULE_RESET, GFP_ATOMIC);
 4027	kfree(buf);
 4028
 4029	return nfnetlink_unicast(skb2, net, portid);
 4030}
 4031
 4032void nf_tables_rule_destroy(const struct nft_ctx *ctx, struct nft_rule *rule)
 4033{
 4034	struct nft_expr *expr, *next;
 4035
 4036	/*
 4037	 * Careful: some expressions might not be initialized in case this
 4038	 * is called on error from nf_tables_newrule().
 4039	 */
 4040	expr = nft_expr_first(rule);
 4041	while (nft_expr_more(rule, expr)) {
 4042		next = nft_expr_next(expr);
 4043		nf_tables_expr_destroy(ctx, expr);
 4044		expr = next;
 4045	}
 4046	kfree(rule);
 4047}
 4048
 4049/* can only be used if rule is no longer visible to dumps */
 4050static void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
 4051{
 4052	WARN_ON_ONCE(!lockdep_commit_lock_is_held(ctx->net));
 4053
 4054	nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
 4055	nf_tables_rule_destroy(ctx, rule);
 4056}
 4057
 4058static void nft_chain_vstate_update(const struct nft_ctx *ctx, struct nft_chain *chain)
 4059{
 4060	const struct nft_base_chain *base_chain;
 4061	enum nft_chain_types type;
 4062	u8 hooknum;
 4063
 4064	/* ctx->chain must hold the calling base chain. */
 4065	if (WARN_ON_ONCE(!nft_is_base_chain(ctx->chain))) {
 4066		memset(&chain->vstate, 0, sizeof(chain->vstate));
 4067		return;
 4068	}
 4069
 4070	base_chain = nft_base_chain(ctx->chain);
 4071	hooknum = base_chain->ops.hooknum;
 4072	type = base_chain->type->type;
 4073
 4074	BUILD_BUG_ON(BIT(NF_INET_NUMHOOKS) > U8_MAX);
 4075
 4076	chain->vstate.hook_mask[type] |= BIT(hooknum);
 4077	if (chain->vstate.depth < ctx->level)
 4078		chain->vstate.depth = ctx->level;
 4079}
 4080
 4081/** nft_chain_validate - loop detection and hook validation
 4082 *
 4083 * @ctx: context containing call depth and base chain
 4084 * @chain: chain to validate
 4085 *
 4086 * Walk through the rules of the given chain and chase all jumps/gotos
 4087 * and set lookups until either the jump limit is hit or all reachable
 4088 * chains have been validated.
 4089 */
 4090int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain)
 4091{
 4092	struct nft_expr *expr, *last;
 4093	struct nft_rule *rule;
 4094	int err;
 4095
 4096	BUILD_BUG_ON(NFT_JUMP_STACK_SIZE > 255);
 4097	if (ctx->level == NFT_JUMP_STACK_SIZE)
 4098		return -EMLINK;
 4099
 4100	if (ctx->level > 0) {
 4101		/* jumps to base chains are not allowed. */
 4102		if (nft_is_base_chain(chain))
 4103			return -ELOOP;
 4104
 4105		if (nft_chain_vstate_valid(ctx, chain))
 4106			return 0;
 4107	}
 4108
 4109	list_for_each_entry(rule, &chain->rules, list) {
 4110		if (fatal_signal_pending(current))
 4111			return -EINTR;
 4112
 4113		if (!nft_is_active_next(ctx->net, rule))
 4114			continue;
 4115
 4116		nft_rule_for_each_expr(expr, last, rule) {
 4117			if (!expr->ops->validate)
 4118				continue;
 4119
 4120			/* This may call nft_chain_validate() recursively,
 4121			 * callers that do so must increment ctx->level.
 4122			 */
 4123			err = expr->ops->validate(ctx, expr);
 4124			if (err < 0)
 4125				return err;
 4126		}
 4127
 4128		cond_resched();
 4129	}
 4130
 4131	nft_chain_vstate_update(ctx, chain);
 4132	return 0;
 4133}
 4134EXPORT_SYMBOL_GPL(nft_chain_validate);
 4135
 4136static int nft_table_validate(struct net *net, const struct nft_table *table)
 4137{
 4138	struct nft_chain *chain;
 4139	struct nft_ctx ctx = {
 4140		.net	= net,
 4141		.family	= table->family,
 4142	};
 4143	int err = 0;
 4144
 4145	list_for_each_entry(chain, &table->chains, list) {
 4146		if (!nft_is_base_chain(chain))
 4147			continue;
 4148
 4149		ctx.chain = chain;
 4150		err = nft_chain_validate(&ctx, chain);
 4151		if (err < 0)
 4152			goto err;
 4153	}
 4154
 4155err:
 4156	list_for_each_entry(chain, &table->chains, list)
 4157		memset(&chain->vstate, 0, sizeof(chain->vstate));
 4158
 4159	return err;
 4160}
 4161
 4162int nft_setelem_validate(const struct nft_ctx *ctx, struct nft_set *set,
 4163			 const struct nft_set_iter *iter,
 4164			 struct nft_elem_priv *elem_priv)
 4165{
 4166	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 4167	struct nft_ctx *pctx = (struct nft_ctx *)ctx;
 4168	const struct nft_data *data;
 4169	int err;
 4170
 4171	if (!nft_set_elem_active(ext, iter->genmask))
 4172		return 0;
 4173
 4174	if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
 4175	    *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
 4176		return 0;
 4177
 4178	data = nft_set_ext_data(ext);
 4179	switch (data->verdict.code) {
 4180	case NFT_JUMP:
 4181	case NFT_GOTO:
 4182		pctx->level++;
 4183		err = nft_chain_validate(ctx, data->verdict.chain);
 4184		if (err < 0)
 4185			return err;
 4186		pctx->level--;
 4187		break;
 4188	default:
 4189		break;
 4190	}
 4191
 4192	return 0;
 4193}
 4194
 4195int nft_set_catchall_validate(const struct nft_ctx *ctx, struct nft_set *set)
 4196{
 4197	struct nft_set_iter dummy_iter = {
 4198		.genmask	= nft_genmask_next(ctx->net),
 4199	};
 4200	struct nft_set_elem_catchall *catchall;
 4201
 4202	struct nft_set_ext *ext;
 4203	int ret = 0;
 4204
 4205	list_for_each_entry_rcu(catchall, &set->catchall_list, list,
 4206				lockdep_commit_lock_is_held(ctx->net)) {
 4207		ext = nft_set_elem_ext(set, catchall->elem);
 4208		if (!nft_set_elem_active(ext, dummy_iter.genmask))
 4209			continue;
 4210
 4211		ret = nft_setelem_validate(ctx, set, &dummy_iter, catchall->elem);
 4212		if (ret < 0)
 4213			return ret;
 4214	}
 4215
 4216	return ret;
 4217}
 4218
 4219static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
 4220					     const struct nft_chain *chain,
 4221					     const struct nlattr *nla);
 4222
 4223#define NFT_RULE_MAXEXPRS	128
 4224
 4225static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
 4226			     const struct nlattr * const nla[])
 4227{
 4228	struct nftables_pernet *nft_net = nft_pernet(info->net);
 4229	struct netlink_ext_ack *extack = info->extack;
 4230	unsigned int size, i, n, ulen = 0, usize = 0;
 4231	u8 genmask = nft_genmask_next(info->net);
 4232	struct nft_rule *rule, *old_rule = NULL;
 4233	struct nft_expr_info *expr_info = NULL;
 4234	u8 family = info->nfmsg->nfgen_family;
 4235	struct nft_flow_rule *flow = NULL;
 4236	struct net *net = info->net;
 4237	struct nft_userdata *udata;
 4238	struct nft_table *table;
 4239	struct nft_chain *chain;
 4240	struct nft_trans *trans;
 4241	u64 handle, pos_handle;
 4242	struct nft_expr *expr;
 4243	struct nft_ctx ctx;
 4244	struct nlattr *tmp;
 4245	int err, rem;
 4246
 4247	lockdep_assert_held(&nft_net->commit_mutex);
 4248
 4249	table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
 4250				 NETLINK_CB(skb).portid);
 4251	if (IS_ERR(table)) {
 4252		NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
 4253		return PTR_ERR(table);
 4254	}
 4255
 4256	if (nla[NFTA_RULE_CHAIN]) {
 4257		chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
 4258					 genmask);
 4259		if (IS_ERR(chain)) {
 4260			NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
 4261			return PTR_ERR(chain);
 4262		}
 4263
 4264	} else if (nla[NFTA_RULE_CHAIN_ID]) {
 4265		chain = nft_chain_lookup_byid(net, table, nla[NFTA_RULE_CHAIN_ID],
 4266					      genmask);
 4267		if (IS_ERR(chain)) {
 4268			NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
 4269			return PTR_ERR(chain);
 4270		}
 4271	} else {
 4272		return -EINVAL;
 4273	}
 4274
 4275	if (nft_chain_is_bound(chain))
 4276		return -EOPNOTSUPP;
 4277
 4278	if (nla[NFTA_RULE_HANDLE]) {
 4279		handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
 4280		rule = __nft_rule_lookup(net, chain, handle);
 4281		if (IS_ERR(rule)) {
 4282			NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
 4283			return PTR_ERR(rule);
 4284		}
 4285
 4286		if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
 4287			NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
 4288			return -EEXIST;
 4289		}
 4290		if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
 4291			old_rule = rule;
 4292		else
 4293			return -EOPNOTSUPP;
 4294	} else {
 4295		if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
 4296		    info->nlh->nlmsg_flags & NLM_F_REPLACE)
 4297			return -EINVAL;
 4298		handle = nf_tables_alloc_handle(table);
 4299
 4300		if (nla[NFTA_RULE_POSITION]) {
 4301			pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
 4302			old_rule = __nft_rule_lookup(net, chain, pos_handle);
 4303			if (IS_ERR(old_rule)) {
 4304				NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
 4305				return PTR_ERR(old_rule);
 4306			}
 4307		} else if (nla[NFTA_RULE_POSITION_ID]) {
 4308			old_rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_POSITION_ID]);
 4309			if (IS_ERR(old_rule)) {
 4310				NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
 4311				return PTR_ERR(old_rule);
 4312			}
 4313		}
 4314	}
 4315
 4316	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
 4317
 4318	n = 0;
 4319	size = 0;
 4320	if (nla[NFTA_RULE_EXPRESSIONS]) {
 4321		expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
 4322					   sizeof(struct nft_expr_info),
 4323					   GFP_KERNEL);
 4324		if (!expr_info)
 4325			return -ENOMEM;
 4326
 4327		nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
 4328			err = -EINVAL;
 4329			if (nla_type(tmp) != NFTA_LIST_ELEM)
 4330				goto err_release_expr;
 4331			if (n == NFT_RULE_MAXEXPRS)
 4332				goto err_release_expr;
 4333			err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
 4334			if (err < 0) {
 4335				NL_SET_BAD_ATTR(extack, tmp);
 4336				goto err_release_expr;
 4337			}
 4338			size += expr_info[n].ops->size;
 4339			n++;
 4340		}
 4341	}
 4342	/* Check for overflow of dlen field */
 4343	err = -EFBIG;
 4344	if (size >= 1 << 12)
 4345		goto err_release_expr;
 4346
 4347	if (nla[NFTA_RULE_USERDATA]) {
 4348		ulen = nla_len(nla[NFTA_RULE_USERDATA]);
 4349		if (ulen > 0)
 4350			usize = sizeof(struct nft_userdata) + ulen;
 4351	}
 4352
 4353	err = -ENOMEM;
 4354	rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL_ACCOUNT);
 4355	if (rule == NULL)
 4356		goto err_release_expr;
 4357
 4358	nft_activate_next(net, rule);
 4359
 4360	rule->handle = handle;
 4361	rule->dlen   = size;
 4362	rule->udata  = ulen ? 1 : 0;
 4363
 4364	if (ulen) {
 4365		udata = nft_userdata(rule);
 4366		udata->len = ulen - 1;
 4367		nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
 4368	}
 4369
 4370	expr = nft_expr_first(rule);
 4371	for (i = 0; i < n; i++) {
 4372		err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
 4373		if (err < 0) {
 4374			NL_SET_BAD_ATTR(extack, expr_info[i].attr);
 4375			goto err_release_rule;
 4376		}
 4377
 4378		if (expr_info[i].ops->validate)
 4379			nft_validate_state_update(table, NFT_VALIDATE_NEED);
 4380
 4381		expr_info[i].ops = NULL;
 4382		expr = nft_expr_next(expr);
 4383	}
 4384
 4385	if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 4386		flow = nft_flow_rule_create(net, rule);
 4387		if (IS_ERR(flow)) {
 4388			err = PTR_ERR(flow);
 4389			goto err_release_rule;
 4390		}
 4391	}
 4392
 4393	if (!nft_use_inc(&chain->use)) {
 4394		err = -EMFILE;
 4395		goto err_destroy_flow;
 4396	}
 4397
 4398	if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
 4399		if (nft_chain_binding(chain)) {
 4400			err = -EOPNOTSUPP;
 4401			goto err_destroy_flow_rule;
 4402		}
 4403
 4404		err = nft_delrule(&ctx, old_rule);
 4405		if (err < 0)
 4406			goto err_destroy_flow_rule;
 4407
 4408		trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
 4409		if (trans == NULL) {
 4410			err = -ENOMEM;
 4411			goto err_destroy_flow_rule;
 4412		}
 4413		list_add_tail_rcu(&rule->list, &old_rule->list);
 4414	} else {
 4415		trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
 4416		if (!trans) {
 4417			err = -ENOMEM;
 4418			goto err_destroy_flow_rule;
 4419		}
 4420
 4421		if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
 4422			if (old_rule)
 4423				list_add_rcu(&rule->list, &old_rule->list);
 4424			else
 4425				list_add_tail_rcu(&rule->list, &chain->rules);
 4426		 } else {
 4427			if (old_rule)
 4428				list_add_tail_rcu(&rule->list, &old_rule->list);
 4429			else
 4430				list_add_rcu(&rule->list, &chain->rules);
 4431		}
 4432	}
 4433	kvfree(expr_info);
 4434
 4435	if (flow)
 4436		nft_trans_flow_rule(trans) = flow;
 4437
 4438	if (table->validate_state == NFT_VALIDATE_DO)
 4439		return nft_table_validate(net, table);
 4440
 4441	return 0;
 4442
 4443err_destroy_flow_rule:
 4444	nft_use_dec_restore(&chain->use);
 4445err_destroy_flow:
 4446	if (flow)
 4447		nft_flow_rule_destroy(flow);
 4448err_release_rule:
 4449	nft_rule_expr_deactivate(&ctx, rule, NFT_TRANS_PREPARE_ERROR);
 4450	nf_tables_rule_destroy(&ctx, rule);
 4451err_release_expr:
 4452	for (i = 0; i < n; i++) {
 4453		if (expr_info[i].ops) {
 4454			module_put(expr_info[i].ops->type->owner);
 4455			if (expr_info[i].ops->type->release_ops)
 4456				expr_info[i].ops->type->release_ops(expr_info[i].ops);
 4457		}
 4458	}
 4459	kvfree(expr_info);
 4460
 4461	return err;
 4462}
 4463
 4464static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
 4465					     const struct nft_chain *chain,
 4466					     const struct nlattr *nla)
 4467{
 4468	struct nftables_pernet *nft_net = nft_pernet(net);
 4469	u32 id = ntohl(nla_get_be32(nla));
 4470	struct nft_trans *trans;
 4471
 4472	list_for_each_entry(trans, &nft_net->commit_list, list) {
 4473		if (trans->msg_type == NFT_MSG_NEWRULE &&
 4474		    nft_trans_rule_chain(trans) == chain &&
 4475		    id == nft_trans_rule_id(trans))
 4476			return nft_trans_rule(trans);
 4477	}
 4478	return ERR_PTR(-ENOENT);
 4479}
 4480
 4481static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
 4482			     const struct nlattr * const nla[])
 4483{
 4484	struct netlink_ext_ack *extack = info->extack;
 4485	u8 genmask = nft_genmask_next(info->net);
 4486	u8 family = info->nfmsg->nfgen_family;
 4487	struct nft_chain *chain = NULL;
 4488	struct net *net = info->net;
 4489	struct nft_table *table;
 4490	struct nft_rule *rule;
 4491	struct nft_ctx ctx;
 4492	int err = 0;
 4493
 4494	table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
 4495				 NETLINK_CB(skb).portid);
 4496	if (IS_ERR(table)) {
 4497		NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
 4498		return PTR_ERR(table);
 4499	}
 4500
 4501	if (nla[NFTA_RULE_CHAIN]) {
 4502		chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
 4503					 genmask);
 4504		if (IS_ERR(chain)) {
 4505			if (PTR_ERR(chain) == -ENOENT &&
 4506			    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
 4507				return 0;
 4508
 4509			NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
 4510			return PTR_ERR(chain);
 4511		}
 4512		if (nft_chain_binding(chain))
 4513			return -EOPNOTSUPP;
 4514	}
 4515
 4516	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
 4517
 4518	if (chain) {
 4519		if (nla[NFTA_RULE_HANDLE]) {
 4520			rule = nft_rule_lookup(info->net, chain, nla[NFTA_RULE_HANDLE]);
 4521			if (IS_ERR(rule)) {
 4522				if (PTR_ERR(rule) == -ENOENT &&
 4523				    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYRULE)
 4524					return 0;
 4525
 4526				NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
 4527				return PTR_ERR(rule);
 4528			}
 4529
 4530			err = nft_delrule(&ctx, rule);
 4531		} else if (nla[NFTA_RULE_ID]) {
 4532			rule = nft_rule_lookup_byid(net, chain, nla[NFTA_RULE_ID]);
 4533			if (IS_ERR(rule)) {
 4534				NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
 4535				return PTR_ERR(rule);
 4536			}
 4537
 4538			err = nft_delrule(&ctx, rule);
 4539		} else {
 4540			err = nft_delrule_by_chain(&ctx);
 4541		}
 4542	} else {
 4543		list_for_each_entry(chain, &table->chains, list) {
 4544			if (!nft_is_active_next(net, chain))
 4545				continue;
 4546			if (nft_chain_binding(chain))
 4547				continue;
 4548
 4549			ctx.chain = chain;
 4550			err = nft_delrule_by_chain(&ctx);
 4551			if (err < 0)
 4552				break;
 4553		}
 4554	}
 4555
 4556	return err;
 4557}
 4558
 4559/*
 4560 * Sets
 4561 */
 4562static const struct nft_set_type *nft_set_types[] = {
 4563	&nft_set_hash_fast_type,
 4564	&nft_set_hash_type,
 4565	&nft_set_rhash_type,
 4566	&nft_set_bitmap_type,
 4567	&nft_set_rbtree_type,
 4568#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
 4569	&nft_set_pipapo_avx2_type,
 4570#endif
 4571	&nft_set_pipapo_type,
 4572};
 4573
 4574#define NFT_SET_FEATURES	(NFT_SET_INTERVAL | NFT_SET_MAP | \
 4575				 NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
 4576				 NFT_SET_EVAL)
 4577
 4578static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
 4579{
 4580	return (flags & type->features) == (flags & NFT_SET_FEATURES);
 4581}
 4582
 4583/*
 4584 * Select a set implementation based on the data characteristics and the
 4585 * given policy. The total memory use might not be known if no size is
 4586 * given, in that case the amount of memory per element is used.
 4587 */
 4588static const struct nft_set_ops *
 4589nft_select_set_ops(const struct nft_ctx *ctx, u32 flags,
 4590		   const struct nft_set_desc *desc)
 4591{
 4592	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 4593	const struct nft_set_ops *ops, *bops;
 4594	struct nft_set_estimate est, best;
 4595	const struct nft_set_type *type;
 4596	int i;
 4597
 4598	lockdep_assert_held(&nft_net->commit_mutex);
 4599	lockdep_nfnl_nft_mutex_not_held();
 4600
 4601	bops	    = NULL;
 4602	best.size   = ~0;
 4603	best.lookup = ~0;
 4604	best.space  = ~0;
 4605
 4606	for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
 4607		type = nft_set_types[i];
 4608		ops = &type->ops;
 4609
 4610		if (!nft_set_ops_candidate(type, flags))
 4611			continue;
 4612		if (!ops->estimate(desc, flags, &est))
 4613			continue;
 4614
 4615		switch (desc->policy) {
 4616		case NFT_SET_POL_PERFORMANCE:
 4617			if (est.lookup < best.lookup)
 4618				break;
 4619			if (est.lookup == best.lookup &&
 4620			    est.space < best.space)
 4621				break;
 4622			continue;
 4623		case NFT_SET_POL_MEMORY:
 4624			if (!desc->size) {
 4625				if (est.space < best.space)
 4626					break;
 4627				if (est.space == best.space &&
 4628				    est.lookup < best.lookup)
 4629					break;
 4630			} else if (est.size < best.size || !bops) {
 4631				break;
 4632			}
 4633			continue;
 4634		default:
 4635			break;
 4636		}
 4637
 4638		bops = ops;
 4639		best = est;
 4640	}
 4641
 4642	if (bops != NULL)
 4643		return bops;
 4644
 4645	return ERR_PTR(-EOPNOTSUPP);
 4646}
 4647
 4648static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
 4649	[NFTA_SET_TABLE]		= { .type = NLA_STRING,
 4650					    .len = NFT_TABLE_MAXNAMELEN - 1 },
 4651	[NFTA_SET_NAME]			= { .type = NLA_STRING,
 4652					    .len = NFT_SET_MAXNAMELEN - 1 },
 4653	[NFTA_SET_FLAGS]		= { .type = NLA_U32 },
 4654	[NFTA_SET_KEY_TYPE]		= { .type = NLA_U32 },
 4655	[NFTA_SET_KEY_LEN]		= { .type = NLA_U32 },
 4656	[NFTA_SET_DATA_TYPE]		= { .type = NLA_U32 },
 4657	[NFTA_SET_DATA_LEN]		= { .type = NLA_U32 },
 4658	[NFTA_SET_POLICY]		= { .type = NLA_U32 },
 4659	[NFTA_SET_DESC]			= { .type = NLA_NESTED },
 4660	[NFTA_SET_ID]			= { .type = NLA_U32 },
 4661	[NFTA_SET_TIMEOUT]		= { .type = NLA_U64 },
 4662	[NFTA_SET_GC_INTERVAL]		= { .type = NLA_U32 },
 4663	[NFTA_SET_USERDATA]		= { .type = NLA_BINARY,
 4664					    .len  = NFT_USERDATA_MAXLEN },
 4665	[NFTA_SET_OBJ_TYPE]		= { .type = NLA_U32 },
 4666	[NFTA_SET_HANDLE]		= { .type = NLA_U64 },
 4667	[NFTA_SET_EXPR]			= { .type = NLA_NESTED },
 4668	[NFTA_SET_EXPRESSIONS]		= NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
 4669	[NFTA_SET_TYPE]			= { .type = NLA_REJECT },
 4670	[NFTA_SET_COUNT]		= { .type = NLA_REJECT },
 4671};
 4672
 4673static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
 4674	[NFTA_SET_FIELD_LEN]	= { .type = NLA_U32 },
 4675};
 4676
 4677static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
 4678	[NFTA_SET_DESC_SIZE]		= { .type = NLA_U32 },
 4679	[NFTA_SET_DESC_CONCAT]		= NLA_POLICY_NESTED_ARRAY(nft_concat_policy),
 4680};
 4681
 4682static struct nft_set *nft_set_lookup(const struct net *net,
 4683				      const struct nft_table *table,
 4684				      const struct nlattr *nla, u8 genmask)
 4685{
 4686	struct nft_set *set;
 4687
 4688	if (nla == NULL)
 4689		return ERR_PTR(-EINVAL);
 4690
 4691	list_for_each_entry_rcu(set, &table->sets, list,
 4692				lockdep_commit_lock_is_held(net)) {
 4693		if (!nla_strcmp(nla, set->name) &&
 4694		    nft_active_genmask(set, genmask))
 4695			return set;
 4696	}
 4697	return ERR_PTR(-ENOENT);
 4698}
 4699
 4700static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
 4701					       const struct nlattr *nla,
 4702					       u8 genmask)
 4703{
 4704	struct nft_set *set;
 4705
 4706	list_for_each_entry(set, &table->sets, list) {
 4707		if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
 4708		    nft_active_genmask(set, genmask))
 4709			return set;
 4710	}
 4711	return ERR_PTR(-ENOENT);
 4712}
 4713
 4714static struct nft_set *nft_set_lookup_byid(const struct net *net,
 4715					   const struct nft_table *table,
 4716					   const struct nlattr *nla, u8 genmask)
 4717{
 4718	struct nftables_pernet *nft_net = nft_pernet(net);
 4719	u32 id = ntohl(nla_get_be32(nla));
 4720	struct nft_trans_set *trans;
 4721
 4722	/* its likely the id we need is at the tail, not at start */
 4723	list_for_each_entry_reverse(trans, &nft_net->commit_set_list, list_trans_newset) {
 4724		struct nft_set *set = trans->set;
 4725
 4726		if (id == trans->set_id &&
 4727		    set->table == table &&
 4728		    nft_active_genmask(set, genmask))
 4729			return set;
 4730	}
 4731	return ERR_PTR(-ENOENT);
 4732}
 4733
 4734struct nft_set *nft_set_lookup_global(const struct net *net,
 4735				      const struct nft_table *table,
 4736				      const struct nlattr *nla_set_name,
 4737				      const struct nlattr *nla_set_id,
 4738				      u8 genmask)
 4739{
 4740	struct nft_set *set;
 4741
 4742	set = nft_set_lookup(net, table, nla_set_name, genmask);
 4743	if (IS_ERR(set)) {
 4744		if (!nla_set_id)
 4745			return set;
 4746
 4747		set = nft_set_lookup_byid(net, table, nla_set_id, genmask);
 4748	}
 4749	return set;
 4750}
 4751EXPORT_SYMBOL_GPL(nft_set_lookup_global);
 4752
 4753static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
 4754				    const char *name)
 4755{
 4756	const struct nft_set *i;
 4757	const char *p;
 4758	unsigned long *inuse;
 4759	unsigned int n = 0, min = 0;
 4760
 4761	p = strchr(name, '%');
 4762	if (p != NULL) {
 4763		if (p[1] != 'd' || strchr(p + 2, '%'))
 4764			return -EINVAL;
 4765
 4766		if (strnlen(name, NFT_SET_MAX_ANONLEN) >= NFT_SET_MAX_ANONLEN)
 4767			return -EINVAL;
 4768
 4769		inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 4770		if (inuse == NULL)
 4771			return -ENOMEM;
 4772cont:
 4773		list_for_each_entry(i, &ctx->table->sets, list) {
 4774			int tmp;
 4775
 4776			if (!nft_is_active_next(ctx->net, i))
 4777				continue;
 4778			if (!sscanf(i->name, name, &tmp))
 4779				continue;
 4780			if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
 4781				continue;
 4782
 4783			set_bit(tmp - min, inuse);
 4784		}
 4785
 4786		n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
 4787		if (n >= BITS_PER_BYTE * PAGE_SIZE) {
 4788			min += BITS_PER_BYTE * PAGE_SIZE;
 4789			memset(inuse, 0, PAGE_SIZE);
 4790			goto cont;
 4791		}
 4792		free_page((unsigned long)inuse);
 4793	}
 4794
 4795	set->name = kasprintf(GFP_KERNEL_ACCOUNT, name, min + n);
 4796	if (!set->name)
 4797		return -ENOMEM;
 4798
 4799	list_for_each_entry(i, &ctx->table->sets, list) {
 4800		if (!nft_is_active_next(ctx->net, i))
 4801			continue;
 4802		if (!strcmp(set->name, i->name)) {
 4803			kfree(set->name);
 4804			set->name = NULL;
 4805			return -ENFILE;
 4806		}
 4807	}
 4808	return 0;
 4809}
 4810
 4811int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
 4812{
 4813	u64 ms = be64_to_cpu(nla_get_be64(nla));
 4814	u64 max = (u64)(~((u64)0));
 4815
 4816	max = div_u64(max, NSEC_PER_MSEC);
 4817	if (ms >= max)
 4818		return -ERANGE;
 4819
 4820	ms *= NSEC_PER_MSEC;
 4821	*result = nsecs_to_jiffies64(ms) ? : !!ms;
 4822	return 0;
 4823}
 4824
 4825__be64 nf_jiffies64_to_msecs(u64 input)
 4826{
 4827	return cpu_to_be64(jiffies64_to_msecs(input));
 4828}
 4829
 4830static int nf_tables_fill_set_concat(struct sk_buff *skb,
 4831				     const struct nft_set *set)
 4832{
 4833	struct nlattr *concat, *field;
 4834	int i;
 4835
 4836	concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
 4837	if (!concat)
 4838		return -ENOMEM;
 4839
 4840	for (i = 0; i < set->field_count; i++) {
 4841		field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
 4842		if (!field)
 4843			return -ENOMEM;
 4844
 4845		if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
 4846				 htonl(set->field_len[i])))
 4847			return -ENOMEM;
 4848
 4849		nla_nest_end(skb, field);
 4850	}
 4851
 4852	nla_nest_end(skb, concat);
 4853
 4854	return 0;
 4855}
 4856
 4857static u32 nft_set_userspace_size(const struct nft_set_ops *ops, u32 size)
 4858{
 4859	if (ops->usize)
 4860		return ops->usize(size);
 4861
 4862	return size;
 4863}
 4864
 4865static noinline_for_stack int
 4866nf_tables_fill_set_info(struct sk_buff *skb, const struct nft_set *set)
 4867{
 4868	unsigned int nelems;
 4869	char str[40];
 4870	int ret;
 4871
 4872	ret = snprintf(str, sizeof(str), "%ps", set->ops);
 4873
 4874	/* Not expected to happen and harmless: NFTA_SET_TYPE is dumped
 4875	 * to userspace purely for informational/debug purposes.
 4876	 */
 4877	DEBUG_NET_WARN_ON_ONCE(ret >= sizeof(str));
 4878
 4879	if (nla_put_string(skb, NFTA_SET_TYPE, str))
 4880		return -EMSGSIZE;
 4881
 4882	nelems = nft_set_userspace_size(set->ops, atomic_read(&set->nelems));
 4883	return nla_put_be32(skb, NFTA_SET_COUNT, htonl(nelems));
 4884}
 4885
 4886static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
 4887			      const struct nft_set *set, u16 event, u16 flags)
 4888{
 4889	u64 timeout = READ_ONCE(set->timeout);
 4890	u32 gc_int = READ_ONCE(set->gc_int);
 4891	u32 portid = ctx->portid;
 4892	struct nlmsghdr *nlh;
 4893	struct nlattr *nest;
 4894	u32 seq = ctx->seq;
 4895	int i;
 4896
 4897	nlh = nfnl_msg_put(skb, portid, seq,
 4898			   nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event),
 4899			   flags, ctx->family, NFNETLINK_V0,
 4900			   nft_base_seq_be16(ctx->net));
 4901	if (!nlh)
 4902		goto nla_put_failure;
 4903
 4904	if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
 4905		goto nla_put_failure;
 4906	if (nla_put_string(skb, NFTA_SET_NAME, set->name))
 4907		goto nla_put_failure;
 4908	if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
 4909			 NFTA_SET_PAD))
 4910		goto nla_put_failure;
 4911
 4912	if (event == NFT_MSG_DELSET ||
 4913	    event == NFT_MSG_DESTROYSET) {
 4914		nlmsg_end(skb, nlh);
 4915		return 0;
 4916	}
 4917
 4918	if (set->flags != 0)
 4919		if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
 4920			goto nla_put_failure;
 4921
 4922	if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
 4923		goto nla_put_failure;
 4924	if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
 4925		goto nla_put_failure;
 4926	if (set->flags & NFT_SET_MAP) {
 4927		if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
 4928			goto nla_put_failure;
 4929		if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
 4930			goto nla_put_failure;
 4931	}
 4932	if (set->flags & NFT_SET_OBJECT &&
 4933	    nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
 4934		goto nla_put_failure;
 4935
 4936	if (timeout &&
 4937	    nla_put_be64(skb, NFTA_SET_TIMEOUT,
 4938			 nf_jiffies64_to_msecs(timeout),
 4939			 NFTA_SET_PAD))
 4940		goto nla_put_failure;
 4941	if (gc_int &&
 4942	    nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(gc_int)))
 4943		goto nla_put_failure;
 4944
 4945	if (set->policy != NFT_SET_POL_PERFORMANCE) {
 4946		if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
 4947			goto nla_put_failure;
 4948	}
 4949
 4950	if (set->udata &&
 4951	    nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
 4952		goto nla_put_failure;
 4953
 4954	nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
 4955	if (!nest)
 4956		goto nla_put_failure;
 4957	if (set->size &&
 4958	    nla_put_be32(skb, NFTA_SET_DESC_SIZE,
 4959			 htonl(nft_set_userspace_size(set->ops, set->size))))
 4960		goto nla_put_failure;
 4961
 4962	if (set->field_count > 1 &&
 4963	    nf_tables_fill_set_concat(skb, set))
 4964		goto nla_put_failure;
 4965
 4966	nla_nest_end(skb, nest);
 4967
 4968	if (nf_tables_fill_set_info(skb, set))
 4969		goto nla_put_failure;
 4970
 4971	if (set->num_exprs == 1) {
 4972		nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
 4973		if (nf_tables_fill_expr_info(skb, set->exprs[0], false) < 0)
 4974			goto nla_put_failure;
 4975
 4976		nla_nest_end(skb, nest);
 4977	} else if (set->num_exprs > 1) {
 4978		nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
 4979		if (nest == NULL)
 4980			goto nla_put_failure;
 4981
 4982		for (i = 0; i < set->num_exprs; i++) {
 4983			if (nft_expr_dump(skb, NFTA_LIST_ELEM,
 4984					  set->exprs[i], false) < 0)
 4985				goto nla_put_failure;
 4986		}
 4987		nla_nest_end(skb, nest);
 4988	}
 4989
 4990	nlmsg_end(skb, nlh);
 4991	return 0;
 4992
 4993nla_put_failure:
 4994	nlmsg_trim(skb, nlh);
 4995	return -1;
 4996}
 4997
 4998static void nf_tables_set_notify(const struct nft_ctx *ctx,
 4999				 const struct nft_set *set, int event,
 5000			         gfp_t gfp_flags)
 5001{
 5002	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 5003	u32 portid = ctx->portid;
 5004	struct sk_buff *skb;
 5005	u16 flags = 0;
 5006	int err;
 5007
 5008	if (!ctx->report &&
 5009	    !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 5010		return;
 5011
 5012	skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
 5013	if (skb == NULL)
 5014		goto err;
 5015
 5016	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 5017		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 5018
 5019	err = nf_tables_fill_set(skb, ctx, set, event, flags);
 5020	if (err < 0) {
 5021		kfree_skb(skb);
 5022		goto err;
 5023	}
 5024
 5025	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 5026	return;
 5027err:
 5028	nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
 5029}
 5030
 5031static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
 5032{
 5033	const struct nft_set *set;
 5034	unsigned int idx, s_idx = cb->args[0];
 5035	struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
 5036	struct net *net = sock_net(skb->sk);
 5037	struct nft_ctx *ctx = cb->data, ctx_set;
 5038	struct nftables_pernet *nft_net;
 5039
 5040	if (cb->args[1])
 5041		return skb->len;
 5042
 5043	rcu_read_lock();
 5044	nft_net = nft_pernet(net);
 5045	cb->seq = nft_base_seq(net);
 5046
 5047	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 5048		if (ctx->family != NFPROTO_UNSPEC &&
 5049		    ctx->family != table->family)
 5050			continue;
 5051
 5052		if (ctx->table && ctx->table != table)
 5053			continue;
 5054
 5055		if (cur_table) {
 5056			if (cur_table != table)
 5057				continue;
 5058
 5059			cur_table = NULL;
 5060		}
 5061		idx = 0;
 5062		list_for_each_entry_rcu(set, &table->sets, list) {
 5063			if (idx < s_idx)
 5064				goto cont;
 5065			if (!nft_is_active(net, set))
 5066				goto cont;
 5067
 5068			ctx_set = *ctx;
 5069			ctx_set.table = table;
 5070			ctx_set.family = table->family;
 5071
 5072			if (nf_tables_fill_set(skb, &ctx_set, set,
 5073					       NFT_MSG_NEWSET,
 5074					       NLM_F_MULTI) < 0) {
 5075				cb->args[0] = idx;
 5076				cb->args[2] = (unsigned long) table;
 5077				goto done;
 5078			}
 5079			nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 5080cont:
 5081			idx++;
 5082		}
 5083		if (s_idx)
 5084			s_idx = 0;
 5085	}
 5086	cb->args[1] = 1;
 5087done:
 5088	rcu_read_unlock();
 5089	return skb->len;
 5090}
 5091
 5092static int nf_tables_dump_sets_start(struct netlink_callback *cb)
 5093{
 5094	struct nft_ctx *ctx_dump = NULL;
 5095
 5096	ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
 5097	if (ctx_dump == NULL)
 5098		return -ENOMEM;
 5099
 5100	cb->data = ctx_dump;
 5101	return 0;
 5102}
 5103
 5104static int nf_tables_dump_sets_done(struct netlink_callback *cb)
 5105{
 5106	kfree(cb->data);
 5107	return 0;
 5108}
 5109
 5110/* called with rcu_read_lock held */
 5111static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
 5112			    const struct nlattr * const nla[])
 5113{
 5114	struct netlink_ext_ack *extack = info->extack;
 5115	u8 genmask = nft_genmask_cur(info->net);
 5116	u8 family = info->nfmsg->nfgen_family;
 5117	struct nft_table *table = NULL;
 5118	struct net *net = info->net;
 5119	const struct nft_set *set;
 5120	struct sk_buff *skb2;
 5121	struct nft_ctx ctx;
 5122	int err;
 5123
 5124	if (nla[NFTA_SET_TABLE]) {
 5125		table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
 5126					 genmask, 0);
 5127		if (IS_ERR(table)) {
 5128			NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
 5129			return PTR_ERR(table);
 5130		}
 5131	}
 5132
 5133	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 5134
 5135	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 5136		struct netlink_dump_control c = {
 5137			.start = nf_tables_dump_sets_start,
 5138			.dump = nf_tables_dump_sets,
 5139			.done = nf_tables_dump_sets_done,
 5140			.data = &ctx,
 5141			.module = THIS_MODULE,
 5142		};
 5143
 5144		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 5145	}
 5146
 5147	/* Only accept unspec with dump */
 5148	if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
 5149		return -EAFNOSUPPORT;
 5150	if (!nla[NFTA_SET_TABLE])
 5151		return -EINVAL;
 5152
 5153	set = nft_set_lookup(net, table, nla[NFTA_SET_NAME], genmask);
 5154	if (IS_ERR(set)) {
 5155		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
 5156		return PTR_ERR(set);
 5157	}
 5158
 5159	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 5160	if (skb2 == NULL)
 5161		return -ENOMEM;
 5162
 5163	err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
 5164	if (err < 0)
 5165		goto err_fill_set_info;
 5166
 5167	return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 5168
 5169err_fill_set_info:
 5170	kfree_skb(skb2);
 5171	return err;
 5172}
 5173
 5174static int nft_set_desc_concat_parse(const struct nlattr *attr,
 5175				     struct nft_set_desc *desc)
 5176{
 5177	struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
 5178	u32 len;
 5179	int err;
 5180
 5181	if (desc->field_count >= ARRAY_SIZE(desc->field_len))
 5182		return -E2BIG;
 5183
 5184	err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
 5185					  nft_concat_policy, NULL);
 5186	if (err < 0)
 5187		return err;
 5188
 5189	if (!tb[NFTA_SET_FIELD_LEN])
 5190		return -EINVAL;
 5191
 5192	len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
 5193	if (!len || len > U8_MAX)
 5194		return -EINVAL;
 5195
 5196	desc->field_len[desc->field_count++] = len;
 5197
 5198	return 0;
 5199}
 5200
 5201static int nft_set_desc_concat(struct nft_set_desc *desc,
 5202			       const struct nlattr *nla)
 5203{
 5204	u32 len = 0, num_regs;
 5205	struct nlattr *attr;
 5206	int rem, err, i;
 5207
 5208	nla_for_each_nested(attr, nla, rem) {
 5209		if (nla_type(attr) != NFTA_LIST_ELEM)
 5210			return -EINVAL;
 5211
 5212		err = nft_set_desc_concat_parse(attr, desc);
 5213		if (err < 0)
 5214			return err;
 5215	}
 5216
 5217	for (i = 0; i < desc->field_count; i++)
 5218		len += round_up(desc->field_len[i], sizeof(u32));
 5219
 5220	if (len != desc->klen)
 5221		return -EINVAL;
 5222
 5223	num_regs = DIV_ROUND_UP(desc->klen, sizeof(u32));
 5224	if (num_regs > NFT_REG32_COUNT)
 5225		return -E2BIG;
 5226
 5227	return 0;
 5228}
 5229
 5230static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
 5231				    const struct nlattr *nla)
 5232{
 5233	struct nlattr *da[NFTA_SET_DESC_MAX + 1];
 5234	int err;
 5235
 5236	err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
 5237					  nft_set_desc_policy, NULL);
 5238	if (err < 0)
 5239		return err;
 5240
 5241	if (da[NFTA_SET_DESC_SIZE] != NULL)
 5242		desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
 5243	if (da[NFTA_SET_DESC_CONCAT])
 5244		err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
 5245
 5246	return err;
 5247}
 5248
 5249static int nft_set_expr_alloc(struct nft_ctx *ctx, struct nft_set *set,
 5250			      const struct nlattr * const *nla,
 5251			      struct nft_expr **exprs, int *num_exprs,
 5252			      u32 flags)
 5253{
 5254	struct nft_expr *expr;
 5255	int err, i;
 5256
 5257	if (nla[NFTA_SET_EXPR]) {
 5258		expr = nft_set_elem_expr_alloc(ctx, set, nla[NFTA_SET_EXPR]);
 5259		if (IS_ERR(expr)) {
 5260			err = PTR_ERR(expr);
 5261			goto err_set_expr_alloc;
 5262		}
 5263		exprs[0] = expr;
 5264		(*num_exprs)++;
 5265	} else if (nla[NFTA_SET_EXPRESSIONS]) {
 5266		struct nlattr *tmp;
 5267		int left;
 5268
 5269		if (!(flags & NFT_SET_EXPR)) {
 5270			err = -EINVAL;
 5271			goto err_set_expr_alloc;
 5272		}
 5273		i = 0;
 5274		nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
 5275			if (i == NFT_SET_EXPR_MAX) {
 5276				err = -E2BIG;
 5277				goto err_set_expr_alloc;
 5278			}
 5279			if (nla_type(tmp) != NFTA_LIST_ELEM) {
 5280				err = -EINVAL;
 5281				goto err_set_expr_alloc;
 5282			}
 5283			expr = nft_set_elem_expr_alloc(ctx, set, tmp);
 5284			if (IS_ERR(expr)) {
 5285				err = PTR_ERR(expr);
 5286				goto err_set_expr_alloc;
 5287			}
 5288			exprs[i++] = expr;
 5289			(*num_exprs)++;
 5290		}
 5291	}
 5292
 5293	return 0;
 5294
 5295err_set_expr_alloc:
 5296	for (i = 0; i < *num_exprs; i++)
 5297		nft_expr_destroy(ctx, exprs[i]);
 5298
 5299	return err;
 5300}
 5301
 5302static bool nft_set_is_same(const struct nft_set *set,
 5303			    const struct nft_set_desc *desc,
 5304			    struct nft_expr *exprs[], u32 num_exprs, u32 flags)
 5305{
 5306	int i;
 5307
 5308	if (set->ktype != desc->ktype ||
 5309	    set->dtype != desc->dtype ||
 5310	    set->flags != flags ||
 5311	    set->klen != desc->klen ||
 5312	    set->dlen != desc->dlen ||
 5313	    set->field_count != desc->field_count ||
 5314	    set->num_exprs != num_exprs)
 5315		return false;
 5316
 5317	for (i = 0; i < desc->field_count; i++) {
 5318		if (set->field_len[i] != desc->field_len[i])
 5319			return false;
 5320	}
 5321
 5322	for (i = 0; i < num_exprs; i++) {
 5323		if (set->exprs[i]->ops != exprs[i]->ops)
 5324			return false;
 5325	}
 5326
 5327	return true;
 5328}
 5329
 5330static u32 nft_set_kernel_size(const struct nft_set_ops *ops,
 5331			       const struct nft_set_desc *desc)
 5332{
 5333	if (ops->ksize)
 5334		return ops->ksize(desc->size);
 5335
 5336	return desc->size;
 5337}
 5338
 5339static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
 5340			    const struct nlattr * const nla[])
 5341{
 5342	struct netlink_ext_ack *extack = info->extack;
 5343	u8 genmask = nft_genmask_next(info->net);
 5344	u8 family = info->nfmsg->nfgen_family;
 5345	const struct nft_set_ops *ops;
 5346	struct net *net = info->net;
 5347	struct nft_set_desc desc;
 5348	struct nft_table *table;
 5349	unsigned char *udata;
 5350	struct nft_set *set;
 5351	struct nft_ctx ctx;
 5352	size_t alloc_size;
 5353	int num_exprs = 0;
 5354	char *name;
 5355	int err, i;
 5356	u16 udlen;
 5357	u32 flags;
 5358	u64 size;
 5359
 5360	if (nla[NFTA_SET_TABLE] == NULL ||
 5361	    nla[NFTA_SET_NAME] == NULL ||
 5362	    nla[NFTA_SET_KEY_LEN] == NULL ||
 5363	    nla[NFTA_SET_ID] == NULL)
 5364		return -EINVAL;
 5365
 5366	memset(&desc, 0, sizeof(desc));
 5367
 5368	desc.ktype = NFT_DATA_VALUE;
 5369	if (nla[NFTA_SET_KEY_TYPE] != NULL) {
 5370		desc.ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
 5371		if ((desc.ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
 5372			return -EINVAL;
 5373	}
 5374
 5375	desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
 5376	if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
 5377		return -EINVAL;
 5378
 5379	flags = 0;
 5380	if (nla[NFTA_SET_FLAGS] != NULL) {
 5381		flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
 5382		if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
 5383			      NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
 5384			      NFT_SET_MAP | NFT_SET_EVAL |
 5385			      NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
 5386			return -EOPNOTSUPP;
 5387		/* Only one of these operations is supported */
 5388		if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
 5389			     (NFT_SET_MAP | NFT_SET_OBJECT))
 5390			return -EOPNOTSUPP;
 5391		if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
 5392			     (NFT_SET_EVAL | NFT_SET_OBJECT))
 5393			return -EOPNOTSUPP;
 5394		if ((flags & (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT | NFT_SET_EVAL)) ==
 5395			     (NFT_SET_ANONYMOUS | NFT_SET_TIMEOUT))
 5396			return -EOPNOTSUPP;
 5397		if ((flags & (NFT_SET_CONSTANT | NFT_SET_TIMEOUT)) ==
 5398			     (NFT_SET_CONSTANT | NFT_SET_TIMEOUT))
 5399			return -EOPNOTSUPP;
 5400	}
 5401
 5402	desc.dtype = 0;
 5403	if (nla[NFTA_SET_DATA_TYPE] != NULL) {
 5404		if (!(flags & NFT_SET_MAP))
 5405			return -EINVAL;
 5406
 5407		desc.dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
 5408		if ((desc.dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
 5409		    desc.dtype != NFT_DATA_VERDICT)
 5410			return -EINVAL;
 5411
 5412		if (desc.dtype != NFT_DATA_VERDICT) {
 5413			if (nla[NFTA_SET_DATA_LEN] == NULL)
 5414				return -EINVAL;
 5415			desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
 5416			if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
 5417				return -EINVAL;
 5418		} else
 5419			desc.dlen = sizeof(struct nft_verdict);
 5420	} else if (flags & NFT_SET_MAP)
 5421		return -EINVAL;
 5422
 5423	if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
 5424		if (!(flags & NFT_SET_OBJECT))
 5425			return -EINVAL;
 5426
 5427		desc.objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
 5428		if (desc.objtype == NFT_OBJECT_UNSPEC ||
 5429		    desc.objtype > NFT_OBJECT_MAX)
 5430			return -EOPNOTSUPP;
 5431	} else if (flags & NFT_SET_OBJECT)
 5432		return -EINVAL;
 5433	else
 5434		desc.objtype = NFT_OBJECT_UNSPEC;
 5435
 5436	desc.timeout = 0;
 5437	if (nla[NFTA_SET_TIMEOUT] != NULL) {
 5438		if (!(flags & NFT_SET_TIMEOUT))
 5439			return -EINVAL;
 5440
 5441		if (flags & NFT_SET_ANONYMOUS)
 5442			return -EOPNOTSUPP;
 5443
 5444		err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &desc.timeout);
 5445		if (err)
 5446			return err;
 5447	}
 5448	desc.gc_int = 0;
 5449	if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
 5450		if (!(flags & NFT_SET_TIMEOUT))
 5451			return -EINVAL;
 5452
 5453		if (flags & NFT_SET_ANONYMOUS)
 5454			return -EOPNOTSUPP;
 5455
 5456		desc.gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
 5457	}
 5458
 5459	desc.policy = NFT_SET_POL_PERFORMANCE;
 5460	if (nla[NFTA_SET_POLICY] != NULL) {
 5461		desc.policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
 5462		switch (desc.policy) {
 5463		case NFT_SET_POL_PERFORMANCE:
 5464		case NFT_SET_POL_MEMORY:
 5465			break;
 5466		default:
 5467			return -EOPNOTSUPP;
 5468		}
 5469	}
 5470
 5471	if (nla[NFTA_SET_DESC] != NULL) {
 5472		err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
 5473		if (err < 0)
 5474			return err;
 5475
 5476		if (desc.field_count > 1) {
 5477			if (!(flags & NFT_SET_CONCAT))
 5478				return -EINVAL;
 5479		} else if (flags & NFT_SET_CONCAT) {
 5480			return -EINVAL;
 5481		}
 5482	} else if (flags & NFT_SET_CONCAT) {
 5483		return -EINVAL;
 5484	}
 5485
 5486	if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
 5487		desc.expr = true;
 5488
 5489	table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
 5490				 NETLINK_CB(skb).portid);
 5491	if (IS_ERR(table)) {
 5492		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
 5493		return PTR_ERR(table);
 5494	}
 5495
 5496	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 5497
 5498	set = nft_set_lookup(net, table, nla[NFTA_SET_NAME], genmask);
 5499	if (IS_ERR(set)) {
 5500		if (PTR_ERR(set) != -ENOENT) {
 5501			NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
 5502			return PTR_ERR(set);
 5503		}
 5504	} else {
 5505		struct nft_expr *exprs[NFT_SET_EXPR_MAX] = {};
 5506
 5507		if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
 5508			NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
 5509			return -EEXIST;
 5510		}
 5511		if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
 5512			return -EOPNOTSUPP;
 5513
 5514		if (nft_set_is_anonymous(set))
 5515			return -EOPNOTSUPP;
 5516
 5517		err = nft_set_expr_alloc(&ctx, set, nla, exprs, &num_exprs, flags);
 5518		if (err < 0)
 5519			return err;
 5520
 5521		if (desc.size)
 5522			desc.size = nft_set_kernel_size(set->ops, &desc);
 5523
 5524		err = 0;
 5525		if (!nft_set_is_same(set, &desc, exprs, num_exprs, flags)) {
 5526			NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
 5527			err = -EEXIST;
 5528		}
 5529
 5530		for (i = 0; i < num_exprs; i++)
 5531			nft_expr_destroy(&ctx, exprs[i]);
 5532
 5533		if (err < 0)
 5534			return err;
 5535
 5536		return __nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set, &desc);
 5537	}
 5538
 5539	if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
 5540		return -ENOENT;
 5541
 5542	ops = nft_select_set_ops(&ctx, flags, &desc);
 5543	if (IS_ERR(ops))
 5544		return PTR_ERR(ops);
 5545
 5546	if (desc.size)
 5547		desc.size = nft_set_kernel_size(ops, &desc);
 5548
 5549	udlen = 0;
 5550	if (nla[NFTA_SET_USERDATA])
 5551		udlen = nla_len(nla[NFTA_SET_USERDATA]);
 5552
 5553	size = 0;
 5554	if (ops->privsize != NULL)
 5555		size = ops->privsize(nla, &desc);
 5556	alloc_size = sizeof(*set) + size + udlen;
 5557	if (alloc_size < size || alloc_size > INT_MAX)
 5558		return -ENOMEM;
 5559
 5560	if (!nft_use_inc(&table->use))
 5561		return -EMFILE;
 5562
 5563	set = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT);
 5564	if (!set) {
 5565		err = -ENOMEM;
 5566		goto err_alloc;
 5567	}
 5568
 5569	name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL_ACCOUNT);
 5570	if (!name) {
 5571		err = -ENOMEM;
 5572		goto err_set_name;
 5573	}
 5574
 5575	err = nf_tables_set_alloc_name(&ctx, set, name);
 5576	kfree(name);
 5577	if (err < 0)
 5578		goto err_set_name;
 5579
 5580	udata = NULL;
 5581	if (udlen) {
 5582		udata = set->data + size;
 5583		nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
 5584	}
 5585
 5586	INIT_LIST_HEAD(&set->bindings);
 5587	INIT_LIST_HEAD(&set->catchall_list);
 5588	refcount_set(&set->refs, 1);
 5589	set->table = table;
 5590	write_pnet(&set->net, net);
 5591	set->ops = ops;
 5592	set->ktype = desc.ktype;
 5593	set->klen = desc.klen;
 5594	set->dtype = desc.dtype;
 5595	set->objtype = desc.objtype;
 5596	set->dlen = desc.dlen;
 5597	set->flags = flags;
 5598	set->size = desc.size;
 5599	set->policy = desc.policy;
 5600	set->udlen = udlen;
 5601	set->udata = udata;
 5602	set->timeout = desc.timeout;
 5603	set->gc_int = desc.gc_int;
 5604
 5605	set->field_count = desc.field_count;
 5606	for (i = 0; i < desc.field_count; i++)
 5607		set->field_len[i] = desc.field_len[i];
 5608
 5609	err = ops->init(set, &desc, nla);
 5610	if (err < 0)
 5611		goto err_set_init;
 5612
 5613	err = nft_set_expr_alloc(&ctx, set, nla, set->exprs, &num_exprs, flags);
 5614	if (err < 0)
 5615		goto err_set_destroy;
 5616
 5617	set->num_exprs = num_exprs;
 5618	set->handle = nf_tables_alloc_handle(table);
 5619	INIT_LIST_HEAD(&set->pending_update);
 5620
 5621	err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
 5622	if (err < 0)
 5623		goto err_set_expr_alloc;
 5624
 5625	list_add_tail_rcu(&set->list, &table->sets);
 5626
 5627	return 0;
 5628
 5629err_set_expr_alloc:
 5630	for (i = 0; i < set->num_exprs; i++)
 5631		nft_expr_destroy(&ctx, set->exprs[i]);
 5632err_set_destroy:
 5633	ops->destroy(&ctx, set);
 5634err_set_init:
 5635	kfree(set->name);
 5636err_set_name:
 5637	kvfree(set);
 5638err_alloc:
 5639	nft_use_dec_restore(&table->use);
 5640
 5641	return err;
 5642}
 5643
 5644static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
 5645				     struct nft_set *set)
 5646{
 5647	struct nft_set_elem_catchall *next, *catchall;
 5648
 5649	list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
 5650		list_del_rcu(&catchall->list);
 5651		nf_tables_set_elem_destroy(ctx, set, catchall->elem);
 5652		kfree_rcu(catchall, rcu);
 5653	}
 5654}
 5655
 5656static void nft_set_put(struct nft_set *set)
 5657{
 5658	if (refcount_dec_and_test(&set->refs)) {
 5659		kfree(set->name);
 5660		kvfree(set);
 5661	}
 5662}
 5663
 5664static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
 5665{
 5666	int i;
 5667
 5668	if (WARN_ON(set->use > 0))
 5669		return;
 5670
 5671	for (i = 0; i < set->num_exprs; i++)
 5672		nft_expr_destroy(ctx, set->exprs[i]);
 5673
 5674	set->ops->destroy(ctx, set);
 5675	nft_set_catchall_destroy(ctx, set);
 5676	nft_set_put(set);
 5677}
 5678
 5679static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
 5680			    const struct nlattr * const nla[])
 5681{
 5682	struct netlink_ext_ack *extack = info->extack;
 5683	u8 genmask = nft_genmask_next(info->net);
 5684	u8 family = info->nfmsg->nfgen_family;
 5685	struct net *net = info->net;
 5686	const struct nlattr *attr;
 5687	struct nft_table *table;
 5688	struct nft_set *set;
 5689	struct nft_ctx ctx;
 5690
 5691	if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
 5692		return -EAFNOSUPPORT;
 5693
 5694	table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
 5695				 genmask, NETLINK_CB(skb).portid);
 5696	if (IS_ERR(table)) {
 5697		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
 5698		return PTR_ERR(table);
 5699	}
 5700
 5701	if (nla[NFTA_SET_HANDLE]) {
 5702		attr = nla[NFTA_SET_HANDLE];
 5703		set = nft_set_lookup_byhandle(table, attr, genmask);
 5704	} else {
 5705		attr = nla[NFTA_SET_NAME];
 5706		set = nft_set_lookup(net, table, attr, genmask);
 5707	}
 5708
 5709	if (IS_ERR(set)) {
 5710		if (PTR_ERR(set) == -ENOENT &&
 5711		    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSET)
 5712			return 0;
 5713
 5714		NL_SET_BAD_ATTR(extack, attr);
 5715		return PTR_ERR(set);
 5716	}
 5717	if (set->use ||
 5718	    (info->nlh->nlmsg_flags & NLM_F_NONREC &&
 5719	     atomic_read(&set->nelems) > 0)) {
 5720		NL_SET_BAD_ATTR(extack, attr);
 5721		return -EBUSY;
 5722	}
 5723
 5724	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 5725
 5726	return nft_delset(&ctx, set);
 5727}
 5728
 5729static int nft_validate_register_store(const struct nft_ctx *ctx,
 5730				       enum nft_registers reg,
 5731				       const struct nft_data *data,
 5732				       enum nft_data_types type,
 5733				       unsigned int len);
 5734
 5735static int nft_setelem_data_validate(const struct nft_ctx *ctx,
 5736				     struct nft_set *set,
 5737				     struct nft_elem_priv *elem_priv)
 5738{
 5739	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 5740	enum nft_registers dreg;
 5741
 5742	dreg = nft_type_to_reg(set->dtype);
 5743	return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
 5744					   set->dtype == NFT_DATA_VERDICT ?
 5745					   NFT_DATA_VERDICT : NFT_DATA_VALUE,
 5746					   set->dlen);
 5747}
 5748
 5749static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
 5750					struct nft_set *set,
 5751					const struct nft_set_iter *iter,
 5752					struct nft_elem_priv *elem_priv)
 5753{
 5754	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 5755
 5756	if (!nft_set_elem_active(ext, iter->genmask))
 5757		return 0;
 5758
 5759	return nft_setelem_data_validate(ctx, set, elem_priv);
 5760}
 5761
 5762static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
 5763				       struct nft_set *set)
 5764{
 5765	u8 genmask = nft_genmask_next(ctx->net);
 5766	struct nft_set_elem_catchall *catchall;
 5767	struct nft_set_ext *ext;
 5768	int ret = 0;
 5769
 5770	list_for_each_entry_rcu(catchall, &set->catchall_list, list,
 5771				lockdep_commit_lock_is_held(ctx->net)) {
 5772		ext = nft_set_elem_ext(set, catchall->elem);
 5773		if (!nft_set_elem_active(ext, genmask))
 5774			continue;
 5775
 5776		ret = nft_setelem_data_validate(ctx, set, catchall->elem);
 5777		if (ret < 0)
 5778			break;
 5779	}
 5780
 5781	return ret;
 5782}
 5783
 5784int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
 5785		       struct nft_set_binding *binding)
 5786{
 5787	struct nft_set_binding *i;
 5788	struct nft_set_iter iter = {
 5789		.genmask	= nft_genmask_next(ctx->net),
 5790		.type		= NFT_ITER_UPDATE,
 5791		.fn		= nf_tables_bind_check_setelem,
 5792	};
 5793
 5794	if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
 5795		return -EBUSY;
 5796
 5797	if (binding->flags & NFT_SET_MAP) {
 5798		/* If the set is already bound to the same chain all
 5799		 * jumps are already validated for that chain.
 5800		 */
 5801		list_for_each_entry(i, &set->bindings, list) {
 5802			if (i->flags & NFT_SET_MAP &&
 5803			    i->chain == binding->chain)
 5804				goto bind;
 5805		}
 5806
 5807		set->ops->walk(ctx, set, &iter);
 5808		if (!iter.err)
 5809			iter.err = nft_set_catchall_bind_check(ctx, set);
 5810
 5811		if (iter.err < 0)
 5812			return iter.err;
 5813	}
 5814bind:
 5815	if (!nft_use_inc(&set->use))
 5816		return -EMFILE;
 5817
 5818	binding->chain = ctx->chain;
 5819	list_add_tail_rcu(&binding->list, &set->bindings);
 5820	nft_set_trans_bind(ctx, set);
 5821
 5822	return 0;
 5823}
 5824EXPORT_SYMBOL_GPL(nf_tables_bind_set);
 5825
 5826static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
 5827				 struct nft_set_binding *binding, bool event)
 5828{
 5829	list_del_rcu(&binding->list);
 5830
 5831	if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
 5832		list_del_rcu(&set->list);
 5833		set->dead = 1;
 5834		if (event)
 5835			nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
 5836					     GFP_KERNEL);
 5837	}
 5838}
 5839
 5840static void nft_setelem_data_activate(const struct net *net,
 5841				      const struct nft_set *set,
 5842				      struct nft_elem_priv *elem_priv);
 5843
 5844static int nft_mapelem_activate(const struct nft_ctx *ctx,
 5845				struct nft_set *set,
 5846				const struct nft_set_iter *iter,
 5847				struct nft_elem_priv *elem_priv)
 5848{
 5849	struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 5850
 5851	/* called from abort path, reverse check to undo changes. */
 5852	if (nft_set_elem_active(ext, iter->genmask))
 5853		return 0;
 5854
 5855	nft_clear(ctx->net, ext);
 5856	nft_setelem_data_activate(ctx->net, set, elem_priv);
 5857
 5858	return 0;
 5859}
 5860
 5861static void nft_map_catchall_activate(const struct nft_ctx *ctx,
 5862				      struct nft_set *set)
 5863{
 5864	u8 genmask = nft_genmask_next(ctx->net);
 5865	struct nft_set_elem_catchall *catchall;
 5866	struct nft_set_ext *ext;
 5867
 5868	list_for_each_entry(catchall, &set->catchall_list, list) {
 5869		ext = nft_set_elem_ext(set, catchall->elem);
 5870		if (nft_set_elem_active(ext, genmask))
 5871			continue;
 5872
 5873		nft_clear(ctx->net, ext);
 5874		nft_setelem_data_activate(ctx->net, set, catchall->elem);
 5875	}
 5876}
 5877
 5878static void nft_map_activate(const struct nft_ctx *ctx, struct nft_set *set)
 5879{
 5880	struct nft_set_iter iter = {
 5881		.genmask	= nft_genmask_next(ctx->net),
 5882		.type		= NFT_ITER_UPDATE,
 5883		.fn		= nft_mapelem_activate,
 5884	};
 5885
 5886	set->ops->walk(ctx, set, &iter);
 5887	WARN_ON_ONCE(iter.err);
 5888
 5889	nft_map_catchall_activate(ctx, set);
 5890}
 5891
 5892void nf_tables_activate_set(const struct nft_ctx *ctx, struct nft_set *set)
 5893{
 5894	if (nft_set_is_anonymous(set)) {
 5895		if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 5896			nft_map_activate(ctx, set);
 5897
 5898		nft_clear(ctx->net, set);
 5899	}
 5900
 5901	nft_use_inc_restore(&set->use);
 5902}
 5903EXPORT_SYMBOL_GPL(nf_tables_activate_set);
 5904
 5905void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
 5906			      struct nft_set_binding *binding,
 5907			      enum nft_trans_phase phase)
 5908{
 5909	WARN_ON_ONCE(!lockdep_commit_lock_is_held(ctx->net));
 5910
 5911	switch (phase) {
 5912	case NFT_TRANS_PREPARE_ERROR:
 5913		nft_set_trans_unbind(ctx, set);
 5914		if (nft_set_is_anonymous(set))
 5915			nft_deactivate_next(ctx->net, set);
 5916		else
 5917			list_del_rcu(&binding->list);
 5918
 5919		nft_use_dec(&set->use);
 5920		break;
 5921	case NFT_TRANS_PREPARE:
 5922		if (nft_set_is_anonymous(set)) {
 5923			if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 5924				nft_map_deactivate(ctx, set);
 5925
 5926			nft_deactivate_next(ctx->net, set);
 5927		}
 5928		nft_use_dec(&set->use);
 5929		return;
 5930	case NFT_TRANS_ABORT:
 5931	case NFT_TRANS_RELEASE:
 5932		if (nft_set_is_anonymous(set) &&
 5933		    set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
 5934			nft_map_deactivate(ctx, set);
 5935
 5936		nft_use_dec(&set->use);
 5937		fallthrough;
 5938	default:
 5939		nf_tables_unbind_set(ctx, set, binding,
 5940				     phase == NFT_TRANS_COMMIT);
 5941	}
 5942}
 5943EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
 5944
 5945void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
 5946{
 5947	if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
 5948		nft_set_destroy(ctx, set);
 5949}
 5950EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
 5951
 5952const struct nft_set_ext_type nft_set_ext_types[] = {
 5953	[NFT_SET_EXT_KEY]		= {
 5954		.align	= __alignof__(u32),
 5955	},
 5956	[NFT_SET_EXT_DATA]		= {
 5957		.align	= __alignof__(u32),
 5958	},
 5959	[NFT_SET_EXT_EXPRESSIONS]	= {
 5960		.align	= __alignof__(struct nft_set_elem_expr),
 5961	},
 5962	[NFT_SET_EXT_OBJREF]		= {
 5963		.len	= sizeof(struct nft_object *),
 5964		.align	= __alignof__(struct nft_object *),
 5965	},
 5966	[NFT_SET_EXT_FLAGS]		= {
 5967		.len	= sizeof(u8),
 5968		.align	= __alignof__(u8),
 5969	},
 5970	[NFT_SET_EXT_TIMEOUT]		= {
 5971		.len	= sizeof(struct nft_timeout),
 5972		.align	= __alignof__(struct nft_timeout),
 5973	},
 5974	[NFT_SET_EXT_USERDATA]		= {
 5975		.len	= sizeof(struct nft_userdata),
 5976		.align	= __alignof__(struct nft_userdata),
 5977	},
 5978	[NFT_SET_EXT_KEY_END]		= {
 5979		.align	= __alignof__(u32),
 5980	},
 5981};
 5982
 5983/*
 5984 * Set elements
 5985 */
 5986
 5987static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
 5988	[NFTA_SET_ELEM_KEY]		= { .type = NLA_NESTED },
 5989	[NFTA_SET_ELEM_DATA]		= { .type = NLA_NESTED },
 5990	[NFTA_SET_ELEM_FLAGS]		= { .type = NLA_U32 },
 5991	[NFTA_SET_ELEM_TIMEOUT]		= { .type = NLA_U64 },
 5992	[NFTA_SET_ELEM_EXPIRATION]	= { .type = NLA_U64 },
 5993	[NFTA_SET_ELEM_USERDATA]	= { .type = NLA_BINARY,
 5994					    .len = NFT_USERDATA_MAXLEN },
 5995	[NFTA_SET_ELEM_EXPR]		= { .type = NLA_NESTED },
 5996	[NFTA_SET_ELEM_OBJREF]		= { .type = NLA_STRING,
 5997					    .len = NFT_OBJ_MAXNAMELEN - 1 },
 5998	[NFTA_SET_ELEM_KEY_END]		= { .type = NLA_NESTED },
 5999	[NFTA_SET_ELEM_EXPRESSIONS]	= NLA_POLICY_NESTED_ARRAY(nft_expr_policy),
 6000};
 6001
 6002static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
 6003	[NFTA_SET_ELEM_LIST_TABLE]	= { .type = NLA_STRING,
 6004					    .len = NFT_TABLE_MAXNAMELEN - 1 },
 6005	[NFTA_SET_ELEM_LIST_SET]	= { .type = NLA_STRING,
 6006					    .len = NFT_SET_MAXNAMELEN - 1 },
 6007	[NFTA_SET_ELEM_LIST_ELEMENTS]	= NLA_POLICY_NESTED_ARRAY(nft_set_elem_policy),
 6008	[NFTA_SET_ELEM_LIST_SET_ID]	= { .type = NLA_U32 },
 6009};
 6010
 6011static int nft_set_elem_expr_dump(struct sk_buff *skb,
 6012				  const struct nft_set *set,
 6013				  const struct nft_set_ext *ext,
 6014				  bool reset)
 6015{
 6016	struct nft_set_elem_expr *elem_expr;
 6017	u32 size, num_exprs = 0;
 6018	struct nft_expr *expr;
 6019	struct nlattr *nest;
 6020
 6021	elem_expr = nft_set_ext_expr(ext);
 6022	nft_setelem_expr_foreach(expr, elem_expr, size)
 6023		num_exprs++;
 6024
 6025	if (num_exprs == 1) {
 6026		expr = nft_setelem_expr_at(elem_expr, 0);
 6027		if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr, reset) < 0)
 6028			return -1;
 6029
 6030		return 0;
 6031	} else if (num_exprs > 1) {
 6032		nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
 6033		if (nest == NULL)
 6034			goto nla_put_failure;
 6035
 6036		nft_setelem_expr_foreach(expr, elem_expr, size) {
 6037			expr = nft_setelem_expr_at(elem_expr, size);
 6038			if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr, reset) < 0)
 6039				goto nla_put_failure;
 6040		}
 6041		nla_nest_end(skb, nest);
 6042	}
 6043	return 0;
 6044
 6045nla_put_failure:
 6046	return -1;
 6047}
 6048
 6049static int nf_tables_fill_setelem(struct sk_buff *skb,
 6050				  const struct nft_set *set,
 6051				  const struct nft_elem_priv *elem_priv,
 6052				  bool reset)
 6053{
 6054	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 6055	unsigned char *b = skb_tail_pointer(skb);
 6056	struct nlattr *nest;
 6057
 6058	nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
 6059	if (nest == NULL)
 6060		goto nla_put_failure;
 6061
 6062	if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
 6063	    nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
 6064			  NFT_DATA_VALUE, set->klen) < 0)
 6065		goto nla_put_failure;
 6066
 6067	if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
 6068	    nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
 6069			  NFT_DATA_VALUE, set->klen) < 0)
 6070		goto nla_put_failure;
 6071
 6072	if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
 6073	    nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
 6074			  nft_set_datatype(set), set->dlen) < 0)
 6075		goto nla_put_failure;
 6076
 6077	if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
 6078	    nft_set_elem_expr_dump(skb, set, ext, reset))
 6079		goto nla_put_failure;
 6080
 6081	if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
 6082	    nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
 6083			   (*nft_set_ext_obj(ext))->key.name) < 0)
 6084		goto nla_put_failure;
 6085
 6086	if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
 6087	    nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
 6088		         htonl(*nft_set_ext_flags(ext))))
 6089		goto nla_put_failure;
 6090
 6091	if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT)) {
 6092		u64 timeout = READ_ONCE(nft_set_ext_timeout(ext)->timeout);
 6093		u64 set_timeout = READ_ONCE(set->timeout);
 6094		__be64 msecs = 0;
 6095
 6096		if (set_timeout != timeout) {
 6097			msecs = nf_jiffies64_to_msecs(timeout);
 6098			if (nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT, msecs,
 6099					 NFTA_SET_ELEM_PAD))
 6100				goto nla_put_failure;
 6101		}
 6102
 6103		if (timeout > 0) {
 6104			u64 expires, now = get_jiffies_64();
 6105
 6106			expires = READ_ONCE(nft_set_ext_timeout(ext)->expiration);
 6107			if (time_before64(now, expires))
 6108				expires -= now;
 6109			else
 6110				expires = 0;
 6111
 6112			if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
 6113					 nf_jiffies64_to_msecs(expires),
 6114					 NFTA_SET_ELEM_PAD))
 6115				goto nla_put_failure;
 6116		}
 6117	}
 6118
 6119	if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
 6120		struct nft_userdata *udata;
 6121
 6122		udata = nft_set_ext_userdata(ext);
 6123		if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
 6124			    udata->len + 1, udata->data))
 6125			goto nla_put_failure;
 6126	}
 6127
 6128	nla_nest_end(skb, nest);
 6129	return 0;
 6130
 6131nla_put_failure:
 6132	nlmsg_trim(skb, b);
 6133	return -EMSGSIZE;
 6134}
 6135
 6136struct nft_set_dump_args {
 6137	const struct netlink_callback	*cb;
 6138	struct nft_set_iter		iter;
 6139	struct sk_buff			*skb;
 6140	bool				reset;
 6141};
 6142
 6143static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
 6144				  struct nft_set *set,
 6145				  const struct nft_set_iter *iter,
 6146				  struct nft_elem_priv *elem_priv)
 6147{
 6148	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 6149	struct nft_set_dump_args *args;
 6150
 6151	if (!nft_set_elem_active(ext, iter->genmask))
 6152		return 0;
 6153
 6154	if (nft_set_elem_expired(ext) || nft_set_elem_is_dead(ext))
 6155		return 0;
 6156
 6157	args = container_of(iter, struct nft_set_dump_args, iter);
 6158	return nf_tables_fill_setelem(args->skb, set, elem_priv, args->reset);
 6159}
 6160
 6161static void audit_log_nft_set_reset(const struct nft_table *table,
 6162				    unsigned int base_seq,
 6163				    unsigned int nentries)
 6164{
 6165	char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
 6166
 6167	audit_log_nfcfg(buf, table->family, nentries,
 6168			AUDIT_NFT_OP_SETELEM_RESET, GFP_ATOMIC);
 6169	kfree(buf);
 6170}
 6171
 6172struct nft_set_dump_ctx {
 6173	const struct nft_set	*set;
 6174	struct nft_ctx		ctx;
 6175	bool			reset;
 6176};
 6177
 6178static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
 6179				 const struct nft_set *set, bool reset,
 6180				 unsigned int base_seq)
 6181{
 6182	struct nft_set_elem_catchall *catchall;
 6183	u8 genmask = nft_genmask_cur(net);
 6184	struct nft_set_ext *ext;
 6185	int ret = 0;
 6186
 6187	list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
 6188		ext = nft_set_elem_ext(set, catchall->elem);
 6189		if (!nft_set_elem_active(ext, genmask) ||
 6190		    nft_set_elem_expired(ext))
 6191			continue;
 6192
 6193		ret = nf_tables_fill_setelem(skb, set, catchall->elem, reset);
 6194		if (reset && !ret)
 6195			audit_log_nft_set_reset(set->table, base_seq, 1);
 6196		break;
 6197	}
 6198
 6199	return ret;
 6200}
 6201
 6202static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
 6203{
 6204	struct nft_set_dump_ctx *dump_ctx = cb->data;
 6205	struct net *net = sock_net(skb->sk);
 6206	struct nftables_pernet *nft_net;
 6207	struct nft_table *table;
 6208	struct nft_set *set;
 6209	struct nft_set_dump_args args = {
 6210		.cb = cb,
 6211		.skb = skb,
 6212		.reset = dump_ctx->reset,
 6213		.iter = {
 6214			.genmask = nft_genmask_cur(net),
 6215			.type = NFT_ITER_READ,
 6216			.skip = cb->args[0],
 6217			.fn = nf_tables_dump_setelem,
 6218		},
 6219	};
 6220	bool set_found = false;
 6221	struct nlmsghdr *nlh;
 6222	struct nlattr *nest;
 6223	u32 portid, seq;
 6224	int event;
 6225
 6226	rcu_read_lock();
 6227	nft_net = nft_pernet(net);
 6228	cb->seq = nft_base_seq(net);
 6229
 6230	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 6231		if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
 6232		    dump_ctx->ctx.family != table->family)
 6233			continue;
 6234
 6235		if (table != dump_ctx->ctx.table)
 6236			continue;
 6237
 6238		list_for_each_entry_rcu(set, &table->sets, list) {
 6239			if (set == dump_ctx->set) {
 6240				set_found = true;
 6241				break;
 6242			}
 6243		}
 6244		break;
 6245	}
 6246
 6247	if (!set_found) {
 6248		rcu_read_unlock();
 6249		return -ENOENT;
 6250	}
 6251
 6252	event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
 6253	portid = NETLINK_CB(cb->skb).portid;
 6254	seq    = cb->nlh->nlmsg_seq;
 6255
 6256	nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
 6257			   table->family, NFNETLINK_V0, nft_base_seq_be16(net));
 6258	if (!nlh)
 6259		goto nla_put_failure;
 6260
 6261	if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
 6262		goto nla_put_failure;
 6263	if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
 6264		goto nla_put_failure;
 6265
 6266	nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
 6267	if (nest == NULL)
 6268		goto nla_put_failure;
 6269
 6270	set->ops->walk(&dump_ctx->ctx, set, &args.iter);
 6271
 6272	if (!args.iter.err && args.iter.count == cb->args[0])
 6273		args.iter.err = nft_set_catchall_dump(net, skb, set,
 6274						      dump_ctx->reset, cb->seq);
 6275	nla_nest_end(skb, nest);
 6276	nlmsg_end(skb, nlh);
 6277
 6278	if (dump_ctx->reset && args.iter.count > args.iter.skip)
 6279		audit_log_nft_set_reset(table, cb->seq,
 6280					args.iter.count - args.iter.skip);
 6281
 6282	rcu_read_unlock();
 6283
 6284	if (args.iter.err && args.iter.err != -EMSGSIZE)
 6285		return args.iter.err;
 6286	if (args.iter.count == cb->args[0])
 6287		return 0;
 6288
 6289	cb->args[0] = args.iter.count;
 6290	return skb->len;
 6291
 6292nla_put_failure:
 6293	rcu_read_unlock();
 6294	return -ENOSPC;
 6295}
 6296
 6297static int nf_tables_dump_set_start(struct netlink_callback *cb)
 6298{
 6299	struct nft_set_dump_ctx *dump_ctx = cb->data;
 6300
 6301	cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
 6302
 6303	return cb->data ? 0 : -ENOMEM;
 6304}
 6305
 6306static int nf_tables_dump_set_done(struct netlink_callback *cb)
 6307{
 6308	kfree(cb->data);
 6309	return 0;
 6310}
 6311
 6312static int nf_tables_fill_setelem_info(struct sk_buff *skb,
 6313				       const struct nft_ctx *ctx, u32 seq,
 6314				       u32 portid, int event, u16 flags,
 6315				       const struct nft_set *set,
 6316				       const struct nft_elem_priv *elem_priv,
 6317				       bool reset)
 6318{
 6319	struct nlmsghdr *nlh;
 6320	struct nlattr *nest;
 6321	int err;
 6322
 6323	event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 6324	nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
 6325			   NFNETLINK_V0, nft_base_seq_be16(ctx->net));
 6326	if (!nlh)
 6327		goto nla_put_failure;
 6328
 6329	if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
 6330		goto nla_put_failure;
 6331	if (nla_put_string(skb, NFTA_SET_NAME, set->name))
 6332		goto nla_put_failure;
 6333
 6334	nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
 6335	if (nest == NULL)
 6336		goto nla_put_failure;
 6337
 6338	err = nf_tables_fill_setelem(skb, set, elem_priv, reset);
 6339	if (err < 0)
 6340		goto nla_put_failure;
 6341
 6342	nla_nest_end(skb, nest);
 6343
 6344	nlmsg_end(skb, nlh);
 6345	return 0;
 6346
 6347nla_put_failure:
 6348	nlmsg_trim(skb, nlh);
 6349	return -1;
 6350}
 6351
 6352static int nft_setelem_parse_flags(const struct nft_set *set,
 6353				   const struct nlattr *attr, u32 *flags)
 6354{
 6355	if (attr == NULL)
 6356		return 0;
 6357
 6358	*flags = ntohl(nla_get_be32(attr));
 6359	if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
 6360		return -EOPNOTSUPP;
 6361	if (!(set->flags & NFT_SET_INTERVAL) &&
 6362	    *flags & NFT_SET_ELEM_INTERVAL_END)
 6363		return -EINVAL;
 6364	if ((*flags & (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL)) ==
 6365	    (NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
 6366		return -EINVAL;
 6367
 6368	return 0;
 6369}
 6370
 6371static int nft_setelem_parse_key(struct nft_ctx *ctx, const struct nft_set *set,
 6372				 struct nft_data *key, struct nlattr *attr)
 6373{
 6374	struct nft_data_desc desc = {
 6375		.type	= NFT_DATA_VALUE,
 6376		.size	= NFT_DATA_VALUE_MAXLEN,
 6377		.len	= set->klen,
 6378	};
 6379
 6380	return nft_data_init(ctx, key, &desc, attr);
 6381}
 6382
 6383static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
 6384				  struct nft_data_desc *desc,
 6385				  struct nft_data *data,
 6386				  struct nlattr *attr)
 6387{
 6388	u32 dtype;
 6389
 6390	if (set->dtype == NFT_DATA_VERDICT)
 6391		dtype = NFT_DATA_VERDICT;
 6392	else
 6393		dtype = NFT_DATA_VALUE;
 6394
 6395	desc->type = dtype;
 6396	desc->size = NFT_DATA_VALUE_MAXLEN;
 6397	desc->len = set->dlen;
 6398	desc->flags = NFT_DATA_DESC_SETELEM;
 6399
 6400	return nft_data_init(ctx, data, desc, attr);
 6401}
 6402
 6403static void *nft_setelem_catchall_get(const struct net *net,
 6404				      const struct nft_set *set)
 6405{
 6406	struct nft_set_elem_catchall *catchall;
 6407	u8 genmask = nft_genmask_cur(net);
 6408	struct nft_set_ext *ext;
 6409	void *priv = NULL;
 6410
 6411	list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
 6412		ext = nft_set_elem_ext(set, catchall->elem);
 6413		if (!nft_set_elem_active(ext, genmask) ||
 6414		    nft_set_elem_expired(ext))
 6415			continue;
 6416
 6417		priv = catchall->elem;
 6418		break;
 6419	}
 6420
 6421	return priv;
 6422}
 6423
 6424static int nft_setelem_get(struct nft_ctx *ctx, const struct nft_set *set,
 6425			   struct nft_set_elem *elem, u32 flags)
 6426{
 6427	void *priv;
 6428
 6429	if (!(flags & NFT_SET_ELEM_CATCHALL)) {
 6430		priv = set->ops->get(ctx->net, set, elem, flags);
 6431		if (IS_ERR(priv))
 6432			return PTR_ERR(priv);
 6433	} else {
 6434		priv = nft_setelem_catchall_get(ctx->net, set);
 6435		if (!priv)
 6436			return -ENOENT;
 6437	}
 6438	elem->priv = priv;
 6439
 6440	return 0;
 6441}
 6442
 6443static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set,
 6444			    const struct nlattr *attr, bool reset)
 6445{
 6446	struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
 6447	struct nft_set_elem elem;
 6448	struct sk_buff *skb;
 6449	uint32_t flags = 0;
 6450	int err;
 6451
 6452	err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
 6453					  nft_set_elem_policy, NULL);
 6454	if (err < 0)
 6455		return err;
 6456
 6457	err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
 6458	if (err < 0)
 6459		return err;
 6460
 6461	if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
 6462		return -EINVAL;
 6463
 6464	if (nla[NFTA_SET_ELEM_KEY]) {
 6465		err = nft_setelem_parse_key(ctx, set, &elem.key.val,
 6466					    nla[NFTA_SET_ELEM_KEY]);
 6467		if (err < 0)
 6468			return err;
 6469	}
 6470
 6471	if (nla[NFTA_SET_ELEM_KEY_END]) {
 6472		err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
 6473					    nla[NFTA_SET_ELEM_KEY_END]);
 6474		if (err < 0)
 6475			return err;
 6476	}
 6477
 6478	err = nft_setelem_get(ctx, set, &elem, flags);
 6479	if (err < 0)
 6480		return err;
 6481
 6482	err = -ENOMEM;
 6483	skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
 6484	if (skb == NULL)
 6485		return err;
 6486
 6487	err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
 6488					  NFT_MSG_NEWSETELEM, 0, set, elem.priv,
 6489					  reset);
 6490	if (err < 0)
 6491		goto err_fill_setelem;
 6492
 6493	return nfnetlink_unicast(skb, ctx->net, ctx->portid);
 6494
 6495err_fill_setelem:
 6496	kfree_skb(skb);
 6497	return err;
 6498}
 6499
 6500static int nft_set_dump_ctx_init(struct nft_set_dump_ctx *dump_ctx,
 6501				 const struct sk_buff *skb,
 6502				 const struct nfnl_info *info,
 6503				 const struct nlattr * const nla[],
 6504				 bool reset)
 6505{
 6506	struct netlink_ext_ack *extack = info->extack;
 6507	u8 genmask = nft_genmask_cur(info->net);
 6508	u8 family = info->nfmsg->nfgen_family;
 6509	struct net *net = info->net;
 6510	struct nft_table *table;
 6511	struct nft_set *set;
 6512
 6513	table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
 6514				 genmask, 0);
 6515	if (IS_ERR(table)) {
 6516		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
 6517		return PTR_ERR(table);
 6518	}
 6519
 6520	set = nft_set_lookup(net, table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
 6521	if (IS_ERR(set)) {
 6522		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
 6523		return PTR_ERR(set);
 6524	}
 6525
 6526	nft_ctx_init(&dump_ctx->ctx, net, skb,
 6527		     info->nlh, family, table, NULL, nla);
 6528	dump_ctx->set = set;
 6529	dump_ctx->reset = reset;
 6530	return 0;
 6531}
 6532
 6533/* called with rcu_read_lock held */
 6534static int nf_tables_getsetelem(struct sk_buff *skb,
 6535				const struct nfnl_info *info,
 6536				const struct nlattr * const nla[])
 6537{
 6538	struct netlink_ext_ack *extack = info->extack;
 6539	struct nft_set_dump_ctx dump_ctx;
 6540	int rem, err = 0, nelems = 0;
 6541	struct net *net = info->net;
 6542	struct nlattr *attr;
 6543	bool reset = false;
 6544
 6545	if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET)
 6546		reset = true;
 6547
 6548	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 6549		struct netlink_dump_control c = {
 6550			.start = nf_tables_dump_set_start,
 6551			.dump = nf_tables_dump_set,
 6552			.done = nf_tables_dump_set_done,
 6553			.module = THIS_MODULE,
 6554		};
 6555
 6556		err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset);
 6557		if (err)
 6558			return err;
 6559
 6560		c.data = &dump_ctx;
 6561		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 6562	}
 6563
 6564	if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
 6565		return -EINVAL;
 6566
 6567	err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset);
 6568	if (err)
 6569		return err;
 6570
 6571	nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
 6572		err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, reset);
 6573		if (err < 0) {
 6574			NL_SET_BAD_ATTR(extack, attr);
 6575			break;
 6576		}
 6577		nelems++;
 6578	}
 6579	if (reset)
 6580		audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(net),
 6581					nelems);
 6582
 6583	return err;
 6584}
 6585
 6586static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
 6587				     const struct nft_set *set,
 6588				     const struct nft_elem_priv *elem_priv,
 6589				     int event)
 6590{
 6591	struct nftables_pernet *nft_net;
 6592	struct net *net = ctx->net;
 6593	u32 portid = ctx->portid;
 6594	struct sk_buff *skb;
 6595	u16 flags = 0;
 6596	int err;
 6597
 6598	if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
 6599		return;
 6600
 6601	skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 6602	if (skb == NULL)
 6603		goto err;
 6604
 6605	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 6606		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 6607
 6608	err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
 6609					  set, elem_priv, false);
 6610	if (err < 0) {
 6611		kfree_skb(skb);
 6612		goto err;
 6613	}
 6614
 6615	nft_net = nft_pernet(net);
 6616	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 6617	return;
 6618err:
 6619	nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
 6620}
 6621
 6622static struct nft_trans *nft_trans_elem_alloc(const struct nft_ctx *ctx,
 6623					      int msg_type,
 6624					      struct nft_set *set)
 6625{
 6626	struct nft_trans_elem *te;
 6627	struct nft_trans *trans;
 6628
 6629	trans = nft_trans_alloc(ctx, msg_type, struct_size(te, elems, 1));
 6630	if (trans == NULL)
 6631		return NULL;
 6632
 6633	te = nft_trans_container_elem(trans);
 6634	te->nelems = 1;
 6635	te->set = set;
 6636
 6637	return trans;
 6638}
 6639
 6640struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
 6641					 const struct nft_set *set,
 6642					 const struct nlattr *attr)
 6643{
 6644	struct nft_expr *expr;
 6645	int err;
 6646
 6647	expr = nft_expr_init(ctx, attr);
 6648	if (IS_ERR(expr))
 6649		return expr;
 6650
 6651	err = -EOPNOTSUPP;
 6652	if (expr->ops->type->flags & NFT_EXPR_GC) {
 6653		if (set->flags & NFT_SET_TIMEOUT)
 6654			goto err_set_elem_expr;
 6655		if (!set->ops->gc_init)
 6656			goto err_set_elem_expr;
 6657		set->ops->gc_init(set);
 6658	}
 6659
 6660	return expr;
 6661
 6662err_set_elem_expr:
 6663	nft_expr_destroy(ctx, expr);
 6664	return ERR_PTR(err);
 6665}
 6666
 6667static int nft_set_ext_check(const struct nft_set_ext_tmpl *tmpl, u8 id, u32 len)
 6668{
 6669	len += nft_set_ext_types[id].len;
 6670	if (len > tmpl->ext_len[id] ||
 6671	    len > U8_MAX)
 6672		return -1;
 6673
 6674	return 0;
 6675}
 6676
 6677static int nft_set_ext_memcpy(const struct nft_set_ext_tmpl *tmpl, u8 id,
 6678			      void *to, const void *from, u32 len)
 6679{
 6680	if (nft_set_ext_check(tmpl, id, len) < 0)
 6681		return -1;
 6682
 6683	memcpy(to, from, len);
 6684
 6685	return 0;
 6686}
 6687
 6688struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set,
 6689					const struct nft_set_ext_tmpl *tmpl,
 6690					const u32 *key, const u32 *key_end,
 6691					const u32 *data,
 6692					u64 timeout, u64 expiration, gfp_t gfp)
 6693{
 6694	struct nft_set_ext *ext;
 6695	void *elem;
 6696
 6697	elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
 6698	if (elem == NULL)
 6699		return ERR_PTR(-ENOMEM);
 6700
 6701	ext = nft_set_elem_ext(set, elem);
 6702	nft_set_ext_init(ext, tmpl);
 6703
 6704	if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
 6705	    nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY,
 6706			       nft_set_ext_key(ext), key, set->klen) < 0)
 6707		goto err_ext_check;
 6708
 6709	if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
 6710	    nft_set_ext_memcpy(tmpl, NFT_SET_EXT_KEY_END,
 6711			       nft_set_ext_key_end(ext), key_end, set->klen) < 0)
 6712		goto err_ext_check;
 6713
 6714	if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
 6715	    nft_set_ext_memcpy(tmpl, NFT_SET_EXT_DATA,
 6716			       nft_set_ext_data(ext), data, set->dlen) < 0)
 6717		goto err_ext_check;
 6718
 6719	if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT)) {
 6720		nft_set_ext_timeout(ext)->timeout = timeout;
 6721
 6722		if (expiration == 0)
 6723			expiration = timeout;
 6724
 6725		nft_set_ext_timeout(ext)->expiration = get_jiffies_64() + expiration;
 6726	}
 6727
 6728	return elem;
 6729
 6730err_ext_check:
 6731	kfree(elem);
 6732
 6733	return ERR_PTR(-EINVAL);
 6734}
 6735
 6736static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
 6737					struct nft_expr *expr)
 6738{
 6739	if (expr->ops->destroy_clone) {
 6740		expr->ops->destroy_clone(ctx, expr);
 6741		module_put(expr->ops->type->owner);
 6742	} else {
 6743		nf_tables_expr_destroy(ctx, expr);
 6744	}
 6745}
 6746
 6747void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
 6748			       struct nft_set_elem_expr *elem_expr)
 6749{
 6750	struct nft_expr *expr;
 6751	u32 size;
 6752
 6753	nft_setelem_expr_foreach(expr, elem_expr, size)
 6754		__nft_set_elem_expr_destroy(ctx, expr);
 6755}
 6756
 6757/* Drop references and destroy. Called from gc, dynset and abort path. */
 6758static void __nft_set_elem_destroy(const struct nft_ctx *ctx,
 6759				   const struct nft_set *set,
 6760				   const struct nft_elem_priv *elem_priv,
 6761				   bool destroy_expr)
 6762{
 6763	struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 6764
 6765	nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
 6766	if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
 6767		nft_data_release(nft_set_ext_data(ext), set->dtype);
 6768	if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
 6769		nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
 6770	if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
 6771		nft_use_dec(&(*nft_set_ext_obj(ext))->use);
 6772
 6773	kfree(elem_priv);
 6774}
 6775
 6776/* Drop references and destroy. Called from gc and dynset. */
 6777void nft_set_elem_destroy(const struct nft_set *set,
 6778			  const struct nft_elem_priv *elem_priv,
 6779			  bool destroy_expr)
 6780{
 6781	struct nft_ctx ctx = {
 6782		.net	= read_pnet(&set->net),
 6783		.family	= set->table->family,
 6784	};
 6785
 6786	__nft_set_elem_destroy(&ctx, set, elem_priv, destroy_expr);
 6787}
 6788EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
 6789
 6790/* Drop references and destroy. Called from abort path. */
 6791static void nft_trans_set_elem_destroy(const struct nft_ctx *ctx, struct nft_trans_elem *te)
 6792{
 6793	int i;
 6794
 6795	for (i = 0; i < te->nelems; i++) {
 6796		/* skip update request, see nft_trans_elems_new_abort() */
 6797		if (!te->elems[i].priv)
 6798			continue;
 6799
 6800		__nft_set_elem_destroy(ctx, te->set, te->elems[i].priv, true);
 6801	}
 6802}
 6803
 6804/* Destroy element. References have been already dropped in the preparation
 6805 * path via nft_setelem_data_deactivate().
 6806 */
 6807void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
 6808				const struct nft_set *set,
 6809				const struct nft_elem_priv *elem_priv)
 6810{
 6811	struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 6812
 6813	if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
 6814		nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
 6815
 6816	kfree(elem_priv);
 6817}
 6818
 6819static void nft_trans_elems_destroy(const struct nft_ctx *ctx,
 6820				    const struct nft_trans_elem *te)
 6821{
 6822	int i;
 6823
 6824	for (i = 0; i < te->nelems; i++)
 6825		nf_tables_set_elem_destroy(ctx, te->set, te->elems[i].priv);
 6826}
 6827
 6828int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
 6829			    struct nft_expr *expr_array[])
 6830{
 6831	struct nft_expr *expr;
 6832	int err, i, k;
 6833
 6834	for (i = 0; i < set->num_exprs; i++) {
 6835		expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
 6836		if (!expr)
 6837			goto err_expr;
 6838
 6839		err = nft_expr_clone(expr, set->exprs[i], GFP_KERNEL_ACCOUNT);
 6840		if (err < 0) {
 6841			kfree(expr);
 6842			goto err_expr;
 6843		}
 6844		expr_array[i] = expr;
 6845	}
 6846
 6847	return 0;
 6848
 6849err_expr:
 6850	for (k = i - 1; k >= 0; k--)
 6851		nft_expr_destroy(ctx, expr_array[k]);
 6852
 6853	return -ENOMEM;
 6854}
 6855
 6856static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
 6857				   const struct nft_set_ext_tmpl *tmpl,
 6858				   const struct nft_set_ext *ext,
 6859				   struct nft_expr *expr_array[],
 6860				   u32 num_exprs)
 6861{
 6862	struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
 6863	u32 len = sizeof(struct nft_set_elem_expr);
 6864	struct nft_expr *expr;
 6865	int i, err;
 6866
 6867	if (num_exprs == 0)
 6868		return 0;
 6869
 6870	for (i = 0; i < num_exprs; i++)
 6871		len += expr_array[i]->ops->size;
 6872
 6873	if (nft_set_ext_check(tmpl, NFT_SET_EXT_EXPRESSIONS, len) < 0)
 6874		return -EINVAL;
 6875
 6876	for (i = 0; i < num_exprs; i++) {
 6877		expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
 6878		err = nft_expr_clone(expr, expr_array[i], GFP_KERNEL_ACCOUNT);
 6879		if (err < 0)
 6880			goto err_elem_expr_setup;
 6881
 6882		elem_expr->size += expr_array[i]->ops->size;
 6883		nft_expr_destroy(ctx, expr_array[i]);
 6884		expr_array[i] = NULL;
 6885	}
 6886
 6887	return 0;
 6888
 6889err_elem_expr_setup:
 6890	for (; i < num_exprs; i++) {
 6891		nft_expr_destroy(ctx, expr_array[i]);
 6892		expr_array[i] = NULL;
 6893	}
 6894
 6895	return -ENOMEM;
 6896}
 6897
 6898struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
 6899					    const struct nft_set *set)
 6900{
 6901	struct nft_set_elem_catchall *catchall;
 6902	u8 genmask = nft_genmask_cur(net);
 6903	struct nft_set_ext *ext;
 6904
 6905	list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
 6906		ext = nft_set_elem_ext(set, catchall->elem);
 6907		if (nft_set_elem_active(ext, genmask) &&
 6908		    !nft_set_elem_expired(ext) &&
 6909		    !nft_set_elem_is_dead(ext))
 6910			return ext;
 6911	}
 6912
 6913	return NULL;
 6914}
 6915EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
 6916
 6917static int nft_setelem_catchall_insert(const struct net *net,
 6918				       struct nft_set *set,
 6919				       const struct nft_set_elem *elem,
 6920				       struct nft_elem_priv **priv)
 6921{
 6922	struct nft_set_elem_catchall *catchall;
 6923	u8 genmask = nft_genmask_next(net);
 6924	struct nft_set_ext *ext;
 6925
 6926	list_for_each_entry(catchall, &set->catchall_list, list) {
 6927		ext = nft_set_elem_ext(set, catchall->elem);
 6928		if (nft_set_elem_active(ext, genmask)) {
 6929			*priv = catchall->elem;
 6930			return -EEXIST;
 6931		}
 6932	}
 6933
 6934	catchall = kmalloc(sizeof(*catchall), GFP_KERNEL_ACCOUNT);
 6935	if (!catchall)
 6936		return -ENOMEM;
 6937
 6938	catchall->elem = elem->priv;
 6939	list_add_tail_rcu(&catchall->list, &set->catchall_list);
 6940
 6941	return 0;
 6942}
 6943
 6944static int nft_setelem_insert(const struct net *net,
 6945			      struct nft_set *set,
 6946			      const struct nft_set_elem *elem,
 6947			      struct nft_elem_priv **elem_priv,
 6948			      unsigned int flags)
 6949{
 6950	int ret;
 6951
 6952	if (flags & NFT_SET_ELEM_CATCHALL)
 6953		ret = nft_setelem_catchall_insert(net, set, elem, elem_priv);
 6954	else
 6955		ret = set->ops->insert(net, set, elem, elem_priv);
 6956
 6957	return ret;
 6958}
 6959
 6960static bool nft_setelem_is_catchall(const struct nft_set *set,
 6961				    const struct nft_elem_priv *elem_priv)
 6962{
 6963	struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 6964
 6965	if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
 6966	    *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
 6967		return true;
 6968
 6969	return false;
 6970}
 6971
 6972static void nft_setelem_activate(struct net *net, struct nft_set *set,
 6973				 struct nft_elem_priv *elem_priv)
 6974{
 6975	struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 6976
 6977	if (nft_setelem_is_catchall(set, elem_priv)) {
 6978		nft_clear(net, ext);
 6979	} else {
 6980		set->ops->activate(net, set, elem_priv);
 6981	}
 6982}
 6983
 6984static void nft_trans_elem_update(const struct nft_set *set,
 6985				  const struct nft_trans_one_elem *elem)
 6986{
 6987	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
 6988	const struct nft_elem_update *update = elem->update;
 6989
 6990	if (update->flags & NFT_TRANS_UPD_TIMEOUT)
 6991		WRITE_ONCE(nft_set_ext_timeout(ext)->timeout, update->timeout);
 6992
 6993	if (update->flags & NFT_TRANS_UPD_EXPIRATION)
 6994		WRITE_ONCE(nft_set_ext_timeout(ext)->expiration, get_jiffies_64() + update->expiration);
 6995}
 6996
 6997static void nft_trans_elems_add(const struct nft_ctx *ctx,
 6998				struct nft_trans_elem *te)
 6999{
 7000	int i;
 7001
 7002	for (i = 0; i < te->nelems; i++) {
 7003		struct nft_trans_one_elem *elem = &te->elems[i];
 7004
 7005		if (elem->update)
 7006			nft_trans_elem_update(te->set, elem);
 7007		else
 7008			nft_setelem_activate(ctx->net, te->set, elem->priv);
 7009
 7010		nf_tables_setelem_notify(ctx, te->set, elem->priv,
 7011					 NFT_MSG_NEWSETELEM);
 7012		kfree(elem->update);
 7013	}
 7014}
 7015
 7016static int nft_setelem_catchall_deactivate(const struct net *net,
 7017					   struct nft_set *set,
 7018					   struct nft_set_elem *elem)
 7019{
 7020	struct nft_set_elem_catchall *catchall;
 7021	struct nft_set_ext *ext;
 7022
 7023	list_for_each_entry(catchall, &set->catchall_list, list) {
 7024		ext = nft_set_elem_ext(set, catchall->elem);
 7025		if (!nft_is_active_next(net, ext))
 7026			continue;
 7027
 7028		kfree(elem->priv);
 7029		elem->priv = catchall->elem;
 7030		nft_set_elem_change_active(net, set, ext);
 7031		return 0;
 7032	}
 7033
 7034	return -ENOENT;
 7035}
 7036
 7037static int __nft_setelem_deactivate(const struct net *net,
 7038				    struct nft_set *set,
 7039				    struct nft_set_elem *elem)
 7040{
 7041	void *priv;
 7042
 7043	priv = set->ops->deactivate(net, set, elem);
 7044	if (!priv)
 7045		return -ENOENT;
 7046
 7047	kfree(elem->priv);
 7048	elem->priv = priv;
 7049	set->ndeact++;
 7050
 7051	return 0;
 7052}
 7053
 7054static int nft_setelem_deactivate(const struct net *net,
 7055				  struct nft_set *set,
 7056				  struct nft_set_elem *elem, u32 flags)
 7057{
 7058	int ret;
 7059
 7060	if (flags & NFT_SET_ELEM_CATCHALL)
 7061		ret = nft_setelem_catchall_deactivate(net, set, elem);
 7062	else
 7063		ret = __nft_setelem_deactivate(net, set, elem);
 7064
 7065	return ret;
 7066}
 7067
 7068static void nft_setelem_catchall_destroy(struct nft_set_elem_catchall *catchall)
 7069{
 7070	list_del_rcu(&catchall->list);
 7071	kfree_rcu(catchall, rcu);
 7072}
 7073
 7074static void nft_setelem_catchall_remove(const struct net *net,
 7075					const struct nft_set *set,
 7076					struct nft_elem_priv *elem_priv)
 7077{
 7078	struct nft_set_elem_catchall *catchall, *next;
 7079
 7080	list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
 7081		if (catchall->elem == elem_priv) {
 7082			nft_setelem_catchall_destroy(catchall);
 7083			break;
 7084		}
 7085	}
 7086}
 7087
 7088static void nft_setelem_remove(const struct net *net,
 7089			       const struct nft_set *set,
 7090			       struct nft_elem_priv *elem_priv)
 7091{
 7092	if (nft_setelem_is_catchall(set, elem_priv))
 7093		nft_setelem_catchall_remove(net, set, elem_priv);
 7094	else
 7095		set->ops->remove(net, set, elem_priv);
 7096}
 7097
 7098static void nft_trans_elems_remove(const struct nft_ctx *ctx,
 7099				   const struct nft_trans_elem *te)
 7100{
 7101	int i;
 7102
 7103	for (i = 0; i < te->nelems; i++) {
 7104		WARN_ON_ONCE(te->elems[i].update);
 7105
 7106		nf_tables_setelem_notify(ctx, te->set,
 7107					 te->elems[i].priv,
 7108					 te->nft_trans.msg_type);
 7109
 7110		nft_setelem_remove(ctx->net, te->set, te->elems[i].priv);
 7111		if (!nft_setelem_is_catchall(te->set, te->elems[i].priv)) {
 7112			atomic_dec(&te->set->nelems);
 7113			te->set->ndeact--;
 7114		}
 7115	}
 7116}
 7117
 7118static bool nft_setelem_valid_key_end(const struct nft_set *set,
 7119				      struct nlattr **nla, u32 flags)
 7120{
 7121	if ((set->flags & (NFT_SET_CONCAT | NFT_SET_INTERVAL)) ==
 7122			  (NFT_SET_CONCAT | NFT_SET_INTERVAL)) {
 7123		if (flags & NFT_SET_ELEM_INTERVAL_END)
 7124			return false;
 7125
 7126		if (nla[NFTA_SET_ELEM_KEY_END] &&
 7127		    flags & NFT_SET_ELEM_CATCHALL)
 7128			return false;
 7129	} else {
 7130		if (nla[NFTA_SET_ELEM_KEY_END])
 7131			return false;
 7132	}
 7133
 7134	return true;
 7135}
 7136
 7137static u32 nft_set_maxsize(const struct nft_set *set)
 7138{
 7139	u32 maxsize, delta;
 7140
 7141	if (!set->size)
 7142		return UINT_MAX;
 7143
 7144	if (set->ops->adjust_maxsize)
 7145		delta = set->ops->adjust_maxsize(set);
 7146	else
 7147		delta = 0;
 7148
 7149	if (check_add_overflow(set->size, set->ndeact, &maxsize))
 7150		return UINT_MAX;
 7151
 7152	if (check_add_overflow(maxsize, delta, &maxsize))
 7153		return UINT_MAX;
 7154
 7155	return maxsize;
 7156}
 7157
 7158static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
 7159			    const struct nlattr *attr, u32 nlmsg_flags)
 7160{
 7161	struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
 7162	struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
 7163	u8 genmask = nft_genmask_next(ctx->net);
 7164	u32 flags = 0, size = 0, num_exprs = 0;
 7165	struct nft_set_ext_tmpl tmpl;
 7166	struct nft_set_ext *ext, *ext2;
 7167	struct nft_set_elem elem;
 7168	struct nft_set_binding *binding;
 7169	struct nft_elem_priv *elem_priv;
 7170	struct nft_object *obj = NULL;
 7171	struct nft_userdata *udata;
 7172	struct nft_data_desc desc;
 7173	enum nft_registers dreg;
 7174	struct nft_trans *trans;
 7175	bool set_full = false;
 7176	u64 expiration;
 7177	u64 timeout;
 7178	int err, i;
 7179	u8 ulen;
 7180
 7181	err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
 7182					  nft_set_elem_policy, NULL);
 7183	if (err < 0)
 7184		return err;
 7185
 7186	nft_set_ext_prepare(&tmpl);
 7187
 7188	err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
 7189	if (err < 0)
 7190		return err;
 7191
 7192	if (((flags & NFT_SET_ELEM_CATCHALL) && nla[NFTA_SET_ELEM_KEY]) ||
 7193	    (!(flags & NFT_SET_ELEM_CATCHALL) && !nla[NFTA_SET_ELEM_KEY]))
 7194		return -EINVAL;
 7195
 7196	if (flags != 0) {
 7197		err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
 7198		if (err < 0)
 7199			return err;
 7200	}
 7201
 7202	if (set->flags & NFT_SET_MAP) {
 7203		if (nla[NFTA_SET_ELEM_DATA] == NULL &&
 7204		    !(flags & NFT_SET_ELEM_INTERVAL_END))
 7205			return -EINVAL;
 7206	} else {
 7207		if (nla[NFTA_SET_ELEM_DATA] != NULL)
 7208			return -EINVAL;
 7209	}
 7210
 7211	if (set->flags & NFT_SET_OBJECT) {
 7212		if (!nla[NFTA_SET_ELEM_OBJREF] &&
 7213		    !(flags & NFT_SET_ELEM_INTERVAL_END))
 7214			return -EINVAL;
 7215	} else {
 7216		if (nla[NFTA_SET_ELEM_OBJREF])
 7217			return -EINVAL;
 7218	}
 7219
 7220	if (!nft_setelem_valid_key_end(set, nla, flags))
 7221		return -EINVAL;
 7222
 7223	if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
 7224	     (nla[NFTA_SET_ELEM_DATA] ||
 7225	      nla[NFTA_SET_ELEM_OBJREF] ||
 7226	      nla[NFTA_SET_ELEM_TIMEOUT] ||
 7227	      nla[NFTA_SET_ELEM_EXPIRATION] ||
 7228	      nla[NFTA_SET_ELEM_USERDATA] ||
 7229	      nla[NFTA_SET_ELEM_EXPR] ||
 7230	      nla[NFTA_SET_ELEM_KEY_END] ||
 7231	      nla[NFTA_SET_ELEM_EXPRESSIONS]))
 7232		return -EINVAL;
 7233
 7234	timeout = 0;
 7235	if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
 7236		if (!(set->flags & NFT_SET_TIMEOUT))
 7237			return -EINVAL;
 7238		err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
 7239					    &timeout);
 7240		if (err)
 7241			return err;
 7242	} else if (set->flags & NFT_SET_TIMEOUT &&
 7243		   !(flags & NFT_SET_ELEM_INTERVAL_END)) {
 7244		timeout = set->timeout;
 7245	}
 7246
 7247	expiration = 0;
 7248	if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
 7249		if (!(set->flags & NFT_SET_TIMEOUT))
 7250			return -EINVAL;
 7251		if (timeout == 0)
 7252			return -EOPNOTSUPP;
 7253
 7254		err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
 7255					    &expiration);
 7256		if (err)
 7257			return err;
 7258
 7259		if (expiration > timeout)
 7260			return -ERANGE;
 7261	}
 7262
 7263	if (nla[NFTA_SET_ELEM_EXPR]) {
 7264		struct nft_expr *expr;
 7265
 7266		if (set->num_exprs && set->num_exprs != 1)
 7267			return -EOPNOTSUPP;
 7268
 7269		expr = nft_set_elem_expr_alloc(ctx, set,
 7270					       nla[NFTA_SET_ELEM_EXPR]);
 7271		if (IS_ERR(expr))
 7272			return PTR_ERR(expr);
 7273
 7274		expr_array[0] = expr;
 7275		num_exprs = 1;
 7276
 7277		if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
 7278			err = -EOPNOTSUPP;
 7279			goto err_set_elem_expr;
 7280		}
 7281	} else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
 7282		struct nft_expr *expr;
 7283		struct nlattr *tmp;
 7284		int left;
 7285
 7286		i = 0;
 7287		nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
 7288			if (i == NFT_SET_EXPR_MAX ||
 7289			    (set->num_exprs && set->num_exprs == i)) {
 7290				err = -E2BIG;
 7291				goto err_set_elem_expr;
 7292			}
 7293			if (nla_type(tmp) != NFTA_LIST_ELEM) {
 7294				err = -EINVAL;
 7295				goto err_set_elem_expr;
 7296			}
 7297			expr = nft_set_elem_expr_alloc(ctx, set, tmp);
 7298			if (IS_ERR(expr)) {
 7299				err = PTR_ERR(expr);
 7300				goto err_set_elem_expr;
 7301			}
 7302			expr_array[i] = expr;
 7303			num_exprs++;
 7304
 7305			if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
 7306				err = -EOPNOTSUPP;
 7307				goto err_set_elem_expr;
 7308			}
 7309			i++;
 7310		}
 7311		if (set->num_exprs && set->num_exprs != i) {
 7312			err = -EOPNOTSUPP;
 7313			goto err_set_elem_expr;
 7314		}
 7315	} else if (set->num_exprs > 0 &&
 7316		   !(flags & NFT_SET_ELEM_INTERVAL_END)) {
 7317		err = nft_set_elem_expr_clone(ctx, set, expr_array);
 7318		if (err < 0)
 7319			goto err_set_elem_expr_clone;
 7320
 7321		num_exprs = set->num_exprs;
 7322	}
 7323
 7324	if (nla[NFTA_SET_ELEM_KEY]) {
 7325		err = nft_setelem_parse_key(ctx, set, &elem.key.val,
 7326					    nla[NFTA_SET_ELEM_KEY]);
 7327		if (err < 0)
 7328			goto err_set_elem_expr;
 7329
 7330		err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
 7331		if (err < 0)
 7332			goto err_parse_key;
 7333	}
 7334
 7335	if (nla[NFTA_SET_ELEM_KEY_END]) {
 7336		err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
 7337					    nla[NFTA_SET_ELEM_KEY_END]);
 7338		if (err < 0)
 7339			goto err_parse_key;
 7340
 7341		err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
 7342		if (err < 0)
 7343			goto err_parse_key_end;
 7344	}
 7345
 7346	if (set->flags & NFT_SET_TIMEOUT) {
 7347		err = nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
 7348		if (err < 0)
 7349			goto err_parse_key_end;
 7350	}
 7351
 7352	if (num_exprs) {
 7353		for (i = 0; i < num_exprs; i++)
 7354			size += expr_array[i]->ops->size;
 7355
 7356		err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
 7357					     sizeof(struct nft_set_elem_expr) + size);
 7358		if (err < 0)
 7359			goto err_parse_key_end;
 7360	}
 7361
 7362	if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
 7363		obj = nft_obj_lookup(ctx->net, ctx->table,
 7364				     nla[NFTA_SET_ELEM_OBJREF],
 7365				     set->objtype, genmask);
 7366		if (IS_ERR(obj)) {
 7367			err = PTR_ERR(obj);
 7368			obj = NULL;
 7369			goto err_parse_key_end;
 7370		}
 7371
 7372		if (!nft_use_inc(&obj->use)) {
 7373			err = -EMFILE;
 7374			obj = NULL;
 7375			goto err_parse_key_end;
 7376		}
 7377
 7378		err = nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
 7379		if (err < 0)
 7380			goto err_parse_key_end;
 7381	}
 7382
 7383	if (nla[NFTA_SET_ELEM_DATA] != NULL) {
 7384		err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
 7385					     nla[NFTA_SET_ELEM_DATA]);
 7386		if (err < 0)
 7387			goto err_parse_key_end;
 7388
 7389		dreg = nft_type_to_reg(set->dtype);
 7390		list_for_each_entry(binding, &set->bindings, list) {
 7391			struct nft_ctx bind_ctx = {
 7392				.net	= ctx->net,
 7393				.family	= ctx->family,
 7394				.table	= ctx->table,
 7395				.chain	= (struct nft_chain *)binding->chain,
 7396			};
 7397
 7398			if (!(binding->flags & NFT_SET_MAP))
 7399				continue;
 7400
 7401			err = nft_validate_register_store(&bind_ctx, dreg,
 7402							  &elem.data.val,
 7403							  desc.type, desc.len);
 7404			if (err < 0)
 7405				goto err_parse_data;
 7406
 7407			if (desc.type == NFT_DATA_VERDICT &&
 7408			    (elem.data.val.verdict.code == NFT_GOTO ||
 7409			     elem.data.val.verdict.code == NFT_JUMP))
 7410				nft_validate_state_update(ctx->table,
 7411							  NFT_VALIDATE_NEED);
 7412		}
 7413
 7414		err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
 7415		if (err < 0)
 7416			goto err_parse_data;
 7417	}
 7418
 7419	/* The full maximum length of userdata can exceed the maximum
 7420	 * offset value (U8_MAX) for following extensions, therefor it
 7421	 * must be the last extension added.
 7422	 */
 7423	ulen = 0;
 7424	if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
 7425		ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
 7426		if (ulen > 0) {
 7427			err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
 7428						     ulen);
 7429			if (err < 0)
 7430				goto err_parse_data;
 7431		}
 7432	}
 7433
 7434	elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
 7435				      elem.key_end.val.data, elem.data.val.data,
 7436				      timeout, expiration, GFP_KERNEL_ACCOUNT);
 7437	if (IS_ERR(elem.priv)) {
 7438		err = PTR_ERR(elem.priv);
 7439		goto err_parse_data;
 7440	}
 7441
 7442	ext = nft_set_elem_ext(set, elem.priv);
 7443	if (flags)
 7444		*nft_set_ext_flags(ext) = flags;
 7445
 7446	if (obj)
 7447		*nft_set_ext_obj(ext) = obj;
 7448
 7449	if (ulen > 0) {
 7450		if (nft_set_ext_check(&tmpl, NFT_SET_EXT_USERDATA, ulen) < 0) {
 7451			err = -EINVAL;
 7452			goto err_elem_free;
 7453		}
 7454		udata = nft_set_ext_userdata(ext);
 7455		udata->len = ulen - 1;
 7456		nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
 7457	}
 7458	err = nft_set_elem_expr_setup(ctx, &tmpl, ext, expr_array, num_exprs);
 7459	if (err < 0)
 7460		goto err_elem_free;
 7461
 7462	if (!(flags & NFT_SET_ELEM_CATCHALL)) {
 7463		unsigned int max = nft_set_maxsize(set), nelems;
 7464
 7465		nelems = atomic_inc_return(&set->nelems);
 7466		if (nelems > max)
 7467			set_full = true;
 7468	}
 7469
 7470	trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
 7471	if (trans == NULL) {
 7472		err = -ENOMEM;
 7473		goto err_set_size;
 7474	}
 7475
 7476	ext->genmask = nft_genmask_cur(ctx->net);
 7477
 7478	err = nft_setelem_insert(ctx->net, set, &elem, &elem_priv, flags);
 7479	if (err) {
 7480		if (err == -EEXIST) {
 7481			ext2 = nft_set_elem_ext(set, elem_priv);
 7482			if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
 7483			    nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
 7484			    nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
 7485			    nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
 7486				goto err_element_clash;
 7487			if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
 7488			     nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
 7489			     memcmp(nft_set_ext_data(ext),
 7490				    nft_set_ext_data(ext2), set->dlen) != 0) ||
 7491			    (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
 7492			     nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
 7493			     *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
 7494				goto err_element_clash;
 7495			else if (!(nlmsg_flags & NLM_F_EXCL)) {
 7496				err = 0;
 7497				if (nft_set_ext_exists(ext2, NFT_SET_EXT_TIMEOUT)) {
 7498					struct nft_elem_update update = { };
 7499
 7500					if (timeout != nft_set_ext_timeout(ext2)->timeout) {
 7501						update.timeout = timeout;
 7502						if (expiration == 0)
 7503							expiration = timeout;
 7504
 7505						update.flags |= NFT_TRANS_UPD_TIMEOUT;
 7506					}
 7507					if (expiration) {
 7508						update.expiration = expiration;
 7509						update.flags |= NFT_TRANS_UPD_EXPIRATION;
 7510					}
 7511
 7512					if (update.flags) {
 7513						struct nft_trans_one_elem *ue;
 7514
 7515						ue = &nft_trans_container_elem(trans)->elems[0];
 7516
 7517						ue->update = kmemdup(&update, sizeof(update), GFP_KERNEL);
 7518						if (!ue->update) {
 7519							err = -ENOMEM;
 7520							goto err_element_clash;
 7521						}
 7522
 7523						ue->priv = elem_priv;
 7524						nft_trans_commit_list_add_elem(ctx->net, trans);
 7525						goto err_set_size;
 7526					}
 7527				}
 7528			}
 7529		} else if (err == -ENOTEMPTY) {
 7530			/* ENOTEMPTY reports overlapping between this element
 7531			 * and an existing one.
 7532			 */
 7533			err = -EEXIST;
 7534		} else if (err == -ECANCELED) {
 7535			/* ECANCELED reports an existing nul-element in
 7536			 * interval sets.
 7537			 */
 7538			err = 0;
 7539		}
 7540		goto err_element_clash;
 7541	}
 7542
 7543	nft_trans_container_elem(trans)->elems[0].priv = elem.priv;
 7544	nft_trans_commit_list_add_elem(ctx->net, trans);
 7545
 7546	return set_full ? -ENFILE : 0;
 7547
 7548err_element_clash:
 7549	kfree(trans);
 7550err_set_size:
 7551	if (!(flags & NFT_SET_ELEM_CATCHALL))
 7552		atomic_dec(&set->nelems);
 7553err_elem_free:
 7554	nf_tables_set_elem_destroy(ctx, set, elem.priv);
 7555err_parse_data:
 7556	if (nla[NFTA_SET_ELEM_DATA] != NULL)
 7557		nft_data_release(&elem.data.val, desc.type);
 7558err_parse_key_end:
 7559	if (obj)
 7560		nft_use_dec_restore(&obj->use);
 7561
 7562	nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
 7563err_parse_key:
 7564	nft_data_release(&elem.key.val, NFT_DATA_VALUE);
 7565err_set_elem_expr:
 7566	for (i = 0; i < num_exprs && expr_array[i]; i++)
 7567		nft_expr_destroy(ctx, expr_array[i]);
 7568err_set_elem_expr_clone:
 7569	return err;
 7570}
 7571
 7572static int nf_tables_newsetelem(struct sk_buff *skb,
 7573				const struct nfnl_info *info,
 7574				const struct nlattr * const nla[])
 7575{
 7576	struct netlink_ext_ack *extack = info->extack;
 7577	u8 genmask = nft_genmask_next(info->net);
 7578	u8 family = info->nfmsg->nfgen_family;
 7579	struct net *net = info->net;
 7580	const struct nlattr *attr;
 7581	struct nft_table *table;
 7582	struct nft_set *set;
 7583	struct nft_ctx ctx;
 7584	int rem, err;
 7585
 7586	if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
 7587		return -EINVAL;
 7588
 7589	table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
 7590				 genmask, NETLINK_CB(skb).portid);
 7591	if (IS_ERR(table)) {
 7592		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
 7593		return PTR_ERR(table);
 7594	}
 7595
 7596	set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
 7597				    nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
 7598	if (IS_ERR(set)) {
 7599		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
 7600		return PTR_ERR(set);
 7601	}
 7602
 7603	if (!list_empty(&set->bindings) &&
 7604	    (set->flags & (NFT_SET_CONSTANT | NFT_SET_ANONYMOUS)))
 7605		return -EBUSY;
 7606
 7607	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 7608
 7609	nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
 7610		err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
 7611		if (err < 0) {
 7612			NL_SET_BAD_ATTR(extack, attr);
 7613			return err;
 7614		}
 7615	}
 7616
 7617	if (table->validate_state == NFT_VALIDATE_DO)
 7618		return nft_table_validate(net, table);
 7619
 7620	return 0;
 7621}
 7622
 7623/**
 7624 *	nft_data_hold - hold a nft_data item
 7625 *
 7626 *	@data: struct nft_data to release
 7627 *	@type: type of data
 7628 *
 7629 *	Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
 7630 *	NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
 7631 *	NFT_GOTO verdicts. This function must be called on active data objects
 7632 *	from the second phase of the commit protocol.
 7633 */
 7634void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
 7635{
 7636	struct nft_chain *chain;
 7637
 7638	if (type == NFT_DATA_VERDICT) {
 7639		switch (data->verdict.code) {
 7640		case NFT_JUMP:
 7641		case NFT_GOTO:
 7642			chain = data->verdict.chain;
 7643			nft_use_inc_restore(&chain->use);
 7644			break;
 7645		}
 7646	}
 7647}
 7648
 7649static int nft_setelem_active_next(const struct net *net,
 7650				   const struct nft_set *set,
 7651				   struct nft_elem_priv *elem_priv)
 7652{
 7653	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 7654	u8 genmask = nft_genmask_next(net);
 7655
 7656	return nft_set_elem_active(ext, genmask);
 7657}
 7658
 7659static void nft_setelem_data_activate(const struct net *net,
 7660				      const struct nft_set *set,
 7661				      struct nft_elem_priv *elem_priv)
 7662{
 7663	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 7664
 7665	if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
 7666		nft_data_hold(nft_set_ext_data(ext), set->dtype);
 7667	if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
 7668		nft_use_inc_restore(&(*nft_set_ext_obj(ext))->use);
 7669}
 7670
 7671void nft_setelem_data_deactivate(const struct net *net,
 7672				 const struct nft_set *set,
 7673				 struct nft_elem_priv *elem_priv)
 7674{
 7675	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 7676
 7677	if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
 7678		nft_data_release(nft_set_ext_data(ext), set->dtype);
 7679	if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
 7680		nft_use_dec(&(*nft_set_ext_obj(ext))->use);
 7681}
 7682
 7683/* similar to nft_trans_elems_remove, but called from abort path to undo newsetelem.
 7684 * No notifications and no ndeact changes.
 7685 *
 7686 * Returns true if set had been added to (i.e., elements need to be removed again).
 7687 */
 7688static bool nft_trans_elems_new_abort(const struct nft_ctx *ctx,
 7689				      struct nft_trans_elem *te)
 7690{
 7691	bool removed = false;
 7692	int i;
 7693
 7694	for (i = 0; i < te->nelems; i++) {
 7695		if (te->elems[i].update) {
 7696			kfree(te->elems[i].update);
 7697			te->elems[i].update = NULL;
 7698			/* Update request, so do not release this element */
 7699			te->elems[i].priv = NULL;
 7700			continue;
 7701		}
 7702
 7703		if (!te->set->ops->abort_skip_removal ||
 7704		    nft_setelem_is_catchall(te->set, te->elems[i].priv))
 7705			nft_setelem_remove(ctx->net, te->set, te->elems[i].priv);
 7706
 7707		if (!nft_setelem_is_catchall(te->set, te->elems[i].priv))
 7708			atomic_dec(&te->set->nelems);
 7709
 7710		removed = true;
 7711	}
 7712
 7713	return removed;
 7714}
 7715
 7716/* Called from abort path to undo DELSETELEM/DESTROYSETELEM. */
 7717static void nft_trans_elems_destroy_abort(const struct nft_ctx *ctx,
 7718					  const struct nft_trans_elem *te)
 7719{
 7720	int i;
 7721
 7722	for (i = 0; i < te->nelems; i++) {
 7723		if (!nft_setelem_active_next(ctx->net, te->set, te->elems[i].priv)) {
 7724			nft_setelem_data_activate(ctx->net, te->set, te->elems[i].priv);
 7725			nft_setelem_activate(ctx->net, te->set, te->elems[i].priv);
 7726		}
 7727
 7728		if (!nft_setelem_is_catchall(te->set, te->elems[i].priv))
 7729			te->set->ndeact--;
 7730	}
 7731}
 7732
 7733static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
 7734			   const struct nlattr *attr)
 7735{
 7736	struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
 7737	struct nft_set_ext_tmpl tmpl;
 7738	struct nft_set_elem elem;
 7739	struct nft_set_ext *ext;
 7740	struct nft_trans *trans;
 7741	u32 flags = 0;
 7742	int err;
 7743
 7744	err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
 7745					  nft_set_elem_policy, NULL);
 7746	if (err < 0)
 7747		return err;
 7748
 7749	err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
 7750	if (err < 0)
 7751		return err;
 7752
 7753	if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
 7754		return -EINVAL;
 7755
 7756	if (!nft_setelem_valid_key_end(set, nla, flags))
 7757		return -EINVAL;
 7758
 7759	nft_set_ext_prepare(&tmpl);
 7760
 7761	if (flags != 0) {
 7762		err = nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
 7763		if (err < 0)
 7764			return err;
 7765	}
 7766
 7767	if (nla[NFTA_SET_ELEM_KEY]) {
 7768		err = nft_setelem_parse_key(ctx, set, &elem.key.val,
 7769					    nla[NFTA_SET_ELEM_KEY]);
 7770		if (err < 0)
 7771			return err;
 7772
 7773		err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
 7774		if (err < 0)
 7775			goto fail_elem;
 7776	}
 7777
 7778	if (nla[NFTA_SET_ELEM_KEY_END]) {
 7779		err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
 7780					    nla[NFTA_SET_ELEM_KEY_END]);
 7781		if (err < 0)
 7782			goto fail_elem;
 7783
 7784		err = nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
 7785		if (err < 0)
 7786			goto fail_elem_key_end;
 7787	}
 7788
 7789	err = -ENOMEM;
 7790	elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
 7791				      elem.key_end.val.data, NULL, 0, 0,
 7792				      GFP_KERNEL_ACCOUNT);
 7793	if (IS_ERR(elem.priv)) {
 7794		err = PTR_ERR(elem.priv);
 7795		goto fail_elem_key_end;
 7796	}
 7797
 7798	ext = nft_set_elem_ext(set, elem.priv);
 7799	if (flags)
 7800		*nft_set_ext_flags(ext) = flags;
 7801
 7802	trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
 7803	if (trans == NULL)
 7804		goto fail_trans;
 7805
 7806	err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
 7807	if (err < 0)
 7808		goto fail_ops;
 7809
 7810	nft_setelem_data_deactivate(ctx->net, set, elem.priv);
 7811
 7812	nft_trans_container_elem(trans)->elems[0].priv = elem.priv;
 7813	nft_trans_commit_list_add_elem(ctx->net, trans);
 7814	return 0;
 7815
 7816fail_ops:
 7817	kfree(trans);
 7818fail_trans:
 7819	kfree(elem.priv);
 7820fail_elem_key_end:
 7821	nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
 7822fail_elem:
 7823	nft_data_release(&elem.key.val, NFT_DATA_VALUE);
 7824	return err;
 7825}
 7826
 7827static int nft_setelem_flush(const struct nft_ctx *ctx,
 7828			     struct nft_set *set,
 7829			     const struct nft_set_iter *iter,
 7830			     struct nft_elem_priv *elem_priv)
 7831{
 7832	const struct nft_set_ext *ext = nft_set_elem_ext(set, elem_priv);
 7833	struct nft_trans *trans;
 7834
 7835	if (!nft_set_elem_active(ext, iter->genmask))
 7836		return 0;
 7837
 7838	trans = nft_trans_alloc(ctx, NFT_MSG_DELSETELEM,
 7839				struct_size_t(struct nft_trans_elem, elems, 1));
 7840	if (!trans)
 7841		return -ENOMEM;
 7842
 7843	set->ops->flush(ctx->net, set, elem_priv);
 7844	set->ndeact++;
 7845
 7846	nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 7847	nft_trans_elem_set(trans) = set;
 7848	nft_trans_container_elem(trans)->nelems = 1;
 7849	nft_trans_container_elem(trans)->elems[0].priv = elem_priv;
 7850	nft_trans_commit_list_add_elem(ctx->net, trans);
 7851
 7852	return 0;
 7853}
 7854
 7855static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
 7856				    struct nft_set *set,
 7857				    struct nft_elem_priv *elem_priv)
 7858{
 7859	struct nft_trans *trans;
 7860
 7861	trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
 7862	if (!trans)
 7863		return -ENOMEM;
 7864
 7865	nft_setelem_data_deactivate(ctx->net, set, elem_priv);
 7866	nft_trans_container_elem(trans)->elems[0].priv = elem_priv;
 7867	nft_trans_commit_list_add_elem(ctx->net, trans);
 7868
 7869	return 0;
 7870}
 7871
 7872static int nft_set_catchall_flush(const struct nft_ctx *ctx,
 7873				  struct nft_set *set)
 7874{
 7875	u8 genmask = nft_genmask_next(ctx->net);
 7876	struct nft_set_elem_catchall *catchall;
 7877	struct nft_set_ext *ext;
 7878	int ret = 0;
 7879
 7880	list_for_each_entry_rcu(catchall, &set->catchall_list, list,
 7881				lockdep_commit_lock_is_held(ctx->net)) {
 7882		ext = nft_set_elem_ext(set, catchall->elem);
 7883		if (!nft_set_elem_active(ext, genmask))
 7884			continue;
 7885
 7886		ret = __nft_set_catchall_flush(ctx, set, catchall->elem);
 7887		if (ret < 0)
 7888			break;
 7889		nft_set_elem_change_active(ctx->net, set, ext);
 7890	}
 7891
 7892	return ret;
 7893}
 7894
 7895static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
 7896{
 7897	/* The set backend might need to clone the set, do it now from the
 7898	 * preparation phase, use NFT_ITER_UPDATE_CLONE iterator type.
 7899	 */
 7900	struct nft_set_iter iter = {
 7901		.genmask	= genmask,
 7902		.type		= NFT_ITER_UPDATE_CLONE,
 7903		.fn		= nft_setelem_flush,
 7904	};
 7905
 7906	set->ops->walk(ctx, set, &iter);
 7907	if (!iter.err)
 7908		iter.err = nft_set_catchall_flush(ctx, set);
 7909
 7910	return iter.err;
 7911}
 7912
 7913static int nf_tables_delsetelem(struct sk_buff *skb,
 7914				const struct nfnl_info *info,
 7915				const struct nlattr * const nla[])
 7916{
 7917	struct netlink_ext_ack *extack = info->extack;
 7918	u8 genmask = nft_genmask_next(info->net);
 7919	u8 family = info->nfmsg->nfgen_family;
 7920	struct net *net = info->net;
 7921	const struct nlattr *attr;
 7922	struct nft_table *table;
 7923	struct nft_set *set;
 7924	struct nft_ctx ctx;
 7925	int rem, err = 0;
 7926
 7927	table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
 7928				 genmask, NETLINK_CB(skb).portid);
 7929	if (IS_ERR(table)) {
 7930		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
 7931		return PTR_ERR(table);
 7932	}
 7933
 7934	set = nft_set_lookup(net, table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
 7935	if (IS_ERR(set)) {
 7936		NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_SET]);
 7937		return PTR_ERR(set);
 7938	}
 7939
 7940	if (nft_set_is_anonymous(set))
 7941		return -EOPNOTSUPP;
 7942
 7943	if (!list_empty(&set->bindings) && (set->flags & NFT_SET_CONSTANT))
 7944		return -EBUSY;
 7945
 7946	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 7947
 7948	if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
 7949		return nft_set_flush(&ctx, set, genmask);
 7950
 7951	nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
 7952		err = nft_del_setelem(&ctx, set, attr);
 7953		if (err == -ENOENT &&
 7954		    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYSETELEM)
 7955			continue;
 7956
 7957		if (err < 0) {
 7958			NL_SET_BAD_ATTR(extack, attr);
 7959			return err;
 7960		}
 7961	}
 7962
 7963	return 0;
 7964}
 7965
 7966/*
 7967 * Stateful objects
 7968 */
 7969
 7970/**
 7971 *	nft_register_obj- register nf_tables stateful object type
 7972 *	@obj_type: object type
 7973 *
 7974 *	Registers the object type for use with nf_tables. Returns zero on
 7975 *	success or a negative errno code otherwise.
 7976 */
 7977int nft_register_obj(struct nft_object_type *obj_type)
 7978{
 7979	if (obj_type->type == NFT_OBJECT_UNSPEC)
 7980		return -EINVAL;
 7981
 7982	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 7983	list_add_rcu(&obj_type->list, &nf_tables_objects);
 7984	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 7985	return 0;
 7986}
 7987EXPORT_SYMBOL_GPL(nft_register_obj);
 7988
 7989/**
 7990 *	nft_unregister_obj - unregister nf_tables object type
 7991 *	@obj_type: object type
 7992 *
 7993 * 	Unregisters the object type for use with nf_tables.
 7994 */
 7995void nft_unregister_obj(struct nft_object_type *obj_type)
 7996{
 7997	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 7998	list_del_rcu(&obj_type->list);
 7999	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 8000}
 8001EXPORT_SYMBOL_GPL(nft_unregister_obj);
 8002
 8003struct nft_object *nft_obj_lookup(const struct net *net,
 8004				  const struct nft_table *table,
 8005				  const struct nlattr *nla, u32 objtype,
 8006				  u8 genmask)
 8007{
 8008	struct nft_object_hash_key k = { .table = table };
 8009	char search[NFT_OBJ_MAXNAMELEN];
 8010	struct rhlist_head *tmp, *list;
 8011	struct nft_object *obj;
 8012
 8013	nla_strscpy(search, nla, sizeof(search));
 8014	k.name = search;
 8015
 8016	WARN_ON_ONCE(!rcu_read_lock_held() &&
 8017		     !lockdep_commit_lock_is_held(net));
 8018
 8019	rcu_read_lock();
 8020	list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
 8021	if (!list)
 8022		goto out;
 8023
 8024	rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
 8025		if (objtype == obj->ops->type->type &&
 8026		    nft_active_genmask(obj, genmask)) {
 8027			rcu_read_unlock();
 8028			return obj;
 8029		}
 8030	}
 8031out:
 8032	rcu_read_unlock();
 8033	return ERR_PTR(-ENOENT);
 8034}
 8035EXPORT_SYMBOL_GPL(nft_obj_lookup);
 8036
 8037static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
 8038						  const struct nlattr *nla,
 8039						  u32 objtype, u8 genmask)
 8040{
 8041	struct nft_object *obj;
 8042
 8043	list_for_each_entry(obj, &table->objects, list) {
 8044		if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
 8045		    objtype == obj->ops->type->type &&
 8046		    nft_active_genmask(obj, genmask))
 8047			return obj;
 8048	}
 8049	return ERR_PTR(-ENOENT);
 8050}
 8051
 8052static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
 8053	[NFTA_OBJ_TABLE]	= { .type = NLA_STRING,
 8054				    .len = NFT_TABLE_MAXNAMELEN - 1 },
 8055	[NFTA_OBJ_NAME]		= { .type = NLA_STRING,
 8056				    .len = NFT_OBJ_MAXNAMELEN - 1 },
 8057	[NFTA_OBJ_TYPE]		= { .type = NLA_U32 },
 8058	[NFTA_OBJ_DATA]		= { .type = NLA_NESTED },
 8059	[NFTA_OBJ_HANDLE]	= { .type = NLA_U64},
 8060	[NFTA_OBJ_USERDATA]	= { .type = NLA_BINARY,
 8061				    .len = NFT_USERDATA_MAXLEN },
 8062};
 8063
 8064static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
 8065				       const struct nft_object_type *type,
 8066				       const struct nlattr *attr)
 8067{
 8068	struct nlattr **tb;
 8069	const struct nft_object_ops *ops;
 8070	struct nft_object *obj;
 8071	int err = -ENOMEM;
 8072
 8073	tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
 8074	if (!tb)
 8075		goto err1;
 8076
 8077	if (attr) {
 8078		err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
 8079						  type->policy, NULL);
 8080		if (err < 0)
 8081			goto err2;
 8082	} else {
 8083		memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
 8084	}
 8085
 8086	if (type->select_ops) {
 8087		ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
 8088		if (IS_ERR(ops)) {
 8089			err = PTR_ERR(ops);
 8090			goto err2;
 8091		}
 8092	} else {
 8093		ops = type->ops;
 8094	}
 8095
 8096	err = -ENOMEM;
 8097	obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL_ACCOUNT);
 8098	if (!obj)
 8099		goto err2;
 8100
 8101	err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
 8102	if (err < 0)
 8103		goto err3;
 8104
 8105	obj->ops = ops;
 8106
 8107	kfree(tb);
 8108	return obj;
 8109err3:
 8110	kfree(obj);
 8111err2:
 8112	kfree(tb);
 8113err1:
 8114	return ERR_PTR(err);
 8115}
 8116
 8117static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
 8118			   struct nft_object *obj, bool reset)
 8119{
 8120	struct nlattr *nest;
 8121
 8122	nest = nla_nest_start_noflag(skb, attr);
 8123	if (!nest)
 8124		goto nla_put_failure;
 8125	if (obj->ops->dump(skb, obj, reset) < 0)
 8126		goto nla_put_failure;
 8127	nla_nest_end(skb, nest);
 8128	return 0;
 8129
 8130nla_put_failure:
 8131	return -1;
 8132}
 8133
 8134static const struct nft_object_type *__nft_obj_type_get(u32 objtype, u8 family)
 8135{
 8136	const struct nft_object_type *type;
 8137
 8138	list_for_each_entry_rcu(type, &nf_tables_objects, list) {
 8139		if (type->family != NFPROTO_UNSPEC &&
 8140		    type->family != family)
 8141			continue;
 8142
 8143		if (objtype == type->type)
 8144			return type;
 8145	}
 8146	return NULL;
 8147}
 8148
 8149static const struct nft_object_type *
 8150nft_obj_type_get(struct net *net, u32 objtype, u8 family)
 8151{
 8152	const struct nft_object_type *type;
 8153
 8154	rcu_read_lock();
 8155	type = __nft_obj_type_get(objtype, family);
 8156	if (type != NULL && try_module_get(type->owner)) {
 8157		rcu_read_unlock();
 8158		return type;
 8159	}
 8160	rcu_read_unlock();
 8161
 8162	lockdep_nfnl_nft_mutex_not_held();
 8163#ifdef CONFIG_MODULES
 8164	if (type == NULL) {
 8165		if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
 8166			return ERR_PTR(-EAGAIN);
 8167	}
 8168#endif
 8169	return ERR_PTR(-ENOENT);
 8170}
 8171
 8172static int nf_tables_updobj(const struct nft_ctx *ctx,
 8173			    const struct nft_object_type *type,
 8174			    const struct nlattr *attr,
 8175			    struct nft_object *obj)
 8176{
 8177	struct nft_object *newobj;
 8178	struct nft_trans *trans;
 8179	int err = -ENOMEM;
 8180
 8181	/* caller must have obtained type->owner reference. */
 8182	trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
 8183				sizeof(struct nft_trans_obj));
 8184	if (!trans)
 8185		goto err_trans;
 8186
 8187	newobj = nft_obj_init(ctx, type, attr);
 8188	if (IS_ERR(newobj)) {
 8189		err = PTR_ERR(newobj);
 8190		goto err_free_trans;
 8191	}
 8192
 8193	nft_trans_obj(trans) = obj;
 8194	nft_trans_obj_update(trans) = true;
 8195	nft_trans_obj_newobj(trans) = newobj;
 8196	nft_trans_commit_list_add_tail(ctx->net, trans);
 8197
 8198	return 0;
 8199
 8200err_free_trans:
 8201	kfree(trans);
 8202err_trans:
 8203	module_put(type->owner);
 8204	return err;
 8205}
 8206
 8207static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
 8208			    const struct nlattr * const nla[])
 8209{
 8210	struct netlink_ext_ack *extack = info->extack;
 8211	u8 genmask = nft_genmask_next(info->net);
 8212	u8 family = info->nfmsg->nfgen_family;
 8213	const struct nft_object_type *type;
 8214	struct net *net = info->net;
 8215	struct nft_table *table;
 8216	struct nft_object *obj;
 8217	struct nft_ctx ctx;
 8218	u32 objtype;
 8219	int err;
 8220
 8221	if (!nla[NFTA_OBJ_TYPE] ||
 8222	    !nla[NFTA_OBJ_NAME] ||
 8223	    !nla[NFTA_OBJ_DATA])
 8224		return -EINVAL;
 8225
 8226	table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
 8227				 NETLINK_CB(skb).portid);
 8228	if (IS_ERR(table)) {
 8229		NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
 8230		return PTR_ERR(table);
 8231	}
 8232
 8233	objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
 8234	obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
 8235	if (IS_ERR(obj)) {
 8236		err = PTR_ERR(obj);
 8237		if (err != -ENOENT) {
 8238			NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
 8239			return err;
 8240		}
 8241	} else {
 8242		if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
 8243			NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
 8244			return -EEXIST;
 8245		}
 8246		if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
 8247			return -EOPNOTSUPP;
 8248
 8249		if (!obj->ops->update)
 8250			return 0;
 8251
 8252		type = nft_obj_type_get(net, objtype, family);
 8253		if (WARN_ON_ONCE(IS_ERR(type)))
 8254			return PTR_ERR(type);
 8255
 8256		nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 8257
 8258		/* type->owner reference is put when transaction object is released. */
 8259		return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
 8260	}
 8261
 8262	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 8263
 8264	if (!nft_use_inc(&table->use))
 8265		return -EMFILE;
 8266
 8267	type = nft_obj_type_get(net, objtype, family);
 8268	if (IS_ERR(type)) {
 8269		err = PTR_ERR(type);
 8270		goto err_type;
 8271	}
 8272
 8273	obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
 8274	if (IS_ERR(obj)) {
 8275		err = PTR_ERR(obj);
 8276		goto err_init;
 8277	}
 8278	obj->key.table = table;
 8279	obj->handle = nf_tables_alloc_handle(table);
 8280
 8281	obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL_ACCOUNT);
 8282	if (!obj->key.name) {
 8283		err = -ENOMEM;
 8284		goto err_strdup;
 8285	}
 8286
 8287	if (nla[NFTA_OBJ_USERDATA]) {
 8288		obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL_ACCOUNT);
 8289		if (obj->udata == NULL)
 8290			goto err_userdata;
 8291
 8292		obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
 8293	}
 8294
 8295	err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
 8296	if (err < 0)
 8297		goto err_trans;
 8298
 8299	err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
 8300			      nft_objname_ht_params);
 8301	if (err < 0)
 8302		goto err_obj_ht;
 8303
 8304	list_add_tail_rcu(&obj->list, &table->objects);
 8305
 8306	return 0;
 8307err_obj_ht:
 8308	/* queued in transaction log */
 8309	INIT_LIST_HEAD(&obj->list);
 8310	return err;
 8311err_trans:
 8312	kfree(obj->udata);
 8313err_userdata:
 8314	kfree(obj->key.name);
 8315err_strdup:
 8316	if (obj->ops->destroy)
 8317		obj->ops->destroy(&ctx, obj);
 8318	kfree(obj);
 8319err_init:
 8320	module_put(type->owner);
 8321err_type:
 8322	nft_use_dec_restore(&table->use);
 8323
 8324	return err;
 8325}
 8326
 8327static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
 8328				   u32 portid, u32 seq, int event, u32 flags,
 8329				   int family, const struct nft_table *table,
 8330				   struct nft_object *obj, bool reset)
 8331{
 8332	struct nlmsghdr *nlh;
 8333
 8334	nlh = nfnl_msg_put(skb, portid, seq,
 8335			   nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event),
 8336			   flags, family, NFNETLINK_V0, nft_base_seq_be16(net));
 8337	if (!nlh)
 8338		goto nla_put_failure;
 8339
 8340	if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
 8341	    nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
 8342	    nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
 8343	    nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
 8344			 NFTA_OBJ_PAD))
 8345		goto nla_put_failure;
 8346
 8347	if (event == NFT_MSG_DELOBJ ||
 8348	    event == NFT_MSG_DESTROYOBJ) {
 8349		nlmsg_end(skb, nlh);
 8350		return 0;
 8351	}
 8352
 8353	if (nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
 8354	    nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset))
 8355		goto nla_put_failure;
 8356
 8357	if (obj->udata &&
 8358	    nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
 8359		goto nla_put_failure;
 8360
 8361	nlmsg_end(skb, nlh);
 8362	return 0;
 8363
 8364nla_put_failure:
 8365	nlmsg_trim(skb, nlh);
 8366	return -1;
 8367}
 8368
 8369static void audit_log_obj_reset(const struct nft_table *table,
 8370				unsigned int base_seq, unsigned int nentries)
 8371{
 8372	char *buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, base_seq);
 8373
 8374	audit_log_nfcfg(buf, table->family, nentries,
 8375			AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
 8376	kfree(buf);
 8377}
 8378
 8379struct nft_obj_dump_ctx {
 8380	unsigned int	s_idx;
 8381	char		*table;
 8382	u32		type;
 8383	bool		reset;
 8384};
 8385
 8386static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
 8387{
 8388	const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 8389	struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
 8390	struct net *net = sock_net(skb->sk);
 8391	int family = nfmsg->nfgen_family;
 8392	struct nftables_pernet *nft_net;
 8393	const struct nft_table *table;
 8394	unsigned int entries = 0;
 8395	struct nft_object *obj;
 8396	unsigned int idx = 0;
 8397	int rc = 0;
 8398
 8399	rcu_read_lock();
 8400	nft_net = nft_pernet(net);
 8401	cb->seq = nft_base_seq(net);
 8402
 8403	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 8404		if (family != NFPROTO_UNSPEC && family != table->family)
 8405			continue;
 8406
 8407		entries = 0;
 8408		list_for_each_entry_rcu(obj, &table->objects, list) {
 8409			if (!nft_is_active(net, obj))
 8410				goto cont;
 8411			if (idx < ctx->s_idx)
 8412				goto cont;
 8413			if (ctx->table && strcmp(ctx->table, table->name))
 8414				goto cont;
 8415			if (ctx->type != NFT_OBJECT_UNSPEC &&
 8416			    obj->ops->type->type != ctx->type)
 8417				goto cont;
 8418
 8419			rc = nf_tables_fill_obj_info(skb, net,
 8420						     NETLINK_CB(cb->skb).portid,
 8421						     cb->nlh->nlmsg_seq,
 8422						     NFT_MSG_NEWOBJ,
 8423						     NLM_F_MULTI | NLM_F_APPEND,
 8424						     table->family, table,
 8425						     obj, ctx->reset);
 8426			if (rc < 0)
 8427				break;
 8428
 8429			entries++;
 8430			nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 8431cont:
 8432			idx++;
 8433		}
 8434		if (ctx->reset && entries)
 8435			audit_log_obj_reset(table, nft_base_seq(net), entries);
 8436		if (rc < 0)
 8437			break;
 8438	}
 8439	rcu_read_unlock();
 8440
 8441	ctx->s_idx = idx;
 8442	return skb->len;
 8443}
 8444
 8445static int nf_tables_dump_obj_start(struct netlink_callback *cb)
 8446{
 8447	struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
 8448	const struct nlattr * const *nla = cb->data;
 8449
 8450	BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
 8451
 8452	if (nla[NFTA_OBJ_TABLE]) {
 8453		ctx->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
 8454		if (!ctx->table)
 8455			return -ENOMEM;
 8456	}
 8457
 8458	if (nla[NFTA_OBJ_TYPE])
 8459		ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
 8460
 8461	if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
 8462		ctx->reset = true;
 8463
 8464	return 0;
 8465}
 8466
 8467static int nf_tables_dump_obj_done(struct netlink_callback *cb)
 8468{
 8469	struct nft_obj_dump_ctx *ctx = (void *)cb->ctx;
 8470
 8471	kfree(ctx->table);
 8472
 8473	return 0;
 8474}
 8475
 8476/* Caller must hold rcu read lock or transaction mutex */
 8477static struct sk_buff *
 8478nf_tables_getobj_single(u32 portid, const struct nfnl_info *info,
 8479			const struct nlattr * const nla[], bool reset)
 8480{
 8481	struct netlink_ext_ack *extack = info->extack;
 8482	u8 genmask = nft_genmask_cur(info->net);
 8483	u8 family = info->nfmsg->nfgen_family;
 8484	const struct nft_table *table;
 8485	struct net *net = info->net;
 8486	struct nft_object *obj;
 8487	struct sk_buff *skb2;
 8488	u32 objtype;
 8489	int err;
 8490
 8491	if (!nla[NFTA_OBJ_NAME] ||
 8492	    !nla[NFTA_OBJ_TYPE])
 8493		return ERR_PTR(-EINVAL);
 8494
 8495	table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
 8496	if (IS_ERR(table)) {
 8497		NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
 8498		return ERR_CAST(table);
 8499	}
 8500
 8501	objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
 8502	obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
 8503	if (IS_ERR(obj)) {
 8504		NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
 8505		return ERR_CAST(obj);
 8506	}
 8507
 8508	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 8509	if (!skb2)
 8510		return ERR_PTR(-ENOMEM);
 8511
 8512	err = nf_tables_fill_obj_info(skb2, net, portid,
 8513				      info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
 8514				      family, table, obj, reset);
 8515	if (err < 0) {
 8516		kfree_skb(skb2);
 8517		return ERR_PTR(err);
 8518	}
 8519
 8520	return skb2;
 8521}
 8522
 8523static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
 8524			    const struct nlattr * const nla[])
 8525{
 8526	u32 portid = NETLINK_CB(skb).portid;
 8527	struct net *net = info->net;
 8528	struct sk_buff *skb2;
 8529	bool reset = false;
 8530	char *buf;
 8531
 8532	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 8533		struct netlink_dump_control c = {
 8534			.start = nf_tables_dump_obj_start,
 8535			.dump = nf_tables_dump_obj,
 8536			.done = nf_tables_dump_obj_done,
 8537			.module = THIS_MODULE,
 8538			.data = (void *)nla,
 8539		};
 8540
 8541		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 8542	}
 8543
 8544	if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
 8545		reset = true;
 8546
 8547	skb2 = nf_tables_getobj_single(portid, info, nla, reset);
 8548	if (IS_ERR(skb2))
 8549		return PTR_ERR(skb2);
 8550
 8551	if (!reset)
 8552		return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 8553
 8554	buf = kasprintf(GFP_ATOMIC, "%.*s:%u",
 8555			nla_len(nla[NFTA_OBJ_TABLE]),
 8556			(char *)nla_data(nla[NFTA_OBJ_TABLE]),
 8557			nft_base_seq(net));
 8558	audit_log_nfcfg(buf, info->nfmsg->nfgen_family, 1,
 8559			AUDIT_NFT_OP_OBJ_RESET, GFP_ATOMIC);
 8560	kfree(buf);
 8561
 8562	return nfnetlink_unicast(skb2, net, portid);
 8563}
 8564
 8565static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
 8566{
 8567	if (obj->ops->destroy)
 8568		obj->ops->destroy(ctx, obj);
 8569
 8570	module_put(obj->ops->type->owner);
 8571	kfree(obj->key.name);
 8572	kfree(obj->udata);
 8573	kfree(obj);
 8574}
 8575
 8576static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
 8577			    const struct nlattr * const nla[])
 8578{
 8579	struct netlink_ext_ack *extack = info->extack;
 8580	u8 genmask = nft_genmask_next(info->net);
 8581	u8 family = info->nfmsg->nfgen_family;
 8582	struct net *net = info->net;
 8583	const struct nlattr *attr;
 8584	struct nft_table *table;
 8585	struct nft_object *obj;
 8586	struct nft_ctx ctx;
 8587	u32 objtype;
 8588
 8589	if (!nla[NFTA_OBJ_TYPE] ||
 8590	    (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
 8591		return -EINVAL;
 8592
 8593	table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
 8594				 NETLINK_CB(skb).portid);
 8595	if (IS_ERR(table)) {
 8596		NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
 8597		return PTR_ERR(table);
 8598	}
 8599
 8600	objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
 8601	if (nla[NFTA_OBJ_HANDLE]) {
 8602		attr = nla[NFTA_OBJ_HANDLE];
 8603		obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
 8604	} else {
 8605		attr = nla[NFTA_OBJ_NAME];
 8606		obj = nft_obj_lookup(net, table, attr, objtype, genmask);
 8607	}
 8608
 8609	if (IS_ERR(obj)) {
 8610		if (PTR_ERR(obj) == -ENOENT &&
 8611		    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYOBJ)
 8612			return 0;
 8613
 8614		NL_SET_BAD_ATTR(extack, attr);
 8615		return PTR_ERR(obj);
 8616	}
 8617	if (obj->use > 0) {
 8618		NL_SET_BAD_ATTR(extack, attr);
 8619		return -EBUSY;
 8620	}
 8621
 8622	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 8623
 8624	return nft_delobj(&ctx, obj);
 8625}
 8626
 8627static void
 8628__nft_obj_notify(struct net *net, const struct nft_table *table,
 8629		 struct nft_object *obj, u32 portid, u32 seq, int event,
 8630		 u16 flags, int family, int report, gfp_t gfp)
 8631{
 8632	struct nftables_pernet *nft_net = nft_pernet(net);
 8633	struct sk_buff *skb;
 8634	int err;
 8635
 8636	if (!report &&
 8637	    !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
 8638		return;
 8639
 8640	skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
 8641	if (skb == NULL)
 8642		goto err;
 8643
 8644	err = nf_tables_fill_obj_info(skb, net, portid, seq, event,
 8645				      flags & (NLM_F_CREATE | NLM_F_EXCL),
 8646				      family, table, obj, false);
 8647	if (err < 0) {
 8648		kfree_skb(skb);
 8649		goto err;
 8650	}
 8651
 8652	nft_notify_enqueue(skb, report, &nft_net->notify_list);
 8653	return;
 8654err:
 8655	nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
 8656}
 8657
 8658void nft_obj_notify(struct net *net, const struct nft_table *table,
 8659		    struct nft_object *obj, u32 portid, u32 seq, int event,
 8660		    u16 flags, int family, int report, gfp_t gfp)
 8661{
 8662	char *buf = kasprintf(gfp, "%s:%u",
 8663			      table->name, nft_base_seq(net));
 8664
 8665	audit_log_nfcfg(buf,
 8666			family,
 8667			obj->handle,
 8668			event == NFT_MSG_NEWOBJ ?
 8669				 AUDIT_NFT_OP_OBJ_REGISTER :
 8670				 AUDIT_NFT_OP_OBJ_UNREGISTER,
 8671			gfp);
 8672	kfree(buf);
 8673
 8674	__nft_obj_notify(net, table, obj, portid, seq, event,
 8675			 flags, family, report, gfp);
 8676}
 8677EXPORT_SYMBOL_GPL(nft_obj_notify);
 8678
 8679static void nf_tables_obj_notify(const struct nft_ctx *ctx,
 8680				 struct nft_object *obj, int event)
 8681{
 8682	__nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid,
 8683			 ctx->seq, event, ctx->flags, ctx->family,
 8684			 ctx->report, GFP_KERNEL);
 8685}
 8686
 8687/*
 8688 * Flow tables
 8689 */
 8690void nft_register_flowtable_type(struct nf_flowtable_type *type)
 8691{
 8692	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 8693	list_add_tail_rcu(&type->list, &nf_tables_flowtables);
 8694	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 8695}
 8696EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
 8697
 8698void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
 8699{
 8700	nfnl_lock(NFNL_SUBSYS_NFTABLES);
 8701	list_del_rcu(&type->list);
 8702	nfnl_unlock(NFNL_SUBSYS_NFTABLES);
 8703}
 8704EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
 8705
 8706static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
 8707	[NFTA_FLOWTABLE_TABLE]		= { .type = NLA_STRING,
 8708					    .len = NFT_NAME_MAXLEN - 1 },
 8709	[NFTA_FLOWTABLE_NAME]		= { .type = NLA_STRING,
 8710					    .len = NFT_NAME_MAXLEN - 1 },
 8711	[NFTA_FLOWTABLE_HOOK]		= { .type = NLA_NESTED },
 8712	[NFTA_FLOWTABLE_HANDLE]		= { .type = NLA_U64 },
 8713	[NFTA_FLOWTABLE_FLAGS]		= { .type = NLA_U32 },
 8714};
 8715
 8716struct nft_flowtable *nft_flowtable_lookup(const struct net *net,
 8717					   const struct nft_table *table,
 8718					   const struct nlattr *nla, u8 genmask)
 8719{
 8720	struct nft_flowtable *flowtable;
 8721
 8722	list_for_each_entry_rcu(flowtable, &table->flowtables, list,
 8723				lockdep_commit_lock_is_held(net)) {
 8724		if (!nla_strcmp(nla, flowtable->name) &&
 8725		    nft_active_genmask(flowtable, genmask))
 8726			return flowtable;
 8727	}
 8728	return ERR_PTR(-ENOENT);
 8729}
 8730EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
 8731
 8732void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
 8733				    struct nft_flowtable *flowtable,
 8734				    enum nft_trans_phase phase)
 8735{
 8736	switch (phase) {
 8737	case NFT_TRANS_PREPARE_ERROR:
 8738	case NFT_TRANS_PREPARE:
 8739	case NFT_TRANS_ABORT:
 8740	case NFT_TRANS_RELEASE:
 8741		nft_use_dec(&flowtable->use);
 8742		fallthrough;
 8743	default:
 8744		return;
 8745	}
 8746}
 8747EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
 8748
 8749static struct nft_flowtable *
 8750nft_flowtable_lookup_byhandle(const struct nft_table *table,
 8751			      const struct nlattr *nla, u8 genmask)
 8752{
 8753       struct nft_flowtable *flowtable;
 8754
 8755       list_for_each_entry(flowtable, &table->flowtables, list) {
 8756               if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
 8757                   nft_active_genmask(flowtable, genmask))
 8758                       return flowtable;
 8759       }
 8760       return ERR_PTR(-ENOENT);
 8761}
 8762
 8763struct nft_flowtable_hook {
 8764	u32			num;
 8765	int			priority;
 8766	struct list_head	list;
 8767};
 8768
 8769static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
 8770	[NFTA_FLOWTABLE_HOOK_NUM]	= { .type = NLA_U32 },
 8771	[NFTA_FLOWTABLE_HOOK_PRIORITY]	= { .type = NLA_U32 },
 8772	[NFTA_FLOWTABLE_HOOK_DEVS]	= { .type = NLA_NESTED },
 8773};
 8774
 8775static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
 8776				    const struct nlattr * const nla[],
 8777				    struct nft_flowtable_hook *flowtable_hook,
 8778				    struct nft_flowtable *flowtable,
 8779				    struct netlink_ext_ack *extack, bool add)
 8780{
 8781	struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
 8782	struct nf_hook_ops *ops;
 8783	struct nft_hook *hook;
 8784	int hooknum, priority;
 8785	int err;
 8786
 8787	INIT_LIST_HEAD(&flowtable_hook->list);
 8788
 8789	err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX,
 8790					  nla[NFTA_FLOWTABLE_HOOK],
 8791					  nft_flowtable_hook_policy, NULL);
 8792	if (err < 0)
 8793		return err;
 8794
 8795	if (add) {
 8796		if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
 8797		    !tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
 8798			NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
 8799			return -ENOENT;
 8800		}
 8801
 8802		hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
 8803		if (hooknum != NF_NETDEV_INGRESS)
 8804			return -EOPNOTSUPP;
 8805
 8806		priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
 8807
 8808		flowtable_hook->priority	= priority;
 8809		flowtable_hook->num		= hooknum;
 8810	} else {
 8811		if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
 8812			hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
 8813			if (hooknum != flowtable->hooknum)
 8814				return -EOPNOTSUPP;
 8815		}
 8816
 8817		if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
 8818			priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
 8819			if (priority != flowtable->data.priority)
 8820				return -EOPNOTSUPP;
 8821		}
 8822
 8823		flowtable_hook->priority	= flowtable->data.priority;
 8824		flowtable_hook->num		= flowtable->hooknum;
 8825	}
 8826
 8827	if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
 8828		err = nf_tables_parse_netdev_hooks(ctx->net,
 8829						   tb[NFTA_FLOWTABLE_HOOK_DEVS],
 8830						   &flowtable_hook->list,
 8831						   extack);
 8832		if (err < 0)
 8833			return err;
 8834	}
 8835
 8836	list_for_each_entry(hook, &flowtable_hook->list, list) {
 8837		list_for_each_entry(ops, &hook->ops_list, list) {
 8838			ops->pf			= NFPROTO_NETDEV;
 8839			ops->hooknum		= flowtable_hook->num;
 8840			ops->priority		= flowtable_hook->priority;
 8841			ops->priv		= &flowtable->data;
 8842			ops->hook		= flowtable->data.type->hook;
 8843			ops->hook_ops_type	= NF_HOOK_OP_NFT_FT;
 8844		}
 8845	}
 8846
 8847	return err;
 8848}
 8849
 8850/* call under rcu_read_lock */
 8851static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
 8852{
 8853	const struct nf_flowtable_type *type;
 8854
 8855	list_for_each_entry_rcu(type, &nf_tables_flowtables, list) {
 8856		if (family == type->family)
 8857			return type;
 8858	}
 8859	return NULL;
 8860}
 8861
 8862static const struct nf_flowtable_type *
 8863nft_flowtable_type_get(struct net *net, u8 family)
 8864{
 8865	const struct nf_flowtable_type *type;
 8866
 8867	rcu_read_lock();
 8868	type = __nft_flowtable_type_get(family);
 8869	if (type != NULL && try_module_get(type->owner)) {
 8870		rcu_read_unlock();
 8871		return type;
 8872	}
 8873	rcu_read_unlock();
 8874
 8875	lockdep_nfnl_nft_mutex_not_held();
 8876#ifdef CONFIG_MODULES
 8877	if (type == NULL) {
 8878		if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
 8879			return ERR_PTR(-EAGAIN);
 8880	}
 8881#endif
 8882	return ERR_PTR(-ENOENT);
 8883}
 8884
 8885/* Only called from error and netdev event paths. */
 8886static void nft_unregister_flowtable_ops(struct net *net,
 8887					 struct nft_flowtable *flowtable,
 8888					 struct nf_hook_ops *ops)
 8889{
 8890	nf_unregister_net_hook(net, ops);
 8891	flowtable->data.type->setup(&flowtable->data, ops->dev,
 8892				    FLOW_BLOCK_UNBIND);
 8893}
 8894
 8895static void __nft_unregister_flowtable_net_hooks(struct net *net,
 8896						 struct nft_flowtable *flowtable,
 8897						 struct list_head *hook_list,
 8898					         bool release_netdev)
 8899{
 8900	struct nft_hook *hook, *next;
 8901	struct nf_hook_ops *ops;
 8902
 8903	list_for_each_entry_safe(hook, next, hook_list, list) {
 8904		list_for_each_entry(ops, &hook->ops_list, list)
 8905			nft_unregister_flowtable_ops(net, flowtable, ops);
 8906		if (release_netdev) {
 8907			list_del(&hook->list);
 8908			nft_netdev_hook_free_rcu(hook);
 8909		}
 8910	}
 8911}
 8912
 8913static void nft_unregister_flowtable_net_hooks(struct net *net,
 8914					       struct nft_flowtable *flowtable,
 8915					       struct list_head *hook_list)
 8916{
 8917	__nft_unregister_flowtable_net_hooks(net, flowtable, hook_list, false);
 8918}
 8919
 8920static int nft_register_flowtable_ops(struct net *net,
 8921				      struct nft_flowtable *flowtable,
 8922				      struct nf_hook_ops *ops)
 8923{
 8924	int err;
 8925
 8926	err = flowtable->data.type->setup(&flowtable->data,
 8927					  ops->dev, FLOW_BLOCK_BIND);
 8928	if (err < 0)
 8929		return err;
 8930
 8931	err = nf_register_net_hook(net, ops);
 8932	if (!err)
 8933		return 0;
 8934
 8935	flowtable->data.type->setup(&flowtable->data,
 8936				    ops->dev, FLOW_BLOCK_UNBIND);
 8937	return err;
 8938}
 8939
 8940static int nft_register_flowtable_net_hooks(struct net *net,
 8941					    struct nft_table *table,
 8942					    struct list_head *hook_list,
 8943					    struct nft_flowtable *flowtable)
 8944{
 8945	struct nft_hook *hook, *next;
 8946	struct nft_flowtable *ft;
 8947	struct nf_hook_ops *ops;
 8948	int err, i = 0;
 8949
 8950	list_for_each_entry(hook, hook_list, list) {
 8951		list_for_each_entry(ft, &table->flowtables, list) {
 8952			if (!nft_is_active_next(net, ft))
 8953				continue;
 8954
 8955			if (nft_hook_list_find(&ft->hook_list, hook)) {
 8956				err = -EEXIST;
 8957				goto err_unregister_net_hooks;
 8958			}
 8959		}
 8960
 8961		list_for_each_entry(ops, &hook->ops_list, list) {
 8962			err = nft_register_flowtable_ops(net, flowtable, ops);
 8963			if (err < 0)
 8964				goto err_unregister_net_hooks;
 8965
 8966			i++;
 8967		}
 8968	}
 8969
 8970	return 0;
 8971
 8972err_unregister_net_hooks:
 8973	list_for_each_entry_safe(hook, next, hook_list, list) {
 8974		list_for_each_entry(ops, &hook->ops_list, list) {
 8975			if (i-- <= 0)
 8976				break;
 8977
 8978			nft_unregister_flowtable_ops(net, flowtable, ops);
 8979		}
 8980		list_del_rcu(&hook->list);
 8981		nft_netdev_hook_free_rcu(hook);
 8982	}
 8983
 8984	return err;
 8985}
 8986
 8987static void nft_hooks_destroy(struct list_head *hook_list)
 8988{
 8989	struct nft_hook *hook, *next;
 8990
 8991	list_for_each_entry_safe(hook, next, hook_list, list) {
 8992		list_del_rcu(&hook->list);
 8993		nft_netdev_hook_free_rcu(hook);
 8994	}
 8995}
 8996
 8997static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
 8998				struct nft_flowtable *flowtable,
 8999				struct netlink_ext_ack *extack)
 9000{
 9001	const struct nlattr * const *nla = ctx->nla;
 9002	struct nft_flowtable_hook flowtable_hook;
 9003	struct nftables_pernet *nft_net;
 9004	struct nft_hook *hook, *next;
 9005	struct nf_hook_ops *ops;
 9006	struct nft_trans *trans;
 9007	bool unregister = false;
 9008	u32 flags;
 9009	int err;
 9010
 9011	err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
 9012				       extack, false);
 9013	if (err < 0)
 9014		return err;
 9015
 9016	list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
 9017		if (nft_hook_list_find(&flowtable->hook_list, hook)) {
 9018			list_del(&hook->list);
 9019			nft_netdev_hook_free(hook);
 9020			continue;
 9021		}
 9022
 9023		nft_net = nft_pernet(ctx->net);
 9024		list_for_each_entry(trans, &nft_net->commit_list, list) {
 9025			if (trans->msg_type != NFT_MSG_NEWFLOWTABLE ||
 9026			    trans->table != ctx->table ||
 9027			    !nft_trans_flowtable_update(trans))
 9028				continue;
 9029
 9030			if (nft_hook_list_find(&nft_trans_flowtable_hooks(trans), hook)) {
 9031				err = -EEXIST;
 9032				goto err_flowtable_update_hook;
 9033			}
 9034		}
 9035	}
 9036
 9037	if (nla[NFTA_FLOWTABLE_FLAGS]) {
 9038		flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
 9039		if (flags & ~NFT_FLOWTABLE_MASK) {
 9040			err = -EOPNOTSUPP;
 9041			goto err_flowtable_update_hook;
 9042		}
 9043		if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
 9044		    (flags & NFT_FLOWTABLE_HW_OFFLOAD)) {
 9045			err = -EOPNOTSUPP;
 9046			goto err_flowtable_update_hook;
 9047		}
 9048	} else {
 9049		flags = flowtable->data.flags;
 9050	}
 9051
 9052	err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
 9053					       &flowtable_hook.list, flowtable);
 9054	if (err < 0)
 9055		goto err_flowtable_update_hook;
 9056
 9057	trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
 9058				sizeof(struct nft_trans_flowtable));
 9059	if (!trans) {
 9060		unregister = true;
 9061		err = -ENOMEM;
 9062		goto err_flowtable_update_hook;
 9063	}
 9064
 9065	nft_trans_flowtable_flags(trans) = flags;
 9066	nft_trans_flowtable(trans) = flowtable;
 9067	nft_trans_flowtable_update(trans) = true;
 9068	INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 9069	list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
 9070
 9071	nft_trans_commit_list_add_tail(ctx->net, trans);
 9072
 9073	return 0;
 9074
 9075err_flowtable_update_hook:
 9076	list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
 9077		if (unregister) {
 9078			list_for_each_entry(ops, &hook->ops_list, list)
 9079				nft_unregister_flowtable_ops(ctx->net,
 9080							     flowtable, ops);
 9081		}
 9082		list_del_rcu(&hook->list);
 9083		nft_netdev_hook_free_rcu(hook);
 9084	}
 9085
 9086	return err;
 9087
 9088}
 9089
 9090static int nf_tables_newflowtable(struct sk_buff *skb,
 9091				  const struct nfnl_info *info,
 9092				  const struct nlattr * const nla[])
 9093{
 9094	struct netlink_ext_ack *extack = info->extack;
 9095	struct nft_flowtable_hook flowtable_hook;
 9096	u8 genmask = nft_genmask_next(info->net);
 9097	u8 family = info->nfmsg->nfgen_family;
 9098	const struct nf_flowtable_type *type;
 9099	struct nft_flowtable *flowtable;
 9100	struct net *net = info->net;
 9101	struct nft_table *table;
 9102	struct nft_trans *trans;
 9103	struct nft_ctx ctx;
 9104	int err;
 9105
 9106	if (!nla[NFTA_FLOWTABLE_TABLE] ||
 9107	    !nla[NFTA_FLOWTABLE_NAME] ||
 9108	    !nla[NFTA_FLOWTABLE_HOOK])
 9109		return -EINVAL;
 9110
 9111	table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
 9112				 genmask, NETLINK_CB(skb).portid);
 9113	if (IS_ERR(table)) {
 9114		NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
 9115		return PTR_ERR(table);
 9116	}
 9117
 9118	flowtable = nft_flowtable_lookup(net, table, nla[NFTA_FLOWTABLE_NAME],
 9119					 genmask);
 9120	if (IS_ERR(flowtable)) {
 9121		err = PTR_ERR(flowtable);
 9122		if (err != -ENOENT) {
 9123			NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
 9124			return err;
 9125		}
 9126	} else {
 9127		if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
 9128			NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
 9129			return -EEXIST;
 9130		}
 9131
 9132		nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 9133
 9134		return nft_flowtable_update(&ctx, info->nlh, flowtable, extack);
 9135	}
 9136
 9137	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 9138
 9139	if (!nft_use_inc(&table->use))
 9140		return -EMFILE;
 9141
 9142	flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL_ACCOUNT);
 9143	if (!flowtable) {
 9144		err = -ENOMEM;
 9145		goto flowtable_alloc;
 9146	}
 9147
 9148	flowtable->table = table;
 9149	flowtable->handle = nf_tables_alloc_handle(table);
 9150	INIT_LIST_HEAD(&flowtable->hook_list);
 9151
 9152	flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL_ACCOUNT);
 9153	if (!flowtable->name) {
 9154		err = -ENOMEM;
 9155		goto err1;
 9156	}
 9157
 9158	type = nft_flowtable_type_get(net, family);
 9159	if (IS_ERR(type)) {
 9160		err = PTR_ERR(type);
 9161		goto err2;
 9162	}
 9163
 9164	if (nla[NFTA_FLOWTABLE_FLAGS]) {
 9165		flowtable->data.flags =
 9166			ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
 9167		if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
 9168			err = -EOPNOTSUPP;
 9169			goto err3;
 9170		}
 9171	}
 9172
 9173	write_pnet(&flowtable->data.net, net);
 9174	flowtable->data.type = type;
 9175	err = type->init(&flowtable->data);
 9176	if (err < 0)
 9177		goto err3;
 9178
 9179	err = nft_flowtable_parse_hook(&ctx, nla, &flowtable_hook, flowtable,
 9180				       extack, true);
 9181	if (err < 0)
 9182		goto err_flowtable_parse_hooks;
 9183
 9184	list_splice(&flowtable_hook.list, &flowtable->hook_list);
 9185	flowtable->data.priority = flowtable_hook.priority;
 9186	flowtable->hooknum = flowtable_hook.num;
 9187
 9188	trans = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
 9189	if (IS_ERR(trans)) {
 9190		err = PTR_ERR(trans);
 9191		goto err_flowtable_trans;
 9192	}
 9193
 9194	/* This must be LAST to ensure no packets are walking over this flowtable. */
 9195	err = nft_register_flowtable_net_hooks(ctx.net, table,
 9196					       &flowtable->hook_list,
 9197					       flowtable);
 9198	if (err < 0)
 9199		goto err_flowtable_hooks;
 9200
 9201	list_add_tail_rcu(&flowtable->list, &table->flowtables);
 9202
 9203	return 0;
 9204
 9205err_flowtable_hooks:
 9206	synchronize_rcu();
 9207	nft_trans_destroy(trans);
 9208err_flowtable_trans:
 9209	nft_hooks_destroy(&flowtable->hook_list);
 9210err_flowtable_parse_hooks:
 9211	flowtable->data.type->free(&flowtable->data);
 9212err3:
 9213	module_put(type->owner);
 9214err2:
 9215	kfree(flowtable->name);
 9216err1:
 9217	kfree(flowtable);
 9218flowtable_alloc:
 9219	nft_use_dec_restore(&table->use);
 9220
 9221	return err;
 9222}
 9223
 9224static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
 9225{
 9226	struct nft_hook *this, *next;
 9227
 9228	list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
 9229		list_del(&this->list);
 9230		nft_netdev_hook_free(this);
 9231	}
 9232}
 9233
 9234static int nft_delflowtable_hook(struct nft_ctx *ctx,
 9235				 struct nft_flowtable *flowtable,
 9236				 struct netlink_ext_ack *extack)
 9237{
 9238	const struct nlattr * const *nla = ctx->nla;
 9239	struct nft_flowtable_hook flowtable_hook;
 9240	LIST_HEAD(flowtable_del_list);
 9241	struct nft_hook *this, *hook;
 9242	struct nft_trans *trans;
 9243	int err;
 9244
 9245	err = nft_flowtable_parse_hook(ctx, nla, &flowtable_hook, flowtable,
 9246				       extack, false);
 9247	if (err < 0)
 9248		return err;
 9249
 9250	list_for_each_entry(this, &flowtable_hook.list, list) {
 9251		hook = nft_hook_list_find(&flowtable->hook_list, this);
 9252		if (!hook) {
 9253			err = -ENOENT;
 9254			goto err_flowtable_del_hook;
 9255		}
 9256		list_move(&hook->list, &flowtable_del_list);
 9257	}
 9258
 9259	trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
 9260				sizeof(struct nft_trans_flowtable));
 9261	if (!trans) {
 9262		err = -ENOMEM;
 9263		goto err_flowtable_del_hook;
 9264	}
 9265
 9266	nft_trans_flowtable(trans) = flowtable;
 9267	nft_trans_flowtable_update(trans) = true;
 9268	INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
 9269	list_splice(&flowtable_del_list, &nft_trans_flowtable_hooks(trans));
 9270	nft_flowtable_hook_release(&flowtable_hook);
 9271
 9272	nft_trans_commit_list_add_tail(ctx->net, trans);
 9273
 9274	return 0;
 9275
 9276err_flowtable_del_hook:
 9277	list_splice(&flowtable_del_list, &flowtable->hook_list);
 9278	nft_flowtable_hook_release(&flowtable_hook);
 9279
 9280	return err;
 9281}
 9282
 9283static int nf_tables_delflowtable(struct sk_buff *skb,
 9284				  const struct nfnl_info *info,
 9285				  const struct nlattr * const nla[])
 9286{
 9287	struct netlink_ext_ack *extack = info->extack;
 9288	u8 genmask = nft_genmask_next(info->net);
 9289	u8 family = info->nfmsg->nfgen_family;
 9290	struct nft_flowtable *flowtable;
 9291	struct net *net = info->net;
 9292	const struct nlattr *attr;
 9293	struct nft_table *table;
 9294	struct nft_ctx ctx;
 9295
 9296	if (!nla[NFTA_FLOWTABLE_TABLE] ||
 9297	    (!nla[NFTA_FLOWTABLE_NAME] &&
 9298	     !nla[NFTA_FLOWTABLE_HANDLE]))
 9299		return -EINVAL;
 9300
 9301	table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
 9302				 genmask, NETLINK_CB(skb).portid);
 9303	if (IS_ERR(table)) {
 9304		NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
 9305		return PTR_ERR(table);
 9306	}
 9307
 9308	if (nla[NFTA_FLOWTABLE_HANDLE]) {
 9309		attr = nla[NFTA_FLOWTABLE_HANDLE];
 9310		flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
 9311	} else {
 9312		attr = nla[NFTA_FLOWTABLE_NAME];
 9313		flowtable = nft_flowtable_lookup(net, table, attr, genmask);
 9314	}
 9315
 9316	if (IS_ERR(flowtable)) {
 9317		if (PTR_ERR(flowtable) == -ENOENT &&
 9318		    NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_DESTROYFLOWTABLE)
 9319			return 0;
 9320
 9321		NL_SET_BAD_ATTR(extack, attr);
 9322		return PTR_ERR(flowtable);
 9323	}
 9324
 9325	nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
 9326
 9327	if (nla[NFTA_FLOWTABLE_HOOK])
 9328		return nft_delflowtable_hook(&ctx, flowtable, extack);
 9329
 9330	if (flowtable->use > 0) {
 9331		NL_SET_BAD_ATTR(extack, attr);
 9332		return -EBUSY;
 9333	}
 9334
 9335	return nft_delflowtable(&ctx, flowtable);
 9336}
 9337
 9338static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
 9339					 u32 portid, u32 seq, int event,
 9340					 u32 flags, int family,
 9341					 struct nft_flowtable *flowtable,
 9342					 struct list_head *hook_list)
 9343{
 9344	struct nlattr *nest, *nest_devs;
 9345	struct nft_hook *hook;
 9346	struct nlmsghdr *nlh;
 9347
 9348	nlh = nfnl_msg_put(skb, portid, seq,
 9349			   nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event),
 9350			   flags, family, NFNETLINK_V0, nft_base_seq_be16(net));
 9351	if (!nlh)
 9352		goto nla_put_failure;
 9353
 9354	if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
 9355	    nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
 9356	    nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
 9357			 NFTA_FLOWTABLE_PAD))
 9358		goto nla_put_failure;
 9359
 9360	if (!hook_list &&
 9361	    (event == NFT_MSG_DELFLOWTABLE ||
 9362	     event == NFT_MSG_DESTROYFLOWTABLE)) {
 9363		nlmsg_end(skb, nlh);
 9364		return 0;
 9365	}
 9366
 9367	if (nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
 9368	    nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
 9369		goto nla_put_failure;
 9370
 9371	nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
 9372	if (!nest)
 9373		goto nla_put_failure;
 9374	if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
 9375	    nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
 9376		goto nla_put_failure;
 9377
 9378	nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
 9379	if (!nest_devs)
 9380		goto nla_put_failure;
 9381
 9382	if (!hook_list)
 9383		hook_list = &flowtable->hook_list;
 9384
 9385	list_for_each_entry_rcu(hook, hook_list, list,
 9386				lockdep_commit_lock_is_held(net)) {
 9387		if (nft_nla_put_hook_dev(skb, hook))
 9388			goto nla_put_failure;
 9389	}
 9390	nla_nest_end(skb, nest_devs);
 9391	nla_nest_end(skb, nest);
 9392
 9393	nlmsg_end(skb, nlh);
 9394	return 0;
 9395
 9396nla_put_failure:
 9397	nlmsg_trim(skb, nlh);
 9398	return -1;
 9399}
 9400
 9401struct nft_flowtable_filter {
 9402	char		*table;
 9403};
 9404
 9405static int nf_tables_dump_flowtable(struct sk_buff *skb,
 9406				    struct netlink_callback *cb)
 9407{
 9408	const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 9409	struct nft_flowtable_filter *filter = cb->data;
 9410	unsigned int idx = 0, s_idx = cb->args[0];
 9411	struct net *net = sock_net(skb->sk);
 9412	int family = nfmsg->nfgen_family;
 9413	struct nft_flowtable *flowtable;
 9414	struct nftables_pernet *nft_net;
 9415	const struct nft_table *table;
 9416
 9417	rcu_read_lock();
 9418	nft_net = nft_pernet(net);
 9419	cb->seq = nft_base_seq(net);
 9420
 9421	list_for_each_entry_rcu(table, &nft_net->tables, list) {
 9422		if (family != NFPROTO_UNSPEC && family != table->family)
 9423			continue;
 9424
 9425		list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
 9426			if (!nft_is_active(net, flowtable))
 9427				goto cont;
 9428			if (idx < s_idx)
 9429				goto cont;
 9430			if (idx > s_idx)
 9431				memset(&cb->args[1], 0,
 9432				       sizeof(cb->args) - sizeof(cb->args[0]));
 9433			if (filter && filter->table &&
 9434			    strcmp(filter->table, table->name))
 9435				goto cont;
 9436
 9437			if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
 9438							  cb->nlh->nlmsg_seq,
 9439							  NFT_MSG_NEWFLOWTABLE,
 9440							  NLM_F_MULTI | NLM_F_APPEND,
 9441							  table->family,
 9442							  flowtable, NULL) < 0)
 9443				goto done;
 9444
 9445			nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 9446cont:
 9447			idx++;
 9448		}
 9449	}
 9450done:
 9451	rcu_read_unlock();
 9452
 9453	cb->args[0] = idx;
 9454	return skb->len;
 9455}
 9456
 9457static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
 9458{
 9459	const struct nlattr * const *nla = cb->data;
 9460	struct nft_flowtable_filter *filter = NULL;
 9461
 9462	if (nla[NFTA_FLOWTABLE_TABLE]) {
 9463		filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
 9464		if (!filter)
 9465			return -ENOMEM;
 9466
 9467		filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
 9468					   GFP_ATOMIC);
 9469		if (!filter->table) {
 9470			kfree(filter);
 9471			return -ENOMEM;
 9472		}
 9473	}
 9474
 9475	cb->data = filter;
 9476	return 0;
 9477}
 9478
 9479static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
 9480{
 9481	struct nft_flowtable_filter *filter = cb->data;
 9482
 9483	if (!filter)
 9484		return 0;
 9485
 9486	kfree(filter->table);
 9487	kfree(filter);
 9488
 9489	return 0;
 9490}
 9491
 9492/* called with rcu_read_lock held */
 9493static int nf_tables_getflowtable(struct sk_buff *skb,
 9494				  const struct nfnl_info *info,
 9495				  const struct nlattr * const nla[])
 9496{
 9497	struct netlink_ext_ack *extack = info->extack;
 9498	u8 genmask = nft_genmask_cur(info->net);
 9499	u8 family = info->nfmsg->nfgen_family;
 9500	struct nft_flowtable *flowtable;
 9501	const struct nft_table *table;
 9502	struct net *net = info->net;
 9503	struct sk_buff *skb2;
 9504	int err;
 9505
 9506	if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 9507		struct netlink_dump_control c = {
 9508			.start = nf_tables_dump_flowtable_start,
 9509			.dump = nf_tables_dump_flowtable,
 9510			.done = nf_tables_dump_flowtable_done,
 9511			.module = THIS_MODULE,
 9512			.data = (void *)nla,
 9513		};
 9514
 9515		return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 9516	}
 9517
 9518	if (!nla[NFTA_FLOWTABLE_NAME])
 9519		return -EINVAL;
 9520
 9521	table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
 9522				 genmask, 0);
 9523	if (IS_ERR(table)) {
 9524		NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
 9525		return PTR_ERR(table);
 9526	}
 9527
 9528	flowtable = nft_flowtable_lookup(net, table, nla[NFTA_FLOWTABLE_NAME],
 9529					 genmask);
 9530	if (IS_ERR(flowtable)) {
 9531		NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
 9532		return PTR_ERR(flowtable);
 9533	}
 9534
 9535	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 9536	if (!skb2)
 9537		return -ENOMEM;
 9538
 9539	err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
 9540					    info->nlh->nlmsg_seq,
 9541					    NFT_MSG_NEWFLOWTABLE, 0, family,
 9542					    flowtable, NULL);
 9543	if (err < 0)
 9544		goto err_fill_flowtable_info;
 9545
 9546	return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 9547
 9548err_fill_flowtable_info:
 9549	kfree_skb(skb2);
 9550	return err;
 9551}
 9552
 9553static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
 9554				       struct nft_flowtable *flowtable,
 9555				       struct list_head *hook_list, int event)
 9556{
 9557	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
 9558	struct sk_buff *skb;
 9559	u16 flags = 0;
 9560	int err;
 9561
 9562	if (!ctx->report &&
 9563	    !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 9564		return;
 9565
 9566	skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 9567	if (skb == NULL)
 9568		goto err;
 9569
 9570	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
 9571		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);
 9572
 9573	err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
 9574					    ctx->seq, event, flags,
 9575					    ctx->family, flowtable, hook_list);
 9576	if (err < 0) {
 9577		kfree_skb(skb);
 9578		goto err;
 9579	}
 9580
 9581	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 9582	return;
 9583err:
 9584	nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 9585}
 9586
 9587static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
 9588{
 9589	struct nft_hook *hook, *next;
 9590
 9591	flowtable->data.type->free(&flowtable->data);
 9592	list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
 9593		list_del_rcu(&hook->list);
 9594		nft_netdev_hook_free_rcu(hook);
 9595	}
 9596	kfree(flowtable->name);
 9597	module_put(flowtable->data.type->owner);
 9598	kfree(flowtable);
 9599}
 9600
 9601static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
 9602				   u32 portid, u32 seq)
 9603{
 9604	struct nlmsghdr *nlh;
 9605	char buf[TASK_COMM_LEN];
 9606	int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
 9607
 9608	nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
 9609			   NFNETLINK_V0, nft_base_seq_be16(net));
 9610	if (!nlh)
 9611		goto nla_put_failure;
 9612
 9613	if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_base_seq(net))) ||
 9614	    nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
 9615	    nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
 9616		goto nla_put_failure;
 9617
 9618	nlmsg_end(skb, nlh);
 9619	return 0;
 9620
 9621nla_put_failure:
 9622	nlmsg_trim(skb, nlh);
 9623	return -EMSGSIZE;
 9624}
 9625
 9626struct nf_hook_ops *nft_hook_find_ops(const struct nft_hook *hook,
 9627				      const struct net_device *dev)
 9628{
 9629	struct nf_hook_ops *ops;
 9630
 9631	list_for_each_entry(ops, &hook->ops_list, list) {
 9632		if (ops->dev == dev)
 9633			return ops;
 9634	}
 9635	return NULL;
 9636}
 9637EXPORT_SYMBOL_GPL(nft_hook_find_ops);
 9638
 9639struct nf_hook_ops *nft_hook_find_ops_rcu(const struct nft_hook *hook,
 9640					  const struct net_device *dev)
 9641{
 9642	struct nf_hook_ops *ops;
 9643
 9644	list_for_each_entry_rcu(ops, &hook->ops_list, list) {
 9645		if (ops->dev == dev)
 9646			return ops;
 9647	}
 9648	return NULL;
 9649}
 9650EXPORT_SYMBOL_GPL(nft_hook_find_ops_rcu);
 9651
 9652static int nft_flowtable_event(unsigned long event, struct net_device *dev,
 9653			       struct nft_flowtable *flowtable, bool changename)
 9654{
 9655	struct nf_hook_ops *ops;
 9656	struct nft_hook *hook;
 9657	bool match;
 9658
 9659	list_for_each_entry(hook, &flowtable->hook_list, list) {
 9660		ops = nft_hook_find_ops(hook, dev);
 9661		match = !strncmp(hook->ifname, dev->name, hook->ifnamelen);
 9662
 9663		switch (event) {
 9664		case NETDEV_UNREGISTER:
 9665			/* NOP if not found or new name still matching */
 9666			if (!ops || (changename && match))
 9667				continue;
 9668
 9669			/* flow_offload_netdev_event() cleans up entries for us. */
 9670			nft_unregister_flowtable_ops(dev_net(dev),
 9671						     flowtable, ops);
 9672			list_del_rcu(&ops->list);
 9673			kfree_rcu(ops, rcu);
 9674			break;
 9675		case NETDEV_REGISTER:
 9676			/* NOP if not matching or already registered */
 9677			if (!match || ops)
 9678				continue;
 9679
 9680			ops = kzalloc(sizeof(struct nf_hook_ops),
 9681				      GFP_KERNEL_ACCOUNT);
 9682			if (!ops)
 9683				return 1;
 9684
 9685			ops->pf			= NFPROTO_NETDEV;
 9686			ops->hooknum		= flowtable->hooknum;
 9687			ops->priority		= flowtable->data.priority;
 9688			ops->priv		= &flowtable->data;
 9689			ops->hook		= flowtable->data.type->hook;
 9690			ops->hook_ops_type	= NF_HOOK_OP_NFT_FT;
 9691			ops->dev		= dev;
 9692			if (nft_register_flowtable_ops(dev_net(dev),
 9693						       flowtable, ops)) {
 9694				kfree(ops);
 9695				return 1;
 9696			}
 9697			list_add_tail_rcu(&ops->list, &hook->ops_list);
 9698			break;
 9699		}
 9700		break;
 9701	}
 9702	return 0;
 9703}
 9704
 9705static int __nf_tables_flowtable_event(unsigned long event,
 9706				       struct net_device *dev,
 9707				       bool changename)
 9708{
 9709	struct nftables_pernet *nft_net = nft_pernet(dev_net(dev));
 9710	struct nft_flowtable *flowtable;
 9711	struct nft_table *table;
 9712
 9713	list_for_each_entry(table, &nft_net->tables, list) {
 9714		list_for_each_entry(flowtable, &table->flowtables, list) {
 9715			if (nft_flowtable_event(event, dev,
 9716						flowtable, changename))
 9717				return 1;
 9718		}
 9719	}
 9720	return 0;
 9721}
 9722
 9723static int nf_tables_flowtable_event(struct notifier_block *this,
 9724				     unsigned long event, void *ptr)
 9725{
 9726	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 9727	struct nftables_pernet *nft_net;
 9728	int ret = NOTIFY_DONE;
 9729	struct net *net;
 9730
 9731	if (event != NETDEV_REGISTER &&
 9732	    event != NETDEV_UNREGISTER &&
 9733	    event != NETDEV_CHANGENAME)
 9734		return NOTIFY_DONE;
 9735
 9736	net = dev_net(dev);
 9737	nft_net = nft_pernet(net);
 9738	mutex_lock(&nft_net->commit_mutex);
 9739
 9740	if (event == NETDEV_CHANGENAME) {
 9741		if (__nf_tables_flowtable_event(NETDEV_REGISTER, dev, true)) {
 9742			ret = NOTIFY_BAD;
 9743			goto out_unlock;
 9744		}
 9745		__nf_tables_flowtable_event(NETDEV_UNREGISTER, dev, true);
 9746	} else if (__nf_tables_flowtable_event(event, dev, false)) {
 9747		ret = NOTIFY_BAD;
 9748	}
 9749out_unlock:
 9750	mutex_unlock(&nft_net->commit_mutex);
 9751	return ret;
 9752}
 9753
 9754static struct notifier_block nf_tables_flowtable_notifier = {
 9755	.notifier_call	= nf_tables_flowtable_event,
 9756};
 9757
 9758static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
 9759				 int event)
 9760{
 9761	struct nlmsghdr *nlh = nlmsg_hdr(skb);
 9762	struct sk_buff *skb2;
 9763	int err;
 9764
 9765	if (!nlmsg_report(nlh) &&
 9766	    !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
 9767		return;
 9768
 9769	skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 9770	if (skb2 == NULL)
 9771		goto err;
 9772
 9773	err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
 9774				      nlh->nlmsg_seq);
 9775	if (err < 0) {
 9776		kfree_skb(skb2);
 9777		goto err;
 9778	}
 9779
 9780	nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
 9781		       nlmsg_report(nlh), GFP_KERNEL);
 9782	return;
 9783err:
 9784	nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
 9785			  -ENOBUFS);
 9786}
 9787
 9788static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
 9789			    const struct nlattr * const nla[])
 9790{
 9791	struct sk_buff *skb2;
 9792	int err;
 9793
 9794	skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 9795	if (skb2 == NULL)
 9796		return -ENOMEM;
 9797
 9798	err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
 9799				      info->nlh->nlmsg_seq);
 9800	if (err < 0)
 9801		goto err_fill_gen_info;
 9802
 9803	return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
 9804
 9805err_fill_gen_info:
 9806	kfree_skb(skb2);
 9807	return err;
 9808}
 9809
 9810static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
 9811	[NFT_MSG_NEWTABLE] = {
 9812		.call		= nf_tables_newtable,
 9813		.type		= NFNL_CB_BATCH,
 9814		.attr_count	= NFTA_TABLE_MAX,
 9815		.policy		= nft_table_policy,
 9816	},
 9817	[NFT_MSG_GETTABLE] = {
 9818		.call		= nf_tables_gettable,
 9819		.type		= NFNL_CB_RCU,
 9820		.attr_count	= NFTA_TABLE_MAX,
 9821		.policy		= nft_table_policy,
 9822	},
 9823	[NFT_MSG_DELTABLE] = {
 9824		.call		= nf_tables_deltable,
 9825		.type		= NFNL_CB_BATCH,
 9826		.attr_count	= NFTA_TABLE_MAX,
 9827		.policy		= nft_table_policy,
 9828	},
 9829	[NFT_MSG_DESTROYTABLE] = {
 9830		.call		= nf_tables_deltable,
 9831		.type		= NFNL_CB_BATCH,
 9832		.attr_count	= NFTA_TABLE_MAX,
 9833		.policy		= nft_table_policy,
 9834	},
 9835	[NFT_MSG_NEWCHAIN] = {
 9836		.call		= nf_tables_newchain,
 9837		.type		= NFNL_CB_BATCH,
 9838		.attr_count	= NFTA_CHAIN_MAX,
 9839		.policy		= nft_chain_policy,
 9840	},
 9841	[NFT_MSG_GETCHAIN] = {
 9842		.call		= nf_tables_getchain,
 9843		.type		= NFNL_CB_RCU,
 9844		.attr_count	= NFTA_CHAIN_MAX,
 9845		.policy		= nft_chain_policy,
 9846	},
 9847	[NFT_MSG_DELCHAIN] = {
 9848		.call		= nf_tables_delchain,
 9849		.type		= NFNL_CB_BATCH,
 9850		.attr_count	= NFTA_CHAIN_MAX,
 9851		.policy		= nft_chain_policy,
 9852	},
 9853	[NFT_MSG_DESTROYCHAIN] = {
 9854		.call		= nf_tables_delchain,
 9855		.type		= NFNL_CB_BATCH,
 9856		.attr_count	= NFTA_CHAIN_MAX,
 9857		.policy		= nft_chain_policy,
 9858	},
 9859	[NFT_MSG_NEWRULE] = {
 9860		.call		= nf_tables_newrule,
 9861		.type		= NFNL_CB_BATCH,
 9862		.attr_count	= NFTA_RULE_MAX,
 9863		.policy		= nft_rule_policy,
 9864	},
 9865	[NFT_MSG_GETRULE] = {
 9866		.call		= nf_tables_getrule,
 9867		.type		= NFNL_CB_RCU,
 9868		.attr_count	= NFTA_RULE_MAX,
 9869		.policy		= nft_rule_policy,
 9870	},
 9871	[NFT_MSG_GETRULE_RESET] = {
 9872		.call		= nf_tables_getrule,
 9873		.type		= NFNL_CB_RCU,
 9874		.attr_count	= NFTA_RULE_MAX,
 9875		.policy		= nft_rule_policy,
 9876	},
 9877	[NFT_MSG_DELRULE] = {
 9878		.call		= nf_tables_delrule,
 9879		.type		= NFNL_CB_BATCH,
 9880		.attr_count	= NFTA_RULE_MAX,
 9881		.policy		= nft_rule_policy,
 9882	},
 9883	[NFT_MSG_DESTROYRULE] = {
 9884		.call		= nf_tables_delrule,
 9885		.type		= NFNL_CB_BATCH,
 9886		.attr_count	= NFTA_RULE_MAX,
 9887		.policy		= nft_rule_policy,
 9888	},
 9889	[NFT_MSG_NEWSET] = {
 9890		.call		= nf_tables_newset,
 9891		.type		= NFNL_CB_BATCH,
 9892		.attr_count	= NFTA_SET_MAX,
 9893		.policy		= nft_set_policy,
 9894	},
 9895	[NFT_MSG_GETSET] = {
 9896		.call		= nf_tables_getset,
 9897		.type		= NFNL_CB_RCU,
 9898		.attr_count	= NFTA_SET_MAX,
 9899		.policy		= nft_set_policy,
 9900	},
 9901	[NFT_MSG_DELSET] = {
 9902		.call		= nf_tables_delset,
 9903		.type		= NFNL_CB_BATCH,
 9904		.attr_count	= NFTA_SET_MAX,
 9905		.policy		= nft_set_policy,
 9906	},
 9907	[NFT_MSG_DESTROYSET] = {
 9908		.call		= nf_tables_delset,
 9909		.type		= NFNL_CB_BATCH,
 9910		.attr_count	= NFTA_SET_MAX,
 9911		.policy		= nft_set_policy,
 9912	},
 9913	[NFT_MSG_NEWSETELEM] = {
 9914		.call		= nf_tables_newsetelem,
 9915		.type		= NFNL_CB_BATCH,
 9916		.attr_count	= NFTA_SET_ELEM_LIST_MAX,
 9917		.policy		= nft_set_elem_list_policy,
 9918	},
 9919	[NFT_MSG_GETSETELEM] = {
 9920		.call		= nf_tables_getsetelem,
 9921		.type		= NFNL_CB_RCU,
 9922		.attr_count	= NFTA_SET_ELEM_LIST_MAX,
 9923		.policy		= nft_set_elem_list_policy,
 9924	},
 9925	[NFT_MSG_GETSETELEM_RESET] = {
 9926		.call		= nf_tables_getsetelem,
 9927		.type		= NFNL_CB_RCU,
 9928		.attr_count	= NFTA_SET_ELEM_LIST_MAX,
 9929		.policy		= nft_set_elem_list_policy,
 9930	},
 9931	[NFT_MSG_DELSETELEM] = {
 9932		.call		= nf_tables_delsetelem,
 9933		.type		= NFNL_CB_BATCH,
 9934		.attr_count	= NFTA_SET_ELEM_LIST_MAX,
 9935		.policy		= nft_set_elem_list_policy,
 9936	},
 9937	[NFT_MSG_DESTROYSETELEM] = {
 9938		.call		= nf_tables_delsetelem,
 9939		.type		= NFNL_CB_BATCH,
 9940		.attr_count	= NFTA_SET_ELEM_LIST_MAX,
 9941		.policy		= nft_set_elem_list_policy,
 9942	},
 9943	[NFT_MSG_GETGEN] = {
 9944		.call		= nf_tables_getgen,
 9945		.type		= NFNL_CB_RCU,
 9946	},
 9947	[NFT_MSG_NEWOBJ] = {
 9948		.call		= nf_tables_newobj,
 9949		.type		= NFNL_CB_BATCH,
 9950		.attr_count	= NFTA_OBJ_MAX,
 9951		.policy		= nft_obj_policy,
 9952	},
 9953	[NFT_MSG_GETOBJ] = {
 9954		.call		= nf_tables_getobj,
 9955		.type		= NFNL_CB_RCU,
 9956		.attr_count	= NFTA_OBJ_MAX,
 9957		.policy		= nft_obj_policy,
 9958	},
 9959	[NFT_MSG_DELOBJ] = {
 9960		.call		= nf_tables_delobj,
 9961		.type		= NFNL_CB_BATCH,
 9962		.attr_count	= NFTA_OBJ_MAX,
 9963		.policy		= nft_obj_policy,
 9964	},
 9965	[NFT_MSG_DESTROYOBJ] = {
 9966		.call		= nf_tables_delobj,
 9967		.type		= NFNL_CB_BATCH,
 9968		.attr_count	= NFTA_OBJ_MAX,
 9969		.policy		= nft_obj_policy,
 9970	},
 9971	[NFT_MSG_GETOBJ_RESET] = {
 9972		.call		= nf_tables_getobj,
 9973		.type		= NFNL_CB_RCU,
 9974		.attr_count	= NFTA_OBJ_MAX,
 9975		.policy		= nft_obj_policy,
 9976	},
 9977	[NFT_MSG_NEWFLOWTABLE] = {
 9978		.call		= nf_tables_newflowtable,
 9979		.type		= NFNL_CB_BATCH,
 9980		.attr_count	= NFTA_FLOWTABLE_MAX,
 9981		.policy		= nft_flowtable_policy,
 9982	},
 9983	[NFT_MSG_GETFLOWTABLE] = {
 9984		.call		= nf_tables_getflowtable,
 9985		.type		= NFNL_CB_RCU,
 9986		.attr_count	= NFTA_FLOWTABLE_MAX,
 9987		.policy		= nft_flowtable_policy,
 9988	},
 9989	[NFT_MSG_DELFLOWTABLE] = {
 9990		.call		= nf_tables_delflowtable,
 9991		.type		= NFNL_CB_BATCH,
 9992		.attr_count	= NFTA_FLOWTABLE_MAX,
 9993		.policy		= nft_flowtable_policy,
 9994	},
 9995	[NFT_MSG_DESTROYFLOWTABLE] = {
 9996		.call		= nf_tables_delflowtable,
 9997		.type		= NFNL_CB_BATCH,
 9998		.attr_count	= NFTA_FLOWTABLE_MAX,
 9999		.policy		= nft_flowtable_policy,
10000	},
10001};
10002
10003static int nf_tables_validate(struct net *net)
10004{
10005	struct nftables_pernet *nft_net = nft_pernet(net);
10006	struct nft_table *table;
10007
10008	list_for_each_entry(table, &nft_net->tables, list) {
10009		switch (table->validate_state) {
10010		case NFT_VALIDATE_SKIP:
10011			continue;
10012		case NFT_VALIDATE_NEED:
10013			nft_validate_state_update(table, NFT_VALIDATE_DO);
10014			fallthrough;
10015		case NFT_VALIDATE_DO:
10016			if (nft_table_validate(net, table) < 0)
10017				return -EAGAIN;
10018
10019			nft_validate_state_update(table, NFT_VALIDATE_SKIP);
10020			break;
10021		}
10022	}
10023
10024	return 0;
10025}
10026
10027/* a drop policy has to be deferred until all rules have been activated,
10028 * otherwise a large ruleset that contains a drop-policy base chain will
10029 * cause all packets to get dropped until the full transaction has been
10030 * processed.
10031 *
10032 * We defer the drop policy until the transaction has been finalized.
10033 */
10034static void nft_chain_commit_drop_policy(struct nft_trans_chain *trans)
10035{
10036	struct nft_base_chain *basechain;
10037
10038	if (trans->policy != NF_DROP)
10039		return;
10040
10041	if (!nft_is_base_chain(trans->chain))
10042		return;
10043
10044	basechain = nft_base_chain(trans->chain);
10045	basechain->policy = NF_DROP;
10046}
10047
10048static void nft_chain_commit_update(struct nft_trans_chain *trans)
10049{
10050	struct nft_table *table = trans->nft_trans_binding.nft_trans.table;
10051	struct nft_base_chain *basechain;
10052
10053	if (trans->name) {
10054		rhltable_remove(&table->chains_ht,
10055				&trans->chain->rhlhead,
10056				nft_chain_ht_params);
10057		swap(trans->chain->name, trans->name);
10058		rhltable_insert_key(&table->chains_ht,
10059				    trans->chain->name,
10060				    &trans->chain->rhlhead,
10061				    nft_chain_ht_params);
10062	}
10063
10064	if (!nft_is_base_chain(trans->chain))
10065		return;
10066
10067	nft_chain_stats_replace(trans);
10068
10069	basechain = nft_base_chain(trans->chain);
10070
10071	switch (trans->policy) {
10072	case NF_DROP:
10073	case NF_ACCEPT:
10074		basechain->policy = trans->policy;
10075		break;
10076	}
10077}
10078
10079static void nft_obj_commit_update(const struct nft_ctx *ctx,
10080				  struct nft_trans *trans)
10081{
10082	struct nft_object *newobj;
10083	struct nft_object *obj;
10084
10085	obj = nft_trans_obj(trans);
10086	newobj = nft_trans_obj_newobj(trans);
10087
10088	if (WARN_ON_ONCE(!obj->ops->update))
10089		return;
10090
10091	obj->ops->update(obj, newobj);
10092	nft_obj_destroy(ctx, newobj);
10093}
10094
10095static void nft_commit_release(struct nft_trans *trans)
10096{
10097	struct nft_ctx ctx = {
10098		.net = trans->net,
10099	};
10100
10101	nft_ctx_update(&ctx, trans);
10102
10103	switch (trans->msg_type) {
10104	case NFT_MSG_DELTABLE:
10105	case NFT_MSG_DESTROYTABLE:
10106		nf_tables_table_destroy(trans->table);
10107		break;
10108	case NFT_MSG_NEWCHAIN:
10109		free_percpu(nft_trans_chain_stats(trans));
10110		kfree(nft_trans_chain_name(trans));
10111		break;
10112	case NFT_MSG_DELCHAIN:
10113	case NFT_MSG_DESTROYCHAIN:
10114		if (nft_trans_chain_update(trans))
10115			nft_hooks_destroy(&nft_trans_chain_hooks(trans));
10116		else
10117			nf_tables_chain_destroy(nft_trans_chain(trans));
10118		break;
10119	case NFT_MSG_DELRULE:
10120	case NFT_MSG_DESTROYRULE:
10121		nf_tables_rule_destroy(&ctx, nft_trans_rule(trans));
10122		break;
10123	case NFT_MSG_DELSET:
10124	case NFT_MSG_DESTROYSET:
10125		nft_set_destroy(&ctx, nft_trans_set(trans));
10126		break;
10127	case NFT_MSG_DELSETELEM:
10128	case NFT_MSG_DESTROYSETELEM:
10129		nft_trans_elems_destroy(&ctx, nft_trans_container_elem(trans));
10130		break;
10131	case NFT_MSG_DELOBJ:
10132	case NFT_MSG_DESTROYOBJ:
10133		nft_obj_destroy(&ctx, nft_trans_obj(trans));
10134		break;
10135	case NFT_MSG_DELFLOWTABLE:
10136	case NFT_MSG_DESTROYFLOWTABLE:
10137		if (nft_trans_flowtable_update(trans))
10138			nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
10139		else
10140			nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
10141		break;
10142	}
10143
10144	if (trans->put_net)
10145		put_net(trans->net);
10146
10147	kfree(trans);
10148}
10149
10150static void nf_tables_trans_destroy_work(struct work_struct *w)
10151{
10152	struct nftables_pernet *nft_net = container_of(w, struct nftables_pernet, destroy_work);
10153	struct nft_trans *trans, *next;
10154	LIST_HEAD(head);
10155
10156	spin_lock(&nf_tables_destroy_list_lock);
10157	list_splice_init(&nft_net->destroy_list, &head);
10158	spin_unlock(&nf_tables_destroy_list_lock);
10159
10160	if (list_empty(&head))
10161		return;
10162
10163	synchronize_rcu();
10164
10165	list_for_each_entry_safe(trans, next, &head, list) {
10166		nft_trans_list_del(trans);
10167		nft_commit_release(trans);
10168	}
10169}
10170
10171void nf_tables_trans_destroy_flush_work(struct net *net)
10172{
10173	struct nftables_pernet *nft_net = nft_pernet(net);
10174
10175	flush_work(&nft_net->destroy_work);
10176}
10177EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
10178
10179static bool nft_expr_reduce(struct nft_regs_track *track,
10180			    const struct nft_expr *expr)
10181{
10182	return false;
10183}
10184
10185static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
10186{
10187	const struct nft_expr *expr, *last;
10188	struct nft_regs_track track = {};
10189	unsigned int size, data_size;
10190	void *data, *data_boundary;
10191	struct nft_rule_dp *prule;
10192	struct nft_rule *rule;
10193
10194	/* already handled or inactive chain? */
10195	if (chain->blob_next || !nft_is_active_next(net, chain))
10196		return 0;
10197
10198	data_size = 0;
10199	list_for_each_entry(rule, &chain->rules, list) {
10200		if (nft_is_active_next(net, rule)) {
10201			data_size += sizeof(*prule) + rule->dlen;
10202			if (data_size > INT_MAX)
10203				return -ENOMEM;
10204		}
10205	}
10206
10207	chain->blob_next = nf_tables_chain_alloc_rules(chain, data_size);
10208	if (!chain->blob_next)
10209		return -ENOMEM;
10210
10211	data = (void *)chain->blob_next->data;
10212	data_boundary = data + data_size;
10213	size = 0;
10214
10215	list_for_each_entry(rule, &chain->rules, list) {
10216		if (!nft_is_active_next(net, rule))
10217			continue;
10218
10219		prule = (struct nft_rule_dp *)data;
10220		data += offsetof(struct nft_rule_dp, data);
10221		if (WARN_ON_ONCE(data > data_boundary))
10222			return -ENOMEM;
10223
10224		size = 0;
10225		track.last = nft_expr_last(rule);
10226		nft_rule_for_each_expr(expr, last, rule) {
10227			track.cur = expr;
10228
10229			if (nft_expr_reduce(&track, expr)) {
10230				expr = track.cur;
10231				continue;
10232			}
10233
10234			if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary))
10235				return -ENOMEM;
10236
10237			memcpy(data + size, expr, expr->ops->size);
10238			size += expr->ops->size;
10239		}
10240		if (WARN_ON_ONCE(size >= 1 << 12))
10241			return -ENOMEM;
10242
10243		prule->handle = rule->handle;
10244		prule->dlen = size;
10245		prule->is_last = 0;
10246
10247		data += size;
10248		size = 0;
10249		chain->blob_next->size += (unsigned long)(data - (void *)prule);
10250	}
10251
10252	if (WARN_ON_ONCE(data > data_boundary))
10253		return -ENOMEM;
10254
10255	prule = (struct nft_rule_dp *)data;
10256	nft_last_rule(chain, prule);
10257
10258	return 0;
10259}
10260
10261static void nf_tables_commit_chain_prepare_cancel(struct net *net)
10262{
10263	struct nftables_pernet *nft_net = nft_pernet(net);
10264	struct nft_trans *trans, *next;
10265
10266	list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10267		if (trans->msg_type == NFT_MSG_NEWRULE ||
10268		    trans->msg_type == NFT_MSG_DELRULE) {
10269			struct nft_chain *chain = nft_trans_rule_chain(trans);
10270
10271			kvfree(chain->blob_next);
10272			chain->blob_next = NULL;
10273		}
10274	}
10275}
10276
10277static void __nf_tables_commit_chain_free_rules(struct rcu_head *h)
10278{
10279	struct nft_rule_dp_last *l = container_of(h, struct nft_rule_dp_last, h);
10280
10281	kvfree(l->blob);
10282}
10283
10284static void nf_tables_commit_chain_free_rules_old(struct nft_rule_blob *blob)
10285{
10286	struct nft_rule_dp_last *last;
10287
10288	/* last rule trailer is after end marker */
10289	last = (void *)blob + sizeof(*blob) + blob->size;
10290	last->blob = blob;
10291
10292	call_rcu(&last->h, __nf_tables_commit_chain_free_rules);
10293}
10294
10295static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
10296{
10297	struct nft_rule_blob *g0, *g1;
10298	bool next_genbit;
10299
10300	next_genbit = nft_gencursor_next(net);
10301
10302	g0 = rcu_dereference_protected(chain->blob_gen_0,
10303				       lockdep_commit_lock_is_held(net));
10304	g1 = rcu_dereference_protected(chain->blob_gen_1,
10305				       lockdep_commit_lock_is_held(net));
10306
10307	/* No changes to this chain? */
10308	if (chain->blob_next == NULL) {
10309		/* chain had no change in last or next generation */
10310		if (g0 == g1)
10311			return;
10312		/*
10313		 * chain had no change in this generation; make sure next
10314		 * one uses same rules as current generation.
10315		 */
10316		if (next_genbit) {
10317			rcu_assign_pointer(chain->blob_gen_1, g0);
10318			nf_tables_commit_chain_free_rules_old(g1);
10319		} else {
10320			rcu_assign_pointer(chain->blob_gen_0, g1);
10321			nf_tables_commit_chain_free_rules_old(g0);
10322		}
10323
10324		return;
10325	}
10326
10327	if (next_genbit)
10328		rcu_assign_pointer(chain->blob_gen_1, chain->blob_next);
10329	else
10330		rcu_assign_pointer(chain->blob_gen_0, chain->blob_next);
10331
10332	chain->blob_next = NULL;
10333
10334	if (g0 == g1)
10335		return;
10336
10337	if (next_genbit)
10338		nf_tables_commit_chain_free_rules_old(g1);
10339	else
10340		nf_tables_commit_chain_free_rules_old(g0);
10341}
10342
10343static void nft_obj_del(struct nft_object *obj)
10344{
10345	rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
10346	list_del_rcu(&obj->list);
10347}
10348
10349void nft_chain_del(struct nft_chain *chain)
10350{
10351	struct nft_table *table = chain->table;
10352
10353	WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
10354				     nft_chain_ht_params));
10355	list_del_rcu(&chain->list);
10356}
10357
10358static void nft_trans_gc_setelem_remove(struct nft_ctx *ctx,
10359					struct nft_trans_gc *trans)
10360{
10361	struct nft_elem_priv **priv = trans->priv;
10362	unsigned int i;
10363
10364	for (i = 0; i < trans->count; i++) {
10365		nft_setelem_data_deactivate(ctx->net, trans->set, priv[i]);
10366		nft_setelem_remove(ctx->net, trans->set, priv[i]);
10367	}
10368}
10369
10370void nft_trans_gc_destroy(struct nft_trans_gc *trans)
10371{
10372	nft_set_put(trans->set);
10373	put_net(trans->net);
10374	kfree(trans);
10375}
10376
10377static void nft_trans_gc_trans_free(struct rcu_head *rcu)
10378{
10379	struct nft_elem_priv *elem_priv;
10380	struct nft_trans_gc *trans;
10381	struct nft_ctx ctx = {};
10382	unsigned int i;
10383
10384	trans = container_of(rcu, struct nft_trans_gc, rcu);
10385	ctx.net	= read_pnet(&trans->set->net);
10386
10387	for (i = 0; i < trans->count; i++) {
10388		elem_priv = trans->priv[i];
10389		if (!nft_setelem_is_catchall(trans->set, elem_priv))
10390			atomic_dec(&trans->set->nelems);
10391
10392		nf_tables_set_elem_destroy(&ctx, trans->set, elem_priv);
10393	}
10394
10395	nft_trans_gc_destroy(trans);
10396}
10397
10398static bool nft_trans_gc_work_done(struct nft_trans_gc *trans)
10399{
10400	struct nftables_pernet *nft_net;
10401	struct nft_ctx ctx = {};
10402
10403	nft_net = nft_pernet(trans->net);
10404
10405	mutex_lock(&nft_net->commit_mutex);
10406
10407	/* Check for race with transaction, otherwise this batch refers to
10408	 * stale objects that might not be there anymore. Skip transaction if
10409	 * set has been destroyed from control plane transaction in case gc
10410	 * worker loses race.
10411	 */
10412	if (READ_ONCE(nft_net->gc_seq) != trans->seq || trans->set->dead) {
10413		mutex_unlock(&nft_net->commit_mutex);
10414		return false;
10415	}
10416
10417	ctx.net = trans->net;
10418	ctx.table = trans->set->table;
10419
10420	nft_trans_gc_setelem_remove(&ctx, trans);
10421	mutex_unlock(&nft_net->commit_mutex);
10422
10423	return true;
10424}
10425
10426static void nft_trans_gc_work(struct work_struct *work)
10427{
10428	struct nft_trans_gc *trans, *next;
10429	LIST_HEAD(trans_gc_list);
10430
10431	spin_lock(&nf_tables_gc_list_lock);
10432	list_splice_init(&nf_tables_gc_list, &trans_gc_list);
10433	spin_unlock(&nf_tables_gc_list_lock);
10434
10435	list_for_each_entry_safe(trans, next, &trans_gc_list, list) {
10436		list_del(&trans->list);
10437		if (!nft_trans_gc_work_done(trans)) {
10438			nft_trans_gc_destroy(trans);
10439			continue;
10440		}
10441		call_rcu(&trans->rcu, nft_trans_gc_trans_free);
10442	}
10443}
10444
10445struct nft_trans_gc *nft_trans_gc_alloc(struct nft_set *set,
10446					unsigned int gc_seq, gfp_t gfp)
10447{
10448	struct net *net = read_pnet(&set->net);
10449	struct nft_trans_gc *trans;
10450
10451	trans = kzalloc(sizeof(*trans), gfp);
10452	if (!trans)
10453		return NULL;
10454
10455	trans->net = maybe_get_net(net);
10456	if (!trans->net) {
10457		kfree(trans);
10458		return NULL;
10459	}
10460
10461	refcount_inc(&set->refs);
10462	trans->set = set;
10463	trans->seq = gc_seq;
10464
10465	return trans;
10466}
10467
10468void nft_trans_gc_elem_add(struct nft_trans_gc *trans, void *priv)
10469{
10470	trans->priv[trans->count++] = priv;
10471}
10472
10473static void nft_trans_gc_queue_work(struct nft_trans_gc *trans)
10474{
10475	spin_lock(&nf_tables_gc_list_lock);
10476	list_add_tail(&trans->list, &nf_tables_gc_list);
10477	spin_unlock(&nf_tables_gc_list_lock);
10478
10479	schedule_work(&trans_gc_work);
10480}
10481
10482struct nft_trans_gc *nft_trans_gc_queue_async(struct nft_trans_gc *gc,
10483					      unsigned int gc_seq, gfp_t gfp)
10484{
10485	struct nft_set *set;
10486
10487	if (nft_trans_gc_space(gc))
10488		return gc;
10489
10490	set = gc->set;
10491	nft_trans_gc_queue_work(gc);
10492
10493	return nft_trans_gc_alloc(set, gc_seq, gfp);
10494}
10495
10496void nft_trans_gc_queue_async_done(struct nft_trans_gc *trans)
10497{
10498	if (trans->count == 0) {
10499		nft_trans_gc_destroy(trans);
10500		return;
10501	}
10502
10503	nft_trans_gc_queue_work(trans);
10504}
10505
10506struct nft_trans_gc *nft_trans_gc_queue_sync(struct nft_trans_gc *gc, gfp_t gfp)
10507{
10508	struct nft_set *set;
10509
10510	if (WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net)))
10511		return NULL;
10512
10513	if (nft_trans_gc_space(gc))
10514		return gc;
10515
10516	set = gc->set;
10517	call_rcu(&gc->rcu, nft_trans_gc_trans_free);
10518
10519	return nft_trans_gc_alloc(set, 0, gfp);
10520}
10521
10522void nft_trans_gc_queue_sync_done(struct nft_trans_gc *trans)
10523{
10524	WARN_ON_ONCE(!lockdep_commit_lock_is_held(trans->net));
10525
10526	if (trans->count == 0) {
10527		nft_trans_gc_destroy(trans);
10528		return;
10529	}
10530
10531	call_rcu(&trans->rcu, nft_trans_gc_trans_free);
10532}
10533
10534struct nft_trans_gc *nft_trans_gc_catchall_async(struct nft_trans_gc *gc,
10535						 unsigned int gc_seq)
10536{
10537	struct nft_set_elem_catchall *catchall;
10538	const struct nft_set *set = gc->set;
10539	struct nft_set_ext *ext;
10540
10541	list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
10542		ext = nft_set_elem_ext(set, catchall->elem);
10543
10544		if (!nft_set_elem_expired(ext))
10545			continue;
10546		if (nft_set_elem_is_dead(ext))
10547			goto dead_elem;
10548
10549		nft_set_elem_dead(ext);
10550dead_elem:
10551		gc = nft_trans_gc_queue_async(gc, gc_seq, GFP_ATOMIC);
10552		if (!gc)
10553			return NULL;
10554
10555		nft_trans_gc_elem_add(gc, catchall->elem);
10556	}
10557
10558	return gc;
10559}
10560
10561struct nft_trans_gc *nft_trans_gc_catchall_sync(struct nft_trans_gc *gc)
10562{
10563	struct nft_set_elem_catchall *catchall, *next;
10564	u64 tstamp = nft_net_tstamp(gc->net);
10565	const struct nft_set *set = gc->set;
10566	struct nft_elem_priv *elem_priv;
10567	struct nft_set_ext *ext;
10568
10569	WARN_ON_ONCE(!lockdep_commit_lock_is_held(gc->net));
10570
10571	list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
10572		ext = nft_set_elem_ext(set, catchall->elem);
10573
10574		if (!__nft_set_elem_expired(ext, tstamp))
10575			continue;
10576
10577		gc = nft_trans_gc_queue_sync(gc, GFP_KERNEL);
10578		if (!gc)
10579			return NULL;
10580
10581		elem_priv = catchall->elem;
10582		nft_setelem_data_deactivate(gc->net, gc->set, elem_priv);
10583		nft_setelem_catchall_destroy(catchall);
10584		nft_trans_gc_elem_add(gc, elem_priv);
10585	}
10586
10587	return gc;
10588}
10589
10590static void nf_tables_module_autoload_cleanup(struct net *net)
10591{
10592	struct nftables_pernet *nft_net = nft_pernet(net);
10593	struct nft_module_request *req, *next;
10594
10595	WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
10596	list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
10597		WARN_ON_ONCE(!req->done);
10598		list_del(&req->list);
10599		kfree(req);
10600	}
10601}
10602
10603static void nf_tables_commit_release(struct net *net)
10604{
10605	struct nftables_pernet *nft_net = nft_pernet(net);
10606	struct nft_trans *trans;
10607
10608	/* all side effects have to be made visible.
10609	 * For example, if a chain named 'foo' has been deleted, a
10610	 * new transaction must not find it anymore.
10611	 *
10612	 * Memory reclaim happens asynchronously from work queue
10613	 * to prevent expensive synchronize_rcu() in commit phase.
10614	 */
10615	if (list_empty(&nft_net->commit_list)) {
10616		nf_tables_module_autoload_cleanup(net);
10617		mutex_unlock(&nft_net->commit_mutex);
10618		return;
10619	}
10620
10621	trans = list_last_entry(&nft_net->commit_list,
10622				struct nft_trans, list);
10623	get_net(trans->net);
10624	WARN_ON_ONCE(trans->put_net);
10625
10626	trans->put_net = true;
10627	spin_lock(&nf_tables_destroy_list_lock);
10628	list_splice_tail_init(&nft_net->commit_list, &nft_net->destroy_list);
10629	spin_unlock(&nf_tables_destroy_list_lock);
10630
10631	nf_tables_module_autoload_cleanup(net);
10632	schedule_work(&nft_net->destroy_work);
10633
10634	mutex_unlock(&nft_net->commit_mutex);
10635}
10636
10637static void nft_commit_notify(struct net *net, u32 portid)
10638{
10639	struct nftables_pernet *nft_net = nft_pernet(net);
10640	struct sk_buff *batch_skb = NULL, *nskb, *skb;
10641	unsigned char *data;
10642	int len;
10643
10644	list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
10645		if (!batch_skb) {
10646new_batch:
10647			batch_skb = skb;
10648			len = NLMSG_GOODSIZE - skb->len;
10649			list_del(&skb->list);
10650			continue;
10651		}
10652		len -= skb->len;
10653		if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
10654			data = skb_put(batch_skb, skb->len);
10655			memcpy(data, skb->data, skb->len);
10656			list_del(&skb->list);
10657			kfree_skb(skb);
10658			continue;
10659		}
10660		nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10661			       NFT_CB(batch_skb).report, GFP_KERNEL);
10662		goto new_batch;
10663	}
10664
10665	if (batch_skb) {
10666		nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
10667			       NFT_CB(batch_skb).report, GFP_KERNEL);
10668	}
10669
10670	WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
10671}
10672
10673static int nf_tables_commit_audit_alloc(struct list_head *adl,
10674					struct nft_table *table)
10675{
10676	struct nft_audit_data *adp;
10677
10678	list_for_each_entry(adp, adl, list) {
10679		if (adp->table == table)
10680			return 0;
10681	}
10682	adp = kzalloc(sizeof(*adp), GFP_KERNEL);
10683	if (!adp)
10684		return -ENOMEM;
10685	adp->table = table;
10686	list_add(&adp->list, adl);
10687	return 0;
10688}
10689
10690static void nf_tables_commit_audit_free(struct list_head *adl)
10691{
10692	struct nft_audit_data *adp, *adn;
10693
10694	list_for_each_entry_safe(adp, adn, adl, list) {
10695		list_del(&adp->list);
10696		kfree(adp);
10697	}
10698}
10699
10700/* nft audit emits the number of elements that get added/removed/updated,
10701 * so NEW/DELSETELEM needs to increment based on the total elem count.
10702 */
10703static unsigned int nf_tables_commit_audit_entrycount(const struct nft_trans *trans)
10704{
10705	switch (trans->msg_type) {
10706	case NFT_MSG_NEWSETELEM:
10707	case NFT_MSG_DELSETELEM:
10708		return nft_trans_container_elem(trans)->nelems;
10709	}
10710
10711	return 1;
10712}
10713
10714static void nf_tables_commit_audit_collect(struct list_head *adl,
10715					   const struct nft_trans *trans, u32 op)
10716{
10717	const struct nft_table *table = trans->table;
10718	struct nft_audit_data *adp;
10719
10720	list_for_each_entry(adp, adl, list) {
10721		if (adp->table == table)
10722			goto found;
10723	}
10724	WARN_ONCE(1, "table=%s not expected in commit list", table->name);
10725	return;
10726found:
10727	adp->entries += nf_tables_commit_audit_entrycount(trans);
10728	if (!adp->op || adp->op > op)
10729		adp->op = op;
10730}
10731
10732#define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
10733
10734static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
10735{
10736	struct nft_audit_data *adp, *adn;
10737	char aubuf[AUNFTABLENAMELEN];
10738
10739	list_for_each_entry_safe(adp, adn, adl, list) {
10740		snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
10741			 generation);
10742		audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
10743				nft2audit_op[adp->op], GFP_KERNEL);
10744		list_del(&adp->list);
10745		kfree(adp);
10746	}
10747}
10748
10749static void nft_set_commit_update(struct list_head *set_update_list)
10750{
10751	struct nft_set *set, *next;
10752
10753	list_for_each_entry_safe(set, next, set_update_list, pending_update) {
10754		list_del_init(&set->pending_update);
10755
10756		if (!set->ops->commit || set->dead)
10757			continue;
10758
10759		set->ops->commit(set);
10760	}
10761}
10762
10763static unsigned int nft_gc_seq_begin(struct nftables_pernet *nft_net)
10764{
10765	unsigned int gc_seq;
10766
10767	/* Bump gc counter, it becomes odd, this is the busy mark. */
10768	gc_seq = READ_ONCE(nft_net->gc_seq);
10769	WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10770
10771	return gc_seq;
10772}
10773
10774static void nft_gc_seq_end(struct nftables_pernet *nft_net, unsigned int gc_seq)
10775{
10776	WRITE_ONCE(nft_net->gc_seq, ++gc_seq);
10777}
10778
10779static int nf_tables_commit(struct net *net, struct sk_buff *skb)
10780{
10781	struct nftables_pernet *nft_net = nft_pernet(net);
10782	const struct nlmsghdr *nlh = nlmsg_hdr(skb);
10783	struct nft_trans_binding *trans_binding;
10784	struct nft_trans *trans, *next;
10785	unsigned int base_seq, gc_seq;
10786	LIST_HEAD(set_update_list);
10787	struct nft_trans_elem *te;
10788	struct nft_chain *chain;
10789	struct nft_table *table;
10790	struct nft_ctx ctx;
10791	LIST_HEAD(adl);
10792	int err;
10793
10794	if (list_empty(&nft_net->commit_list)) {
10795		mutex_unlock(&nft_net->commit_mutex);
10796		return 0;
10797	}
10798
10799	nft_ctx_init(&ctx, net, skb, nlh, NFPROTO_UNSPEC, NULL, NULL, NULL);
10800
10801	list_for_each_entry(trans_binding, &nft_net->binding_list, binding_list) {
10802		trans = &trans_binding->nft_trans;
10803		switch (trans->msg_type) {
10804		case NFT_MSG_NEWSET:
10805			if (!nft_trans_set_update(trans) &&
10806			    nft_set_is_anonymous(nft_trans_set(trans)) &&
10807			    !nft_trans_set_bound(trans)) {
10808				pr_warn_once("nftables ruleset with unbound set\n");
10809				return -EINVAL;
10810			}
10811			break;
10812		case NFT_MSG_NEWCHAIN:
10813			if (!nft_trans_chain_update(trans) &&
10814			    nft_chain_binding(nft_trans_chain(trans)) &&
10815			    !nft_trans_chain_bound(trans)) {
10816				pr_warn_once("nftables ruleset with unbound chain\n");
10817				return -EINVAL;
10818			}
10819			break;
10820		default:
10821			WARN_ONCE(1, "Unhandled bind type %d", trans->msg_type);
10822			break;
10823		}
10824	}
10825
10826	/* 0. Validate ruleset, otherwise roll back for error reporting. */
10827	if (nf_tables_validate(net) < 0) {
10828		nft_net->validate_state = NFT_VALIDATE_DO;
10829		return -EAGAIN;
10830	}
10831
10832	err = nft_flow_rule_offload_commit(net);
10833	if (err < 0)
10834		return err;
10835
10836	/* 1.  Allocate space for next generation rules_gen_X[] */
10837	list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10838		struct nft_table *table = trans->table;
10839		int ret;
10840
10841		ret = nf_tables_commit_audit_alloc(&adl, table);
10842		if (ret) {
10843			nf_tables_commit_chain_prepare_cancel(net);
10844			nf_tables_commit_audit_free(&adl);
10845			return ret;
10846		}
10847		if (trans->msg_type == NFT_MSG_NEWRULE ||
10848		    trans->msg_type == NFT_MSG_DELRULE) {
10849			chain = nft_trans_rule_chain(trans);
10850
10851			ret = nf_tables_commit_chain_prepare(net, chain);
10852			if (ret < 0) {
10853				nf_tables_commit_chain_prepare_cancel(net);
10854				nf_tables_commit_audit_free(&adl);
10855				return ret;
10856			}
10857		}
10858	}
10859
10860	/* step 2.  Make rules_gen_X visible to packet path */
10861	list_for_each_entry(table, &nft_net->tables, list) {
10862		list_for_each_entry(chain, &table->chains, list)
10863			nf_tables_commit_chain(net, chain);
10864	}
10865
10866	/*
10867	 * Bump generation counter, invalidate any dump in progress.
10868	 * Cannot fail after this point.
10869	 */
10870	base_seq = nft_base_seq(net);
10871	while (++base_seq == 0)
10872		;
10873
10874	/* pairs with smp_load_acquire in nft_lookup_eval */
10875	smp_store_release(&net->nft.base_seq, base_seq);
10876
10877	gc_seq = nft_gc_seq_begin(nft_net);
10878
10879	/* step 3. Start new generation, rules_gen_X now in use. */
10880	net->nft.gencursor = nft_gencursor_next(net);
10881
10882	list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
10883		struct nft_table *table = trans->table;
10884
10885		nft_ctx_update(&ctx, trans);
10886
10887		nf_tables_commit_audit_collect(&adl, trans, trans->msg_type);
10888		switch (trans->msg_type) {
10889		case NFT_MSG_NEWTABLE:
10890			if (nft_trans_table_update(trans)) {
10891				if (!(table->flags & __NFT_TABLE_F_UPDATE)) {
10892					nft_trans_destroy(trans);
10893					break;
10894				}
10895				if (table->flags & NFT_TABLE_F_DORMANT)
10896					nf_tables_table_disable(net, table);
10897
10898				table->flags &= ~__NFT_TABLE_F_UPDATE;
10899			} else {
10900				nft_clear(net, table);
10901			}
10902			nf_tables_table_notify(&ctx, NFT_MSG_NEWTABLE);
10903			nft_trans_destroy(trans);
10904			break;
10905		case NFT_MSG_DELTABLE:
10906		case NFT_MSG_DESTROYTABLE:
10907			list_del_rcu(&table->list);
10908			nf_tables_table_notify(&ctx, trans->msg_type);
10909			break;
10910		case NFT_MSG_NEWCHAIN:
10911			if (nft_trans_chain_update(trans)) {
10912				nft_chain_commit_update(nft_trans_container_chain(trans));
10913				nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN,
10914						       &nft_trans_chain_hooks(trans));
10915				list_splice(&nft_trans_chain_hooks(trans),
10916					    &nft_trans_basechain(trans)->hook_list);
10917				/* trans destroyed after rcu grace period */
10918			} else {
10919				nft_chain_commit_drop_policy(nft_trans_container_chain(trans));
10920				nft_clear(net, nft_trans_chain(trans));
10921				nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, NULL);
10922				nft_trans_destroy(trans);
10923			}
10924			break;
10925		case NFT_MSG_DELCHAIN:
10926		case NFT_MSG_DESTROYCHAIN:
10927			if (nft_trans_chain_update(trans)) {
10928				nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN,
10929						       &nft_trans_chain_hooks(trans));
10930				if (!(table->flags & NFT_TABLE_F_DORMANT)) {
10931					nft_netdev_unregister_hooks(net,
10932								    &nft_trans_chain_hooks(trans),
10933								    true);
10934				}
10935			} else {
10936				nft_chain_del(nft_trans_chain(trans));
10937				nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN,
10938						       NULL);
10939				nf_tables_unregister_hook(ctx.net, ctx.table,
10940							  nft_trans_chain(trans));
10941			}
10942			break;
10943		case NFT_MSG_NEWRULE:
10944			nft_clear(net, nft_trans_rule(trans));
10945			nf_tables_rule_notify(&ctx, nft_trans_rule(trans),
10946					      NFT_MSG_NEWRULE);
10947			if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
10948				nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10949
10950			nft_trans_destroy(trans);
10951			break;
10952		case NFT_MSG_DELRULE:
10953		case NFT_MSG_DESTROYRULE:
10954			list_del_rcu(&nft_trans_rule(trans)->list);
10955			nf_tables_rule_notify(&ctx, nft_trans_rule(trans),
10956					      trans->msg_type);
10957			nft_rule_expr_deactivate(&ctx, nft_trans_rule(trans),
10958						 NFT_TRANS_COMMIT);
10959
10960			if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
10961				nft_flow_rule_destroy(nft_trans_flow_rule(trans));
10962			break;
10963		case NFT_MSG_NEWSET:
10964			list_del(&nft_trans_container_set(trans)->list_trans_newset);
10965			if (nft_trans_set_update(trans)) {
10966				struct nft_set *set = nft_trans_set(trans);
10967
10968				WRITE_ONCE(set->timeout, nft_trans_set_timeout(trans));
10969				WRITE_ONCE(set->gc_int, nft_trans_set_gc_int(trans));
10970
10971				if (nft_trans_set_size(trans))
10972					WRITE_ONCE(set->size, nft_trans_set_size(trans));
10973			} else {
10974				nft_clear(net, nft_trans_set(trans));
10975				/* This avoids hitting -EBUSY when deleting the table
10976				 * from the transaction.
10977				 */
10978				if (nft_set_is_anonymous(nft_trans_set(trans)) &&
10979				    !list_empty(&nft_trans_set(trans)->bindings))
10980					nft_use_dec(&table->use);
10981			}
10982			nf_tables_set_notify(&ctx, nft_trans_set(trans),
10983					     NFT_MSG_NEWSET, GFP_KERNEL);
10984			nft_trans_destroy(trans);
10985			break;
10986		case NFT_MSG_DELSET:
10987		case NFT_MSG_DESTROYSET:
10988			nft_trans_set(trans)->dead = 1;
10989			list_del_rcu(&nft_trans_set(trans)->list);
10990			nf_tables_set_notify(&ctx, nft_trans_set(trans),
10991					     trans->msg_type, GFP_KERNEL);
10992			break;
10993		case NFT_MSG_NEWSETELEM:
10994			te = nft_trans_container_elem(trans);
10995
10996			nft_trans_elems_add(&ctx, te);
10997
10998			if (te->set->ops->commit &&
10999			    list_empty(&te->set->pending_update)) {
11000				list_add_tail(&te->set->pending_update,
11001					      &set_update_list);
11002			}
11003			nft_trans_destroy(trans);
11004			break;
11005		case NFT_MSG_DELSETELEM:
11006		case NFT_MSG_DESTROYSETELEM:
11007			te = nft_trans_container_elem(trans);
11008
11009			nft_trans_elems_remove(&ctx, te);
11010
11011			if (te->set->ops->commit &&
11012			    list_empty(&te->set->pending_update)) {
11013				list_add_tail(&te->set->pending_update,
11014					      &set_update_list);
11015			}
11016			break;
11017		case NFT_MSG_NEWOBJ:
11018			if (nft_trans_obj_update(trans)) {
11019				nft_obj_commit_update(&ctx, trans);
11020				nf_tables_obj_notify(&ctx,
11021						     nft_trans_obj(trans),
11022						     NFT_MSG_NEWOBJ);
11023			} else {
11024				nft_clear(net, nft_trans_obj(trans));
11025				nf_tables_obj_notify(&ctx,
11026						     nft_trans_obj(trans),
11027						     NFT_MSG_NEWOBJ);
11028				nft_trans_destroy(trans);
11029			}
11030			break;
11031		case NFT_MSG_DELOBJ:
11032		case NFT_MSG_DESTROYOBJ:
11033			nft_obj_del(nft_trans_obj(trans));
11034			nf_tables_obj_notify(&ctx, nft_trans_obj(trans),
11035					     trans->msg_type);
11036			break;
11037		case NFT_MSG_NEWFLOWTABLE:
11038			if (nft_trans_flowtable_update(trans)) {
11039				nft_trans_flowtable(trans)->data.flags =
11040					nft_trans_flowtable_flags(trans);
11041				nf_tables_flowtable_notify(&ctx,
11042							   nft_trans_flowtable(trans),
11043							   &nft_trans_flowtable_hooks(trans),
11044							   NFT_MSG_NEWFLOWTABLE);
11045				list_splice(&nft_trans_flowtable_hooks(trans),
11046					    &nft_trans_flowtable(trans)->hook_list);
11047			} else {
11048				nft_clear(net, nft_trans_flowtable(trans));
11049				nf_tables_flowtable_notify(&ctx,
11050							   nft_trans_flowtable(trans),
11051							   NULL,
11052							   NFT_MSG_NEWFLOWTABLE);
11053			}
11054			nft_trans_destroy(trans);
11055			break;
11056		case NFT_MSG_DELFLOWTABLE:
11057		case NFT_MSG_DESTROYFLOWTABLE:
11058			if (nft_trans_flowtable_update(trans)) {
11059				nf_tables_flowtable_notify(&ctx,
11060							   nft_trans_flowtable(trans),
11061							   &nft_trans_flowtable_hooks(trans),
11062							   trans->msg_type);
11063				nft_unregister_flowtable_net_hooks(net,
11064								   nft_trans_flowtable(trans),
11065								   &nft_trans_flowtable_hooks(trans));
11066			} else {
11067				list_del_rcu(&nft_trans_flowtable(trans)->list);
11068				nf_tables_flowtable_notify(&ctx,
11069							   nft_trans_flowtable(trans),
11070							   NULL,
11071							   trans->msg_type);
11072				nft_unregister_flowtable_net_hooks(net,
11073						nft_trans_flowtable(trans),
11074						&nft_trans_flowtable(trans)->hook_list);
11075			}
11076			break;
11077		}
11078	}
11079
11080	nft_set_commit_update(&set_update_list);
11081
11082	nft_commit_notify(net, NETLINK_CB(skb).portid);
11083	nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
11084	nf_tables_commit_audit_log(&adl, nft_base_seq(net));
11085
11086	nft_gc_seq_end(nft_net, gc_seq);
11087	nft_net->validate_state = NFT_VALIDATE_SKIP;
11088	nf_tables_commit_release(net);
11089
11090	return 0;
11091}
11092
11093static void nf_tables_module_autoload(struct net *net)
11094{
11095	struct nftables_pernet *nft_net = nft_pernet(net);
11096	struct nft_module_request *req, *next;
11097	LIST_HEAD(module_list);
11098
11099	list_splice_init(&nft_net->module_list, &module_list);
11100	mutex_unlock(&nft_net->commit_mutex);
11101	list_for_each_entry_safe(req, next, &module_list, list) {
11102		request_module("%s", req->module);
11103		req->done = true;
11104	}
11105	mutex_lock(&nft_net->commit_mutex);
11106	list_splice(&module_list, &nft_net->module_list);
11107}
11108
11109static void nf_tables_abort_release(struct nft_trans *trans)
11110{
11111	struct nft_ctx ctx = { };
11112
11113	nft_ctx_update(&ctx, trans);
11114
11115	switch (trans->msg_type) {
11116	case NFT_MSG_NEWTABLE:
11117		nf_tables_table_destroy(trans->table);
11118		break;
11119	case NFT_MSG_NEWCHAIN:
11120		if (nft_trans_chain_update(trans))
11121			nft_hooks_destroy(&nft_trans_chain_hooks(trans));
11122		else
11123			nf_tables_chain_destroy(nft_trans_chain(trans));
11124		break;
11125	case NFT_MSG_NEWRULE:
11126		nf_tables_rule_destroy(&ctx, nft_trans_rule(trans));
11127		break;
11128	case NFT_MSG_NEWSET:
11129		nft_set_destroy(&ctx, nft_trans_set(trans));
11130		break;
11131	case NFT_MSG_NEWSETELEM:
11132		nft_trans_set_elem_destroy(&ctx, nft_trans_container_elem(trans));
11133		break;
11134	case NFT_MSG_NEWOBJ:
11135		nft_obj_destroy(&ctx, nft_trans_obj(trans));
11136		break;
11137	case NFT_MSG_NEWFLOWTABLE:
11138		if (nft_trans_flowtable_update(trans))
11139			nft_hooks_destroy(&nft_trans_flowtable_hooks(trans));
11140		else
11141			nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
11142		break;
11143	}
11144	kfree(trans);
11145}
11146
11147static void nft_set_abort_update(struct list_head *set_update_list)
11148{
11149	struct nft_set *set, *next;
11150
11151	list_for_each_entry_safe(set, next, set_update_list, pending_update) {
11152		list_del_init(&set->pending_update);
11153
11154		if (!set->ops->abort)
11155			continue;
11156
11157		set->ops->abort(set);
11158	}
11159}
11160
11161static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
11162{
11163	struct nftables_pernet *nft_net = nft_pernet(net);
11164	struct nft_trans *trans, *next;
11165	LIST_HEAD(set_update_list);
11166	struct nft_trans_elem *te;
11167	struct nft_ctx ctx = {
11168		.net = net,
11169	};
11170	int err = 0;
11171
11172	if (action == NFNL_ABORT_VALIDATE &&
11173	    nf_tables_validate(net) < 0)
11174		err = -EAGAIN;
11175
11176	list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
11177					 list) {
11178		struct nft_table *table = trans->table;
11179
11180		nft_ctx_update(&ctx, trans);
11181
11182		switch (trans->msg_type) {
11183		case NFT_MSG_NEWTABLE:
11184			if (nft_trans_table_update(trans)) {
11185				if (!(table->flags & __NFT_TABLE_F_UPDATE)) {
11186					nft_trans_destroy(trans);
11187					break;
11188				}
11189				if (table->flags & __NFT_TABLE_F_WAS_DORMANT) {
11190					nf_tables_table_disable(net, table);
11191					table->flags |= NFT_TABLE_F_DORMANT;
11192				} else if (table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
11193					table->flags &= ~NFT_TABLE_F_DORMANT;
11194				}
11195				if (table->flags & __NFT_TABLE_F_WAS_ORPHAN) {
11196					table->flags &= ~NFT_TABLE_F_OWNER;
11197					table->nlpid = 0;
11198				}
11199				table->flags &= ~__NFT_TABLE_F_UPDATE;
11200				nft_trans_destroy(trans);
11201			} else {
11202				list_del_rcu(&table->list);
11203			}
11204			break;
11205		case NFT_MSG_DELTABLE:
11206		case NFT_MSG_DESTROYTABLE:
11207			nft_clear(trans->net, table);
11208			nft_trans_destroy(trans);
11209			break;
11210		case NFT_MSG_NEWCHAIN:
11211			if (nft_trans_chain_update(trans)) {
11212				if (!(table->flags & NFT_TABLE_F_DORMANT)) {
11213					nft_netdev_unregister_hooks(net,
11214								    &nft_trans_chain_hooks(trans),
11215								    true);
11216				}
11217				free_percpu(nft_trans_chain_stats(trans));
11218				kfree(nft_trans_chain_name(trans));
11219				nft_trans_destroy(trans);
11220			} else {
11221				if (nft_trans_chain_bound(trans)) {
11222					nft_trans_destroy(trans);
11223					break;
11224				}
11225				nft_use_dec_restore(&table->use);
11226				nft_chain_del(nft_trans_chain(trans));
11227				nf_tables_unregister_hook(trans->net, table,
11228							  nft_trans_chain(trans));
11229			}
11230			break;
11231		case NFT_MSG_DELCHAIN:
11232		case NFT_MSG_DESTROYCHAIN:
11233			if (nft_trans_chain_update(trans)) {
11234				list_splice(&nft_trans_chain_hooks(trans),
11235					    &nft_trans_basechain(trans)->hook_list);
11236			} else {
11237				nft_use_inc_restore(&table->use);
11238				nft_clear(trans->net, nft_trans_chain(trans));
11239			}
11240			nft_trans_destroy(trans);
11241			break;
11242		case NFT_MSG_NEWRULE:
11243			if (nft_trans_rule_bound(trans)) {
11244				nft_trans_destroy(trans);
11245				break;
11246			}
11247			nft_use_dec_restore(&nft_trans_rule_chain(trans)->use);
11248			list_del_rcu(&nft_trans_rule(trans)->list);
11249			nft_rule_expr_deactivate(&ctx,
11250						 nft_trans_rule(trans),
11251						 NFT_TRANS_ABORT);
11252			if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
11253				nft_flow_rule_destroy(nft_trans_flow_rule(trans));
11254			break;
11255		case NFT_MSG_DELRULE:
11256		case NFT_MSG_DESTROYRULE:
11257			nft_use_inc_restore(&nft_trans_rule_chain(trans)->use);
11258			nft_clear(trans->net, nft_trans_rule(trans));
11259			nft_rule_expr_activate(&ctx, nft_trans_rule(trans));
11260			if (nft_trans_rule_chain(trans)->flags & NFT_CHAIN_HW_OFFLOAD)
11261				nft_flow_rule_destroy(nft_trans_flow_rule(trans));
11262
11263			nft_trans_destroy(trans);
11264			break;
11265		case NFT_MSG_NEWSET:
11266			list_del(&nft_trans_container_set(trans)->list_trans_newset);
11267			if (nft_trans_set_update(trans)) {
11268				nft_trans_destroy(trans);
11269				break;
11270			}
11271			nft_use_dec_restore(&table->use);
11272			if (nft_trans_set_bound(trans)) {
11273				nft_trans_destroy(trans);
11274				break;
11275			}
11276			nft_trans_set(trans)->dead = 1;
11277			list_del_rcu(&nft_trans_set(trans)->list);
11278			break;
11279		case NFT_MSG_DELSET:
11280		case NFT_MSG_DESTROYSET:
11281			nft_use_inc_restore(&table->use);
11282			nft_clear(trans->net, nft_trans_set(trans));
11283			if (nft_trans_set(trans)->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11284				nft_map_activate(&ctx, nft_trans_set(trans));
11285
11286			nft_trans_destroy(trans);
11287			break;
11288		case NFT_MSG_NEWSETELEM:
11289			if (nft_trans_elem_set_bound(trans)) {
11290				nft_trans_destroy(trans);
11291				break;
11292			}
11293			te = nft_trans_container_elem(trans);
11294			if (!nft_trans_elems_new_abort(&ctx, te)) {
11295				nft_trans_destroy(trans);
11296				break;
11297			}
11298
11299			if (te->set->ops->abort &&
11300			    list_empty(&te->set->pending_update)) {
11301				list_add_tail(&te->set->pending_update,
11302					      &set_update_list);
11303			}
11304			break;
11305		case NFT_MSG_DELSETELEM:
11306		case NFT_MSG_DESTROYSETELEM:
11307			te = nft_trans_container_elem(trans);
11308
11309			nft_trans_elems_destroy_abort(&ctx, te);
11310
11311			if (te->set->ops->abort &&
11312			    list_empty(&te->set->pending_update)) {
11313				list_add_tail(&te->set->pending_update,
11314					      &set_update_list);
11315			}
11316			nft_trans_destroy(trans);
11317			break;
11318		case NFT_MSG_NEWOBJ:
11319			if (nft_trans_obj_update(trans)) {
11320				nft_obj_destroy(&ctx, nft_trans_obj_newobj(trans));
11321				nft_trans_destroy(trans);
11322			} else {
11323				nft_use_dec_restore(&table->use);
11324				nft_obj_del(nft_trans_obj(trans));
11325			}
11326			break;
11327		case NFT_MSG_DELOBJ:
11328		case NFT_MSG_DESTROYOBJ:
11329			nft_use_inc_restore(&table->use);
11330			nft_clear(trans->net, nft_trans_obj(trans));
11331			nft_trans_destroy(trans);
11332			break;
11333		case NFT_MSG_NEWFLOWTABLE:
11334			if (nft_trans_flowtable_update(trans)) {
11335				nft_unregister_flowtable_net_hooks(net,
11336						nft_trans_flowtable(trans),
11337						&nft_trans_flowtable_hooks(trans));
11338			} else {
11339				nft_use_dec_restore(&table->use);
11340				list_del_rcu(&nft_trans_flowtable(trans)->list);
11341				nft_unregister_flowtable_net_hooks(net,
11342						nft_trans_flowtable(trans),
11343						&nft_trans_flowtable(trans)->hook_list);
11344			}
11345			break;
11346		case NFT_MSG_DELFLOWTABLE:
11347		case NFT_MSG_DESTROYFLOWTABLE:
11348			if (nft_trans_flowtable_update(trans)) {
11349				list_splice(&nft_trans_flowtable_hooks(trans),
11350					    &nft_trans_flowtable(trans)->hook_list);
11351			} else {
11352				nft_use_inc_restore(&table->use);
11353				nft_clear(trans->net, nft_trans_flowtable(trans));
11354			}
11355			nft_trans_destroy(trans);
11356			break;
11357		}
11358	}
11359
11360	WARN_ON_ONCE(!list_empty(&nft_net->commit_set_list));
11361
11362	nft_set_abort_update(&set_update_list);
11363
11364	synchronize_rcu();
11365
11366	list_for_each_entry_safe_reverse(trans, next,
11367					 &nft_net->commit_list, list) {
11368		nft_trans_list_del(trans);
11369		nf_tables_abort_release(trans);
11370	}
11371
11372	return err;
11373}
11374
11375static int nf_tables_abort(struct net *net, struct sk_buff *skb,
11376			   enum nfnl_abort_action action)
11377{
11378	struct nftables_pernet *nft_net = nft_pernet(net);
11379	unsigned int gc_seq;
11380	int ret;
11381
11382	gc_seq = nft_gc_seq_begin(nft_net);
11383	ret = __nf_tables_abort(net, action);
11384	nft_gc_seq_end(nft_net, gc_seq);
11385
11386	if (action == NFNL_ABORT_NONE) {
11387		struct nft_table *table;
11388
11389		list_for_each_entry(table, &nft_net->tables, list)
11390			table->validate_state = NFT_VALIDATE_SKIP;
11391	}
11392
11393	WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
11394
11395	/* module autoload needs to happen after GC sequence update because it
11396	 * temporarily releases and grabs mutex again.
11397	 */
11398	if (action == NFNL_ABORT_AUTOLOAD)
11399		nf_tables_module_autoload(net);
11400	else
11401		nf_tables_module_autoload_cleanup(net);
11402
11403	mutex_unlock(&nft_net->commit_mutex);
11404
11405	return ret;
11406}
11407
11408static bool nf_tables_valid_genid(struct net *net, u32 genid)
11409{
11410	struct nftables_pernet *nft_net = nft_pernet(net);
11411	bool genid_ok;
11412
11413	mutex_lock(&nft_net->commit_mutex);
11414	nft_net->tstamp = get_jiffies_64();
11415
11416	genid_ok = genid == 0 || nft_base_seq(net) == genid;
11417	if (!genid_ok)
11418		mutex_unlock(&nft_net->commit_mutex);
11419
11420	/* else, commit mutex has to be released by commit or abort function */
11421	return genid_ok;
11422}
11423
11424static const struct nfnetlink_subsystem nf_tables_subsys = {
11425	.name		= "nf_tables",
11426	.subsys_id	= NFNL_SUBSYS_NFTABLES,
11427	.cb_count	= NFT_MSG_MAX,
11428	.cb		= nf_tables_cb,
11429	.commit		= nf_tables_commit,
11430	.abort		= nf_tables_abort,
11431	.valid_genid	= nf_tables_valid_genid,
11432	.owner		= THIS_MODULE,
11433};
11434
11435int nft_chain_validate_dependency(const struct nft_chain *chain,
11436				  enum nft_chain_types type)
11437{
11438	const struct nft_base_chain *basechain;
11439
11440	if (nft_is_base_chain(chain)) {
11441		basechain = nft_base_chain(chain);
11442		if (basechain->type->type != type)
11443			return -EOPNOTSUPP;
11444	}
11445	return 0;
11446}
11447EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
11448
11449int nft_chain_validate_hooks(const struct nft_chain *chain,
11450			     unsigned int hook_flags)
11451{
11452	struct nft_base_chain *basechain;
11453
11454	if (nft_is_base_chain(chain)) {
11455		basechain = nft_base_chain(chain);
11456
11457		if ((1 << basechain->ops.hooknum) & hook_flags)
11458			return 0;
11459
11460		return -EOPNOTSUPP;
11461	}
11462
11463	return 0;
11464}
11465EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
11466
11467/**
11468 *	nft_parse_u32_check - fetch u32 attribute and check for maximum value
11469 *
11470 *	@attr: netlink attribute to fetch value from
11471 *	@max: maximum value to be stored in dest
11472 *	@dest: pointer to the variable
11473 *
11474 *	Parse, check and store a given u32 netlink attribute into variable.
11475 *	This function returns -ERANGE if the value goes over maximum value.
11476 *	Otherwise a 0 is returned and the attribute value is stored in the
11477 *	destination variable.
11478 */
11479int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
11480{
11481	u32 val;
11482
11483	val = ntohl(nla_get_be32(attr));
11484	if (val > max)
11485		return -ERANGE;
11486
11487	*dest = val;
11488	return 0;
11489}
11490EXPORT_SYMBOL_GPL(nft_parse_u32_check);
11491
11492static int nft_parse_register(const struct nlattr *attr, u32 *preg)
11493{
11494	unsigned int reg;
11495
11496	reg = ntohl(nla_get_be32(attr));
11497	switch (reg) {
11498	case NFT_REG_VERDICT...NFT_REG_4:
11499		*preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
11500		break;
11501	case NFT_REG32_00...NFT_REG32_15:
11502		*preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
11503		break;
11504	default:
11505		return -ERANGE;
11506	}
11507
11508	return 0;
11509}
11510
11511/**
11512 *	nft_dump_register - dump a register value to a netlink attribute
11513 *
11514 *	@skb: socket buffer
11515 *	@attr: attribute number
11516 *	@reg: register number
11517 *
11518 *	Construct a netlink attribute containing the register number. For
11519 *	compatibility reasons, register numbers being a multiple of 4 are
11520 *	translated to the corresponding 128 bit register numbers.
11521 */
11522int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
11523{
11524	if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
11525		reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
11526	else
11527		reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
11528
11529	return nla_put_be32(skb, attr, htonl(reg));
11530}
11531EXPORT_SYMBOL_GPL(nft_dump_register);
11532
11533static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
11534{
11535	if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11536		return -EINVAL;
11537	if (len == 0)
11538		return -EINVAL;
11539	if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
11540		return -ERANGE;
11541
11542	return 0;
11543}
11544
11545int nft_parse_register_load(const struct nft_ctx *ctx,
11546			    const struct nlattr *attr, u8 *sreg, u32 len)
11547{
11548	int err, invalid_reg;
11549	u32 reg, next_register;
11550
11551	err = nft_parse_register(attr, &reg);
11552	if (err < 0)
11553		return err;
11554
11555	err = nft_validate_register_load(reg, len);
11556	if (err < 0)
11557		return err;
11558
11559	next_register = DIV_ROUND_UP(len, NFT_REG32_SIZE) + reg;
11560
11561	/* Can't happen: nft_validate_register_load() should have failed */
11562	if (WARN_ON_ONCE(next_register > NFT_REG32_NUM))
11563		return -EINVAL;
11564
11565	/* find first register that did not see an earlier store. */
11566	invalid_reg = find_next_zero_bit(ctx->reg_inited, NFT_REG32_NUM, reg);
11567
11568	/* invalid register within the range that we're loading from? */
11569	if (invalid_reg < next_register)
11570		return -ENODATA;
11571
11572	*sreg = reg;
11573	return 0;
11574}
11575EXPORT_SYMBOL_GPL(nft_parse_register_load);
11576
11577static void nft_saw_register_store(const struct nft_ctx *__ctx,
11578				   int reg, unsigned int len)
11579{
11580	unsigned int registers = DIV_ROUND_UP(len, NFT_REG32_SIZE);
11581	struct nft_ctx *ctx = (struct nft_ctx *)__ctx;
11582
11583	if (WARN_ON_ONCE(len == 0 || reg < 0))
11584		return;
11585
11586	bitmap_set(ctx->reg_inited, reg, registers);
11587}
11588
11589static int nft_validate_register_store(const struct nft_ctx *ctx,
11590				       enum nft_registers reg,
11591				       const struct nft_data *data,
11592				       enum nft_data_types type,
11593				       unsigned int len)
11594{
11595	switch (reg) {
11596	case NFT_REG_VERDICT:
11597		if (type != NFT_DATA_VERDICT)
11598			return -EINVAL;
11599		break;
11600	default:
11601		if (type != NFT_DATA_VALUE)
11602			return -EINVAL;
11603
11604		if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
11605			return -EINVAL;
11606		if (len == 0)
11607			return -EINVAL;
11608		if (reg * NFT_REG32_SIZE + len >
11609		    sizeof_field(struct nft_regs, data))
11610			return -ERANGE;
11611
11612		break;
11613	}
11614
11615	nft_saw_register_store(ctx, reg, len);
11616	return 0;
11617}
11618
11619int nft_parse_register_store(const struct nft_ctx *ctx,
11620			     const struct nlattr *attr, u8 *dreg,
11621			     const struct nft_data *data,
11622			     enum nft_data_types type, unsigned int len)
11623{
11624	int err;
11625	u32 reg;
11626
11627	err = nft_parse_register(attr, &reg);
11628	if (err < 0)
11629		return err;
11630
11631	err = nft_validate_register_store(ctx, reg, data, type, len);
11632	if (err < 0)
11633		return err;
11634
11635	*dreg = reg;
11636	return 0;
11637}
11638EXPORT_SYMBOL_GPL(nft_parse_register_store);
11639
11640static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
11641	[NFTA_VERDICT_CODE]	= { .type = NLA_U32 },
11642	[NFTA_VERDICT_CHAIN]	= { .type = NLA_STRING,
11643				    .len = NFT_CHAIN_MAXNAMELEN - 1 },
11644	[NFTA_VERDICT_CHAIN_ID]	= { .type = NLA_U32 },
11645};
11646
11647static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
11648			    struct nft_data_desc *desc, const struct nlattr *nla)
11649{
11650	u8 genmask = nft_genmask_next(ctx->net);
11651	struct nlattr *tb[NFTA_VERDICT_MAX + 1];
11652	struct nft_chain *chain;
11653	int err;
11654
11655	err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
11656					  nft_verdict_policy, NULL);
11657	if (err < 0)
11658		return err;
11659
11660	if (!tb[NFTA_VERDICT_CODE])
11661		return -EINVAL;
11662
11663	/* zero padding hole for memcmp */
11664	memset(data, 0, sizeof(*data));
11665	data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
11666
11667	switch (data->verdict.code) {
11668	case NF_ACCEPT:
11669	case NF_DROP:
11670	case NFT_CONTINUE:
11671	case NFT_BREAK:
11672	case NFT_RETURN:
11673		break;
11674	case NFT_JUMP:
11675	case NFT_GOTO:
11676		if (tb[NFTA_VERDICT_CHAIN]) {
11677			chain = nft_chain_lookup(ctx->net, ctx->table,
11678						 tb[NFTA_VERDICT_CHAIN],
11679						 genmask);
11680		} else if (tb[NFTA_VERDICT_CHAIN_ID]) {
11681			chain = nft_chain_lookup_byid(ctx->net, ctx->table,
11682						      tb[NFTA_VERDICT_CHAIN_ID],
11683						      genmask);
11684			if (IS_ERR(chain))
11685				return PTR_ERR(chain);
11686		} else {
11687			return -EINVAL;
11688		}
11689
11690		if (IS_ERR(chain))
11691			return PTR_ERR(chain);
11692		if (nft_is_base_chain(chain))
11693			return -EOPNOTSUPP;
11694		if (nft_chain_is_bound(chain))
11695			return -EINVAL;
11696		if (desc->flags & NFT_DATA_DESC_SETELEM &&
11697		    chain->flags & NFT_CHAIN_BINDING)
11698			return -EINVAL;
11699		if (!nft_use_inc(&chain->use))
11700			return -EMFILE;
11701
11702		data->verdict.chain = chain;
11703		break;
11704	case NF_QUEUE:
11705		/* The nft_queue expression is used for this purpose, an
11706		 * immediate NF_QUEUE verdict should not ever be seen here.
11707		 */
11708		fallthrough;
11709	default:
11710		return -EINVAL;
11711	}
11712
11713	desc->len = sizeof(data->verdict);
11714
11715	return 0;
11716}
11717
11718static void nft_verdict_uninit(const struct nft_data *data)
11719{
11720	struct nft_chain *chain;
11721
11722	switch (data->verdict.code) {
11723	case NFT_JUMP:
11724	case NFT_GOTO:
11725		chain = data->verdict.chain;
11726		nft_use_dec(&chain->use);
11727		break;
11728	}
11729}
11730
11731int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
11732{
11733	struct nlattr *nest;
11734
11735	nest = nla_nest_start_noflag(skb, type);
11736	if (!nest)
11737		goto nla_put_failure;
11738
11739	if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
11740		goto nla_put_failure;
11741
11742	switch (v->code) {
11743	case NFT_JUMP:
11744	case NFT_GOTO:
11745		if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
11746				   v->chain->name))
11747			goto nla_put_failure;
11748	}
11749	nla_nest_end(skb, nest);
11750	return 0;
11751
11752nla_put_failure:
11753	return -1;
11754}
11755
11756static int nft_value_init(const struct nft_ctx *ctx,
11757			  struct nft_data *data, struct nft_data_desc *desc,
11758			  const struct nlattr *nla)
11759{
11760	unsigned int len;
11761
11762	len = nla_len(nla);
11763	if (len == 0)
11764		return -EINVAL;
11765	if (len > desc->size)
11766		return -EOVERFLOW;
11767	if (desc->len) {
11768		if (len != desc->len)
11769			return -EINVAL;
11770	} else {
11771		desc->len = len;
11772	}
11773
11774	nla_memcpy(data->data, nla, len);
11775
11776	return 0;
11777}
11778
11779static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
11780			  unsigned int len)
11781{
11782	return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
11783}
11784
11785static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
11786	[NFTA_DATA_VALUE]	= { .type = NLA_BINARY },
11787	[NFTA_DATA_VERDICT]	= { .type = NLA_NESTED },
11788};
11789
11790/**
11791 *	nft_data_init - parse nf_tables data netlink attributes
11792 *
11793 *	@ctx: context of the expression using the data
11794 *	@data: destination struct nft_data
11795 *	@desc: data description
11796 *	@nla: netlink attribute containing data
11797 *
11798 *	Parse the netlink data attributes and initialize a struct nft_data.
11799 *	The type and length of data are returned in the data description.
11800 *
11801 *	The caller can indicate that it only wants to accept data of type
11802 *	NFT_DATA_VALUE by passing NULL for the ctx argument.
11803 */
11804int nft_data_init(const struct nft_ctx *ctx, struct nft_data *data,
11805		  struct nft_data_desc *desc, const struct nlattr *nla)
11806{
11807	struct nlattr *tb[NFTA_DATA_MAX + 1];
11808	int err;
11809
11810	if (WARN_ON_ONCE(!desc->size))
11811		return -EINVAL;
11812
11813	err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
11814					  nft_data_policy, NULL);
11815	if (err < 0)
11816		return err;
11817
11818	if (tb[NFTA_DATA_VALUE]) {
11819		if (desc->type != NFT_DATA_VALUE)
11820			return -EINVAL;
11821
11822		err = nft_value_init(ctx, data, desc, tb[NFTA_DATA_VALUE]);
11823	} else if (tb[NFTA_DATA_VERDICT] && ctx != NULL) {
11824		if (desc->type != NFT_DATA_VERDICT)
11825			return -EINVAL;
11826
11827		err = nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
11828	} else {
11829		err = -EINVAL;
11830	}
11831
11832	return err;
11833}
11834EXPORT_SYMBOL_GPL(nft_data_init);
11835
11836/**
11837 *	nft_data_release - release a nft_data item
11838 *
11839 *	@data: struct nft_data to release
11840 *	@type: type of data
11841 *
11842 *	Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
11843 *	all others need to be released by calling this function.
11844 */
11845void nft_data_release(const struct nft_data *data, enum nft_data_types type)
11846{
11847	if (type < NFT_DATA_VERDICT)
11848		return;
11849	switch (type) {
11850	case NFT_DATA_VERDICT:
11851		return nft_verdict_uninit(data);
11852	default:
11853		WARN_ON(1);
11854	}
11855}
11856EXPORT_SYMBOL_GPL(nft_data_release);
11857
11858int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
11859		  enum nft_data_types type, unsigned int len)
11860{
11861	struct nlattr *nest;
11862	int err;
11863
11864	nest = nla_nest_start_noflag(skb, attr);
11865	if (nest == NULL)
11866		return -1;
11867
11868	switch (type) {
11869	case NFT_DATA_VALUE:
11870		err = nft_value_dump(skb, data, len);
11871		break;
11872	case NFT_DATA_VERDICT:
11873		err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
11874		break;
11875	default:
11876		err = -EINVAL;
11877		WARN_ON(1);
11878	}
11879
11880	nla_nest_end(skb, nest);
11881	return err;
11882}
11883EXPORT_SYMBOL_GPL(nft_data_dump);
11884
11885static void __nft_release_hook(struct net *net, struct nft_table *table)
11886{
11887	struct nft_flowtable *flowtable;
11888	struct nft_chain *chain;
11889
11890	list_for_each_entry(chain, &table->chains, list)
11891		__nf_tables_unregister_hook(net, table, chain, true);
11892	list_for_each_entry(flowtable, &table->flowtables, list)
11893		__nft_unregister_flowtable_net_hooks(net, flowtable,
11894						     &flowtable->hook_list,
11895						     true);
11896}
11897
11898static void __nft_release_hooks(struct net *net)
11899{
11900	struct nftables_pernet *nft_net = nft_pernet(net);
11901	struct nft_table *table;
11902
11903	list_for_each_entry(table, &nft_net->tables, list) {
11904		if (nft_table_has_owner(table))
11905			continue;
11906
11907		__nft_release_hook(net, table);
11908	}
11909}
11910
11911static void __nft_release_table(struct net *net, struct nft_table *table)
11912{
11913	struct nft_flowtable *flowtable, *nf;
11914	struct nft_chain *chain, *nc;
11915	struct nft_object *obj, *ne;
11916	struct nft_rule *rule, *nr;
11917	struct nft_set *set, *ns;
11918	struct nft_ctx ctx = {
11919		.net	= net,
11920		.family	= NFPROTO_NETDEV,
11921	};
11922
11923	ctx.family = table->family;
11924	ctx.table = table;
11925	list_for_each_entry(chain, &table->chains, list) {
11926		if (nft_chain_binding(chain))
11927			continue;
11928
11929		ctx.chain = chain;
11930		list_for_each_entry_safe(rule, nr, &chain->rules, list) {
11931			list_del(&rule->list);
11932			nft_use_dec(&chain->use);
11933			nf_tables_rule_release(&ctx, rule);
11934		}
11935	}
11936	list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
11937		list_del(&flowtable->list);
11938		nft_use_dec(&table->use);
11939		nf_tables_flowtable_destroy(flowtable);
11940	}
11941	list_for_each_entry_safe(set, ns, &table->sets, list) {
11942		list_del(&set->list);
11943		nft_use_dec(&table->use);
11944		if (set->flags & (NFT_SET_MAP | NFT_SET_OBJECT))
11945			nft_map_deactivate(&ctx, set);
11946
11947		nft_set_destroy(&ctx, set);
11948	}
11949	list_for_each_entry_safe(obj, ne, &table->objects, list) {
11950		nft_obj_del(obj);
11951		nft_use_dec(&table->use);
11952		nft_obj_destroy(&ctx, obj);
11953	}
11954	list_for_each_entry_safe(chain, nc, &table->chains, list) {
11955		nft_chain_del(chain);
11956		nft_use_dec(&table->use);
11957		nf_tables_chain_destroy(chain);
11958	}
11959	nf_tables_table_destroy(table);
11960}
11961
11962static void __nft_release_tables(struct net *net)
11963{
11964	struct nftables_pernet *nft_net = nft_pernet(net);
11965	struct nft_table *table, *nt;
11966
11967	list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
11968		if (nft_table_has_owner(table))
11969			continue;
11970
11971		list_del(&table->list);
11972
11973		__nft_release_table(net, table);
11974	}
11975}
11976
11977static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
11978			    void *ptr)
11979{
11980	struct nft_table *table, *to_delete[8];
11981	struct nftables_pernet *nft_net;
11982	struct netlink_notify *n = ptr;
11983	struct net *net = n->net;
11984	unsigned int deleted;
11985	bool restart = false;
11986	unsigned int gc_seq;
11987
11988	if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
11989		return NOTIFY_DONE;
11990
11991	nft_net = nft_pernet(net);
11992	deleted = 0;
11993	mutex_lock(&nft_net->commit_mutex);
11994
11995	gc_seq = nft_gc_seq_begin(nft_net);
11996
11997	nf_tables_trans_destroy_flush_work(net);
11998again:
11999	list_for_each_entry(table, &nft_net->tables, list) {
12000		if (nft_table_has_owner(table) &&
12001		    n->portid == table->nlpid) {
12002			if (table->flags & NFT_TABLE_F_PERSIST) {
12003				table->flags &= ~NFT_TABLE_F_OWNER;
12004				continue;
12005			}
12006			__nft_release_hook(net, table);
12007			list_del_rcu(&table->list);
12008			to_delete[deleted++] = table;
12009			if (deleted >= ARRAY_SIZE(to_delete))
12010				break;
12011		}
12012	}
12013	if (deleted) {
12014		restart = deleted >= ARRAY_SIZE(to_delete);
12015		synchronize_rcu();
12016		while (deleted)
12017			__nft_release_table(net, to_delete[--deleted]);
12018
12019		if (restart)
12020			goto again;
12021	}
12022	nft_gc_seq_end(nft_net, gc_seq);
12023
12024	mutex_unlock(&nft_net->commit_mutex);
12025
12026	return NOTIFY_DONE;
12027}
12028
12029static struct notifier_block nft_nl_notifier = {
12030	.notifier_call  = nft_rcv_nl_event,
12031};
12032
12033static int __net_init nf_tables_init_net(struct net *net)
12034{
12035	struct nftables_pernet *nft_net = nft_pernet(net);
12036
12037	INIT_LIST_HEAD(&nft_net->tables);
12038	INIT_LIST_HEAD(&nft_net->commit_list);
12039	INIT_LIST_HEAD(&nft_net->destroy_list);
12040	INIT_LIST_HEAD(&nft_net->commit_set_list);
12041	INIT_LIST_HEAD(&nft_net->binding_list);
12042	INIT_LIST_HEAD(&nft_net->module_list);
12043	INIT_LIST_HEAD(&nft_net->notify_list);
12044	mutex_init(&nft_net->commit_mutex);
12045	net->nft.base_seq = 1;
12046	nft_net->gc_seq = 0;
12047	nft_net->validate_state = NFT_VALIDATE_SKIP;
12048	INIT_WORK(&nft_net->destroy_work, nf_tables_trans_destroy_work);
12049
12050	return 0;
12051}
12052
12053static void __net_exit nf_tables_pre_exit_net(struct net *net)
12054{
12055	struct nftables_pernet *nft_net = nft_pernet(net);
12056
12057	mutex_lock(&nft_net->commit_mutex);
12058	__nft_release_hooks(net);
12059	mutex_unlock(&nft_net->commit_mutex);
12060}
12061
12062static void __net_exit nf_tables_exit_net(struct net *net)
12063{
12064	struct nftables_pernet *nft_net = nft_pernet(net);
12065	unsigned int gc_seq;
12066
12067	mutex_lock(&nft_net->commit_mutex);
12068
12069	gc_seq = nft_gc_seq_begin(nft_net);
12070
12071	WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
12072	WARN_ON_ONCE(!list_empty(&nft_net->commit_set_list));
12073
12074	if (!list_empty(&nft_net->module_list))
12075		nf_tables_module_autoload_cleanup(net);
12076
12077	cancel_work_sync(&nft_net->destroy_work);
12078	__nft_release_tables(net);
12079
12080	nft_gc_seq_end(nft_net, gc_seq);
12081
12082	mutex_unlock(&nft_net->commit_mutex);
12083
12084	WARN_ON_ONCE(!list_empty(&nft_net->tables));
12085	WARN_ON_ONCE(!list_empty(&nft_net->module_list));
12086	WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
12087	WARN_ON_ONCE(!list_empty(&nft_net->destroy_list));
12088}
12089
12090static void nf_tables_exit_batch(struct list_head *net_exit_list)
12091{
12092	flush_work(&trans_gc_work);
12093}
12094
12095static struct pernet_operations nf_tables_net_ops = {
12096	.init		= nf_tables_init_net,
12097	.pre_exit	= nf_tables_pre_exit_net,
12098	.exit		= nf_tables_exit_net,
12099	.exit_batch	= nf_tables_exit_batch,
12100	.id		= &nf_tables_net_id,
12101	.size		= sizeof(struct nftables_pernet),
12102};
12103
12104static int __init nf_tables_module_init(void)
12105{
12106	int err;
12107
12108	BUILD_BUG_ON(offsetof(struct nft_trans_table, nft_trans) != 0);
12109	BUILD_BUG_ON(offsetof(struct nft_trans_chain, nft_trans_binding.nft_trans) != 0);
12110	BUILD_BUG_ON(offsetof(struct nft_trans_rule, nft_trans) != 0);
12111	BUILD_BUG_ON(offsetof(struct nft_trans_set, nft_trans_binding.nft_trans) != 0);
12112	BUILD_BUG_ON(offsetof(struct nft_trans_elem, nft_trans) != 0);
12113	BUILD_BUG_ON(offsetof(struct nft_trans_obj, nft_trans) != 0);
12114	BUILD_BUG_ON(offsetof(struct nft_trans_flowtable, nft_trans) != 0);
12115
12116	err = register_pernet_subsys(&nf_tables_net_ops);
12117	if (err < 0)
12118		return err;
12119
12120	err = nft_chain_filter_init();
12121	if (err < 0)
12122		goto err_chain_filter;
12123
12124	err = nf_tables_core_module_init();
12125	if (err < 0)
12126		goto err_core_module;
12127
12128	err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
12129	if (err < 0)
12130		goto err_netdev_notifier;
12131
12132	err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
12133	if (err < 0)
12134		goto err_rht_objname;
12135
12136	err = nft_offload_init();
12137	if (err < 0)
12138		goto err_offload;
12139
12140	err = netlink_register_notifier(&nft_nl_notifier);
12141	if (err < 0)
12142		goto err_netlink_notifier;
12143
12144	/* must be last */
12145	err = nfnetlink_subsys_register(&nf_tables_subsys);
12146	if (err < 0)
12147		goto err_nfnl_subsys;
12148
12149	nft_chain_route_init();
12150
12151	return err;
12152
12153err_nfnl_subsys:
12154	netlink_unregister_notifier(&nft_nl_notifier);
12155err_netlink_notifier:
12156	nft_offload_exit();
12157err_offload:
12158	rhltable_destroy(&nft_objname_ht);
12159err_rht_objname:
12160	unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
12161err_netdev_notifier:
12162	nf_tables_core_module_exit();
12163err_core_module:
12164	nft_chain_filter_fini();
12165err_chain_filter:
12166	unregister_pernet_subsys(&nf_tables_net_ops);
12167	return err;
12168}
12169
12170static void __exit nf_tables_module_exit(void)
12171{
12172	nfnetlink_subsys_unregister(&nf_tables_subsys);
12173	netlink_unregister_notifier(&nft_nl_notifier);
12174	nft_offload_exit();
12175	unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
12176	nft_chain_filter_fini();
12177	nft_chain_route_fini();
12178	unregister_pernet_subsys(&nf_tables_net_ops);
12179	cancel_work_sync(&trans_gc_work);
12180	rcu_barrier();
12181	rhltable_destroy(&nft_objname_ht);
12182	nf_tables_core_module_exit();
12183}
12184
12185module_init(nf_tables_module_init);
12186module_exit(nf_tables_module_exit);
12187
12188MODULE_LICENSE("GPL");
12189MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
12190MODULE_DESCRIPTION("Framework for packet filtering and classification");
12191MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);