cls_api.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * net/sched/cls_api.c	Packet classifier API.
   4 *
   5 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
   6 *
   7 * Changes:
   8 *
   9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  10 */
  11
  12#include <linux/module.h>
  13#include <linux/types.h>
  14#include <linux/kernel.h>
  15#include <linux/string.h>
  16#include <linux/errno.h>
  17#include <linux/err.h>
  18#include <linux/skbuff.h>
  19#include <linux/init.h>
  20#include <linux/kmod.h>
  21#include <linux/slab.h>
  22#include <linux/idr.h>
  23#include <linux/jhash.h>
  24#include <linux/rculist.h>
  25#include <linux/rhashtable.h>
  26#include <net/net_namespace.h>
  27#include <net/sock.h>
  28#include <net/netlink.h>
  29#include <net/pkt_sched.h>
  30#include <net/pkt_cls.h>
  31#include <net/tc_act/tc_pedit.h>
  32#include <net/tc_act/tc_mirred.h>
  33#include <net/tc_act/tc_vlan.h>
  34#include <net/tc_act/tc_tunnel_key.h>
  35#include <net/tc_act/tc_csum.h>
  36#include <net/tc_act/tc_gact.h>
  37#include <net/tc_act/tc_police.h>
  38#include <net/tc_act/tc_sample.h>
  39#include <net/tc_act/tc_skbedit.h>
  40#include <net/tc_act/tc_ct.h>
  41#include <net/tc_act/tc_mpls.h>
  42#include <net/tc_act/tc_gate.h>
  43#include <net/flow_offload.h>
  44#include <net/tc_wrapper.h>
  45
  46/* The list of all installed classifier types */
  47static LIST_HEAD(tcf_proto_base);
  48
  49/* Protects list of registered TC modules. It is pure SMP lock. */
  50static DEFINE_RWLOCK(cls_mod_lock);
  51
  52static struct xarray tcf_exts_miss_cookies_xa;
  53struct tcf_exts_miss_cookie_node {
  54	const struct tcf_chain *chain;
  55	const struct tcf_proto *tp;
  56	const struct tcf_exts *exts;
  57	u32 chain_index;
  58	u32 tp_prio;
  59	u32 handle;
  60	u32 miss_cookie_base;
  61	struct rcu_head rcu;
  62};
  63
  64/* Each tc action entry cookie will be comprised of 32bit miss_cookie_base +
  65 * action index in the exts tc actions array.
  66 */
  67union tcf_exts_miss_cookie {
  68	struct {
  69		u32 miss_cookie_base;
  70		u32 act_index;
  71	};
  72	u64 miss_cookie;
  73};
  74
  75#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
  76static int
  77tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
  78				u32 handle)
  79{
  80	struct tcf_exts_miss_cookie_node *n;
  81	static u32 next;
  82	int err;
  83
  84	if (WARN_ON(!handle || !tp->ops->get_exts))
  85		return -EINVAL;
  86
  87	n = kzalloc(sizeof(*n), GFP_KERNEL);
  88	if (!n)
  89		return -ENOMEM;
  90
  91	n->chain_index = tp->chain->index;
  92	n->chain = tp->chain;
  93	n->tp_prio = tp->prio;
  94	n->tp = tp;
  95	n->exts = exts;
  96	n->handle = handle;
  97
  98	err = xa_alloc_cyclic(&tcf_exts_miss_cookies_xa, &n->miss_cookie_base,
  99			      n, xa_limit_32b, &next, GFP_KERNEL);
 100	if (err < 0)
 101		goto err_xa_alloc;
 102
 103	exts->miss_cookie_node = n;
 104	return 0;
 105
 106err_xa_alloc:
 107	kfree(n);
 108	return err;
 109}
 110
 111static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
 112{
 113	struct tcf_exts_miss_cookie_node *n;
 114
 115	if (!exts->miss_cookie_node)
 116		return;
 117
 118	n = exts->miss_cookie_node;
 119	xa_erase(&tcf_exts_miss_cookies_xa, n->miss_cookie_base);
 120	kfree_rcu(n, rcu);
 121}
 122
 123static struct tcf_exts_miss_cookie_node *
 124tcf_exts_miss_cookie_lookup(u64 miss_cookie, int *act_index)
 125{
 126	union tcf_exts_miss_cookie mc = { .miss_cookie = miss_cookie, };
 127
 128	*act_index = mc.act_index;
 129	return xa_load(&tcf_exts_miss_cookies_xa, mc.miss_cookie_base);
 130}
 131#else /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
 132static int
 133tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp,
 134				u32 handle)
 135{
 136	return 0;
 137}
 138
 139static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts)
 140{
 141}
 142#endif /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */
 143
 144static u64 tcf_exts_miss_cookie_get(u32 miss_cookie_base, int act_index)
 145{
 146	union tcf_exts_miss_cookie mc = { .act_index = act_index, };
 147
 148	if (!miss_cookie_base)
 149		return 0;
 150
 151	mc.miss_cookie_base = miss_cookie_base;
 152	return mc.miss_cookie;
 153}
 154
 155#ifdef CONFIG_NET_CLS_ACT
 156DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc);
 157EXPORT_SYMBOL(tc_skb_ext_tc);
 158
 159void tc_skb_ext_tc_enable(void)
 160{
 161	static_branch_inc(&tc_skb_ext_tc);
 162}
 163EXPORT_SYMBOL(tc_skb_ext_tc_enable);
 164
 165void tc_skb_ext_tc_disable(void)
 166{
 167	static_branch_dec(&tc_skb_ext_tc);
 168}
 169EXPORT_SYMBOL(tc_skb_ext_tc_disable);
 170#endif
 171
 172static u32 destroy_obj_hashfn(const struct tcf_proto *tp)
 173{
 174	return jhash_3words(tp->chain->index, tp->prio,
 175			    (__force __u32)tp->protocol, 0);
 176}
 177
 178static void tcf_proto_signal_destroying(struct tcf_chain *chain,
 179					struct tcf_proto *tp)
 180{
 181	struct tcf_block *block = chain->block;
 182
 183	mutex_lock(&block->proto_destroy_lock);
 184	hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node,
 185		     destroy_obj_hashfn(tp));
 186	mutex_unlock(&block->proto_destroy_lock);
 187}
 188
 189static bool tcf_proto_cmp(const struct tcf_proto *tp1,
 190			  const struct tcf_proto *tp2)
 191{
 192	return tp1->chain->index == tp2->chain->index &&
 193	       tp1->prio == tp2->prio &&
 194	       tp1->protocol == tp2->protocol;
 195}
 196
 197static bool tcf_proto_exists_destroying(struct tcf_chain *chain,
 198					struct tcf_proto *tp)
 199{
 200	u32 hash = destroy_obj_hashfn(tp);
 201	struct tcf_proto *iter;
 202	bool found = false;
 203
 204	rcu_read_lock();
 205	hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter,
 206				   destroy_ht_node, hash) {
 207		if (tcf_proto_cmp(tp, iter)) {
 208			found = true;
 209			break;
 210		}
 211	}
 212	rcu_read_unlock();
 213
 214	return found;
 215}
 216
 217static void
 218tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp)
 219{
 220	struct tcf_block *block = chain->block;
 221
 222	mutex_lock(&block->proto_destroy_lock);
 223	if (hash_hashed(&tp->destroy_ht_node))
 224		hash_del_rcu(&tp->destroy_ht_node);
 225	mutex_unlock(&block->proto_destroy_lock);
 226}
 227
 228/* Find classifier type by string name */
 229
 230static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
 231{
 232	const struct tcf_proto_ops *t, *res = NULL;
 233
 234	if (kind) {
 235		read_lock(&cls_mod_lock);
 236		list_for_each_entry(t, &tcf_proto_base, head) {
 237			if (strcmp(kind, t->kind) == 0) {
 238				if (try_module_get(t->owner))
 239					res = t;
 240				break;
 241			}
 242		}
 243		read_unlock(&cls_mod_lock);
 244	}
 245	return res;
 246}
 247
 248static const struct tcf_proto_ops *
 249tcf_proto_lookup_ops(const char *kind, bool rtnl_held,
 250		     struct netlink_ext_ack *extack)
 251{
 252	const struct tcf_proto_ops *ops;
 253
 254	ops = __tcf_proto_lookup_ops(kind);
 255	if (ops)
 256		return ops;
 257#ifdef CONFIG_MODULES
 258	if (rtnl_held)
 259		rtnl_unlock();
 260	request_module(NET_CLS_ALIAS_PREFIX "%s", kind);
 261	if (rtnl_held)
 262		rtnl_lock();
 263	ops = __tcf_proto_lookup_ops(kind);
 264	/* We dropped the RTNL semaphore in order to perform
 265	 * the module load. So, even if we succeeded in loading
 266	 * the module we have to replay the request. We indicate
 267	 * this using -EAGAIN.
 268	 */
 269	if (ops) {
 270		module_put(ops->owner);
 271		return ERR_PTR(-EAGAIN);
 272	}
 273#endif
 274	NL_SET_ERR_MSG(extack, "TC classifier not found");
 275	return ERR_PTR(-ENOENT);
 276}
 277
 278/* Register(unregister) new classifier type */
 279
 280int register_tcf_proto_ops(struct tcf_proto_ops *ops)
 281{
 282	struct tcf_proto_ops *t;
 283	int rc = -EEXIST;
 284
 285	write_lock(&cls_mod_lock);
 286	list_for_each_entry(t, &tcf_proto_base, head)
 287		if (!strcmp(ops->kind, t->kind))
 288			goto out;
 289
 290	list_add_tail(&ops->head, &tcf_proto_base);
 291	rc = 0;
 292out:
 293	write_unlock(&cls_mod_lock);
 294	return rc;
 295}
 296EXPORT_SYMBOL(register_tcf_proto_ops);
 297
 298static struct workqueue_struct *tc_filter_wq;
 299
 300void unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
 301{
 302	struct tcf_proto_ops *t;
 303	int rc = -ENOENT;
 304
 305	/* Wait for outstanding call_rcu()s, if any, from a
 306	 * tcf_proto_ops's destroy() handler.
 307	 */
 308	rcu_barrier();
 309	flush_workqueue(tc_filter_wq);
 310
 311	write_lock(&cls_mod_lock);
 312	list_for_each_entry(t, &tcf_proto_base, head) {
 313		if (t == ops) {
 314			list_del(&t->head);
 315			rc = 0;
 316			break;
 317		}
 318	}
 319	write_unlock(&cls_mod_lock);
 320
 321	WARN(rc, "unregister tc filter kind(%s) failed %d\n", ops->kind, rc);
 322}
 323EXPORT_SYMBOL(unregister_tcf_proto_ops);
 324
 325bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
 326{
 327	INIT_RCU_WORK(rwork, func);
 328	return queue_rcu_work(tc_filter_wq, rwork);
 329}
 330EXPORT_SYMBOL(tcf_queue_work);
 331
 332/* Select new prio value from the range, managed by kernel. */
 333
 334static inline u32 tcf_auto_prio(struct tcf_proto *tp)
 335{
 336	u32 first = TC_H_MAKE(0xC0000000U, 0U);
 337
 338	if (tp)
 339		first = tp->prio - 1;
 340
 341	return TC_H_MAJ(first);
 342}
 343
 344static bool tcf_proto_check_kind(struct nlattr *kind, char *name)
 345{
 346	if (kind)
 347		return nla_strscpy(name, kind, IFNAMSIZ) < 0;
 348	memset(name, 0, IFNAMSIZ);
 349	return false;
 350}
 351
 352static bool tcf_proto_is_unlocked(const char *kind)
 353{
 354	const struct tcf_proto_ops *ops;
 355	bool ret;
 356
 357	if (strlen(kind) == 0)
 358		return false;
 359
 360	ops = tcf_proto_lookup_ops(kind, false, NULL);
 361	/* On error return false to take rtnl lock. Proto lookup/create
 362	 * functions will perform lookup again and properly handle errors.
 363	 */
 364	if (IS_ERR(ops))
 365		return false;
 366
 367	ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED);
 368	module_put(ops->owner);
 369	return ret;
 370}
 371
 372static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
 373					  u32 prio, struct tcf_chain *chain,
 374					  bool rtnl_held,
 375					  struct netlink_ext_ack *extack)
 376{
 377	struct tcf_proto *tp;
 378	int err;
 379
 380	tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 381	if (!tp)
 382		return ERR_PTR(-ENOBUFS);
 383
 384	tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack);
 385	if (IS_ERR(tp->ops)) {
 386		err = PTR_ERR(tp->ops);
 387		goto errout;
 388	}
 389	tp->classify = tp->ops->classify;
 390	tp->protocol = protocol;
 391	tp->prio = prio;
 392	tp->chain = chain;
 393	tp->usesw = !tp->ops->reoffload;
 394	spin_lock_init(&tp->lock);
 395	refcount_set(&tp->refcnt, 1);
 396
 397	err = tp->ops->init(tp);
 398	if (err) {
 399		module_put(tp->ops->owner);
 400		goto errout;
 401	}
 402	return tp;
 403
 404errout:
 405	kfree(tp);
 406	return ERR_PTR(err);
 407}
 408
 409static void tcf_proto_get(struct tcf_proto *tp)
 410{
 411	refcount_inc(&tp->refcnt);
 412}
 413
 414static void tcf_proto_count_usesw(struct tcf_proto *tp, bool add)
 415{
 416#ifdef CONFIG_NET_CLS_ACT
 417	struct tcf_block *block = tp->chain->block;
 418	bool counted = false;
 419
 420	if (!add) {
 421		if (tp->usesw && tp->counted) {
 422			if (!atomic_dec_return(&block->useswcnt))
 423				static_branch_dec(&tcf_sw_enabled_key);
 424			tp->counted = false;
 425		}
 426		return;
 427	}
 428
 429	spin_lock(&tp->lock);
 430	if (tp->usesw && !tp->counted) {
 431		counted = true;
 432		tp->counted = true;
 433	}
 434	spin_unlock(&tp->lock);
 435
 436	if (counted && atomic_inc_return(&block->useswcnt) == 1)
 437		static_branch_inc(&tcf_sw_enabled_key);
 438#endif
 439}
 440
 441static void tcf_chain_put(struct tcf_chain *chain);
 442
 443static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held,
 444			      bool sig_destroy, struct netlink_ext_ack *extack)
 445{
 446	tp->ops->destroy(tp, rtnl_held, extack);
 447	tcf_proto_count_usesw(tp, false);
 448	if (sig_destroy)
 449		tcf_proto_signal_destroyed(tp->chain, tp);
 450	tcf_chain_put(tp->chain);
 451	module_put(tp->ops->owner);
 452	kfree_rcu(tp, rcu);
 453}
 454
 455static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held,
 456			  struct netlink_ext_ack *extack)
 457{
 458	if (refcount_dec_and_test(&tp->refcnt))
 459		tcf_proto_destroy(tp, rtnl_held, true, extack);
 460}
 461
 462static bool tcf_proto_check_delete(struct tcf_proto *tp)
 463{
 464	if (tp->ops->delete_empty)
 465		return tp->ops->delete_empty(tp);
 466
 467	tp->deleting = true;
 468	return tp->deleting;
 469}
 470
 471static void tcf_proto_mark_delete(struct tcf_proto *tp)
 472{
 473	spin_lock(&tp->lock);
 474	tp->deleting = true;
 475	spin_unlock(&tp->lock);
 476}
 477
 478static bool tcf_proto_is_deleting(struct tcf_proto *tp)
 479{
 480	bool deleting;
 481
 482	spin_lock(&tp->lock);
 483	deleting = tp->deleting;
 484	spin_unlock(&tp->lock);
 485
 486	return deleting;
 487}
 488
 489#define ASSERT_BLOCK_LOCKED(block)					\
 490	lockdep_assert_held(&(block)->lock)
 491
 492struct tcf_filter_chain_list_item {
 493	struct list_head list;
 494	tcf_chain_head_change_t *chain_head_change;
 495	void *chain_head_change_priv;
 496};
 497
 498static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
 499					  u32 chain_index)
 500{
 501	struct tcf_chain *chain;
 502
 503	ASSERT_BLOCK_LOCKED(block);
 504
 505	chain = kzalloc(sizeof(*chain), GFP_KERNEL);
 506	if (!chain)
 507		return NULL;
 508	list_add_tail_rcu(&chain->list, &block->chain_list);
 509	mutex_init(&chain->filter_chain_lock);
 510	chain->block = block;
 511	chain->index = chain_index;
 512	chain->refcnt = 1;
 513	if (!chain->index)
 514		block->chain0.chain = chain;
 515	return chain;
 516}
 517
 518static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
 519				       struct tcf_proto *tp_head)
 520{
 521	if (item->chain_head_change)
 522		item->chain_head_change(tp_head, item->chain_head_change_priv);
 523}
 524
 525static void tcf_chain0_head_change(struct tcf_chain *chain,
 526				   struct tcf_proto *tp_head)
 527{
 528	struct tcf_filter_chain_list_item *item;
 529	struct tcf_block *block = chain->block;
 530
 531	if (chain->index)
 532		return;
 533
 534	mutex_lock(&block->lock);
 535	list_for_each_entry(item, &block->chain0.filter_chain_list, list)
 536		tcf_chain_head_change_item(item, tp_head);
 537	mutex_unlock(&block->lock);
 538}
 539
 540/* Returns true if block can be safely freed. */
 541
 542static bool tcf_chain_detach(struct tcf_chain *chain)
 543{
 544	struct tcf_block *block = chain->block;
 545
 546	ASSERT_BLOCK_LOCKED(block);
 547
 548	list_del_rcu(&chain->list);
 549	if (!chain->index)
 550		block->chain0.chain = NULL;
 551
 552	if (list_empty(&block->chain_list) &&
 553	    refcount_read(&block->refcnt) == 0)
 554		return true;
 555
 556	return false;
 557}
 558
 559static void tcf_block_destroy(struct tcf_block *block)
 560{
 561	mutex_destroy(&block->lock);
 562	mutex_destroy(&block->proto_destroy_lock);
 563	xa_destroy(&block->ports);
 564	kfree_rcu(block, rcu);
 565}
 566
 567static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
 568{
 569	struct tcf_block *block = chain->block;
 570
 571	mutex_destroy(&chain->filter_chain_lock);
 572	kfree_rcu(chain, rcu);
 573	if (free_block)
 574		tcf_block_destroy(block);
 575}
 576
 577static void tcf_chain_hold(struct tcf_chain *chain)
 578{
 579	ASSERT_BLOCK_LOCKED(chain->block);
 580
 581	++chain->refcnt;
 582}
 583
 584static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
 585{
 586	ASSERT_BLOCK_LOCKED(chain->block);
 587
 588	/* In case all the references are action references, this
 589	 * chain should not be shown to the user.
 590	 */
 591	return chain->refcnt == chain->action_refcnt;
 592}
 593
 594static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
 595					  u32 chain_index)
 596{
 597	struct tcf_chain *chain;
 598
 599	ASSERT_BLOCK_LOCKED(block);
 600
 601	list_for_each_entry(chain, &block->chain_list, list) {
 602		if (chain->index == chain_index)
 603			return chain;
 604	}
 605	return NULL;
 606}
 607
 608#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
 609static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block,
 610					      u32 chain_index)
 611{
 612	struct tcf_chain *chain;
 613
 614	list_for_each_entry_rcu(chain, &block->chain_list, list) {
 615		if (chain->index == chain_index)
 616			return chain;
 617	}
 618	return NULL;
 619}
 620#endif
 621
 622static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
 623			   u32 seq, u16 flags, int event, bool unicast,
 624			   struct netlink_ext_ack *extack);
 625
 626static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
 627					 u32 chain_index, bool create,
 628					 bool by_act)
 629{
 630	struct tcf_chain *chain = NULL;
 631	bool is_first_reference;
 632
 633	mutex_lock(&block->lock);
 634	chain = tcf_chain_lookup(block, chain_index);
 635	if (chain) {
 636		tcf_chain_hold(chain);
 637	} else {
 638		if (!create)
 639			goto errout;
 640		chain = tcf_chain_create(block, chain_index);
 641		if (!chain)
 642			goto errout;
 643	}
 644
 645	if (by_act)
 646		++chain->action_refcnt;
 647	is_first_reference = chain->refcnt - chain->action_refcnt == 1;
 648	mutex_unlock(&block->lock);
 649
 650	/* Send notification only in case we got the first
 651	 * non-action reference. Until then, the chain acts only as
 652	 * a placeholder for actions pointing to it and user ought
 653	 * not know about them.
 654	 */
 655	if (is_first_reference && !by_act)
 656		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
 657				RTM_NEWCHAIN, false, NULL);
 658
 659	return chain;
 660
 661errout:
 662	mutex_unlock(&block->lock);
 663	return chain;
 664}
 665
 666static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
 667				       bool create)
 668{
 669	return __tcf_chain_get(block, chain_index, create, false);
 670}
 671
 672struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
 673{
 674	return __tcf_chain_get(block, chain_index, true, true);
 675}
 676EXPORT_SYMBOL(tcf_chain_get_by_act);
 677
 678static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
 679			       void *tmplt_priv);
 680static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
 681				  void *tmplt_priv, u32 chain_index,
 682				  struct tcf_block *block, struct sk_buff *oskb,
 683				  u32 seq, u16 flags);
 684
 685static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
 686			    bool explicitly_created)
 687{
 688	struct tcf_block *block = chain->block;
 689	const struct tcf_proto_ops *tmplt_ops;
 690	unsigned int refcnt, non_act_refcnt;
 691	bool free_block = false;
 692	void *tmplt_priv;
 693
 694	mutex_lock(&block->lock);
 695	if (explicitly_created) {
 696		if (!chain->explicitly_created) {
 697			mutex_unlock(&block->lock);
 698			return;
 699		}
 700		chain->explicitly_created = false;
 701	}
 702
 703	if (by_act)
 704		chain->action_refcnt--;
 705
 706	/* tc_chain_notify_delete can't be called while holding block lock.
 707	 * However, when block is unlocked chain can be changed concurrently, so
 708	 * save these to temporary variables.
 709	 */
 710	refcnt = --chain->refcnt;
 711	non_act_refcnt = refcnt - chain->action_refcnt;
 712	tmplt_ops = chain->tmplt_ops;
 713	tmplt_priv = chain->tmplt_priv;
 714
 715	if (non_act_refcnt == chain->explicitly_created && !by_act) {
 716		if (non_act_refcnt == 0)
 717			tc_chain_notify_delete(tmplt_ops, tmplt_priv,
 718					       chain->index, block, NULL, 0, 0);
 719		/* Last reference to chain, no need to lock. */
 720		chain->flushing = false;
 721	}
 722
 723	if (refcnt == 0)
 724		free_block = tcf_chain_detach(chain);
 725	mutex_unlock(&block->lock);
 726
 727	if (refcnt == 0) {
 728		tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
 729		tcf_chain_destroy(chain, free_block);
 730	}
 731}
 732
 733static void tcf_chain_put(struct tcf_chain *chain)
 734{
 735	__tcf_chain_put(chain, false, false);
 736}
 737
 738void tcf_chain_put_by_act(struct tcf_chain *chain)
 739{
 740	__tcf_chain_put(chain, true, false);
 741}
 742EXPORT_SYMBOL(tcf_chain_put_by_act);
 743
 744static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
 745{
 746	__tcf_chain_put(chain, false, true);
 747}
 748
 749static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held)
 750{
 751	struct tcf_proto *tp, *tp_next;
 752
 753	mutex_lock(&chain->filter_chain_lock);
 754	tp = tcf_chain_dereference(chain->filter_chain, chain);
 755	while (tp) {
 756		tp_next = rcu_dereference_protected(tp->next, 1);
 757		tcf_proto_signal_destroying(chain, tp);
 758		tp = tp_next;
 759	}
 760	tp = tcf_chain_dereference(chain->filter_chain, chain);
 761	RCU_INIT_POINTER(chain->filter_chain, NULL);
 762	tcf_chain0_head_change(chain, NULL);
 763	chain->flushing = true;
 764	mutex_unlock(&chain->filter_chain_lock);
 765
 766	while (tp) {
 767		tp_next = rcu_dereference_protected(tp->next, 1);
 768		tcf_proto_put(tp, rtnl_held, NULL);
 769		tp = tp_next;
 770	}
 771}
 772
 773static int tcf_block_setup(struct tcf_block *block,
 774			   struct flow_block_offload *bo);
 775
 776static void tcf_block_offload_init(struct flow_block_offload *bo,
 777				   struct net_device *dev, struct Qdisc *sch,
 778				   enum flow_block_command command,
 779				   enum flow_block_binder_type binder_type,
 780				   struct flow_block *flow_block,
 781				   bool shared, struct netlink_ext_ack *extack)
 782{
 783	bo->net = dev_net(dev);
 784	bo->command = command;
 785	bo->binder_type = binder_type;
 786	bo->block = flow_block;
 787	bo->block_shared = shared;
 788	bo->extack = extack;
 789	bo->sch = sch;
 790	bo->cb_list_head = &flow_block->cb_list;
 791	INIT_LIST_HEAD(&bo->cb_list);
 792}
 793
 794static void tcf_block_unbind(struct tcf_block *block,
 795			     struct flow_block_offload *bo);
 796
 797static void tc_block_indr_cleanup(struct flow_block_cb *block_cb)
 798{
 799	struct tcf_block *block = block_cb->indr.data;
 800	struct net_device *dev = block_cb->indr.dev;
 801	struct Qdisc *sch = block_cb->indr.sch;
 802	struct netlink_ext_ack extack = {};
 803	struct flow_block_offload bo = {};
 804
 805	tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND,
 806			       block_cb->indr.binder_type,
 807			       &block->flow_block, tcf_block_shared(block),
 808			       &extack);
 809	rtnl_lock();
 810	down_write(&block->cb_lock);
 811	list_del(&block_cb->driver_list);
 812	list_move(&block_cb->list, &bo.cb_list);
 813	tcf_block_unbind(block, &bo);
 814	up_write(&block->cb_lock);
 815	rtnl_unlock();
 816}
 817
 818static bool tcf_block_offload_in_use(struct tcf_block *block)
 819{
 820	return atomic_read(&block->offloadcnt);
 821}
 822
 823static int tcf_block_offload_cmd(struct tcf_block *block,
 824				 struct net_device *dev, struct Qdisc *sch,
 825				 struct tcf_block_ext_info *ei,
 826				 enum flow_block_command command,
 827				 struct netlink_ext_ack *extack)
 828{
 829	struct flow_block_offload bo = {};
 830
 831	tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type,
 832			       &block->flow_block, tcf_block_shared(block),
 833			       extack);
 834
 835	if (dev->netdev_ops->ndo_setup_tc) {
 836		int err;
 837
 838		err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 839		if (err < 0) {
 840			if (err != -EOPNOTSUPP)
 841				NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed");
 842			return err;
 843		}
 844
 845		return tcf_block_setup(block, &bo);
 846	}
 847
 848	flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo,
 849				    tc_block_indr_cleanup);
 850	tcf_block_setup(block, &bo);
 851
 852	return -EOPNOTSUPP;
 853}
 854
 855static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
 856				  struct tcf_block_ext_info *ei,
 857				  struct netlink_ext_ack *extack)
 858{
 859	struct net_device *dev = q->dev_queue->dev;
 860	int err;
 861
 862	down_write(&block->cb_lock);
 863
 864	/* If tc offload feature is disabled and the block we try to bind
 865	 * to already has some offloaded filters, forbid to bind.
 866	 */
 867	if (dev->netdev_ops->ndo_setup_tc &&
 868	    !tc_can_offload(dev) &&
 869	    tcf_block_offload_in_use(block)) {
 870		NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
 871		err = -EOPNOTSUPP;
 872		goto err_unlock;
 873	}
 874
 875	err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack);
 876	if (err == -EOPNOTSUPP)
 877		goto no_offload_dev_inc;
 878	if (err)
 879		goto err_unlock;
 880
 881	up_write(&block->cb_lock);
 882	return 0;
 883
 884no_offload_dev_inc:
 885	if (tcf_block_offload_in_use(block))
 886		goto err_unlock;
 887
 888	err = 0;
 889	block->nooffloaddevcnt++;
 890err_unlock:
 891	up_write(&block->cb_lock);
 892	return err;
 893}
 894
 895static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
 896				     struct tcf_block_ext_info *ei)
 897{
 898	struct net_device *dev = q->dev_queue->dev;
 899	int err;
 900
 901	down_write(&block->cb_lock);
 902	err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL);
 903	if (err == -EOPNOTSUPP)
 904		goto no_offload_dev_dec;
 905	up_write(&block->cb_lock);
 906	return;
 907
 908no_offload_dev_dec:
 909	WARN_ON(block->nooffloaddevcnt-- == 0);
 910	up_write(&block->cb_lock);
 911}
 912
 913static int
 914tcf_chain0_head_change_cb_add(struct tcf_block *block,
 915			      struct tcf_block_ext_info *ei,
 916			      struct netlink_ext_ack *extack)
 917{
 918	struct tcf_filter_chain_list_item *item;
 919	struct tcf_chain *chain0;
 920
 921	item = kmalloc(sizeof(*item), GFP_KERNEL);
 922	if (!item) {
 923		NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
 924		return -ENOMEM;
 925	}
 926	item->chain_head_change = ei->chain_head_change;
 927	item->chain_head_change_priv = ei->chain_head_change_priv;
 928
 929	mutex_lock(&block->lock);
 930	chain0 = block->chain0.chain;
 931	if (chain0)
 932		tcf_chain_hold(chain0);
 933	else
 934		list_add(&item->list, &block->chain0.filter_chain_list);
 935	mutex_unlock(&block->lock);
 936
 937	if (chain0) {
 938		struct tcf_proto *tp_head;
 939
 940		mutex_lock(&chain0->filter_chain_lock);
 941
 942		tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
 943		if (tp_head)
 944			tcf_chain_head_change_item(item, tp_head);
 945
 946		mutex_lock(&block->lock);
 947		list_add(&item->list, &block->chain0.filter_chain_list);
 948		mutex_unlock(&block->lock);
 949
 950		mutex_unlock(&chain0->filter_chain_lock);
 951		tcf_chain_put(chain0);
 952	}
 953
 954	return 0;
 955}
 956
 957static void
 958tcf_chain0_head_change_cb_del(struct tcf_block *block,
 959			      struct tcf_block_ext_info *ei)
 960{
 961	struct tcf_filter_chain_list_item *item;
 962
 963	mutex_lock(&block->lock);
 964	list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
 965		if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
 966		    (item->chain_head_change == ei->chain_head_change &&
 967		     item->chain_head_change_priv == ei->chain_head_change_priv)) {
 968			if (block->chain0.chain)
 969				tcf_chain_head_change_item(item, NULL);
 970			list_del(&item->list);
 971			mutex_unlock(&block->lock);
 972
 973			kfree(item);
 974			return;
 975		}
 976	}
 977	mutex_unlock(&block->lock);
 978	WARN_ON(1);
 979}
 980
 981struct tcf_net {
 982	spinlock_t idr_lock; /* Protects idr */
 983	struct idr idr;
 984};
 985
 986static unsigned int tcf_net_id;
 987
 988static int tcf_block_insert(struct tcf_block *block, struct net *net,
 989			    struct netlink_ext_ack *extack)
 990{
 991	struct tcf_net *tn = net_generic(net, tcf_net_id);
 992	int err;
 993
 994	idr_preload(GFP_KERNEL);
 995	spin_lock(&tn->idr_lock);
 996	err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
 997			    GFP_NOWAIT);
 998	spin_unlock(&tn->idr_lock);
 999	idr_preload_end();
1000
1001	return err;
1002}
1003
1004static void tcf_block_remove(struct tcf_block *block, struct net *net)
1005{
1006	struct tcf_net *tn = net_generic(net, tcf_net_id);
1007
1008	spin_lock(&tn->idr_lock);
1009	idr_remove(&tn->idr, block->index);
1010	spin_unlock(&tn->idr_lock);
1011}
1012
1013static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
1014					  u32 block_index,
1015					  struct netlink_ext_ack *extack)
1016{
1017	struct tcf_block *block;
1018
1019	block = kzalloc(sizeof(*block), GFP_KERNEL);
1020	if (!block) {
1021		NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
1022		return ERR_PTR(-ENOMEM);
1023	}
1024	mutex_init(&block->lock);
1025	mutex_init(&block->proto_destroy_lock);
1026	init_rwsem(&block->cb_lock);
1027	flow_block_init(&block->flow_block);
1028	INIT_LIST_HEAD(&block->chain_list);
1029	INIT_LIST_HEAD(&block->owner_list);
1030	INIT_LIST_HEAD(&block->chain0.filter_chain_list);
1031
1032	refcount_set(&block->refcnt, 1);
1033	block->net = net;
1034	block->index = block_index;
1035	xa_init(&block->ports);
1036
1037	/* Don't store q pointer for blocks which are shared */
1038	if (!tcf_block_shared(block))
1039		block->q = q;
1040	return block;
1041}
1042
1043struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
1044{
1045	struct tcf_net *tn = net_generic(net, tcf_net_id);
1046
1047	return idr_find(&tn->idr, block_index);
1048}
1049EXPORT_SYMBOL(tcf_block_lookup);
1050
1051static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
1052{
1053	struct tcf_block *block;
1054
1055	rcu_read_lock();
1056	block = tcf_block_lookup(net, block_index);
1057	if (block && !refcount_inc_not_zero(&block->refcnt))
1058		block = NULL;
1059	rcu_read_unlock();
1060
1061	return block;
1062}
1063
1064static struct tcf_chain *
1065__tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1066{
1067	mutex_lock(&block->lock);
1068	if (chain)
1069		chain = list_is_last(&chain->list, &block->chain_list) ?
1070			NULL : list_next_entry(chain, list);
1071	else
1072		chain = list_first_entry_or_null(&block->chain_list,
1073						 struct tcf_chain, list);
1074
1075	/* skip all action-only chains */
1076	while (chain && tcf_chain_held_by_acts_only(chain))
1077		chain = list_is_last(&chain->list, &block->chain_list) ?
1078			NULL : list_next_entry(chain, list);
1079
1080	if (chain)
1081		tcf_chain_hold(chain);
1082	mutex_unlock(&block->lock);
1083
1084	return chain;
1085}
1086
1087/* Function to be used by all clients that want to iterate over all chains on
1088 * block. It properly obtains block->lock and takes reference to chain before
1089 * returning it. Users of this function must be tolerant to concurrent chain
1090 * insertion/deletion or ensure that no concurrent chain modification is
1091 * possible. Note that all netlink dump callbacks cannot guarantee to provide
1092 * consistent dump because rtnl lock is released each time skb is filled with
1093 * data and sent to user-space.
1094 */
1095
1096struct tcf_chain *
1097tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1098{
1099	struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
1100
1101	if (chain)
1102		tcf_chain_put(chain);
1103
1104	return chain_next;
1105}
1106EXPORT_SYMBOL(tcf_get_next_chain);
1107
1108static struct tcf_proto *
1109__tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1110{
1111	u32 prio = 0;
1112
1113	ASSERT_RTNL();
1114	mutex_lock(&chain->filter_chain_lock);
1115
1116	if (!tp) {
1117		tp = tcf_chain_dereference(chain->filter_chain, chain);
1118	} else if (tcf_proto_is_deleting(tp)) {
1119		/* 'deleting' flag is set and chain->filter_chain_lock was
1120		 * unlocked, which means next pointer could be invalid. Restart
1121		 * search.
1122		 */
1123		prio = tp->prio + 1;
1124		tp = tcf_chain_dereference(chain->filter_chain, chain);
1125
1126		for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1127			if (!tp->deleting && tp->prio >= prio)
1128				break;
1129	} else {
1130		tp = tcf_chain_dereference(tp->next, chain);
1131	}
1132
1133	if (tp)
1134		tcf_proto_get(tp);
1135
1136	mutex_unlock(&chain->filter_chain_lock);
1137
1138	return tp;
1139}
1140
1141/* Function to be used by all clients that want to iterate over all tp's on
1142 * chain. Users of this function must be tolerant to concurrent tp
1143 * insertion/deletion or ensure that no concurrent chain modification is
1144 * possible. Note that all netlink dump callbacks cannot guarantee to provide
1145 * consistent dump because rtnl lock is released each time skb is filled with
1146 * data and sent to user-space.
1147 */
1148
1149struct tcf_proto *
1150tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1151{
1152	struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1153
1154	if (tp)
1155		tcf_proto_put(tp, true, NULL);
1156
1157	return tp_next;
1158}
1159EXPORT_SYMBOL(tcf_get_next_proto);
1160
1161static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held)
1162{
1163	struct tcf_chain *chain;
1164
1165	/* Last reference to block. At this point chains cannot be added or
1166	 * removed concurrently.
1167	 */
1168	for (chain = tcf_get_next_chain(block, NULL);
1169	     chain;
1170	     chain = tcf_get_next_chain(block, chain)) {
1171		tcf_chain_put_explicitly_created(chain);
1172		tcf_chain_flush(chain, rtnl_held);
1173	}
1174}
1175
1176/* Lookup Qdisc and increments its reference counter.
1177 * Set parent, if necessary.
1178 */
1179
1180static int __tcf_qdisc_find(struct net *net, struct Qdisc **q,
1181			    u32 *parent, int ifindex, bool rtnl_held,
1182			    struct netlink_ext_ack *extack)
1183{
1184	const struct Qdisc_class_ops *cops;
1185	struct net_device *dev;
1186	int err = 0;
1187
1188	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1189		return 0;
1190
1191	rcu_read_lock();
1192
1193	/* Find link */
1194	dev = dev_get_by_index_rcu(net, ifindex);
1195	if (!dev) {
1196		rcu_read_unlock();
1197		return -ENODEV;
1198	}
1199
1200	/* Find qdisc */
1201	if (!*parent) {
1202		*q = rcu_dereference(dev->qdisc);
1203		*parent = (*q)->handle;
1204	} else {
1205		*q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1206		if (!*q) {
1207			NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1208			err = -EINVAL;
1209			goto errout_rcu;
1210		}
1211	}
1212
1213	*q = qdisc_refcount_inc_nz(*q);
1214	if (!*q) {
1215		NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1216		err = -EINVAL;
1217		goto errout_rcu;
1218	}
1219
1220	/* Is it classful? */
1221	cops = (*q)->ops->cl_ops;
1222	if (!cops) {
1223		NL_SET_ERR_MSG(extack, "Qdisc not classful");
1224		err = -EINVAL;
1225		goto errout_qdisc;
1226	}
1227
1228	if (!cops->tcf_block) {
1229		NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1230		err = -EOPNOTSUPP;
1231		goto errout_qdisc;
1232	}
1233
1234errout_rcu:
1235	/* At this point we know that qdisc is not noop_qdisc,
1236	 * which means that qdisc holds a reference to net_device
1237	 * and we hold a reference to qdisc, so it is safe to release
1238	 * rcu read lock.
1239	 */
1240	rcu_read_unlock();
1241	return err;
1242
1243errout_qdisc:
1244	rcu_read_unlock();
1245
1246	if (rtnl_held)
1247		qdisc_put(*q);
1248	else
1249		qdisc_put_unlocked(*q);
1250	*q = NULL;
1251
1252	return err;
1253}
1254
1255static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl,
1256			       int ifindex, struct netlink_ext_ack *extack)
1257{
1258	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK)
1259		return 0;
1260
1261	/* Do we search for filter, attached to class? */
1262	if (TC_H_MIN(parent)) {
1263		const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1264
1265		*cl = cops->find(q, parent);
1266		if (*cl == 0) {
1267			NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1268			return -ENOENT;
1269		}
1270	}
1271
1272	return 0;
1273}
1274
1275static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q,
1276					  unsigned long cl, int ifindex,
1277					  u32 block_index,
1278					  struct netlink_ext_ack *extack)
1279{
1280	struct tcf_block *block;
1281
1282	if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1283		block = tcf_block_refcnt_get(net, block_index);
1284		if (!block) {
1285			NL_SET_ERR_MSG(extack, "Block of given index was not found");
1286			return ERR_PTR(-EINVAL);
1287		}
1288	} else {
1289		const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1290
1291		block = cops->tcf_block(q, cl, extack);
1292		if (!block)
1293			return ERR_PTR(-EINVAL);
1294
1295		if (tcf_block_shared(block)) {
1296			NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1297			return ERR_PTR(-EOPNOTSUPP);
1298		}
1299
1300		/* Always take reference to block in order to support execution
1301		 * of rules update path of cls API without rtnl lock. Caller
1302		 * must release block when it is finished using it. 'if' block
1303		 * of this conditional obtain reference to block by calling
1304		 * tcf_block_refcnt_get().
1305		 */
1306		refcount_inc(&block->refcnt);
1307	}
1308
1309	return block;
1310}
1311
1312static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
1313			    struct tcf_block_ext_info *ei, bool rtnl_held)
1314{
1315	if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
1316		/* Flushing/putting all chains will cause the block to be
1317		 * deallocated when last chain is freed. However, if chain_list
1318		 * is empty, block has to be manually deallocated. After block
1319		 * reference counter reached 0, it is no longer possible to
1320		 * increment it or add new chains to block.
1321		 */
1322		bool free_block = list_empty(&block->chain_list);
1323
1324		mutex_unlock(&block->lock);
1325		if (tcf_block_shared(block))
1326			tcf_block_remove(block, block->net);
1327
1328		if (q)
1329			tcf_block_offload_unbind(block, q, ei);
1330
1331		if (free_block)
1332			tcf_block_destroy(block);
1333		else
1334			tcf_block_flush_all_chains(block, rtnl_held);
1335	} else if (q) {
1336		tcf_block_offload_unbind(block, q, ei);
1337	}
1338}
1339
1340static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held)
1341{
1342	__tcf_block_put(block, NULL, NULL, rtnl_held);
1343}
1344
1345/* Find tcf block.
1346 * Set q, parent, cl when appropriate.
1347 */
1348
1349static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1350					u32 *parent, unsigned long *cl,
1351					int ifindex, u32 block_index,
1352					struct netlink_ext_ack *extack)
1353{
1354	struct tcf_block *block;
1355	int err = 0;
1356
1357	ASSERT_RTNL();
1358
1359	err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack);
1360	if (err)
1361		goto errout;
1362
1363	err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack);
1364	if (err)
1365		goto errout_qdisc;
1366
1367	block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack);
1368	if (IS_ERR(block)) {
1369		err = PTR_ERR(block);
1370		goto errout_qdisc;
1371	}
1372
1373	return block;
1374
1375errout_qdisc:
1376	if (*q)
1377		qdisc_put(*q);
1378errout:
1379	*q = NULL;
1380	return ERR_PTR(err);
1381}
1382
1383static void tcf_block_release(struct Qdisc *q, struct tcf_block *block,
1384			      bool rtnl_held)
1385{
1386	if (!IS_ERR_OR_NULL(block))
1387		tcf_block_refcnt_put(block, rtnl_held);
1388
1389	if (q) {
1390		if (rtnl_held)
1391			qdisc_put(q);
1392		else
1393			qdisc_put_unlocked(q);
1394	}
1395}
1396
1397struct tcf_block_owner_item {
1398	struct list_head list;
1399	struct Qdisc *q;
1400	enum flow_block_binder_type binder_type;
1401};
1402
1403static void
1404tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1405			       struct Qdisc *q,
1406			       enum flow_block_binder_type binder_type)
1407{
1408	if (block->keep_dst &&
1409	    binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1410	    binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1411		netif_keep_dst(qdisc_dev(q));
1412}
1413
1414void tcf_block_netif_keep_dst(struct tcf_block *block)
1415{
1416	struct tcf_block_owner_item *item;
1417
1418	block->keep_dst = true;
1419	list_for_each_entry(item, &block->owner_list, list)
1420		tcf_block_owner_netif_keep_dst(block, item->q,
1421					       item->binder_type);
1422}
1423EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1424
1425static int tcf_block_owner_add(struct tcf_block *block,
1426			       struct Qdisc *q,
1427			       enum flow_block_binder_type binder_type)
1428{
1429	struct tcf_block_owner_item *item;
1430
1431	item = kmalloc(sizeof(*item), GFP_KERNEL);
1432	if (!item)
1433		return -ENOMEM;
1434	item->q = q;
1435	item->binder_type = binder_type;
1436	list_add(&item->list, &block->owner_list);
1437	return 0;
1438}
1439
1440static void tcf_block_owner_del(struct tcf_block *block,
1441				struct Qdisc *q,
1442				enum flow_block_binder_type binder_type)
1443{
1444	struct tcf_block_owner_item *item;
1445
1446	list_for_each_entry(item, &block->owner_list, list) {
1447		if (item->q == q && item->binder_type == binder_type) {
1448			list_del(&item->list);
1449			kfree(item);
1450			return;
1451		}
1452	}
1453	WARN_ON(1);
1454}
1455
1456static bool tcf_block_tracks_dev(struct tcf_block *block,
1457				 struct tcf_block_ext_info *ei)
1458{
1459	return tcf_block_shared(block) &&
1460	       (ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS ||
1461		ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS);
1462}
1463
1464int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1465		      struct tcf_block_ext_info *ei,
1466		      struct netlink_ext_ack *extack)
1467{
1468	struct net_device *dev = qdisc_dev(q);
1469	struct net *net = qdisc_net(q);
1470	struct tcf_block *block = NULL;
1471	int err;
1472
1473	if (ei->block_index)
1474		/* block_index not 0 means the shared block is requested */
1475		block = tcf_block_refcnt_get(net, ei->block_index);
1476
1477	if (!block) {
1478		block = tcf_block_create(net, q, ei->block_index, extack);
1479		if (IS_ERR(block))
1480			return PTR_ERR(block);
1481		if (tcf_block_shared(block)) {
1482			err = tcf_block_insert(block, net, extack);
1483			if (err)
1484				goto err_block_insert;
1485		}
1486	}
1487
1488	err = tcf_block_owner_add(block, q, ei->binder_type);
1489	if (err)
1490		goto err_block_owner_add;
1491
1492	tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1493
1494	err = tcf_chain0_head_change_cb_add(block, ei, extack);
1495	if (err)
1496		goto err_chain0_head_change_cb_add;
1497
1498	err = tcf_block_offload_bind(block, q, ei, extack);
1499	if (err)
1500		goto err_block_offload_bind;
1501
1502	if (tcf_block_tracks_dev(block, ei)) {
1503		err = xa_insert(&block->ports, dev->ifindex, dev, GFP_KERNEL);
1504		if (err) {
1505			NL_SET_ERR_MSG(extack, "block dev insert failed");
1506			goto err_dev_insert;
1507		}
1508	}
1509
1510	*p_block = block;
1511	return 0;
1512
1513err_dev_insert:
1514	tcf_block_offload_unbind(block, q, ei);
1515err_block_offload_bind:
1516	tcf_chain0_head_change_cb_del(block, ei);
1517err_chain0_head_change_cb_add:
1518	tcf_block_owner_del(block, q, ei->binder_type);
1519err_block_owner_add:
1520err_block_insert:
1521	tcf_block_refcnt_put(block, true);
1522	return err;
1523}
1524EXPORT_SYMBOL(tcf_block_get_ext);
1525
1526static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1527{
1528	struct tcf_proto __rcu **p_filter_chain = priv;
1529
1530	rcu_assign_pointer(*p_filter_chain, tp_head);
1531}
1532
1533int tcf_block_get(struct tcf_block **p_block,
1534		  struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1535		  struct netlink_ext_ack *extack)
1536{
1537	struct tcf_block_ext_info ei = {
1538		.chain_head_change = tcf_chain_head_change_dflt,
1539		.chain_head_change_priv = p_filter_chain,
1540	};
1541
1542	WARN_ON(!p_filter_chain);
1543	return tcf_block_get_ext(p_block, q, &ei, extack);
1544}
1545EXPORT_SYMBOL(tcf_block_get);
1546
1547/* XXX: Standalone actions are not allowed to jump to any chain, and bound
1548 * actions should be all removed after flushing.
1549 */
1550void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1551		       struct tcf_block_ext_info *ei)
1552{
1553	struct net_device *dev = qdisc_dev(q);
1554
1555	if (!block)
1556		return;
1557	if (tcf_block_tracks_dev(block, ei))
1558		xa_erase(&block->ports, dev->ifindex);
1559	tcf_chain0_head_change_cb_del(block, ei);
1560	tcf_block_owner_del(block, q, ei->binder_type);
1561
1562	__tcf_block_put(block, q, ei, true);
1563}
1564EXPORT_SYMBOL(tcf_block_put_ext);
1565
1566void tcf_block_put(struct tcf_block *block)
1567{
1568	struct tcf_block_ext_info ei = {0, };
1569
1570	if (!block)
1571		return;
1572	tcf_block_put_ext(block, block->q, &ei);
1573}
1574
1575EXPORT_SYMBOL(tcf_block_put);
1576
1577static int
1578tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb,
1579			    void *cb_priv, bool add, bool offload_in_use,
1580			    struct netlink_ext_ack *extack)
1581{
1582	struct tcf_chain *chain, *chain_prev;
1583	struct tcf_proto *tp, *tp_prev;
1584	int err;
1585
1586	lockdep_assert_held(&block->cb_lock);
1587
1588	for (chain = __tcf_get_next_chain(block, NULL);
1589	     chain;
1590	     chain_prev = chain,
1591		     chain = __tcf_get_next_chain(block, chain),
1592		     tcf_chain_put(chain_prev)) {
1593		if (chain->tmplt_ops && add)
1594			chain->tmplt_ops->tmplt_reoffload(chain, true, cb,
1595							  cb_priv);
1596		for (tp = __tcf_get_next_proto(chain, NULL); tp;
1597		     tp_prev = tp,
1598			     tp = __tcf_get_next_proto(chain, tp),
1599			     tcf_proto_put(tp_prev, true, NULL)) {
1600			if (tp->ops->reoffload) {
1601				err = tp->ops->reoffload(tp, add, cb, cb_priv,
1602							 extack);
1603				if (err && add)
1604					goto err_playback_remove;
1605			} else if (add && offload_in_use) {
1606				err = -EOPNOTSUPP;
1607				NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1608				goto err_playback_remove;
1609			}
1610		}
1611		if (chain->tmplt_ops && !add)
1612			chain->tmplt_ops->tmplt_reoffload(chain, false, cb,
1613							  cb_priv);
1614	}
1615
1616	return 0;
1617
1618err_playback_remove:
1619	tcf_proto_put(tp, true, NULL);
1620	tcf_chain_put(chain);
1621	tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1622				    extack);
1623	return err;
1624}
1625
1626static int tcf_block_bind(struct tcf_block *block,
1627			  struct flow_block_offload *bo)
1628{
1629	struct flow_block_cb *block_cb, *next;
1630	int err, i = 0;
1631
1632	lockdep_assert_held(&block->cb_lock);
1633
1634	list_for_each_entry(block_cb, &bo->cb_list, list) {
1635		err = tcf_block_playback_offloads(block, block_cb->cb,
1636						  block_cb->cb_priv, true,
1637						  tcf_block_offload_in_use(block),
1638						  bo->extack);
1639		if (err)
1640			goto err_unroll;
1641		if (!bo->unlocked_driver_cb)
1642			block->lockeddevcnt++;
1643
1644		i++;
1645	}
1646	list_splice(&bo->cb_list, &block->flow_block.cb_list);
1647
1648	return 0;
1649
1650err_unroll:
1651	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1652		list_del(&block_cb->driver_list);
1653		if (i-- > 0) {
1654			list_del(&block_cb->list);
1655			tcf_block_playback_offloads(block, block_cb->cb,
1656						    block_cb->cb_priv, false,
1657						    tcf_block_offload_in_use(block),
1658						    NULL);
1659			if (!bo->unlocked_driver_cb)
1660				block->lockeddevcnt--;
1661		}
1662		flow_block_cb_free(block_cb);
1663	}
1664
1665	return err;
1666}
1667
1668static void tcf_block_unbind(struct tcf_block *block,
1669			     struct flow_block_offload *bo)
1670{
1671	struct flow_block_cb *block_cb, *next;
1672
1673	lockdep_assert_held(&block->cb_lock);
1674
1675	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
1676		tcf_block_playback_offloads(block, block_cb->cb,
1677					    block_cb->cb_priv, false,
1678					    tcf_block_offload_in_use(block),
1679					    NULL);
1680		list_del(&block_cb->list);
1681		flow_block_cb_free(block_cb);
1682		if (!bo->unlocked_driver_cb)
1683			block->lockeddevcnt--;
1684	}
1685}
1686
1687static int tcf_block_setup(struct tcf_block *block,
1688			   struct flow_block_offload *bo)
1689{
1690	int err;
1691
1692	switch (bo->command) {
1693	case FLOW_BLOCK_BIND:
1694		err = tcf_block_bind(block, bo);
1695		break;
1696	case FLOW_BLOCK_UNBIND:
1697		err = 0;
1698		tcf_block_unbind(block, bo);
1699		break;
1700	default:
1701		WARN_ON_ONCE(1);
1702		err = -EOPNOTSUPP;
1703	}
1704
1705	return err;
1706}
1707
1708/* Main classifier routine: scans classifier chain attached
1709 * to this qdisc, (optionally) tests for protocol and asks
1710 * specific classifiers.
1711 */
1712static inline int __tcf_classify(struct sk_buff *skb,
1713				 const struct tcf_proto *tp,
1714				 const struct tcf_proto *orig_tp,
1715				 struct tcf_result *res,
1716				 bool compat_mode,
1717				 struct tcf_exts_miss_cookie_node *n,
1718				 int act_index,
1719				 u32 *last_executed_chain)
1720{
1721#ifdef CONFIG_NET_CLS_ACT
1722	const int max_reclassify_loop = 16;
1723	const struct tcf_proto *first_tp;
1724	int limit = 0;
1725
1726reclassify:
1727#endif
1728	for (; tp; tp = rcu_dereference_bh(tp->next)) {
1729		__be16 protocol = skb_protocol(skb, false);
1730		int err = 0;
1731
1732		if (n) {
1733			struct tcf_exts *exts;
1734
1735			if (n->tp_prio != tp->prio)
1736				continue;
1737
1738			/* We re-lookup the tp and chain based on index instead
1739			 * of having hard refs and locks to them, so do a sanity
1740			 * check if any of tp,chain,exts was replaced by the
1741			 * time we got here with a cookie from hardware.
1742			 */
1743			if (unlikely(n->tp != tp || n->tp->chain != n->chain ||
1744				     !tp->ops->get_exts)) {
1745				tcf_set_drop_reason(skb,
1746						    SKB_DROP_REASON_TC_COOKIE_ERROR);
1747				return TC_ACT_SHOT;
1748			}
1749
1750			exts = tp->ops->get_exts(tp, n->handle);
1751			if (unlikely(!exts || n->exts != exts)) {
1752				tcf_set_drop_reason(skb,
1753						    SKB_DROP_REASON_TC_COOKIE_ERROR);
1754				return TC_ACT_SHOT;
1755			}
1756
1757			n = NULL;
1758			err = tcf_exts_exec_ex(skb, exts, act_index, res);
1759		} else {
1760			if (tp->protocol != protocol &&
1761			    tp->protocol != htons(ETH_P_ALL))
1762				continue;
1763
1764			err = tc_classify(skb, tp, res);
1765		}
1766#ifdef CONFIG_NET_CLS_ACT
1767		if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1768			first_tp = orig_tp;
1769			*last_executed_chain = first_tp->chain->index;
1770			goto reset;
1771		} else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1772			first_tp = res->goto_tp;
1773			*last_executed_chain = err & TC_ACT_EXT_VAL_MASK;
1774			goto reset;
1775		}
1776#endif
1777		if (err >= 0)
1778			return err;
1779	}
1780
1781	if (unlikely(n)) {
1782		tcf_set_drop_reason(skb,
1783				    SKB_DROP_REASON_TC_COOKIE_ERROR);
1784		return TC_ACT_SHOT;
1785	}
1786
1787	return TC_ACT_UNSPEC; /* signal: continue lookup */
1788#ifdef CONFIG_NET_CLS_ACT
1789reset:
1790	if (unlikely(limit++ >= max_reclassify_loop)) {
1791		net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1792				       tp->chain->block->index,
1793				       tp->prio & 0xffff,
1794				       ntohs(tp->protocol));
1795		tcf_set_drop_reason(skb,
1796				    SKB_DROP_REASON_TC_RECLASSIFY_LOOP);
1797		return TC_ACT_SHOT;
1798	}
1799
1800	tp = first_tp;
1801	goto reclassify;
1802#endif
1803}
1804
1805int tcf_classify(struct sk_buff *skb,
1806		 const struct tcf_block *block,
1807		 const struct tcf_proto *tp,
1808		 struct tcf_result *res, bool compat_mode)
1809{
1810#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1811	u32 last_executed_chain = 0;
1812
1813	return __tcf_classify(skb, tp, tp, res, compat_mode, NULL, 0,
1814			      &last_executed_chain);
1815#else
1816	u32 last_executed_chain = tp ? tp->chain->index : 0;
1817	struct tcf_exts_miss_cookie_node *n = NULL;
1818	const struct tcf_proto *orig_tp = tp;
1819	struct tc_skb_ext *ext;
1820	int act_index = 0;
1821	int ret;
1822
1823	if (block) {
1824		ext = skb_ext_find(skb, TC_SKB_EXT);
1825
1826		if (ext && (ext->chain || ext->act_miss)) {
1827			struct tcf_chain *fchain;
1828			u32 chain;
1829
1830			if (ext->act_miss) {
1831				n = tcf_exts_miss_cookie_lookup(ext->act_miss_cookie,
1832								&act_index);
1833				if (!n) {
1834					tcf_set_drop_reason(skb,
1835							    SKB_DROP_REASON_TC_COOKIE_ERROR);
1836					return TC_ACT_SHOT;
1837				}
1838
1839				chain = n->chain_index;
1840			} else {
1841				chain = ext->chain;
1842			}
1843
1844			fchain = tcf_chain_lookup_rcu(block, chain);
1845			if (!fchain) {
1846				tcf_set_drop_reason(skb,
1847						    SKB_DROP_REASON_TC_CHAIN_NOTFOUND);
1848
1849				return TC_ACT_SHOT;
1850			}
1851
1852			/* Consume, so cloned/redirect skbs won't inherit ext */
1853			skb_ext_del(skb, TC_SKB_EXT);
1854
1855			tp = rcu_dereference_bh(fchain->filter_chain);
1856			last_executed_chain = fchain->index;
1857		}
1858	}
1859
1860	ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, n, act_index,
1861			     &last_executed_chain);
1862
1863	if (tc_skb_ext_tc_enabled()) {
1864		/* If we missed on some chain */
1865		if (ret == TC_ACT_UNSPEC && last_executed_chain) {
1866			struct tc_skb_cb *cb = tc_skb_cb(skb);
1867
1868			ext = tc_skb_ext_alloc(skb);
1869			if (!ext) {
1870				tcf_set_drop_reason(skb, SKB_DROP_REASON_NOMEM);
1871				return TC_ACT_SHOT;
1872			}
1873			ext->chain = last_executed_chain;
1874			ext->mru = cb->mru;
1875			ext->post_ct = qdisc_skb_cb(skb)->post_ct;
1876			ext->post_ct_snat = qdisc_skb_cb(skb)->post_ct_snat;
1877			ext->post_ct_dnat = qdisc_skb_cb(skb)->post_ct_dnat;
1878			ext->zone = cb->zone;
1879		}
1880	}
1881
1882	return ret;
1883#endif
1884}
1885EXPORT_SYMBOL(tcf_classify);
1886
1887struct tcf_chain_info {
1888	struct tcf_proto __rcu **pprev;
1889	struct tcf_proto __rcu *next;
1890};
1891
1892static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1893					   struct tcf_chain_info *chain_info)
1894{
1895	return tcf_chain_dereference(*chain_info->pprev, chain);
1896}
1897
1898static int tcf_chain_tp_insert(struct tcf_chain *chain,
1899			       struct tcf_chain_info *chain_info,
1900			       struct tcf_proto *tp)
1901{
1902	if (chain->flushing)
1903		return -EAGAIN;
1904
1905	RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1906	if (*chain_info->pprev == chain->filter_chain)
1907		tcf_chain0_head_change(chain, tp);
1908	tcf_proto_get(tp);
1909	rcu_assign_pointer(*chain_info->pprev, tp);
1910
1911	return 0;
1912}
1913
1914static void tcf_chain_tp_remove(struct tcf_chain *chain,
1915				struct tcf_chain_info *chain_info,
1916				struct tcf_proto *tp)
1917{
1918	struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1919
1920	tcf_proto_mark_delete(tp);
1921	if (tp == chain->filter_chain)
1922		tcf_chain0_head_change(chain, next);
1923	RCU_INIT_POINTER(*chain_info->pprev, next);
1924}
1925
1926static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1927					   struct tcf_chain_info *chain_info,
1928					   u32 protocol, u32 prio,
1929					   bool prio_allocate,
1930					   struct netlink_ext_ack *extack);
1931
1932/* Try to insert new proto.
1933 * If proto with specified priority already exists, free new proto
1934 * and return existing one.
1935 */
1936
1937static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1938						    struct tcf_proto *tp_new,
1939						    u32 protocol, u32 prio,
1940						    bool rtnl_held)
1941{
1942	struct tcf_chain_info chain_info;
1943	struct tcf_proto *tp;
1944	int err = 0;
1945
1946	mutex_lock(&chain->filter_chain_lock);
1947
1948	if (tcf_proto_exists_destroying(chain, tp_new)) {
1949		mutex_unlock(&chain->filter_chain_lock);
1950		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1951		return ERR_PTR(-EAGAIN);
1952	}
1953
1954	tp = tcf_chain_tp_find(chain, &chain_info, protocol, prio, false, NULL);
1955	if (!tp)
1956		err = tcf_chain_tp_insert(chain, &chain_info, tp_new);
1957	mutex_unlock(&chain->filter_chain_lock);
1958
1959	if (tp) {
1960		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1961		tp_new = tp;
1962	} else if (err) {
1963		tcf_proto_destroy(tp_new, rtnl_held, false, NULL);
1964		tp_new = ERR_PTR(err);
1965	}
1966
1967	return tp_new;
1968}
1969
1970static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1971				      struct tcf_proto *tp, bool rtnl_held,
1972				      struct netlink_ext_ack *extack)
1973{
1974	struct tcf_chain_info chain_info;
1975	struct tcf_proto *tp_iter;
1976	struct tcf_proto **pprev;
1977	struct tcf_proto *next;
1978
1979	mutex_lock(&chain->filter_chain_lock);
1980
1981	/* Atomically find and remove tp from chain. */
1982	for (pprev = &chain->filter_chain;
1983	     (tp_iter = tcf_chain_dereference(*pprev, chain));
1984	     pprev = &tp_iter->next) {
1985		if (tp_iter == tp) {
1986			chain_info.pprev = pprev;
1987			chain_info.next = tp_iter->next;
1988			WARN_ON(tp_iter->deleting);
1989			break;
1990		}
1991	}
1992	/* Verify that tp still exists and no new filters were inserted
1993	 * concurrently.
1994	 * Mark tp for deletion if it is empty.
1995	 */
1996	if (!tp_iter || !tcf_proto_check_delete(tp)) {
1997		mutex_unlock(&chain->filter_chain_lock);
1998		return;
1999	}
2000
2001	tcf_proto_signal_destroying(chain, tp);
2002	next = tcf_chain_dereference(chain_info.next, chain);
2003	if (tp == chain->filter_chain)
2004		tcf_chain0_head_change(chain, next);
2005	RCU_INIT_POINTER(*chain_info.pprev, next);
2006	mutex_unlock(&chain->filter_chain_lock);
2007
2008	tcf_proto_put(tp, rtnl_held, extack);
2009}
2010
2011static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
2012					   struct tcf_chain_info *chain_info,
2013					   u32 protocol, u32 prio,
2014					   bool prio_allocate,
2015					   struct netlink_ext_ack *extack)
2016{
2017	struct tcf_proto **pprev;
2018	struct tcf_proto *tp;
2019
2020	/* Check the chain for existence of proto-tcf with this priority */
2021	for (pprev = &chain->filter_chain;
2022	     (tp = tcf_chain_dereference(*pprev, chain));
2023	     pprev = &tp->next) {
2024		if (tp->prio >= prio) {
2025			if (tp->prio == prio) {
2026				if (prio_allocate) {
2027					NL_SET_ERR_MSG(extack, "Lowest ID from auto-alloc range already in use");
2028					return ERR_PTR(-ENOSPC);
2029				}
2030				if (tp->protocol != protocol && protocol) {
2031					NL_SET_ERR_MSG(extack, "Protocol mismatch for filter with specified priority");
2032					return ERR_PTR(-EINVAL);
2033				}
2034			} else {
2035				tp = NULL;
2036			}
2037			break;
2038		}
2039	}
2040	chain_info->pprev = pprev;
2041	if (tp) {
2042		chain_info->next = tp->next;
2043		tcf_proto_get(tp);
2044	} else {
2045		chain_info->next = NULL;
2046	}
2047	return tp;
2048}
2049
2050static int tcf_fill_node(struct net *net, struct sk_buff *skb,
2051			 struct tcf_proto *tp, struct tcf_block *block,
2052			 struct Qdisc *q, u32 parent, void *fh,
2053			 u32 portid, u32 seq, u16 flags, int event,
2054			 bool terse_dump, bool rtnl_held,
2055			 struct netlink_ext_ack *extack)
2056{
2057	struct tcmsg *tcm;
2058	struct nlmsghdr  *nlh;
2059	unsigned char *b = skb_tail_pointer(skb);
2060	int ret = -EMSGSIZE;
2061
2062	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2063	if (!nlh)
2064		goto out_nlmsg_trim;
2065	tcm = nlmsg_data(nlh);
2066	tcm->tcm_family = AF_UNSPEC;
2067	tcm->tcm__pad1 = 0;
2068	tcm->tcm__pad2 = 0;
2069	if (q) {
2070		tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
2071		tcm->tcm_parent = parent;
2072	} else {
2073		tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2074		tcm->tcm_block_index = block->index;
2075	}
2076	tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
2077	if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
2078		goto nla_put_failure;
2079	if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
2080		goto nla_put_failure;
2081	if (!fh) {
2082		tcm->tcm_handle = 0;
2083	} else if (terse_dump) {
2084		if (tp->ops->terse_dump) {
2085			if (tp->ops->terse_dump(net, tp, fh, skb, tcm,
2086						rtnl_held) < 0)
2087				goto nla_put_failure;
2088		} else {
2089			goto cls_op_not_supp;
2090		}
2091	} else {
2092		if (tp->ops->dump &&
2093		    tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0)
2094			goto nla_put_failure;
2095	}
2096
2097	if (extack && extack->_msg &&
2098	    nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg))
2099		goto nla_put_failure;
2100
2101	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2102
2103	return skb->len;
2104
2105cls_op_not_supp:
2106	ret = -EOPNOTSUPP;
2107out_nlmsg_trim:
2108nla_put_failure:
2109	nlmsg_trim(skb, b);
2110	return ret;
2111}
2112
2113static struct sk_buff *tfilter_notify_prep(struct net *net,
2114					   struct sk_buff *oskb,
2115					   struct nlmsghdr *n,
2116					   struct tcf_proto *tp,
2117					   struct tcf_block *block,
2118					   struct Qdisc *q, u32 parent,
2119					   void *fh, int event,
2120					   u32 portid, bool rtnl_held,
2121					   struct netlink_ext_ack *extack)
2122{
2123	unsigned int size = oskb ? max(NLMSG_GOODSIZE, oskb->len) : NLMSG_GOODSIZE;
2124	struct sk_buff *skb;
2125	int ret;
2126
2127retry:
2128	skb = alloc_skb(size, GFP_KERNEL);
2129	if (!skb)
2130		return ERR_PTR(-ENOBUFS);
2131
2132	ret = tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
2133			    n->nlmsg_seq, n->nlmsg_flags, event, false,
2134			    rtnl_held, extack);
2135	if (ret <= 0) {
2136		kfree_skb(skb);
2137		if (ret == -EMSGSIZE) {
2138			size += NLMSG_GOODSIZE;
2139			goto retry;
2140		}
2141		return ERR_PTR(-EINVAL);
2142	}
2143	return skb;
2144}
2145
2146static int tfilter_notify(struct net *net, struct sk_buff *oskb,
2147			  struct nlmsghdr *n, struct tcf_proto *tp,
2148			  struct tcf_block *block, struct Qdisc *q,
2149			  u32 parent, void *fh, int event, bool unicast,
2150			  bool rtnl_held, struct netlink_ext_ack *extack)
2151{
2152	struct sk_buff *skb;
2153	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2154	int err = 0;
2155
2156	if (!unicast && !rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC))
2157		return 0;
2158
2159	skb = tfilter_notify_prep(net, oskb, n, tp, block, q, parent, fh, event,
2160				  portid, rtnl_held, extack);
2161	if (IS_ERR(skb))
2162		return PTR_ERR(skb);
2163
2164	if (unicast)
2165		err = rtnl_unicast(skb, net, portid);
2166	else
2167		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2168				     n->nlmsg_flags & NLM_F_ECHO);
2169	return err;
2170}
2171
2172static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
2173			      struct nlmsghdr *n, struct tcf_proto *tp,
2174			      struct tcf_block *block, struct Qdisc *q,
2175			      u32 parent, void *fh, bool *last, bool rtnl_held,
2176			      struct netlink_ext_ack *extack)
2177{
2178	struct sk_buff *skb;
2179	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2180	int err;
2181
2182	if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC))
2183		return tp->ops->delete(tp, fh, last, rtnl_held, extack);
2184
2185	skb = tfilter_notify_prep(net, oskb, n, tp, block, q, parent, fh,
2186				  RTM_DELTFILTER, portid, rtnl_held, extack);
2187	if (IS_ERR(skb)) {
2188		NL_SET_ERR_MSG(extack, "Failed to build del event notification");
2189		return PTR_ERR(skb);
2190	}
2191
2192	err = tp->ops->delete(tp, fh, last, rtnl_held, extack);
2193	if (err) {
2194		kfree_skb(skb);
2195		return err;
2196	}
2197
2198	err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
2199			     n->nlmsg_flags & NLM_F_ECHO);
2200	if (err < 0)
2201		NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
2202
2203	return err;
2204}
2205
2206static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
2207				 struct tcf_block *block, struct Qdisc *q,
2208				 u32 parent, struct nlmsghdr *n,
2209				 struct tcf_chain *chain, int event,
2210				 struct netlink_ext_ack *extack)
2211{
2212	struct tcf_proto *tp;
2213
2214	for (tp = tcf_get_next_proto(chain, NULL);
2215	     tp; tp = tcf_get_next_proto(chain, tp))
2216		tfilter_notify(net, oskb, n, tp, block, q, parent, NULL,
2217			       event, false, true, extack);
2218}
2219
2220static void tfilter_put(struct tcf_proto *tp, void *fh)
2221{
2222	if (tp->ops->put && fh)
2223		tp->ops->put(tp, fh);
2224}
2225
2226static bool is_qdisc_ingress(__u32 classid)
2227{
2228	return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS));
2229}
2230
2231static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q)
2232{
2233	return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS));
2234}
2235
2236static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2237			  struct netlink_ext_ack *extack)
2238{
2239	struct net *net = sock_net(skb->sk);
2240	struct nlattr *tca[TCA_MAX + 1];
2241	char name[IFNAMSIZ];
2242	struct tcmsg *t;
2243	u32 protocol;
2244	u32 prio;
2245	bool prio_allocate;
2246	u32 parent;
2247	u32 chain_index;
2248	struct Qdisc *q;
2249	struct tcf_chain_info chain_info;
2250	struct tcf_chain *chain;
2251	struct tcf_block *block;
2252	struct tcf_proto *tp;
2253	unsigned long cl;
2254	void *fh;
2255	int err;
2256	int tp_created;
2257	bool rtnl_held = false;
2258	u32 flags;
2259
2260replay:
2261	tp_created = 0;
2262
2263	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2264				     rtm_tca_policy, extack);
2265	if (err < 0)
2266		return err;
2267
2268	t = nlmsg_data(n);
2269	protocol = TC_H_MIN(t->tcm_info);
2270	prio = TC_H_MAJ(t->tcm_info);
2271	prio_allocate = false;
2272	parent = t->tcm_parent;
2273	tp = NULL;
2274	cl = 0;
2275	block = NULL;
2276	q = NULL;
2277	chain = NULL;
2278	flags = 0;
2279
2280	if (prio == 0) {
2281		/* If no priority is provided by the user,
2282		 * we allocate one.
2283		 */
2284		if (n->nlmsg_flags & NLM_F_CREATE) {
2285			prio = TC_H_MAKE(0x80000000U, 0U);
2286			prio_allocate = true;
2287		} else {
2288			NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2289			return -ENOENT;
2290		}
2291	}
2292
2293	/* Find head of filter chain. */
2294
2295	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2296	if (err)
2297		return err;
2298
2299	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2300		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2301		err = -EINVAL;
2302		goto errout;
2303	}
2304
2305	/* Take rtnl mutex if rtnl_held was set to true on previous iteration,
2306	 * block is shared (no qdisc found), qdisc is not unlocked, classifier
2307	 * type is not specified, classifier is not unlocked.
2308	 */
2309	if (rtnl_held ||
2310	    (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2311	    !tcf_proto_is_unlocked(name)) {
2312		rtnl_held = true;
2313		rtnl_lock();
2314	}
2315
2316	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2317	if (err)
2318		goto errout;
2319
2320	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2321				 extack);
2322	if (IS_ERR(block)) {
2323		err = PTR_ERR(block);
2324		goto errout;
2325	}
2326	block->classid = parent;
2327
2328	chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0);
2329	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2330		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2331		err = -EINVAL;
2332		goto errout;
2333	}
2334	chain = tcf_chain_get(block, chain_index, true);
2335	if (!chain) {
2336		NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
2337		err = -ENOMEM;
2338		goto errout;
2339	}
2340
2341	mutex_lock(&chain->filter_chain_lock);
2342	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2343			       prio, prio_allocate, extack);
2344	if (IS_ERR(tp)) {
2345		err = PTR_ERR(tp);
2346		goto errout_locked;
2347	}
2348
2349	if (tp == NULL) {
2350		struct tcf_proto *tp_new = NULL;
2351
2352		if (chain->flushing) {
2353			err = -EAGAIN;
2354			goto errout_locked;
2355		}
2356
2357		/* Proto-tcf does not exist, create new one */
2358
2359		if (tca[TCA_KIND] == NULL || !protocol) {
2360			NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
2361			err = -EINVAL;
2362			goto errout_locked;
2363		}
2364
2365		if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2366			NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2367			err = -ENOENT;
2368			goto errout_locked;
2369		}
2370
2371		if (prio_allocate)
2372			prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
2373							       &chain_info));
2374
2375		mutex_unlock(&chain->filter_chain_lock);
2376		tp_new = tcf_proto_create(name, protocol, prio, chain,
2377					  rtnl_held, extack);
2378		if (IS_ERR(tp_new)) {
2379			err = PTR_ERR(tp_new);
2380			goto errout_tp;
2381		}
2382
2383		tp_created = 1;
2384		tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio,
2385						rtnl_held);
2386		if (IS_ERR(tp)) {
2387			err = PTR_ERR(tp);
2388			goto errout_tp;
2389		}
2390	} else {
2391		mutex_unlock(&chain->filter_chain_lock);
2392	}
2393
2394	if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2395		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2396		err = -EINVAL;
2397		goto errout;
2398	}
2399
2400	fh = tp->ops->get(tp, t->tcm_handle);
2401
2402	if (!fh) {
2403		if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2404			NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
2405			err = -ENOENT;
2406			goto errout;
2407		}
2408	} else if (n->nlmsg_flags & NLM_F_EXCL) {
2409		tfilter_put(tp, fh);
2410		NL_SET_ERR_MSG(extack, "Filter already exists");
2411		err = -EEXIST;
2412		goto errout;
2413	}
2414
2415	if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
2416		tfilter_put(tp, fh);
2417		NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2418		err = -EINVAL;
2419		goto errout;
2420	}
2421
2422	if (!(n->nlmsg_flags & NLM_F_CREATE))
2423		flags |= TCA_ACT_FLAGS_REPLACE;
2424	if (!rtnl_held)
2425		flags |= TCA_ACT_FLAGS_NO_RTNL;
2426	if (is_qdisc_ingress(parent))
2427		flags |= TCA_ACT_FLAGS_AT_INGRESS;
2428	if (is_ingress_or_clsact(block, q))
2429		flags |= TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT;
2430	err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2431			      flags, extack);
2432	if (err == 0) {
2433		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2434			       RTM_NEWTFILTER, false, rtnl_held, extack);
2435		tfilter_put(tp, fh);
2436		tcf_proto_count_usesw(tp, true);
2437		/* q pointer is NULL for shared blocks */
2438		if (q)
2439			q->flags &= ~TCQ_F_CAN_BYPASS;
2440	}
2441
2442errout:
2443	if (err && tp_created)
2444		tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
2445errout_tp:
2446	if (chain) {
2447		if (tp && !IS_ERR(tp))
2448			tcf_proto_put(tp, rtnl_held, NULL);
2449		if (!tp_created)
2450			tcf_chain_put(chain);
2451	}
2452	tcf_block_release(q, block, rtnl_held);
2453
2454	if (rtnl_held)
2455		rtnl_unlock();
2456
2457	if (err == -EAGAIN) {
2458		/* Take rtnl lock in case EAGAIN is caused by concurrent flush
2459		 * of target chain.
2460		 */
2461		rtnl_held = true;
2462		/* Replay the request. */
2463		goto replay;
2464	}
2465	return err;
2466
2467errout_locked:
2468	mutex_unlock(&chain->filter_chain_lock);
2469	goto errout;
2470}
2471
2472static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2473			  struct netlink_ext_ack *extack)
2474{
2475	struct net *net = sock_net(skb->sk);
2476	struct nlattr *tca[TCA_MAX + 1];
2477	char name[IFNAMSIZ];
2478	struct tcmsg *t;
2479	u32 protocol;
2480	u32 prio;
2481	u32 parent;
2482	u32 chain_index;
2483	struct Qdisc *q = NULL;
2484	struct tcf_chain_info chain_info;
2485	struct tcf_chain *chain = NULL;
2486	struct tcf_block *block = NULL;
2487	struct tcf_proto *tp = NULL;
2488	unsigned long cl = 0;
2489	void *fh = NULL;
2490	int err;
2491	bool rtnl_held = false;
2492
2493	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2494				     rtm_tca_policy, extack);
2495	if (err < 0)
2496		return err;
2497
2498	t = nlmsg_data(n);
2499	protocol = TC_H_MIN(t->tcm_info);
2500	prio = TC_H_MAJ(t->tcm_info);
2501	parent = t->tcm_parent;
2502
2503	if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2504		NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2505		return -ENOENT;
2506	}
2507
2508	/* Find head of filter chain. */
2509
2510	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2511	if (err)
2512		return err;
2513
2514	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2515		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2516		err = -EINVAL;
2517		goto errout;
2518	}
2519	/* Take rtnl mutex if flushing whole chain, block is shared (no qdisc
2520	 * found), qdisc is not unlocked, classifier type is not specified,
2521	 * classifier is not unlocked.
2522	 */
2523	if (!prio ||
2524	    (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2525	    !tcf_proto_is_unlocked(name)) {
2526		rtnl_held = true;
2527		rtnl_lock();
2528	}
2529
2530	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2531	if (err)
2532		goto errout;
2533
2534	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2535				 extack);
2536	if (IS_ERR(block)) {
2537		err = PTR_ERR(block);
2538		goto errout;
2539	}
2540
2541	chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0);
2542	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2543		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2544		err = -EINVAL;
2545		goto errout;
2546	}
2547	chain = tcf_chain_get(block, chain_index, false);
2548	if (!chain) {
2549		/* User requested flush on non-existent chain. Nothing to do,
2550		 * so just return success.
2551		 */
2552		if (prio == 0) {
2553			err = 0;
2554			goto errout;
2555		}
2556		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2557		err = -ENOENT;
2558		goto errout;
2559	}
2560
2561	if (prio == 0) {
2562		tfilter_notify_chain(net, skb, block, q, parent, n,
2563				     chain, RTM_DELTFILTER, extack);
2564		tcf_chain_flush(chain, rtnl_held);
2565		err = 0;
2566		goto errout;
2567	}
2568
2569	mutex_lock(&chain->filter_chain_lock);
2570	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2571			       prio, false, extack);
2572	if (!tp) {
2573		err = -ENOENT;
2574		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2575		goto errout_locked;
2576	} else if (IS_ERR(tp)) {
2577		err = PTR_ERR(tp);
2578		goto errout_locked;
2579	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2580		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2581		err = -EINVAL;
2582		goto errout_locked;
2583	} else if (t->tcm_handle == 0) {
2584		tcf_proto_signal_destroying(chain, tp);
2585		tcf_chain_tp_remove(chain, &chain_info, tp);
2586		mutex_unlock(&chain->filter_chain_lock);
2587
2588		tcf_proto_put(tp, rtnl_held, NULL);
2589		tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2590			       RTM_DELTFILTER, false, rtnl_held, extack);
2591		err = 0;
2592		goto errout;
2593	}
2594	mutex_unlock(&chain->filter_chain_lock);
2595
2596	fh = tp->ops->get(tp, t->tcm_handle);
2597
2598	if (!fh) {
2599		NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2600		err = -ENOENT;
2601	} else {
2602		bool last;
2603
2604		err = tfilter_del_notify(net, skb, n, tp, block, q, parent, fh,
2605					 &last, rtnl_held, extack);
2606
2607		if (err)
2608			goto errout;
2609		if (last)
2610			tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack);
2611	}
2612
2613errout:
2614	if (chain) {
2615		if (tp && !IS_ERR(tp))
2616			tcf_proto_put(tp, rtnl_held, NULL);
2617		tcf_chain_put(chain);
2618	}
2619	tcf_block_release(q, block, rtnl_held);
2620
2621	if (rtnl_held)
2622		rtnl_unlock();
2623
2624	return err;
2625
2626errout_locked:
2627	mutex_unlock(&chain->filter_chain_lock);
2628	goto errout;
2629}
2630
2631static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2632			  struct netlink_ext_ack *extack)
2633{
2634	struct net *net = sock_net(skb->sk);
2635	struct nlattr *tca[TCA_MAX + 1];
2636	char name[IFNAMSIZ];
2637	struct tcmsg *t;
2638	u32 protocol;
2639	u32 prio;
2640	u32 parent;
2641	u32 chain_index;
2642	struct Qdisc *q = NULL;
2643	struct tcf_chain_info chain_info;
2644	struct tcf_chain *chain = NULL;
2645	struct tcf_block *block = NULL;
2646	struct tcf_proto *tp = NULL;
2647	unsigned long cl = 0;
2648	void *fh = NULL;
2649	int err;
2650	bool rtnl_held = false;
2651
2652	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
2653				     rtm_tca_policy, extack);
2654	if (err < 0)
2655		return err;
2656
2657	t = nlmsg_data(n);
2658	protocol = TC_H_MIN(t->tcm_info);
2659	prio = TC_H_MAJ(t->tcm_info);
2660	parent = t->tcm_parent;
2661
2662	if (prio == 0) {
2663		NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2664		return -ENOENT;
2665	}
2666
2667	/* Find head of filter chain. */
2668
2669	err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack);
2670	if (err)
2671		return err;
2672
2673	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
2674		NL_SET_ERR_MSG(extack, "Specified TC filter name too long");
2675		err = -EINVAL;
2676		goto errout;
2677	}
2678	/* Take rtnl mutex if block is shared (no qdisc found), qdisc is not
2679	 * unlocked, classifier type is not specified, classifier is not
2680	 * unlocked.
2681	 */
2682	if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) ||
2683	    !tcf_proto_is_unlocked(name)) {
2684		rtnl_held = true;
2685		rtnl_lock();
2686	}
2687
2688	err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack);
2689	if (err)
2690		goto errout;
2691
2692	block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index,
2693				 extack);
2694	if (IS_ERR(block)) {
2695		err = PTR_ERR(block);
2696		goto errout;
2697	}
2698
2699	chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0);
2700	if (chain_index > TC_ACT_EXT_VAL_MASK) {
2701		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2702		err = -EINVAL;
2703		goto errout;
2704	}
2705	chain = tcf_chain_get(block, chain_index, false);
2706	if (!chain) {
2707		NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2708		err = -EINVAL;
2709		goto errout;
2710	}
2711
2712	mutex_lock(&chain->filter_chain_lock);
2713	tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2714			       prio, false, extack);
2715	mutex_unlock(&chain->filter_chain_lock);
2716	if (!tp) {
2717		err = -ENOENT;
2718		NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2719		goto errout;
2720	} else if (IS_ERR(tp)) {
2721		err = PTR_ERR(tp);
2722		goto errout;
2723	} else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2724		NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2725		err = -EINVAL;
2726		goto errout;
2727	}
2728
2729	fh = tp->ops->get(tp, t->tcm_handle);
2730
2731	if (!fh) {
2732		NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2733		err = -ENOENT;
2734	} else {
2735		err = tfilter_notify(net, skb, n, tp, block, q, parent,
2736				     fh, RTM_NEWTFILTER, true, rtnl_held, NULL);
2737		if (err < 0)
2738			NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2739	}
2740
2741	tfilter_put(tp, fh);
2742errout:
2743	if (chain) {
2744		if (tp && !IS_ERR(tp))
2745			tcf_proto_put(tp, rtnl_held, NULL);
2746		tcf_chain_put(chain);
2747	}
2748	tcf_block_release(q, block, rtnl_held);
2749
2750	if (rtnl_held)
2751		rtnl_unlock();
2752
2753	return err;
2754}
2755
2756struct tcf_dump_args {
2757	struct tcf_walker w;
2758	struct sk_buff *skb;
2759	struct netlink_callback *cb;
2760	struct tcf_block *block;
2761	struct Qdisc *q;
2762	u32 parent;
2763	bool terse_dump;
2764};
2765
2766static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2767{
2768	struct tcf_dump_args *a = (void *)arg;
2769	struct net *net = sock_net(a->skb->sk);
2770
2771	return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2772			     n, NETLINK_CB(a->cb->skb).portid,
2773			     a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2774			     RTM_NEWTFILTER, a->terse_dump, true, NULL);
2775}
2776
2777static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2778			   struct sk_buff *skb, struct netlink_callback *cb,
2779			   long index_start, long *p_index, bool terse)
2780{
2781	struct net *net = sock_net(skb->sk);
2782	struct tcf_block *block = chain->block;
2783	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2784	struct tcf_proto *tp, *tp_prev;
2785	struct tcf_dump_args arg;
2786
2787	for (tp = __tcf_get_next_proto(chain, NULL);
2788	     tp;
2789	     tp_prev = tp,
2790		     tp = __tcf_get_next_proto(chain, tp),
2791		     tcf_proto_put(tp_prev, true, NULL),
2792		     (*p_index)++) {
2793		if (*p_index < index_start)
2794			continue;
2795		if (TC_H_MAJ(tcm->tcm_info) &&
2796		    TC_H_MAJ(tcm->tcm_info) != tp->prio)
2797			continue;
2798		if (TC_H_MIN(tcm->tcm_info) &&
2799		    TC_H_MIN(tcm->tcm_info) != tp->protocol)
2800			continue;
2801		if (*p_index > index_start)
2802			memset(&cb->args[1], 0,
2803			       sizeof(cb->args) - sizeof(cb->args[0]));
2804		if (cb->args[1] == 0) {
2805			if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2806					  NETLINK_CB(cb->skb).portid,
2807					  cb->nlh->nlmsg_seq, NLM_F_MULTI,
2808					  RTM_NEWTFILTER, false, true, NULL) <= 0)
2809				goto errout;
2810			cb->args[1] = 1;
2811		}
2812		if (!tp->ops->walk)
2813			continue;
2814		arg.w.fn = tcf_node_dump;
2815		arg.skb = skb;
2816		arg.cb = cb;
2817		arg.block = block;
2818		arg.q = q;
2819		arg.parent = parent;
2820		arg.w.stop = 0;
2821		arg.w.skip = cb->args[1] - 1;
2822		arg.w.count = 0;
2823		arg.w.cookie = cb->args[2];
2824		arg.terse_dump = terse;
2825		tp->ops->walk(tp, &arg.w, true);
2826		cb->args[2] = arg.w.cookie;
2827		cb->args[1] = arg.w.count + 1;
2828		if (arg.w.stop)
2829			goto errout;
2830	}
2831	return true;
2832
2833errout:
2834	tcf_proto_put(tp, true, NULL);
2835	return false;
2836}
2837
2838static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = {
2839	[TCA_CHAIN]      = { .type = NLA_U32 },
2840	[TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE),
2841};
2842
2843/* called with RTNL */
2844static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2845{
2846	struct tcf_chain *chain, *chain_prev;
2847	struct net *net = sock_net(skb->sk);
2848	struct nlattr *tca[TCA_MAX + 1];
2849	struct Qdisc *q = NULL;
2850	struct tcf_block *block;
2851	struct tcmsg *tcm = nlmsg_data(cb->nlh);
2852	bool terse_dump = false;
2853	long index_start;
2854	long index;
2855	u32 parent;
2856	int err;
2857
2858	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2859		return skb->len;
2860
2861	err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
2862				     tcf_tfilter_dump_policy, cb->extack);
2863	if (err)
2864		return err;
2865
2866	if (tca[TCA_DUMP_FLAGS]) {
2867		struct nla_bitfield32 flags =
2868			nla_get_bitfield32(tca[TCA_DUMP_FLAGS]);
2869
2870		terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE;
2871	}
2872
2873	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2874		block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2875		if (!block)
2876			goto out;
2877		/* If we work with block index, q is NULL and parent value
2878		 * will never be used in the following code. The check
2879		 * in tcf_fill_node prevents it. However, compiler does not
2880		 * see that far, so set parent to zero to silence the warning
2881		 * about parent being uninitialized.
2882		 */
2883		parent = 0;
2884	} else {
2885		const struct Qdisc_class_ops *cops;
2886		struct net_device *dev;
2887		unsigned long cl = 0;
2888
2889		dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2890		if (!dev)
2891			return skb->len;
2892
2893		parent = tcm->tcm_parent;
2894		if (!parent)
2895			q = rtnl_dereference(dev->qdisc);
2896		else
2897			q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2898		if (!q)
2899			goto out;
2900		cops = q->ops->cl_ops;
2901		if (!cops)
2902			goto out;
2903		if (!cops->tcf_block)
2904			goto out;
2905		if (TC_H_MIN(tcm->tcm_parent)) {
2906			cl = cops->find(q, tcm->tcm_parent);
2907			if (cl == 0)
2908				goto out;
2909		}
2910		block = cops->tcf_block(q, cl, NULL);
2911		if (!block)
2912			goto out;
2913		parent = block->classid;
2914		if (tcf_block_shared(block))
2915			q = NULL;
2916	}
2917
2918	index_start = cb->args[0];
2919	index = 0;
2920
2921	for (chain = __tcf_get_next_chain(block, NULL);
2922	     chain;
2923	     chain_prev = chain,
2924		     chain = __tcf_get_next_chain(block, chain),
2925		     tcf_chain_put(chain_prev)) {
2926		if (tca[TCA_CHAIN] &&
2927		    nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2928			continue;
2929		if (!tcf_chain_dump(chain, q, parent, skb, cb,
2930				    index_start, &index, terse_dump)) {
2931			tcf_chain_put(chain);
2932			err = -EMSGSIZE;
2933			break;
2934		}
2935	}
2936
2937	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2938		tcf_block_refcnt_put(block, true);
2939	cb->args[0] = index;
2940
2941out:
2942	/* If we did no progress, the error (EMSGSIZE) is real */
2943	if (skb->len == 0 && err)
2944		return err;
2945	return skb->len;
2946}
2947
2948static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2949			      void *tmplt_priv, u32 chain_index,
2950			      struct net *net, struct sk_buff *skb,
2951			      struct tcf_block *block,
2952			      u32 portid, u32 seq, u16 flags, int event,
2953			      struct netlink_ext_ack *extack)
2954{
2955	unsigned char *b = skb_tail_pointer(skb);
2956	const struct tcf_proto_ops *ops;
2957	struct nlmsghdr *nlh;
2958	struct tcmsg *tcm;
2959	void *priv;
2960
2961	ops = tmplt_ops;
2962	priv = tmplt_priv;
2963
2964	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2965	if (!nlh)
2966		goto out_nlmsg_trim;
2967	tcm = nlmsg_data(nlh);
2968	tcm->tcm_family = AF_UNSPEC;
2969	tcm->tcm__pad1 = 0;
2970	tcm->tcm__pad2 = 0;
2971	tcm->tcm_handle = 0;
2972	tcm->tcm_info = 0;
2973	if (block->q) {
2974		tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2975		tcm->tcm_parent = block->q->handle;
2976	} else {
2977		tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2978		tcm->tcm_block_index = block->index;
2979	}
2980
2981	if (nla_put_u32(skb, TCA_CHAIN, chain_index))
2982		goto nla_put_failure;
2983
2984	if (ops) {
2985		if (nla_put_string(skb, TCA_KIND, ops->kind))
2986			goto nla_put_failure;
2987		if (ops->tmplt_dump(skb, net, priv) < 0)
2988			goto nla_put_failure;
2989	}
2990
2991	if (extack && extack->_msg &&
2992	    nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg))
2993		goto out_nlmsg_trim;
2994
2995	nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2996
2997	return skb->len;
2998
2999out_nlmsg_trim:
3000nla_put_failure:
3001	nlmsg_trim(skb, b);
3002	return -EMSGSIZE;
3003}
3004
3005static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
3006			   u32 seq, u16 flags, int event, bool unicast,
3007			   struct netlink_ext_ack *extack)
3008{
3009	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
3010	struct tcf_block *block = chain->block;
3011	struct net *net = block->net;
3012	struct sk_buff *skb;
3013	int err = 0;
3014
3015	if (!unicast && !rtnl_notify_needed(net, flags, RTNLGRP_TC))
3016		return 0;
3017
3018	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
3019	if (!skb)
3020		return -ENOBUFS;
3021
3022	if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3023			       chain->index, net, skb, block, portid,
3024			       seq, flags, event, extack) <= 0) {
3025		kfree_skb(skb);
3026		return -EINVAL;
3027	}
3028
3029	if (unicast)
3030		err = rtnl_unicast(skb, net, portid);
3031	else
3032		err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
3033				     flags & NLM_F_ECHO);
3034
3035	return err;
3036}
3037
3038static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
3039				  void *tmplt_priv, u32 chain_index,
3040				  struct tcf_block *block, struct sk_buff *oskb,
3041				  u32 seq, u16 flags)
3042{
3043	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
3044	struct net *net = block->net;
3045	struct sk_buff *skb;
3046
3047	if (!rtnl_notify_needed(net, flags, RTNLGRP_TC))
3048		return 0;
3049
3050	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
3051	if (!skb)
3052		return -ENOBUFS;
3053
3054	if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
3055			       block, portid, seq, flags, RTM_DELCHAIN, NULL) <= 0) {
3056		kfree_skb(skb);
3057		return -EINVAL;
3058	}
3059
3060	return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
3061}
3062
3063static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
3064			      struct nlattr **tca,
3065			      struct netlink_ext_ack *extack)
3066{
3067	const struct tcf_proto_ops *ops;
3068	char name[IFNAMSIZ];
3069	void *tmplt_priv;
3070
3071	/* If kind is not set, user did not specify template. */
3072	if (!tca[TCA_KIND])
3073		return 0;
3074
3075	if (tcf_proto_check_kind(tca[TCA_KIND], name)) {
3076		NL_SET_ERR_MSG(extack, "Specified TC chain template name too long");
3077		return -EINVAL;
3078	}
3079
3080	ops = tcf_proto_lookup_ops(name, true, extack);
3081	if (IS_ERR(ops))
3082		return PTR_ERR(ops);
3083	if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump ||
3084	    !ops->tmplt_reoffload) {
3085		NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
3086		module_put(ops->owner);
3087		return -EOPNOTSUPP;
3088	}
3089
3090	tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
3091	if (IS_ERR(tmplt_priv)) {
3092		module_put(ops->owner);
3093		return PTR_ERR(tmplt_priv);
3094	}
3095	chain->tmplt_ops = ops;
3096	chain->tmplt_priv = tmplt_priv;
3097	return 0;
3098}
3099
3100static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
3101			       void *tmplt_priv)
3102{
3103	/* If template ops are set, no work to do for us. */
3104	if (!tmplt_ops)
3105		return;
3106
3107	tmplt_ops->tmplt_destroy(tmplt_priv);
3108	module_put(tmplt_ops->owner);
3109}
3110
3111/* Add/delete/get a chain */
3112
3113static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
3114			struct netlink_ext_ack *extack)
3115{
3116	struct net *net = sock_net(skb->sk);
3117	struct nlattr *tca[TCA_MAX + 1];
3118	struct tcmsg *t;
3119	u32 parent;
3120	u32 chain_index;
3121	struct Qdisc *q;
3122	struct tcf_chain *chain;
3123	struct tcf_block *block;
3124	unsigned long cl;
3125	int err;
3126
3127replay:
3128	q = NULL;
3129	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
3130				     rtm_tca_policy, extack);
3131	if (err < 0)
3132		return err;
3133
3134	t = nlmsg_data(n);
3135	parent = t->tcm_parent;
3136	cl = 0;
3137
3138	block = tcf_block_find(net, &q, &parent, &cl,
3139			       t->tcm_ifindex, t->tcm_block_index, extack);
3140	if (IS_ERR(block))
3141		return PTR_ERR(block);
3142
3143	chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0);
3144	if (chain_index > TC_ACT_EXT_VAL_MASK) {
3145		NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
3146		err = -EINVAL;
3147		goto errout_block;
3148	}
3149
3150	mutex_lock(&block->lock);
3151	chain = tcf_chain_lookup(block, chain_index);
3152	if (n->nlmsg_type == RTM_NEWCHAIN) {
3153		if (chain) {
3154			if (tcf_chain_held_by_acts_only(chain)) {
3155				/* The chain exists only because there is
3156				 * some action referencing it.
3157				 */
3158				tcf_chain_hold(chain);
3159			} else {
3160				NL_SET_ERR_MSG(extack, "Filter chain already exists");
3161				err = -EEXIST;
3162				goto errout_block_locked;
3163			}
3164		} else {
3165			if (!(n->nlmsg_flags & NLM_F_CREATE)) {
3166				NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
3167				err = -ENOENT;
3168				goto errout_block_locked;
3169			}
3170			chain = tcf_chain_create(block, chain_index);
3171			if (!chain) {
3172				NL_SET_ERR_MSG(extack, "Failed to create filter chain");
3173				err = -ENOMEM;
3174				goto errout_block_locked;
3175			}
3176		}
3177	} else {
3178		if (!chain || tcf_chain_held_by_acts_only(chain)) {
3179			NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
3180			err = -EINVAL;
3181			goto errout_block_locked;
3182		}
3183		tcf_chain_hold(chain);
3184	}
3185
3186	if (n->nlmsg_type == RTM_NEWCHAIN) {
3187		/* Modifying chain requires holding parent block lock. In case
3188		 * the chain was successfully added, take a reference to the
3189		 * chain. This ensures that an empty chain does not disappear at
3190		 * the end of this function.
3191		 */
3192		tcf_chain_hold(chain);
3193		chain->explicitly_created = true;
3194	}
3195	mutex_unlock(&block->lock);
3196
3197	switch (n->nlmsg_type) {
3198	case RTM_NEWCHAIN:
3199		err = tc_chain_tmplt_add(chain, net, tca, extack);
3200		if (err) {
3201			tcf_chain_put_explicitly_created(chain);
3202			goto errout;
3203		}
3204
3205		tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
3206				RTM_NEWCHAIN, false, extack);
3207		break;
3208	case RTM_DELCHAIN:
3209		tfilter_notify_chain(net, skb, block, q, parent, n,
3210				     chain, RTM_DELTFILTER, extack);
3211		/* Flush the chain first as the user requested chain removal. */
3212		tcf_chain_flush(chain, true);
3213		/* In case the chain was successfully deleted, put a reference
3214		 * to the chain previously taken during addition.
3215		 */
3216		tcf_chain_put_explicitly_created(chain);
3217		break;
3218	case RTM_GETCHAIN:
3219		err = tc_chain_notify(chain, skb, n->nlmsg_seq,
3220				      n->nlmsg_flags, n->nlmsg_type, true, extack);
3221		if (err < 0)
3222			NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
3223		break;
3224	default:
3225		err = -EOPNOTSUPP;
3226		NL_SET_ERR_MSG(extack, "Unsupported message type");
3227		goto errout;
3228	}
3229
3230errout:
3231	tcf_chain_put(chain);
3232errout_block:
3233	tcf_block_release(q, block, true);
3234	if (err == -EAGAIN)
3235		/* Replay the request. */
3236		goto replay;
3237	return err;
3238
3239errout_block_locked:
3240	mutex_unlock(&block->lock);
3241	goto errout_block;
3242}
3243
3244/* called with RTNL */
3245static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
3246{
3247	struct net *net = sock_net(skb->sk);
3248	struct nlattr *tca[TCA_MAX + 1];
3249	struct Qdisc *q = NULL;
3250	struct tcf_block *block;
3251	struct tcmsg *tcm = nlmsg_data(cb->nlh);
3252	struct tcf_chain *chain;
3253	long index_start;
3254	long index;
3255	int err;
3256
3257	if (nlmsg_len(cb->nlh) < sizeof(*tcm))
3258		return skb->len;
3259
3260	err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX,
3261				     rtm_tca_policy, cb->extack);
3262	if (err)
3263		return err;
3264
3265	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
3266		block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
3267		if (!block)
3268			goto out;
3269	} else {
3270		const struct Qdisc_class_ops *cops;
3271		struct net_device *dev;
3272		unsigned long cl = 0;
3273
3274		dev = __dev_get_by_index(net, tcm->tcm_ifindex);
3275		if (!dev)
3276			return skb->len;
3277
3278		if (!tcm->tcm_parent)
3279			q = rtnl_dereference(dev->qdisc);
3280		else
3281			q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
3282
3283		if (!q)
3284			goto out;
3285		cops = q->ops->cl_ops;
3286		if (!cops)
3287			goto out;
3288		if (!cops->tcf_block)
3289			goto out;
3290		if (TC_H_MIN(tcm->tcm_parent)) {
3291			cl = cops->find(q, tcm->tcm_parent);
3292			if (cl == 0)
3293				goto out;
3294		}
3295		block = cops->tcf_block(q, cl, NULL);
3296		if (!block)
3297			goto out;
3298		if (tcf_block_shared(block))
3299			q = NULL;
3300	}
3301
3302	index_start = cb->args[0];
3303	index = 0;
3304
3305	mutex_lock(&block->lock);
3306	list_for_each_entry(chain, &block->chain_list, list) {
3307		if ((tca[TCA_CHAIN] &&
3308		     nla_get_u32(tca[TCA_CHAIN]) != chain->index))
3309			continue;
3310		if (index < index_start) {
3311			index++;
3312			continue;
3313		}
3314		if (tcf_chain_held_by_acts_only(chain))
3315			continue;
3316		err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
3317					 chain->index, net, skb, block,
3318					 NETLINK_CB(cb->skb).portid,
3319					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
3320					 RTM_NEWCHAIN, NULL);
3321		if (err <= 0)
3322			break;
3323		index++;
3324	}
3325	mutex_unlock(&block->lock);
3326
3327	if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
3328		tcf_block_refcnt_put(block, true);
3329	cb->args[0] = index;
3330
3331out:
3332	/* If we did no progress, the error (EMSGSIZE) is real */
3333	if (skb->len == 0 && err)
3334		return err;
3335	return skb->len;
3336}
3337
3338int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action,
3339		     int police, struct tcf_proto *tp, u32 handle,
3340		     bool use_action_miss)
3341{
3342	int err = 0;
3343
3344#ifdef CONFIG_NET_CLS_ACT
3345	exts->type = 0;
3346	exts->nr_actions = 0;
3347	exts->miss_cookie_node = NULL;
3348	/* Note: we do not own yet a reference on net.
3349	 * This reference might be taken later from tcf_exts_get_net().
3350	 */
3351	exts->net = net;
3352	exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *),
3353				GFP_KERNEL);
3354	if (!exts->actions)
3355		return -ENOMEM;
3356#endif
3357
3358	exts->action = action;
3359	exts->police = police;
3360
3361	if (!use_action_miss)
3362		return 0;
3363
3364	err = tcf_exts_miss_cookie_base_alloc(exts, tp, handle);
3365	if (err)
3366		goto err_miss_alloc;
3367
3368	return 0;
3369
3370err_miss_alloc:
3371	tcf_exts_destroy(exts);
3372#ifdef CONFIG_NET_CLS_ACT
3373	exts->actions = NULL;
3374#endif
3375	return err;
3376}
3377EXPORT_SYMBOL(tcf_exts_init_ex);
3378
3379void tcf_exts_destroy(struct tcf_exts *exts)
3380{
3381	tcf_exts_miss_cookie_base_destroy(exts);
3382
3383#ifdef CONFIG_NET_CLS_ACT
3384	if (exts->actions) {
3385		tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
3386		kfree(exts->actions);
3387	}
3388	exts->nr_actions = 0;
3389#endif
3390}
3391EXPORT_SYMBOL(tcf_exts_destroy);
3392
3393int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3394			 struct nlattr *rate_tlv, struct tcf_exts *exts,
3395			 u32 flags, u32 fl_flags, struct netlink_ext_ack *extack)
3396{
3397#ifdef CONFIG_NET_CLS_ACT
3398	{
3399		int init_res[TCA_ACT_MAX_PRIO] = {};
3400		struct tc_action *act;
3401		size_t attr_size = 0;
3402
3403		if (exts->police && tb[exts->police]) {
3404			struct tc_action_ops *a_o;
3405
3406			flags |= TCA_ACT_FLAGS_POLICE | TCA_ACT_FLAGS_BIND;
3407			a_o = tc_action_load_ops(tb[exts->police], flags,
3408						 extack);
3409			if (IS_ERR(a_o))
3410				return PTR_ERR(a_o);
3411			act = tcf_action_init_1(net, tp, tb[exts->police],
3412						rate_tlv, a_o, init_res, flags,
3413						extack);
3414			module_put(a_o->owner);
3415			if (IS_ERR(act))
3416				return PTR_ERR(act);
3417
3418			act->type = exts->type = TCA_OLD_COMPAT;
3419			exts->actions[0] = act;
3420			exts->nr_actions = 1;
3421			tcf_idr_insert_many(exts->actions, init_res);
3422		} else if (exts->action && tb[exts->action]) {
3423			int err;
3424
3425			flags |= TCA_ACT_FLAGS_BIND;
3426			err = tcf_action_init(net, tp, tb[exts->action],
3427					      rate_tlv, exts->actions, init_res,
3428					      &attr_size, flags, fl_flags,
3429					      extack);
3430			if (err < 0)
3431				return err;
3432			exts->nr_actions = err;
3433		}
3434	}
3435#else
3436	if ((exts->action && tb[exts->action]) ||
3437	    (exts->police && tb[exts->police])) {
3438		NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
3439		return -EOPNOTSUPP;
3440	}
3441#endif
3442
3443	return 0;
3444}
3445EXPORT_SYMBOL(tcf_exts_validate_ex);
3446
3447int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
3448		      struct nlattr *rate_tlv, struct tcf_exts *exts,
3449		      u32 flags, struct netlink_ext_ack *extack)
3450{
3451	return tcf_exts_validate_ex(net, tp, tb, rate_tlv, exts,
3452				    flags, 0, extack);
3453}
3454EXPORT_SYMBOL(tcf_exts_validate);
3455
3456void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
3457{
3458#ifdef CONFIG_NET_CLS_ACT
3459	struct tcf_exts old = *dst;
3460
3461	*dst = *src;
3462	tcf_exts_destroy(&old);
3463#endif
3464}
3465EXPORT_SYMBOL(tcf_exts_change);
3466
3467#ifdef CONFIG_NET_CLS_ACT
3468static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
3469{
3470	if (exts->nr_actions == 0)
3471		return NULL;
3472	else
3473		return exts->actions[0];
3474}
3475#endif
3476
3477int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
3478{
3479#ifdef CONFIG_NET_CLS_ACT
3480	struct nlattr *nest;
3481
3482	if (exts->action && tcf_exts_has_actions(exts)) {
3483		/*
3484		 * again for backward compatible mode - we want
3485		 * to work with both old and new modes of entering
3486		 * tc data even if iproute2  was newer - jhs
3487		 */
3488		if (exts->type != TCA_OLD_COMPAT) {
3489			nest = nla_nest_start_noflag(skb, exts->action);
3490			if (nest == NULL)
3491				goto nla_put_failure;
3492
3493			if (tcf_action_dump(skb, exts->actions, 0, 0, false)
3494			    < 0)
3495				goto nla_put_failure;
3496			nla_nest_end(skb, nest);
3497		} else if (exts->police) {
3498			struct tc_action *act = tcf_exts_first_act(exts);
3499			nest = nla_nest_start_noflag(skb, exts->police);
3500			if (nest == NULL || !act)
3501				goto nla_put_failure;
3502			if (tcf_action_dump_old(skb, act, 0, 0) < 0)
3503				goto nla_put_failure;
3504			nla_nest_end(skb, nest);
3505		}
3506	}
3507	return 0;
3508
3509nla_put_failure:
3510	nla_nest_cancel(skb, nest);
3511	return -1;
3512#else
3513	return 0;
3514#endif
3515}
3516EXPORT_SYMBOL(tcf_exts_dump);
3517
3518int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts)
3519{
3520#ifdef CONFIG_NET_CLS_ACT
3521	struct nlattr *nest;
3522
3523	if (!exts->action || !tcf_exts_has_actions(exts))
3524		return 0;
3525
3526	nest = nla_nest_start_noflag(skb, exts->action);
3527	if (!nest)
3528		goto nla_put_failure;
3529
3530	if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0)
3531		goto nla_put_failure;
3532	nla_nest_end(skb, nest);
3533	return 0;
3534
3535nla_put_failure:
3536	nla_nest_cancel(skb, nest);
3537	return -1;
3538#else
3539	return 0;
3540#endif
3541}
3542EXPORT_SYMBOL(tcf_exts_terse_dump);
3543
3544int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
3545{
3546#ifdef CONFIG_NET_CLS_ACT
3547	struct tc_action *a = tcf_exts_first_act(exts);
3548	if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
3549		return -1;
3550#endif
3551	return 0;
3552}
3553EXPORT_SYMBOL(tcf_exts_dump_stats);
3554
3555static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags)
3556{
3557	if (*flags & TCA_CLS_FLAGS_IN_HW)
3558		return;
3559	*flags |= TCA_CLS_FLAGS_IN_HW;
3560	atomic_inc(&block->offloadcnt);
3561}
3562
3563static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags)
3564{
3565	if (!(*flags & TCA_CLS_FLAGS_IN_HW))
3566		return;
3567	*flags &= ~TCA_CLS_FLAGS_IN_HW;
3568	atomic_dec(&block->offloadcnt);
3569}
3570
3571static void tc_cls_offload_cnt_update(struct tcf_block *block,
3572				      struct tcf_proto *tp, u32 *cnt,
3573				      u32 *flags, u32 diff, bool add)
3574{
3575	lockdep_assert_held(&block->cb_lock);
3576
3577	spin_lock(&tp->lock);
3578	if (add) {
3579		if (!*cnt)
3580			tcf_block_offload_inc(block, flags);
3581		*cnt += diff;
3582	} else {
3583		*cnt -= diff;
3584		if (!*cnt)
3585			tcf_block_offload_dec(block, flags);
3586	}
3587	spin_unlock(&tp->lock);
3588}
3589
3590static void
3591tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp,
3592			 u32 *cnt, u32 *flags)
3593{
3594	lockdep_assert_held(&block->cb_lock);
3595
3596	spin_lock(&tp->lock);
3597	tcf_block_offload_dec(block, flags);
3598	*cnt = 0;
3599	spin_unlock(&tp->lock);
3600}
3601
3602static int
3603__tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3604		   void *type_data, bool err_stop)
3605{
3606	struct flow_block_cb *block_cb;
3607	int ok_count = 0;
3608	int err;
3609
3610	list_for_each_entry(block_cb, &block->flow_block.cb_list, list) {
3611		err = block_cb->cb(type, type_data, block_cb->cb_priv);
3612		if (err) {
3613			if (err_stop)
3614				return err;
3615		} else {
3616			ok_count++;
3617		}
3618	}
3619	return ok_count;
3620}
3621
3622int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
3623		     void *type_data, bool err_stop, bool rtnl_held)
3624{
3625	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3626	int ok_count;
3627
3628retry:
3629	if (take_rtnl)
3630		rtnl_lock();
3631	down_read(&block->cb_lock);
3632	/* Need to obtain rtnl lock if block is bound to devs that require it.
3633	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3634	 * obtain the locks in same order here.
3635	 */
3636	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3637		up_read(&block->cb_lock);
3638		take_rtnl = true;
3639		goto retry;
3640	}
3641
3642	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3643
3644	up_read(&block->cb_lock);
3645	if (take_rtnl)
3646		rtnl_unlock();
3647	return ok_count;
3648}
3649EXPORT_SYMBOL(tc_setup_cb_call);
3650
3651/* Non-destructive filter add. If filter that wasn't already in hardware is
3652 * successfully offloaded, increment block offloads counter. On failure,
3653 * previously offloaded filter is considered to be intact and offloads counter
3654 * is not decremented.
3655 */
3656
3657int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp,
3658		    enum tc_setup_type type, void *type_data, bool err_stop,
3659		    u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3660{
3661	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3662	int ok_count;
3663
3664retry:
3665	if (take_rtnl)
3666		rtnl_lock();
3667	down_read(&block->cb_lock);
3668	/* Need to obtain rtnl lock if block is bound to devs that require it.
3669	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3670	 * obtain the locks in same order here.
3671	 */
3672	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3673		up_read(&block->cb_lock);
3674		take_rtnl = true;
3675		goto retry;
3676	}
3677
3678	/* Make sure all netdevs sharing this block are offload-capable. */
3679	if (block->nooffloaddevcnt && err_stop) {
3680		ok_count = -EOPNOTSUPP;
3681		goto err_unlock;
3682	}
3683
3684	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3685	if (ok_count < 0)
3686		goto err_unlock;
3687
3688	if (tp->ops->hw_add)
3689		tp->ops->hw_add(tp, type_data);
3690	if (ok_count > 0)
3691		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags,
3692					  ok_count, true);
3693err_unlock:
3694	up_read(&block->cb_lock);
3695	if (take_rtnl)
3696		rtnl_unlock();
3697	return min(ok_count, 0);
3698}
3699EXPORT_SYMBOL(tc_setup_cb_add);
3700
3701/* Destructive filter replace. If filter that wasn't already in hardware is
3702 * successfully offloaded, increment block offload counter. On failure,
3703 * previously offloaded filter is considered to be destroyed and offload counter
3704 * is decremented.
3705 */
3706
3707int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp,
3708			enum tc_setup_type type, void *type_data, bool err_stop,
3709			u32 *old_flags, unsigned int *old_in_hw_count,
3710			u32 *new_flags, unsigned int *new_in_hw_count,
3711			bool rtnl_held)
3712{
3713	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3714	int ok_count;
3715
3716retry:
3717	if (take_rtnl)
3718		rtnl_lock();
3719	down_read(&block->cb_lock);
3720	/* Need to obtain rtnl lock if block is bound to devs that require it.
3721	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3722	 * obtain the locks in same order here.
3723	 */
3724	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3725		up_read(&block->cb_lock);
3726		take_rtnl = true;
3727		goto retry;
3728	}
3729
3730	/* Make sure all netdevs sharing this block are offload-capable. */
3731	if (block->nooffloaddevcnt && err_stop) {
3732		ok_count = -EOPNOTSUPP;
3733		goto err_unlock;
3734	}
3735
3736	tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags);
3737	if (tp->ops->hw_del)
3738		tp->ops->hw_del(tp, type_data);
3739
3740	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3741	if (ok_count < 0)
3742		goto err_unlock;
3743
3744	if (tp->ops->hw_add)
3745		tp->ops->hw_add(tp, type_data);
3746	if (ok_count > 0)
3747		tc_cls_offload_cnt_update(block, tp, new_in_hw_count,
3748					  new_flags, ok_count, true);
3749err_unlock:
3750	up_read(&block->cb_lock);
3751	if (take_rtnl)
3752		rtnl_unlock();
3753	return min(ok_count, 0);
3754}
3755EXPORT_SYMBOL(tc_setup_cb_replace);
3756
3757/* Destroy filter and decrement block offload counter, if filter was previously
3758 * offloaded.
3759 */
3760
3761int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp,
3762			enum tc_setup_type type, void *type_data, bool err_stop,
3763			u32 *flags, unsigned int *in_hw_count, bool rtnl_held)
3764{
3765	bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held;
3766	int ok_count;
3767
3768retry:
3769	if (take_rtnl)
3770		rtnl_lock();
3771	down_read(&block->cb_lock);
3772	/* Need to obtain rtnl lock if block is bound to devs that require it.
3773	 * In block bind code cb_lock is obtained while holding rtnl, so we must
3774	 * obtain the locks in same order here.
3775	 */
3776	if (!rtnl_held && !take_rtnl && block->lockeddevcnt) {
3777		up_read(&block->cb_lock);
3778		take_rtnl = true;
3779		goto retry;
3780	}
3781
3782	ok_count = __tc_setup_cb_call(block, type, type_data, err_stop);
3783
3784	tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags);
3785	if (tp->ops->hw_del)
3786		tp->ops->hw_del(tp, type_data);
3787
3788	up_read(&block->cb_lock);
3789	if (take_rtnl)
3790		rtnl_unlock();
3791	return min(ok_count, 0);
3792}
3793EXPORT_SYMBOL(tc_setup_cb_destroy);
3794
3795int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp,
3796			  bool add, flow_setup_cb_t *cb,
3797			  enum tc_setup_type type, void *type_data,
3798			  void *cb_priv, u32 *flags, unsigned int *in_hw_count)
3799{
3800	int err = cb(type, type_data, cb_priv);
3801
3802	if (err) {
3803		if (add && tc_skip_sw(*flags))
3804			return err;
3805	} else {
3806		tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1,
3807					  add);
3808	}
3809
3810	return 0;
3811}
3812EXPORT_SYMBOL(tc_setup_cb_reoffload);
3813
3814static int tcf_act_get_user_cookie(struct flow_action_entry *entry,
3815				   const struct tc_action *act)
3816{
3817	struct tc_cookie *user_cookie;
3818	int err = 0;
3819
3820	rcu_read_lock();
3821	user_cookie = rcu_dereference(act->user_cookie);
3822	if (user_cookie) {
3823		entry->user_cookie = flow_action_cookie_create(user_cookie->data,
3824							       user_cookie->len,
3825							       GFP_ATOMIC);
3826		if (!entry->user_cookie)
3827			err = -ENOMEM;
3828	}
3829	rcu_read_unlock();
3830	return err;
3831}
3832
3833static void tcf_act_put_user_cookie(struct flow_action_entry *entry)
3834{
3835	flow_action_cookie_destroy(entry->user_cookie);
3836}
3837
3838void tc_cleanup_offload_action(struct flow_action *flow_action)
3839{
3840	struct flow_action_entry *entry;
3841	int i;
3842
3843	flow_action_for_each(i, entry, flow_action) {
3844		tcf_act_put_user_cookie(entry);
3845		if (entry->destructor)
3846			entry->destructor(entry->destructor_priv);
3847	}
3848}
3849EXPORT_SYMBOL(tc_cleanup_offload_action);
3850
3851static int tc_setup_offload_act(struct tc_action *act,
3852				struct flow_action_entry *entry,
3853				u32 *index_inc,
3854				struct netlink_ext_ack *extack)
3855{
3856#ifdef CONFIG_NET_CLS_ACT
3857	if (act->ops->offload_act_setup) {
3858		return act->ops->offload_act_setup(act, entry, index_inc, true,
3859						   extack);
3860	} else {
3861		NL_SET_ERR_MSG(extack, "Action does not support offload");
3862		return -EOPNOTSUPP;
3863	}
3864#else
3865	return 0;
3866#endif
3867}
3868
3869int tc_setup_action(struct flow_action *flow_action,
3870		    struct tc_action *actions[],
3871		    u32 miss_cookie_base,
3872		    struct netlink_ext_ack *extack)
3873{
3874	int i, j, k, index, err = 0;
3875	struct tc_action *act;
3876
3877	BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY);
3878	BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE);
3879	BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED);
3880
3881	if (!actions)
3882		return 0;
3883
3884	j = 0;
3885	tcf_act_for_each_action(i, act, actions) {
3886		struct flow_action_entry *entry;
3887
3888		entry = &flow_action->entries[j];
3889		spin_lock_bh(&act->tcfa_lock);
3890		err = tcf_act_get_user_cookie(entry, act);
3891		if (err)
3892			goto err_out_locked;
3893
3894		index = 0;
3895		err = tc_setup_offload_act(act, entry, &index, extack);
3896		if (err)
3897			goto err_out_locked;
3898
3899		for (k = 0; k < index ; k++) {
3900			entry[k].hw_stats = tc_act_hw_stats(act->hw_stats);
3901			entry[k].hw_index = act->tcfa_index;
3902			entry[k].cookie = (unsigned long)act;
3903			entry[k].miss_cookie =
3904				tcf_exts_miss_cookie_get(miss_cookie_base, i);
3905		}
3906
3907		j += index;
3908
3909		spin_unlock_bh(&act->tcfa_lock);
3910	}
3911
3912err_out:
3913	if (err)
3914		tc_cleanup_offload_action(flow_action);
3915
3916	return err;
3917err_out_locked:
3918	spin_unlock_bh(&act->tcfa_lock);
3919	goto err_out;
3920}
3921
3922int tc_setup_offload_action(struct flow_action *flow_action,
3923			    const struct tcf_exts *exts,
3924			    struct netlink_ext_ack *extack)
3925{
3926#ifdef CONFIG_NET_CLS_ACT
3927	u32 miss_cookie_base;
3928
3929	if (!exts)
3930		return 0;
3931
3932	miss_cookie_base = exts->miss_cookie_node ?
3933			   exts->miss_cookie_node->miss_cookie_base : 0;
3934	return tc_setup_action(flow_action, exts->actions, miss_cookie_base,
3935			       extack);
3936#else
3937	return 0;
3938#endif
3939}
3940EXPORT_SYMBOL(tc_setup_offload_action);
3941
3942unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3943{
3944	unsigned int num_acts = 0;
3945	struct tc_action *act;
3946	int i;
3947
3948	tcf_exts_for_each_action(i, act, exts) {
3949		if (is_tcf_pedit(act))
3950			num_acts += tcf_pedit_nkeys(act);
3951		else
3952			num_acts++;
3953	}
3954	return num_acts;
3955}
3956EXPORT_SYMBOL(tcf_exts_num_actions);
3957
3958#ifdef CONFIG_NET_CLS_ACT
3959static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
3960					u32 *p_block_index,
3961					struct netlink_ext_ack *extack)
3962{
3963	*p_block_index = nla_get_u32(block_index_attr);
3964	if (!*p_block_index) {
3965		NL_SET_ERR_MSG(extack, "Block number may not be zero");
3966		return -EINVAL;
3967	}
3968
3969	return 0;
3970}
3971
3972int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
3973		    enum flow_block_binder_type binder_type,
3974		    struct nlattr *block_index_attr,
3975		    struct netlink_ext_ack *extack)
3976{
3977	u32 block_index;
3978	int err;
3979
3980	if (!block_index_attr)
3981		return 0;
3982
3983	err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
3984	if (err)
3985		return err;
3986
3987	qe->info.binder_type = binder_type;
3988	qe->info.chain_head_change = tcf_chain_head_change_dflt;
3989	qe->info.chain_head_change_priv = &qe->filter_chain;
3990	qe->info.block_index = block_index;
3991
3992	return tcf_block_get_ext(&qe->block, sch, &qe->info, extack);
3993}
3994EXPORT_SYMBOL(tcf_qevent_init);
3995
3996void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch)
3997{
3998	if (qe->info.block_index)
3999		tcf_block_put_ext(qe->block, sch, &qe->info);
4000}
4001EXPORT_SYMBOL(tcf_qevent_destroy);
4002
4003int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr,
4004			       struct netlink_ext_ack *extack)
4005{
4006	u32 block_index;
4007	int err;
4008
4009	if (!block_index_attr)
4010		return 0;
4011
4012	err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack);
4013	if (err)
4014		return err;
4015
4016	/* Bounce newly-configured block or change in block. */
4017	if (block_index != qe->info.block_index) {
4018		NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
4019		return -EINVAL;
4020	}
4021
4022	return 0;
4023}
4024EXPORT_SYMBOL(tcf_qevent_validate_change);
4025
4026struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb,
4027				  struct sk_buff **to_free, int *ret)
4028{
4029	struct tcf_result cl_res;
4030	struct tcf_proto *fl;
4031
4032	if (!qe->info.block_index)
4033		return skb;
4034
4035	fl = rcu_dereference_bh(qe->filter_chain);
4036
4037	switch (tcf_classify(skb, NULL, fl, &cl_res, false)) {
4038	case TC_ACT_SHOT:
4039		qdisc_qstats_drop(sch);
4040		__qdisc_drop(skb, to_free);
4041		*ret = __NET_XMIT_BYPASS;
4042		return NULL;
4043	case TC_ACT_STOLEN:
4044	case TC_ACT_QUEUED:
4045	case TC_ACT_TRAP:
4046		__qdisc_drop(skb, to_free);
4047		*ret = __NET_XMIT_STOLEN;
4048		return NULL;
4049	case TC_ACT_REDIRECT:
4050		skb_do_redirect(skb);
4051		*ret = __NET_XMIT_STOLEN;
4052		return NULL;
4053	}
4054
4055	return skb;
4056}
4057EXPORT_SYMBOL(tcf_qevent_handle);
4058
4059int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe)
4060{
4061	if (!qe->info.block_index)
4062		return 0;
4063	return nla_put_u32(skb, attr_name, qe->info.block_index);
4064}
4065EXPORT_SYMBOL(tcf_qevent_dump);
4066#endif
4067
4068static __net_init int tcf_net_init(struct net *net)
4069{
4070	struct tcf_net *tn = net_generic(net, tcf_net_id);
4071
4072	spin_lock_init(&tn->idr_lock);
4073	idr_init(&tn->idr);
4074	return 0;
4075}
4076
4077static void __net_exit tcf_net_exit(struct net *net)
4078{
4079	struct tcf_net *tn = net_generic(net, tcf_net_id);
4080
4081	idr_destroy(&tn->idr);
4082}
4083
4084static struct pernet_operations tcf_net_ops = {
4085	.init = tcf_net_init,
4086	.exit = tcf_net_exit,
4087	.id   = &tcf_net_id,
4088	.size = sizeof(struct tcf_net),
4089};
4090
4091static const struct rtnl_msg_handler tc_filter_rtnl_msg_handlers[] __initconst = {
4092	{.msgtype = RTM_NEWTFILTER, .doit = tc_new_tfilter,
4093	 .flags = RTNL_FLAG_DOIT_UNLOCKED},
4094	{.msgtype = RTM_DELTFILTER, .doit = tc_del_tfilter,
4095	 .flags = RTNL_FLAG_DOIT_UNLOCKED},
4096	{.msgtype = RTM_GETTFILTER, .doit = tc_get_tfilter,
4097	 .dumpit = tc_dump_tfilter, .flags = RTNL_FLAG_DOIT_UNLOCKED},
4098	{.msgtype = RTM_NEWCHAIN, .doit = tc_ctl_chain},
4099	{.msgtype = RTM_DELCHAIN, .doit = tc_ctl_chain},
4100	{.msgtype = RTM_GETCHAIN, .doit = tc_ctl_chain,
4101	 .dumpit = tc_dump_chain},
4102};
4103
4104static int __init tc_filter_init(void)
4105{
4106	int err;
4107
4108	tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
4109	if (!tc_filter_wq)
4110		return -ENOMEM;
4111
4112	err = register_pernet_subsys(&tcf_net_ops);
4113	if (err)
4114		goto err_register_pernet_subsys;
4115
4116	xa_init_flags(&tcf_exts_miss_cookies_xa, XA_FLAGS_ALLOC1);
4117	rtnl_register_many(tc_filter_rtnl_msg_handlers);
4118
4119	return 0;
4120
4121err_register_pernet_subsys:
4122	destroy_workqueue(tc_filter_wq);
4123	return err;
4124}
4125
4126subsys_initcall(tc_filter_init);