sch_generic.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef __NET_SCHED_GENERIC_H
   3#define __NET_SCHED_GENERIC_H
   4
   5#include <linux/netdevice.h>
   6#include <linux/types.h>
   7#include <linux/rcupdate.h>
   8#include <linux/pkt_sched.h>
   9#include <linux/pkt_cls.h>
  10#include <linux/percpu.h>
  11#include <linux/dynamic_queue_limits.h>
  12#include <linux/list.h>
  13#include <linux/refcount.h>
  14#include <linux/workqueue.h>
  15#include <linux/mutex.h>
  16#include <linux/rwsem.h>
  17#include <linux/atomic.h>
  18#include <linux/hashtable.h>
  19#include <net/gen_stats.h>
  20#include <net/rtnetlink.h>
  21#include <net/flow_offload.h>
  22#include <linux/xarray.h>
  23
  24struct Qdisc_ops;
  25struct qdisc_walker;
  26struct tcf_walker;
  27struct module;
  28struct bpf_flow_keys;
  29
  30struct qdisc_rate_table {
  31	struct tc_ratespec rate;
  32	u32		data[256];
  33	struct qdisc_rate_table *next;
  34	int		refcnt;
  35};
  36
  37enum qdisc_state_t {
  38	__QDISC_STATE_SCHED,
  39	__QDISC_STATE_DEACTIVATED,
  40	__QDISC_STATE_MISSED,
  41	__QDISC_STATE_DRAINING,
  42};
  43
  44#define QDISC_STATE_MISSED	BIT(__QDISC_STATE_MISSED)
  45#define QDISC_STATE_DRAINING	BIT(__QDISC_STATE_DRAINING)
  46
  47#define QDISC_STATE_NON_EMPTY	(QDISC_STATE_MISSED | \
  48					QDISC_STATE_DRAINING)
  49
  50struct qdisc_size_table {
  51	struct rcu_head		rcu;
  52	struct list_head	list;
  53	struct tc_sizespec	szopts;
  54	int			refcnt;
  55	u16			data[];
  56};
  57
  58/* similar to sk_buff_head, but skb->prev pointer is undefined. */
  59struct qdisc_skb_head {
  60	struct sk_buff	*head;
  61	struct sk_buff	*tail;
  62	__u32		qlen;
  63	spinlock_t	lock;
  64};
  65
  66struct Qdisc {
  67	int 			(*enqueue)(struct sk_buff *skb,
  68					   struct Qdisc *sch,
  69					   struct sk_buff **to_free);
  70	struct sk_buff *	(*dequeue)(struct Qdisc *sch);
  71	unsigned int		flags;
  72#define TCQ_F_BUILTIN		1
  73#define TCQ_F_INGRESS		2
  74#define TCQ_F_CAN_BYPASS	4
  75#define TCQ_F_MQROOT		8
  76#define TCQ_F_ONETXQUEUE	0x10 /* dequeue_skb() can assume all skbs are for
  77				      * q->dev_queue : It can test
  78				      * netif_xmit_frozen_or_stopped() before
  79				      * dequeueing next packet.
  80				      * Its true for MQ/MQPRIO slaves, or non
  81				      * multiqueue device.
  82				      */
  83#define TCQ_F_WARN_NONWC	(1 << 16)
  84#define TCQ_F_CPUSTATS		0x20 /* run using percpu statistics */
  85#define TCQ_F_NOPARENT		0x40 /* root of its hierarchy :
  86				      * qdisc_tree_decrease_qlen() should stop.
  87				      */
  88#define TCQ_F_INVISIBLE		0x80 /* invisible by default in dump */
  89#define TCQ_F_NOLOCK		0x100 /* qdisc does not require locking */
  90#define TCQ_F_OFFLOADED		0x200 /* qdisc is offloaded to HW */
  91#define TCQ_F_DEQUEUE_DROPS	0x400 /* ->dequeue() can drop packets in q->to_free */
  92
  93	u32			limit;
  94	const struct Qdisc_ops	*ops;
  95	struct qdisc_size_table	__rcu *stab;
  96	struct hlist_node       hash;
  97	u32			handle;
  98	u32			parent;
  99
 100	struct netdev_queue	*dev_queue;
 101
 102	struct net_rate_estimator __rcu *rate_est;
 103	struct gnet_stats_basic_sync __percpu *cpu_bstats;
 104	struct gnet_stats_queue	__percpu *cpu_qstats;
 105	int			pad;
 106	refcount_t		refcnt;
 107
 108	/* Cache line potentially dirtied in dequeue() or __netif_reschedule(). */
 109	__cacheline_group_begin(Qdisc_read_mostly) ____cacheline_aligned;
 110		struct sk_buff_head	gso_skb;
 111		struct Qdisc		*next_sched;
 112		struct sk_buff_head	skb_bad_txq;
 113	__cacheline_group_end(Qdisc_read_mostly);
 114
 115	/* Fields dirtied in dequeue() fast path. */
 116	__cacheline_group_begin(Qdisc_write) ____cacheline_aligned;
 117		struct qdisc_skb_head	q;
 118		unsigned long		state;
 119		struct gnet_stats_basic_sync bstats;
 120		bool			running; /* must be written under qdisc spinlock */
 121
 122		/* Note : we only change qstats.backlog in fast path. */
 123		struct gnet_stats_queue	qstats;
 124
 125		struct sk_buff		*to_free;
 126	__cacheline_group_end(Qdisc_write);
 127
 128
 129	atomic_long_t		defer_count ____cacheline_aligned_in_smp;
 130	struct llist_head	defer_list;
 131
 132	spinlock_t		seqlock;
 133
 134	struct rcu_head		rcu;
 135	netdevice_tracker	dev_tracker;
 136	struct lock_class_key	root_lock_key;
 137	/* private data */
 138	long privdata[] ____cacheline_aligned;
 139};
 140
 141static inline void qdisc_refcount_inc(struct Qdisc *qdisc)
 142{
 143	if (qdisc->flags & TCQ_F_BUILTIN)
 144		return;
 145	refcount_inc(&qdisc->refcnt);
 146}
 147
 148static inline bool qdisc_refcount_dec_if_one(struct Qdisc *qdisc)
 149{
 150	if (qdisc->flags & TCQ_F_BUILTIN)
 151		return true;
 152	return refcount_dec_if_one(&qdisc->refcnt);
 153}
 154
 155/* Intended to be used by unlocked users, when concurrent qdisc release is
 156 * possible.
 157 */
 158
 159static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc)
 160{
 161	if (qdisc->flags & TCQ_F_BUILTIN)
 162		return qdisc;
 163	if (refcount_inc_not_zero(&qdisc->refcnt))
 164		return qdisc;
 165	return NULL;
 166}
 167
 168/* For !TCQ_F_NOLOCK qdisc: callers must either call this within a qdisc
 169 * root_lock section, or provide their own memory barriers -- ordering
 170 * against qdisc_run_begin/end() atomic bit operations.
 171 */
 172static inline bool qdisc_is_running(struct Qdisc *qdisc)
 173{
 174	if (qdisc->flags & TCQ_F_NOLOCK)
 175		return spin_is_locked(&qdisc->seqlock);
 176	return READ_ONCE(qdisc->running);
 177}
 178
 179static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc)
 180{
 181	return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY);
 182}
 183
 184static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
 185{
 186	return q->flags & TCQ_F_CPUSTATS;
 187}
 188
 189static inline bool qdisc_is_empty(const struct Qdisc *qdisc)
 190{
 191	if (qdisc_is_percpu_stats(qdisc))
 192		return nolock_qdisc_is_empty(qdisc);
 193	return !READ_ONCE(qdisc->q.qlen);
 194}
 195
 196/* For !TCQ_F_NOLOCK qdisc, qdisc_run_begin/end() must be invoked with
 197 * the qdisc root lock acquired.
 198 */
 199static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 200{
 201	if (qdisc->flags & TCQ_F_NOLOCK) {
 202		if (spin_trylock(&qdisc->seqlock))
 203			return true;
 204
 205		/* No need to insist if the MISSED flag was already set.
 206		 * Note that test_and_set_bit() also gives us memory ordering
 207		 * guarantees wrt potential earlier enqueue() and below
 208		 * spin_trylock(), both of which are necessary to prevent races
 209		 */
 210		if (test_and_set_bit(__QDISC_STATE_MISSED, &qdisc->state))
 211			return false;
 212
 213		/* Try to take the lock again to make sure that we will either
 214		 * grab it or the CPU that still has it will see MISSED set
 215		 * when testing it in qdisc_run_end()
 216		 */
 217		return spin_trylock(&qdisc->seqlock);
 218	}
 219	if (READ_ONCE(qdisc->running))
 220		return false;
 221	WRITE_ONCE(qdisc->running, true);
 222	return true;
 223}
 224
 225static inline struct sk_buff *qdisc_run_end(struct Qdisc *qdisc)
 226{
 227	struct sk_buff *to_free = NULL;
 228
 229	if (qdisc->flags & TCQ_F_NOLOCK) {
 230		spin_unlock(&qdisc->seqlock);
 231
 232		/* spin_unlock() only has store-release semantic. The unlock
 233		 * and test_bit() ordering is a store-load ordering, so a full
 234		 * memory barrier is needed here.
 235		 */
 236		smp_mb();
 237
 238		if (unlikely(test_bit(__QDISC_STATE_MISSED,
 239				      &qdisc->state)))
 240			__netif_schedule(qdisc);
 241		return NULL;
 242	}
 243
 244	if (qdisc->flags & TCQ_F_DEQUEUE_DROPS) {
 245		to_free = qdisc->to_free;
 246		if (to_free)
 247			qdisc->to_free = NULL;
 248	}
 249	WRITE_ONCE(qdisc->running, false);
 250	return to_free;
 251}
 252
 253static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)
 254{
 255	return qdisc->flags & TCQ_F_ONETXQUEUE;
 256}
 257
 258static inline int qdisc_avail_bulklimit(const struct netdev_queue *txq)
 259{
 260	return netdev_queue_dql_avail(txq);
 261}
 262
 263struct Qdisc_class_ops {
 264	unsigned int		flags;
 265	/* Child qdisc manipulation */
 266	struct netdev_queue *	(*select_queue)(struct Qdisc *, struct tcmsg *);
 267	int			(*graft)(struct Qdisc *, unsigned long cl,
 268					struct Qdisc *, struct Qdisc **,
 269					struct netlink_ext_ack *extack);
 270	struct Qdisc *		(*leaf)(struct Qdisc *, unsigned long cl);
 271	void			(*qlen_notify)(struct Qdisc *, unsigned long);
 272
 273	/* Class manipulation routines */
 274	unsigned long		(*find)(struct Qdisc *, u32 classid);
 275	int			(*change)(struct Qdisc *, u32, u32,
 276					struct nlattr **, unsigned long *,
 277					struct netlink_ext_ack *);
 278	int			(*delete)(struct Qdisc *, unsigned long,
 279					  struct netlink_ext_ack *);
 280	void			(*walk)(struct Qdisc *, struct qdisc_walker * arg);
 281
 282	/* Filter manipulation */
 283	struct tcf_block *	(*tcf_block)(struct Qdisc *sch,
 284					     unsigned long arg,
 285					     struct netlink_ext_ack *extack);
 286	unsigned long		(*bind_tcf)(struct Qdisc *, unsigned long,
 287					u32 classid);
 288	void			(*unbind_tcf)(struct Qdisc *, unsigned long);
 289
 290	/* rtnetlink specific */
 291	int			(*dump)(struct Qdisc *, unsigned long,
 292					struct sk_buff *skb, struct tcmsg*);
 293	int			(*dump_stats)(struct Qdisc *, unsigned long,
 294					struct gnet_dump *);
 295};
 296
 297/* Qdisc_class_ops flag values */
 298
 299/* Implements API that doesn't require rtnl lock */
 300enum qdisc_class_ops_flags {
 301	QDISC_CLASS_OPS_DOIT_UNLOCKED = 1,
 302};
 303
 304struct Qdisc_ops {
 305	struct Qdisc_ops	*next;
 306	const struct Qdisc_class_ops	*cl_ops;
 307	char			id[IFNAMSIZ];
 308	int			priv_size;
 309	unsigned int		static_flags;
 310
 311	int 			(*enqueue)(struct sk_buff *skb,
 312					   struct Qdisc *sch,
 313					   struct sk_buff **to_free);
 314	struct sk_buff *	(*dequeue)(struct Qdisc *);
 315	struct sk_buff *	(*peek)(struct Qdisc *);
 316
 317	int			(*init)(struct Qdisc *sch, struct nlattr *arg,
 318					struct netlink_ext_ack *extack);
 319	void			(*reset)(struct Qdisc *);
 320	void			(*destroy)(struct Qdisc *);
 321	int			(*change)(struct Qdisc *sch,
 322					  struct nlattr *arg,
 323					  struct netlink_ext_ack *extack);
 324	void			(*attach)(struct Qdisc *sch);
 325	int			(*change_tx_queue_len)(struct Qdisc *, unsigned int);
 326	void			(*change_real_num_tx)(struct Qdisc *sch,
 327						      unsigned int new_real_tx);
 328
 329	int			(*dump)(struct Qdisc *, struct sk_buff *);
 330	int			(*dump_stats)(struct Qdisc *, struct gnet_dump *);
 331
 332	void			(*ingress_block_set)(struct Qdisc *sch,
 333						     u32 block_index);
 334	void			(*egress_block_set)(struct Qdisc *sch,
 335						    u32 block_index);
 336	u32			(*ingress_block_get)(struct Qdisc *sch);
 337	u32			(*egress_block_get)(struct Qdisc *sch);
 338
 339	struct module		*owner;
 340};
 341
 342struct tcf_result {
 343	union {
 344		struct {
 345			unsigned long	class;
 346			u32		classid;
 347		};
 348		const struct tcf_proto *goto_tp;
 349	};
 350};
 351
 352struct tcf_chain;
 353
 354struct tcf_proto_ops {
 355	struct list_head	head;
 356	char			kind[IFNAMSIZ];
 357
 358	int			(*classify)(struct sk_buff *,
 359					    const struct tcf_proto *,
 360					    struct tcf_result *);
 361	int			(*init)(struct tcf_proto*);
 362	void			(*destroy)(struct tcf_proto *tp, bool rtnl_held,
 363					   struct netlink_ext_ack *extack);
 364
 365	void*			(*get)(struct tcf_proto*, u32 handle);
 366	void			(*put)(struct tcf_proto *tp, void *f);
 367	int			(*change)(struct net *net, struct sk_buff *,
 368					struct tcf_proto*, unsigned long,
 369					u32 handle, struct nlattr **,
 370					void **, u32,
 371					struct netlink_ext_ack *);
 372	int			(*delete)(struct tcf_proto *tp, void *arg,
 373					  bool *last, bool rtnl_held,
 374					  struct netlink_ext_ack *);
 375	bool			(*delete_empty)(struct tcf_proto *tp);
 376	void			(*walk)(struct tcf_proto *tp,
 377					struct tcf_walker *arg, bool rtnl_held);
 378	int			(*reoffload)(struct tcf_proto *tp, bool add,
 379					     flow_setup_cb_t *cb, void *cb_priv,
 380					     struct netlink_ext_ack *extack);
 381	void			(*hw_add)(struct tcf_proto *tp,
 382					  void *type_data);
 383	void			(*hw_del)(struct tcf_proto *tp,
 384					  void *type_data);
 385	void			(*bind_class)(void *, u32, unsigned long,
 386					      void *, unsigned long);
 387	void *			(*tmplt_create)(struct net *net,
 388						struct tcf_chain *chain,
 389						struct nlattr **tca,
 390						struct netlink_ext_ack *extack);
 391	void			(*tmplt_destroy)(void *tmplt_priv);
 392	void			(*tmplt_reoffload)(struct tcf_chain *chain,
 393						   bool add,
 394						   flow_setup_cb_t *cb,
 395						   void *cb_priv);
 396	struct tcf_exts *	(*get_exts)(const struct tcf_proto *tp,
 397					    u32 handle);
 398
 399	/* rtnetlink specific */
 400	int			(*dump)(struct net*, struct tcf_proto*, void *,
 401					struct sk_buff *skb, struct tcmsg*,
 402					bool);
 403	int			(*terse_dump)(struct net *net,
 404					      struct tcf_proto *tp, void *fh,
 405					      struct sk_buff *skb,
 406					      struct tcmsg *t, bool rtnl_held);
 407	int			(*tmplt_dump)(struct sk_buff *skb,
 408					      struct net *net,
 409					      void *tmplt_priv);
 410
 411	struct module		*owner;
 412	int			flags;
 413};
 414
 415/* Classifiers setting TCF_PROTO_OPS_DOIT_UNLOCKED in tcf_proto_ops->flags
 416 * are expected to implement tcf_proto_ops->delete_empty(), otherwise race
 417 * conditions can occur when filters are inserted/deleted simultaneously.
 418 */
 419enum tcf_proto_ops_flags {
 420	TCF_PROTO_OPS_DOIT_UNLOCKED = 1,
 421};
 422
 423struct tcf_proto {
 424	/* Fast access part */
 425	struct tcf_proto __rcu	*next;
 426	void __rcu		*root;
 427
 428	/* called under RCU BH lock*/
 429	int			(*classify)(struct sk_buff *,
 430					    const struct tcf_proto *,
 431					    struct tcf_result *);
 432	__be16			protocol;
 433
 434	/* All the rest */
 435	u32			prio;
 436	void			*data;
 437	const struct tcf_proto_ops	*ops;
 438	struct tcf_chain	*chain;
 439	/* Lock protects tcf_proto shared state and can be used by unlocked
 440	 * classifiers to protect their private data.
 441	 */
 442	spinlock_t		lock;
 443	bool			deleting;
 444	bool			counted;
 445	bool			usesw;
 446	refcount_t		refcnt;
 447	struct rcu_head		rcu;
 448	struct hlist_node	destroy_ht_node;
 449};
 450
 451struct qdisc_skb_cb {
 452	unsigned int		pkt_len;
 453	u16			pkt_segs;
 454	u16			tc_classid;
 455#define QDISC_CB_PRIV_LEN 20
 456	unsigned char		data[QDISC_CB_PRIV_LEN];
 457
 458	u16			slave_dev_queue_mapping;
 459	u8			post_ct:1;
 460	u8			post_ct_snat:1;
 461	u8			post_ct_dnat:1;
 462};
 463
 464typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv);
 465
 466struct tcf_chain {
 467	/* Protects filter_chain. */
 468	struct mutex filter_chain_lock;
 469	struct tcf_proto __rcu *filter_chain;
 470	struct list_head list;
 471	struct tcf_block *block;
 472	u32 index; /* chain index */
 473	unsigned int refcnt;
 474	unsigned int action_refcnt;
 475	bool explicitly_created;
 476	bool flushing;
 477	const struct tcf_proto_ops *tmplt_ops;
 478	void *tmplt_priv;
 479	struct rcu_head rcu;
 480};
 481
 482struct tcf_block {
 483	struct xarray ports; /* datapath accessible */
 484	/* Lock protects tcf_block and lifetime-management data of chains
 485	 * attached to the block (refcnt, action_refcnt, explicitly_created).
 486	 */
 487	struct mutex lock;
 488	struct list_head chain_list;
 489	u32 index; /* block index for shared blocks */
 490	u32 classid; /* which class this block belongs to */
 491	refcount_t refcnt;
 492	struct net *net;
 493	struct Qdisc *q;
 494	struct rw_semaphore cb_lock; /* protects cb_list and offload counters */
 495	struct flow_block flow_block;
 496	struct list_head owner_list;
 497	bool keep_dst;
 498	atomic_t useswcnt;
 499	atomic_t offloadcnt; /* Number of oddloaded filters */
 500	unsigned int nooffloaddevcnt; /* Number of devs unable to do offload */
 501	unsigned int lockeddevcnt; /* Number of devs that require rtnl lock. */
 502	struct {
 503		struct tcf_chain *chain;
 504		struct list_head filter_chain_list;
 505	} chain0;
 506	struct rcu_head rcu;
 507	DECLARE_HASHTABLE(proto_destroy_ht, 7);
 508	struct mutex proto_destroy_lock; /* Lock for proto_destroy hashtable. */
 509};
 510
 511struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index);
 512
 513static inline bool lockdep_tcf_chain_is_locked(struct tcf_chain *chain)
 514{
 515	return lockdep_is_held(&chain->filter_chain_lock);
 516}
 517
 518static inline bool lockdep_tcf_proto_is_locked(struct tcf_proto *tp)
 519{
 520	return lockdep_is_held(&tp->lock);
 521}
 522
 523#define tcf_chain_dereference(p, chain)					\
 524	rcu_dereference_protected(p, lockdep_tcf_chain_is_locked(chain))
 525
 526#define tcf_proto_dereference(p, tp)					\
 527	rcu_dereference_protected(p, lockdep_tcf_proto_is_locked(tp))
 528
 529static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz)
 530{
 531	struct qdisc_skb_cb *qcb;
 532
 533	BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*qcb));
 534	BUILD_BUG_ON(sizeof(qcb->data) < sz);
 535}
 536
 537static inline int qdisc_qlen(const struct Qdisc *q)
 538{
 539	return q->q.qlen;
 540}
 541
 542static inline int qdisc_qlen_sum(const struct Qdisc *q)
 543{
 544	__u32 qlen = q->qstats.qlen;
 545	int i;
 546
 547	if (qdisc_is_percpu_stats(q)) {
 548		for_each_possible_cpu(i)
 549			qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen;
 550	} else {
 551		qlen += q->q.qlen;
 552	}
 553
 554	return qlen;
 555}
 556
 557static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb)
 558{
 559	return (struct qdisc_skb_cb *)skb->cb;
 560}
 561
 562static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc)
 563{
 564	return &qdisc->q.lock;
 565}
 566
 567static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc)
 568{
 569	struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc);
 570
 571	return q;
 572}
 573
 574static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc)
 575{
 576	return rcu_dereference_bh(qdisc->dev_queue->qdisc);
 577}
 578
 579static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc)
 580{
 581	return rcu_dereference_rtnl(qdisc->dev_queue->qdisc_sleeping);
 582}
 583
 584static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)
 585{
 586	struct Qdisc *root = qdisc_root_sleeping(qdisc);
 587
 588	ASSERT_RTNL();
 589	return qdisc_lock(root);
 590}
 591
 592static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)
 593{
 594	return qdisc->dev_queue->dev;
 595}
 596
 597static inline void sch_tree_lock(struct Qdisc *q)
 598{
 599	if (q->flags & TCQ_F_MQROOT)
 600		spin_lock_bh(qdisc_lock(q));
 601	else
 602		spin_lock_bh(qdisc_root_sleeping_lock(q));
 603}
 604
 605static inline void sch_tree_unlock(struct Qdisc *q)
 606{
 607	if (q->flags & TCQ_F_MQROOT)
 608		spin_unlock_bh(qdisc_lock(q));
 609	else
 610		spin_unlock_bh(qdisc_root_sleeping_lock(q));
 611}
 612
 613extern struct Qdisc noop_qdisc;
 614extern struct Qdisc_ops noop_qdisc_ops;
 615extern struct Qdisc_ops pfifo_fast_ops;
 616extern const u8 sch_default_prio2band[TC_PRIO_MAX + 1];
 617extern struct Qdisc_ops mq_qdisc_ops;
 618extern struct Qdisc_ops noqueue_qdisc_ops;
 619extern const struct Qdisc_ops *default_qdisc_ops;
 620static inline const struct Qdisc_ops *
 621get_default_qdisc_ops(const struct net_device *dev, int ntx)
 622{
 623	return ntx < dev->real_num_tx_queues ?
 624			default_qdisc_ops : &pfifo_fast_ops;
 625}
 626
 627struct Qdisc_class_common {
 628	u32			classid;
 629	unsigned int		filter_cnt;
 630	struct hlist_node	hnode;
 631};
 632
 633struct Qdisc_class_hash {
 634	struct hlist_head	*hash;
 635	unsigned int		hashsize;
 636	unsigned int		hashmask;
 637	unsigned int		hashelems;
 638};
 639
 640static inline unsigned int qdisc_class_hash(u32 id, u32 mask)
 641{
 642	id ^= id >> 8;
 643	id ^= id >> 4;
 644	return id & mask;
 645}
 646
 647static inline struct Qdisc_class_common *
 648qdisc_class_find(const struct Qdisc_class_hash *hash, u32 id)
 649{
 650	struct Qdisc_class_common *cl;
 651	unsigned int h;
 652
 653	if (!id)
 654		return NULL;
 655
 656	h = qdisc_class_hash(id, hash->hashmask);
 657	hlist_for_each_entry(cl, &hash->hash[h], hnode) {
 658		if (cl->classid == id)
 659			return cl;
 660	}
 661	return NULL;
 662}
 663
 664static inline bool qdisc_class_in_use(const struct Qdisc_class_common *cl)
 665{
 666	return cl->filter_cnt > 0;
 667}
 668
 669static inline void qdisc_class_get(struct Qdisc_class_common *cl)
 670{
 671	unsigned int res;
 672
 673	if (check_add_overflow(cl->filter_cnt, 1, &res))
 674		WARN(1, "Qdisc class overflow");
 675
 676	cl->filter_cnt = res;
 677}
 678
 679static inline void qdisc_class_put(struct Qdisc_class_common *cl)
 680{
 681	unsigned int res;
 682
 683	if (check_sub_overflow(cl->filter_cnt, 1, &res))
 684		WARN(1, "Qdisc class underflow");
 685
 686	cl->filter_cnt = res;
 687}
 688
 689static inline int tc_classid_to_hwtc(struct net_device *dev, u32 classid)
 690{
 691	u32 hwtc = TC_H_MIN(classid) - TC_H_MIN_PRIORITY;
 692
 693	return (hwtc < netdev_get_num_tc(dev)) ? hwtc : -EINVAL;
 694}
 695
 696int qdisc_class_hash_init(struct Qdisc_class_hash *);
 697void qdisc_class_hash_insert(struct Qdisc_class_hash *,
 698			     struct Qdisc_class_common *);
 699void qdisc_class_hash_remove(struct Qdisc_class_hash *,
 700			     struct Qdisc_class_common *);
 701void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);
 702void qdisc_class_hash_destroy(struct Qdisc_class_hash *);
 703
 704int dev_qdisc_change_tx_queue_len(struct net_device *dev);
 705void dev_qdisc_change_real_num_tx(struct net_device *dev,
 706				  unsigned int new_real_tx);
 707void dev_init_scheduler(struct net_device *dev);
 708void dev_shutdown(struct net_device *dev);
 709void dev_activate(struct net_device *dev);
 710void dev_deactivate(struct net_device *dev);
 711void dev_deactivate_many(struct list_head *head);
 712struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
 713			      struct Qdisc *qdisc);
 714void qdisc_reset(struct Qdisc *qdisc);
 715void qdisc_destroy(struct Qdisc *qdisc);
 716void qdisc_put(struct Qdisc *qdisc);
 717void qdisc_put_unlocked(struct Qdisc *qdisc);
 718void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
 719
 720static inline void dev_reset_queue(struct net_device *dev,
 721				   struct netdev_queue *dev_queue,
 722				   void *_unused)
 723{
 724	struct Qdisc *qdisc;
 725	bool nolock;
 726
 727	qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
 728	if (!qdisc)
 729		return;
 730
 731	nolock = qdisc->flags & TCQ_F_NOLOCK;
 732
 733	if (nolock)
 734		spin_lock_bh(&qdisc->seqlock);
 735	spin_lock_bh(qdisc_lock(qdisc));
 736
 737	qdisc_reset(qdisc);
 738
 739	spin_unlock_bh(qdisc_lock(qdisc));
 740	if (nolock) {
 741		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
 742		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
 743		spin_unlock_bh(&qdisc->seqlock);
 744	}
 745}
 746
 747#ifdef CONFIG_NET_SCHED
 748int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
 749			      void *type_data);
 750void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
 751				struct Qdisc *new, struct Qdisc *old,
 752				enum tc_setup_type type, void *type_data,
 753				struct netlink_ext_ack *extack);
 754#else
 755static inline int
 756qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
 757			  void *type_data)
 758{
 759	q->flags &= ~TCQ_F_OFFLOADED;
 760	return 0;
 761}
 762
 763static inline void
 764qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
 765			   struct Qdisc *new, struct Qdisc *old,
 766			   enum tc_setup_type type, void *type_data,
 767			   struct netlink_ext_ack *extack)
 768{
 769}
 770#endif
 771void qdisc_offload_query_caps(struct net_device *dev,
 772			      enum tc_setup_type type,
 773			      void *caps, size_t caps_len);
 774struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
 775			  const struct Qdisc_ops *ops,
 776			  struct netlink_ext_ack *extack);
 777void qdisc_free(struct Qdisc *qdisc);
 778struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
 779				const struct Qdisc_ops *ops, u32 parentid,
 780				struct netlink_ext_ack *extack);
 781void __qdisc_calculate_pkt_len(struct sk_buff *skb,
 782			       const struct qdisc_size_table *stab);
 783int skb_do_redirect(struct sk_buff *);
 784
 785static inline bool skb_at_tc_ingress(const struct sk_buff *skb)
 786{
 787#ifdef CONFIG_NET_XGRESS
 788	return skb->tc_at_ingress;
 789#else
 790	return false;
 791#endif
 792}
 793
 794static inline bool skb_skip_tc_classify(struct sk_buff *skb)
 795{
 796#ifdef CONFIG_NET_CLS_ACT
 797	if (skb->tc_skip_classify) {
 798		skb->tc_skip_classify = 0;
 799		return true;
 800	}
 801#endif
 802	return false;
 803}
 804
 805/* Reset all TX qdiscs greater than index of a device.  */
 806static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i)
 807{
 808	struct Qdisc *qdisc;
 809	bool nolock;
 810
 811	for (; i < dev->num_tx_queues; i++) {
 812		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc);
 813		if (qdisc) {
 814			nolock = qdisc->flags & TCQ_F_NOLOCK;
 815
 816			if (nolock)
 817				spin_lock_bh(&qdisc->seqlock);
 818			spin_lock_bh(qdisc_lock(qdisc));
 819			qdisc_reset(qdisc);
 820			spin_unlock_bh(qdisc_lock(qdisc));
 821			if (nolock) {
 822				clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
 823				clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
 824				spin_unlock_bh(&qdisc->seqlock);
 825			}
 826		}
 827	}
 828}
 829
 830/* Are all TX queues of the device empty?  */
 831static inline bool qdisc_all_tx_empty(const struct net_device *dev)
 832{
 833	unsigned int i;
 834
 835	rcu_read_lock();
 836	for (i = 0; i < dev->num_tx_queues; i++) {
 837		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
 838		const struct Qdisc *q = rcu_dereference(txq->qdisc);
 839
 840		if (!qdisc_is_empty(q)) {
 841			rcu_read_unlock();
 842			return false;
 843		}
 844	}
 845	rcu_read_unlock();
 846	return true;
 847}
 848
 849/* Are any of the TX qdiscs changing?  */
 850static inline bool qdisc_tx_changing(const struct net_device *dev)
 851{
 852	unsigned int i;
 853
 854	for (i = 0; i < dev->num_tx_queues; i++) {
 855		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
 856
 857		if (rcu_access_pointer(txq->qdisc) !=
 858		    rcu_access_pointer(txq->qdisc_sleeping))
 859			return true;
 860	}
 861	return false;
 862}
 863
 864/* "noqueue" qdisc identified by not having any enqueue, see noqueue_init() */
 865static inline bool qdisc_txq_has_no_queue(const struct netdev_queue *txq)
 866{
 867	struct Qdisc *qdisc = rcu_access_pointer(txq->qdisc);
 868
 869	return qdisc->enqueue == NULL;
 870}
 871
 872/* Is the device using the noop qdisc on all queues?  */
 873static inline bool qdisc_tx_is_noop(const struct net_device *dev)
 874{
 875	unsigned int i;
 876
 877	for (i = 0; i < dev->num_tx_queues; i++) {
 878		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
 879		if (rcu_access_pointer(txq->qdisc) != &noop_qdisc)
 880			return false;
 881	}
 882	return true;
 883}
 884
 885static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb)
 886{
 887	return qdisc_skb_cb(skb)->pkt_len;
 888}
 889
 890static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb)
 891{
 892	u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;
 893
 894	DEBUG_NET_WARN_ON_ONCE(pkt_segs !=
 895			(skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1));
 896	return pkt_segs;
 897}
 898
 899/* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
 900enum net_xmit_qdisc_t {
 901	__NET_XMIT_STOLEN = 0x00010000,
 902	__NET_XMIT_BYPASS = 0x00020000,
 903};
 904
 905#ifdef CONFIG_NET_CLS_ACT
 906#define net_xmit_drop_count(e)	((e) & __NET_XMIT_STOLEN ? 0 : 1)
 907#else
 908#define net_xmit_drop_count(e)	(1)
 909#endif
 910
 911static inline void qdisc_calculate_pkt_len(struct sk_buff *skb,
 912					   const struct Qdisc *sch)
 913{
 914#ifdef CONFIG_NET_SCHED
 915	struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab);
 916
 917	if (stab)
 918		__qdisc_calculate_pkt_len(skb, stab);
 919#endif
 920}
 921
 922static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 923				struct sk_buff **to_free)
 924{
 925	return sch->enqueue(skb, sch, to_free);
 926}
 927
 928static inline void _bstats_update(struct gnet_stats_basic_sync *bstats,
 929				  __u64 bytes, __u64 packets)
 930{
 931	u64_stats_update_begin(&bstats->syncp);
 932	u64_stats_add(&bstats->bytes, bytes);
 933	u64_stats_add(&bstats->packets, packets);
 934	u64_stats_update_end(&bstats->syncp);
 935}
 936
 937static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
 938				 const struct sk_buff *skb)
 939{
 940	_bstats_update(bstats, qdisc_pkt_len(skb), qdisc_pkt_segs(skb));
 941}
 942
 943static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,
 944					   const struct sk_buff *skb)
 945{
 946	bstats_update(this_cpu_ptr(sch->cpu_bstats), skb);
 947}
 948
 949static inline void qdisc_bstats_update(struct Qdisc *sch,
 950				       const struct sk_buff *skb)
 951{
 952	bstats_update(&sch->bstats, skb);
 953}
 954
 955static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch,
 956					    const struct sk_buff *skb)
 957{
 958	sch->qstats.backlog -= qdisc_pkt_len(skb);
 959}
 960
 961static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
 962						const struct sk_buff *skb)
 963{
 964	this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
 965}
 966
 967static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch,
 968					    const struct sk_buff *skb)
 969{
 970	sch->qstats.backlog += qdisc_pkt_len(skb);
 971}
 972
 973static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
 974						const struct sk_buff *skb)
 975{
 976	this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
 977}
 978
 979static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch)
 980{
 981	this_cpu_inc(sch->cpu_qstats->qlen);
 982}
 983
 984static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch)
 985{
 986	this_cpu_dec(sch->cpu_qstats->qlen);
 987}
 988
 989static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch)
 990{
 991	this_cpu_inc(sch->cpu_qstats->requeues);
 992}
 993
 994static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count)
 995{
 996	sch->qstats.drops += count;
 997}
 998
 999static inline void qstats_drop_inc(struct gnet_stats_queue *qstats)
1000{
1001	qstats->drops++;
1002}
1003
1004static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats)
1005{
1006	qstats->overlimits++;
1007}
1008
1009static inline void qdisc_qstats_drop(struct Qdisc *sch)
1010{
1011	qstats_drop_inc(&sch->qstats);
1012}
1013
1014static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch)
1015{
1016	this_cpu_inc(sch->cpu_qstats->drops);
1017}
1018
1019static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
1020{
1021	sch->qstats.overlimits++;
1022}
1023
1024static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)
1025{
1026	__u32 qlen = qdisc_qlen_sum(sch);
1027
1028	return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen);
1029}
1030
1031static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch,  __u32 *qlen,
1032					     __u32 *backlog)
1033{
1034	struct gnet_stats_queue qstats = { 0 };
1035
1036	gnet_stats_add_queue(&qstats, sch->cpu_qstats, &sch->qstats);
1037	*qlen = qstats.qlen + qdisc_qlen(sch);
1038	*backlog = qstats.backlog;
1039}
1040
1041static inline void qdisc_purge_queue(struct Qdisc *sch)
1042{
1043	__u32 qlen, backlog;
1044
1045	qdisc_qstats_qlen_backlog(sch, &qlen, &backlog);
1046	qdisc_reset(sch);
1047	qdisc_tree_reduce_backlog(sch, qlen, backlog);
1048}
1049
1050static inline void __qdisc_enqueue_tail(struct sk_buff *skb,
1051					struct qdisc_skb_head *qh)
1052{
1053	struct sk_buff *last = qh->tail;
1054
1055	if (last) {
1056		skb->next = NULL;
1057		last->next = skb;
1058		qh->tail = skb;
1059	} else {
1060		qh->tail = skb;
1061		qh->head = skb;
1062	}
1063	qh->qlen++;
1064}
1065
1066static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
1067{
1068	__qdisc_enqueue_tail(skb, &sch->q);
1069	qdisc_qstats_backlog_inc(sch, skb);
1070	return NET_XMIT_SUCCESS;
1071}
1072
1073static inline void __qdisc_enqueue_head(struct sk_buff *skb,
1074					struct qdisc_skb_head *qh)
1075{
1076	skb->next = qh->head;
1077
1078	if (!qh->head)
1079		qh->tail = skb;
1080	qh->head = skb;
1081	qh->qlen++;
1082}
1083
1084static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
1085{
1086	struct sk_buff *skb = qh->head;
1087
1088	if (likely(skb != NULL)) {
1089		qh->head = skb->next;
1090		qh->qlen--;
1091		if (qh->head == NULL)
1092			qh->tail = NULL;
1093		skb->next = NULL;
1094	}
1095
1096	return skb;
1097}
1098
1099static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool direct)
1100{
1101	struct sk_buff *skb;
1102
1103	skb = __skb_dequeue(&sch->gso_skb);
1104	if (skb) {
1105		sch->q.qlen--;
1106		qdisc_qstats_backlog_dec(sch, skb);
1107		return skb;
1108	}
1109	if (direct) {
1110		skb = __qdisc_dequeue_head(&sch->q);
1111		if (skb)
1112			qdisc_qstats_backlog_dec(sch, skb);
1113		return skb;
1114	} else {
1115		return sch->dequeue(sch);
1116	}
1117}
1118
1119static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch)
1120{
1121	struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
1122
1123	if (likely(skb != NULL)) {
1124		qdisc_qstats_backlog_dec(sch, skb);
1125		qdisc_bstats_update(sch, skb);
1126	}
1127
1128	return skb;
1129}
1130
1131struct tc_skb_cb {
1132	struct qdisc_skb_cb qdisc_cb;
1133	u32 drop_reason;
1134
1135	u16 zone; /* Only valid if qdisc_skb_cb(skb)->post_ct = true */
1136	u16 mru;
1137};
1138
1139static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb)
1140{
1141	struct tc_skb_cb *cb = (struct tc_skb_cb *)skb->cb;
1142
1143	BUILD_BUG_ON(sizeof(*cb) > sizeof_field(struct sk_buff, cb));
1144	return cb;
1145}
1146
1147static inline enum skb_drop_reason
1148tcf_get_drop_reason(const struct sk_buff *skb)
1149{
1150	return tc_skb_cb(skb)->drop_reason;
1151}
1152
1153static inline void tcf_set_drop_reason(const struct sk_buff *skb,
1154				       enum skb_drop_reason reason)
1155{
1156	tc_skb_cb(skb)->drop_reason = reason;
1157}
1158
1159static inline void tcf_kfree_skb_list(struct sk_buff *skb)
1160{
1161	while (unlikely(skb)) {
1162		struct sk_buff *next = skb->next;
1163
1164		prefetch(next);
1165		kfree_skb_reason(skb, tcf_get_drop_reason(skb));
1166		skb = next;
1167	}
1168}
1169
1170static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb,
1171				      enum skb_drop_reason reason)
1172{
1173	DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS));
1174	DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK);
1175
1176	tcf_set_drop_reason(skb, reason);
1177	skb->next = q->to_free;
1178	q->to_free = skb;
1179}
1180
1181/* Instead of calling kfree_skb() while root qdisc lock is held,
1182 * queue the skb for future freeing at end of __dev_xmit_skb()
1183 */
1184static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free)
1185{
1186	skb->next = *to_free;
1187	*to_free = skb;
1188}
1189
1190static inline void __qdisc_drop_all(struct sk_buff *skb,
1191				    struct sk_buff **to_free)
1192{
1193	if (skb->prev)
1194		skb->prev->next = *to_free;
1195	else
1196		skb->next = *to_free;
1197	*to_free = skb;
1198}
1199
1200static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,
1201						   struct qdisc_skb_head *qh,
1202						   struct sk_buff **to_free)
1203{
1204	struct sk_buff *skb = __qdisc_dequeue_head(qh);
1205
1206	if (likely(skb != NULL)) {
1207		unsigned int len = qdisc_pkt_len(skb);
1208
1209		qdisc_qstats_backlog_dec(sch, skb);
1210		__qdisc_drop(skb, to_free);
1211		return len;
1212	}
1213
1214	return 0;
1215}
1216
1217static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
1218{
1219	const struct qdisc_skb_head *qh = &sch->q;
1220
1221	return qh->head;
1222}
1223
1224/* generic pseudo peek method for non-work-conserving qdisc */
1225static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)
1226{
1227	struct sk_buff *skb = skb_peek(&sch->gso_skb);
1228
1229	/* we can reuse ->gso_skb because peek isn't called for root qdiscs */
1230	if (!skb) {
1231		skb = sch->dequeue(sch);
1232
1233		if (skb) {
1234			__skb_queue_head(&sch->gso_skb, skb);
1235			/* it's still part of the queue */
1236			qdisc_qstats_backlog_inc(sch, skb);
1237			sch->q.qlen++;
1238		}
1239	}
1240
1241	return skb;
1242}
1243
1244static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch,
1245						 struct sk_buff *skb)
1246{
1247	if (qdisc_is_percpu_stats(sch)) {
1248		qdisc_qstats_cpu_backlog_dec(sch, skb);
1249		qdisc_bstats_cpu_update(sch, skb);
1250		qdisc_qstats_cpu_qlen_dec(sch);
1251	} else {
1252		qdisc_qstats_backlog_dec(sch, skb);
1253		qdisc_bstats_update(sch, skb);
1254		sch->q.qlen--;
1255	}
1256}
1257
1258static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,
1259						 unsigned int pkt_len)
1260{
1261	if (qdisc_is_percpu_stats(sch)) {
1262		qdisc_qstats_cpu_qlen_inc(sch);
1263		this_cpu_add(sch->cpu_qstats->backlog, pkt_len);
1264	} else {
1265		sch->qstats.backlog += pkt_len;
1266		sch->q.qlen++;
1267	}
1268}
1269
1270/* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */
1271static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
1272{
1273	struct sk_buff *skb = skb_peek(&sch->gso_skb);
1274
1275	if (skb) {
1276		skb = __skb_dequeue(&sch->gso_skb);
1277		if (qdisc_is_percpu_stats(sch)) {
1278			qdisc_qstats_cpu_backlog_dec(sch, skb);
1279			qdisc_qstats_cpu_qlen_dec(sch);
1280		} else {
1281			qdisc_qstats_backlog_dec(sch, skb);
1282			sch->q.qlen--;
1283		}
1284	} else {
1285		skb = sch->dequeue(sch);
1286	}
1287
1288	return skb;
1289}
1290
1291static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh)
1292{
1293	/*
1294	 * We do not know the backlog in bytes of this list, it
1295	 * is up to the caller to correct it
1296	 */
1297	ASSERT_RTNL();
1298	if (qh->qlen) {
1299		rtnl_kfree_skbs(qh->head, qh->tail);
1300
1301		qh->head = NULL;
1302		qh->tail = NULL;
1303		qh->qlen = 0;
1304	}
1305}
1306
1307static inline void qdisc_reset_queue(struct Qdisc *sch)
1308{
1309	__qdisc_reset_queue(&sch->q);
1310}
1311
1312static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new,
1313					  struct Qdisc **pold)
1314{
1315	struct Qdisc *old;
1316
1317	sch_tree_lock(sch);
1318	old = *pold;
1319	*pold = new;
1320	if (old != NULL)
1321		qdisc_purge_queue(old);
1322	sch_tree_unlock(sch);
1323
1324	return old;
1325}
1326
1327static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
1328{
1329	rtnl_kfree_skbs(skb, skb);
1330	qdisc_qstats_drop(sch);
1331}
1332
1333static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch,
1334				 struct sk_buff **to_free)
1335{
1336	__qdisc_drop(skb, to_free);
1337	qdisc_qstats_cpu_drop(sch);
1338
1339	return NET_XMIT_DROP;
1340}
1341
1342static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,
1343			     struct sk_buff **to_free)
1344{
1345	__qdisc_drop(skb, to_free);
1346	qdisc_qstats_drop(sch);
1347
1348	return NET_XMIT_DROP;
1349}
1350
1351static inline int qdisc_drop_reason(struct sk_buff *skb, struct Qdisc *sch,
1352				    struct sk_buff **to_free,
1353				    enum skb_drop_reason reason)
1354{
1355	tcf_set_drop_reason(skb, reason);
1356	return qdisc_drop(skb, sch, to_free);
1357}
1358
1359static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch,
1360				 struct sk_buff **to_free)
1361{
1362	__qdisc_drop_all(skb, to_free);
1363	qdisc_qstats_drop(sch);
1364
1365	return NET_XMIT_DROP;
1366}
1367
1368struct psched_ratecfg {
1369	u64	rate_bytes_ps; /* bytes per second */
1370	u32	mult;
1371	u16	overhead;
1372	u16	mpu;
1373	u8	linklayer;
1374	u8	shift;
1375};
1376
1377static inline u64 psched_l2t_ns(const struct psched_ratecfg *r,
1378				unsigned int len)
1379{
1380	len += r->overhead;
1381
1382	if (len < r->mpu)
1383		len = r->mpu;
1384
1385	if (unlikely(r->linklayer == TC_LINKLAYER_ATM))
1386		return ((u64)(DIV_ROUND_UP(len,48)*53) * r->mult) >> r->shift;
1387
1388	return ((u64)len * r->mult) >> r->shift;
1389}
1390
1391void psched_ratecfg_precompute(struct psched_ratecfg *r,
1392			       const struct tc_ratespec *conf,
1393			       u64 rate64);
1394
1395static inline void psched_ratecfg_getrate(struct tc_ratespec *res,
1396					  const struct psched_ratecfg *r)
1397{
1398	memset(res, 0, sizeof(*res));
1399
1400	/* legacy struct tc_ratespec has a 32bit @rate field
1401	 * Qdisc using 64bit rate should add new attributes
1402	 * in order to maintain compatibility.
1403	 */
1404	res->rate = min_t(u64, r->rate_bytes_ps, ~0U);
1405
1406	res->overhead = r->overhead;
1407	res->mpu = r->mpu;
1408	res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);
1409}
1410
1411struct psched_pktrate {
1412	u64	rate_pkts_ps; /* packets per second */
1413	u32	mult;
1414	u8	shift;
1415};
1416
1417static inline u64 psched_pkt2t_ns(const struct psched_pktrate *r,
1418				  unsigned int pkt_num)
1419{
1420	return ((u64)pkt_num * r->mult) >> r->shift;
1421}
1422
1423void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64);
1424
1425/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
1426 * The fast path only needs to access filter list and to update stats
1427 */
1428struct mini_Qdisc {
1429	struct tcf_proto *filter_list;
1430	struct tcf_block *block;
1431	struct gnet_stats_basic_sync __percpu *cpu_bstats;
1432	struct gnet_stats_queue	__percpu *cpu_qstats;
1433	unsigned long rcu_state;
1434};
1435
1436static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq,
1437						const struct sk_buff *skb)
1438{
1439	bstats_update(this_cpu_ptr(miniq->cpu_bstats), skb);
1440}
1441
1442static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq)
1443{
1444	this_cpu_inc(miniq->cpu_qstats->drops);
1445}
1446
1447struct mini_Qdisc_pair {
1448	struct mini_Qdisc miniq1;
1449	struct mini_Qdisc miniq2;
1450	struct mini_Qdisc __rcu **p_miniq;
1451};
1452
1453void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
1454			  struct tcf_proto *tp_head);
1455void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1456			  struct mini_Qdisc __rcu **p_miniq);
1457void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
1458				struct tcf_block *block);
1459
1460static inline bool mini_qdisc_pair_inited(struct mini_Qdisc_pair *miniqp)
1461{
1462	return !!miniqp->p_miniq;
1463}
1464
1465void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx);
1466
1467int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb));
1468
1469/* Make sure qdisc is no longer in SCHED state. */
1470static inline void qdisc_synchronize(const struct Qdisc *q)
1471{
1472	while (test_bit(__QDISC_STATE_SCHED, &q->state))
1473		msleep(1);
1474}
1475
1476#endif