sch_ets.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * net/sched/sch_ets.c         Enhanced Transmission Selection scheduler
  4 *
  5 * Description
  6 * -----------
  7 *
  8 * The Enhanced Transmission Selection scheduler is a classful queuing
  9 * discipline that merges functionality of PRIO and DRR qdiscs in one scheduler.
 10 * ETS makes it easy to configure a set of strict and bandwidth-sharing bands to
 11 * implement the transmission selection described in 802.1Qaz.
 12 *
 13 * Although ETS is technically classful, it's not possible to add and remove
 14 * classes at will. Instead one specifies number of classes, how many are
 15 * PRIO-like and how many DRR-like, and quanta for the latter.
 16 *
 17 * Algorithm
 18 * ---------
 19 *
 20 * The strict classes, if any, are tried for traffic first: first band 0, if it
 21 * has no traffic then band 1, etc.
 22 *
 23 * When there is no traffic in any of the strict queues, the bandwidth-sharing
 24 * ones are tried next. Each band is assigned a deficit counter, initialized to
 25 * "quantum" of that band. ETS maintains a list of active bandwidth-sharing
 26 * bands whose qdiscs are non-empty. A packet is dequeued from the band at the
 27 * head of the list if the packet size is smaller or equal to the deficit
 28 * counter. If the counter is too small, it is increased by "quantum" and the
 29 * scheduler moves on to the next band in the active list.
 30 */
 31
 32#include <linux/module.h>
 33#include <net/gen_stats.h>
 34#include <net/netlink.h>
 35#include <net/pkt_cls.h>
 36#include <net/pkt_sched.h>
 37#include <net/sch_generic.h>
 38
 39struct ets_class {
 40	struct list_head alist; /* In struct ets_sched.active. */
 41	struct Qdisc *qdisc;
 42	u32 quantum;
 43	u32 deficit;
 44	struct gnet_stats_basic_sync bstats;
 45	struct gnet_stats_queue qstats;
 46};
 47
 48struct ets_sched {
 49	struct list_head active;
 50	struct tcf_proto __rcu *filter_list;
 51	struct tcf_block *block;
 52	unsigned int nbands;
 53	unsigned int nstrict;
 54	u8 prio2band[TC_PRIO_MAX + 1];
 55	struct ets_class classes[TCQ_ETS_MAX_BANDS];
 56};
 57
 58static const struct nla_policy ets_policy[TCA_ETS_MAX + 1] = {
 59	[TCA_ETS_NBANDS] = { .type = NLA_U8 },
 60	[TCA_ETS_NSTRICT] = { .type = NLA_U8 },
 61	[TCA_ETS_QUANTA] = { .type = NLA_NESTED },
 62	[TCA_ETS_PRIOMAP] = { .type = NLA_NESTED },
 63};
 64
 65static const struct nla_policy ets_priomap_policy[TCA_ETS_MAX + 1] = {
 66	[TCA_ETS_PRIOMAP_BAND] = { .type = NLA_U8 },
 67};
 68
 69static const struct nla_policy ets_quanta_policy[TCA_ETS_MAX + 1] = {
 70	[TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 },
 71};
 72
 73static const struct nla_policy ets_class_policy[TCA_ETS_MAX + 1] = {
 74	[TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 },
 75};
 76
 77static bool cl_is_active(struct ets_class *cl)
 78{
 79	return !list_empty(&cl->alist);
 80}
 81
 82static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr,
 83			     unsigned int *quantum,
 84			     struct netlink_ext_ack *extack)
 85{
 86	*quantum = nla_get_u32(attr);
 87	if (!*quantum) {
 88		NL_SET_ERR_MSG(extack, "ETS quantum cannot be zero");
 89		return -EINVAL;
 90	}
 91	return 0;
 92}
 93
 94static struct ets_class *
 95ets_class_from_arg(struct Qdisc *sch, unsigned long arg)
 96{
 97	struct ets_sched *q = qdisc_priv(sch);
 98
 99	if (arg == 0 || arg > q->nbands)
100		return NULL;
101	return &q->classes[arg - 1];
102}
103
104static u32 ets_class_id(struct Qdisc *sch, const struct ets_class *cl)
105{
106	struct ets_sched *q = qdisc_priv(sch);
107	int band = cl - q->classes;
108
109	return TC_H_MAKE(sch->handle, band + 1);
110}
111
112static void ets_offload_change(struct Qdisc *sch)
113{
114	struct net_device *dev = qdisc_dev(sch);
115	struct ets_sched *q = qdisc_priv(sch);
116	struct tc_ets_qopt_offload qopt;
117	unsigned int w_psum_prev = 0;
118	unsigned int quantum;
119	unsigned int w_psum;
120	unsigned int weight;
121	unsigned int i;
122	u64 q_psum = 0;
123	u64 q_sum = 0;
124
125	if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
126		return;
127
128	qopt.command = TC_ETS_REPLACE;
129	qopt.handle = sch->handle;
130	qopt.parent = sch->parent;
131	qopt.replace_params.bands = q->nbands;
132	qopt.replace_params.qstats = &sch->qstats;
133	memcpy(&qopt.replace_params.priomap,
134	       q->prio2band, sizeof(q->prio2band));
135
136	for (i = 0; i < q->nbands; i++)
137		q_sum += q->classes[i].quantum;
138
139	for (i = 0; i < q->nbands; i++) {
140		quantum = q->classes[i].quantum;
141		if (quantum) {
142			q_psum += quantum;
143			w_psum = div64_u64(q_psum * 100, q_sum);
144		} else {
145			w_psum = 0;
146		}
147		weight = w_psum - w_psum_prev;
148		w_psum_prev = w_psum;
149
150		qopt.replace_params.quanta[i] = quantum;
151		qopt.replace_params.weights[i] = weight;
152	}
153
154	dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt);
155}
156
157static void ets_offload_destroy(struct Qdisc *sch)
158{
159	struct net_device *dev = qdisc_dev(sch);
160	struct tc_ets_qopt_offload qopt;
161
162	if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
163		return;
164
165	qopt.command = TC_ETS_DESTROY;
166	qopt.handle = sch->handle;
167	qopt.parent = sch->parent;
168	dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt);
169}
170
171static void ets_offload_graft(struct Qdisc *sch, struct Qdisc *new,
172			      struct Qdisc *old, unsigned long arg,
173			      struct netlink_ext_ack *extack)
174{
175	struct net_device *dev = qdisc_dev(sch);
176	struct tc_ets_qopt_offload qopt;
177
178	qopt.command = TC_ETS_GRAFT;
179	qopt.handle = sch->handle;
180	qopt.parent = sch->parent;
181	qopt.graft_params.band = arg - 1;
182	qopt.graft_params.child_handle = new->handle;
183
184	qdisc_offload_graft_helper(dev, sch, new, old, TC_SETUP_QDISC_ETS,
185				   &qopt, extack);
186}
187
188static int ets_offload_dump(struct Qdisc *sch)
189{
190	struct tc_ets_qopt_offload qopt;
191
192	qopt.command = TC_ETS_STATS;
193	qopt.handle = sch->handle;
194	qopt.parent = sch->parent;
195	qopt.stats.bstats = &sch->bstats;
196	qopt.stats.qstats = &sch->qstats;
197
198	return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_ETS, &qopt);
199}
200
201static bool ets_class_is_strict(struct ets_sched *q, const struct ets_class *cl)
202{
203	unsigned int band = cl - q->classes;
204
205	return band < q->nstrict;
206}
207
208static int ets_class_change(struct Qdisc *sch, u32 classid, u32 parentid,
209			    struct nlattr **tca, unsigned long *arg,
210			    struct netlink_ext_ack *extack)
211{
212	struct ets_class *cl = ets_class_from_arg(sch, *arg);
213	struct ets_sched *q = qdisc_priv(sch);
214	struct nlattr *opt = tca[TCA_OPTIONS];
215	struct nlattr *tb[TCA_ETS_MAX + 1];
216	unsigned int quantum;
217	int err;
218
219	/* Classes can be added and removed only through Qdisc_ops.change
220	 * interface.
221	 */
222	if (!cl) {
223		NL_SET_ERR_MSG(extack, "Fine-grained class addition and removal is not supported");
224		return -EOPNOTSUPP;
225	}
226
227	if (!opt) {
228		NL_SET_ERR_MSG(extack, "ETS options are required for this operation");
229		return -EINVAL;
230	}
231
232	err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_class_policy, extack);
233	if (err < 0)
234		return err;
235
236	if (!tb[TCA_ETS_QUANTA_BAND])
237		/* Nothing to configure. */
238		return 0;
239
240	if (ets_class_is_strict(q, cl)) {
241		NL_SET_ERR_MSG(extack, "Strict bands do not have a configurable quantum");
242		return -EINVAL;
243	}
244
245	err = ets_quantum_parse(sch, tb[TCA_ETS_QUANTA_BAND], &quantum,
246				extack);
247	if (err)
248		return err;
249
250	sch_tree_lock(sch);
251	cl->quantum = quantum;
252	sch_tree_unlock(sch);
253
254	ets_offload_change(sch);
255	return 0;
256}
257
258static int ets_class_graft(struct Qdisc *sch, unsigned long arg,
259			   struct Qdisc *new, struct Qdisc **old,
260			   struct netlink_ext_ack *extack)
261{
262	struct ets_class *cl = ets_class_from_arg(sch, arg);
263
264	if (!new) {
265		new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
266					ets_class_id(sch, cl), NULL);
267		if (!new)
268			new = &noop_qdisc;
269		else
270			qdisc_hash_add(new, true);
271	}
272
273	*old = qdisc_replace(sch, new, &cl->qdisc);
274	ets_offload_graft(sch, new, *old, arg, extack);
275	return 0;
276}
277
278static struct Qdisc *ets_class_leaf(struct Qdisc *sch, unsigned long arg)
279{
280	struct ets_class *cl = ets_class_from_arg(sch, arg);
281
282	return cl->qdisc;
283}
284
285static unsigned long ets_class_find(struct Qdisc *sch, u32 classid)
286{
287	unsigned long band = TC_H_MIN(classid);
288	struct ets_sched *q = qdisc_priv(sch);
289
290	if (band - 1 >= q->nbands)
291		return 0;
292	return band;
293}
294
295static void ets_class_qlen_notify(struct Qdisc *sch, unsigned long arg)
296{
297	struct ets_class *cl = ets_class_from_arg(sch, arg);
298	struct ets_sched *q = qdisc_priv(sch);
299
300	/* We get notified about zero-length child Qdiscs as well if they are
301	 * offloaded. Those aren't on the active list though, so don't attempt
302	 * to remove them.
303	 */
304	if (!ets_class_is_strict(q, cl) && sch->q.qlen)
305		list_del_init(&cl->alist);
306}
307
308static int ets_class_dump(struct Qdisc *sch, unsigned long arg,
309			  struct sk_buff *skb, struct tcmsg *tcm)
310{
311	struct ets_class *cl = ets_class_from_arg(sch, arg);
312	struct ets_sched *q = qdisc_priv(sch);
313	struct nlattr *nest;
314
315	tcm->tcm_parent = TC_H_ROOT;
316	tcm->tcm_handle = ets_class_id(sch, cl);
317	tcm->tcm_info = cl->qdisc->handle;
318
319	nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
320	if (!nest)
321		goto nla_put_failure;
322	if (!ets_class_is_strict(q, cl)) {
323		if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, cl->quantum))
324			goto nla_put_failure;
325	}
326	return nla_nest_end(skb, nest);
327
328nla_put_failure:
329	nla_nest_cancel(skb, nest);
330	return -EMSGSIZE;
331}
332
333static int ets_class_dump_stats(struct Qdisc *sch, unsigned long arg,
334				struct gnet_dump *d)
335{
336	struct ets_class *cl = ets_class_from_arg(sch, arg);
337	struct Qdisc *cl_q = cl->qdisc;
338
339	if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats, true) < 0 ||
340	    qdisc_qstats_copy(d, cl_q) < 0)
341		return -1;
342
343	return 0;
344}
345
346static void ets_qdisc_walk(struct Qdisc *sch, struct qdisc_walker *arg)
347{
348	struct ets_sched *q = qdisc_priv(sch);
349	int i;
350
351	if (arg->stop)
352		return;
353
354	for (i = 0; i < q->nbands; i++) {
355		if (!tc_qdisc_stats_dump(sch, i + 1, arg))
356			break;
357	}
358}
359
360static struct tcf_block *
361ets_qdisc_tcf_block(struct Qdisc *sch, unsigned long cl,
362		    struct netlink_ext_ack *extack)
363{
364	struct ets_sched *q = qdisc_priv(sch);
365
366	if (cl) {
367		NL_SET_ERR_MSG(extack, "ETS classid must be zero");
368		return NULL;
369	}
370
371	return q->block;
372}
373
374static unsigned long ets_qdisc_bind_tcf(struct Qdisc *sch, unsigned long parent,
375					u32 classid)
376{
377	return ets_class_find(sch, classid);
378}
379
380static void ets_qdisc_unbind_tcf(struct Qdisc *sch, unsigned long arg)
381{
382}
383
384static struct ets_class *ets_classify(struct sk_buff *skb, struct Qdisc *sch,
385				      int *qerr)
386{
387	struct ets_sched *q = qdisc_priv(sch);
388	u32 band = skb->priority;
389	struct tcf_result res;
390	struct tcf_proto *fl;
391	int err;
392
393	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
394	if (TC_H_MAJ(skb->priority) != sch->handle) {
395		fl = rcu_dereference_bh(q->filter_list);
396		err = tcf_classify(skb, NULL, fl, &res, false);
397#ifdef CONFIG_NET_CLS_ACT
398		switch (err) {
399		case TC_ACT_STOLEN:
400		case TC_ACT_QUEUED:
401		case TC_ACT_TRAP:
402			*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
403			fallthrough;
404		case TC_ACT_SHOT:
405			return NULL;
406		}
407#endif
408		if (!fl || err < 0) {
409			if (TC_H_MAJ(band))
410				band = 0;
411			return &q->classes[q->prio2band[band & TC_PRIO_MAX]];
412		}
413		band = res.classid;
414	}
415	band = TC_H_MIN(band) - 1;
416	if (band >= q->nbands)
417		return &q->classes[q->prio2band[0]];
418	return &q->classes[band];
419}
420
421static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
422			     struct sk_buff **to_free)
423{
424	unsigned int len = qdisc_pkt_len(skb);
425	struct ets_sched *q = qdisc_priv(sch);
426	struct ets_class *cl;
427	int err = 0;
428
429	cl = ets_classify(skb, sch, &err);
430	if (!cl) {
431		if (err & __NET_XMIT_BYPASS)
432			qdisc_qstats_drop(sch);
433		__qdisc_drop(skb, to_free);
434		return err;
435	}
436
437	err = qdisc_enqueue(skb, cl->qdisc, to_free);
438	if (unlikely(err != NET_XMIT_SUCCESS)) {
439		if (net_xmit_drop_count(err)) {
440			cl->qstats.drops++;
441			qdisc_qstats_drop(sch);
442		}
443		return err;
444	}
445
446	if (!cl_is_active(cl) && !ets_class_is_strict(q, cl)) {
447		list_add_tail(&cl->alist, &q->active);
448		cl->deficit = cl->quantum;
449	}
450
451	sch->qstats.backlog += len;
452	sch->q.qlen++;
453	return err;
454}
455
456static struct sk_buff *
457ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb)
458{
459	qdisc_bstats_update(sch, skb);
460	qdisc_qstats_backlog_dec(sch, skb);
461	sch->q.qlen--;
462	return skb;
463}
464
465static struct sk_buff *ets_qdisc_dequeue(struct Qdisc *sch)
466{
467	struct ets_sched *q = qdisc_priv(sch);
468	struct ets_class *cl;
469	struct sk_buff *skb;
470	unsigned int band;
471	unsigned int len;
472
473	while (1) {
474		for (band = 0; band < q->nstrict; band++) {
475			cl = &q->classes[band];
476			skb = qdisc_dequeue_peeked(cl->qdisc);
477			if (skb)
478				return ets_qdisc_dequeue_skb(sch, skb);
479		}
480
481		if (list_empty(&q->active))
482			goto out;
483
484		cl = list_first_entry(&q->active, struct ets_class, alist);
485		skb = cl->qdisc->ops->peek(cl->qdisc);
486		if (!skb) {
487			qdisc_warn_nonwc(__func__, cl->qdisc);
488			goto out;
489		}
490
491		len = qdisc_pkt_len(skb);
492		if (len <= cl->deficit) {
493			cl->deficit -= len;
494			skb = qdisc_dequeue_peeked(cl->qdisc);
495			if (unlikely(!skb))
496				goto out;
497			if (cl->qdisc->q.qlen == 0)
498				list_del_init(&cl->alist);
499			return ets_qdisc_dequeue_skb(sch, skb);
500		}
501
502		cl->deficit += cl->quantum;
503		list_move_tail(&cl->alist, &q->active);
504	}
505out:
506	return NULL;
507}
508
509static int ets_qdisc_priomap_parse(struct nlattr *priomap_attr,
510				   unsigned int nbands, u8 *priomap,
511				   struct netlink_ext_ack *extack)
512{
513	const struct nlattr *attr;
514	int prio = 0;
515	u8 band;
516	int rem;
517	int err;
518
519	err = __nla_validate_nested(priomap_attr, TCA_ETS_MAX,
520				    ets_priomap_policy, NL_VALIDATE_STRICT,
521				    extack);
522	if (err)
523		return err;
524
525	nla_for_each_nested(attr, priomap_attr, rem) {
526		switch (nla_type(attr)) {
527		case TCA_ETS_PRIOMAP_BAND:
528			if (prio > TC_PRIO_MAX) {
529				NL_SET_ERR_MSG_MOD(extack, "Too many priorities in ETS priomap");
530				return -EINVAL;
531			}
532			band = nla_get_u8(attr);
533			if (band >= nbands) {
534				NL_SET_ERR_MSG_MOD(extack, "Invalid band number in ETS priomap");
535				return -EINVAL;
536			}
537			priomap[prio++] = band;
538			break;
539		default:
540			WARN_ON_ONCE(1); /* Validate should have caught this. */
541			return -EINVAL;
542		}
543	}
544
545	return 0;
546}
547
548static int ets_qdisc_quanta_parse(struct Qdisc *sch, struct nlattr *quanta_attr,
549				  unsigned int nbands, unsigned int nstrict,
550				  unsigned int *quanta,
551				  struct netlink_ext_ack *extack)
552{
553	const struct nlattr *attr;
554	int band = nstrict;
555	int rem;
556	int err;
557
558	err = __nla_validate_nested(quanta_attr, TCA_ETS_MAX,
559				    ets_quanta_policy, NL_VALIDATE_STRICT,
560				    extack);
561	if (err < 0)
562		return err;
563
564	nla_for_each_nested(attr, quanta_attr, rem) {
565		switch (nla_type(attr)) {
566		case TCA_ETS_QUANTA_BAND:
567			if (band >= nbands) {
568				NL_SET_ERR_MSG_MOD(extack, "ETS quanta has more values than bands");
569				return -EINVAL;
570			}
571			err = ets_quantum_parse(sch, attr, &quanta[band++],
572						extack);
573			if (err)
574				return err;
575			break;
576		default:
577			WARN_ON_ONCE(1); /* Validate should have caught this. */
578			return -EINVAL;
579		}
580	}
581
582	return 0;
583}
584
585static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt,
586			    struct netlink_ext_ack *extack)
587{
588	unsigned int quanta[TCQ_ETS_MAX_BANDS] = {0};
589	struct Qdisc *queues[TCQ_ETS_MAX_BANDS];
590	struct ets_sched *q = qdisc_priv(sch);
591	struct nlattr *tb[TCA_ETS_MAX + 1];
592	unsigned int oldbands = q->nbands;
593	u8 priomap[TC_PRIO_MAX + 1];
594	unsigned int nstrict = 0;
595	unsigned int nbands;
596	unsigned int i;
597	int err;
598
599	err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_policy, extack);
600	if (err < 0)
601		return err;
602
603	if (!tb[TCA_ETS_NBANDS]) {
604		NL_SET_ERR_MSG_MOD(extack, "Number of bands is a required argument");
605		return -EINVAL;
606	}
607	nbands = nla_get_u8(tb[TCA_ETS_NBANDS]);
608	if (nbands < 1 || nbands > TCQ_ETS_MAX_BANDS) {
609		NL_SET_ERR_MSG_MOD(extack, "Invalid number of bands");
610		return -EINVAL;
611	}
612	/* Unless overridden, traffic goes to the last band. */
613	memset(priomap, nbands - 1, sizeof(priomap));
614
615	if (tb[TCA_ETS_NSTRICT]) {
616		nstrict = nla_get_u8(tb[TCA_ETS_NSTRICT]);
617		if (nstrict > nbands) {
618			NL_SET_ERR_MSG_MOD(extack, "Invalid number of strict bands");
619			return -EINVAL;
620		}
621	}
622
623	if (tb[TCA_ETS_PRIOMAP]) {
624		err = ets_qdisc_priomap_parse(tb[TCA_ETS_PRIOMAP],
625					      nbands, priomap, extack);
626		if (err)
627			return err;
628	}
629
630	if (tb[TCA_ETS_QUANTA]) {
631		err = ets_qdisc_quanta_parse(sch, tb[TCA_ETS_QUANTA],
632					     nbands, nstrict, quanta, extack);
633		if (err)
634			return err;
635	}
636	/* If there are more bands than strict + quanta provided, the remaining
637	 * ones are ETS with quantum of MTU. Initialize the missing values here.
638	 */
639	for (i = nstrict; i < nbands; i++) {
640		if (!quanta[i])
641			quanta[i] = psched_mtu(qdisc_dev(sch));
642	}
643
644	/* Before commit, make sure we can allocate all new qdiscs */
645	for (i = oldbands; i < nbands; i++) {
646		queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
647					      ets_class_id(sch, &q->classes[i]),
648					      extack);
649		if (!queues[i]) {
650			while (i > oldbands)
651				qdisc_put(queues[--i]);
652			return -ENOMEM;
653		}
654	}
655
656	sch_tree_lock(sch);
657
658	for (i = nbands; i < oldbands; i++) {
659		if (cl_is_active(&q->classes[i]))
660			list_del_init(&q->classes[i].alist);
661		qdisc_purge_queue(q->classes[i].qdisc);
662	}
663
664	WRITE_ONCE(q->nbands, nbands);
665	for (i = nstrict; i < q->nstrict; i++) {
666		if (q->classes[i].qdisc->q.qlen) {
667			list_add_tail(&q->classes[i].alist, &q->active);
668			q->classes[i].deficit = quanta[i];
669		}
670	}
671	for (i = q->nstrict; i < nstrict; i++) {
672		if (cl_is_active(&q->classes[i]))
673			list_del_init(&q->classes[i].alist);
674	}
675	WRITE_ONCE(q->nstrict, nstrict);
676	memcpy(q->prio2band, priomap, sizeof(priomap));
677
678	for (i = 0; i < q->nbands; i++)
679		WRITE_ONCE(q->classes[i].quantum, quanta[i]);
680
681	for (i = oldbands; i < q->nbands; i++) {
682		q->classes[i].qdisc = queues[i];
683		if (q->classes[i].qdisc != &noop_qdisc)
684			qdisc_hash_add(q->classes[i].qdisc, true);
685	}
686
687	sch_tree_unlock(sch);
688
689	ets_offload_change(sch);
690	for (i = q->nbands; i < oldbands; i++) {
691		qdisc_put(q->classes[i].qdisc);
692		q->classes[i].qdisc = NULL;
693		WRITE_ONCE(q->classes[i].quantum, 0);
694		q->classes[i].deficit = 0;
695		gnet_stats_basic_sync_init(&q->classes[i].bstats);
696		memset(&q->classes[i].qstats, 0, sizeof(q->classes[i].qstats));
697	}
698	return 0;
699}
700
701static int ets_qdisc_init(struct Qdisc *sch, struct nlattr *opt,
702			  struct netlink_ext_ack *extack)
703{
704	struct ets_sched *q = qdisc_priv(sch);
705	int err, i;
706
707	if (!opt)
708		return -EINVAL;
709
710	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
711	if (err)
712		return err;
713
714	INIT_LIST_HEAD(&q->active);
715	for (i = 0; i < TCQ_ETS_MAX_BANDS; i++)
716		INIT_LIST_HEAD(&q->classes[i].alist);
717
718	return ets_qdisc_change(sch, opt, extack);
719}
720
721static void ets_qdisc_reset(struct Qdisc *sch)
722{
723	struct ets_sched *q = qdisc_priv(sch);
724	int band;
725
726	for (band = q->nstrict; band < q->nbands; band++) {
727		if (q->classes[band].qdisc->q.qlen)
728			list_del_init(&q->classes[band].alist);
729	}
730	for (band = 0; band < q->nbands; band++)
731		qdisc_reset(q->classes[band].qdisc);
732}
733
734static void ets_qdisc_destroy(struct Qdisc *sch)
735{
736	struct ets_sched *q = qdisc_priv(sch);
737	int band;
738
739	ets_offload_destroy(sch);
740	tcf_block_put(q->block);
741	for (band = 0; band < q->nbands; band++)
742		qdisc_put(q->classes[band].qdisc);
743}
744
745static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb)
746{
747	struct ets_sched *q = qdisc_priv(sch);
748	struct nlattr *opts;
749	struct nlattr *nest;
750	u8 nbands, nstrict;
751	int band;
752	int prio;
753	int err;
754
755	err = ets_offload_dump(sch);
756	if (err)
757		return err;
758
759	opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
760	if (!opts)
761		goto nla_err;
762
763	nbands = READ_ONCE(q->nbands);
764	if (nla_put_u8(skb, TCA_ETS_NBANDS, nbands))
765		goto nla_err;
766
767	nstrict = READ_ONCE(q->nstrict);
768	if (nstrict && nla_put_u8(skb, TCA_ETS_NSTRICT, nstrict))
769		goto nla_err;
770
771	if (nbands > nstrict) {
772		nest = nla_nest_start(skb, TCA_ETS_QUANTA);
773		if (!nest)
774			goto nla_err;
775
776		for (band = nstrict; band < nbands; band++) {
777			if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND,
778					READ_ONCE(q->classes[band].quantum)))
779				goto nla_err;
780		}
781
782		nla_nest_end(skb, nest);
783	}
784
785	nest = nla_nest_start(skb, TCA_ETS_PRIOMAP);
786	if (!nest)
787		goto nla_err;
788
789	for (prio = 0; prio <= TC_PRIO_MAX; prio++) {
790		if (nla_put_u8(skb, TCA_ETS_PRIOMAP_BAND,
791			       READ_ONCE(q->prio2band[prio])))
792			goto nla_err;
793	}
794
795	nla_nest_end(skb, nest);
796
797	return nla_nest_end(skb, opts);
798
799nla_err:
800	nla_nest_cancel(skb, opts);
801	return -EMSGSIZE;
802}
803
804static const struct Qdisc_class_ops ets_class_ops = {
805	.change		= ets_class_change,
806	.graft		= ets_class_graft,
807	.leaf		= ets_class_leaf,
808	.find		= ets_class_find,
809	.qlen_notify	= ets_class_qlen_notify,
810	.dump		= ets_class_dump,
811	.dump_stats	= ets_class_dump_stats,
812	.walk		= ets_qdisc_walk,
813	.tcf_block	= ets_qdisc_tcf_block,
814	.bind_tcf	= ets_qdisc_bind_tcf,
815	.unbind_tcf	= ets_qdisc_unbind_tcf,
816};
817
818static struct Qdisc_ops ets_qdisc_ops __read_mostly = {
819	.cl_ops		= &ets_class_ops,
820	.id		= "ets",
821	.priv_size	= sizeof(struct ets_sched),
822	.enqueue	= ets_qdisc_enqueue,
823	.dequeue	= ets_qdisc_dequeue,
824	.peek		= qdisc_peek_dequeued,
825	.change		= ets_qdisc_change,
826	.init		= ets_qdisc_init,
827	.reset		= ets_qdisc_reset,
828	.destroy	= ets_qdisc_destroy,
829	.dump		= ets_qdisc_dump,
830	.owner		= THIS_MODULE,
831};
832MODULE_ALIAS_NET_SCH("ets");
833
834static int __init ets_init(void)
835{
836	return register_qdisc(&ets_qdisc_ops);
837}
838
839static void __exit ets_exit(void)
840{
841	unregister_qdisc(&ets_qdisc_ops);
842}
843
844module_init(ets_init);
845module_exit(ets_exit);
846MODULE_LICENSE("GPL");
847MODULE_DESCRIPTION("Enhanced Transmission Selection(ETS) scheduler");