qos.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
  3#include <net/sch_generic.h>
  4
  5#include <net/pkt_cls.h>
  6#include "en.h"
  7#include "params.h"
  8#include "../qos.h"
  9#include "en/htb.h"
 10
 11struct qos_sq_callback_params {
 12	struct mlx5e_priv *priv;
 13	struct mlx5e_channels *chs;
 14};
 15
 16int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes)
 17{
 18	if (nbytes < BYTES_IN_MBIT) {
 19		qos_warn(mdev, "Input rate (%llu Bytes/sec) below minimum supported (%u Bytes/sec)\n",
 20			 nbytes, BYTES_IN_MBIT);
 21		return -EINVAL;
 22	}
 23	return 0;
 24}
 25
 26static u32 mlx5e_qos_bytes2mbits(struct mlx5_core_dev *mdev, u64 nbytes)
 27{
 28	return div_u64(nbytes, BYTES_IN_MBIT);
 29}
 30
 31int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev)
 32{
 33	return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev));
 34}
 35
 36/* TX datapath API */
 37
 38u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid)
 39{
 40	/* These channel params are safe to access from the datapath, because:
 41	 * 1. This function is called only after checking selq->htb_maj_id != 0,
 42	 *    and the number of queues can't change while HTB offload is active.
 43	 * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for
 44	 *    mlx5e_select_queue to finish while holding priv->state_lock,
 45	 *    preventing other code from changing the number of queues.
 46	 */
 47	bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS);
 48
 49	return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid;
 50}
 51
 52/* SQ lifecycle */
 53
 54static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid)
 55{
 56	struct mlx5e_params *params = &priv->channels.params;
 57	struct mlx5e_txqsq __rcu **qos_sqs;
 58	struct mlx5e_channel *c;
 59	int ix;
 60
 61	ix = qid % params->num_channels;
 62	qid /= params->num_channels;
 63	c = priv->channels.c[ix];
 64
 65	qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
 66	return mlx5e_state_dereference(priv, qos_sqs[qid]);
 67}
 68
 69int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs,
 70		      u16 node_qid, u32 hw_id)
 71{
 72	struct mlx5e_create_cq_param ccp = {};
 73	struct mlx5e_txqsq __rcu **qos_sqs;
 74	struct mlx5e_sq_param param_sq;
 75	struct mlx5e_cq_param param_cq;
 76	int txq_ix, ix, qid, err = 0;
 77	struct mlx5e_params *params;
 78	struct mlx5e_channel *c;
 79	struct mlx5e_txqsq *sq;
 80	u32 tisn;
 81
 82	params = &chs->params;
 83
 84	txq_ix = mlx5e_qid_from_qos(chs, node_qid);
 85
 86	WARN_ON(node_qid >= mlx5e_htb_cur_leaf_nodes(priv->htb));
 87	if (!priv->htb_qos_sq_stats) {
 88		struct mlx5e_sq_stats **stats_list;
 89
 90		stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev),
 91				      sizeof(*stats_list), GFP_KERNEL);
 92		if (!stats_list)
 93			return -ENOMEM;
 94
 95		WRITE_ONCE(priv->htb_qos_sq_stats, stats_list);
 96	}
 97
 98	if (!priv->htb_qos_sq_stats[node_qid]) {
 99		struct mlx5e_sq_stats *stats;
100
101		stats = kzalloc(sizeof(*stats), GFP_KERNEL);
102		if (!stats)
103			return -ENOMEM;
104
105		WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats);
106		/* Order htb_max_qos_sqs increment after writing the array pointer.
107		 * Pairs with smp_load_acquire in en_stats.c.
108		 */
109		smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1);
110	}
111
112	ix = node_qid % params->num_channels;
113	qid = node_qid / params->num_channels;
114	c = chs->c[ix];
115
116	qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
117	sq = kzalloc(sizeof(*sq), GFP_KERNEL);
118
119	if (!sq)
120		return -ENOMEM;
121
122	mlx5e_build_create_cq_param(&ccp, c);
123
124	memset(&param_sq, 0, sizeof(param_sq));
125	memset(&param_cq, 0, sizeof(param_cq));
126	mlx5e_build_sq_param(c->mdev, params, &param_sq);
127	mlx5e_build_tx_cq_param(c->mdev, params, &param_cq);
128	err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &param_cq, &ccp, &sq->cq);
129	if (err)
130		goto err_free_sq;
131
132	tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile,
133				      c->lag_port, 0);
134	err = mlx5e_open_txqsq(c, tisn, txq_ix, params, &param_sq, sq, 0, hw_id,
135			       priv->htb_qos_sq_stats[node_qid]);
136	if (err)
137		goto err_close_cq;
138
139	rcu_assign_pointer(qos_sqs[qid], sq);
140
141	return 0;
142
143err_close_cq:
144	mlx5e_close_cq(&sq->cq);
145err_free_sq:
146	kfree(sq);
147	return err;
148}
149
150static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id)
151{
152	struct qos_sq_callback_params *cb_params = data;
153
154	return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id);
155}
156
157int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id)
158{
159	struct mlx5e_priv *priv = data;
160	struct mlx5e_txqsq *sq;
161	u16 qid;
162
163	sq = mlx5e_get_qos_sq(priv, node_qid);
164
165	qid = mlx5e_qid_from_qos(&priv->channels, node_qid);
166
167	/* If it's a new queue, it will be marked as started at this point.
168	 * Stop it before updating txq2sq.
169	 */
170	mlx5e_tx_disable_queue(netdev_get_tx_queue(priv->netdev, qid));
171
172	priv->txq2sq[qid] = sq;
173	priv->txq2sq_stats[qid] = sq->stats;
174
175	/* Make the change to txq2sq visible before the queue is started.
176	 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
177	 * which pairs with this barrier.
178	 */
179	smp_wmb();
180
181	qos_dbg(sq->mdev, "Activate QoS SQ qid %u\n", node_qid);
182	mlx5e_activate_txqsq(sq);
183
184	return 0;
185}
186
187void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid)
188{
189	struct mlx5e_txqsq *sq;
190	u16 txq_ix;
191
192	sq = mlx5e_get_qos_sq(priv, qid);
193	if (!sq) /* Handle the case when the SQ failed to open. */
194		return;
195
196	qos_dbg(sq->mdev, "Deactivate QoS SQ qid %u\n", qid);
197	mlx5e_deactivate_txqsq(sq);
198
199	txq_ix = mlx5e_qid_from_qos(&priv->channels, qid);
200
201	priv->txq2sq[txq_ix] = NULL;
202	priv->txq2sq_stats[txq_ix] = NULL;
203
204	/* Make the change to txq2sq visible before the queue is started again.
205	 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
206	 * which pairs with this barrier.
207	 */
208	smp_wmb();
209}
210
211void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid)
212{
213	struct mlx5e_txqsq __rcu **qos_sqs;
214	struct mlx5e_params *params;
215	struct mlx5e_channel *c;
216	struct mlx5e_txqsq *sq;
217	int ix;
218
219	params = &priv->channels.params;
220
221	ix = qid % params->num_channels;
222	qid /= params->num_channels;
223	c = priv->channels.c[ix];
224	qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs);
225	sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock));
226	if (!sq) /* Handle the case when the SQ failed to open. */
227		return;
228
229	synchronize_rcu(); /* Sync with NAPI. */
230
231	mlx5e_close_txqsq(sq);
232	mlx5e_close_cq(&sq->cq);
233	kfree(sq);
234}
235
236void mlx5e_qos_close_queues(struct mlx5e_channel *c)
237{
238	struct mlx5e_txqsq __rcu **qos_sqs;
239	int i;
240
241	qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock));
242	if (!qos_sqs)
243		return;
244	synchronize_rcu(); /* Sync with NAPI. */
245
246	for (i = 0; i < c->qos_sqs_size; i++) {
247		struct mlx5e_txqsq *sq;
248
249		sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
250		if (!sq) /* Handle the case when the SQ failed to open. */
251			continue;
252
253		mlx5e_close_txqsq(sq);
254		mlx5e_close_cq(&sq->cq);
255		kfree(sq);
256	}
257
258	kvfree(qos_sqs);
259}
260
261void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs)
262{
263	int i;
264
265	for (i = 0; i < chs->num; i++)
266		mlx5e_qos_close_queues(chs->c[i]);
267}
268
269int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
270{
271	u16 qos_sqs_size;
272	int i;
273
274	qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num);
275
276	for (i = 0; i < chs->num; i++) {
277		struct mlx5e_txqsq **sqs;
278
279		sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL);
280		if (!sqs)
281			goto err_free;
282
283		WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size);
284		smp_wmb(); /* Pairs with mlx5e_napi_poll. */
285		rcu_assign_pointer(chs->c[i]->qos_sqs, sqs);
286	}
287
288	return 0;
289
290err_free:
291	while (--i >= 0) {
292		struct mlx5e_txqsq **sqs;
293
294		sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL,
295					  lockdep_is_held(&priv->state_lock));
296
297		synchronize_rcu(); /* Sync with NAPI. */
298		kvfree(sqs);
299	}
300	return -ENOMEM;
301}
302
303int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
304{
305	struct qos_sq_callback_params callback_params;
306	int err;
307
308	err = mlx5e_qos_alloc_queues(priv, chs);
309	if (err)
310		return err;
311
312	callback_params.priv = priv;
313	callback_params.chs = chs;
314
315	err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params);
316	if (err) {
317		mlx5e_qos_close_all_queues(chs);
318		return err;
319	}
320
321	return 0;
322}
323
324void mlx5e_qos_activate_queues(struct mlx5e_priv *priv)
325{
326	mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv);
327}
328
329void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c)
330{
331	struct mlx5e_params *params = &c->priv->channels.params;
332	struct mlx5e_txqsq __rcu **qos_sqs;
333	u16 txq_ix;
334	int i;
335
336	qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs);
337	if (!qos_sqs)
338		return;
339
340	for (i = 0; i < c->qos_sqs_size; i++) {
341		u16 qid = params->num_channels * i + c->ix;
342		struct mlx5e_txqsq *sq;
343
344		sq = mlx5e_state_dereference(c->priv, qos_sqs[i]);
345		if (!sq) /* Handle the case when the SQ failed to open. */
346			continue;
347
348		qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid);
349		mlx5e_deactivate_txqsq(sq);
350
351		txq_ix = mlx5e_qid_from_qos(&c->priv->channels, qid);
352
353		/* The queue is disabled, no synchronization with datapath is needed. */
354		c->priv->txq2sq[txq_ix] = NULL;
355		c->priv->txq2sq_stats[txq_ix] = NULL;
356	}
357}
358
359void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs)
360{
361	int i;
362
363	for (i = 0; i < chs->num; i++)
364		mlx5e_qos_deactivate_queues(chs->c[i]);
365}
366
367void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq)
368{
369	qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid);
370	netdev_tx_reset_queue(txq);
371	netif_tx_start_queue(txq);
372}
373
374void mlx5e_reset_qdisc(struct net_device *dev, u16 qid)
375{
376	struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid);
377	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
378
379	if (!qdisc)
380		return;
381
382	spin_lock_bh(qdisc_lock(qdisc));
383	qdisc_reset(qdisc);
384	spin_unlock_bh(qdisc_lock(qdisc));
385}
386
387int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt)
388{
389	struct mlx5e_htb *htb = priv->htb;
390	int res;
391
392	if (!htb && htb_qopt->command != TC_HTB_CREATE)
393		return -EINVAL;
394
395	if (htb_qopt->prio || htb_qopt->quantum) {
396		NL_SET_ERR_MSG_MOD(htb_qopt->extack,
397				   "prio and quantum parameters are not supported by device with HTB offload enabled.");
398		return -EOPNOTSUPP;
399	}
400
401	switch (htb_qopt->command) {
402	case TC_HTB_CREATE:
403		if (!mlx5_qos_is_supported(priv->mdev)) {
404			NL_SET_ERR_MSG_MOD(htb_qopt->extack,
405					   "Missing QoS capabilities. Try disabling SRIOV or use a supported device.");
406			return -EOPNOTSUPP;
407		}
408		priv->htb = mlx5e_htb_alloc();
409		htb = priv->htb;
410		if (!htb)
411			return -ENOMEM;
412		res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv);
413		if (res) {
414			mlx5e_htb_free(htb);
415			priv->htb = NULL;
416		}
417		return res;
418	case TC_HTB_DESTROY:
419		mlx5e_htb_cleanup(htb);
420		mlx5e_htb_free(htb);
421		priv->htb = NULL;
422		return 0;
423	case TC_HTB_LEAF_ALLOC_QUEUE:
424		res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid,
425						 htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack);
426		if (res < 0)
427			return res;
428		htb_qopt->qid = res;
429		return 0;
430	case TC_HTB_LEAF_TO_INNER:
431		return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid,
432					       htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack);
433	case TC_HTB_LEAF_DEL:
434		return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack);
435	case TC_HTB_LEAF_DEL_LAST:
436	case TC_HTB_LEAF_DEL_LAST_FORCE:
437		return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid,
438					       htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE,
439					       htb_qopt->extack);
440	case TC_HTB_NODE_MODIFY:
441		return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil,
442					     htb_qopt->extack);
443	case TC_HTB_LEAF_QUERY_QUEUE:
444		res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid);
445		if (res < 0)
446			return res;
447		htb_qopt->qid = res;
448		return 0;
449	default:
450		return -EOPNOTSUPP;
451	}
452}
453
454struct mlx5e_mqprio_rl {
455	struct mlx5_core_dev *mdev;
456	u32 root_id;
457	u32 *leaves_id;
458	u8 num_tc;
459};
460
461struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_alloc(void)
462{
463	return kvzalloc(sizeof(struct mlx5e_mqprio_rl), GFP_KERNEL);
464}
465
466void mlx5e_mqprio_rl_free(struct mlx5e_mqprio_rl *rl)
467{
468	kvfree(rl);
469}
470
471int mlx5e_mqprio_rl_init(struct mlx5e_mqprio_rl *rl, struct mlx5_core_dev *mdev, u8 num_tc,
472			 u64 max_rate[])
473{
474	int err;
475	int tc;
476
477	if (!mlx5_qos_is_supported(mdev)) {
478		qos_warn(mdev, "Missing QoS capabilities. Try disabling SRIOV or use a supported device.");
479		return -EOPNOTSUPP;
480	}
481	if (num_tc > mlx5e_qos_max_leaf_nodes(mdev))
482		return -EINVAL;
483
484	rl->mdev = mdev;
485	rl->num_tc = num_tc;
486	rl->leaves_id = kvcalloc(num_tc, sizeof(*rl->leaves_id), GFP_KERNEL);
487	if (!rl->leaves_id)
488		return -ENOMEM;
489
490	err = mlx5_qos_create_root_node(mdev, &rl->root_id);
491	if (err)
492		goto err_free_leaves;
493
494	qos_dbg(mdev, "Root created, id %#x\n", rl->root_id);
495
496	for (tc = 0; tc < num_tc; tc++) {
497		u32 max_average_bw;
498
499		max_average_bw = mlx5e_qos_bytes2mbits(mdev, max_rate[tc]);
500		err = mlx5_qos_create_leaf_node(mdev, rl->root_id, 0, max_average_bw,
501						&rl->leaves_id[tc]);
502		if (err)
503			goto err_destroy_leaves;
504
505		qos_dbg(mdev, "Leaf[%d] created, id %#x, max average bw %u Mbits/sec\n",
506			tc, rl->leaves_id[tc], max_average_bw);
507	}
508	return 0;
509
510err_destroy_leaves:
511	while (--tc >= 0)
512		mlx5_qos_destroy_node(mdev, rl->leaves_id[tc]);
513	mlx5_qos_destroy_node(mdev, rl->root_id);
514err_free_leaves:
515	kvfree(rl->leaves_id);
516	return err;
517}
518
519void mlx5e_mqprio_rl_cleanup(struct mlx5e_mqprio_rl *rl)
520{
521	int tc;
522
523	for (tc = 0; tc < rl->num_tc; tc++)
524		mlx5_qos_destroy_node(rl->mdev, rl->leaves_id[tc]);
525	mlx5_qos_destroy_node(rl->mdev, rl->root_id);
526	kvfree(rl->leaves_id);
527}
528
529int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_id)
530{
531	if (tc >= rl->num_tc)
532		return -EINVAL;
533
534	*hw_id = rl->leaves_id[tc];
535	return 0;
536}