meter.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  3
  4#include <linux/iopoll.h>
  5#include <linux/math64.h>
  6#include "lib/aso.h"
  7#include "en/tc/post_act.h"
  8#include "meter.h"
  9#include "en/tc_priv.h"
 10
 11#define MLX5_START_COLOR_SHIFT 28
 12#define MLX5_METER_MODE_SHIFT 24
 13#define MLX5_CBS_EXP_SHIFT 24
 14#define MLX5_CBS_MAN_SHIFT 16
 15#define MLX5_CIR_EXP_SHIFT 8
 16
 17/* cir = 8*(10^9)*cir_mantissa/(2^cir_exponent)) bits/s */
 18#define MLX5_CONST_CIR 8000000000ULL
 19#define MLX5_CALC_CIR(m, e)  ((MLX5_CONST_CIR * (m)) >> (e))
 20#define MLX5_MAX_CIR ((MLX5_CONST_CIR * 0x100) - 1)
 21
 22/* cbs = cbs_mantissa*2^cbs_exponent */
 23#define MLX5_CALC_CBS(m, e)  ((m) << (e))
 24#define MLX5_MAX_CBS ((0x100ULL << 0x1F) - 1)
 25#define MLX5_MAX_HW_CBS 0x7FFFFFFF
 26
 27struct mlx5e_flow_meter_aso_obj {
 28	struct list_head entry;
 29	int base_id;
 30	int total_meters;
 31
 32	unsigned long meters_map[]; /* must be at the end of this struct */
 33};
 34
 35struct mlx5e_flow_meters {
 36	enum mlx5_flow_namespace_type ns_type;
 37	struct mlx5_aso *aso;
 38	struct mutex aso_lock; /* Protects aso operations */
 39	int log_granularity;
 40	u32 pdn;
 41
 42	DECLARE_HASHTABLE(hashtbl, 8);
 43
 44	struct mutex sync_lock; /* protect flow meter operations */
 45	struct list_head partial_list;
 46	struct list_head full_list;
 47
 48	struct mlx5_core_dev *mdev;
 49	struct mlx5e_post_act *post_act;
 50};
 51
 52static void
 53mlx5e_flow_meter_cir_calc(u64 cir, u8 *man, u8 *exp)
 54{
 55	s64 _cir, _delta, delta = S64_MAX;
 56	u8 e, _man = 0, _exp = 0;
 57	u64 m;
 58
 59	for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 60		m = cir << e;
 61		if ((s64)m < 0) /* overflow */
 62			break;
 63		m = div64_u64(m, MLX5_CONST_CIR);
 64		if (m > 0xFF) /* man width 8 bit */
 65			continue;
 66		_cir = MLX5_CALC_CIR(m, e);
 67		_delta = cir - _cir;
 68		if (_delta < delta) {
 69			_man = m;
 70			_exp = e;
 71			if (!_delta)
 72				goto found;
 73			delta = _delta;
 74		}
 75	}
 76
 77found:
 78	*man = _man;
 79	*exp = _exp;
 80}
 81
 82static void
 83mlx5e_flow_meter_cbs_calc(u64 cbs, u8 *man, u8 *exp)
 84{
 85	s64 _cbs, _delta, delta = S64_MAX;
 86	u8 e, _man = 0, _exp = 0;
 87	u64 m;
 88
 89	for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 90		m = cbs >> e;
 91		if (m > 0xFF) /* man width 8 bit */
 92			continue;
 93		_cbs = MLX5_CALC_CBS(m, e);
 94		_delta = cbs - _cbs;
 95		if (_delta < delta) {
 96			_man = m;
 97			_exp = e;
 98			if (!_delta)
 99				goto found;
100			delta = _delta;
101		}
102	}
103
104found:
105	*man = _man;
106	*exp = _exp;
107}
108
109int
110mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev,
111		      struct mlx5e_flow_meter_handle *meter,
112		      struct mlx5e_flow_meter_params *meter_params)
113{
114	struct mlx5_wqe_aso_ctrl_seg *aso_ctrl;
115	struct mlx5_wqe_aso_data_seg *aso_data;
116	struct mlx5e_flow_meters *flow_meters;
117	u8 cir_man, cir_exp, cbs_man, cbs_exp;
118	struct mlx5_aso_wqe *aso_wqe;
119	struct mlx5_aso *aso;
120	u64 rate, burst;
121	u8 ds_cnt;
122	int err;
123
124	rate = meter_params->rate;
125	burst = meter_params->burst;
126
127	/* HW treats each packet as 128 bytes in PPS mode */
128	if (meter_params->mode == MLX5_RATE_LIMIT_PPS) {
129		rate <<= 10;
130		burst <<= 7;
131	}
132
133	if (!rate || rate > MLX5_MAX_CIR || !burst || burst > MLX5_MAX_CBS)
134		return -EINVAL;
135
136	/* HW has limitation of total 31 bits for cbs */
137	if (burst > MLX5_MAX_HW_CBS) {
138		mlx5_core_warn(mdev,
139			       "burst(%lld) is too large, use HW allowed value(%d)\n",
140			       burst, MLX5_MAX_HW_CBS);
141		burst = MLX5_MAX_HW_CBS;
142	}
143
144	mlx5_core_dbg(mdev, "meter mode=%d\n", meter_params->mode);
145	mlx5e_flow_meter_cir_calc(rate, &cir_man, &cir_exp);
146	mlx5_core_dbg(mdev, "rate=%lld, cir=%lld, exp=%d, man=%d\n",
147		      rate, MLX5_CALC_CIR(cir_man, cir_exp), cir_exp, cir_man);
148	mlx5e_flow_meter_cbs_calc(burst, &cbs_man, &cbs_exp);
149	mlx5_core_dbg(mdev, "burst=%lld, cbs=%lld, exp=%d, man=%d\n",
150		      burst, MLX5_CALC_CBS((u64)cbs_man, cbs_exp), cbs_exp, cbs_man);
151
152	if (!cir_man || !cbs_man)
153		return -EINVAL;
154
155	flow_meters = meter->flow_meters;
156	aso = flow_meters->aso;
157
158	mutex_lock(&flow_meters->aso_lock);
159	aso_wqe = mlx5_aso_get_wqe(aso);
160	ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_DS);
161	mlx5_aso_build_wqe(aso, ds_cnt, aso_wqe, meter->obj_id,
162			   MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER);
163
164	aso_ctrl = &aso_wqe->aso_ctrl;
165	aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6;
166	aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE |
167					  MLX5_ASO_ALWAYS_TRUE << 4;
168	aso_ctrl->data_offset_condition_operand = MLX5_ASO_LOGICAL_OR << 6;
169	aso_ctrl->data_mask = cpu_to_be64(0x80FFFFFFULL << (meter->idx ? 0 : 32));
170
171	aso_data = (struct mlx5_wqe_aso_data_seg *)(aso_wqe + 1);
172	memset(aso_data, 0, sizeof(*aso_data));
173	aso_data->bytewise_data[meter->idx * 8] = cpu_to_be32((0x1 << 31) | /* valid */
174					(MLX5_FLOW_METER_COLOR_GREEN << MLX5_START_COLOR_SHIFT));
175	if (meter_params->mode == MLX5_RATE_LIMIT_PPS)
176		aso_data->bytewise_data[meter->idx * 8] |=
177			cpu_to_be32(MLX5_FLOW_METER_MODE_NUM_PACKETS << MLX5_METER_MODE_SHIFT);
178	else
179		aso_data->bytewise_data[meter->idx * 8] |=
180			cpu_to_be32(MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH << MLX5_METER_MODE_SHIFT);
181
182	aso_data->bytewise_data[meter->idx * 8 + 2] = cpu_to_be32((cbs_exp << MLX5_CBS_EXP_SHIFT) |
183								  (cbs_man << MLX5_CBS_MAN_SHIFT) |
184								  (cir_exp << MLX5_CIR_EXP_SHIFT) |
185								  cir_man);
186
187	mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl);
188
189	/* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */
190	read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC,
191			  false, aso, true);
192	mutex_unlock(&flow_meters->aso_lock);
193
194	return err;
195}
196
197static int
198mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_id)
199{
200	u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {};
201	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
202	struct mlx5_core_dev *mdev = flow_meters->mdev;
203	void *obj, *param;
204	int err;
205
206	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
207	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
208		 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO);
209	param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
210	MLX5_SET(general_obj_create_param, param, log_obj_range,
211		 flow_meters->log_granularity);
212
213	obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj);
214	MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn);
215
216	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
217	if (!err) {
218		*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
219		mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) created\n", *obj_id);
220	}
221
222	return err;
223}
224
225static void
226mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id)
227{
228	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
229	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
230
231	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
232	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
233		 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO);
234	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
235
236	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
237	mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) destroyed\n", obj_id);
238}
239
240static struct mlx5e_flow_meter_handle *
241__mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters, bool alloc_aso)
242{
243	struct mlx5_core_dev *mdev = flow_meters->mdev;
244	struct mlx5e_flow_meter_aso_obj *meters_obj;
245	struct mlx5e_flow_meter_handle *meter;
246	struct mlx5_fc *counter;
247	int err, pos, total;
248	u32 id;
249
250	meter = kzalloc(sizeof(*meter), GFP_KERNEL);
251	if (!meter)
252		return ERR_PTR(-ENOMEM);
253
254	counter = mlx5_fc_create(mdev, true);
255	if (IS_ERR(counter)) {
256		err = PTR_ERR(counter);
257		goto err_drop_counter;
258	}
259	meter->drop_counter = counter;
260
261	counter = mlx5_fc_create(mdev, true);
262	if (IS_ERR(counter)) {
263		err = PTR_ERR(counter);
264		goto err_act_counter;
265	}
266	meter->act_counter = counter;
267
268	if (!alloc_aso)
269		goto no_aso;
270
271	meters_obj = list_first_entry_or_null(&flow_meters->partial_list,
272					      struct mlx5e_flow_meter_aso_obj,
273					      entry);
274	/* 2 meters in one object */
275	total = 1 << (flow_meters->log_granularity + 1);
276	if (!meters_obj) {
277		err = mlx5e_flow_meter_create_aso_obj(flow_meters, &id);
278		if (err) {
279			mlx5_core_err(mdev, "Failed to create flow meter ASO object\n");
280			goto err_create;
281		}
282
283		meters_obj = kzalloc(sizeof(*meters_obj) + BITS_TO_BYTES(total),
284				     GFP_KERNEL);
285		if (!meters_obj) {
286			err = -ENOMEM;
287			goto err_mem;
288		}
289
290		meters_obj->base_id = id;
291		meters_obj->total_meters = total;
292		list_add(&meters_obj->entry, &flow_meters->partial_list);
293		pos = 0;
294	} else {
295		pos = find_first_zero_bit(meters_obj->meters_map, total);
296		if (bitmap_weight(meters_obj->meters_map, total) == total - 1) {
297			list_del(&meters_obj->entry);
298			list_add(&meters_obj->entry, &flow_meters->full_list);
299		}
300	}
301
302	bitmap_set(meters_obj->meters_map, pos, 1);
303	meter->meters_obj = meters_obj;
304	meter->obj_id = meters_obj->base_id + pos / 2;
305	meter->idx = pos % 2;
306
307no_aso:
308	meter->flow_meters = flow_meters;
309	mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n",
310		      meter->obj_id, meter->idx);
311
312	return meter;
313
314err_mem:
315	mlx5e_flow_meter_destroy_aso_obj(mdev, id);
316err_create:
317	mlx5_fc_destroy(mdev, meter->act_counter);
318err_act_counter:
319	mlx5_fc_destroy(mdev, meter->drop_counter);
320err_drop_counter:
321	kfree(meter);
322	return ERR_PTR(err);
323}
324
325static void
326__mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter)
327{
328	struct mlx5e_flow_meters *flow_meters = meter->flow_meters;
329	struct mlx5_core_dev *mdev = flow_meters->mdev;
330	struct mlx5e_flow_meter_aso_obj *meters_obj;
331	int n, pos;
332
333	mlx5_fc_destroy(mdev, meter->act_counter);
334	mlx5_fc_destroy(mdev, meter->drop_counter);
335
336	if (meter->params.mtu)
337		goto out_no_aso;
338
339	meters_obj = meter->meters_obj;
340	pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx;
341	bitmap_clear(meters_obj->meters_map, pos, 1);
342	n = bitmap_weight(meters_obj->meters_map, meters_obj->total_meters);
343	if (n == 0) {
344		list_del(&meters_obj->entry);
345		mlx5e_flow_meter_destroy_aso_obj(mdev, meters_obj->base_id);
346		kfree(meters_obj);
347	} else if (n == meters_obj->total_meters - 1) {
348		list_del(&meters_obj->entry);
349		list_add(&meters_obj->entry, &flow_meters->partial_list);
350	}
351
352out_no_aso:
353	mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n",
354		      meter->obj_id, meter->idx);
355	kfree(meter);
356}
357
358static struct mlx5e_flow_meter_handle *
359__mlx5e_tc_meter_get(struct mlx5e_flow_meters *flow_meters, u32 index)
360{
361	struct mlx5e_flow_meter_handle *meter;
362
363	hash_for_each_possible(flow_meters->hashtbl, meter, hlist, index)
364		if (meter->params.index == index)
365			goto add_ref;
366
367	return ERR_PTR(-ENOENT);
368
369add_ref:
370	meter->refcnt++;
371
372	return meter;
373}
374
375struct mlx5e_flow_meter_handle *
376mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params)
377{
378	struct mlx5e_flow_meters *flow_meters;
379	struct mlx5e_flow_meter_handle *meter;
380
381	flow_meters = mlx5e_get_flow_meters(mdev);
382	if (!flow_meters)
383		return ERR_PTR(-EOPNOTSUPP);
384
385	mutex_lock(&flow_meters->sync_lock);
386	meter = __mlx5e_tc_meter_get(flow_meters, params->index);
387	mutex_unlock(&flow_meters->sync_lock);
388
389	return meter;
390}
391
392static void
393__mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter)
394{
395	if (--meter->refcnt == 0) {
396		hash_del(&meter->hlist);
397		__mlx5e_flow_meter_free(meter);
398	}
399}
400
401void
402mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter)
403{
404	struct mlx5e_flow_meters *flow_meters = meter->flow_meters;
405
406	mutex_lock(&flow_meters->sync_lock);
407	__mlx5e_tc_meter_put(meter);
408	mutex_unlock(&flow_meters->sync_lock);
409}
410
411static struct mlx5e_flow_meter_handle *
412mlx5e_tc_meter_alloc(struct mlx5e_flow_meters *flow_meters,
413		     struct mlx5e_flow_meter_params *params)
414{
415	struct mlx5e_flow_meter_handle *meter;
416
417	meter = __mlx5e_flow_meter_alloc(flow_meters, !params->mtu);
418	if (IS_ERR(meter))
419		return meter;
420
421	hash_add(flow_meters->hashtbl, &meter->hlist, params->index);
422	meter->params.index = params->index;
423	meter->params.mtu = params->mtu;
424	meter->refcnt++;
425
426	return meter;
427}
428
429static int
430__mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter,
431			struct mlx5e_flow_meter_params *params)
432{
433	struct mlx5_core_dev *mdev = meter->flow_meters->mdev;
434	int err = 0;
435
436	if (meter->params.mode != params->mode || meter->params.rate != params->rate ||
437	    meter->params.burst != params->burst) {
438		err = mlx5e_tc_meter_modify(mdev, meter, params);
439		if (err)
440			goto out;
441
442		meter->params.mode = params->mode;
443		meter->params.rate = params->rate;
444		meter->params.burst = params->burst;
445	}
446
447out:
448	return err;
449}
450
451int
452mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter,
453		      struct mlx5e_flow_meter_params *params)
454{
455	struct mlx5_core_dev *mdev = meter->flow_meters->mdev;
456	struct mlx5e_flow_meters *flow_meters;
457	int err;
458
459	flow_meters = mlx5e_get_flow_meters(mdev);
460	if (!flow_meters)
461		return -EOPNOTSUPP;
462
463	mutex_lock(&flow_meters->sync_lock);
464	err = __mlx5e_tc_meter_update(meter, params);
465	mutex_unlock(&flow_meters->sync_lock);
466	return err;
467}
468
469struct mlx5e_flow_meter_handle *
470mlx5e_tc_meter_replace(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params)
471{
472	struct mlx5e_flow_meters *flow_meters;
473	struct mlx5e_flow_meter_handle *meter;
474	int err;
475
476	flow_meters = mlx5e_get_flow_meters(mdev);
477	if (!flow_meters)
478		return ERR_PTR(-EOPNOTSUPP);
479
480	mutex_lock(&flow_meters->sync_lock);
481	meter = __mlx5e_tc_meter_get(flow_meters, params->index);
482	if (IS_ERR(meter)) {
483		meter = mlx5e_tc_meter_alloc(flow_meters, params);
484		if (IS_ERR(meter)) {
485			err = PTR_ERR(meter);
486			goto err_get;
487		}
488	}
489
490	err = __mlx5e_tc_meter_update(meter, params);
491	if (err)
492		goto err_update;
493
494	mutex_unlock(&flow_meters->sync_lock);
495	return meter;
496
497err_update:
498	__mlx5e_tc_meter_put(meter);
499err_get:
500	mutex_unlock(&flow_meters->sync_lock);
501	return ERR_PTR(err);
502}
503
504enum mlx5_flow_namespace_type
505mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters *flow_meters)
506{
507	return flow_meters->ns_type;
508}
509
510struct mlx5e_flow_meters *
511mlx5e_flow_meters_init(struct mlx5e_priv *priv,
512		       enum mlx5_flow_namespace_type ns_type,
513		       struct mlx5e_post_act *post_act)
514{
515	struct mlx5_core_dev *mdev = priv->mdev;
516	struct mlx5e_flow_meters *flow_meters;
517	int err;
518
519	if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
520	      MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO))
521		return ERR_PTR(-EOPNOTSUPP);
522
523	if (IS_ERR_OR_NULL(post_act)) {
524		netdev_dbg(priv->netdev,
525			   "flow meter offload is not supported, post action is missing\n");
526		return ERR_PTR(-EOPNOTSUPP);
527	}
528
529	flow_meters = kzalloc(sizeof(*flow_meters), GFP_KERNEL);
530	if (!flow_meters)
531		return ERR_PTR(-ENOMEM);
532
533	err = mlx5_core_alloc_pd(mdev, &flow_meters->pdn);
534	if (err) {
535		mlx5_core_err(mdev, "Failed to alloc pd for flow meter aso, err=%d\n", err);
536		goto err_out;
537	}
538
539	flow_meters->aso = mlx5_aso_create(mdev, flow_meters->pdn);
540	if (IS_ERR(flow_meters->aso)) {
541		mlx5_core_warn(mdev, "Failed to create aso wqe for flow meter\n");
542		err = PTR_ERR(flow_meters->aso);
543		goto err_sq;
544	}
545
546	mutex_init(&flow_meters->sync_lock);
547	INIT_LIST_HEAD(&flow_meters->partial_list);
548	INIT_LIST_HEAD(&flow_meters->full_list);
549
550	flow_meters->ns_type = ns_type;
551	flow_meters->mdev = mdev;
552	flow_meters->post_act = post_act;
553	mutex_init(&flow_meters->aso_lock);
554	flow_meters->log_granularity = min_t(int, 6,
555					     MLX5_CAP_QOS(mdev, log_meter_aso_max_alloc));
556
557	return flow_meters;
558
559err_sq:
560	mlx5_core_dealloc_pd(mdev, flow_meters->pdn);
561err_out:
562	kfree(flow_meters);
563	return ERR_PTR(err);
564}
565
566void
567mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters)
568{
569	if (IS_ERR_OR_NULL(flow_meters))
570		return;
571
572	mlx5_aso_destroy(flow_meters->aso);
573	mlx5_core_dealloc_pd(flow_meters->mdev, flow_meters->pdn);
574	kfree(flow_meters);
575}
576
577void
578mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter,
579			 u64 *bytes, u64 *packets, u64 *drops, u64 *lastuse)
580{
581	u64 bytes1, packets1, lastuse1;
582	u64 bytes2, packets2, lastuse2;
583
584	mlx5_fc_query_cached(meter->act_counter, &bytes1, &packets1, &lastuse1);
585	mlx5_fc_query_cached(meter->drop_counter, &bytes2, &packets2, &lastuse2);
586
587	*bytes = bytes1 + bytes2;
588	*packets = packets1 + packets2;
589	*drops = packets2;
590	*lastuse = max_t(u64, lastuse1, lastuse2);
591}
592
593int mlx5e_flow_meter_get_base_id(struct mlx5e_flow_meter_handle *meter)
594{
595	return meter->meters_obj->base_id;
596}