pcie_cong_event.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2// Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
  3
  4#include "../devlink.h"
  5#include "en.h"
  6#include "pcie_cong_event.h"
  7
  8#define MLX5E_CONG_HIGH_STATE 0x7
  9
 10enum {
 11	MLX5E_INBOUND_CONG  = BIT(0),
 12	MLX5E_OUTBOUND_CONG = BIT(1),
 13};
 14
 15struct mlx5e_pcie_cong_thresh {
 16	u16 inbound_high;
 17	u16 inbound_low;
 18	u16 outbound_high;
 19	u16 outbound_low;
 20};
 21
 22struct mlx5e_pcie_cong_stats {
 23	u32 pci_bw_inbound_high;
 24	u32 pci_bw_inbound_low;
 25	u32 pci_bw_outbound_high;
 26	u32 pci_bw_outbound_low;
 27	u32 pci_bw_stale_event;
 28};
 29
 30struct mlx5e_pcie_cong_event {
 31	u64 obj_id;
 32
 33	struct mlx5e_priv *priv;
 34
 35	/* For event notifier and workqueue. */
 36	struct work_struct work;
 37	struct mlx5_nb nb;
 38
 39	/* Stores last read state. */
 40	u8 state;
 41
 42	/* For ethtool stats group. */
 43	struct mlx5e_pcie_cong_stats stats;
 44};
 45
 46
 47static const struct counter_desc mlx5e_pcie_cong_stats_desc[] = {
 48	{ MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats,
 49			     pci_bw_inbound_high) },
 50	{ MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats,
 51			     pci_bw_inbound_low) },
 52	{ MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats,
 53			     pci_bw_outbound_high) },
 54	{ MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats,
 55			     pci_bw_outbound_low) },
 56	{ MLX5E_DECLARE_STAT(struct mlx5e_pcie_cong_stats,
 57			     pci_bw_stale_event) },
 58};
 59
 60#define NUM_PCIE_CONG_COUNTERS ARRAY_SIZE(mlx5e_pcie_cong_stats_desc)
 61
 62static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(pcie_cong)
 63{
 64	return priv->cong_event ? NUM_PCIE_CONG_COUNTERS : 0;
 65}
 66
 67static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(pcie_cong) {}
 68
 69static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(pcie_cong)
 70{
 71	if (!priv->cong_event)
 72		return;
 73
 74	for (int i = 0; i < NUM_PCIE_CONG_COUNTERS; i++)
 75		ethtool_puts(data, mlx5e_pcie_cong_stats_desc[i].format);
 76}
 77
 78static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(pcie_cong)
 79{
 80	if (!priv->cong_event)
 81		return;
 82
 83	for (int i = 0; i < NUM_PCIE_CONG_COUNTERS; i++) {
 84		u32 ctr = MLX5E_READ_CTR32_CPU(&priv->cong_event->stats,
 85					       mlx5e_pcie_cong_stats_desc,
 86					       i);
 87
 88		mlx5e_ethtool_put_stat(data, ctr);
 89	}
 90}
 91
 92MLX5E_DEFINE_STATS_GRP(pcie_cong, 0);
 93
 94static int
 95mlx5_cmd_pcie_cong_event_set(struct mlx5_core_dev *dev,
 96			     const struct mlx5e_pcie_cong_thresh *config,
 97			     u64 *obj_id)
 98{
 99	u32 in[MLX5_ST_SZ_DW(pcie_cong_event_cmd_in)] = {};
100	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
101	void *cong_obj;
102	void *hdr;
103	int err;
104
105	hdr = MLX5_ADDR_OF(pcie_cong_event_cmd_in, in, hdr);
106	cong_obj = MLX5_ADDR_OF(pcie_cong_event_cmd_in, in, cong_obj);
107
108	MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
109		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
110
111	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
112		 MLX5_GENERAL_OBJECT_TYPES_PCIE_CONG_EVENT);
113
114	MLX5_SET(pcie_cong_event_obj, cong_obj, inbound_event_en, 1);
115	MLX5_SET(pcie_cong_event_obj, cong_obj, outbound_event_en, 1);
116
117	MLX5_SET(pcie_cong_event_obj, cong_obj,
118		 inbound_cong_high_threshold, config->inbound_high);
119	MLX5_SET(pcie_cong_event_obj, cong_obj,
120		 inbound_cong_low_threshold, config->inbound_low);
121
122	MLX5_SET(pcie_cong_event_obj, cong_obj,
123		 outbound_cong_high_threshold, config->outbound_high);
124	MLX5_SET(pcie_cong_event_obj, cong_obj,
125		 outbound_cong_low_threshold, config->outbound_low);
126
127	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
128	if (err)
129		return err;
130
131	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
132
133	mlx5_core_dbg(dev, "PCIe congestion event (obj_id=%llu) created. Config: in: [%u, %u], out: [%u, %u]\n",
134		      *obj_id,
135		      config->inbound_high, config->inbound_low,
136		      config->outbound_high, config->outbound_low);
137
138	return 0;
139}
140
141static int mlx5_cmd_pcie_cong_event_destroy(struct mlx5_core_dev *dev,
142					    u64 obj_id)
143{
144	u32 in[MLX5_ST_SZ_DW(pcie_cong_event_cmd_in)] = {};
145	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
146	void *hdr;
147
148	hdr = MLX5_ADDR_OF(pcie_cong_event_cmd_in, in, hdr);
149	MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
150		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
151	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
152		 MLX5_GENERAL_OBJECT_TYPES_PCIE_CONG_EVENT);
153	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, obj_id);
154
155	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
156}
157
158static int mlx5_cmd_pcie_cong_event_query(struct mlx5_core_dev *dev,
159					  u64 obj_id,
160					  u32 *state)
161{
162	u32 in[MLX5_ST_SZ_DW(pcie_cong_event_cmd_in)] = {};
163	u32 out[MLX5_ST_SZ_DW(pcie_cong_event_cmd_out)];
164	void *obj;
165	void *hdr;
166	u8 cong;
167	int err;
168
169	hdr = MLX5_ADDR_OF(pcie_cong_event_cmd_in, in, hdr);
170
171	MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
172		 MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
173	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
174		 MLX5_GENERAL_OBJECT_TYPES_PCIE_CONG_EVENT);
175	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, obj_id);
176
177	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
178	if (err)
179		return err;
180
181	obj = MLX5_ADDR_OF(pcie_cong_event_cmd_out, out, cong_obj);
182
183	if (state) {
184		cong = MLX5_GET(pcie_cong_event_obj, obj, inbound_cong_state);
185		if (cong == MLX5E_CONG_HIGH_STATE)
186			*state |= MLX5E_INBOUND_CONG;
187
188		cong = MLX5_GET(pcie_cong_event_obj, obj, outbound_cong_state);
189		if (cong == MLX5E_CONG_HIGH_STATE)
190			*state |= MLX5E_OUTBOUND_CONG;
191	}
192
193	return 0;
194}
195
196static void mlx5e_pcie_cong_event_work(struct work_struct *work)
197{
198	struct mlx5e_pcie_cong_event *cong_event;
199	struct mlx5_core_dev *dev;
200	struct mlx5e_priv *priv;
201	u32 new_cong_state = 0;
202	u32 changes;
203	int err;
204
205	cong_event = container_of(work, struct mlx5e_pcie_cong_event, work);
206	priv = cong_event->priv;
207	dev = priv->mdev;
208
209	err = mlx5_cmd_pcie_cong_event_query(dev, cong_event->obj_id,
210					     &new_cong_state);
211	if (err) {
212		mlx5_core_warn(dev, "Error %d when querying PCIe cong event object (obj_id=%llu).\n",
213			       err, cong_event->obj_id);
214		return;
215	}
216
217	changes = cong_event->state ^ new_cong_state;
218	if (!changes) {
219		cong_event->stats.pci_bw_stale_event++;
220		return;
221	}
222
223	cong_event->state = new_cong_state;
224
225	if (changes & MLX5E_INBOUND_CONG) {
226		if (new_cong_state & MLX5E_INBOUND_CONG)
227			cong_event->stats.pci_bw_inbound_high++;
228		else
229			cong_event->stats.pci_bw_inbound_low++;
230	}
231
232	if (changes & MLX5E_OUTBOUND_CONG) {
233		if (new_cong_state & MLX5E_OUTBOUND_CONG)
234			cong_event->stats.pci_bw_outbound_high++;
235		else
236			cong_event->stats.pci_bw_outbound_low++;
237	}
238}
239
240static int mlx5e_pcie_cong_event_handler(struct notifier_block *nb,
241					 unsigned long event, void *eqe)
242{
243	struct mlx5e_pcie_cong_event *cong_event;
244
245	cong_event = mlx5_nb_cof(nb, struct mlx5e_pcie_cong_event, nb);
246	queue_work(cong_event->priv->wq, &cong_event->work);
247
248	return NOTIFY_OK;
249}
250
251static int
252mlx5e_pcie_cong_get_thresh_config(struct mlx5_core_dev *dev,
253				  struct mlx5e_pcie_cong_thresh *config)
254{
255	u32 ids[4] = {
256		MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_LOW,
257		MLX5_DEVLINK_PARAM_ID_PCIE_CONG_IN_HIGH,
258		MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_LOW,
259		MLX5_DEVLINK_PARAM_ID_PCIE_CONG_OUT_HIGH,
260	};
261	struct devlink *devlink = priv_to_devlink(dev);
262	union devlink_param_value val[4];
263
264	for (int i = 0; i < 4; i++) {
265		u32 id = ids[i];
266		int err;
267
268		err = devl_param_driverinit_value_get(devlink, id, &val[i]);
269		if (err)
270			return err;
271	}
272
273	config->inbound_low = val[0].vu16;
274	config->inbound_high = val[1].vu16;
275	config->outbound_low = val[2].vu16;
276	config->outbound_high = val[3].vu16;
277
278	return 0;
279}
280
281static int
282mlx5e_thresh_config_validate(struct mlx5_core_dev *mdev,
283			     const struct mlx5e_pcie_cong_thresh *config)
284{
285	int err = 0;
286
287	if (config->inbound_low >= config->inbound_high) {
288		err = -EINVAL;
289		mlx5_core_err(mdev, "PCIe inbound congestion threshold configuration invalid: low (%u) >= high (%u).\n",
290			      config->inbound_low, config->inbound_high);
291	}
292
293	if (config->outbound_low >= config->outbound_high) {
294		err = -EINVAL;
295		mlx5_core_err(mdev, "PCIe outbound congestion threshold configuration invalid: low (%u) >= high (%u).\n",
296			      config->outbound_low, config->outbound_high);
297	}
298
299	return err;
300}
301
302int mlx5e_pcie_cong_event_init(struct mlx5e_priv *priv)
303{
304	struct mlx5e_pcie_cong_thresh thresh_config = {};
305	struct mlx5e_pcie_cong_event *cong_event;
306	struct mlx5_core_dev *mdev = priv->mdev;
307	int err;
308
309	if (!mlx5_pcie_cong_event_supported(mdev))
310		return 0;
311
312	err = mlx5e_pcie_cong_get_thresh_config(mdev, &thresh_config);
313	if (WARN_ON(err))
314		return err;
315
316	err = mlx5e_thresh_config_validate(mdev, &thresh_config);
317	if (err) {
318		mlx5_core_err(mdev, "PCIe congestion event feature disabled\n");
319		return err;
320	}
321
322	cong_event = kvzalloc_node(sizeof(*cong_event), GFP_KERNEL,
323				   mdev->priv.numa_node);
324	if (!cong_event)
325		return -ENOMEM;
326
327	INIT_WORK(&cong_event->work, mlx5e_pcie_cong_event_work);
328	MLX5_NB_INIT(&cong_event->nb, mlx5e_pcie_cong_event_handler,
329		     OBJECT_CHANGE);
330
331	cong_event->priv = priv;
332
333	err = mlx5_cmd_pcie_cong_event_set(mdev, &thresh_config,
334					   &cong_event->obj_id);
335	if (err) {
336		mlx5_core_warn(mdev, "Error creating a PCIe congestion event object\n");
337		goto err_free;
338	}
339
340	err = mlx5_eq_notifier_register(mdev, &cong_event->nb);
341	if (err) {
342		mlx5_core_warn(mdev, "Error registering notifier for the PCIe congestion event\n");
343		goto err_obj_destroy;
344	}
345
346	priv->cong_event = cong_event;
347
348	return 0;
349
350err_obj_destroy:
351	mlx5_cmd_pcie_cong_event_destroy(mdev, cong_event->obj_id);
352err_free:
353	kvfree(cong_event);
354
355	return err;
356}
357
358void mlx5e_pcie_cong_event_cleanup(struct mlx5e_priv *priv)
359{
360	struct mlx5e_pcie_cong_event *cong_event = priv->cong_event;
361	struct mlx5_core_dev *mdev = priv->mdev;
362
363	if (!cong_event)
364		return;
365
366	priv->cong_event = NULL;
367
368	mlx5_eq_notifier_unregister(mdev, &cong_event->nb);
369	cancel_work_sync(&cong_event->work);
370
371	if (mlx5_cmd_pcie_cong_event_destroy(mdev, cong_event->obj_id))
372		mlx5_core_warn(mdev, "Error destroying PCIe congestion event (obj_id=%llu)\n",
373			       cong_event->obj_id);
374
375	kvfree(cong_event);
376}