reporter_tx.c

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* Copyright (c) 2019 Mellanox Technologies. */
  3
  4#include <net/netdev_lock.h>
  5
  6#include "health.h"
  7#include "en/ptp.h"
  8#include "en/devlink.h"
  9#include "lib/tout.h"
 10
 11/* Keep this string array consistent with the MLX5E_SQ_STATE_* enums in en.h */
 12static const char * const sq_sw_state_type_name[] = {
 13	[MLX5E_SQ_STATE_ENABLED] = "enabled",
 14	[MLX5E_SQ_STATE_MPWQE] = "mpwqe",
 15	[MLX5E_SQ_STATE_RECOVERING] = "recovering",
 16	[MLX5E_SQ_STATE_IPSEC] = "ipsec",
 17	[MLX5E_SQ_STATE_DIM] = "dim",
 18	[MLX5E_SQ_STATE_PENDING_XSK_TX] = "pending_xsk_tx",
 19	[MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC] = "pending_tls_rx_resync",
 20};
 21
 22static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq)
 23{
 24	struct mlx5_core_dev *dev = sq->mdev;
 25	unsigned long exp_time;
 26
 27	exp_time = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, FLUSH_ON_ERROR));
 28
 29	while (time_before(jiffies, exp_time)) {
 30		if (sq->cc == sq->pc)
 31			return 0;
 32
 33		msleep(20);
 34	}
 35
 36	netdev_err(sq->netdev,
 37		   "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n",
 38		   sq->sqn, sq->cc, sq->pc);
 39
 40	return -ETIMEDOUT;
 41}
 42
 43static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq)
 44{
 45	WARN_ONCE(sq->cc != sq->pc,
 46		  "SQ 0x%x: cc (0x%x) != pc (0x%x)\n",
 47		  sq->sqn, sq->cc, sq->pc);
 48	sq->cc = 0;
 49	sq->pc = 0;
 50}
 51
 52static void mlx5e_health_sq_put_sw_state(struct devlink_fmsg *fmsg, struct mlx5e_txqsq *sq)
 53{
 54	int i;
 55
 56	BUILD_BUG_ON_MSG(ARRAY_SIZE(sq_sw_state_type_name) != MLX5E_NUM_SQ_STATES,
 57			 "sq_sw_state_type_name string array must be consistent with MLX5E_SQ_STATE_* enum in en.h");
 58	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SW State");
 59
 60	for (i = 0; i < ARRAY_SIZE(sq_sw_state_type_name); ++i)
 61		devlink_fmsg_u32_pair_put(fmsg, sq_sw_state_type_name[i],
 62					  test_bit(i, &sq->state));
 63
 64	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
 65}
 66
 67static int mlx5e_tx_reporter_err_cqe_recover(void *ctx)
 68{
 69	struct mlx5_core_dev *mdev;
 70	struct net_device *dev;
 71	struct mlx5e_txqsq *sq;
 72	u8 state;
 73	int err;
 74
 75	sq = ctx;
 76	mdev = sq->mdev;
 77	dev = sq->netdev;
 78
 79	if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state))
 80		return 0;
 81
 82	/* Recovering queues means re-enabling NAPI, which requires the netdev
 83	 * instance lock. However, SQ closing flows have to wait for work tasks
 84	 * to finish while also holding the netdev instance lock. So either get
 85	 * the lock or find that the SQ is no longer enabled and thus this work
 86	 * is not relevant anymore.
 87	 */
 88	while (!netdev_trylock(dev)) {
 89		if (!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))
 90			return 0;
 91		msleep(20);
 92	}
 93
 94	err = mlx5_core_query_sq_state(mdev, sq->sqn, &state);
 95	if (err) {
 96		netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n",
 97			   sq->sqn, err);
 98		goto out;
 99	}
100
101	if (state != MLX5_SQC_STATE_ERR)
102		goto out;
103
104	mlx5e_tx_disable_queue(sq->txq);
105
106	err = mlx5e_wait_for_sq_flush(sq);
107	if (err)
108		goto out;
109
110	/* At this point, no new packets will arrive from the stack as TXQ is
111	 * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all
112	 * pending WQEs. SQ can safely reset the SQ.
113	 */
114
115	err = mlx5e_health_sq_to_ready(mdev, dev, sq->sqn);
116	if (err)
117		goto out;
118
119	mlx5e_reset_txqsq_cc_pc(sq);
120	sq->stats->recover++;
121	clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state);
122	mlx5e_activate_txqsq(sq);
123
124	if (sq->channel)
125		mlx5e_trigger_napi_icosq(sq->channel);
126	else
127		mlx5e_trigger_napi_sched(sq->cq.napi);
128
129	netdev_unlock(dev);
130	return 0;
131out:
132	clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state);
133	netdev_unlock(dev);
134	return err;
135}
136
137struct mlx5e_tx_timeout_ctx {
138	struct mlx5e_txqsq *sq;
139	signed int status;
140};
141
142static int mlx5e_tx_reporter_timeout_recover(void *ctx)
143{
144	struct mlx5e_tx_timeout_ctx *to_ctx;
145	struct mlx5e_priv *priv;
146	struct mlx5_eq_comp *eq;
147	struct mlx5e_txqsq *sq;
148	int err;
149
150	to_ctx = ctx;
151	sq = to_ctx->sq;
152	eq = sq->cq.mcq.eq;
153	priv = sq->priv;
154
155	/* Recovering the TX queues implies re-enabling NAPI, which requires
156	 * the netdev instance lock.
157	 * However, channel closing flows have to wait for this work to finish
158	 * while holding the same lock. So either get the lock or find that
159	 * channels are being closed for other reason and this work is not
160	 * relevant anymore.
161	 */
162	while (!netdev_trylock(sq->netdev)) {
163		if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state))
164			return 0;
165		msleep(20);
166	}
167
168	err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats);
169	if (!err) {
170		to_ctx->status = 0; /* this sq recovered */
171		goto out;
172	}
173
174	mutex_lock(&priv->state_lock);
175	err = mlx5e_safe_reopen_channels(priv);
176	mutex_unlock(&priv->state_lock);
177	if (!err) {
178		to_ctx->status = 1; /* all channels recovered */
179		goto out;
180	}
181
182	to_ctx->status = err;
183	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
184	netdev_err(priv->netdev,
185		   "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n",
186		   err);
187out:
188	netdev_unlock(sq->netdev);
189	return err;
190}
191
192static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx)
193{
194	struct mlx5e_ptpsq *ptpsq = ctx;
195	struct mlx5e_channels *chs;
196	struct net_device *netdev;
197	struct mlx5e_priv *priv;
198	int carrier_ok;
199	int err;
200
201	if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &ptpsq->txqsq.state))
202		return 0;
203
204	priv = ptpsq->txqsq.priv;
205	netdev = priv->netdev;
206
207	/* Recovering the PTP SQ means re-enabling NAPI, which requires the
208	 * netdev instance lock. However, SQ closing has to wait for this work
209	 * task to finish while also holding the same lock. So either get the
210	 * lock or find that the SQ is no longer enabled and thus this work is
211	 * not relevant anymore.
212	 */
213	while (!netdev_trylock(netdev)) {
214		if (!test_bit(MLX5E_SQ_STATE_ENABLED, &ptpsq->txqsq.state))
215			return 0;
216		msleep(20);
217	}
218
219	mutex_lock(&priv->state_lock);
220	chs = &priv->channels;
221
222	carrier_ok = netif_carrier_ok(netdev);
223	netif_carrier_off(netdev);
224
225	mlx5e_deactivate_priv_channels(priv);
226
227	mlx5e_ptp_close(chs->ptp);
228	err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp);
229
230	mlx5e_activate_priv_channels(priv);
231
232	/* return carrier back if needed */
233	if (carrier_ok)
234		netif_carrier_on(netdev);
235
236	mutex_unlock(&priv->state_lock);
237	netdev_unlock(netdev);
238
239	return err;
240}
241
242/* state lock cannot be grabbed within this function.
243 * It can cause a dead lock or a read-after-free.
244 */
245static int mlx5e_tx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx)
246{
247	return err_ctx->recover(err_ctx->ctx);
248}
249
250static int mlx5e_tx_reporter_recover(struct devlink_health_reporter *reporter,
251				     void *context,
252				     struct netlink_ext_ack *extack)
253{
254	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);
255	struct mlx5e_err_ctx *err_ctx = context;
256
257	return err_ctx ? mlx5e_tx_reporter_recover_from_ctx(err_ctx) :
258			 mlx5e_health_recover_channels(priv);
259}
260
261static void
262mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg,
263						  struct mlx5e_txqsq *sq, int tc)
264{
265	bool stopped = netif_xmit_stopped(sq->txq);
266	u8 state;
267	int err;
268
269	devlink_fmsg_u32_pair_put(fmsg, "tc", tc);
270	devlink_fmsg_u32_pair_put(fmsg, "txq ix", sq->txq_ix);
271	devlink_fmsg_u32_pair_put(fmsg, "sqn", sq->sqn);
272
273	err = mlx5_core_query_sq_state(sq->mdev, sq->sqn, &state);
274	if (!err)
275		devlink_fmsg_u8_pair_put(fmsg, "HW state", state);
276
277	devlink_fmsg_bool_pair_put(fmsg, "stopped", stopped);
278	devlink_fmsg_u32_pair_put(fmsg, "cc", sq->cc);
279	devlink_fmsg_u32_pair_put(fmsg, "pc", sq->pc);
280	mlx5e_health_sq_put_sw_state(fmsg, sq);
281	mlx5e_health_cq_diag_fmsg(&sq->cq, fmsg);
282	mlx5e_health_eq_diag_fmsg(sq->cq.mcq.eq, fmsg);
283}
284
285static void
286mlx5e_tx_reporter_build_diagnose_output(struct devlink_fmsg *fmsg,
287					struct mlx5e_txqsq *sq, int tc)
288{
289	devlink_fmsg_obj_nest_start(fmsg);
290	devlink_fmsg_u32_pair_put(fmsg, "channel ix", sq->ch_ix);
291	mlx5e_tx_reporter_build_diagnose_output_sq_common(fmsg, sq, tc);
292	devlink_fmsg_obj_nest_end(fmsg);
293}
294
295static void
296mlx5e_tx_reporter_build_diagnose_output_ptpsq(struct devlink_fmsg *fmsg,
297					      struct mlx5e_ptpsq *ptpsq, int tc)
298{
299	devlink_fmsg_obj_nest_start(fmsg);
300	devlink_fmsg_string_pair_put(fmsg, "channel", "ptp");
301	mlx5e_tx_reporter_build_diagnose_output_sq_common(fmsg, &ptpsq->txqsq, tc);
302	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Port TS");
303	mlx5e_health_cq_diag_fmsg(&ptpsq->ts_cq, fmsg);
304	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
305	devlink_fmsg_obj_nest_end(fmsg);
306}
307
308static void
309mlx5e_tx_reporter_diagnose_generic_txqsq(struct devlink_fmsg *fmsg,
310					 struct mlx5e_txqsq *txqsq)
311{
312	bool real_time =  mlx5_is_real_time_sq(txqsq->mdev);
313	u32 sq_sz = mlx5_wq_cyc_get_size(&txqsq->wq);
314	u32 sq_stride = MLX5_SEND_WQE_BB;
315
316	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SQ");
317	devlink_fmsg_u64_pair_put(fmsg, "stride size", sq_stride);
318	devlink_fmsg_u32_pair_put(fmsg, "size", sq_sz);
319	devlink_fmsg_string_pair_put(fmsg, "ts_format", real_time ? "RT" : "FRC");
320	mlx5e_health_cq_common_diag_fmsg(&txqsq->cq, fmsg);
321	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
322}
323
324static void
325mlx5e_tx_reporter_diagnose_generic_tx_port_ts(struct devlink_fmsg *fmsg,
326					      struct mlx5e_ptpsq *ptpsq)
327{
328	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Port TS");
329	mlx5e_health_cq_common_diag_fmsg(&ptpsq->ts_cq, fmsg);
330	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
331}
332
333static void
334mlx5e_tx_reporter_diagnose_common_config(struct devlink_health_reporter *reporter,
335					 struct devlink_fmsg *fmsg)
336{
337	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);
338	struct mlx5e_txqsq *generic_sq = priv->txq2sq[0];
339	struct mlx5e_ptp *ptp_ch = priv->channels.ptp;
340	struct mlx5e_ptpsq *generic_ptpsq;
341
342	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common Config");
343	mlx5e_tx_reporter_diagnose_generic_txqsq(fmsg, generic_sq);
344
345	if (!ptp_ch || !test_bit(MLX5E_PTP_STATE_TX, ptp_ch->state))
346		goto out;
347
348	generic_ptpsq = &ptp_ch->ptpsq[0];
349	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "PTP");
350	mlx5e_tx_reporter_diagnose_generic_txqsq(fmsg, &generic_ptpsq->txqsq);
351	mlx5e_tx_reporter_diagnose_generic_tx_port_ts(fmsg, generic_ptpsq);
352	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
353out:
354	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
355}
356
357static void
358mlx5e_tx_reporter_diagnose_tis_config(struct devlink_health_reporter *reporter,
359				      struct devlink_fmsg *fmsg)
360{
361	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);
362	u8 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params);
363	u32 tc, i, tisn;
364
365	devlink_fmsg_arr_pair_nest_start(fmsg, "TIS Config");
366	for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) {
367		for (tc = 0; tc < num_tc; tc++) {
368			tisn = mlx5e_profile_get_tisn(priv->mdev, priv,
369						      priv->profile, i, tc);
370
371			devlink_fmsg_obj_nest_start(fmsg);
372			devlink_fmsg_u32_pair_put(fmsg, "lag port", i);
373			devlink_fmsg_u32_pair_put(fmsg, "tc", tc);
374			devlink_fmsg_u32_pair_put(fmsg, "tisn", tisn);
375			devlink_fmsg_obj_nest_end(fmsg);
376		}
377	}
378	devlink_fmsg_arr_pair_nest_end(fmsg);
379}
380
381static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter,
382				      struct devlink_fmsg *fmsg,
383				      struct netlink_ext_ack *extack)
384{
385	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);
386	struct mlx5e_ptp *ptp_ch = priv->channels.ptp;
387
388	int i, tc;
389
390	mutex_lock(&priv->state_lock);
391
392	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
393		goto unlock;
394
395	mlx5e_tx_reporter_diagnose_common_config(reporter, fmsg);
396	mlx5e_tx_reporter_diagnose_tis_config(reporter, fmsg);
397	devlink_fmsg_arr_pair_nest_start(fmsg, "SQs");
398
399	for (i = 0; i < priv->channels.num; i++) {
400		struct mlx5e_channel *c = priv->channels.c[i];
401
402		for (tc = 0; tc < mlx5e_get_dcb_num_tc(&priv->channels.params); tc++) {
403			struct mlx5e_txqsq *sq = &c->sq[tc];
404
405			mlx5e_tx_reporter_build_diagnose_output(fmsg, sq, tc);
406		}
407	}
408
409	if (!ptp_ch || !test_bit(MLX5E_PTP_STATE_TX, ptp_ch->state))
410		goto close_sqs_nest;
411
412	for (tc = 0; tc < mlx5e_get_dcb_num_tc(&priv->channels.params); tc++)
413		mlx5e_tx_reporter_build_diagnose_output_ptpsq(fmsg,
414							      &ptp_ch->ptpsq[tc],
415							      tc);
416
417close_sqs_nest:
418	devlink_fmsg_arr_pair_nest_end(fmsg);
419unlock:
420	mutex_unlock(&priv->state_lock);
421	return 0;
422}
423
424static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg,
425				     void *ctx)
426{
427	struct mlx5_rsc_key key = {};
428	struct mlx5e_txqsq *sq = ctx;
429
430	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
431		return 0;
432
433	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice");
434	key.size = PAGE_SIZE;
435	key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL;
436	mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg);
437	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
438
439	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SQ");
440	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC");
441	key.rsc = MLX5_SGMT_TYPE_FULL_QPC;
442	key.index1 = sq->sqn;
443	key.num_of_obj1 = 1;
444	mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg);
445	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
446
447	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff");
448	key.rsc = MLX5_SGMT_TYPE_SND_BUFF;
449	key.num_of_obj2 = MLX5_RSC_DUMP_ALL;
450	mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg);
451	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
452
453	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
454
455	return 0;
456}
457
458static int mlx5e_tx_reporter_timeout_dump(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg,
459					  void *ctx)
460{
461	struct mlx5e_tx_timeout_ctx *to_ctx = ctx;
462
463	return mlx5e_tx_reporter_dump_sq(priv, fmsg, to_ctx->sq);
464}
465
466static int mlx5e_tx_reporter_ptpsq_unhealthy_dump(struct mlx5e_priv *priv,
467						  struct devlink_fmsg *fmsg,
468						  void *ctx)
469{
470	struct mlx5e_ptpsq *ptpsq = ctx;
471
472	return mlx5e_tx_reporter_dump_sq(priv, fmsg, &ptpsq->txqsq);
473}
474
475static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv,
476					  struct devlink_fmsg *fmsg)
477{
478	struct mlx5e_ptp *ptp_ch = priv->channels.ptp;
479	struct mlx5_rsc_key key = {};
480	int i, tc;
481
482	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
483		return 0;
484
485	mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice");
486	key.size = PAGE_SIZE;
487	key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL;
488	mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg);
489	mlx5e_health_fmsg_named_obj_nest_end(fmsg);
490	devlink_fmsg_arr_pair_nest_start(fmsg, "SQs");
491
492	for (i = 0; i < priv->channels.num; i++) {
493		struct mlx5e_channel *c = priv->channels.c[i];
494
495		for (tc = 0; tc < mlx5e_get_dcb_num_tc(&priv->channels.params); tc++) {
496			struct mlx5e_txqsq *sq = &c->sq[tc];
497
498			mlx5e_health_queue_dump(priv, fmsg, sq->sqn, "SQ");
499		}
500	}
501
502	if (ptp_ch && test_bit(MLX5E_PTP_STATE_TX, ptp_ch->state)) {
503		for (tc = 0; tc < mlx5e_get_dcb_num_tc(&priv->channels.params); tc++) {
504			struct mlx5e_txqsq *sq = &ptp_ch->ptpsq[tc].txqsq;
505
506			mlx5e_health_queue_dump(priv, fmsg, sq->sqn, "PTP SQ");
507		}
508	}
509
510	devlink_fmsg_arr_pair_nest_end(fmsg);
511	return 0;
512}
513
514static int mlx5e_tx_reporter_dump_from_ctx(struct mlx5e_priv *priv,
515					   struct mlx5e_err_ctx *err_ctx,
516					   struct devlink_fmsg *fmsg)
517{
518	return err_ctx->dump(priv, fmsg, err_ctx->ctx);
519}
520
521static int mlx5e_tx_reporter_dump(struct devlink_health_reporter *reporter,
522				  struct devlink_fmsg *fmsg, void *context,
523				  struct netlink_ext_ack *extack)
524{
525	struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);
526	struct mlx5e_err_ctx *err_ctx = context;
527
528	return err_ctx ? mlx5e_tx_reporter_dump_from_ctx(priv, err_ctx, fmsg) :
529			 mlx5e_tx_reporter_dump_all_sqs(priv, fmsg);
530}
531
532void mlx5e_reporter_tx_err_cqe(struct mlx5e_txqsq *sq)
533{
534	char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN];
535	struct mlx5e_priv *priv = sq->priv;
536	struct mlx5e_err_ctx err_ctx = {};
537
538	err_ctx.ctx = sq;
539	err_ctx.recover = mlx5e_tx_reporter_err_cqe_recover;
540	err_ctx.dump = mlx5e_tx_reporter_dump_sq;
541	snprintf(err_str, sizeof(err_str), "ERR CQE on SQ: 0x%x", sq->sqn);
542
543	mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx);
544}
545
546int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq)
547{
548	char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN];
549	struct mlx5e_tx_timeout_ctx to_ctx = {};
550	struct mlx5e_priv *priv = sq->priv;
551	struct mlx5e_err_ctx err_ctx = {};
552
553	to_ctx.sq = sq;
554	err_ctx.ctx = &to_ctx;
555	err_ctx.recover = mlx5e_tx_reporter_timeout_recover;
556	err_ctx.dump = mlx5e_tx_reporter_timeout_dump;
557	snprintf(err_str, sizeof(err_str),
558		 "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u",
559		 sq->ch_ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,
560		 jiffies_to_usecs(jiffies - READ_ONCE(sq->txq->trans_start)));
561
562	mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx);
563	return to_ctx.status;
564}
565
566void mlx5e_reporter_tx_ptpsq_unhealthy(struct mlx5e_ptpsq *ptpsq)
567{
568	struct mlx5e_ptp_metadata_map *map = &ptpsq->metadata_map;
569	char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN];
570	struct mlx5e_txqsq *txqsq = &ptpsq->txqsq;
571	struct mlx5e_cq *ts_cq = &ptpsq->ts_cq;
572	struct mlx5e_priv *priv = txqsq->priv;
573	struct mlx5e_err_ctx err_ctx = {};
574
575	err_ctx.ctx = ptpsq;
576	err_ctx.recover = mlx5e_tx_reporter_ptpsq_unhealthy_recover;
577	err_ctx.dump = mlx5e_tx_reporter_ptpsq_unhealthy_dump;
578	snprintf(err_str, sizeof(err_str),
579		 "Unhealthy TX port TS queue: %d, SQ: 0x%x, CQ: 0x%x, Undelivered CQEs: %u Map Capacity: %u",
580		 txqsq->ch_ix, txqsq->sqn, ts_cq->mcq.cqn, map->undelivered_counter, map->capacity);
581
582	mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx);
583}
584
585#define MLX5E_REPORTER_TX_GRACEFUL_PERIOD 500
586#define MLX5E_REPORTER_TX_BURST_PERIOD 500
587
588static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = {
589		.name = "tx",
590		.recover = mlx5e_tx_reporter_recover,
591		.diagnose = mlx5e_tx_reporter_diagnose,
592		.dump = mlx5e_tx_reporter_dump,
593		.default_graceful_period = MLX5E_REPORTER_TX_GRACEFUL_PERIOD,
594		.default_burst_period = MLX5E_REPORTER_TX_BURST_PERIOD,
595};
596
597void mlx5e_reporter_tx_create(struct mlx5e_priv *priv)
598{
599	struct devlink_port *port = priv->netdev->devlink_port;
600	struct devlink_health_reporter *reporter;
601
602	reporter = devlink_port_health_reporter_create(port,
603						       &mlx5_tx_reporter_ops,
604						       priv);
605	if (IS_ERR(reporter)) {
606		netdev_warn(priv->netdev,
607			    "Failed to create tx reporter, err = %pe\n",
608			    reporter);
609		return;
610	}
611	priv->tx_reporter = reporter;
612}
613
614void mlx5e_reporter_tx_destroy(struct mlx5e_priv *priv)
615{
616	if (!priv->tx_reporter)
617		return;
618
619	devlink_health_reporter_destroy(priv->tx_reporter);
620	priv->tx_reporter = NULL;
621}