health.c

  1/*
  2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#include <linux/kernel.h>
 34#include <linux/random.h>
 35#include <linux/vmalloc.h>
 36#include <linux/hardirq.h>
 37#include <linux/mlx5/driver.h>
 38#include <linux/kern_levels.h>
 39#include "mlx5_core.h"
 40#include "lib/eq.h"
 41#include "lib/mlx5.h"
 42#include "lib/events.h"
 43#include "lib/pci_vsc.h"
 44#include "lib/tout.h"
 45#include "diag/fw_tracer.h"
 46#include "diag/reporter_vnic.h"
 47
 48enum {
 49	MAX_MISSES			= 3,
 50};
 51
 52enum {
 53	MLX5_DROP_HEALTH_WORK,
 54};
 55
 56enum  {
 57	MLX5_SENSOR_NO_ERR		= 0,
 58	MLX5_SENSOR_PCI_COMM_ERR	= 1,
 59	MLX5_SENSOR_PCI_ERR		= 2,
 60	MLX5_SENSOR_NIC_DISABLED	= 3,
 61	MLX5_SENSOR_NIC_SW_RESET	= 4,
 62	MLX5_SENSOR_FW_SYND_RFR		= 5,
 63};
 64
 65enum {
 66	MLX5_SEVERITY_MASK		= 0x7,
 67	MLX5_SEVERITY_VALID_MASK	= 0x8,
 68};
 69
 70u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
 71{
 72	return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7;
 73}
 74
 75void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
 76{
 77	u32 cur_cmdq_addr_l_sz;
 78
 79	cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz);
 80	iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) |
 81		    state << MLX5_NIC_IFC_OFFSET,
 82		    &dev->iseg->cmdq_addr_l_sz);
 83}
 84
 85static bool sensor_pci_not_working(struct mlx5_core_dev *dev)
 86{
 87	struct mlx5_core_health *health = &dev->priv.health;
 88	struct health_buffer __iomem *h = health->health;
 89
 90	/* Offline PCI reads return 0xffffffff */
 91	return (ioread32be(&h->fw_ver) == 0xffffffff);
 92}
 93
 94static int mlx5_health_get_rfr(u8 rfr_severity)
 95{
 96	return rfr_severity >> MLX5_RFR_BIT_OFFSET;
 97}
 98
 99static int mlx5_health_get_crr(u8 rfr_severity)
100{
101	return (rfr_severity >> MLX5_CRR_BIT_OFFSET) & 0x01;
102}
103
104static bool sensor_fw_synd_rfr(struct mlx5_core_dev *dev)
105{
106	struct mlx5_core_health *health = &dev->priv.health;
107	struct health_buffer __iomem *h = health->health;
108	u8 synd = ioread8(&h->synd);
109	u8 rfr;
110
111	rfr = mlx5_health_get_rfr(ioread8(&h->rfr_severity));
112
113	if (rfr && synd)
114		mlx5_core_dbg(dev, "FW requests reset, synd: %d\n", synd);
115	return rfr && synd;
116}
117
118u32 mlx5_health_check_fatal_sensors(struct mlx5_core_dev *dev)
119{
120	if (sensor_pci_not_working(dev))
121		return MLX5_SENSOR_PCI_COMM_ERR;
122	if (pci_channel_offline(dev->pdev))
123		return MLX5_SENSOR_PCI_ERR;
124	if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED)
125		return MLX5_SENSOR_NIC_DISABLED;
126	if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET)
127		return MLX5_SENSOR_NIC_SW_RESET;
128	if (sensor_fw_synd_rfr(dev))
129		return MLX5_SENSOR_FW_SYND_RFR;
130
131	return MLX5_SENSOR_NO_ERR;
132}
133
134static int lock_sem_sw_reset(struct mlx5_core_dev *dev, bool lock)
135{
136	enum mlx5_vsc_state state;
137	int ret;
138
139	if (!mlx5_core_is_pf(dev))
140		return -EBUSY;
141
142	/* Try to lock GW access, this stage doesn't return
143	 * EBUSY because locked GW does not mean that other PF
144	 * already started the reset.
145	 */
146	ret = mlx5_vsc_gw_lock(dev);
147	if (ret == -EBUSY)
148		return -EINVAL;
149	if (ret)
150		return ret;
151
152	state = lock ? MLX5_VSC_LOCK : MLX5_VSC_UNLOCK;
153	/* At this stage, if the return status == EBUSY, then we know
154	 * for sure that another PF started the reset, so don't allow
155	 * another reset.
156	 */
157	ret = mlx5_vsc_sem_set_space(dev, MLX5_SEMAPHORE_SW_RESET, state);
158	if (ret)
159		mlx5_core_warn(dev, "Failed to lock SW reset semaphore\n");
160
161	/* Unlock GW access */
162	mlx5_vsc_gw_unlock(dev);
163
164	return ret;
165}
166
167static bool reset_fw_if_needed(struct mlx5_core_dev *dev)
168{
169	bool supported = (ioread32be(&dev->iseg->initializing) >>
170			  MLX5_FW_RESET_SUPPORTED_OFFSET) & 1;
171	u32 fatal_error;
172
173	if (!supported)
174		return false;
175
176	/* The reset only needs to be issued by one PF. The health buffer is
177	 * shared between all functions, and will be cleared during a reset.
178	 * Check again to avoid a redundant 2nd reset. If the fatal errors was
179	 * PCI related a reset won't help.
180	 */
181	fatal_error = mlx5_health_check_fatal_sensors(dev);
182	if (fatal_error == MLX5_SENSOR_PCI_COMM_ERR ||
183	    fatal_error == MLX5_SENSOR_NIC_DISABLED ||
184	    fatal_error == MLX5_SENSOR_NIC_SW_RESET) {
185		mlx5_core_warn(dev, "Not issuing FW reset. Either it's already done or won't help.");
186		return false;
187	}
188
189	mlx5_core_warn(dev, "Issuing FW Reset\n");
190	/* Write the NIC interface field to initiate the reset, the command
191	 * interface address also resides here, don't overwrite it.
192	 */
193	mlx5_set_nic_state(dev, MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET);
194
195	return true;
196}
197
198static void enter_error_state(struct mlx5_core_dev *dev, bool force)
199{
200	if (mlx5_health_check_fatal_sensors(dev) || force) { /* protected state setting */
201		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
202		mlx5_cmd_flush(dev);
203	}
204
205	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
206}
207
208void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
209{
210	bool err_detected = false;
211
212	/* Mark the device as fatal in order to abort FW commands */
213	if ((mlx5_health_check_fatal_sensors(dev) || force) &&
214	    dev->state == MLX5_DEVICE_STATE_UP) {
215		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
216		err_detected = true;
217	}
218	mutex_lock(&dev->intf_state_mutex);
219	if (!err_detected && dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
220		goto unlock;/* a previous error is still being handled */
221
222	enter_error_state(dev, force);
223unlock:
224	mutex_unlock(&dev->intf_state_mutex);
225}
226
227void mlx5_error_sw_reset(struct mlx5_core_dev *dev)
228{
229	unsigned long end, delay_ms = mlx5_tout_ms(dev, PCI_TOGGLE);
230	int lock = -EBUSY;
231
232	mutex_lock(&dev->intf_state_mutex);
233	if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR)
234		goto unlock;
235
236	mlx5_core_err(dev, "start\n");
237
238	if (mlx5_health_check_fatal_sensors(dev) == MLX5_SENSOR_FW_SYND_RFR) {
239		/* Get cr-dump and reset FW semaphore */
240		lock = lock_sem_sw_reset(dev, true);
241
242		if (lock == -EBUSY) {
243			delay_ms = mlx5_tout_ms(dev, FULL_CRDUMP);
244			goto recover_from_sw_reset;
245		}
246		/* Execute SW reset */
247		reset_fw_if_needed(dev);
248	}
249
250recover_from_sw_reset:
251	/* Recover from SW reset */
252	end = jiffies + msecs_to_jiffies(delay_ms);
253	do {
254		if (mlx5_get_nic_state(dev) == MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED)
255			break;
256		if (pci_channel_offline(dev->pdev)) {
257			mlx5_core_err(dev, "PCI channel offline, stop waiting for NIC IFC\n");
258			goto unlock;
259		}
260
261		msleep(20);
262	} while (!time_after(jiffies, end));
263
264	if (mlx5_get_nic_state(dev) != MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED) {
265		dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n",
266			mlx5_get_nic_state(dev), delay_ms);
267	}
268
269	/* Release FW semaphore if you are the lock owner */
270	if (!lock)
271		lock_sem_sw_reset(dev, false);
272
273	mlx5_core_err(dev, "end\n");
274
275unlock:
276	mutex_unlock(&dev->intf_state_mutex);
277}
278
279static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
280{
281	u8 nic_interface = mlx5_get_nic_state(dev);
282
283	switch (nic_interface) {
284	case MLX5_INITIAL_SEG_NIC_INTERFACE_FULL_DRIVER:
285		mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
286		break;
287
288	case MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED:
289		mlx5_core_warn(dev, "starting teardown\n");
290		break;
291
292	case MLX5_INITIAL_SEG_NIC_INTERFACE_NO_DRAM_NIC:
293		mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
294		break;
295
296	case MLX5_INITIAL_SEG_NIC_INTERFACE_SW_RESET:
297		/* The IFC mode field is 3 bits, so it will read 0x7 in 2 cases:
298		 * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded
299		 *    and this is a VF), this is not recoverable by SW reset.
300		 *    Logging of this is handled elsewhere.
301		 * 2. FW reset has been issued by another function, driver can
302		 *    be reloaded to recover after the mode switches to
303		 *    MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED.
304		 */
305		if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR)
306			mlx5_core_warn(dev, "NIC SW reset in progress\n");
307		break;
308
309	default:
310		mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
311			       nic_interface);
312	}
313
314	mlx5_disable_device(dev);
315}
316
317int mlx5_health_wait_pci_up(struct mlx5_core_dev *dev)
318{
319	unsigned long end;
320
321	end = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, FW_RESET));
322	while (sensor_pci_not_working(dev)) {
323		if (time_after(jiffies, end))
324			return -ETIMEDOUT;
325		if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
326			mlx5_core_warn(dev, "device is being removed, stop waiting for PCI\n");
327			return -ENODEV;
328		}
329		if (pci_channel_offline(dev->pdev)) {
330			mlx5_core_err(dev, "PCI channel offline, stop waiting for PCI\n");
331			return -EACCES;
332		}
333		msleep(100);
334	}
335	return 0;
336}
337
338static int mlx5_health_try_recover(struct mlx5_core_dev *dev)
339{
340	mlx5_core_warn(dev, "handling bad device here\n");
341	mlx5_handle_bad_state(dev);
342	if (mlx5_health_wait_pci_up(dev)) {
343		mlx5_core_err(dev, "health recovery flow aborted, PCI reads still not working\n");
344		return -EIO;
345	}
346	mlx5_core_err(dev, "starting health recovery flow\n");
347	if (mlx5_recover_device(dev) || mlx5_health_check_fatal_sensors(dev)) {
348		mlx5_core_err(dev, "health recovery failed\n");
349		return -EIO;
350	}
351
352	mlx5_core_info(dev, "health recovery succeeded\n");
353	return 0;
354}
355
356static const char *hsynd_str(u8 synd)
357{
358	switch (synd) {
359	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_FW_INTERNAL_ERR:
360		return "firmware internal error";
361	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_DEAD_IRISC:
362		return "irisc not responding";
363	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_HW_FATAL_ERR:
364		return "unrecoverable hardware error";
365	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_FW_CRC_ERR:
366		return "firmware CRC error";
367	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_ICM_FETCH_PCI_ERR:
368		return "ICM fetch PCI error";
369	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_ICM_PAGE_ERR:
370		return "HW fatal error\n";
371	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_ASYNCHRONOUS_EQ_BUF_OVERRUN:
372		return "async EQ buffer overrun";
373	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_EQ_IN_ERR:
374		return "EQ error";
375	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_EQ_INV:
376		return "Invalid EQ referenced";
377	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_FFSER_ERR:
378		return "FFSER error";
379	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_HIGH_TEMP_ERR:
380		return "High temperature";
381	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_ICM_PCI_POISONED_ERR:
382		return "ICM fetch PCI data poisoned error";
383	case MLX5_INITIAL_SEG_HEALTH_SYNDROME_TRUST_LOCKDOWN_ERR:
384		return "Trust lockdown error";
385	default:
386		return "unrecognized error";
387	}
388}
389
390static const char *mlx5_loglevel_str(int level)
391{
392	switch (level) {
393	case LOGLEVEL_EMERG:
394		return "EMERGENCY";
395	case LOGLEVEL_ALERT:
396		return "ALERT";
397	case LOGLEVEL_CRIT:
398		return "CRITICAL";
399	case LOGLEVEL_ERR:
400		return "ERROR";
401	case LOGLEVEL_WARNING:
402		return "WARNING";
403	case LOGLEVEL_NOTICE:
404		return "NOTICE";
405	case LOGLEVEL_INFO:
406		return "INFO";
407	case LOGLEVEL_DEBUG:
408		return "DEBUG";
409	}
410	return "Unknown log level";
411}
412
413static int mlx5_health_get_severity(u8 rfr_severity)
414{
415	return rfr_severity & MLX5_SEVERITY_VALID_MASK ?
416	       rfr_severity & MLX5_SEVERITY_MASK : LOGLEVEL_ERR;
417}
418
419static void print_health_info(struct mlx5_core_dev *dev)
420{
421	struct mlx5_core_health *health = &dev->priv.health;
422	struct health_buffer __iomem *h = health->health;
423	u8 rfr_severity;
424	int severity;
425	int i;
426
427	/* If the syndrome is 0, the device is OK and no need to print buffer */
428	if (!ioread8(&h->synd))
429		return;
430
431	if (ioread32be(&h->fw_ver) == 0xFFFFFFFF) {
432		mlx5_log(dev, LOGLEVEL_ERR, "PCI slot is unavailable\n");
433		return;
434	}
435
436	rfr_severity = ioread8(&h->rfr_severity);
437	severity  = mlx5_health_get_severity(rfr_severity);
438	mlx5_log(dev, severity, "Health issue observed, %s, severity(%d) %s:\n",
439		 hsynd_str(ioread8(&h->synd)), severity, mlx5_loglevel_str(severity));
440
441	for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
442		mlx5_log(dev, severity, "assert_var[%d] 0x%08x\n", i,
443			 ioread32be(h->assert_var + i));
444
445	mlx5_log(dev, severity, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr));
446	mlx5_log(dev, severity, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra));
447	mlx5_log(dev, severity, "fw_ver %d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev),
448		 fw_rev_sub(dev));
449	mlx5_log(dev, severity, "time %u\n", ioread32be(&h->time));
450	mlx5_log(dev, severity, "hw_id 0x%08x\n", ioread32be(&h->hw_id));
451	mlx5_log(dev, severity, "rfr %d\n", mlx5_health_get_rfr(rfr_severity));
452	mlx5_log(dev, severity, "crr %d\n", mlx5_health_get_crr(rfr_severity));
453	mlx5_log(dev, severity, "severity %d (%s)\n", severity, mlx5_loglevel_str(severity));
454	mlx5_log(dev, severity, "irisc_index %d\n", ioread8(&h->irisc_index));
455	mlx5_log(dev, severity, "synd 0x%x: %s\n", ioread8(&h->synd),
456		 hsynd_str(ioread8(&h->synd)));
457	mlx5_log(dev, severity, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd));
458	mlx5_log(dev, severity, "raw fw_ver 0x%08x\n", ioread32be(&h->fw_ver));
459	if (mlx5_health_get_crr(rfr_severity))
460		mlx5_core_warn(dev, "Cold reset is required\n");
461}
462
463static int
464mlx5_fw_reporter_diagnose(struct devlink_health_reporter *reporter,
465			  struct devlink_fmsg *fmsg,
466			  struct netlink_ext_ack *extack)
467{
468	struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
469	struct mlx5_core_health *health = &dev->priv.health;
470	struct health_buffer __iomem *h = health->health;
471	u8 synd = ioread8(&h->synd);
472
473	devlink_fmsg_u8_pair_put(fmsg, "Syndrome", synd);
474	if (!synd)
475		return 0;
476
477	devlink_fmsg_string_pair_put(fmsg, "Description", hsynd_str(synd));
478
479	return 0;
480}
481
482struct mlx5_fw_reporter_ctx {
483	u8 err_synd;
484	int miss_counter;
485};
486
487static void
488mlx5_fw_reporter_ctx_pairs_put(struct devlink_fmsg *fmsg,
489			       struct mlx5_fw_reporter_ctx *fw_reporter_ctx)
490{
491	devlink_fmsg_u8_pair_put(fmsg, "syndrome", fw_reporter_ctx->err_synd);
492	devlink_fmsg_u32_pair_put(fmsg, "fw_miss_counter", fw_reporter_ctx->miss_counter);
493}
494
495static void
496mlx5_fw_reporter_heath_buffer_data_put(struct mlx5_core_dev *dev,
497				       struct devlink_fmsg *fmsg)
498{
499	struct mlx5_core_health *health = &dev->priv.health;
500	struct health_buffer __iomem *h = health->health;
501	u8 rfr_severity;
502	int i;
503
504	if (!ioread8(&h->synd))
505		return;
506
507	devlink_fmsg_pair_nest_start(fmsg, "health buffer");
508	devlink_fmsg_obj_nest_start(fmsg);
509	devlink_fmsg_arr_pair_nest_start(fmsg, "assert_var");
510	for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
511		devlink_fmsg_u32_put(fmsg, ioread32be(h->assert_var + i));
512	devlink_fmsg_arr_pair_nest_end(fmsg);
513	devlink_fmsg_u32_pair_put(fmsg, "assert_exit_ptr",
514				  ioread32be(&h->assert_exit_ptr));
515	devlink_fmsg_u32_pair_put(fmsg, "assert_callra",
516				  ioread32be(&h->assert_callra));
517	devlink_fmsg_u32_pair_put(fmsg, "time", ioread32be(&h->time));
518	devlink_fmsg_u32_pair_put(fmsg, "hw_id", ioread32be(&h->hw_id));
519	rfr_severity = ioread8(&h->rfr_severity);
520	devlink_fmsg_u8_pair_put(fmsg, "rfr", mlx5_health_get_rfr(rfr_severity));
521	devlink_fmsg_u8_pair_put(fmsg, "severity", mlx5_health_get_severity(rfr_severity));
522	devlink_fmsg_u8_pair_put(fmsg, "irisc_index", ioread8(&h->irisc_index));
523	devlink_fmsg_u8_pair_put(fmsg, "synd", ioread8(&h->synd));
524	devlink_fmsg_u32_pair_put(fmsg, "ext_synd", ioread16be(&h->ext_synd));
525	devlink_fmsg_u32_pair_put(fmsg, "raw_fw_ver", ioread32be(&h->fw_ver));
526	devlink_fmsg_obj_nest_end(fmsg);
527	devlink_fmsg_pair_nest_end(fmsg);
528}
529
530static int
531mlx5_fw_reporter_dump(struct devlink_health_reporter *reporter,
532		      struct devlink_fmsg *fmsg, void *priv_ctx,
533		      struct netlink_ext_ack *extack)
534{
535	struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
536	int err;
537
538	err = mlx5_fw_tracer_trigger_core_dump_general(dev);
539	if (err)
540		return err;
541
542	if (priv_ctx) {
543		struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx;
544
545		mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx);
546	}
547
548	mlx5_fw_reporter_heath_buffer_data_put(dev, fmsg);
549
550	return mlx5_fw_tracer_get_saved_traces_objects(dev->tracer, fmsg);
551}
552
553static void mlx5_fw_reporter_err_work(struct work_struct *work)
554{
555	struct mlx5_fw_reporter_ctx fw_reporter_ctx;
556	struct mlx5_core_health *health;
557
558	health = container_of(work, struct mlx5_core_health, report_work);
559
560	if (IS_ERR_OR_NULL(health->fw_reporter))
561		return;
562
563	fw_reporter_ctx.err_synd = health->synd;
564	fw_reporter_ctx.miss_counter = health->miss_counter;
565	if (fw_reporter_ctx.err_synd) {
566		devlink_health_report(health->fw_reporter,
567				      "FW syndrome reported", &fw_reporter_ctx);
568		return;
569	}
570	if (fw_reporter_ctx.miss_counter)
571		devlink_health_report(health->fw_reporter,
572				      "FW miss counter reported",
573				      &fw_reporter_ctx);
574}
575
576static const struct devlink_health_reporter_ops mlx5_fw_reporter_pf_ops = {
577		.name = "fw",
578		.diagnose = mlx5_fw_reporter_diagnose,
579		.dump = mlx5_fw_reporter_dump,
580};
581
582static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = {
583		.name = "fw",
584		.diagnose = mlx5_fw_reporter_diagnose,
585};
586
587static int
588mlx5_fw_fatal_reporter_recover(struct devlink_health_reporter *reporter,
589			       void *priv_ctx,
590			       struct netlink_ext_ack *extack)
591{
592	struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
593
594	return mlx5_health_try_recover(dev);
595}
596
597static int
598mlx5_fw_fatal_reporter_dump(struct devlink_health_reporter *reporter,
599			    struct devlink_fmsg *fmsg, void *priv_ctx,
600			    struct netlink_ext_ack *extack)
601{
602	struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
603	u32 crdump_size = dev->priv.health.crdump_size;
604	u32 *cr_data;
605	int err;
606
607	if (!mlx5_core_is_pf(dev))
608		return -EPERM;
609
610	cr_data = kvmalloc(crdump_size, GFP_KERNEL);
611	if (!cr_data)
612		return -ENOMEM;
613	err = mlx5_crdump_collect(dev, cr_data);
614	if (err)
615		goto free_data;
616
617	if (priv_ctx) {
618		struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx;
619
620		mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx);
621	}
622
623	devlink_fmsg_binary_pair_put(fmsg, "crdump_data", cr_data, crdump_size);
624
625free_data:
626	kvfree(cr_data);
627	return err;
628}
629
630static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work)
631{
632	struct mlx5_fw_reporter_ctx fw_reporter_ctx;
633	struct mlx5_core_health *health;
634	struct mlx5_core_dev *dev;
635	struct devlink *devlink;
636	struct mlx5_priv *priv;
637
638	health = container_of(work, struct mlx5_core_health, fatal_report_work);
639	priv = container_of(health, struct mlx5_priv, health);
640	dev = container_of(priv, struct mlx5_core_dev, priv);
641	devlink = priv_to_devlink(dev);
642
643	mutex_lock(&dev->intf_state_mutex);
644	if (test_bit(MLX5_DROP_HEALTH_WORK, &health->flags)) {
645		mlx5_core_err(dev, "health works are not permitted at this stage\n");
646		mutex_unlock(&dev->intf_state_mutex);
647		return;
648	}
649	mutex_unlock(&dev->intf_state_mutex);
650	enter_error_state(dev, false);
651	if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) {
652		devl_lock(devlink);
653		if (mlx5_health_try_recover(dev))
654			mlx5_core_err(dev, "health recovery failed\n");
655		devl_unlock(devlink);
656		return;
657	}
658	fw_reporter_ctx.err_synd = health->synd;
659	fw_reporter_ctx.miss_counter = health->miss_counter;
660	if (devlink_health_report(health->fw_fatal_reporter,
661				  "FW fatal error reported", &fw_reporter_ctx) == -ECANCELED) {
662		/* If recovery wasn't performed, due to grace period,
663		 * unload the driver. This ensures that the driver
664		 * closes all its resources and it is not subjected to
665		 * requests from the kernel.
666		 */
667		mlx5_core_err(dev, "Driver is in error state. Unloading\n");
668		mlx5_unload_one(dev, false);
669	}
670}
671
672#define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000
673#define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000
674#define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000
675#define MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD \
676	MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD
677
678static
679const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ecpf_ops = {
680		.name = "fw_fatal",
681		.recover = mlx5_fw_fatal_reporter_recover,
682		.dump = mlx5_fw_fatal_reporter_dump,
683		.default_graceful_period =
684			MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD,
685};
686
687static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_pf_ops = {
688		.name = "fw_fatal",
689		.recover = mlx5_fw_fatal_reporter_recover,
690		.dump = mlx5_fw_fatal_reporter_dump,
691		.default_graceful_period = MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD,
692};
693
694static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = {
695		.name = "fw_fatal",
696		.recover = mlx5_fw_fatal_reporter_recover,
697		.default_graceful_period =
698			MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD,
699};
700
701void mlx5_fw_reporters_create(struct mlx5_core_dev *dev)
702{
703	const struct devlink_health_reporter_ops *fw_fatal_ops;
704	struct mlx5_core_health *health = &dev->priv.health;
705	const struct devlink_health_reporter_ops *fw_ops;
706	struct devlink *devlink = priv_to_devlink(dev);
707
708	fw_ops = &mlx5_fw_reporter_pf_ops;
709	if (mlx5_core_is_ecpf(dev)) {
710		fw_fatal_ops = &mlx5_fw_fatal_reporter_ecpf_ops;
711	} else if (mlx5_core_is_pf(dev)) {
712		fw_fatal_ops = &mlx5_fw_fatal_reporter_pf_ops;
713	} else {
714		/* VF or SF */
715		fw_fatal_ops = &mlx5_fw_fatal_reporter_ops;
716		fw_ops = &mlx5_fw_reporter_ops;
717	}
718
719	health->fw_reporter = devl_health_reporter_create(devlink, fw_ops, dev);
720	if (IS_ERR(health->fw_reporter))
721		mlx5_core_warn(dev, "Failed to create fw reporter, err = %pe\n",
722			       health->fw_reporter);
723
724	health->fw_fatal_reporter = devl_health_reporter_create(devlink,
725								fw_fatal_ops,
726								dev);
727	if (IS_ERR(health->fw_fatal_reporter))
728		mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %pe\n",
729			       health->fw_fatal_reporter);
730}
731
732static void mlx5_fw_reporters_destroy(struct mlx5_core_dev *dev)
733{
734	struct mlx5_core_health *health = &dev->priv.health;
735
736	if (!IS_ERR_OR_NULL(health->fw_reporter))
737		devlink_health_reporter_destroy(health->fw_reporter);
738
739	if (!IS_ERR_OR_NULL(health->fw_fatal_reporter))
740		devlink_health_reporter_destroy(health->fw_fatal_reporter);
741}
742
743static unsigned long get_next_poll_jiffies(struct mlx5_core_dev *dev)
744{
745	unsigned long next;
746
747	get_random_bytes(&next, sizeof(next));
748	next %= HZ;
749	next += jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, HEALTH_POLL_INTERVAL));
750
751	return next;
752}
753
754void mlx5_trigger_health_work(struct mlx5_core_dev *dev)
755{
756	struct mlx5_core_health *health = &dev->priv.health;
757
758	if (!mlx5_dev_is_lightweight(dev))
759		queue_work(health->wq, &health->fatal_report_work);
760}
761
762#define MLX5_MSEC_PER_HOUR (MSEC_PER_SEC * 60 * 60)
763static void mlx5_health_log_ts_update(struct work_struct *work)
764{
765	struct delayed_work *dwork = to_delayed_work(work);
766	u32 out[MLX5_ST_SZ_DW(mrtc_reg)] = {};
767	u32 in[MLX5_ST_SZ_DW(mrtc_reg)] = {};
768	struct mlx5_core_health *health;
769	struct mlx5_core_dev *dev;
770	struct mlx5_priv *priv;
771	u64 now_us;
772
773	health = container_of(dwork, struct mlx5_core_health, update_fw_log_ts_work);
774	priv = container_of(health, struct mlx5_priv, health);
775	dev = container_of(priv, struct mlx5_core_dev, priv);
776
777	now_us =  ktime_to_us(ktime_get_real());
778
779	MLX5_SET(mrtc_reg, in, time_h, now_us >> 32);
780	MLX5_SET(mrtc_reg, in, time_l, now_us & 0xFFFFFFFF);
781	mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MRTC, 0, 1);
782
783	queue_delayed_work(health->wq, &health->update_fw_log_ts_work,
784			   msecs_to_jiffies(MLX5_MSEC_PER_HOUR));
785}
786
787static void poll_health(struct timer_list *t)
788{
789	struct mlx5_core_dev *dev = timer_container_of(dev, t,
790						       priv.health.timer);
791	struct mlx5_core_health *health = &dev->priv.health;
792	struct health_buffer __iomem *h = health->health;
793	u32 fatal_error;
794	u8 prev_synd;
795	u32 count;
796
797	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
798		goto out;
799
800	fatal_error = mlx5_health_check_fatal_sensors(dev);
801
802	if (fatal_error && !health->fatal_error) {
803		mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error);
804		dev->priv.health.fatal_error = fatal_error;
805		print_health_info(dev);
806		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
807		mlx5_trigger_health_work(dev);
808		return;
809	}
810
811	count = ioread32be(health->health_counter);
812	if (count == health->prev)
813		++health->miss_counter;
814	else
815		health->miss_counter = 0;
816
817	health->prev = count;
818	if (health->miss_counter == MAX_MISSES) {
819		mlx5_core_err(dev, "device's health compromised - reached miss count\n");
820		health->synd = ioread8(&h->synd);
821		print_health_info(dev);
822		queue_work(health->wq, &health->report_work);
823	}
824
825	prev_synd = health->synd;
826	health->synd = ioread8(&h->synd);
827	if (health->synd && health->synd != prev_synd) {
828		print_health_info(dev);
829		queue_work(health->wq, &health->report_work);
830	}
831
832out:
833	mod_timer(&health->timer, get_next_poll_jiffies(dev));
834}
835
836void mlx5_start_health_poll(struct mlx5_core_dev *dev)
837{
838	u64 poll_interval_ms =  mlx5_tout_ms(dev, HEALTH_POLL_INTERVAL);
839	struct mlx5_core_health *health = &dev->priv.health;
840
841	timer_setup(&health->timer, poll_health, 0);
842	health->fatal_error = MLX5_SENSOR_NO_ERR;
843	clear_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
844	health->health = &dev->iseg->health;
845	health->health_counter = &dev->iseg->health_counter;
846
847	health->timer.expires = jiffies + msecs_to_jiffies(poll_interval_ms);
848	add_timer(&health->timer);
849}
850
851void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
852{
853	struct mlx5_core_health *health = &dev->priv.health;
854
855	if (disable_health)
856		set_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
857
858	timer_delete_sync(&health->timer);
859}
860
861void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev)
862{
863	struct mlx5_core_health *health = &dev->priv.health;
864
865	if (mlx5_core_is_pf(dev) && MLX5_CAP_MCAM_REG(dev, mrtc))
866		queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0);
867}
868
869void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
870{
871	struct mlx5_core_health *health = &dev->priv.health;
872
873	set_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
874	cancel_delayed_work_sync(&health->update_fw_log_ts_work);
875	cancel_work_sync(&health->report_work);
876	cancel_work_sync(&health->fatal_report_work);
877}
878
879void mlx5_health_cleanup(struct mlx5_core_dev *dev)
880{
881	struct mlx5_core_health *health = &dev->priv.health;
882
883	cancel_delayed_work_sync(&health->update_fw_log_ts_work);
884	destroy_workqueue(health->wq);
885	mlx5_reporter_vnic_destroy(dev);
886	mlx5_fw_reporters_destroy(dev);
887}
888
889int mlx5_health_init(struct mlx5_core_dev *dev)
890{
891	struct devlink *devlink = priv_to_devlink(dev);
892	struct mlx5_core_health *health;
893	char *name;
894
895	if (!mlx5_dev_is_lightweight(dev)) {
896		devl_lock(devlink);
897		mlx5_fw_reporters_create(dev);
898		devl_unlock(devlink);
899	}
900	mlx5_reporter_vnic_create(dev);
901
902	health = &dev->priv.health;
903	name = kmalloc(64, GFP_KERNEL);
904	if (!name)
905		goto out_err;
906
907	strcpy(name, "mlx5_health");
908	strcat(name, dev_name(dev->device));
909	health->wq = create_singlethread_workqueue(name);
910	kfree(name);
911	if (!health->wq)
912		goto out_err;
913	INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work);
914	INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work);
915	INIT_DELAYED_WORK(&health->update_fw_log_ts_work, mlx5_health_log_ts_update);
916
917	return 0;
918
919out_err:
920	mlx5_reporter_vnic_destroy(dev);
921	mlx5_fw_reporters_destroy(dev);
922	return -ENOMEM;
923}