fw_reset.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved. */
  3
  4#include <devlink.h>
  5
  6#include "fw_reset.h"
  7#include "diag/fw_tracer.h"
  8#include "lib/tout.h"
  9#include "sf/sf.h"
 10
 11enum {
 12	MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
 13	MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
 14	MLX5_FW_RESET_FLAGS_PENDING_COMP,
 15	MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
 16	MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED,
 17	MLX5_FW_RESET_FLAGS_UNLOAD_EVENT,
 18	MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS,
 19};
 20
 21struct mlx5_fw_reset {
 22	struct mlx5_core_dev *dev;
 23	struct mlx5_nb nb;
 24	struct workqueue_struct *wq;
 25	struct work_struct fw_live_patch_work;
 26	struct work_struct reset_request_work;
 27	struct work_struct reset_unload_work;
 28	struct work_struct reset_reload_work;
 29	struct work_struct reset_now_work;
 30	struct work_struct reset_abort_work;
 31	struct delayed_work reset_timeout_work;
 32	unsigned long reset_flags;
 33	u8 reset_method;
 34	struct timer_list timer;
 35	struct completion done;
 36	int ret;
 37};
 38
 39enum {
 40	MLX5_FW_RST_STATE_IDLE = 0,
 41	MLX5_FW_RST_STATE_TOGGLE_REQ = 4,
 42	MLX5_FW_RST_STATE_DROP_MODE = 5,
 43};
 44
 45enum {
 46	MLX5_RST_STATE_BIT_NUM = 12,
 47	MLX5_RST_ACK_BIT_NUM = 22,
 48};
 49
 50static u8 mlx5_get_fw_rst_state(struct mlx5_core_dev *dev)
 51{
 52	return (ioread32be(&dev->iseg->initializing) >> MLX5_RST_STATE_BIT_NUM) & 0xF;
 53}
 54
 55static void mlx5_set_fw_rst_ack(struct mlx5_core_dev *dev)
 56{
 57	iowrite32be(BIT(MLX5_RST_ACK_BIT_NUM), &dev->iseg->initializing);
 58}
 59
 60static int mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink *devlink, u32 id,
 61						     struct devlink_param_gset_ctx *ctx,
 62						     struct netlink_ext_ack *extack)
 63{
 64	struct mlx5_core_dev *dev = devlink_priv(devlink);
 65	struct mlx5_fw_reset *fw_reset;
 66
 67	fw_reset = dev->priv.fw_reset;
 68
 69	if (ctx->val.vbool)
 70		clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
 71	else
 72		set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
 73	return 0;
 74}
 75
 76static int mlx5_fw_reset_enable_remote_dev_reset_get(struct devlink *devlink, u32 id,
 77						     struct devlink_param_gset_ctx *ctx,
 78						     struct netlink_ext_ack *extack)
 79{
 80	struct mlx5_core_dev *dev = devlink_priv(devlink);
 81	struct mlx5_fw_reset *fw_reset;
 82
 83	fw_reset = dev->priv.fw_reset;
 84
 85	ctx->val.vbool = !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
 86				   &fw_reset->reset_flags);
 87	return 0;
 88}
 89
 90static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
 91			     u8 reset_type_sel, u8 sync_resp, bool sync_start)
 92{
 93	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
 94	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
 95
 96	MLX5_SET(mfrl_reg, in, reset_level, reset_level);
 97	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
 98	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
 99	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);
100
101	return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
102}
103
104static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level,
105			       u8 *reset_type, u8 *reset_state, u8 *reset_method)
106{
107	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
108	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
109	int err;
110
111	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
112	if (err)
113		return err;
114
115	if (reset_level)
116		*reset_level = MLX5_GET(mfrl_reg, out, reset_level);
117	if (reset_type)
118		*reset_type = MLX5_GET(mfrl_reg, out, reset_type);
119	if (reset_state)
120		*reset_state = MLX5_GET(mfrl_reg, out, reset_state);
121	if (reset_method)
122		*reset_method = MLX5_GET(mfrl_reg, out, pci_reset_req_method);
123
124	return 0;
125}
126
127int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
128{
129	return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL, NULL);
130}
131
132bool mlx5_fw_reset_in_progress(struct mlx5_core_dev *dev)
133{
134	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
135
136	if (!fw_reset)
137		return false;
138
139	return test_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
140}
141
142static int mlx5_fw_reset_get_reset_method(struct mlx5_core_dev *dev,
143					  u8 *reset_method)
144{
145	if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method)) {
146		*reset_method = MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE;
147		return 0;
148	}
149
150	return mlx5_reg_mfrl_query(dev, NULL, NULL, NULL, reset_method);
151}
152
153static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev,
154					     struct netlink_ext_ack *extack)
155{
156	u8 reset_state;
157
158	if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state, NULL))
159		goto out;
160
161	if (!reset_state)
162		return 0;
163
164	switch (reset_state) {
165	case MLX5_MFRL_REG_RESET_STATE_IN_NEGOTIATION:
166	case MLX5_MFRL_REG_RESET_STATE_RESET_IN_PROGRESS:
167		NL_SET_ERR_MSG_MOD(extack, "Sync reset still in progress");
168		return -EBUSY;
169	case MLX5_MFRL_REG_RESET_STATE_NEG_TIMEOUT:
170		NL_SET_ERR_MSG_MOD(extack, "Sync reset negotiation timeout");
171		return -ETIMEDOUT;
172	case MLX5_MFRL_REG_RESET_STATE_NACK:
173		NL_SET_ERR_MSG_MOD(extack, "One of the hosts disabled reset");
174		return -EPERM;
175	case MLX5_MFRL_REG_RESET_STATE_UNLOAD_TIMEOUT:
176		NL_SET_ERR_MSG_MOD(extack, "Sync reset unload timeout");
177		return -ETIMEDOUT;
178	}
179
180out:
181	NL_SET_ERR_MSG_MOD(extack, "Sync reset failed");
182	return -EIO;
183}
184
185int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel,
186				 struct netlink_ext_ack *extack)
187{
188	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
189	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
190	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
191	int err, rst_res;
192
193	set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
194
195	MLX5_SET(mfrl_reg, in, reset_level, MLX5_MFRL_REG_RESET_LEVEL3);
196	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
197	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, 1);
198	err = mlx5_access_reg(dev, in, sizeof(in), out, sizeof(out),
199			      MLX5_REG_MFRL, 0, 1, false);
200	if (!err)
201		return 0;
202
203	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
204	if (err == -EREMOTEIO && MLX5_CAP_MCAM_FEATURE(dev, reset_state)) {
205		rst_res = mlx5_fw_reset_get_reset_state_err(dev, extack);
206		return rst_res ? rst_res : err;
207	}
208
209	NL_SET_ERR_MSG_MOD(extack, "Sync reset command failed");
210	return mlx5_cmd_check(dev, err, in, out);
211}
212
213int mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev *dev,
214				     struct netlink_ext_ack *extack)
215{
216	u8 rst_state;
217	int err;
218
219	err = mlx5_fw_reset_get_reset_state_err(dev, extack);
220	if (err)
221		return err;
222
223	rst_state = mlx5_get_fw_rst_state(dev);
224	if (!rst_state)
225		return 0;
226
227	mlx5_core_err(dev, "Sync reset did not complete, state=%d\n", rst_state);
228	NL_SET_ERR_MSG_MOD(extack, "Sync reset did not complete successfully");
229	return rst_state;
230}
231
232int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
233{
234	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
235}
236
237static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
238{
239	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
240	struct devlink *devlink = priv_to_devlink(dev);
241
242	/* if this is the driver that initiated the fw reset, devlink completed the reload */
243	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
244		complete(&fw_reset->done);
245	} else {
246		mlx5_sync_reset_unload_flow(dev, false);
247		if (mlx5_health_wait_pci_up(dev))
248			mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
249		else
250			mlx5_load_one(dev, true);
251		devl_lock(devlink);
252		devlink_remote_reload_actions_performed(devlink, 0,
253							BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
254							BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
255		devl_unlock(devlink);
256	}
257
258	clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
259}
260
261static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
262{
263	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
264
265	timer_delete_sync(&fw_reset->timer);
266}
267
268static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
269{
270	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
271
272	if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
273		mlx5_core_warn(dev, "Reset request was already cleared\n");
274		return -EALREADY;
275	}
276
277	if (current_work() != &fw_reset->reset_timeout_work.work)
278		cancel_delayed_work(&fw_reset->reset_timeout_work);
279	mlx5_stop_sync_reset_poll(dev);
280	if (poll_health)
281		mlx5_start_health_poll(dev);
282	return 0;
283}
284
285static void mlx5_sync_reset_reload_work(struct work_struct *work)
286{
287	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
288						      reset_reload_work);
289	struct mlx5_core_dev *dev = fw_reset->dev;
290
291	mlx5_sync_reset_clear_reset_requested(dev, false);
292	mlx5_enter_error_state(dev, true);
293	mlx5_fw_reset_complete_reload(dev);
294}
295
296#define MLX5_RESET_POLL_INTERVAL	(HZ / 10)
297static void poll_sync_reset(struct timer_list *t)
298{
299	struct mlx5_fw_reset *fw_reset = timer_container_of(fw_reset, t,
300							    timer);
301	struct mlx5_core_dev *dev = fw_reset->dev;
302	u32 fatal_error;
303
304	if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
305		return;
306
307	fatal_error = mlx5_health_check_fatal_sensors(dev);
308
309	if (fatal_error) {
310		mlx5_core_warn(dev, "Got Device Reset\n");
311		if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
312			queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
313		else
314			mlx5_core_err(dev, "Device is being removed, Drop new reset work\n");
315		return;
316	}
317
318	mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
319}
320
321static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
322{
323	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
324
325	timer_setup(&fw_reset->timer, poll_sync_reset, 0);
326	fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
327	add_timer(&fw_reset->timer);
328}
329
330static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
331{
332	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
333}
334
335static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
336{
337	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
338}
339
340static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
341{
342	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
343
344	if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
345		mlx5_core_warn(dev, "Reset request was already set\n");
346		return -EALREADY;
347	}
348	mlx5_stop_health_poll(dev, true);
349	mlx5_start_sync_reset_poll(dev);
350
351	if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
352		      &fw_reset->reset_flags))
353		schedule_delayed_work(&fw_reset->reset_timeout_work,
354			msecs_to_jiffies(mlx5_tout_ms(dev, PCI_SYNC_UPDATE)));
355	return 0;
356}
357
358static void mlx5_fw_live_patch_event(struct work_struct *work)
359{
360	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
361						      fw_live_patch_work);
362	struct mlx5_core_dev *dev = fw_reset->dev;
363
364	mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
365		       fw_rev_min(dev), fw_rev_sub(dev));
366
367	if (mlx5_fw_tracer_reload(dev->tracer))
368		mlx5_core_err(dev, "Failed to reload FW tracer\n");
369}
370
371#if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
372static int mlx5_check_hotplug_interrupt(struct mlx5_core_dev *dev,
373					struct pci_dev *bridge)
374{
375	u16 reg16;
376	int err;
377
378	err = pcie_capability_read_word(bridge, PCI_EXP_SLTCTL, &reg16);
379	if (err)
380		return err;
381
382	if ((reg16 & PCI_EXP_SLTCTL_HPIE) && (reg16 & PCI_EXP_SLTCTL_DLLSCE)) {
383		mlx5_core_warn(dev, "FW reset is not supported as HotPlug is enabled\n");
384		return -EOPNOTSUPP;
385	}
386
387	return 0;
388}
389#endif
390
391static const struct pci_device_id mgt_ifc_device_ids[] = {
392	{ PCI_VDEVICE(MELLANOX, 0xc2d2) }, /* BlueField1 MGT interface device ID */
393	{ PCI_VDEVICE(MELLANOX, 0xc2d3) }, /* BlueField2 MGT interface device ID */
394	{ PCI_VDEVICE(MELLANOX, 0xc2d4) }, /* BlueField3-Lx MGT interface device ID */
395	{ PCI_VDEVICE(MELLANOX, 0xc2d5) }, /* BlueField3 MGT interface device ID */
396	{ PCI_VDEVICE(MELLANOX, 0xc2d6) }, /* BlueField4 MGT interface device ID */
397};
398
399static bool mlx5_is_mgt_ifc_pci_device(struct mlx5_core_dev *dev, u16 dev_id)
400{
401	int i;
402
403	for (i = 0; i < ARRAY_SIZE(mgt_ifc_device_ids); ++i)
404		if (mgt_ifc_device_ids[i].device == dev_id)
405			return true;
406
407	return false;
408}
409
410static int mlx5_check_dev_ids(struct mlx5_core_dev *dev, u16 dev_id)
411{
412	struct pci_bus *bridge_bus = dev->pdev->bus;
413	struct pci_dev *sdev;
414	u16 sdev_id;
415	int err;
416
417	/* Check that all functions under the pci bridge are PFs of
418	 * this device otherwise fail this function.
419	 */
420	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
421		err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
422		if (err)
423			return pcibios_err_to_errno(err);
424
425		if (sdev_id == dev_id)
426			continue;
427
428		if (mlx5_is_mgt_ifc_pci_device(dev, sdev_id))
429			continue;
430
431		mlx5_core_warn(dev, "unrecognized dev_id (0x%x)\n", sdev_id);
432		return -EPERM;
433	}
434	return 0;
435}
436
437static bool mlx5_is_reset_now_capable(struct mlx5_core_dev *dev,
438				      u8 reset_method)
439{
440	struct pci_dev *bridge = dev->pdev->bus->self;
441	u16 dev_id;
442	int err;
443
444	if (!bridge) {
445		mlx5_core_warn(dev, "PCI bus bridge is not accessible\n");
446		return false;
447	}
448
449	if (!MLX5_CAP_GEN(dev, fast_teardown)) {
450		mlx5_core_warn(dev, "fast teardown is not supported by firmware\n");
451		return false;
452	}
453
454	if (!mlx5_core_is_ecpf(dev) && !mlx5_sf_table_empty(dev)) {
455		mlx5_core_warn(dev, "SFs should be removed before reset\n");
456		return false;
457	}
458
459#if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
460	if (reset_method != MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET) {
461		err = mlx5_check_hotplug_interrupt(dev, bridge);
462		if (err)
463			return false;
464	}
465#endif
466
467	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
468	if (err)
469		return false;
470	return (!mlx5_check_dev_ids(dev, dev_id));
471}
472
473static void mlx5_sync_reset_request_event(struct work_struct *work)
474{
475	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
476						      reset_request_work);
477	struct mlx5_core_dev *dev = fw_reset->dev;
478	bool nack_request = false;
479	struct devlink *devlink;
480	int err;
481
482	err = mlx5_fw_reset_get_reset_method(dev, &fw_reset->reset_method);
483	if (err) {
484		nack_request = true;
485		mlx5_core_warn(dev, "Failed reading MFRL, err %d\n", err);
486	} else if (!mlx5_is_reset_now_capable(dev, fw_reset->reset_method) ||
487		   test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
488			    &fw_reset->reset_flags)) {
489		nack_request = true;
490	}
491
492	devlink = priv_to_devlink(dev);
493	/* For external resets, try to acquire devl_lock. Skip if devlink reset is
494	 * pending (lock already held)
495	 */
496	if (nack_request ||
497	    (!test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP,
498		       &fw_reset->reset_flags) &&
499	     !devl_trylock(devlink))) {
500		err = mlx5_fw_reset_set_reset_sync_nack(dev);
501		mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
502			       err ? "Failed" : "Sent");
503		return;
504	}
505
506	if (mlx5_sync_reset_set_reset_requested(dev))
507		goto unlock;
508
509	set_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
510
511	err = mlx5_fw_reset_set_reset_sync_ack(dev);
512	if (err)
513		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
514	else
515		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
516
517unlock:
518	if (!test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags))
519		devl_unlock(devlink);
520}
521
522static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev, u16 dev_id)
523{
524	struct pci_bus *bridge_bus = dev->pdev->bus;
525	struct pci_dev *bridge = bridge_bus->self;
526	unsigned long timeout;
527	struct pci_dev *sdev;
528	int cap, err;
529	u16 reg16;
530
531	cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
532	if (!cap)
533		return -EOPNOTSUPP;
534
535	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
536		pci_save_state(sdev);
537		pci_cfg_access_lock(sdev);
538	}
539	/* PCI link toggle */
540	err = pcie_capability_set_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
541	if (err)
542		return pcibios_err_to_errno(err);
543	msleep(500);
544	err = pcie_capability_clear_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
545	if (err)
546		return pcibios_err_to_errno(err);
547
548	/* Check link */
549	if (!bridge->link_active_reporting) {
550		mlx5_core_warn(dev, "No PCI link reporting capability\n");
551		msleep(1000);
552		goto restore;
553	}
554
555	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE));
556	do {
557		err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, &reg16);
558		if (err)
559			return pcibios_err_to_errno(err);
560		if (reg16 & PCI_EXP_LNKSTA_DLLLA)
561			break;
562		msleep(20);
563	} while (!time_after(jiffies, timeout));
564
565	if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
566		mlx5_core_info(dev, "PCI Link up\n");
567	} else {
568		mlx5_core_err(dev, "PCI link not ready (0x%04x) after %llu ms\n",
569			      reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
570		err = -ETIMEDOUT;
571		goto restore;
572	}
573
574	do {
575		err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &reg16);
576		if (err)
577			return pcibios_err_to_errno(err);
578		if (reg16 == dev_id)
579			break;
580		msleep(20);
581	} while (!time_after(jiffies, timeout));
582
583	if (reg16 == dev_id) {
584		mlx5_core_info(dev, "Firmware responds to PCI config cycles again\n");
585	} else {
586		mlx5_core_err(dev, "Firmware is not responsive (0x%04x) after %llu ms\n",
587			      reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
588		err = -ETIMEDOUT;
589	}
590
591restore:
592	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
593		pci_cfg_access_unlock(sdev);
594		pci_restore_state(sdev);
595	}
596
597	return err;
598}
599
600static int mlx5_pci_reset_bus(struct mlx5_core_dev *dev)
601{
602	if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method))
603		return -EOPNOTSUPP;
604
605	return pci_reset_bus(dev->pdev);
606}
607
608static int mlx5_sync_pci_reset(struct mlx5_core_dev *dev, u8 reset_method)
609{
610	u16 dev_id;
611	int err;
612
613	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
614	if (err)
615		return pcibios_err_to_errno(err);
616	err = mlx5_check_dev_ids(dev, dev_id);
617	if (err)
618		return err;
619
620	switch (reset_method) {
621	case MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE:
622		err = mlx5_pci_link_toggle(dev, dev_id);
623		if (err)
624			mlx5_core_warn(dev, "mlx5_pci_link_toggle failed\n");
625		break;
626	case MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET:
627		err = mlx5_pci_reset_bus(dev);
628		if (err)
629			mlx5_core_warn(dev, "mlx5_pci_reset_bus failed\n");
630		break;
631	default:
632		return -EOPNOTSUPP;
633	}
634
635	return err;
636}
637
638void mlx5_sync_reset_unload_flow(struct mlx5_core_dev *dev, bool locked)
639{
640	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
641	unsigned long timeout;
642	int poll_freq = 20;
643	bool reset_action;
644	u8 rst_state;
645	int err;
646
647	if (locked)
648		mlx5_unload_one_devl_locked(dev, false);
649	else
650		mlx5_unload_one(dev, false);
651
652	if (!test_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags))
653		return;
654
655	mlx5_set_fw_rst_ack(dev);
656	mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n");
657
658	reset_action = false;
659	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, RESET_UNLOAD));
660	do {
661		rst_state = mlx5_get_fw_rst_state(dev);
662		if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ ||
663		    rst_state == MLX5_FW_RST_STATE_IDLE) {
664			reset_action = true;
665			break;
666		}
667		if (rst_state == MLX5_FW_RST_STATE_DROP_MODE) {
668			mlx5_core_info(dev, "Sync Reset Drop mode ack\n");
669			mlx5_set_fw_rst_ack(dev);
670			poll_freq = 1000;
671		}
672		msleep(poll_freq);
673	} while (!time_after(jiffies, timeout));
674
675	if (!reset_action) {
676		mlx5_core_err(dev, "Got timeout waiting for sync reset action, state = %u\n",
677			      rst_state);
678		fw_reset->ret = -ETIMEDOUT;
679		goto done;
680	}
681
682	mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n",
683		       rst_state);
684	if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) {
685		err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
686		if (err) {
687			mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, err %d\n",
688				       err);
689			fw_reset->ret = err;
690		}
691	}
692
693done:
694	clear_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags);
695}
696
697static void mlx5_sync_reset_now_event(struct work_struct *work)
698{
699	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
700						      reset_now_work);
701	struct mlx5_core_dev *dev = fw_reset->dev;
702	int err;
703
704	if (mlx5_sync_reset_clear_reset_requested(dev, false))
705		return;
706
707	mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");
708
709	err = mlx5_cmd_fast_teardown_hca(dev);
710	if (err) {
711		mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
712		goto done;
713	}
714
715	err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
716	if (err) {
717		mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, no reset done, err %d\n", err);
718		set_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags);
719	}
720
721	mlx5_enter_error_state(dev, true);
722done:
723	fw_reset->ret = err;
724	mlx5_fw_reset_complete_reload(dev);
725}
726
727static void mlx5_sync_reset_unload_event(struct work_struct *work)
728{
729	struct mlx5_fw_reset *fw_reset;
730	struct mlx5_core_dev *dev;
731	int err;
732
733	fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work);
734	dev = fw_reset->dev;
735
736	if (mlx5_sync_reset_clear_reset_requested(dev, false))
737		return;
738
739	set_bit(MLX5_FW_RESET_FLAGS_UNLOAD_EVENT, &fw_reset->reset_flags);
740	mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n");
741
742	err = mlx5_cmd_fast_teardown_hca(dev);
743	if (err)
744		mlx5_core_warn(dev, "Fast teardown failed, unloading, err %d\n", err);
745	else
746		mlx5_enter_error_state(dev, true);
747
748	mlx5_fw_reset_complete_reload(dev);
749}
750
751static void mlx5_sync_reset_abort_event(struct work_struct *work)
752{
753	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
754						      reset_abort_work);
755	struct mlx5_core_dev *dev = fw_reset->dev;
756
757	if (mlx5_sync_reset_clear_reset_requested(dev, true))
758		return;
759
760	clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
761	mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
762}
763
764static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe)
765{
766	struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
767	u8 sync_event_rst_type;
768
769	sync_fw_update_eqe = &eqe->data.sync_fw_update;
770	sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
771	switch (sync_event_rst_type) {
772	case MLX5_SYNC_RST_STATE_RESET_REQUEST:
773		queue_work(fw_reset->wq, &fw_reset->reset_request_work);
774		break;
775	case MLX5_SYNC_RST_STATE_RESET_UNLOAD:
776		queue_work(fw_reset->wq, &fw_reset->reset_unload_work);
777		break;
778	case MLX5_SYNC_RST_STATE_RESET_NOW:
779		queue_work(fw_reset->wq, &fw_reset->reset_now_work);
780		break;
781	case MLX5_SYNC_RST_STATE_RESET_ABORT:
782		queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
783		break;
784	}
785}
786
787static void mlx5_sync_reset_timeout_work(struct work_struct *work)
788{
789	struct delayed_work *dwork = container_of(work, struct delayed_work,
790						  work);
791	struct mlx5_fw_reset *fw_reset =
792		container_of(dwork, struct mlx5_fw_reset, reset_timeout_work);
793	struct mlx5_core_dev *dev = fw_reset->dev;
794
795	if (mlx5_sync_reset_clear_reset_requested(dev, true))
796		return;
797	clear_bit(MLX5_FW_RESET_FLAGS_RESET_IN_PROGRESS, &fw_reset->reset_flags);
798	mlx5_core_warn(dev, "PCI Sync FW Update Reset Timeout.\n");
799}
800
801static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data)
802{
803	struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
804	struct mlx5_eqe *eqe = data;
805
806	if (test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
807		return NOTIFY_DONE;
808
809	switch (eqe->sub_type) {
810	case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
811		queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
812		break;
813	case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
814		mlx5_sync_reset_events_handle(fw_reset, eqe);
815		break;
816	default:
817		return NOTIFY_DONE;
818	}
819
820	return NOTIFY_OK;
821}
822
823int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
824{
825	unsigned long pci_sync_update_timeout = mlx5_tout_ms(dev, PCI_SYNC_UPDATE);
826	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
827	unsigned long timeout;
828	int err;
829
830	if (MLX5_CAP_GEN(dev, pci_sync_for_fw_update_with_driver_unload))
831		pci_sync_update_timeout += mlx5_tout_ms(dev, RESET_UNLOAD);
832	timeout = msecs_to_jiffies(pci_sync_update_timeout);
833	if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
834		mlx5_core_warn(dev, "FW sync reset timeout after %lu seconds\n",
835			       pci_sync_update_timeout / 1000);
836		err = -ETIMEDOUT;
837		goto out;
838	}
839	err = fw_reset->ret;
840	if (test_and_clear_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags)) {
841		mlx5_unload_one_devl_locked(dev, false);
842		mlx5_load_one_devl_locked(dev, true);
843	}
844out:
845	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
846	return err;
847}
848
849void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
850{
851	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
852
853	if (!fw_reset)
854		return;
855
856	MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
857	mlx5_eq_notifier_register(dev, &fw_reset->nb);
858}
859
860void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
861{
862	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
863
864	if (!fw_reset)
865		return;
866
867	mlx5_eq_notifier_unregister(dev, &fw_reset->nb);
868}
869
870void mlx5_drain_fw_reset(struct mlx5_core_dev *dev)
871{
872	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
873
874	if (!fw_reset)
875		return;
876
877	set_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags);
878	cancel_work_sync(&fw_reset->fw_live_patch_work);
879	cancel_work_sync(&fw_reset->reset_request_work);
880	cancel_work_sync(&fw_reset->reset_unload_work);
881	cancel_work_sync(&fw_reset->reset_reload_work);
882	cancel_work_sync(&fw_reset->reset_now_work);
883	cancel_work_sync(&fw_reset->reset_abort_work);
884	if (test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
885		mlx5_sync_reset_clear_reset_requested(dev, true);
886}
887
888static const struct devlink_param mlx5_fw_reset_devlink_params[] = {
889	DEVLINK_PARAM_GENERIC(ENABLE_REMOTE_DEV_RESET, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
890			      mlx5_fw_reset_enable_remote_dev_reset_get,
891			      mlx5_fw_reset_enable_remote_dev_reset_set, NULL),
892};
893
894int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
895{
896	struct mlx5_fw_reset *fw_reset;
897	int err;
898
899	if (!MLX5_CAP_MCAM_REG(dev, mfrl))
900		return 0;
901
902	fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL);
903	if (!fw_reset)
904		return -ENOMEM;
905	fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
906	if (!fw_reset->wq) {
907		kfree(fw_reset);
908		return -ENOMEM;
909	}
910
911	fw_reset->dev = dev;
912	dev->priv.fw_reset = fw_reset;
913
914	err = devl_params_register(priv_to_devlink(dev),
915				   mlx5_fw_reset_devlink_params,
916				   ARRAY_SIZE(mlx5_fw_reset_devlink_params));
917	if (err) {
918		destroy_workqueue(fw_reset->wq);
919		kfree(fw_reset);
920		return err;
921	}
922
923	INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
924	INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
925	INIT_WORK(&fw_reset->reset_unload_work, mlx5_sync_reset_unload_event);
926	INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
927	INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
928	INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);
929	INIT_DELAYED_WORK(&fw_reset->reset_timeout_work,
930			  mlx5_sync_reset_timeout_work);
931
932	init_completion(&fw_reset->done);
933	return 0;
934}
935
936void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
937{
938	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
939
940	if (!fw_reset)
941		return;
942
943	devl_params_unregister(priv_to_devlink(dev),
944			       mlx5_fw_reset_devlink_params,
945			       ARRAY_SIZE(mlx5_fw_reset_devlink_params));
946	destroy_workqueue(fw_reset->wq);
947	kfree(dev->priv.fw_reset);
948}