fbnic_pci.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) Meta Platforms, Inc. and affiliates. */
  3
  4#include <linux/init.h>
  5#include <linux/module.h>
  6#include <linux/pci.h>
  7#include <linux/rtnetlink.h>
  8#include <linux/types.h>
  9#include <net/devlink.h>
 10
 11#include "fbnic.h"
 12#include "fbnic_drvinfo.h"
 13#include "fbnic_hw_stats.h"
 14#include "fbnic_netdev.h"
 15
 16char fbnic_driver_name[] = DRV_NAME;
 17
 18MODULE_DESCRIPTION(DRV_SUMMARY);
 19MODULE_LICENSE("GPL");
 20
 21static const struct fbnic_info fbnic_asic_info = {
 22	.max_num_queues = FBNIC_MAX_QUEUES,
 23	.bar_mask = BIT(0) | BIT(4)
 24};
 25
 26static const struct fbnic_info *fbnic_info_tbl[] = {
 27	[fbnic_board_asic] = &fbnic_asic_info,
 28};
 29
 30static const struct pci_device_id fbnic_pci_tbl[] = {
 31	{ PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },
 32	/* Required last entry */
 33	{0, }
 34};
 35MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);
 36
 37u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)
 38{
 39	u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);
 40	u32 value;
 41
 42	if (!csr)
 43		return ~0U;
 44
 45	value = readl(csr + reg);
 46
 47	/* If any bits are 0 value should be valid */
 48	if (~value)
 49		return value;
 50
 51	/* All 1's may be valid if ZEROs register still works */
 52	if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))
 53		return value;
 54
 55	/* Hardware is giving us all 1's reads, assume it is gone */
 56	WRITE_ONCE(fbd->uc_addr0, NULL);
 57	WRITE_ONCE(fbd->uc_addr4, NULL);
 58
 59	dev_err(fbd->dev,
 60		"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
 61		reg, reg << 2);
 62
 63	/* Notify stack that device has lost (PCIe) link */
 64	if (!fbnic_init_failure(fbd))
 65		netif_device_detach(fbd->netdev);
 66
 67	return ~0U;
 68}
 69
 70bool fbnic_fw_present(struct fbnic_dev *fbd)
 71{
 72	return !!READ_ONCE(fbd->uc_addr4);
 73}
 74
 75void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)
 76{
 77	u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
 78
 79	if (csr)
 80		writel(val, csr + reg);
 81}
 82
 83u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)
 84{
 85	u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);
 86	u32 value;
 87
 88	if (!csr)
 89		return ~0U;
 90
 91	value = readl(csr + reg);
 92
 93	/* If any bits are 0 value should be valid */
 94	if (~value)
 95		return value;
 96
 97	/* All 1's may be valid if ZEROs register still works */
 98	if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))
 99		return value;
100
101	/* Hardware is giving us all 1's reads, assume it is gone */
102	WRITE_ONCE(fbd->uc_addr0, NULL);
103	WRITE_ONCE(fbd->uc_addr4, NULL);
104
105	dev_err(fbd->dev,
106		"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
107		reg, reg << 2);
108
109	/* Notify stack that device has lost (PCIe) link */
110	if (!fbnic_init_failure(fbd))
111		netif_device_detach(fbd->netdev);
112
113	return ~0U;
114}
115
116static void fbnic_service_task_start(struct fbnic_net *fbn)
117{
118	struct fbnic_dev *fbd = fbn->fbd;
119
120	schedule_delayed_work(&fbd->service_task, HZ);
121}
122
123static void fbnic_service_task_stop(struct fbnic_net *fbn)
124{
125	struct fbnic_dev *fbd = fbn->fbd;
126
127	cancel_delayed_work(&fbd->service_task);
128}
129
130void fbnic_up(struct fbnic_net *fbn)
131{
132	fbnic_enable(fbn);
133
134	fbnic_fill(fbn);
135
136	fbnic_rss_reinit_hw(fbn->fbd, fbn);
137
138	__fbnic_set_rx_mode(fbn->fbd);
139
140	/* Enable Tx/Rx processing */
141	fbnic_napi_enable(fbn);
142	netif_tx_start_all_queues(fbn->netdev);
143
144	fbnic_service_task_start(fbn);
145}
146
147void fbnic_down_noidle(struct fbnic_net *fbn)
148{
149	fbnic_service_task_stop(fbn);
150
151	/* Disable Tx/Rx Processing */
152	fbnic_napi_disable(fbn);
153	netif_tx_disable(fbn->netdev);
154
155	fbnic_clear_rx_mode(fbn->fbd);
156	fbnic_clear_rules(fbn->fbd);
157	fbnic_rss_disable_hw(fbn->fbd);
158	fbnic_disable(fbn);
159}
160
161void fbnic_down(struct fbnic_net *fbn)
162{
163	fbnic_down_noidle(fbn);
164
165	fbnic_wait_all_queues_idle(fbn->fbd, false);
166
167	fbnic_flush(fbn);
168}
169
170static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd)
171{
172	if (fbnic_fw_xmit_ownership_msg(fbd, true)) {
173		dev_err(fbd->dev, "NIC failed to take ownership\n");
174
175		return -1;
176	}
177
178	fbnic_rpc_reset_valid_entries(fbd);
179	__fbnic_set_rx_mode(fbd);
180
181	return 0;
182}
183
184static void fbnic_health_check(struct fbnic_dev *fbd)
185{
186	struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
187
188	/* As long as the heart is beating the FW is healthy */
189	if (fbd->fw_heartbeat_enabled)
190		return;
191
192	/* If the Tx mailbox still has messages sitting in it then there likely
193	 * isn't anything we can do. We will wait until the mailbox is empty to
194	 * report the fault so we can collect the crashlog.
195	 */
196	if (tx_mbx->head != tx_mbx->tail)
197		return;
198
199	fbnic_devlink_fw_report(fbd, "Firmware crash detected!");
200	fbnic_devlink_otp_check(fbd, "error detected after firmware recovery");
201
202	if (fbnic_fw_config_after_crash(fbd))
203		dev_err(fbd->dev, "Firmware recovery failed after crash\n");
204}
205
206static void fbnic_service_task(struct work_struct *work)
207{
208	struct fbnic_dev *fbd = container_of(to_delayed_work(work),
209					     struct fbnic_dev, service_task);
210	struct net_device *netdev = fbd->netdev;
211
212	if (netif_running(netdev))
213		fbnic_phylink_pmd_training_complete_notify(netdev);
214
215	rtnl_lock();
216
217	fbnic_get_hw_stats32(fbd);
218
219	fbnic_fw_check_heartbeat(fbd);
220
221	fbnic_health_check(fbd);
222
223	fbnic_bmc_rpc_check(fbd);
224
225	if (netif_carrier_ok(fbd->netdev)) {
226		netdev_lock(fbd->netdev);
227		fbnic_napi_depletion_check(fbd->netdev);
228		netdev_unlock(fbd->netdev);
229	}
230
231	if (netif_running(netdev))
232		schedule_delayed_work(&fbd->service_task, HZ);
233
234	rtnl_unlock();
235}
236
237/**
238 * fbnic_probe - Device Initialization Routine
239 * @pdev: PCI device information struct
240 * @ent: entry in fbnic_pci_tbl
241 *
242 * Initializes a PCI device identified by a pci_dev structure.
243 * The OS initialization, configuring of the adapter private structure,
244 * and a hardware reset occur.
245 *
246 * Return: 0 on success, negative on failure
247 **/
248static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
249{
250	const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];
251	struct net_device *netdev;
252	struct fbnic_dev *fbd;
253	int err;
254
255	if (pdev->error_state != pci_channel_io_normal) {
256		dev_err(&pdev->dev,
257			"PCI device still in an error state. Unable to load...\n");
258		return -EIO;
259	}
260
261	err = pcim_enable_device(pdev);
262	if (err) {
263		dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
264		return err;
265	}
266
267	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
268	if (err)
269		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
270	if (err) {
271		dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
272		return err;
273	}
274
275	err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);
276	if (err) {
277		dev_err(&pdev->dev,
278			"pci_request_selected_regions failed: %d\n", err);
279		return err;
280	}
281
282	fbd = fbnic_devlink_alloc(pdev);
283	if (!fbd) {
284		dev_err(&pdev->dev, "Devlink allocation failed\n");
285		return -ENOMEM;
286	}
287
288	err = fbnic_devlink_health_create(fbd);
289	if (err)
290		goto free_fbd;
291
292	/* Populate driver with hardware-specific info and handlers */
293	fbd->max_num_queues = info->max_num_queues;
294
295	pci_set_master(pdev);
296	pci_save_state(pdev);
297
298	INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);
299
300	err = fbnic_alloc_irqs(fbd);
301	if (err)
302		goto err_destroy_health;
303
304	err = fbnic_mac_init(fbd);
305	if (err) {
306		dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);
307		goto free_irqs;
308	}
309
310	err = fbnic_fw_log_init(fbd);
311	if (err)
312		dev_warn(fbd->dev,
313			 "Unable to initialize firmware log buffer: %d\n",
314			 err);
315
316	err = fbnic_fw_request_mbx(fbd);
317	if (err) {
318		dev_err(&pdev->dev,
319			"Firmware mailbox initialization failure\n");
320		goto free_fw_log;
321	}
322
323	/* Send the request to enable the FW logging to host. Note if this
324	 * fails we ignore the error and just display a message as it is
325	 * possible the FW is just too old to support the logging and needs
326	 * to be updated.
327	 */
328	fbnic_fw_log_enable(fbd, true);
329
330	fbnic_devlink_register(fbd);
331	fbnic_devlink_otp_check(fbd, "error detected during probe");
332	fbnic_dbg_fbd_init(fbd);
333
334	/* Capture snapshot of hardware stats so netdev can calculate delta */
335	fbnic_init_hw_stats(fbd);
336
337	fbnic_hwmon_register(fbd);
338
339	if (!fbd->dsn) {
340		dev_warn(&pdev->dev, "Reading serial number failed\n");
341		goto init_failure_mode;
342	}
343
344	if (fbnic_mdiobus_create(fbd))
345		goto init_failure_mode;
346
347	netdev = fbnic_netdev_alloc(fbd);
348	if (!netdev) {
349		dev_err(&pdev->dev, "Netdev allocation failed\n");
350		goto init_failure_mode;
351	}
352
353	err = fbnic_ptp_setup(fbd);
354	if (err)
355		goto ifm_free_netdev;
356
357	err = fbnic_netdev_register(netdev);
358	if (err) {
359		dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
360		goto ifm_destroy_ptp;
361	}
362
363	return 0;
364
365ifm_destroy_ptp:
366	fbnic_ptp_destroy(fbd);
367ifm_free_netdev:
368	fbnic_netdev_free(fbd);
369init_failure_mode:
370	dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");
371	 /* Always return 0 even on error so devlink is registered to allow
372	  * firmware updates for fixes.
373	  */
374	return 0;
375free_fw_log:
376	fbnic_fw_log_free(fbd);
377free_irqs:
378	fbnic_free_irqs(fbd);
379err_destroy_health:
380	fbnic_devlink_health_destroy(fbd);
381free_fbd:
382	fbnic_devlink_free(fbd);
383
384	return err;
385}
386
387/**
388 * fbnic_remove - Device Removal Routine
389 * @pdev: PCI device information struct
390 *
391 * Called by the PCI subsystem to alert the driver that it should release
392 * a PCI device.  This could be caused by a Hot-Plug event, or because the
393 * driver is going to be removed from memory.
394 **/
395static void fbnic_remove(struct pci_dev *pdev)
396{
397	struct fbnic_dev *fbd = pci_get_drvdata(pdev);
398
399	if (!fbnic_init_failure(fbd)) {
400		struct net_device *netdev = fbd->netdev;
401
402		fbnic_netdev_unregister(netdev);
403		cancel_delayed_work_sync(&fbd->service_task);
404		fbnic_ptp_destroy(fbd);
405		fbnic_netdev_free(fbd);
406	}
407
408	fbnic_hwmon_unregister(fbd);
409	fbnic_dbg_fbd_exit(fbd);
410	fbnic_devlink_unregister(fbd);
411	fbnic_fw_log_disable(fbd);
412	fbnic_fw_free_mbx(fbd);
413	fbnic_fw_log_free(fbd);
414	fbnic_free_irqs(fbd);
415
416	fbnic_devlink_health_destroy(fbd);
417	fbnic_devlink_free(fbd);
418}
419
420static int fbnic_pm_suspend(struct device *dev)
421{
422	struct fbnic_dev *fbd = dev_get_drvdata(dev);
423	struct net_device *netdev = fbd->netdev;
424
425	if (fbnic_init_failure(fbd))
426		goto null_uc_addr;
427
428	rtnl_lock();
429	netdev_lock(netdev);
430
431	netif_device_detach(netdev);
432
433	if (netif_running(netdev))
434		netdev->netdev_ops->ndo_stop(netdev);
435
436	netdev_unlock(netdev);
437	rtnl_unlock();
438
439null_uc_addr:
440	fbnic_fw_log_disable(fbd);
441
442	devl_lock(priv_to_devlink(fbd));
443
444	fbnic_fw_free_mbx(fbd);
445
446	devl_unlock(priv_to_devlink(fbd));
447
448	/* Free the IRQs so they aren't trying to occupy sleeping CPUs */
449	fbnic_free_irqs(fbd);
450
451	/* Hardware is about to go away, so switch off MMIO access internally */
452	WRITE_ONCE(fbd->uc_addr0, NULL);
453	WRITE_ONCE(fbd->uc_addr4, NULL);
454
455	return 0;
456}
457
458static int __fbnic_pm_resume(struct device *dev)
459{
460	struct fbnic_dev *fbd = dev_get_drvdata(dev);
461	struct net_device *netdev = fbd->netdev;
462	void __iomem * const *iomap_table;
463	struct fbnic_net *fbn;
464	int err;
465
466	/* Restore MMIO access */
467	iomap_table = pcim_iomap_table(to_pci_dev(dev));
468	fbd->uc_addr0 = iomap_table[0];
469	fbd->uc_addr4 = iomap_table[4];
470
471	/* Rerequest the IRQs */
472	err = fbnic_alloc_irqs(fbd);
473	if (err)
474		goto err_invalidate_uc_addr;
475
476	fbd->mac->init_regs(fbd);
477
478	devl_lock(priv_to_devlink(fbd));
479
480	/* Re-enable mailbox */
481	err = fbnic_fw_request_mbx(fbd);
482	devl_unlock(priv_to_devlink(fbd));
483	if (err)
484		goto err_free_irqs;
485
486	/* Only send log history if log buffer is empty to prevent duplicate
487	 * log entries.
488	 */
489	fbnic_fw_log_enable(fbd, list_empty(&fbd->fw_log.entries));
490
491	/* Since the FW should be up, check if it reported OTP errors */
492	fbnic_devlink_otp_check(fbd, "error detected after PM resume");
493
494	/* No netdev means there isn't a network interface to bring up */
495	if (fbnic_init_failure(fbd))
496		return 0;
497
498	fbn = netdev_priv(netdev);
499
500	/* Reset the queues if needed */
501	fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
502
503	rtnl_lock();
504	netdev_lock(netdev);
505
506	if (netif_running(netdev))
507		err = __fbnic_open(fbn);
508
509	netdev_unlock(netdev);
510	rtnl_unlock();
511	if (err)
512		goto err_free_mbx;
513
514	return 0;
515err_free_mbx:
516	fbnic_fw_log_disable(fbd);
517
518	devl_lock(priv_to_devlink(fbd));
519	fbnic_fw_free_mbx(fbd);
520	devl_unlock(priv_to_devlink(fbd));
521err_free_irqs:
522	fbnic_free_irqs(fbd);
523err_invalidate_uc_addr:
524	WRITE_ONCE(fbd->uc_addr0, NULL);
525	WRITE_ONCE(fbd->uc_addr4, NULL);
526	return err;
527}
528
529static void __fbnic_pm_attach(struct device *dev)
530{
531	struct fbnic_dev *fbd = dev_get_drvdata(dev);
532	struct net_device *netdev = fbd->netdev;
533	struct fbnic_net *fbn;
534
535	rtnl_lock();
536	fbnic_reset_hw_stats(fbd);
537	rtnl_unlock();
538
539	if (fbnic_init_failure(fbd))
540		return;
541
542	fbn = netdev_priv(netdev);
543
544	if (netif_running(netdev))
545		fbnic_up(fbn);
546
547	netif_device_attach(netdev);
548}
549
550static int __maybe_unused fbnic_pm_resume(struct device *dev)
551{
552	int err;
553
554	err = __fbnic_pm_resume(dev);
555	if (!err)
556		__fbnic_pm_attach(dev);
557
558	return err;
559}
560
561static const struct dev_pm_ops fbnic_pm_ops = {
562	SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)
563};
564
565static void fbnic_shutdown(struct pci_dev *pdev)
566{
567	fbnic_pm_suspend(&pdev->dev);
568}
569
570static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,
571						 pci_channel_state_t state)
572{
573	/* Disconnect device if failure is not recoverable via reset */
574	if (state == pci_channel_io_perm_failure)
575		return PCI_ERS_RESULT_DISCONNECT;
576
577	fbnic_pm_suspend(&pdev->dev);
578
579	/* Request a slot reset */
580	return PCI_ERS_RESULT_NEED_RESET;
581}
582
583static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)
584{
585	int err;
586
587	pci_set_power_state(pdev, PCI_D0);
588	pci_restore_state(pdev);
589
590	if (pci_enable_device_mem(pdev)) {
591		dev_err(&pdev->dev,
592			"Cannot re-enable PCI device after reset.\n");
593		return PCI_ERS_RESULT_DISCONNECT;
594	}
595
596	/* Restore device to previous state */
597	err = __fbnic_pm_resume(&pdev->dev);
598
599	return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
600}
601
602static void fbnic_err_resume(struct pci_dev *pdev)
603{
604	__fbnic_pm_attach(&pdev->dev);
605}
606
607static const struct pci_error_handlers fbnic_err_handler = {
608	.error_detected	= fbnic_err_error_detected,
609	.slot_reset	= fbnic_err_slot_reset,
610	.resume		= fbnic_err_resume,
611};
612
613static struct pci_driver fbnic_driver = {
614	.name		= fbnic_driver_name,
615	.id_table	= fbnic_pci_tbl,
616	.probe		= fbnic_probe,
617	.remove		= fbnic_remove,
618	.driver.pm	= &fbnic_pm_ops,
619	.shutdown	= fbnic_shutdown,
620	.err_handler	= &fbnic_err_handler,
621};
622
623/**
624 * fbnic_init_module - Driver Registration Routine
625 *
626 * The first routine called when the driver is loaded.  All it does is
627 * register with the PCI subsystem.
628 *
629 * Return: 0 on success, negative on failure
630 **/
631static int __init fbnic_init_module(void)
632{
633	int err;
634
635	fbnic_dbg_init();
636
637	err = pci_register_driver(&fbnic_driver);
638	if (err) {
639		fbnic_dbg_exit();
640		goto out;
641	}
642
643	pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);
644out:
645	return err;
646}
647module_init(fbnic_init_module);
648
649/**
650 * fbnic_exit_module - Driver Exit Cleanup Routine
651 *
652 * Called just before the driver is removed from memory.
653 **/
654static void __exit fbnic_exit_module(void)
655{
656	pci_unregister_driver(&fbnic_driver);
657
658	fbnic_dbg_exit();
659}
660module_exit(fbnic_exit_module);