main.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2023 Advanced Micro Devices, Inc */
  3
  4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  5
  6#include <linux/pci.h>
  7
  8#include <linux/pds/pds_common.h>
  9
 10#include "core.h"
 11
 12MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
 13MODULE_AUTHOR("Advanced Micro Devices, Inc");
 14MODULE_LICENSE("GPL");
 15
 16/* Supported devices */
 17static const struct pci_device_id pdsc_id_table[] = {
 18	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
 19	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
 20	{ 0, }	/* end of table */
 21};
 22MODULE_DEVICE_TABLE(pci, pdsc_id_table);
 23
 24static void pdsc_wdtimer_cb(struct timer_list *t)
 25{
 26	struct pdsc *pdsc = timer_container_of(pdsc, t, wdtimer);
 27
 28	dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
 29	mod_timer(&pdsc->wdtimer,
 30		  round_jiffies(jiffies + pdsc->wdtimer_period));
 31
 32	queue_work(pdsc->wq, &pdsc->health_work);
 33}
 34
 35static void pdsc_unmap_bars(struct pdsc *pdsc)
 36{
 37	struct pdsc_dev_bar *bars = pdsc->bars;
 38	unsigned int i;
 39
 40	pdsc->info_regs = NULL;
 41	pdsc->cmd_regs = NULL;
 42	pdsc->intr_status = NULL;
 43	pdsc->intr_ctrl = NULL;
 44
 45	for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
 46		if (bars[i].vaddr)
 47			pci_iounmap(pdsc->pdev, bars[i].vaddr);
 48		bars[i].vaddr = NULL;
 49	}
 50}
 51
 52static int pdsc_map_bars(struct pdsc *pdsc)
 53{
 54	struct pdsc_dev_bar *bar = pdsc->bars;
 55	struct pci_dev *pdev = pdsc->pdev;
 56	struct device *dev = pdsc->dev;
 57	struct pdsc_dev_bar *bars;
 58	unsigned int i, j;
 59	int num_bars = 0;
 60	int err;
 61	u32 sig;
 62
 63	bars = pdsc->bars;
 64
 65	/* Since the PCI interface in the hardware is configurable,
 66	 * we need to poke into all the bars to find the set we're
 67	 * expecting.
 68	 */
 69	for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
 70		if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
 71			continue;
 72
 73		bars[j].len = pci_resource_len(pdev, i);
 74		bars[j].bus_addr = pci_resource_start(pdev, i);
 75		bars[j].res_index = i;
 76
 77		/* only map the whole bar 0 */
 78		if (j > 0) {
 79			bars[j].vaddr = NULL;
 80		} else {
 81			bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
 82			if (!bars[j].vaddr) {
 83				dev_err(dev, "Cannot map BAR %d, aborting\n", i);
 84				return -ENODEV;
 85			}
 86		}
 87
 88		j++;
 89	}
 90	num_bars = j;
 91
 92	/* BAR0: dev_cmd and interrupts */
 93	if (num_bars < 1) {
 94		dev_err(dev, "No bars found\n");
 95		err = -EFAULT;
 96		goto err_out;
 97	}
 98
 99	if (bar->len < PDS_CORE_BAR0_SIZE) {
100		dev_err(dev, "Resource bar size %lu too small\n", bar->len);
101		err = -EFAULT;
102		goto err_out;
103	}
104
105	pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
106	pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
107	pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
108	pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
109
110	sig = ioread32(&pdsc->info_regs->signature);
111	if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
112		dev_err(dev, "Incompatible firmware signature %x", sig);
113		err = -EFAULT;
114		goto err_out;
115	}
116
117	/* BAR1: doorbells */
118	bar++;
119	if (num_bars < 2) {
120		dev_err(dev, "Doorbell bar missing\n");
121		err = -EFAULT;
122		goto err_out;
123	}
124
125	pdsc->db_pages = bar->vaddr;
126	pdsc->phy_db_pages = bar->bus_addr;
127
128	return 0;
129
130err_out:
131	pdsc_unmap_bars(pdsc);
132	return err;
133}
134
135void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
136{
137	return pci_iomap_range(pdsc->pdev,
138			       pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
139			       (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
140}
141
142static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
143{
144	struct pdsc *pdsc = pci_get_drvdata(pdev);
145	struct device *dev = pdsc->dev;
146	int ret = 0;
147
148	if (num_vfs > 0) {
149		pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
150				    GFP_KERNEL);
151		if (!pdsc->vfs)
152			return -ENOMEM;
153		pdsc->num_vfs = num_vfs;
154
155		ret = pci_enable_sriov(pdev, num_vfs);
156		if (ret) {
157			dev_err(dev, "Cannot enable SRIOV: %pe\n",
158				ERR_PTR(ret));
159			goto no_vfs;
160		}
161
162		return num_vfs;
163	}
164
165no_vfs:
166	pci_disable_sriov(pdev);
167
168	kfree(pdsc->vfs);
169	pdsc->vfs = NULL;
170	pdsc->num_vfs = 0;
171
172	return ret;
173}
174
175static int pdsc_init_vf(struct pdsc *vf)
176{
177	struct devlink *dl;
178	struct pdsc *pf;
179	int err;
180
181	pf = pdsc_get_pf_struct(vf->pdev);
182	if (IS_ERR_OR_NULL(pf))
183		return PTR_ERR(pf) ?: -1;
184
185	vf->vf_id = pci_iov_vf_id(vf->pdev);
186
187	dl = priv_to_devlink(vf);
188	devl_lock(dl);
189	devl_register(dl);
190	devl_unlock(dl);
191
192	pf->vfs[vf->vf_id].vf = vf;
193	err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA,
194				  &pf->vfs[vf->vf_id].padev);
195	if (err) {
196		devl_lock(dl);
197		devl_unregister(dl);
198		devl_unlock(dl);
199	}
200
201	return err;
202}
203
204static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
205	.name = "fw",
206	.diagnose = pdsc_fw_reporter_diagnose,
207};
208
209static const struct devlink_param pdsc_dl_params[] = {
210	DEVLINK_PARAM_GENERIC(ENABLE_VNET,
211			      BIT(DEVLINK_PARAM_CMODE_RUNTIME),
212			      pdsc_dl_enable_get,
213			      pdsc_dl_enable_set,
214			      pdsc_dl_enable_validate),
215};
216
217#define PDSC_WQ_NAME_LEN 24
218
219static int pdsc_init_pf(struct pdsc *pdsc)
220{
221	struct devlink_health_reporter *hr;
222	char wq_name[PDSC_WQ_NAME_LEN];
223	struct devlink *dl;
224	int err;
225
226	pcie_print_link_status(pdsc->pdev);
227
228	err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
229	if (err) {
230		dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
231			ERR_PTR(err));
232		return err;
233	}
234
235	err = pdsc_map_bars(pdsc);
236	if (err)
237		goto err_out_release_regions;
238
239	/* General workqueue and timer, but don't start timer yet */
240	snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
241	pdsc->wq = create_singlethread_workqueue(wq_name);
242	INIT_WORK(&pdsc->health_work, pdsc_health_thread);
243	INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
244	timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
245	pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
246
247	mutex_init(&pdsc->devcmd_lock);
248	mutex_init(&pdsc->config_lock);
249	spin_lock_init(&pdsc->adminq_lock);
250
251	mutex_lock(&pdsc->config_lock);
252	set_bit(PDSC_S_FW_DEAD, &pdsc->state);
253
254	err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
255	if (err) {
256		mutex_unlock(&pdsc->config_lock);
257		goto err_out_unmap_bars;
258	}
259
260	err = pdsc_start(pdsc);
261	if (err) {
262		mutex_unlock(&pdsc->config_lock);
263		goto err_out_teardown;
264	}
265
266	mutex_unlock(&pdsc->config_lock);
267
268	err = pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, &pdsc->padev);
269	if (err)
270		goto err_out_stop;
271
272	dl = priv_to_devlink(pdsc);
273	devl_lock(dl);
274	err = devl_params_register(dl, pdsc_dl_params,
275				   ARRAY_SIZE(pdsc_dl_params));
276	if (err) {
277		devl_unlock(dl);
278		dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
279			 ERR_PTR(err));
280		goto err_out_del_dev;
281	}
282
283	hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, pdsc);
284	if (IS_ERR(hr)) {
285		devl_unlock(dl);
286		dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
287		err = PTR_ERR(hr);
288		goto err_out_unreg_params;
289	}
290	pdsc->fw_reporter = hr;
291
292	devl_register(dl);
293	devl_unlock(dl);
294
295	/* Lastly, start the health check timer */
296	mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
297
298	return 0;
299
300err_out_unreg_params:
301	devlink_params_unregister(dl, pdsc_dl_params,
302				  ARRAY_SIZE(pdsc_dl_params));
303err_out_del_dev:
304	pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
305err_out_stop:
306	pdsc_stop(pdsc);
307err_out_teardown:
308	pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
309err_out_unmap_bars:
310	timer_shutdown_sync(&pdsc->wdtimer);
311	if (pdsc->wq)
312		destroy_workqueue(pdsc->wq);
313	mutex_destroy(&pdsc->config_lock);
314	mutex_destroy(&pdsc->devcmd_lock);
315	pci_free_irq_vectors(pdsc->pdev);
316	pdsc_unmap_bars(pdsc);
317err_out_release_regions:
318	pci_release_regions(pdsc->pdev);
319
320	return err;
321}
322
323static const struct devlink_ops pdsc_dl_ops = {
324	.info_get	= pdsc_dl_info_get,
325	.flash_update	= pdsc_dl_flash_update,
326};
327
328static const struct devlink_ops pdsc_dl_vf_ops = {
329};
330
331static DEFINE_IDA(pdsc_ida);
332
333static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
334{
335	struct device *dev = &pdev->dev;
336	const struct devlink_ops *ops;
337	struct devlink *dl;
338	struct pdsc *pdsc;
339	bool is_pf;
340	int err;
341
342	is_pf = !pdev->is_virtfn;
343	ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
344	dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
345	if (!dl)
346		return -ENOMEM;
347	pdsc = devlink_priv(dl);
348
349	pdsc->pdev = pdev;
350	pdsc->dev = &pdev->dev;
351	set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
352	pci_set_drvdata(pdev, pdsc);
353	pdsc_debugfs_add_dev(pdsc);
354
355	err = ida_alloc(&pdsc_ida, GFP_KERNEL);
356	if (err < 0) {
357		dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
358			__func__, ERR_PTR(err));
359		goto err_out_free_devlink;
360	}
361	pdsc->uid = err;
362
363	/* Query system for DMA addressing limitation for the device. */
364	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
365	if (err) {
366		dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
367			ERR_PTR(err));
368		goto err_out_free_ida;
369	}
370
371	err = pci_enable_device(pdev);
372	if (err) {
373		dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
374		goto err_out_free_ida;
375	}
376	pci_set_master(pdev);
377
378	if (is_pf)
379		err = pdsc_init_pf(pdsc);
380	else
381		err = pdsc_init_vf(pdsc);
382	if (err) {
383		dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
384		goto err_out_disable_device;
385	}
386
387	clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
388	return 0;
389
390err_out_disable_device:
391	pci_disable_device(pdev);
392err_out_free_ida:
393	ida_free(&pdsc_ida, pdsc->uid);
394err_out_free_devlink:
395	pdsc_debugfs_del_dev(pdsc);
396	devlink_free(dl);
397
398	return err;
399}
400
401static void pdsc_remove(struct pci_dev *pdev)
402{
403	struct pdsc *pdsc = pci_get_drvdata(pdev);
404	struct devlink *dl;
405
406	/* Unhook the registrations first to be sure there
407	 * are no requests while we're stopping.
408	 */
409	dl = priv_to_devlink(pdsc);
410	devl_lock(dl);
411	devl_unregister(dl);
412	if (!pdev->is_virtfn) {
413		if (pdsc->fw_reporter) {
414			devl_health_reporter_destroy(pdsc->fw_reporter);
415			pdsc->fw_reporter = NULL;
416		}
417		devl_params_unregister(dl, pdsc_dl_params,
418				       ARRAY_SIZE(pdsc_dl_params));
419	}
420	devl_unlock(dl);
421
422	if (pdev->is_virtfn) {
423		struct pdsc *pf;
424
425		pf = pdsc_get_pf_struct(pdsc->pdev);
426		if (!IS_ERR(pf)) {
427			pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev);
428			pf->vfs[pdsc->vf_id].vf = NULL;
429		}
430	} else {
431		/* Remove the VFs and their aux_bus connections before other
432		 * cleanup so that the clients can use the AdminQ to cleanly
433		 * shut themselves down.
434		 */
435		pdsc_sriov_configure(pdev, 0);
436		pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
437
438		timer_shutdown_sync(&pdsc->wdtimer);
439		if (pdsc->wq)
440			destroy_workqueue(pdsc->wq);
441
442		mutex_lock(&pdsc->config_lock);
443		set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
444
445		pdsc_stop(pdsc);
446		pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
447		mutex_unlock(&pdsc->config_lock);
448		mutex_destroy(&pdsc->config_lock);
449		mutex_destroy(&pdsc->devcmd_lock);
450
451		pdsc_unmap_bars(pdsc);
452		pci_release_regions(pdev);
453	}
454
455	pci_disable_device(pdev);
456
457	ida_free(&pdsc_ida, pdsc->uid);
458	pdsc_debugfs_del_dev(pdsc);
459	devlink_free(dl);
460}
461
462static void pdsc_stop_health_thread(struct pdsc *pdsc)
463{
464	if (pdsc->pdev->is_virtfn)
465		return;
466
467	timer_shutdown_sync(&pdsc->wdtimer);
468	if (pdsc->health_work.func)
469		cancel_work_sync(&pdsc->health_work);
470}
471
472static void pdsc_restart_health_thread(struct pdsc *pdsc)
473{
474	if (pdsc->pdev->is_virtfn)
475		return;
476
477	timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
478	mod_timer(&pdsc->wdtimer, jiffies + 1);
479}
480
481static void pdsc_reset_prepare(struct pci_dev *pdev)
482{
483	struct pdsc *pdsc = pci_get_drvdata(pdev);
484
485	pdsc_stop_health_thread(pdsc);
486	pdsc_fw_down(pdsc);
487
488	if (pdev->is_virtfn) {
489		struct pdsc *pf;
490
491		pf = pdsc_get_pf_struct(pdsc->pdev);
492		if (!IS_ERR(pf))
493			pdsc_auxbus_dev_del(pdsc, pf,
494					    &pf->vfs[pdsc->vf_id].padev);
495	} else {
496		pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
497	}
498
499	pdsc_unmap_bars(pdsc);
500	pci_release_regions(pdev);
501	if (pci_is_enabled(pdev))
502		pci_disable_device(pdev);
503}
504
505static void pdsc_reset_done(struct pci_dev *pdev)
506{
507	struct pdsc *pdsc = pci_get_drvdata(pdev);
508	struct device *dev = pdsc->dev;
509	int err;
510
511	err = pci_enable_device(pdev);
512	if (err) {
513		dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
514		return;
515	}
516	pci_set_master(pdev);
517
518	if (!pdev->is_virtfn) {
519		pcie_print_link_status(pdsc->pdev);
520
521		err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
522		if (err) {
523			dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
524				ERR_PTR(err));
525			return;
526		}
527
528		err = pdsc_map_bars(pdsc);
529		if (err)
530			return;
531	}
532
533	pdsc_fw_up(pdsc);
534	pdsc_restart_health_thread(pdsc);
535
536	if (pdev->is_virtfn) {
537		struct pdsc *pf;
538
539		pf = pdsc_get_pf_struct(pdsc->pdev);
540		if (!IS_ERR(pf))
541			pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA,
542					    &pf->vfs[pdsc->vf_id].padev);
543	} else {
544		pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL,
545				    &pdsc->padev);
546	}
547}
548
549static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev,
550						pci_channel_state_t error)
551{
552	if (error == pci_channel_io_frozen) {
553		pdsc_reset_prepare(pdev);
554		return PCI_ERS_RESULT_NEED_RESET;
555	}
556
557	return PCI_ERS_RESULT_NONE;
558}
559
560static void pdsc_pci_error_resume(struct pci_dev *pdev)
561{
562	struct pdsc *pdsc = pci_get_drvdata(pdev);
563
564	if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))
565		pci_reset_function_locked(pdev);
566}
567
568static const struct pci_error_handlers pdsc_err_handler = {
569	/* FLR handling */
570	.reset_prepare      = pdsc_reset_prepare,
571	.reset_done         = pdsc_reset_done,
572
573	/* AER handling */
574	.error_detected     = pdsc_pci_error_detected,
575	.resume             = pdsc_pci_error_resume,
576};
577
578static struct pci_driver pdsc_driver = {
579	.name = PDS_CORE_DRV_NAME,
580	.id_table = pdsc_id_table,
581	.probe = pdsc_probe,
582	.remove = pdsc_remove,
583	.sriov_configure = pdsc_sriov_configure,
584	.err_handler = &pdsc_err_handler,
585};
586
587void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
588{
589	return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
590}
591EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
592
593static int __init pdsc_init_module(void)
594{
595	if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
596		return -EINVAL;
597
598	pdsc_debugfs_create();
599	return pci_register_driver(&pdsc_driver);
600}
601
602static void __exit pdsc_cleanup_module(void)
603{
604	pci_unregister_driver(&pdsc_driver);
605	pdsc_debugfs_destroy();
606}
607
608module_init(pdsc_init_module);
609module_exit(pdsc_cleanup_module);