card_base.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * IBM Accelerator Family 'GenWQE'
   4 *
   5 * (C) Copyright IBM Corp. 2013
   6 *
   7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
   8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
   9 * Author: Michael Jung <mijung@gmx.net>
  10 * Author: Michael Ruettger <michael@ibmra.de>
  11 */
  12
  13/*
  14 * Module initialization and PCIe setup. Card health monitoring and
  15 * recovery functionality. Character device creation and deletion are
  16 * controlled from here.
  17 */
  18
  19#include <linux/types.h>
  20#include <linux/pci.h>
  21#include <linux/err.h>
  22#include <linux/string.h>
  23#include <linux/sched.h>
  24#include <linux/wait.h>
  25#include <linux/delay.h>
  26#include <linux/dma-mapping.h>
  27#include <linux/module.h>
  28#include <linux/notifier.h>
  29#include <linux/device.h>
  30#include <linux/log2.h>
  31
  32#include "card_base.h"
  33#include "card_ddcb.h"
  34
  35MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>");
  36MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>");
  37MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>");
  38MODULE_AUTHOR("Michael Jung <mijung@gmx.net>");
  39
  40MODULE_DESCRIPTION("GenWQE Card");
  41MODULE_VERSION(DRV_VERSION);
  42MODULE_LICENSE("GPL");
  43
  44static char genwqe_driver_name[] = GENWQE_DEVNAME;
  45
  46static struct dentry *debugfs_genwqe;
  47static struct genwqe_dev *genwqe_devices[GENWQE_CARD_NO_MAX];
  48
  49/* PCI structure for identifying device by PCI vendor and device ID */
  50static const struct pci_device_id genwqe_device_table[] = {
  51	{ .vendor      = PCI_VENDOR_ID_IBM,
  52	  .device      = PCI_DEVICE_GENWQE,
  53	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
  54	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
  55	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
  56	  .class_mask  = ~0,
  57	  .driver_data = 0 },
  58
  59	/* Initial SR-IOV bring-up image */
  60	{ .vendor      = PCI_VENDOR_ID_IBM,
  61	  .device      = PCI_DEVICE_GENWQE,
  62	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
  63	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
  64	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
  65	  .class_mask  = ~0,
  66	  .driver_data = 0 },
  67
  68	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
  69	  .device      = 0x0000,  /* VF Device ID */
  70	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
  71	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
  72	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
  73	  .class_mask  = ~0,
  74	  .driver_data = 0 },
  75
  76	/* Fixed up image */
  77	{ .vendor      = PCI_VENDOR_ID_IBM,
  78	  .device      = PCI_DEVICE_GENWQE,
  79	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
  80	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
  81	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
  82	  .class_mask  = ~0,
  83	  .driver_data = 0 },
  84
  85	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
  86	  .device      = 0x0000,  /* VF Device ID */
  87	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
  88	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
  89	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
  90	  .class_mask  = ~0,
  91	  .driver_data = 0 },
  92
  93	/* Even one more ... */
  94	{ .vendor      = PCI_VENDOR_ID_IBM,
  95	  .device      = PCI_DEVICE_GENWQE,
  96	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
  97	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_NEW,
  98	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
  99	  .class_mask  = ~0,
 100	  .driver_data = 0 },
 101
 102	{ 0, }			/* 0 terminated list. */
 103};
 104
 105MODULE_DEVICE_TABLE(pci, genwqe_device_table);
 106
 107/**
 108 * genwqe_devnode() - Set default access mode for genwqe devices.
 109 * @dev:	Pointer to device (unused)
 110 * @mode:	Carrier to pass-back given mode (permissions)
 111 *
 112 * Default mode should be rw for everybody. Do not change default
 113 * device name.
 114 */
 115static char *genwqe_devnode(const struct device *dev, umode_t *mode)
 116{
 117	if (mode)
 118		*mode = 0666;
 119	return NULL;
 120}
 121
 122static const struct class class_genwqe = {
 123	.name = GENWQE_DEVNAME,
 124	.devnode = genwqe_devnode,
 125};
 126
 127/**
 128 * genwqe_dev_alloc() - Create and prepare a new card descriptor
 129 *
 130 * Return: Pointer to card descriptor, or ERR_PTR(err) on error
 131 */
 132static struct genwqe_dev *genwqe_dev_alloc(void)
 133{
 134	unsigned int i = 0, j;
 135	struct genwqe_dev *cd;
 136
 137	for (i = 0; i < GENWQE_CARD_NO_MAX; i++) {
 138		if (genwqe_devices[i] == NULL)
 139			break;
 140	}
 141	if (i >= GENWQE_CARD_NO_MAX)
 142		return ERR_PTR(-ENODEV);
 143
 144	cd = kzalloc(sizeof(struct genwqe_dev), GFP_KERNEL);
 145	if (!cd)
 146		return ERR_PTR(-ENOMEM);
 147
 148	cd->card_idx = i;
 149	cd->class_genwqe = &class_genwqe;
 150	cd->debugfs_genwqe = debugfs_genwqe;
 151
 152	/*
 153	 * This comes from kernel config option and can be overritten via
 154	 * debugfs.
 155	 */
 156	cd->use_platform_recovery = CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY;
 157
 158	init_waitqueue_head(&cd->queue_waitq);
 159
 160	spin_lock_init(&cd->file_lock);
 161	INIT_LIST_HEAD(&cd->file_list);
 162
 163	cd->card_state = GENWQE_CARD_UNUSED;
 164	spin_lock_init(&cd->print_lock);
 165
 166	cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT;
 167	cd->kill_timeout = GENWQE_KILL_TIMEOUT;
 168
 169	for (j = 0; j < GENWQE_MAX_VFS; j++)
 170		cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC;
 171
 172	genwqe_devices[i] = cd;
 173	return cd;
 174}
 175
 176static void genwqe_dev_free(struct genwqe_dev *cd)
 177{
 178	if (!cd)
 179		return;
 180
 181	genwqe_devices[cd->card_idx] = NULL;
 182	kfree(cd);
 183}
 184
 185/**
 186 * genwqe_bus_reset() - Card recovery
 187 * @cd: GenWQE device information
 188 *
 189 * pci_reset_function() will recover the device and ensure that the
 190 * registers are accessible again when it completes with success. If
 191 * not, the card will stay dead and registers will be unaccessible
 192 * still.
 193 */
 194static int genwqe_bus_reset(struct genwqe_dev *cd)
 195{
 196	int rc = 0;
 197	struct pci_dev *pci_dev = cd->pci_dev;
 198	void __iomem *mmio;
 199
 200	if (cd->err_inject & GENWQE_INJECT_BUS_RESET_FAILURE)
 201		return -EIO;
 202
 203	mmio = cd->mmio;
 204	cd->mmio = NULL;
 205	pci_iounmap(pci_dev, mmio);
 206
 207	pci_release_mem_regions(pci_dev);
 208
 209	/*
 210	 * Firmware/BIOS might change memory mapping during bus reset.
 211	 * Settings like enable bus-mastering, ... are backuped and
 212	 * restored by the pci_reset_function().
 213	 */
 214	dev_dbg(&pci_dev->dev, "[%s] pci_reset function ...\n", __func__);
 215	rc = pci_reset_function(pci_dev);
 216	if (rc) {
 217		dev_err(&pci_dev->dev,
 218			"[%s] err: failed reset func (rc %d)\n", __func__, rc);
 219		return rc;
 220	}
 221	dev_dbg(&pci_dev->dev, "[%s] done with rc=%d\n", __func__, rc);
 222
 223	/*
 224	 * Here is the right spot to clear the register read
 225	 * failure. pci_bus_reset() does this job in real systems.
 226	 */
 227	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
 228			    GENWQE_INJECT_GFIR_FATAL |
 229			    GENWQE_INJECT_GFIR_INFO);
 230
 231	rc = pci_request_mem_regions(pci_dev, genwqe_driver_name);
 232	if (rc) {
 233		dev_err(&pci_dev->dev,
 234			"[%s] err: request bars failed (%d)\n", __func__, rc);
 235		return -EIO;
 236	}
 237
 238	cd->mmio = pci_iomap(pci_dev, 0, 0);
 239	if (cd->mmio == NULL) {
 240		dev_err(&pci_dev->dev,
 241			"[%s] err: mapping BAR0 failed\n", __func__);
 242		return -ENOMEM;
 243	}
 244	return 0;
 245}
 246
 247/*
 248 * Hardware circumvention section. Certain bitstreams in our test-lab
 249 * had different kinds of problems. Here is where we adjust those
 250 * bitstreams to function will with this version of our device driver.
 251 *
 252 * Thise circumventions are applied to the physical function only.
 253 * The magical numbers below are identifying development/manufacturing
 254 * versions of the bitstream used on the card.
 255 *
 256 * Turn off error reporting for old/manufacturing images.
 257 */
 258
 259bool genwqe_need_err_masking(struct genwqe_dev *cd)
 260{
 261	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
 262}
 263
 264static void genwqe_tweak_hardware(struct genwqe_dev *cd)
 265{
 266	struct pci_dev *pci_dev = cd->pci_dev;
 267
 268	/* Mask FIRs for development images */
 269	if (((cd->slu_unitcfg & 0xFFFF0ull) >= 0x32000ull) &&
 270	    ((cd->slu_unitcfg & 0xFFFF0ull) <= 0x33250ull)) {
 271		dev_warn(&pci_dev->dev,
 272			 "FIRs masked due to bitstream %016llx.%016llx\n",
 273			 cd->slu_unitcfg, cd->app_unitcfg);
 274
 275		__genwqe_writeq(cd, IO_APP_SEC_LEM_DEBUG_OVR,
 276				0xFFFFFFFFFFFFFFFFull);
 277
 278		__genwqe_writeq(cd, IO_APP_ERR_ACT_MASK,
 279				0x0000000000000000ull);
 280	}
 281}
 282
 283/**
 284 * genwqe_recovery_on_fatal_gfir_required() - Version depended actions
 285 * @cd: GenWQE device information
 286 *
 287 * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must
 288 * be ignored. This is e.g. true for the bitstream we gave to the card
 289 * manufacturer, but also for some old bitstreams we released to our
 290 * test-lab.
 291 */
 292int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd)
 293{
 294	return (cd->slu_unitcfg & 0xFFFF0ull) >= 0x32170ull;
 295}
 296
 297int genwqe_flash_readback_fails(struct genwqe_dev *cd)
 298{
 299	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
 300}
 301
 302/**
 303 * genwqe_T_psec() - Calculate PF/VF timeout register content
 304 * @cd: GenWQE device information
 305 *
 306 * Note: From a design perspective it turned out to be a bad idea to
 307 * use codes here to specifiy the frequency/speed values. An old
 308 * driver cannot understand new codes and is therefore always a
 309 * problem. Better is to measure out the value or put the
 310 * speed/frequency directly into a register which is always a valid
 311 * value for old as well as for new software.
 312 */
 313/* T = 1/f */
 314static int genwqe_T_psec(struct genwqe_dev *cd)
 315{
 316	u16 speed;	/* 1/f -> 250,  200,  166,  175 */
 317	static const int T[] = { 4000, 5000, 6000, 5714 };
 318
 319	speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
 320	if (speed >= ARRAY_SIZE(T))
 321		return -1;	/* illegal value */
 322
 323	return T[speed];
 324}
 325
 326/**
 327 * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution
 328 * @cd: GenWQE device information
 329 *
 330 * Do this _after_ card_reset() is called. Otherwise the values will
 331 * vanish. The settings need to be done when the queues are inactive.
 332 *
 333 * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16.
 334 * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16.
 335 */
 336static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd)
 337{
 338	u32 T = genwqe_T_psec(cd);
 339	u64 x;
 340
 341	if (GENWQE_PF_JOBTIMEOUT_MSEC == 0)
 342		return false;
 343
 344	/* PF: large value needed, flash update 2sec per block */
 345	x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC *
 346		  16000000000uL/(T * 15)) - 10;
 347
 348	genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
 349			  0xff00 | (x & 0xff), 0);
 350	return true;
 351}
 352
 353/**
 354 * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution
 355 * @cd: GenWQE device information
 356 */
 357static bool genwqe_setup_vf_jtimer(struct genwqe_dev *cd)
 358{
 359	struct pci_dev *pci_dev = cd->pci_dev;
 360	unsigned int vf;
 361	u32 T = genwqe_T_psec(cd);
 362	u64 x;
 363	int totalvfs;
 364
 365	totalvfs = pci_sriov_get_totalvfs(pci_dev);
 366	if (totalvfs <= 0)
 367		return false;
 368
 369	for (vf = 0; vf < totalvfs; vf++) {
 370
 371		if (cd->vf_jobtimeout_msec[vf] == 0)
 372			continue;
 373
 374		x = ilog2(cd->vf_jobtimeout_msec[vf] *
 375			  16000000000uL/(T * 15)) - 10;
 376
 377		genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
 378				  0xff00 | (x & 0xff), vf + 1);
 379	}
 380	return true;
 381}
 382
 383static int genwqe_ffdc_buffs_alloc(struct genwqe_dev *cd)
 384{
 385	unsigned int type, e = 0;
 386
 387	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
 388		switch (type) {
 389		case GENWQE_DBG_UNIT0:
 390			e = genwqe_ffdc_buff_size(cd, 0);
 391			break;
 392		case GENWQE_DBG_UNIT1:
 393			e = genwqe_ffdc_buff_size(cd, 1);
 394			break;
 395		case GENWQE_DBG_UNIT2:
 396			e = genwqe_ffdc_buff_size(cd, 2);
 397			break;
 398		case GENWQE_DBG_REGS:
 399			e = GENWQE_FFDC_REGS;
 400			break;
 401		}
 402
 403		/* currently support only the debug units mentioned here */
 404		cd->ffdc[type].entries = e;
 405		cd->ffdc[type].regs =
 406			kmalloc_array(e, sizeof(struct genwqe_reg),
 407				      GFP_KERNEL);
 408		/*
 409		 * regs == NULL is ok, the using code treats this as no regs,
 410		 * Printing warning is ok in this case.
 411		 */
 412	}
 413	return 0;
 414}
 415
 416static void genwqe_ffdc_buffs_free(struct genwqe_dev *cd)
 417{
 418	unsigned int type;
 419
 420	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
 421		kfree(cd->ffdc[type].regs);
 422		cd->ffdc[type].regs = NULL;
 423	}
 424}
 425
 426static int genwqe_read_ids(struct genwqe_dev *cd)
 427{
 428	int err = 0;
 429	int slu_id;
 430	struct pci_dev *pci_dev = cd->pci_dev;
 431
 432	cd->slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
 433	if (cd->slu_unitcfg == IO_ILLEGAL_VALUE) {
 434		dev_err(&pci_dev->dev,
 435			"err: SLUID=%016llx\n", cd->slu_unitcfg);
 436		err = -EIO;
 437		goto out_err;
 438	}
 439
 440	slu_id = genwqe_get_slu_id(cd);
 441	if (slu_id < GENWQE_SLU_ARCH_REQ || slu_id == 0xff) {
 442		dev_err(&pci_dev->dev,
 443			"err: incompatible SLU Architecture %u\n", slu_id);
 444		err = -ENOENT;
 445		goto out_err;
 446	}
 447
 448	cd->app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
 449	if (cd->app_unitcfg == IO_ILLEGAL_VALUE) {
 450		dev_err(&pci_dev->dev,
 451			"err: APPID=%016llx\n", cd->app_unitcfg);
 452		err = -EIO;
 453		goto out_err;
 454	}
 455	genwqe_read_app_id(cd, cd->app_name, sizeof(cd->app_name));
 456
 457	/*
 458	 * Is access to all registers possible? If we are a VF the
 459	 * answer is obvious. If we run fully virtualized, we need to
 460	 * check if we can access all registers. If we do not have
 461	 * full access we will cause an UR and some informational FIRs
 462	 * in the PF, but that should not harm.
 463	 */
 464	if (pci_dev->is_virtfn)
 465		cd->is_privileged = 0;
 466	else
 467		cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM)
 468				     != IO_ILLEGAL_VALUE);
 469
 470 out_err:
 471	return err;
 472}
 473
 474static int genwqe_start(struct genwqe_dev *cd)
 475{
 476	int err;
 477	struct pci_dev *pci_dev = cd->pci_dev;
 478
 479	err = genwqe_read_ids(cd);
 480	if (err)
 481		return err;
 482
 483	if (genwqe_is_privileged(cd)) {
 484		/* do this after the tweaks. alloc fail is acceptable */
 485		genwqe_ffdc_buffs_alloc(cd);
 486		genwqe_stop_traps(cd);
 487
 488		/* Collect registers e.g. FIRs, UNITIDs, traces ... */
 489		genwqe_read_ffdc_regs(cd, cd->ffdc[GENWQE_DBG_REGS].regs,
 490				      cd->ffdc[GENWQE_DBG_REGS].entries, 0);
 491
 492		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT0,
 493				      cd->ffdc[GENWQE_DBG_UNIT0].regs,
 494				      cd->ffdc[GENWQE_DBG_UNIT0].entries);
 495
 496		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT1,
 497				      cd->ffdc[GENWQE_DBG_UNIT1].regs,
 498				      cd->ffdc[GENWQE_DBG_UNIT1].entries);
 499
 500		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT2,
 501				      cd->ffdc[GENWQE_DBG_UNIT2].regs,
 502				      cd->ffdc[GENWQE_DBG_UNIT2].entries);
 503
 504		genwqe_start_traps(cd);
 505
 506		if (cd->card_state == GENWQE_CARD_FATAL_ERROR) {
 507			dev_warn(&pci_dev->dev,
 508				 "[%s] chip reload/recovery!\n", __func__);
 509
 510			/*
 511			 * Stealth Mode: Reload chip on either hot
 512			 * reset or PERST.
 513			 */
 514			cd->softreset = 0x7Cull;
 515			__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
 516				       cd->softreset);
 517
 518			err = genwqe_bus_reset(cd);
 519			if (err != 0) {
 520				dev_err(&pci_dev->dev,
 521					"[%s] err: bus reset failed!\n",
 522					__func__);
 523				goto out;
 524			}
 525
 526			/*
 527			 * Re-read the IDs because
 528			 * it could happen that the bitstream load
 529			 * failed!
 530			 */
 531			err = genwqe_read_ids(cd);
 532			if (err)
 533				goto out;
 534		}
 535	}
 536
 537	err = genwqe_setup_service_layer(cd);  /* does a reset to the card */
 538	if (err != 0) {
 539		dev_err(&pci_dev->dev,
 540			"[%s] err: could not setup servicelayer!\n", __func__);
 541		err = -ENODEV;
 542		goto out;
 543	}
 544
 545	if (genwqe_is_privileged(cd)) {	 /* code is running _after_ reset */
 546		genwqe_tweak_hardware(cd);
 547
 548		genwqe_setup_pf_jtimer(cd);
 549		genwqe_setup_vf_jtimer(cd);
 550	}
 551
 552	err = genwqe_device_create(cd);
 553	if (err < 0) {
 554		dev_err(&pci_dev->dev,
 555			"err: chdev init failed! (err=%d)\n", err);
 556		goto out_release_service_layer;
 557	}
 558	return 0;
 559
 560 out_release_service_layer:
 561	genwqe_release_service_layer(cd);
 562 out:
 563	if (genwqe_is_privileged(cd))
 564		genwqe_ffdc_buffs_free(cd);
 565	return -EIO;
 566}
 567
 568/**
 569 * genwqe_stop() - Stop card operation
 570 * @cd: GenWQE device information
 571 *
 572 * Recovery notes:
 573 *   As long as genwqe_thread runs we might access registers during
 574 *   error data capture. Same is with the genwqe_health_thread.
 575 *   When genwqe_bus_reset() fails this function might called two times:
 576 *   first by the genwqe_health_thread() and later by genwqe_remove() to
 577 *   unbind the device. We must be able to survive that.
 578 *
 579 * This function must be robust enough to be called twice.
 580 */
 581static int genwqe_stop(struct genwqe_dev *cd)
 582{
 583	genwqe_finish_queue(cd);	    /* no register access */
 584	genwqe_device_remove(cd);	    /* device removed, procs killed */
 585	genwqe_release_service_layer(cd);   /* here genwqe_thread is stopped */
 586
 587	if (genwqe_is_privileged(cd)) {
 588		pci_disable_sriov(cd->pci_dev);	/* access pci config space */
 589		genwqe_ffdc_buffs_free(cd);
 590	}
 591
 592	return 0;
 593}
 594
 595/**
 596 * genwqe_recover_card() - Try to recover the card if it is possible
 597 * @cd: GenWQE device information
 598 * @fatal_err: Indicate whether to attempt soft reset
 599 *
 600 * If fatal_err is set no register access is possible anymore. It is
 601 * likely that genwqe_start fails in that situation. Proper error
 602 * handling is required in this case.
 603 *
 604 * genwqe_bus_reset() will cause the pci code to call genwqe_remove()
 605 * and later genwqe_probe() for all virtual functions.
 606 */
 607static int genwqe_recover_card(struct genwqe_dev *cd, int fatal_err)
 608{
 609	int rc;
 610	struct pci_dev *pci_dev = cd->pci_dev;
 611
 612	genwqe_stop(cd);
 613
 614	/*
 615	 * Make sure chip is not reloaded to maintain FFDC. Write SLU
 616	 * Reset Register, CPLDReset field to 0.
 617	 */
 618	if (!fatal_err) {
 619		cd->softreset = 0x70ull;
 620		__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, cd->softreset);
 621	}
 622
 623	rc = genwqe_bus_reset(cd);
 624	if (rc != 0) {
 625		dev_err(&pci_dev->dev,
 626			"[%s] err: card recovery impossible!\n", __func__);
 627		return rc;
 628	}
 629
 630	rc = genwqe_start(cd);
 631	if (rc < 0) {
 632		dev_err(&pci_dev->dev,
 633			"[%s] err: failed to launch device!\n", __func__);
 634		return rc;
 635	}
 636	return 0;
 637}
 638
 639static int genwqe_health_check_cond(struct genwqe_dev *cd, u64 *gfir)
 640{
 641	*gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
 642	return (*gfir & GFIR_ERR_TRIGGER) &&
 643		genwqe_recovery_on_fatal_gfir_required(cd);
 644}
 645
 646/**
 647 * genwqe_fir_checking() - Check the fault isolation registers of the card
 648 * @cd: GenWQE device information
 649 *
 650 * If this code works ok, can be tried out with help of the genwqe_poke tool:
 651 *   sudo ./tools/genwqe_poke 0x8 0xfefefefefef
 652 *
 653 * Now the relevant FIRs/sFIRs should be printed out and the driver should
 654 * invoke recovery (devices are removed and readded).
 655 */
 656static u64 genwqe_fir_checking(struct genwqe_dev *cd)
 657{
 658	int j, iterations = 0;
 659	u64 mask, fir, fec, uid, gfir, gfir_masked, sfir, sfec;
 660	u32 fir_addr, fir_clr_addr, fec_addr, sfir_addr, sfec_addr;
 661	struct pci_dev *pci_dev = cd->pci_dev;
 662
 663 healthMonitor:
 664	iterations++;
 665	if (iterations > 16) {
 666		dev_err(&pci_dev->dev, "* exit looping after %d times\n",
 667			iterations);
 668		goto fatal_error;
 669	}
 670
 671	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
 672	if (gfir != 0x0)
 673		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n",
 674				    IO_SLC_CFGREG_GFIR, gfir);
 675	if (gfir == IO_ILLEGAL_VALUE)
 676		goto fatal_error;
 677
 678	/*
 679	 * Avoid printing when to GFIR bit is on prevents contignous
 680	 * printout e.g. for the following bug:
 681	 *   FIR set without a 2ndary FIR/FIR cannot be cleared
 682	 * Comment out the following if to get the prints:
 683	 */
 684	if (gfir == 0)
 685		return 0;
 686
 687	gfir_masked = gfir & GFIR_ERR_TRIGGER;  /* fatal errors */
 688
 689	for (uid = 0; uid < GENWQE_MAX_UNITS; uid++) { /* 0..2 in zEDC */
 690
 691		/* read the primary FIR (pfir) */
 692		fir_addr = (uid << 24) + 0x08;
 693		fir = __genwqe_readq(cd, fir_addr);
 694		if (fir == 0x0)
 695			continue;  /* no error in this unit */
 696
 697		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fir_addr, fir);
 698		if (fir == IO_ILLEGAL_VALUE)
 699			goto fatal_error;
 700
 701		/* read primary FEC */
 702		fec_addr = (uid << 24) + 0x18;
 703		fec = __genwqe_readq(cd, fec_addr);
 704
 705		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fec_addr, fec);
 706		if (fec == IO_ILLEGAL_VALUE)
 707			goto fatal_error;
 708
 709		for (j = 0, mask = 1ULL; j < 64; j++, mask <<= 1) {
 710
 711			/* secondary fir empty, skip it */
 712			if ((fir & mask) == 0x0)
 713				continue;
 714
 715			sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
 716			sfir = __genwqe_readq(cd, sfir_addr);
 717
 718			if (sfir == IO_ILLEGAL_VALUE)
 719				goto fatal_error;
 720			dev_err(&pci_dev->dev,
 721				"* 0x%08x 0x%016llx\n", sfir_addr, sfir);
 722
 723			sfec_addr = (uid << 24) + 0x300 + 0x08 * j;
 724			sfec = __genwqe_readq(cd, sfec_addr);
 725
 726			if (sfec == IO_ILLEGAL_VALUE)
 727				goto fatal_error;
 728			dev_err(&pci_dev->dev,
 729				"* 0x%08x 0x%016llx\n", sfec_addr, sfec);
 730
 731			gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
 732			if (gfir == IO_ILLEGAL_VALUE)
 733				goto fatal_error;
 734
 735			/* gfir turned on during routine! get out and
 736			   start over. */
 737			if ((gfir_masked == 0x0) &&
 738			    (gfir & GFIR_ERR_TRIGGER)) {
 739				goto healthMonitor;
 740			}
 741
 742			/* do not clear if we entered with a fatal gfir */
 743			if (gfir_masked == 0x0) {
 744
 745				/* NEW clear by mask the logged bits */
 746				sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
 747				__genwqe_writeq(cd, sfir_addr, sfir);
 748
 749				dev_dbg(&pci_dev->dev,
 750					"[HM] Clearing  2ndary FIR 0x%08x with 0x%016llx\n",
 751					sfir_addr, sfir);
 752
 753				/*
 754				 * note, these cannot be error-Firs
 755				 * since gfir_masked is 0 after sfir
 756				 * was read. Also, it is safe to do
 757				 * this write if sfir=0. Still need to
 758				 * clear the primary. This just means
 759				 * there is no secondary FIR.
 760				 */
 761
 762				/* clear by mask the logged bit. */
 763				fir_clr_addr = (uid << 24) + 0x10;
 764				__genwqe_writeq(cd, fir_clr_addr, mask);
 765
 766				dev_dbg(&pci_dev->dev,
 767					"[HM] Clearing primary FIR 0x%08x with 0x%016llx\n",
 768					fir_clr_addr, mask);
 769			}
 770		}
 771	}
 772	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
 773	if (gfir == IO_ILLEGAL_VALUE)
 774		goto fatal_error;
 775
 776	if ((gfir_masked == 0x0) && (gfir & GFIR_ERR_TRIGGER)) {
 777		/*
 778		 * Check once more that it didn't go on after all the
 779		 * FIRS were cleared.
 780		 */
 781		dev_dbg(&pci_dev->dev, "ACK! Another FIR! Recursing %d!\n",
 782			iterations);
 783		goto healthMonitor;
 784	}
 785	return gfir_masked;
 786
 787 fatal_error:
 788	return IO_ILLEGAL_VALUE;
 789}
 790
 791/**
 792 * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot
 793 * @pci_dev:	PCI device information struct
 794 *
 795 * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this
 796 * reset method will not work in all cases.
 797 *
 798 * Return: 0 on success or error code from pci_set_pcie_reset_state()
 799 */
 800static int genwqe_pci_fundamental_reset(struct pci_dev *pci_dev)
 801{
 802	int rc;
 803
 804	/*
 805	 * lock pci config space access from userspace,
 806	 * save state and issue PCIe fundamental reset
 807	 */
 808	pci_cfg_access_lock(pci_dev);
 809	pci_save_state(pci_dev);
 810	rc = pci_set_pcie_reset_state(pci_dev, pcie_warm_reset);
 811	if (!rc) {
 812		/* keep PCIe reset asserted for 250ms */
 813		msleep(250);
 814		pci_set_pcie_reset_state(pci_dev, pcie_deassert_reset);
 815		/* Wait for 2s to reload flash and train the link */
 816		msleep(2000);
 817	}
 818	pci_restore_state(pci_dev);
 819	pci_cfg_access_unlock(pci_dev);
 820	return rc;
 821}
 822
 823
 824static int genwqe_platform_recovery(struct genwqe_dev *cd)
 825{
 826	struct pci_dev *pci_dev = cd->pci_dev;
 827	int rc;
 828
 829	dev_info(&pci_dev->dev,
 830		 "[%s] resetting card for error recovery\n", __func__);
 831
 832	/* Clear out error injection flags */
 833	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
 834			    GENWQE_INJECT_GFIR_FATAL |
 835			    GENWQE_INJECT_GFIR_INFO);
 836
 837	genwqe_stop(cd);
 838
 839	/* Try recoverying the card with fundamental reset */
 840	rc = genwqe_pci_fundamental_reset(pci_dev);
 841	if (!rc) {
 842		rc = genwqe_start(cd);
 843		if (!rc)
 844			dev_info(&pci_dev->dev,
 845				 "[%s] card recovered\n", __func__);
 846		else
 847			dev_err(&pci_dev->dev,
 848				"[%s] err: cannot start card services! (err=%d)\n",
 849				__func__, rc);
 850	} else {
 851		dev_err(&pci_dev->dev,
 852			"[%s] card reset failed\n", __func__);
 853	}
 854
 855	return rc;
 856}
 857
 858/**
 859 * genwqe_reload_bistream() - reload card bitstream
 860 * @cd: GenWQE device information
 861 *
 862 * Set the appropriate register and call fundamental reset to reaload the card
 863 * bitstream.
 864 *
 865 * Return: 0 on success, error code otherwise
 866 */
 867static int genwqe_reload_bistream(struct genwqe_dev *cd)
 868{
 869	struct pci_dev *pci_dev = cd->pci_dev;
 870	int rc;
 871
 872	dev_info(&pci_dev->dev,
 873		 "[%s] resetting card for bitstream reload\n",
 874		 __func__);
 875
 876	genwqe_stop(cd);
 877
 878	/*
 879	 * Cause a CPLD reprogram with the 'next_bitstream'
 880	 * partition on PCIe hot or fundamental reset
 881	 */
 882	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
 883			(cd->softreset & 0xcull) | 0x70ull);
 884
 885	rc = genwqe_pci_fundamental_reset(pci_dev);
 886	if (rc) {
 887		/*
 888		 * A fundamental reset failure can be caused
 889		 * by lack of support on the arch, so we just
 890		 * log the error and try to start the card
 891		 * again.
 892		 */
 893		dev_err(&pci_dev->dev,
 894			"[%s] err: failed to reset card for bitstream reload\n",
 895			__func__);
 896	}
 897
 898	rc = genwqe_start(cd);
 899	if (rc) {
 900		dev_err(&pci_dev->dev,
 901			"[%s] err: cannot start card services! (err=%d)\n",
 902			__func__, rc);
 903		return rc;
 904	}
 905	dev_info(&pci_dev->dev,
 906		 "[%s] card reloaded\n", __func__);
 907	return 0;
 908}
 909
 910
 911/**
 912 * genwqe_health_thread() - Health checking thread
 913 * @data: GenWQE device information
 914 *
 915 * This thread is only started for the PF of the card.
 916 *
 917 * This thread monitors the health of the card. A critical situation
 918 * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In
 919 * this case we need to be recovered from outside. Writing to
 920 * registers will very likely not work either.
 921 *
 922 * This thread must only exit if kthread_should_stop() becomes true.
 923 *
 924 * Condition for the health-thread to trigger:
 925 *   a) when a kthread_stop() request comes in or
 926 *   b) a critical GFIR occured
 927 *
 928 * Informational GFIRs are checked and potentially printed in
 929 * GENWQE_HEALTH_CHECK_INTERVAL seconds.
 930 */
 931static int genwqe_health_thread(void *data)
 932{
 933	int rc, should_stop = 0;
 934	struct genwqe_dev *cd = data;
 935	struct pci_dev *pci_dev = cd->pci_dev;
 936	u64 gfir, gfir_masked, slu_unitcfg, app_unitcfg;
 937
 938 health_thread_begin:
 939	while (!kthread_should_stop()) {
 940		rc = wait_event_interruptible_timeout(cd->health_waitq,
 941			 (genwqe_health_check_cond(cd, &gfir) ||
 942			  (should_stop = kthread_should_stop())),
 943				GENWQE_HEALTH_CHECK_INTERVAL * HZ);
 944
 945		if (should_stop)
 946			break;
 947
 948		if (gfir == IO_ILLEGAL_VALUE) {
 949			dev_err(&pci_dev->dev,
 950				"[%s] GFIR=%016llx\n", __func__, gfir);
 951			goto fatal_error;
 952		}
 953
 954		slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
 955		if (slu_unitcfg == IO_ILLEGAL_VALUE) {
 956			dev_err(&pci_dev->dev,
 957				"[%s] SLU_UNITCFG=%016llx\n",
 958				__func__, slu_unitcfg);
 959			goto fatal_error;
 960		}
 961
 962		app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
 963		if (app_unitcfg == IO_ILLEGAL_VALUE) {
 964			dev_err(&pci_dev->dev,
 965				"[%s] APP_UNITCFG=%016llx\n",
 966				__func__, app_unitcfg);
 967			goto fatal_error;
 968		}
 969
 970		gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
 971		if (gfir == IO_ILLEGAL_VALUE) {
 972			dev_err(&pci_dev->dev,
 973				"[%s] %s: GFIR=%016llx\n", __func__,
 974				(gfir & GFIR_ERR_TRIGGER) ? "err" : "info",
 975				gfir);
 976			goto fatal_error;
 977		}
 978
 979		gfir_masked = genwqe_fir_checking(cd);
 980		if (gfir_masked == IO_ILLEGAL_VALUE)
 981			goto fatal_error;
 982
 983		/*
 984		 * GFIR ErrorTrigger bits set => reset the card!
 985		 * Never do this for old/manufacturing images!
 986		 */
 987		if ((gfir_masked) && !cd->skip_recovery &&
 988		    genwqe_recovery_on_fatal_gfir_required(cd)) {
 989
 990			cd->card_state = GENWQE_CARD_FATAL_ERROR;
 991
 992			rc = genwqe_recover_card(cd, 0);
 993			if (rc < 0) {
 994				/* FIXME Card is unusable and needs unbind! */
 995				goto fatal_error;
 996			}
 997		}
 998
 999		if (cd->card_state == GENWQE_CARD_RELOAD_BITSTREAM) {
1000			/* Userspace requested card bitstream reload */
1001			rc = genwqe_reload_bistream(cd);
1002			if (rc)
1003				goto fatal_error;
1004		}
1005
1006		cd->last_gfir = gfir;
1007		cond_resched();
1008	}
1009
1010	return 0;
1011
1012 fatal_error:
1013	if (cd->use_platform_recovery) {
1014		/*
1015		 * Since we use raw accessors, EEH errors won't be detected
1016		 * by the platform until we do a non-raw MMIO or config space
1017		 * read
1018		 */
1019		readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1020
1021		/* We do nothing if the card is going over PCI recovery */
1022		if (pci_channel_offline(pci_dev))
1023			return -EIO;
1024
1025		/*
1026		 * If it's supported by the platform, we try a fundamental reset
1027		 * to recover from a fatal error. Otherwise, we continue to wait
1028		 * for an external recovery procedure to take care of it.
1029		 */
1030		rc = genwqe_platform_recovery(cd);
1031		if (!rc)
1032			goto health_thread_begin;
1033	}
1034
1035	dev_err(&pci_dev->dev,
1036		"[%s] card unusable. Please trigger unbind!\n", __func__);
1037
1038	/* Bring down logical devices to inform user space via udev remove. */
1039	cd->card_state = GENWQE_CARD_FATAL_ERROR;
1040	genwqe_stop(cd);
1041
1042	/* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */
1043	while (!kthread_should_stop())
1044		cond_resched();
1045
1046	return -EIO;
1047}
1048
1049static int genwqe_health_check_start(struct genwqe_dev *cd)
1050{
1051	int rc;
1052
1053	if (GENWQE_HEALTH_CHECK_INTERVAL <= 0)
1054		return 0;	/* valid for disabling the service */
1055
1056	/* moved before request_irq() */
1057	/* init_waitqueue_head(&cd->health_waitq); */
1058
1059	cd->health_thread = kthread_run(genwqe_health_thread, cd,
1060					GENWQE_DEVNAME "%d_health",
1061					cd->card_idx);
1062	if (IS_ERR(cd->health_thread)) {
1063		rc = PTR_ERR(cd->health_thread);
1064		cd->health_thread = NULL;
1065		return rc;
1066	}
1067	return 0;
1068}
1069
1070static int genwqe_health_thread_running(struct genwqe_dev *cd)
1071{
1072	return cd->health_thread != NULL;
1073}
1074
1075static int genwqe_health_check_stop(struct genwqe_dev *cd)
1076{
1077	if (!genwqe_health_thread_running(cd))
1078		return -EIO;
1079
1080	kthread_stop(cd->health_thread);
1081	cd->health_thread = NULL;
1082	return 0;
1083}
1084
1085/**
1086 * genwqe_pci_setup() - Allocate PCIe related resources for our card
1087 * @cd: GenWQE device information
1088 */
1089static int genwqe_pci_setup(struct genwqe_dev *cd)
1090{
1091	int err;
1092	struct pci_dev *pci_dev = cd->pci_dev;
1093
1094	err = pci_enable_device_mem(pci_dev);
1095	if (err) {
1096		dev_err(&pci_dev->dev,
1097			"err: failed to enable pci memory (err=%d)\n", err);
1098		goto err_out;
1099	}
1100
1101	/* Reserve PCI I/O and memory resources */
1102	err = pci_request_mem_regions(pci_dev, genwqe_driver_name);
1103	if (err) {
1104		dev_err(&pci_dev->dev,
1105			"[%s] err: request bars failed (%d)\n", __func__, err);
1106		err = -EIO;
1107		goto err_disable_device;
1108	}
1109
1110	/* check for 64-bit DMA address supported (DAC) */
1111	/* check for 32-bit DMA address supported (SAC) */
1112	if (dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)) &&
1113	    dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32))) {
1114		dev_err(&pci_dev->dev,
1115			"err: neither DMA32 nor DMA64 supported\n");
1116		err = -EIO;
1117		goto out_release_resources;
1118	}
1119
1120	pci_set_master(pci_dev);
1121
1122	/* EEH recovery requires PCIe fundamental reset */
1123	pci_dev->needs_freset = 1;
1124
1125	/* request complete BAR-0 space (length = 0) */
1126	cd->mmio_len = pci_resource_len(pci_dev, 0);
1127	cd->mmio = pci_iomap(pci_dev, 0, 0);
1128	if (cd->mmio == NULL) {
1129		dev_err(&pci_dev->dev,
1130			"[%s] err: mapping BAR0 failed\n", __func__);
1131		err = -ENOMEM;
1132		goto out_release_resources;
1133	}
1134
1135	cd->num_vfs = pci_sriov_get_totalvfs(pci_dev);
1136	if (cd->num_vfs < 0)
1137		cd->num_vfs = 0;
1138
1139	err = genwqe_read_ids(cd);
1140	if (err)
1141		goto out_iounmap;
1142
1143	return 0;
1144
1145 out_iounmap:
1146	pci_iounmap(pci_dev, cd->mmio);
1147 out_release_resources:
1148	pci_release_mem_regions(pci_dev);
1149 err_disable_device:
1150	pci_disable_device(pci_dev);
1151 err_out:
1152	return err;
1153}
1154
1155/**
1156 * genwqe_pci_remove() - Free PCIe related resources for our card
1157 * @cd: GenWQE device information
1158 */
1159static void genwqe_pci_remove(struct genwqe_dev *cd)
1160{
1161	struct pci_dev *pci_dev = cd->pci_dev;
1162
1163	if (cd->mmio)
1164		pci_iounmap(pci_dev, cd->mmio);
1165
1166	pci_release_mem_regions(pci_dev);
1167	pci_disable_device(pci_dev);
1168}
1169
1170/**
1171 * genwqe_probe() - Device initialization
1172 * @pci_dev:	PCI device information struct
1173 * @id:		PCI device ID
1174 *
1175 * Callable for multiple cards. This function is called on bind.
1176 *
1177 * Return: 0 if succeeded, < 0 when failed
1178 */
1179static int genwqe_probe(struct pci_dev *pci_dev,
1180			const struct pci_device_id *id)
1181{
1182	int err;
1183	struct genwqe_dev *cd;
1184
1185	genwqe_init_crc32();
1186
1187	cd = genwqe_dev_alloc();
1188	if (IS_ERR(cd)) {
1189		dev_err(&pci_dev->dev, "err: could not alloc mem (err=%d)!\n",
1190			(int)PTR_ERR(cd));
1191		return PTR_ERR(cd);
1192	}
1193
1194	dev_set_drvdata(&pci_dev->dev, cd);
1195	cd->pci_dev = pci_dev;
1196
1197	err = genwqe_pci_setup(cd);
1198	if (err < 0) {
1199		dev_err(&pci_dev->dev,
1200			"err: problems with PCI setup (err=%d)\n", err);
1201		goto out_free_dev;
1202	}
1203
1204	err = genwqe_start(cd);
1205	if (err < 0) {
1206		dev_err(&pci_dev->dev,
1207			"err: cannot start card services! (err=%d)\n", err);
1208		goto out_pci_remove;
1209	}
1210
1211	if (genwqe_is_privileged(cd)) {
1212		err = genwqe_health_check_start(cd);
1213		if (err < 0) {
1214			dev_err(&pci_dev->dev,
1215				"err: cannot start health checking! (err=%d)\n",
1216				err);
1217			goto out_stop_services;
1218		}
1219	}
1220	return 0;
1221
1222 out_stop_services:
1223	genwqe_stop(cd);
1224 out_pci_remove:
1225	genwqe_pci_remove(cd);
1226 out_free_dev:
1227	genwqe_dev_free(cd);
1228	return err;
1229}
1230
1231/**
1232 * genwqe_remove() - Called when device is removed (hot-plugable)
1233 * @pci_dev:	PCI device information struct
1234 *
1235 * Or when driver is unloaded respecitively when unbind is done.
1236 */
1237static void genwqe_remove(struct pci_dev *pci_dev)
1238{
1239	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1240
1241	genwqe_health_check_stop(cd);
1242
1243	/*
1244	 * genwqe_stop() must survive if it is called twice
1245	 * sequentially. This happens when the health thread calls it
1246	 * and fails on genwqe_bus_reset().
1247	 */
1248	genwqe_stop(cd);
1249	genwqe_pci_remove(cd);
1250	genwqe_dev_free(cd);
1251}
1252
1253/**
1254 * genwqe_err_error_detected() - Error detection callback
1255 * @pci_dev:	PCI device information struct
1256 * @state:	PCI channel state
1257 *
1258 * This callback is called by the PCI subsystem whenever a PCI bus
1259 * error is detected.
1260 */
1261static pci_ers_result_t genwqe_err_error_detected(struct pci_dev *pci_dev,
1262						 pci_channel_state_t state)
1263{
1264	struct genwqe_dev *cd;
1265
1266	dev_err(&pci_dev->dev, "[%s] state=%d\n", __func__, state);
1267
1268	cd = dev_get_drvdata(&pci_dev->dev);
1269	if (cd == NULL)
1270		return PCI_ERS_RESULT_DISCONNECT;
1271
1272	/* Stop the card */
1273	genwqe_health_check_stop(cd);
1274	genwqe_stop(cd);
1275
1276	/*
1277	 * On permanent failure, the PCI code will call device remove
1278	 * after the return of this function.
1279	 * genwqe_stop() can be called twice.
1280	 */
1281	if (state == pci_channel_io_perm_failure) {
1282		return PCI_ERS_RESULT_DISCONNECT;
1283	} else {
1284		genwqe_pci_remove(cd);
1285		return PCI_ERS_RESULT_NEED_RESET;
1286	}
1287}
1288
1289static pci_ers_result_t genwqe_err_slot_reset(struct pci_dev *pci_dev)
1290{
1291	int rc;
1292	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1293
1294	rc = genwqe_pci_setup(cd);
1295	if (!rc) {
1296		return PCI_ERS_RESULT_RECOVERED;
1297	} else {
1298		dev_err(&pci_dev->dev,
1299			"err: problems with PCI setup (err=%d)\n", rc);
1300		return PCI_ERS_RESULT_DISCONNECT;
1301	}
1302}
1303
1304static pci_ers_result_t genwqe_err_result_none(struct pci_dev *dev)
1305{
1306	return PCI_ERS_RESULT_NONE;
1307}
1308
1309static void genwqe_err_resume(struct pci_dev *pci_dev)
1310{
1311	int rc;
1312	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1313
1314	rc = genwqe_start(cd);
1315	if (!rc) {
1316		rc = genwqe_health_check_start(cd);
1317		if (rc)
1318			dev_err(&pci_dev->dev,
1319				"err: cannot start health checking! (err=%d)\n",
1320				rc);
1321	} else {
1322		dev_err(&pci_dev->dev,
1323			"err: cannot start card services! (err=%d)\n", rc);
1324	}
1325}
1326
1327static int genwqe_sriov_configure(struct pci_dev *dev, int numvfs)
1328{
1329	int rc;
1330	struct genwqe_dev *cd = dev_get_drvdata(&dev->dev);
1331
1332	if (numvfs > 0) {
1333		genwqe_setup_vf_jtimer(cd);
1334		rc = pci_enable_sriov(dev, numvfs);
1335		if (rc < 0)
1336			return rc;
1337		return numvfs;
1338	}
1339	if (numvfs == 0) {
1340		pci_disable_sriov(dev);
1341		return 0;
1342	}
1343	return 0;
1344}
1345
1346static const struct pci_error_handlers genwqe_err_handler = {
1347	.error_detected = genwqe_err_error_detected,
1348	.mmio_enabled	= genwqe_err_result_none,
1349	.slot_reset	= genwqe_err_slot_reset,
1350	.resume		= genwqe_err_resume,
1351};
1352
1353static struct pci_driver genwqe_driver = {
1354	.name	  = genwqe_driver_name,
1355	.id_table = genwqe_device_table,
1356	.probe	  = genwqe_probe,
1357	.remove	  = genwqe_remove,
1358	.sriov_configure = genwqe_sriov_configure,
1359	.err_handler = &genwqe_err_handler,
1360};
1361
1362/**
1363 * genwqe_init_module() - Driver registration and initialization
1364 */
1365static int __init genwqe_init_module(void)
1366{
1367	int rc;
1368
1369	rc = class_register(&class_genwqe);
1370	if (rc) {
1371		pr_err("[%s] create class failed\n", __func__);
1372		return -ENOMEM;
1373	}
1374
1375	debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL);
1376
1377	rc = pci_register_driver(&genwqe_driver);
1378	if (rc != 0) {
1379		pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__, rc);
1380		goto err_out0;
1381	}
1382
1383	return rc;
1384
1385 err_out0:
1386	debugfs_remove(debugfs_genwqe);
1387	class_unregister(&class_genwqe);
1388	return rc;
1389}
1390
1391/**
1392 * genwqe_exit_module() - Driver exit
1393 */
1394static void __exit genwqe_exit_module(void)
1395{
1396	pci_unregister_driver(&genwqe_driver);
1397	debugfs_remove(debugfs_genwqe);
1398	class_unregister(&class_genwqe);
1399}
1400
1401module_init(genwqe_init_module);
1402module_exit(genwqe_exit_module);