qedi_main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * QLogic iSCSI Offload Driver
   4 * Copyright (c) 2016 Cavium Inc.
   5 */
   6
   7#include <linux/module.h>
   8#include <linux/pci.h>
   9#include <linux/kernel.h>
  10#include <linux/if_arp.h>
  11#include <scsi/iscsi_if.h>
  12#include <linux/inet.h>
  13#include <net/arp.h>
  14#include <linux/list.h>
  15#include <linux/kthread.h>
  16#include <linux/mm.h>
  17#include <linux/if_vlan.h>
  18#include <linux/cpu.h>
  19#include <linux/iscsi_boot_sysfs.h>
  20
  21#include <scsi/scsi_cmnd.h>
  22#include <scsi/scsi_device.h>
  23#include <scsi/scsi_eh.h>
  24#include <scsi/scsi_host.h>
  25#include <scsi/scsi.h>
  26
  27#include "qedi.h"
  28#include "qedi_gbl.h"
  29#include "qedi_iscsi.h"
  30
  31static uint qedi_qed_debug;
  32module_param(qedi_qed_debug, uint, 0644);
  33MODULE_PARM_DESC(qedi_qed_debug, " QED debug level 0 (default)");
  34
  35static uint qedi_fw_debug;
  36module_param(qedi_fw_debug, uint, 0644);
  37MODULE_PARM_DESC(qedi_fw_debug, " Firmware debug level 0(default) to 3");
  38
  39uint qedi_dbg_log = QEDI_LOG_WARN | QEDI_LOG_SCSI_TM;
  40module_param(qedi_dbg_log, uint, 0644);
  41MODULE_PARM_DESC(qedi_dbg_log, " Default debug level");
  42
  43uint qedi_io_tracing;
  44module_param(qedi_io_tracing, uint, 0644);
  45MODULE_PARM_DESC(qedi_io_tracing,
  46		 " Enable logging of SCSI requests/completions into trace buffer. (default off).");
  47
  48static uint qedi_ll2_buf_size = 0x400;
  49module_param(qedi_ll2_buf_size, uint, 0644);
  50MODULE_PARM_DESC(qedi_ll2_buf_size,
  51		 "parameter to set ping packet size, default - 0x400, Jumbo packets - 0x2400.");
  52
  53static uint qedi_flags_override;
  54module_param(qedi_flags_override, uint, 0644);
  55MODULE_PARM_DESC(qedi_flags_override, "Disable/Enable MFW error flags bits action.");
  56
  57const struct qed_iscsi_ops *qedi_ops;
  58static struct scsi_transport_template *qedi_scsi_transport;
  59static struct pci_driver qedi_pci_driver;
  60static DEFINE_PER_CPU(struct qedi_percpu_s, qedi_percpu);
  61static LIST_HEAD(qedi_udev_list);
  62/* Static function declaration */
  63static int qedi_alloc_global_queues(struct qedi_ctx *qedi);
  64static void qedi_free_global_queues(struct qedi_ctx *qedi);
  65static struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid);
  66static void qedi_reset_uio_rings(struct qedi_uio_dev *udev);
  67static void qedi_ll2_free_skbs(struct qedi_ctx *qedi);
  68static struct nvm_iscsi_block *qedi_get_nvram_block(struct qedi_ctx *qedi);
  69static void qedi_recovery_handler(struct work_struct *work);
  70static void qedi_schedule_hw_err_handler(void *dev,
  71					 enum qed_hw_err_type err_type);
  72static int qedi_suspend(struct pci_dev *pdev, pm_message_t state);
  73
  74static int qedi_iscsi_event_cb(void *context, u8 fw_event_code, void *fw_handle)
  75{
  76	struct qedi_ctx *qedi;
  77	struct qedi_endpoint *qedi_ep;
  78	struct iscsi_eqe_data *data;
  79	int rval = 0;
  80
  81	if (!context || !fw_handle) {
  82		QEDI_ERR(NULL, "Recv event with ctx NULL\n");
  83		return -EINVAL;
  84	}
  85
  86	qedi = (struct qedi_ctx *)context;
  87	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
  88		  "Recv Event %d fw_handle %p\n", fw_event_code, fw_handle);
  89
  90	data = (struct iscsi_eqe_data *)fw_handle;
  91	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
  92		  "icid=0x%x conn_id=0x%x err-code=0x%x error-pdu-opcode-reserved=0x%x\n",
  93		   data->icid, data->conn_id, data->error_code,
  94		   data->error_pdu_opcode_reserved);
  95
  96	qedi_ep = qedi->ep_tbl[data->icid];
  97
  98	if (!qedi_ep) {
  99		QEDI_WARN(&qedi->dbg_ctx,
 100			  "Cannot process event, ep already disconnected, cid=0x%x\n",
 101			   data->icid);
 102		WARN_ON(1);
 103		return -ENODEV;
 104	}
 105
 106	switch (fw_event_code) {
 107	case ISCSI_EVENT_TYPE_ASYN_CONNECT_COMPLETE:
 108		if (qedi_ep->state == EP_STATE_OFLDCONN_START)
 109			qedi_ep->state = EP_STATE_OFLDCONN_COMPL;
 110
 111		wake_up_interruptible(&qedi_ep->tcp_ofld_wait);
 112		break;
 113	case ISCSI_EVENT_TYPE_ASYN_TERMINATE_DONE:
 114		qedi_ep->state = EP_STATE_DISCONN_COMPL;
 115		wake_up_interruptible(&qedi_ep->tcp_ofld_wait);
 116		break;
 117	case ISCSI_EVENT_TYPE_ISCSI_CONN_ERROR:
 118		qedi_process_iscsi_error(qedi_ep, data);
 119		break;
 120	case ISCSI_EVENT_TYPE_ASYN_ABORT_RCVD:
 121	case ISCSI_EVENT_TYPE_ASYN_SYN_RCVD:
 122	case ISCSI_EVENT_TYPE_ASYN_MAX_RT_TIME:
 123	case ISCSI_EVENT_TYPE_ASYN_MAX_RT_CNT:
 124	case ISCSI_EVENT_TYPE_ASYN_MAX_KA_PROBES_CNT:
 125	case ISCSI_EVENT_TYPE_ASYN_FIN_WAIT2:
 126	case ISCSI_EVENT_TYPE_TCP_CONN_ERROR:
 127		qedi_process_tcp_error(qedi_ep, data);
 128		break;
 129	default:
 130		QEDI_ERR(&qedi->dbg_ctx, "Recv Unknown Event %u\n",
 131			 fw_event_code);
 132	}
 133
 134	return rval;
 135}
 136
 137static int qedi_uio_open(struct uio_info *uinfo, struct inode *inode)
 138{
 139	struct qedi_uio_dev *udev = uinfo->priv;
 140	struct qedi_ctx *qedi = udev->qedi;
 141
 142	if (!capable(CAP_NET_ADMIN))
 143		return -EPERM;
 144
 145	if (udev->uio_dev != -1)
 146		return -EBUSY;
 147
 148	rtnl_lock();
 149	udev->uio_dev = iminor(inode);
 150	qedi_reset_uio_rings(udev);
 151	set_bit(UIO_DEV_OPENED, &qedi->flags);
 152	rtnl_unlock();
 153
 154	return 0;
 155}
 156
 157static int qedi_uio_close(struct uio_info *uinfo, struct inode *inode)
 158{
 159	struct qedi_uio_dev *udev = uinfo->priv;
 160	struct qedi_ctx *qedi = udev->qedi;
 161
 162	udev->uio_dev = -1;
 163	clear_bit(UIO_DEV_OPENED, &qedi->flags);
 164	qedi_ll2_free_skbs(qedi);
 165	return 0;
 166}
 167
 168static void __qedi_free_uio_rings(struct qedi_uio_dev *udev)
 169{
 170	if (udev->uctrl) {
 171		free_page((unsigned long)udev->uctrl);
 172		udev->uctrl = NULL;
 173	}
 174
 175	if (udev->ll2_ring) {
 176		free_page((unsigned long)udev->ll2_ring);
 177		udev->ll2_ring = NULL;
 178	}
 179
 180	if (udev->ll2_buf) {
 181		free_pages((unsigned long)udev->ll2_buf, 2);
 182		udev->ll2_buf = NULL;
 183	}
 184}
 185
 186static void __qedi_free_uio(struct qedi_uio_dev *udev)
 187{
 188	uio_unregister_device(&udev->qedi_uinfo);
 189
 190	__qedi_free_uio_rings(udev);
 191
 192	pci_dev_put(udev->pdev);
 193	kfree(udev);
 194}
 195
 196static void qedi_free_uio(struct qedi_uio_dev *udev)
 197{
 198	if (!udev)
 199		return;
 200
 201	list_del_init(&udev->list);
 202	__qedi_free_uio(udev);
 203}
 204
 205static void qedi_reset_uio_rings(struct qedi_uio_dev *udev)
 206{
 207	struct qedi_ctx *qedi = NULL;
 208	struct qedi_uio_ctrl *uctrl = NULL;
 209
 210	qedi = udev->qedi;
 211	uctrl = udev->uctrl;
 212
 213	spin_lock_bh(&qedi->ll2_lock);
 214	uctrl->host_rx_cons = 0;
 215	uctrl->hw_rx_prod = 0;
 216	uctrl->hw_rx_bd_prod = 0;
 217	uctrl->host_rx_bd_cons = 0;
 218
 219	memset(udev->ll2_ring, 0, udev->ll2_ring_size);
 220	memset(udev->ll2_buf, 0, udev->ll2_buf_size);
 221	spin_unlock_bh(&qedi->ll2_lock);
 222}
 223
 224static int __qedi_alloc_uio_rings(struct qedi_uio_dev *udev)
 225{
 226	int rc = 0;
 227
 228	if (udev->ll2_ring || udev->ll2_buf)
 229		return rc;
 230
 231	/* Memory for control area.  */
 232	udev->uctrl = (void *)get_zeroed_page(GFP_KERNEL);
 233	if (!udev->uctrl)
 234		return -ENOMEM;
 235
 236	/* Allocating memory for LL2 ring  */
 237	udev->ll2_ring_size = QEDI_PAGE_SIZE;
 238	udev->ll2_ring = (void *)get_zeroed_page(GFP_KERNEL | __GFP_COMP);
 239	if (!udev->ll2_ring) {
 240		rc = -ENOMEM;
 241		goto exit_alloc_ring;
 242	}
 243
 244	/* Allocating memory for Tx/Rx pkt buffer */
 245	udev->ll2_buf_size = TX_RX_RING * qedi_ll2_buf_size;
 246	udev->ll2_buf_size = QEDI_PAGE_ALIGN(udev->ll2_buf_size);
 247	udev->ll2_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_COMP |
 248						 __GFP_ZERO, 2);
 249	if (!udev->ll2_buf) {
 250		rc = -ENOMEM;
 251		goto exit_alloc_buf;
 252	}
 253	return rc;
 254
 255exit_alloc_buf:
 256	free_page((unsigned long)udev->ll2_ring);
 257	udev->ll2_ring = NULL;
 258exit_alloc_ring:
 259	return rc;
 260}
 261
 262static int qedi_alloc_uio_rings(struct qedi_ctx *qedi)
 263{
 264	struct qedi_uio_dev *udev = NULL;
 265	int rc = 0;
 266
 267	list_for_each_entry(udev, &qedi_udev_list, list) {
 268		if (udev->pdev == qedi->pdev) {
 269			udev->qedi = qedi;
 270			if (__qedi_alloc_uio_rings(udev)) {
 271				udev->qedi = NULL;
 272				return -ENOMEM;
 273			}
 274			qedi->udev = udev;
 275			return 0;
 276		}
 277	}
 278
 279	udev = kzalloc(sizeof(*udev), GFP_KERNEL);
 280	if (!udev)
 281		goto err_udev;
 282
 283	udev->uio_dev = -1;
 284
 285	udev->qedi = qedi;
 286	udev->pdev = qedi->pdev;
 287
 288	rc = __qedi_alloc_uio_rings(udev);
 289	if (rc)
 290		goto err_uctrl;
 291
 292	list_add(&udev->list, &qedi_udev_list);
 293
 294	pci_dev_get(udev->pdev);
 295	qedi->udev = udev;
 296
 297	udev->tx_pkt = udev->ll2_buf;
 298	udev->rx_pkt = udev->ll2_buf + qedi_ll2_buf_size;
 299	return 0;
 300
 301 err_uctrl:
 302	kfree(udev);
 303 err_udev:
 304	return -ENOMEM;
 305}
 306
 307static int qedi_init_uio(struct qedi_ctx *qedi)
 308{
 309	struct qedi_uio_dev *udev = qedi->udev;
 310	struct uio_info *uinfo;
 311	int ret = 0;
 312
 313	if (!udev)
 314		return -ENOMEM;
 315
 316	uinfo = &udev->qedi_uinfo;
 317
 318	uinfo->mem[0].addr = (unsigned long)udev->uctrl;
 319	uinfo->mem[0].size = sizeof(struct qedi_uio_ctrl);
 320	uinfo->mem[0].memtype = UIO_MEM_LOGICAL;
 321
 322	uinfo->mem[1].addr = (unsigned long)udev->ll2_ring;
 323	uinfo->mem[1].size = udev->ll2_ring_size;
 324	uinfo->mem[1].memtype = UIO_MEM_LOGICAL;
 325
 326	uinfo->mem[2].addr = (unsigned long)udev->ll2_buf;
 327	uinfo->mem[2].size = udev->ll2_buf_size;
 328	uinfo->mem[2].memtype = UIO_MEM_LOGICAL;
 329
 330	uinfo->name = "qedi_uio";
 331	uinfo->version = QEDI_MODULE_VERSION;
 332	uinfo->irq = UIO_IRQ_CUSTOM;
 333
 334	uinfo->open = qedi_uio_open;
 335	uinfo->release = qedi_uio_close;
 336
 337	if (udev->uio_dev == -1) {
 338		if (!uinfo->priv) {
 339			uinfo->priv = udev;
 340
 341			ret = uio_register_device(&udev->pdev->dev, uinfo);
 342			if (ret) {
 343				QEDI_ERR(&qedi->dbg_ctx,
 344					 "UIO registration failed\n");
 345			}
 346		}
 347	}
 348
 349	return ret;
 350}
 351
 352static int qedi_alloc_and_init_sb(struct qedi_ctx *qedi,
 353				  struct qed_sb_info *sb_info, u16 sb_id)
 354{
 355	struct status_block *sb_virt;
 356	dma_addr_t sb_phys;
 357	int ret;
 358
 359	sb_virt = dma_alloc_coherent(&qedi->pdev->dev,
 360				     sizeof(struct status_block), &sb_phys,
 361				     GFP_KERNEL);
 362	if (!sb_virt) {
 363		QEDI_ERR(&qedi->dbg_ctx,
 364			 "Status block allocation failed for id = %d.\n",
 365			  sb_id);
 366		return -ENOMEM;
 367	}
 368
 369	ret = qedi_ops->common->sb_init(qedi->cdev, sb_info, sb_virt, sb_phys,
 370				       sb_id, QED_SB_TYPE_STORAGE);
 371	if (ret) {
 372		dma_free_coherent(&qedi->pdev->dev, sizeof(*sb_virt), sb_virt, sb_phys);
 373		QEDI_ERR(&qedi->dbg_ctx,
 374			 "Status block initialization failed for id = %d.\n",
 375			  sb_id);
 376		return ret;
 377	}
 378
 379	return 0;
 380}
 381
 382static void qedi_free_sb(struct qedi_ctx *qedi)
 383{
 384	struct qed_sb_info *sb_info;
 385	int id;
 386
 387	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
 388		sb_info = &qedi->sb_array[id];
 389		if (sb_info->sb_virt)
 390			dma_free_coherent(&qedi->pdev->dev,
 391					  sizeof(*sb_info->sb_virt),
 392					  (void *)sb_info->sb_virt,
 393					  sb_info->sb_phys);
 394	}
 395}
 396
 397static void qedi_free_fp(struct qedi_ctx *qedi)
 398{
 399	kfree(qedi->fp_array);
 400	kfree(qedi->sb_array);
 401}
 402
 403static void qedi_destroy_fp(struct qedi_ctx *qedi)
 404{
 405	qedi_free_sb(qedi);
 406	qedi_free_fp(qedi);
 407}
 408
 409static int qedi_alloc_fp(struct qedi_ctx *qedi)
 410{
 411	int ret = 0;
 412
 413	qedi->fp_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),
 414				 sizeof(struct qedi_fastpath), GFP_KERNEL);
 415	if (!qedi->fp_array) {
 416		QEDI_ERR(&qedi->dbg_ctx,
 417			 "fastpath fp array allocation failed.\n");
 418		return -ENOMEM;
 419	}
 420
 421	qedi->sb_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),
 422				 sizeof(struct qed_sb_info), GFP_KERNEL);
 423	if (!qedi->sb_array) {
 424		QEDI_ERR(&qedi->dbg_ctx,
 425			 "fastpath sb array allocation failed.\n");
 426		ret = -ENOMEM;
 427		goto free_fp;
 428	}
 429
 430	return ret;
 431
 432free_fp:
 433	qedi_free_fp(qedi);
 434	return ret;
 435}
 436
 437static void qedi_int_fp(struct qedi_ctx *qedi)
 438{
 439	struct qedi_fastpath *fp;
 440	int id;
 441
 442	memset(qedi->fp_array, 0, MIN_NUM_CPUS_MSIX(qedi) *
 443	       sizeof(*qedi->fp_array));
 444	memset(qedi->sb_array, 0, MIN_NUM_CPUS_MSIX(qedi) *
 445	       sizeof(*qedi->sb_array));
 446
 447	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
 448		fp = &qedi->fp_array[id];
 449		fp->sb_info = &qedi->sb_array[id];
 450		fp->sb_id = id;
 451		fp->qedi = qedi;
 452		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
 453			 "qedi", id);
 454
 455		/* fp_array[i] ---- irq cookie
 456		 * So init data which is needed in int ctx
 457		 */
 458	}
 459}
 460
 461static int qedi_prepare_fp(struct qedi_ctx *qedi)
 462{
 463	struct qedi_fastpath *fp;
 464	int id, ret = 0;
 465
 466	ret = qedi_alloc_fp(qedi);
 467	if (ret)
 468		goto err;
 469
 470	qedi_int_fp(qedi);
 471
 472	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
 473		fp = &qedi->fp_array[id];
 474		ret = qedi_alloc_and_init_sb(qedi, fp->sb_info, fp->sb_id);
 475		if (ret) {
 476			QEDI_ERR(&qedi->dbg_ctx,
 477				 "SB allocation and initialization failed.\n");
 478			ret = -EIO;
 479			goto err_init;
 480		}
 481	}
 482
 483	return 0;
 484
 485err_init:
 486	qedi_free_sb(qedi);
 487	qedi_free_fp(qedi);
 488err:
 489	return ret;
 490}
 491
 492static int qedi_setup_cid_que(struct qedi_ctx *qedi)
 493{
 494	int i;
 495
 496	qedi->cid_que.cid_que_base = kmalloc_array(qedi->max_active_conns,
 497						   sizeof(u32), GFP_KERNEL);
 498	if (!qedi->cid_que.cid_que_base)
 499		return -ENOMEM;
 500
 501	qedi->cid_que.conn_cid_tbl = kmalloc_array(qedi->max_active_conns,
 502						   sizeof(struct qedi_conn *),
 503						   GFP_KERNEL);
 504	if (!qedi->cid_que.conn_cid_tbl) {
 505		kfree(qedi->cid_que.cid_que_base);
 506		qedi->cid_que.cid_que_base = NULL;
 507		return -ENOMEM;
 508	}
 509
 510	qedi->cid_que.cid_que = (u32 *)qedi->cid_que.cid_que_base;
 511	qedi->cid_que.cid_q_prod_idx = 0;
 512	qedi->cid_que.cid_q_cons_idx = 0;
 513	qedi->cid_que.cid_q_max_idx = qedi->max_active_conns;
 514	qedi->cid_que.cid_free_cnt = qedi->max_active_conns;
 515
 516	for (i = 0; i < qedi->max_active_conns; i++) {
 517		qedi->cid_que.cid_que[i] = i;
 518		qedi->cid_que.conn_cid_tbl[i] = NULL;
 519	}
 520
 521	return 0;
 522}
 523
 524static void qedi_release_cid_que(struct qedi_ctx *qedi)
 525{
 526	kfree(qedi->cid_que.cid_que_base);
 527	qedi->cid_que.cid_que_base = NULL;
 528
 529	kfree(qedi->cid_que.conn_cid_tbl);
 530	qedi->cid_que.conn_cid_tbl = NULL;
 531}
 532
 533static int qedi_init_id_tbl(struct qedi_portid_tbl *id_tbl, u16 size,
 534			    u16 start_id, u16 next)
 535{
 536	id_tbl->start = start_id;
 537	id_tbl->max = size;
 538	id_tbl->next = next;
 539	spin_lock_init(&id_tbl->lock);
 540	id_tbl->table = kcalloc(BITS_TO_LONGS(size), sizeof(long), GFP_KERNEL);
 541	if (!id_tbl->table)
 542		return -ENOMEM;
 543
 544	return 0;
 545}
 546
 547static void qedi_free_id_tbl(struct qedi_portid_tbl *id_tbl)
 548{
 549	kfree(id_tbl->table);
 550	id_tbl->table = NULL;
 551}
 552
 553int qedi_alloc_id(struct qedi_portid_tbl *id_tbl, u16 id)
 554{
 555	int ret = -1;
 556
 557	id -= id_tbl->start;
 558	if (id >= id_tbl->max)
 559		return ret;
 560
 561	spin_lock(&id_tbl->lock);
 562	if (!test_bit(id, id_tbl->table)) {
 563		set_bit(id, id_tbl->table);
 564		ret = 0;
 565	}
 566	spin_unlock(&id_tbl->lock);
 567	return ret;
 568}
 569
 570u16 qedi_alloc_new_id(struct qedi_portid_tbl *id_tbl)
 571{
 572	u16 id;
 573
 574	spin_lock(&id_tbl->lock);
 575	id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next);
 576	if (id >= id_tbl->max) {
 577		id = QEDI_LOCAL_PORT_INVALID;
 578		if (id_tbl->next != 0) {
 579			id = find_first_zero_bit(id_tbl->table, id_tbl->next);
 580			if (id >= id_tbl->next)
 581				id = QEDI_LOCAL_PORT_INVALID;
 582		}
 583	}
 584
 585	if (id < id_tbl->max) {
 586		set_bit(id, id_tbl->table);
 587		id_tbl->next = (id + 1) & (id_tbl->max - 1);
 588		id += id_tbl->start;
 589	}
 590
 591	spin_unlock(&id_tbl->lock);
 592
 593	return id;
 594}
 595
 596void qedi_free_id(struct qedi_portid_tbl *id_tbl, u16 id)
 597{
 598	if (id == QEDI_LOCAL_PORT_INVALID)
 599		return;
 600
 601	id -= id_tbl->start;
 602	if (id >= id_tbl->max)
 603		return;
 604
 605	clear_bit(id, id_tbl->table);
 606}
 607
 608static void qedi_cm_free_mem(struct qedi_ctx *qedi)
 609{
 610	kfree(qedi->ep_tbl);
 611	qedi->ep_tbl = NULL;
 612	qedi_free_id_tbl(&qedi->lcl_port_tbl);
 613}
 614
 615static int qedi_cm_alloc_mem(struct qedi_ctx *qedi)
 616{
 617	u16 port_id;
 618
 619	qedi->ep_tbl = kzalloc((qedi->max_active_conns *
 620				sizeof(struct qedi_endpoint *)), GFP_KERNEL);
 621	if (!qedi->ep_tbl)
 622		return -ENOMEM;
 623	port_id = get_random_u32_below(QEDI_LOCAL_PORT_RANGE);
 624	if (qedi_init_id_tbl(&qedi->lcl_port_tbl, QEDI_LOCAL_PORT_RANGE,
 625			     QEDI_LOCAL_PORT_MIN, port_id)) {
 626		qedi_cm_free_mem(qedi);
 627		return -ENOMEM;
 628	}
 629
 630	return 0;
 631}
 632
 633static struct qedi_ctx *qedi_host_alloc(struct pci_dev *pdev)
 634{
 635	struct Scsi_Host *shost;
 636	struct qedi_ctx *qedi = NULL;
 637
 638	shost = iscsi_host_alloc(&qedi_host_template,
 639				 sizeof(struct qedi_ctx), 0);
 640	if (!shost) {
 641		QEDI_ERR(NULL, "Could not allocate shost\n");
 642		goto exit_setup_shost;
 643	}
 644
 645	shost->max_id = QEDI_MAX_ISCSI_CONNS_PER_HBA - 1;
 646	shost->max_channel = 0;
 647	shost->max_lun = ~0;
 648	shost->max_cmd_len = 16;
 649	shost->transportt = qedi_scsi_transport;
 650
 651	qedi = iscsi_host_priv(shost);
 652	memset(qedi, 0, sizeof(*qedi));
 653	qedi->shost = shost;
 654	qedi->dbg_ctx.host_no = shost->host_no;
 655	qedi->pdev = pdev;
 656	qedi->dbg_ctx.pdev = pdev;
 657	qedi->max_active_conns = ISCSI_MAX_SESS_PER_HBA;
 658	qedi->max_sqes = QEDI_SQ_SIZE;
 659
 660	shost->nr_hw_queues = MIN_NUM_CPUS_MSIX(qedi);
 661
 662	pci_set_drvdata(pdev, qedi);
 663
 664exit_setup_shost:
 665	return qedi;
 666}
 667
 668static int qedi_ll2_rx(void *cookie, struct sk_buff *skb, u32 arg1, u32 arg2)
 669{
 670	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
 671	struct skb_work_list *work;
 672	struct ethhdr *eh;
 673
 674	if (!qedi) {
 675		QEDI_ERR(NULL, "qedi is NULL\n");
 676		return -1;
 677	}
 678
 679	if (!test_bit(UIO_DEV_OPENED, &qedi->flags)) {
 680		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UIO,
 681			  "UIO DEV is not opened\n");
 682		kfree_skb(skb);
 683		return 0;
 684	}
 685
 686	eh = (struct ethhdr *)skb->data;
 687	/* Undo VLAN encapsulation */
 688	if (eh->h_proto == htons(ETH_P_8021Q)) {
 689		memmove((u8 *)eh + VLAN_HLEN, eh, ETH_ALEN * 2);
 690		eh = (struct ethhdr *)skb_pull(skb, VLAN_HLEN);
 691		skb_reset_mac_header(skb);
 692	}
 693
 694	/* Filter out non FIP/FCoE frames here to free them faster */
 695	if (eh->h_proto != htons(ETH_P_ARP) &&
 696	    eh->h_proto != htons(ETH_P_IP) &&
 697	    eh->h_proto != htons(ETH_P_IPV6)) {
 698		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 699			  "Dropping frame ethertype [0x%x] len [0x%x].\n",
 700			  eh->h_proto, skb->len);
 701		kfree_skb(skb);
 702		return 0;
 703	}
 704
 705	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 706		  "Allowed frame ethertype [0x%x] len [0x%x].\n",
 707		  eh->h_proto, skb->len);
 708
 709	work = kzalloc(sizeof(*work), GFP_ATOMIC);
 710	if (!work) {
 711		QEDI_WARN(&qedi->dbg_ctx,
 712			  "Could not allocate work so dropping frame.\n");
 713		kfree_skb(skb);
 714		return 0;
 715	}
 716
 717	INIT_LIST_HEAD(&work->list);
 718	work->skb = skb;
 719
 720	if (skb_vlan_tag_present(skb))
 721		work->vlan_id = skb_vlan_tag_get(skb);
 722
 723	if (work->vlan_id)
 724		__vlan_insert_tag(work->skb, htons(ETH_P_8021Q), work->vlan_id);
 725
 726	spin_lock_bh(&qedi->ll2_lock);
 727	list_add_tail(&work->list, &qedi->ll2_skb_list);
 728	spin_unlock_bh(&qedi->ll2_lock);
 729
 730	wake_up_process(qedi->ll2_recv_thread);
 731
 732	return 0;
 733}
 734
 735/* map this skb to iscsiuio mmaped region */
 736static int qedi_ll2_process_skb(struct qedi_ctx *qedi, struct sk_buff *skb,
 737				u16 vlan_id)
 738{
 739	struct qedi_uio_dev *udev = NULL;
 740	struct qedi_uio_ctrl *uctrl = NULL;
 741	struct qedi_rx_bd rxbd;
 742	struct qedi_rx_bd *p_rxbd;
 743	u32 rx_bd_prod;
 744	void *pkt;
 745	int len = 0;
 746	u32 prod;
 747
 748	if (!qedi) {
 749		QEDI_ERR(NULL, "qedi is NULL\n");
 750		return -1;
 751	}
 752
 753	udev = qedi->udev;
 754	uctrl = udev->uctrl;
 755
 756	++uctrl->hw_rx_prod_cnt;
 757	prod = (uctrl->hw_rx_prod + 1) % RX_RING;
 758
 759	pkt = udev->rx_pkt + (prod * qedi_ll2_buf_size);
 760	len = min_t(u32, skb->len, (u32)qedi_ll2_buf_size);
 761	memcpy(pkt, skb->data, len);
 762
 763	memset(&rxbd, 0, sizeof(rxbd));
 764	rxbd.rx_pkt_index = prod;
 765	rxbd.rx_pkt_len = len;
 766	rxbd.vlan_id = vlan_id;
 767
 768	uctrl->hw_rx_bd_prod = (uctrl->hw_rx_bd_prod + 1) % QEDI_NUM_RX_BD;
 769	rx_bd_prod = uctrl->hw_rx_bd_prod;
 770	p_rxbd = (struct qedi_rx_bd *)udev->ll2_ring;
 771	p_rxbd += rx_bd_prod;
 772
 773	memcpy(p_rxbd, &rxbd, sizeof(rxbd));
 774
 775	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 776		  "hw_rx_prod [%d] prod [%d] hw_rx_bd_prod [%d] rx_pkt_idx [%d] rx_len [%d].\n",
 777		  uctrl->hw_rx_prod, prod, uctrl->hw_rx_bd_prod,
 778		  rxbd.rx_pkt_index, rxbd.rx_pkt_len);
 779	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 780		  "host_rx_cons [%d] hw_rx_bd_cons [%d].\n",
 781		  uctrl->host_rx_cons, uctrl->host_rx_bd_cons);
 782
 783	uctrl->hw_rx_prod = prod;
 784
 785	/* notify the iscsiuio about new packet */
 786	uio_event_notify(&udev->qedi_uinfo);
 787
 788	return 0;
 789}
 790
 791static void qedi_ll2_free_skbs(struct qedi_ctx *qedi)
 792{
 793	struct skb_work_list *work, *work_tmp;
 794
 795	spin_lock_bh(&qedi->ll2_lock);
 796	list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list, list) {
 797		list_del(&work->list);
 798		kfree_skb(work->skb);
 799		kfree(work);
 800	}
 801	spin_unlock_bh(&qedi->ll2_lock);
 802}
 803
 804static int qedi_ll2_recv_thread(void *arg)
 805{
 806	struct qedi_ctx *qedi = (struct qedi_ctx *)arg;
 807	struct skb_work_list *work, *work_tmp;
 808
 809	set_user_nice(current, -20);
 810
 811	while (!kthread_should_stop()) {
 812		spin_lock_bh(&qedi->ll2_lock);
 813		list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list,
 814					 list) {
 815			list_del(&work->list);
 816			qedi_ll2_process_skb(qedi, work->skb, work->vlan_id);
 817			kfree_skb(work->skb);
 818			kfree(work);
 819		}
 820		set_current_state(TASK_INTERRUPTIBLE);
 821		spin_unlock_bh(&qedi->ll2_lock);
 822		schedule();
 823	}
 824
 825	__set_current_state(TASK_RUNNING);
 826	return 0;
 827}
 828
 829static int qedi_set_iscsi_pf_param(struct qedi_ctx *qedi)
 830{
 831	u8 num_sq_pages;
 832	u32 log_page_size;
 833	int rval = 0;
 834
 835
 836	num_sq_pages = (MAX_OUTSTANDING_TASKS_PER_CON * 8) / QEDI_PAGE_SIZE;
 837
 838	qedi->num_queues = MIN_NUM_CPUS_MSIX(qedi);
 839
 840	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
 841		  "Number of CQ count is %d\n", qedi->num_queues);
 842
 843	memset(&qedi->pf_params.iscsi_pf_params, 0,
 844	       sizeof(qedi->pf_params.iscsi_pf_params));
 845
 846	qedi->p_cpuq = dma_alloc_coherent(&qedi->pdev->dev,
 847			qedi->num_queues * sizeof(struct qedi_glbl_q_params),
 848			&qedi->hw_p_cpuq, GFP_KERNEL);
 849	if (!qedi->p_cpuq) {
 850		QEDI_ERR(&qedi->dbg_ctx, "dma_alloc_coherent fail\n");
 851		rval = -1;
 852		goto err_alloc_mem;
 853	}
 854
 855	rval = qedi_alloc_global_queues(qedi);
 856	if (rval) {
 857		QEDI_ERR(&qedi->dbg_ctx, "Global queue allocation failed.\n");
 858		rval = -1;
 859		goto err_alloc_mem;
 860	}
 861
 862	qedi->pf_params.iscsi_pf_params.num_cons = QEDI_MAX_ISCSI_CONNS_PER_HBA;
 863	qedi->pf_params.iscsi_pf_params.num_tasks = QEDI_MAX_ISCSI_TASK;
 864	qedi->pf_params.iscsi_pf_params.half_way_close_timeout = 10;
 865	qedi->pf_params.iscsi_pf_params.num_sq_pages_in_ring = num_sq_pages;
 866	qedi->pf_params.iscsi_pf_params.num_r2tq_pages_in_ring = num_sq_pages;
 867	qedi->pf_params.iscsi_pf_params.num_uhq_pages_in_ring = num_sq_pages;
 868	qedi->pf_params.iscsi_pf_params.num_queues = qedi->num_queues;
 869	qedi->pf_params.iscsi_pf_params.debug_mode = qedi_fw_debug;
 870	qedi->pf_params.iscsi_pf_params.two_msl_timer = QED_TWO_MSL_TIMER_DFLT;
 871	qedi->pf_params.iscsi_pf_params.tx_sws_timer = QED_TX_SWS_TIMER_DFLT;
 872	qedi->pf_params.iscsi_pf_params.max_fin_rt = 2;
 873
 874	for (log_page_size = 0 ; log_page_size < 32 ; log_page_size++) {
 875		if ((1 << log_page_size) == QEDI_PAGE_SIZE)
 876			break;
 877	}
 878	qedi->pf_params.iscsi_pf_params.log_page_size = log_page_size;
 879
 880	qedi->pf_params.iscsi_pf_params.glbl_q_params_addr =
 881							   (u64)qedi->hw_p_cpuq;
 882
 883	/* RQ BDQ initializations.
 884	 * rq_num_entries: suggested value for Initiator is 16 (4KB RQ)
 885	 * rqe_log_size: 8 for 256B RQE
 886	 */
 887	qedi->pf_params.iscsi_pf_params.rqe_log_size = 8;
 888	/* BDQ address and size */
 889	qedi->pf_params.iscsi_pf_params.bdq_pbl_base_addr[BDQ_ID_RQ] =
 890							qedi->bdq_pbl_list_dma;
 891	qedi->pf_params.iscsi_pf_params.bdq_pbl_num_entries[BDQ_ID_RQ] =
 892						qedi->bdq_pbl_list_num_entries;
 893	qedi->pf_params.iscsi_pf_params.rq_buffer_size = QEDI_BDQ_BUF_SIZE;
 894
 895	/* cq_num_entries: num_tasks + rq_num_entries */
 896	qedi->pf_params.iscsi_pf_params.cq_num_entries = 2048;
 897
 898	qedi->pf_params.iscsi_pf_params.gl_rq_pi = QEDI_PROTO_CQ_PROD_IDX;
 899	qedi->pf_params.iscsi_pf_params.gl_cmd_pi = 1;
 900
 901err_alloc_mem:
 902	return rval;
 903}
 904
 905/* Free DMA coherent memory for array of queue pointers we pass to qed */
 906static void qedi_free_iscsi_pf_param(struct qedi_ctx *qedi)
 907{
 908	size_t size = 0;
 909
 910	if (qedi->p_cpuq) {
 911		size = qedi->num_queues * sizeof(struct qedi_glbl_q_params);
 912		dma_free_coherent(&qedi->pdev->dev, size, qedi->p_cpuq,
 913				    qedi->hw_p_cpuq);
 914	}
 915
 916	qedi_free_global_queues(qedi);
 917
 918	kfree(qedi->global_queues);
 919}
 920
 921static void qedi_get_boot_tgt_info(struct nvm_iscsi_block *block,
 922				   struct qedi_boot_target *tgt, u8 index)
 923{
 924	u32 ipv6_en;
 925
 926	ipv6_en = !!(block->generic.ctrl_flags &
 927		     NVM_ISCSI_CFG_GEN_IPV6_ENABLED);
 928
 929	snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s",
 930		 block->target[index].target_name.byte);
 931
 932	tgt->ipv6_en = ipv6_en;
 933
 934	if (ipv6_en)
 935		snprintf(tgt->ip_addr, IPV6_LEN, "%pI6\n",
 936			 block->target[index].ipv6_addr.byte);
 937	else
 938		snprintf(tgt->ip_addr, IPV4_LEN, "%pI4\n",
 939			 block->target[index].ipv4_addr.byte);
 940}
 941
 942static int qedi_find_boot_info(struct qedi_ctx *qedi,
 943			       struct qed_mfw_tlv_iscsi *iscsi,
 944			       struct nvm_iscsi_block *block)
 945{
 946	struct qedi_boot_target *pri_tgt = NULL, *sec_tgt = NULL;
 947	u32 pri_ctrl_flags = 0, sec_ctrl_flags = 0, found = 0;
 948	struct iscsi_cls_session *cls_sess;
 949	struct iscsi_cls_conn *cls_conn;
 950	struct qedi_conn *qedi_conn;
 951	struct iscsi_session *sess;
 952	struct iscsi_conn *conn;
 953	char ep_ip_addr[64];
 954	int i, ret = 0;
 955
 956	pri_ctrl_flags = !!(block->target[0].ctrl_flags &
 957					NVM_ISCSI_CFG_TARGET_ENABLED);
 958	if (pri_ctrl_flags) {
 959		pri_tgt = kzalloc(sizeof(*pri_tgt), GFP_KERNEL);
 960		if (!pri_tgt)
 961			return -1;
 962		qedi_get_boot_tgt_info(block, pri_tgt, 0);
 963	}
 964
 965	sec_ctrl_flags = !!(block->target[1].ctrl_flags &
 966					NVM_ISCSI_CFG_TARGET_ENABLED);
 967	if (sec_ctrl_flags) {
 968		sec_tgt = kzalloc(sizeof(*sec_tgt), GFP_KERNEL);
 969		if (!sec_tgt) {
 970			ret = -1;
 971			goto free_tgt;
 972		}
 973		qedi_get_boot_tgt_info(block, sec_tgt, 1);
 974	}
 975
 976	for (i = 0; i < qedi->max_active_conns; i++) {
 977		qedi_conn = qedi_get_conn_from_id(qedi, i);
 978		if (!qedi_conn)
 979			continue;
 980
 981		if (qedi_conn->ep->ip_type == TCP_IPV4)
 982			snprintf(ep_ip_addr, IPV4_LEN, "%pI4\n",
 983				 qedi_conn->ep->dst_addr);
 984		else
 985			snprintf(ep_ip_addr, IPV6_LEN, "%pI6\n",
 986				 qedi_conn->ep->dst_addr);
 987
 988		cls_conn = qedi_conn->cls_conn;
 989		conn = cls_conn->dd_data;
 990		cls_sess = iscsi_conn_to_session(cls_conn);
 991		sess = cls_sess->dd_data;
 992
 993		if (!iscsi_is_session_online(cls_sess))
 994			continue;
 995
 996		if (!sess->targetname)
 997			continue;
 998
 999		if (pri_ctrl_flags) {
1000			if (!strcmp(pri_tgt->iscsi_name, sess->targetname) &&
1001			    !strcmp(pri_tgt->ip_addr, ep_ip_addr)) {
1002				found = 1;
1003				break;
1004			}
1005		}
1006
1007		if (sec_ctrl_flags) {
1008			if (!strcmp(sec_tgt->iscsi_name, sess->targetname) &&
1009			    !strcmp(sec_tgt->ip_addr, ep_ip_addr)) {
1010				found = 1;
1011				break;
1012			}
1013		}
1014	}
1015
1016	if (found) {
1017		if (conn->hdrdgst_en) {
1018			iscsi->header_digest_set = true;
1019			iscsi->header_digest = 1;
1020		}
1021
1022		if (conn->datadgst_en) {
1023			iscsi->data_digest_set = true;
1024			iscsi->data_digest = 1;
1025		}
1026		iscsi->boot_taget_portal_set = true;
1027		iscsi->boot_taget_portal = sess->tpgt;
1028
1029	} else {
1030		ret = -1;
1031	}
1032
1033	if (sec_ctrl_flags)
1034		kfree(sec_tgt);
1035free_tgt:
1036	if (pri_ctrl_flags)
1037		kfree(pri_tgt);
1038
1039	return ret;
1040}
1041
1042static void qedi_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
1043{
1044	struct qedi_ctx *qedi;
1045
1046	if (!dev) {
1047		QEDI_INFO(NULL, QEDI_LOG_EVT,
1048			  "dev is NULL so ignoring get_generic_tlv_data request.\n");
1049		return;
1050	}
1051	qedi = (struct qedi_ctx *)dev;
1052
1053	memset(data, 0, sizeof(struct qed_generic_tlvs));
1054	ether_addr_copy(data->mac[0], qedi->mac);
1055}
1056
1057/*
1058 * Protocol TLV handler
1059 */
1060static void qedi_get_protocol_tlv_data(void *dev, void *data)
1061{
1062	struct qed_mfw_tlv_iscsi *iscsi = data;
1063	struct qed_iscsi_stats *fw_iscsi_stats;
1064	struct nvm_iscsi_block *block = NULL;
1065	u32 chap_en = 0, mchap_en = 0;
1066	struct qedi_ctx *qedi = dev;
1067	int rval = 0;
1068
1069	fw_iscsi_stats = kmalloc(sizeof(*fw_iscsi_stats), GFP_KERNEL);
1070	if (!fw_iscsi_stats) {
1071		QEDI_ERR(&qedi->dbg_ctx,
1072			 "Could not allocate memory for fw_iscsi_stats.\n");
1073		goto exit_get_data;
1074	}
1075
1076	mutex_lock(&qedi->stats_lock);
1077	/* Query firmware for offload stats */
1078	qedi_ops->get_stats(qedi->cdev, fw_iscsi_stats);
1079	mutex_unlock(&qedi->stats_lock);
1080
1081	iscsi->rx_frames_set = true;
1082	iscsi->rx_frames = fw_iscsi_stats->iscsi_rx_packet_cnt;
1083	iscsi->rx_bytes_set = true;
1084	iscsi->rx_bytes = fw_iscsi_stats->iscsi_rx_bytes_cnt;
1085	iscsi->tx_frames_set = true;
1086	iscsi->tx_frames = fw_iscsi_stats->iscsi_tx_packet_cnt;
1087	iscsi->tx_bytes_set = true;
1088	iscsi->tx_bytes = fw_iscsi_stats->iscsi_tx_bytes_cnt;
1089	iscsi->frame_size_set = true;
1090	iscsi->frame_size = qedi->ll2_mtu;
1091	block = qedi_get_nvram_block(qedi);
1092	if (block) {
1093		chap_en = !!(block->generic.ctrl_flags &
1094			     NVM_ISCSI_CFG_GEN_CHAP_ENABLED);
1095		mchap_en = !!(block->generic.ctrl_flags &
1096			      NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED);
1097
1098		iscsi->auth_method_set = (chap_en || mchap_en) ? true : false;
1099		iscsi->auth_method = 1;
1100		if (chap_en)
1101			iscsi->auth_method = 2;
1102		if (mchap_en)
1103			iscsi->auth_method = 3;
1104
1105		iscsi->tx_desc_size_set = true;
1106		iscsi->tx_desc_size = QEDI_SQ_SIZE;
1107		iscsi->rx_desc_size_set = true;
1108		iscsi->rx_desc_size = QEDI_CQ_SIZE;
1109
1110		/* tpgt, hdr digest, data digest */
1111		rval = qedi_find_boot_info(qedi, iscsi, block);
1112		if (rval)
1113			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1114				  "Boot target not set");
1115	}
1116
1117	kfree(fw_iscsi_stats);
1118exit_get_data:
1119	return;
1120}
1121
1122void qedi_schedule_hw_err_handler(void *dev,
1123				  enum qed_hw_err_type err_type)
1124{
1125	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
1126	unsigned long override_flags = qedi_flags_override;
1127
1128	if (override_flags && test_bit(QEDI_ERR_OVERRIDE_EN, &override_flags))
1129		qedi->qedi_err_flags = qedi_flags_override;
1130
1131	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1132		  "HW error handler scheduled, err=%d err_flags=0x%x\n",
1133		  err_type, qedi->qedi_err_flags);
1134
1135	switch (err_type) {
1136	case QED_HW_ERR_FAN_FAIL:
1137		schedule_delayed_work(&qedi->board_disable_work, 0);
1138		break;
1139	case QED_HW_ERR_MFW_RESP_FAIL:
1140	case QED_HW_ERR_HW_ATTN:
1141	case QED_HW_ERR_DMAE_FAIL:
1142	case QED_HW_ERR_RAMROD_FAIL:
1143	case QED_HW_ERR_FW_ASSERT:
1144		/* Prevent HW attentions from being reasserted */
1145		if (test_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags))
1146			qedi_ops->common->attn_clr_enable(qedi->cdev, true);
1147
1148		if (err_type == QED_HW_ERR_RAMROD_FAIL &&
1149		    test_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags))
1150			qedi_ops->common->recovery_process(qedi->cdev);
1151
1152		break;
1153	default:
1154		break;
1155	}
1156}
1157
1158static void qedi_schedule_recovery_handler(void *dev)
1159{
1160	struct qedi_ctx *qedi = dev;
1161
1162	QEDI_ERR(&qedi->dbg_ctx, "Recovery handler scheduled.\n");
1163
1164	if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags))
1165		return;
1166
1167	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
1168
1169	schedule_delayed_work(&qedi->recovery_work, 0);
1170}
1171
1172static void qedi_set_conn_recovery(struct iscsi_cls_session *cls_session)
1173{
1174	struct iscsi_session *session = cls_session->dd_data;
1175	struct iscsi_conn *conn = session->leadconn;
1176	struct qedi_conn *qedi_conn = conn->dd_data;
1177
1178	qedi_start_conn_recovery(qedi_conn->qedi, qedi_conn);
1179}
1180
1181static void qedi_link_update(void *dev, struct qed_link_output *link)
1182{
1183	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
1184
1185	if (link->link_up) {
1186		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "Link Up event.\n");
1187		atomic_set(&qedi->link_state, QEDI_LINK_UP);
1188	} else {
1189		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1190			  "Link Down event.\n");
1191		atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
1192		iscsi_host_for_each_session(qedi->shost, qedi_set_conn_recovery);
1193	}
1194}
1195
1196static struct qed_iscsi_cb_ops qedi_cb_ops = {
1197	{
1198		.link_update =		qedi_link_update,
1199		.schedule_recovery_handler = qedi_schedule_recovery_handler,
1200		.schedule_hw_err_handler = qedi_schedule_hw_err_handler,
1201		.get_protocol_tlv_data = qedi_get_protocol_tlv_data,
1202		.get_generic_tlv_data = qedi_get_generic_tlv_data,
1203	}
1204};
1205
1206static int qedi_queue_cqe(struct qedi_ctx *qedi, union iscsi_cqe *cqe,
1207			  u16 que_idx, struct qedi_percpu_s *p)
1208{
1209	struct qedi_work *qedi_work;
1210	struct qedi_conn *q_conn;
1211	struct qedi_cmd *qedi_cmd;
1212	u32 iscsi_cid;
1213	int rc = 0;
1214
1215	iscsi_cid  = cqe->cqe_common.conn_id;
1216	q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
1217	if (!q_conn) {
1218		QEDI_WARN(&qedi->dbg_ctx,
1219			  "Session no longer exists for cid=0x%x!!\n",
1220			  iscsi_cid);
1221		return -1;
1222	}
1223
1224	switch (cqe->cqe_common.cqe_type) {
1225	case ISCSI_CQE_TYPE_SOLICITED:
1226	case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE:
1227		qedi_cmd = qedi_get_cmd_from_tid(qedi, cqe->cqe_solicited.itid);
1228		if (!qedi_cmd) {
1229			rc = -1;
1230			break;
1231		}
1232		INIT_LIST_HEAD(&qedi_cmd->cqe_work.list);
1233		qedi_cmd->cqe_work.qedi = qedi;
1234		memcpy(&qedi_cmd->cqe_work.cqe, cqe, sizeof(union iscsi_cqe));
1235		qedi_cmd->cqe_work.que_idx = que_idx;
1236		qedi_cmd->cqe_work.is_solicited = true;
1237		list_add_tail(&qedi_cmd->cqe_work.list, &p->work_list);
1238		break;
1239	case ISCSI_CQE_TYPE_UNSOLICITED:
1240	case ISCSI_CQE_TYPE_DUMMY:
1241	case ISCSI_CQE_TYPE_TASK_CLEANUP:
1242		qedi_work = kzalloc(sizeof(*qedi_work), GFP_ATOMIC);
1243		if (!qedi_work) {
1244			rc = -1;
1245			break;
1246		}
1247		INIT_LIST_HEAD(&qedi_work->list);
1248		qedi_work->qedi = qedi;
1249		memcpy(&qedi_work->cqe, cqe, sizeof(union iscsi_cqe));
1250		qedi_work->que_idx = que_idx;
1251		qedi_work->is_solicited = false;
1252		list_add_tail(&qedi_work->list, &p->work_list);
1253		break;
1254	default:
1255		rc = -1;
1256		QEDI_ERR(&qedi->dbg_ctx, "FW Error cqe.\n");
1257	}
1258	return rc;
1259}
1260
1261static bool qedi_process_completions(struct qedi_fastpath *fp)
1262{
1263	struct qedi_ctx *qedi = fp->qedi;
1264	struct qed_sb_info *sb_info = fp->sb_info;
1265	struct status_block *sb = sb_info->sb_virt;
1266	struct qedi_percpu_s *p = NULL;
1267	struct global_queue *que;
1268	u16 prod_idx;
1269	unsigned long flags;
1270	union iscsi_cqe *cqe;
1271	int cpu;
1272	int ret;
1273
1274	/* Get the current firmware producer index */
1275	prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];
1276
1277	if (prod_idx >= QEDI_CQ_SIZE)
1278		prod_idx = prod_idx % QEDI_CQ_SIZE;
1279
1280	que = qedi->global_queues[fp->sb_id];
1281	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1282		  "Before: global queue=%p prod_idx=%d cons_idx=%d, sb_id=%d\n",
1283		  que, prod_idx, que->cq_cons_idx, fp->sb_id);
1284
1285	qedi->intr_cpu = fp->sb_id;
1286	cpu = smp_processor_id();
1287	p = &per_cpu(qedi_percpu, cpu);
1288
1289	if (unlikely(!p->iothread))
1290		WARN_ON(1);
1291
1292	spin_lock_irqsave(&p->p_work_lock, flags);
1293	while (que->cq_cons_idx != prod_idx) {
1294		cqe = &que->cq[que->cq_cons_idx];
1295
1296		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1297			  "cqe=%p prod_idx=%d cons_idx=%d.\n",
1298			  cqe, prod_idx, que->cq_cons_idx);
1299
1300		ret = qedi_queue_cqe(qedi, cqe, fp->sb_id, p);
1301		if (ret)
1302			QEDI_WARN(&qedi->dbg_ctx,
1303				  "Dropping CQE 0x%x for cid=0x%x.\n",
1304				  que->cq_cons_idx, cqe->cqe_common.conn_id);
1305
1306		que->cq_cons_idx++;
1307		if (que->cq_cons_idx == QEDI_CQ_SIZE)
1308			que->cq_cons_idx = 0;
1309	}
1310	wake_up_process(p->iothread);
1311	spin_unlock_irqrestore(&p->p_work_lock, flags);
1312
1313	return true;
1314}
1315
1316static bool qedi_fp_has_work(struct qedi_fastpath *fp)
1317{
1318	struct qedi_ctx *qedi = fp->qedi;
1319	struct global_queue *que;
1320	struct qed_sb_info *sb_info = fp->sb_info;
1321	struct status_block *sb = sb_info->sb_virt;
1322	u16 prod_idx;
1323
1324	barrier();
1325
1326	/* Get the current firmware producer index */
1327	prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];
1328
1329	/* Get the pointer to the global CQ this completion is on */
1330	que = qedi->global_queues[fp->sb_id];
1331
1332	/* prod idx wrap around uint16 */
1333	if (prod_idx >= QEDI_CQ_SIZE)
1334		prod_idx = prod_idx % QEDI_CQ_SIZE;
1335
1336	return (que->cq_cons_idx != prod_idx);
1337}
1338
1339/* MSI-X fastpath handler code */
1340static irqreturn_t qedi_msix_handler(int irq, void *dev_id)
1341{
1342	struct qedi_fastpath *fp = dev_id;
1343	struct qedi_ctx *qedi = fp->qedi;
1344	bool wake_io_thread = true;
1345
1346	qed_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
1347
1348process_again:
1349	wake_io_thread = qedi_process_completions(fp);
1350	if (wake_io_thread) {
1351		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1352			  "process already running\n");
1353	}
1354
1355	if (!qedi_fp_has_work(fp))
1356		qed_sb_update_sb_idx(fp->sb_info);
1357
1358	/* Check for more work */
1359	rmb();
1360
1361	if (!qedi_fp_has_work(fp))
1362		qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
1363	else
1364		goto process_again;
1365
1366	return IRQ_HANDLED;
1367}
1368
1369/* simd handler for MSI/INTa */
1370static void qedi_simd_int_handler(void *cookie)
1371{
1372	/* Cookie is qedi_ctx struct */
1373	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
1374
1375	QEDI_WARN(&qedi->dbg_ctx, "qedi=%p.\n", qedi);
1376}
1377
1378#define QEDI_SIMD_HANDLER_NUM		0
1379static void qedi_sync_free_irqs(struct qedi_ctx *qedi)
1380{
1381	int i;
1382	u16 idx;
1383
1384	if (qedi->int_info.msix_cnt) {
1385		for (i = 0; i < qedi->int_info.used_cnt; i++) {
1386			idx = i * qedi->dev_info.common.num_hwfns +
1387			qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
1388
1389			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1390				  "Freeing IRQ #%d vector_idx=%d.\n", i, idx);
1391
1392			synchronize_irq(qedi->int_info.msix[idx].vector);
1393			irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1394					      NULL);
1395			free_irq(qedi->int_info.msix[idx].vector,
1396				 &qedi->fp_array[i]);
1397		}
1398	} else {
1399		qedi_ops->common->simd_handler_clean(qedi->cdev,
1400						     QEDI_SIMD_HANDLER_NUM);
1401	}
1402
1403	qedi->int_info.used_cnt = 0;
1404	qedi_ops->common->set_fp_int(qedi->cdev, 0);
1405}
1406
1407static int qedi_request_msix_irq(struct qedi_ctx *qedi)
1408{
1409	int i, rc, cpu;
1410	u16 idx;
1411
1412	cpu = cpumask_first(cpu_online_mask);
1413	for (i = 0; i < qedi->msix_count; i++) {
1414		idx = i * qedi->dev_info.common.num_hwfns +
1415			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
1416
1417		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1418			  "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
1419			  qedi->dev_info.common.num_hwfns,
1420			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
1421
1422		rc = request_irq(qedi->int_info.msix[idx].vector,
1423				 qedi_msix_handler, 0, "qedi",
1424				 &qedi->fp_array[i]);
1425		if (rc) {
1426			QEDI_WARN(&qedi->dbg_ctx, "request_irq failed.\n");
1427			qedi_sync_free_irqs(qedi);
1428			return rc;
1429		}
1430		qedi->int_info.used_cnt++;
1431		rc = irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1432					   get_cpu_mask(cpu));
1433		cpu = cpumask_next(cpu, cpu_online_mask);
1434	}
1435
1436	return 0;
1437}
1438
1439static int qedi_setup_int(struct qedi_ctx *qedi)
1440{
1441	int rc = 0;
1442
1443	rc = qedi_ops->common->set_fp_int(qedi->cdev, qedi->num_queues);
1444	if (rc < 0)
1445		goto exit_setup_int;
1446
1447	qedi->msix_count = rc;
1448
1449	rc = qedi_ops->common->get_fp_int(qedi->cdev, &qedi->int_info);
1450	if (rc)
1451		goto exit_setup_int;
1452
1453	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1454		  "Number of msix_cnt = 0x%x num of cpus = 0x%x\n",
1455		   qedi->int_info.msix_cnt, num_online_cpus());
1456
1457	if (qedi->int_info.msix_cnt) {
1458		rc = qedi_request_msix_irq(qedi);
1459		goto exit_setup_int;
1460	} else {
1461		qedi_ops->common->simd_handler_config(qedi->cdev, &qedi,
1462						      QEDI_SIMD_HANDLER_NUM,
1463						      qedi_simd_int_handler);
1464		qedi->int_info.used_cnt = 1;
1465	}
1466
1467exit_setup_int:
1468	return rc;
1469}
1470
1471static void qedi_free_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1472{
1473	if (qedi->iscsi_image)
1474		dma_free_coherent(&qedi->pdev->dev,
1475				  sizeof(struct qedi_nvm_iscsi_image),
1476				  qedi->iscsi_image, qedi->nvm_buf_dma);
1477}
1478
1479static int qedi_alloc_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1480{
1481	qedi->iscsi_image = dma_alloc_coherent(&qedi->pdev->dev,
1482					       sizeof(struct qedi_nvm_iscsi_image),
1483					       &qedi->nvm_buf_dma, GFP_KERNEL);
1484	if (!qedi->iscsi_image) {
1485		QEDI_ERR(&qedi->dbg_ctx, "Could not allocate NVM BUF.\n");
1486		return -ENOMEM;
1487	}
1488	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1489		  "NVM BUF addr=0x%p dma=0x%llx.\n", qedi->iscsi_image,
1490		  qedi->nvm_buf_dma);
1491
1492	return 0;
1493}
1494
1495static void qedi_free_bdq(struct qedi_ctx *qedi)
1496{
1497	int i;
1498
1499	if (qedi->bdq_pbl_list)
1500		dma_free_coherent(&qedi->pdev->dev, QEDI_PAGE_SIZE,
1501				  qedi->bdq_pbl_list, qedi->bdq_pbl_list_dma);
1502
1503	if (qedi->bdq_pbl)
1504		dma_free_coherent(&qedi->pdev->dev, qedi->bdq_pbl_mem_size,
1505				  qedi->bdq_pbl, qedi->bdq_pbl_dma);
1506
1507	for (i = 0; i < QEDI_BDQ_NUM; i++) {
1508		if (qedi->bdq[i].buf_addr) {
1509			dma_free_coherent(&qedi->pdev->dev, QEDI_BDQ_BUF_SIZE,
1510					  qedi->bdq[i].buf_addr,
1511					  qedi->bdq[i].buf_dma);
1512		}
1513	}
1514}
1515
1516static void qedi_free_global_queues(struct qedi_ctx *qedi)
1517{
1518	int i;
1519	struct global_queue **gl = qedi->global_queues;
1520
1521	for (i = 0; i < qedi->num_queues; i++) {
1522		if (!gl[i])
1523			continue;
1524
1525		if (gl[i]->cq)
1526			dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_mem_size,
1527					  gl[i]->cq, gl[i]->cq_dma);
1528		if (gl[i]->cq_pbl)
1529			dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_pbl_size,
1530					  gl[i]->cq_pbl, gl[i]->cq_pbl_dma);
1531
1532		kfree(gl[i]);
1533	}
1534	qedi_free_bdq(qedi);
1535	qedi_free_nvm_iscsi_cfg(qedi);
1536}
1537
1538static int qedi_alloc_bdq(struct qedi_ctx *qedi)
1539{
1540	int i;
1541	struct scsi_bd *pbl;
1542	u64 *list;
1543
1544	/* Alloc dma memory for BDQ buffers */
1545	for (i = 0; i < QEDI_BDQ_NUM; i++) {
1546		qedi->bdq[i].buf_addr =
1547				dma_alloc_coherent(&qedi->pdev->dev,
1548						   QEDI_BDQ_BUF_SIZE,
1549						   &qedi->bdq[i].buf_dma,
1550						   GFP_KERNEL);
1551		if (!qedi->bdq[i].buf_addr) {
1552			QEDI_ERR(&qedi->dbg_ctx,
1553				 "Could not allocate BDQ buffer %d.\n", i);
1554			return -ENOMEM;
1555		}
1556	}
1557
1558	/* Alloc dma memory for BDQ page buffer list */
1559	qedi->bdq_pbl_mem_size = QEDI_BDQ_NUM * sizeof(struct scsi_bd);
1560	qedi->bdq_pbl_mem_size = ALIGN(qedi->bdq_pbl_mem_size, QEDI_PAGE_SIZE);
1561	qedi->rq_num_entries = qedi->bdq_pbl_mem_size / sizeof(struct scsi_bd);
1562
1563	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, "rq_num_entries = %d.\n",
1564		  qedi->rq_num_entries);
1565
1566	qedi->bdq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
1567					   qedi->bdq_pbl_mem_size,
1568					   &qedi->bdq_pbl_dma, GFP_KERNEL);
1569	if (!qedi->bdq_pbl) {
1570		QEDI_ERR(&qedi->dbg_ctx, "Could not allocate BDQ PBL.\n");
1571		return -ENOMEM;
1572	}
1573
1574	/*
1575	 * Populate BDQ PBL with physical and virtual address of individual
1576	 * BDQ buffers
1577	 */
1578	pbl = (struct scsi_bd  *)qedi->bdq_pbl;
1579	for (i = 0; i < QEDI_BDQ_NUM; i++) {
1580		pbl->address.hi =
1581				cpu_to_le32(QEDI_U64_HI(qedi->bdq[i].buf_dma));
1582		pbl->address.lo =
1583				cpu_to_le32(QEDI_U64_LO(qedi->bdq[i].buf_dma));
1584		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1585			  "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx], idx [%d]\n",
1586			  pbl, pbl->address.hi, pbl->address.lo, i);
1587		pbl->opaque.iscsi_opaque.reserved_zero[0] = 0;
1588		pbl->opaque.iscsi_opaque.reserved_zero[1] = 0;
1589		pbl->opaque.iscsi_opaque.reserved_zero[2] = 0;
1590		pbl->opaque.iscsi_opaque.opaque = cpu_to_le16(i);
1591		pbl++;
1592	}
1593
1594	/* Allocate list of PBL pages */
1595	qedi->bdq_pbl_list = dma_alloc_coherent(&qedi->pdev->dev,
1596						QEDI_PAGE_SIZE,
1597						&qedi->bdq_pbl_list_dma,
1598						GFP_KERNEL);
1599	if (!qedi->bdq_pbl_list) {
1600		QEDI_ERR(&qedi->dbg_ctx,
1601			 "Could not allocate list of PBL pages.\n");
1602		return -ENOMEM;
1603	}
1604
1605	/*
1606	 * Now populate PBL list with pages that contain pointers to the
1607	 * individual buffers.
1608	 */
1609	qedi->bdq_pbl_list_num_entries = qedi->bdq_pbl_mem_size /
1610					 QEDI_PAGE_SIZE;
1611	list = (u64 *)qedi->bdq_pbl_list;
1612	for (i = 0; i < qedi->bdq_pbl_list_num_entries; i++) {
1613		*list = qedi->bdq_pbl_dma;
1614		list++;
1615	}
1616
1617	return 0;
1618}
1619
1620static int qedi_alloc_global_queues(struct qedi_ctx *qedi)
1621{
1622	u32 *list;
1623	int i;
1624	int status;
1625	u32 *pbl;
1626	dma_addr_t page;
1627	int num_pages;
1628
1629	/*
1630	 * Number of global queues (CQ / RQ). This should
1631	 * be <= number of available MSIX vectors for the PF
1632	 */
1633	if (!qedi->num_queues) {
1634		QEDI_ERR(&qedi->dbg_ctx, "No MSI-X vectors available!\n");
1635		return -ENOMEM;
1636	}
1637
1638	/* Make sure we allocated the PBL that will contain the physical
1639	 * addresses of our queues
1640	 */
1641	if (!qedi->p_cpuq) {
1642		status = -EINVAL;
1643		goto mem_alloc_failure;
1644	}
1645
1646	qedi->global_queues = kzalloc((sizeof(struct global_queue *) *
1647				       qedi->num_queues), GFP_KERNEL);
1648	if (!qedi->global_queues) {
1649		QEDI_ERR(&qedi->dbg_ctx,
1650			 "Unable to allocate global queues array ptr memory\n");
1651		return -ENOMEM;
1652	}
1653	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1654		  "qedi->global_queues=%p.\n", qedi->global_queues);
1655
1656	/* Allocate DMA coherent buffers for BDQ */
1657	status = qedi_alloc_bdq(qedi);
1658	if (status)
1659		goto mem_alloc_failure;
1660
1661	/* Allocate DMA coherent buffers for NVM_ISCSI_CFG */
1662	status = qedi_alloc_nvm_iscsi_cfg(qedi);
1663	if (status)
1664		goto mem_alloc_failure;
1665
1666	/* Allocate a CQ and an associated PBL for each MSI-X
1667	 * vector.
1668	 */
1669	for (i = 0; i < qedi->num_queues; i++) {
1670		qedi->global_queues[i] =
1671					kzalloc(sizeof(*qedi->global_queues[0]),
1672						GFP_KERNEL);
1673		if (!qedi->global_queues[i]) {
1674			QEDI_ERR(&qedi->dbg_ctx,
1675				 "Unable to allocation global queue %d.\n", i);
1676			status = -ENOMEM;
1677			goto mem_alloc_failure;
1678		}
1679
1680		qedi->global_queues[i]->cq_mem_size =
1681		    (QEDI_CQ_SIZE + 8) * sizeof(union iscsi_cqe);
1682		qedi->global_queues[i]->cq_mem_size =
1683		    (qedi->global_queues[i]->cq_mem_size +
1684		    (QEDI_PAGE_SIZE - 1));
1685
1686		qedi->global_queues[i]->cq_pbl_size =
1687		    (qedi->global_queues[i]->cq_mem_size /
1688		    QEDI_PAGE_SIZE) * sizeof(void *);
1689		qedi->global_queues[i]->cq_pbl_size =
1690		    (qedi->global_queues[i]->cq_pbl_size +
1691		    (QEDI_PAGE_SIZE - 1));
1692
1693		qedi->global_queues[i]->cq = dma_alloc_coherent(&qedi->pdev->dev,
1694								qedi->global_queues[i]->cq_mem_size,
1695								&qedi->global_queues[i]->cq_dma,
1696								GFP_KERNEL);
1697
1698		if (!qedi->global_queues[i]->cq) {
1699			QEDI_WARN(&qedi->dbg_ctx,
1700				  "Could not allocate cq.\n");
1701			status = -ENOMEM;
1702			goto mem_alloc_failure;
1703		}
1704		qedi->global_queues[i]->cq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
1705								    qedi->global_queues[i]->cq_pbl_size,
1706								    &qedi->global_queues[i]->cq_pbl_dma,
1707								    GFP_KERNEL);
1708
1709		if (!qedi->global_queues[i]->cq_pbl) {
1710			QEDI_WARN(&qedi->dbg_ctx,
1711				  "Could not allocate cq PBL.\n");
1712			status = -ENOMEM;
1713			goto mem_alloc_failure;
1714		}
1715
1716		/* Create PBL */
1717		num_pages = qedi->global_queues[i]->cq_mem_size /
1718		    QEDI_PAGE_SIZE;
1719		page = qedi->global_queues[i]->cq_dma;
1720		pbl = (u32 *)qedi->global_queues[i]->cq_pbl;
1721
1722		while (num_pages--) {
1723			*pbl = (u32)page;
1724			pbl++;
1725			*pbl = (u32)((u64)page >> 32);
1726			pbl++;
1727			page += QEDI_PAGE_SIZE;
1728		}
1729	}
1730
1731	list = (u32 *)qedi->p_cpuq;
1732
1733	/*
1734	 * The list is built as follows: CQ#0 PBL pointer, RQ#0 PBL pointer,
1735	 * CQ#1 PBL pointer, RQ#1 PBL pointer, etc.  Each PBL pointer points
1736	 * to the physical address which contains an array of pointers to the
1737	 * physical addresses of the specific queue pages.
1738	 */
1739	for (i = 0; i < qedi->num_queues; i++) {
1740		*list = (u32)qedi->global_queues[i]->cq_pbl_dma;
1741		list++;
1742		*list = (u32)((u64)qedi->global_queues[i]->cq_pbl_dma >> 32);
1743		list++;
1744
1745		*list = (u32)0;
1746		list++;
1747		*list = (u32)((u64)0 >> 32);
1748		list++;
1749	}
1750
1751	return 0;
1752
1753mem_alloc_failure:
1754	qedi_free_global_queues(qedi);
1755	return status;
1756}
1757
1758int qedi_alloc_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)
1759{
1760	int rval = 0;
1761	u32 *pbl;
1762	dma_addr_t page;
1763	int num_pages;
1764
1765	if (!ep)
1766		return -EIO;
1767
1768	/* Calculate appropriate queue and PBL sizes */
1769	ep->sq_mem_size = QEDI_SQ_SIZE * sizeof(struct iscsi_wqe);
1770	ep->sq_mem_size += QEDI_PAGE_SIZE - 1;
1771
1772	ep->sq_pbl_size = (ep->sq_mem_size / QEDI_PAGE_SIZE) * sizeof(void *);
1773	ep->sq_pbl_size = ep->sq_pbl_size + QEDI_PAGE_SIZE;
1774
1775	ep->sq = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_mem_size,
1776				    &ep->sq_dma, GFP_KERNEL);
1777	if (!ep->sq) {
1778		QEDI_WARN(&qedi->dbg_ctx,
1779			  "Could not allocate send queue.\n");
1780		rval = -ENOMEM;
1781		goto out;
1782	}
1783	ep->sq_pbl = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_pbl_size,
1784					&ep->sq_pbl_dma, GFP_KERNEL);
1785	if (!ep->sq_pbl) {
1786		QEDI_WARN(&qedi->dbg_ctx,
1787			  "Could not allocate send queue PBL.\n");
1788		rval = -ENOMEM;
1789		goto out_free_sq;
1790	}
1791
1792	/* Create PBL */
1793	num_pages = ep->sq_mem_size / QEDI_PAGE_SIZE;
1794	page = ep->sq_dma;
1795	pbl = (u32 *)ep->sq_pbl;
1796
1797	while (num_pages--) {
1798		*pbl = (u32)page;
1799		pbl++;
1800		*pbl = (u32)((u64)page >> 32);
1801		pbl++;
1802		page += QEDI_PAGE_SIZE;
1803	}
1804
1805	return rval;
1806
1807out_free_sq:
1808	dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,
1809			  ep->sq_dma);
1810out:
1811	return rval;
1812}
1813
1814void qedi_free_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)
1815{
1816	if (ep->sq_pbl)
1817		dma_free_coherent(&qedi->pdev->dev, ep->sq_pbl_size, ep->sq_pbl,
1818				  ep->sq_pbl_dma);
1819	if (ep->sq)
1820		dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,
1821				  ep->sq_dma);
1822}
1823
1824int qedi_get_task_idx(struct qedi_ctx *qedi)
1825{
1826	s16 tmp_idx;
1827
1828again:
1829	tmp_idx = find_first_zero_bit(qedi->task_idx_map,
1830				      MAX_ISCSI_TASK_ENTRIES);
1831
1832	if (tmp_idx >= MAX_ISCSI_TASK_ENTRIES) {
1833		QEDI_ERR(&qedi->dbg_ctx, "FW task context pool is full.\n");
1834		tmp_idx = -1;
1835		goto err_idx;
1836	}
1837
1838	if (test_and_set_bit(tmp_idx, qedi->task_idx_map))
1839		goto again;
1840
1841err_idx:
1842	return tmp_idx;
1843}
1844
1845void qedi_clear_task_idx(struct qedi_ctx *qedi, int idx)
1846{
1847	if (!test_and_clear_bit(idx, qedi->task_idx_map))
1848		QEDI_ERR(&qedi->dbg_ctx,
1849			 "FW task context, already cleared, tid=0x%x\n", idx);
1850}
1851
1852void qedi_update_itt_map(struct qedi_ctx *qedi, u32 tid, u32 proto_itt,
1853			 struct qedi_cmd *cmd)
1854{
1855	qedi->itt_map[tid].itt = proto_itt;
1856	qedi->itt_map[tid].p_cmd = cmd;
1857
1858	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1859		  "update itt map tid=0x%x, with proto itt=0x%x\n", tid,
1860		  qedi->itt_map[tid].itt);
1861}
1862
1863void qedi_get_task_tid(struct qedi_ctx *qedi, u32 itt, s16 *tid)
1864{
1865	u16 i;
1866
1867	for (i = 0; i < MAX_ISCSI_TASK_ENTRIES; i++) {
1868		if (qedi->itt_map[i].itt == itt) {
1869			*tid = i;
1870			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1871				  "Ref itt=0x%x, found at tid=0x%x\n",
1872				  itt, *tid);
1873			return;
1874		}
1875	}
1876
1877	WARN_ON(1);
1878}
1879
1880struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid)
1881{
1882	struct qedi_cmd *cmd = NULL;
1883
1884	if (tid >= MAX_ISCSI_TASK_ENTRIES)
1885		return NULL;
1886
1887	cmd = qedi->itt_map[tid].p_cmd;
1888	if (cmd->task_id != tid)
1889		return NULL;
1890
1891	qedi->itt_map[tid].p_cmd = NULL;
1892
1893	return cmd;
1894}
1895
1896static int qedi_alloc_itt(struct qedi_ctx *qedi)
1897{
1898	qedi->itt_map = kcalloc(MAX_ISCSI_TASK_ENTRIES,
1899				sizeof(struct qedi_itt_map), GFP_KERNEL);
1900	if (!qedi->itt_map) {
1901		QEDI_ERR(&qedi->dbg_ctx,
1902			 "Unable to allocate itt map array memory\n");
1903		return -ENOMEM;
1904	}
1905	return 0;
1906}
1907
1908static void qedi_free_itt(struct qedi_ctx *qedi)
1909{
1910	kfree(qedi->itt_map);
1911}
1912
1913static struct qed_ll2_cb_ops qedi_ll2_cb_ops = {
1914	.rx_cb = qedi_ll2_rx,
1915	.tx_cb = NULL,
1916};
1917
1918static int qedi_percpu_io_thread(void *arg)
1919{
1920	struct qedi_percpu_s *p = arg;
1921	struct qedi_work *work, *tmp;
1922	unsigned long flags;
1923	LIST_HEAD(work_list);
1924
1925	set_user_nice(current, -20);
1926
1927	while (!kthread_should_stop()) {
1928		spin_lock_irqsave(&p->p_work_lock, flags);
1929		while (!list_empty(&p->work_list)) {
1930			list_splice_init(&p->work_list, &work_list);
1931			spin_unlock_irqrestore(&p->p_work_lock, flags);
1932
1933			list_for_each_entry_safe(work, tmp, &work_list, list) {
1934				list_del_init(&work->list);
1935				qedi_fp_process_cqes(work);
1936				if (!work->is_solicited)
1937					kfree(work);
1938			}
1939			cond_resched();
1940			spin_lock_irqsave(&p->p_work_lock, flags);
1941		}
1942		set_current_state(TASK_INTERRUPTIBLE);
1943		spin_unlock_irqrestore(&p->p_work_lock, flags);
1944		schedule();
1945	}
1946	__set_current_state(TASK_RUNNING);
1947
1948	return 0;
1949}
1950
1951static int qedi_cpu_online(unsigned int cpu)
1952{
1953	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1954	struct task_struct *thread;
1955
1956	thread = kthread_create_on_cpu(qedi_percpu_io_thread, (void *)p,
1957				       cpu, "qedi_thread/%d");
1958	if (IS_ERR(thread))
1959		return PTR_ERR(thread);
1960
1961	p->iothread = thread;
1962	wake_up_process(thread);
1963	return 0;
1964}
1965
1966static int qedi_cpu_offline(unsigned int cpu)
1967{
1968	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1969	struct qedi_work *work, *tmp;
1970	struct task_struct *thread;
1971	unsigned long flags;
1972
1973	spin_lock_irqsave(&p->p_work_lock, flags);
1974	thread = p->iothread;
1975	p->iothread = NULL;
1976
1977	list_for_each_entry_safe(work, tmp, &p->work_list, list) {
1978		list_del_init(&work->list);
1979		qedi_fp_process_cqes(work);
1980		if (!work->is_solicited)
1981			kfree(work);
1982	}
1983
1984	spin_unlock_irqrestore(&p->p_work_lock, flags);
1985	if (thread)
1986		kthread_stop(thread);
1987	return 0;
1988}
1989
1990void qedi_reset_host_mtu(struct qedi_ctx *qedi, u16 mtu)
1991{
1992	struct qed_ll2_params params;
1993
1994	qedi_recover_all_conns(qedi);
1995
1996	qedi_ops->ll2->stop(qedi->cdev);
1997	qedi_ll2_free_skbs(qedi);
1998
1999	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "old MTU %u, new MTU %u\n",
2000		  qedi->ll2_mtu, mtu);
2001	memset(&params, 0, sizeof(params));
2002	qedi->ll2_mtu = mtu;
2003	params.mtu = qedi->ll2_mtu + IPV6_HDR_LEN + TCP_HDR_LEN;
2004	params.drop_ttl0_packets = 0;
2005	params.rx_vlan_stripping = 1;
2006	ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);
2007	qedi_ops->ll2->start(qedi->cdev, &params);
2008}
2009
2010/*
2011 * qedi_get_nvram_block: - Scan through the iSCSI NVRAM block (while accounting
2012 * for gaps) for the matching absolute-pf-id of the QEDI device.
2013 */
2014static struct nvm_iscsi_block *
2015qedi_get_nvram_block(struct qedi_ctx *qedi)
2016{
2017	int i;
2018	u8 pf;
2019	u32 flags;
2020	struct nvm_iscsi_block *block;
2021
2022	pf = qedi->dev_info.common.abs_pf_id;
2023	block = &qedi->iscsi_image->iscsi_cfg.block[0];
2024	for (i = 0; i < NUM_OF_ISCSI_PF_SUPPORTED; i++, block++) {
2025		flags = ((block->id) & NVM_ISCSI_CFG_BLK_CTRL_FLAG_MASK) >>
2026			NVM_ISCSI_CFG_BLK_CTRL_FLAG_OFFSET;
2027		if (flags & (NVM_ISCSI_CFG_BLK_CTRL_FLAG_IS_NOT_EMPTY |
2028				NVM_ISCSI_CFG_BLK_CTRL_FLAG_PF_MAPPED) &&
2029			(pf == (block->id & NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_MASK)
2030				>> NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_OFFSET))
2031			return block;
2032	}
2033	return NULL;
2034}
2035
2036static ssize_t qedi_show_boot_eth_info(void *data, int type, char *buf)
2037{
2038	struct qedi_ctx *qedi = data;
2039	struct nvm_iscsi_initiator *initiator;
2040	int rc = 1;
2041	u32 ipv6_en, dhcp_en, ip_len;
2042	struct nvm_iscsi_block *block;
2043	char *fmt, *ip, *sub, *gw;
2044
2045	block = qedi_get_nvram_block(qedi);
2046	if (!block)
2047		return 0;
2048
2049	initiator = &block->initiator;
2050	ipv6_en = block->generic.ctrl_flags &
2051		  NVM_ISCSI_CFG_GEN_IPV6_ENABLED;
2052	dhcp_en = block->generic.ctrl_flags &
2053		  NVM_ISCSI_CFG_GEN_DHCP_TCPIP_CONFIG_ENABLED;
2054	/* Static IP assignments. */
2055	fmt = ipv6_en ? "%pI6\n" : "%pI4\n";
2056	ip = ipv6_en ? initiator->ipv6.addr.byte : initiator->ipv4.addr.byte;
2057	ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;
2058	sub = ipv6_en ? initiator->ipv6.subnet_mask.byte :
2059	      initiator->ipv4.subnet_mask.byte;
2060	gw = ipv6_en ? initiator->ipv6.gateway.byte :
2061	     initiator->ipv4.gateway.byte;
2062	/* DHCP IP adjustments. */
2063	fmt = dhcp_en ? "%s\n" : fmt;
2064	if (dhcp_en) {
2065		ip = ipv6_en ? "0::0" : "0.0.0.0";
2066		sub = ip;
2067		gw = ip;
2068		ip_len = ipv6_en ? 5 : 8;
2069	}
2070
2071	switch (type) {
2072	case ISCSI_BOOT_ETH_IP_ADDR:
2073		rc = snprintf(buf, ip_len, fmt, ip);
2074		break;
2075	case ISCSI_BOOT_ETH_SUBNET_MASK:
2076		rc = snprintf(buf, ip_len, fmt, sub);
2077		break;
2078	case ISCSI_BOOT_ETH_GATEWAY:
2079		rc = snprintf(buf, ip_len, fmt, gw);
2080		break;
2081	case ISCSI_BOOT_ETH_FLAGS:
2082		rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);
2083		break;
2084	case ISCSI_BOOT_ETH_INDEX:
2085		rc = snprintf(buf, 3, "0\n");
2086		break;
2087	case ISCSI_BOOT_ETH_MAC:
2088		rc = sysfs_format_mac(buf, qedi->mac, ETH_ALEN);
2089		break;
2090	case ISCSI_BOOT_ETH_VLAN:
2091		rc = snprintf(buf, 12, "%d\n",
2092			      GET_FIELD2(initiator->generic_cont0,
2093					 NVM_ISCSI_CFG_INITIATOR_VLAN));
2094		break;
2095	case ISCSI_BOOT_ETH_ORIGIN:
2096		if (dhcp_en)
2097			rc = snprintf(buf, 3, "3\n");
2098		break;
2099	default:
2100		rc = 0;
2101		break;
2102	}
2103
2104	return rc;
2105}
2106
2107static umode_t qedi_eth_get_attr_visibility(void *data, int type)
2108{
2109	int rc = 1;
2110
2111	switch (type) {
2112	case ISCSI_BOOT_ETH_FLAGS:
2113	case ISCSI_BOOT_ETH_MAC:
2114	case ISCSI_BOOT_ETH_INDEX:
2115	case ISCSI_BOOT_ETH_IP_ADDR:
2116	case ISCSI_BOOT_ETH_SUBNET_MASK:
2117	case ISCSI_BOOT_ETH_GATEWAY:
2118	case ISCSI_BOOT_ETH_ORIGIN:
2119	case ISCSI_BOOT_ETH_VLAN:
2120		rc = 0444;
2121		break;
2122	default:
2123		rc = 0;
2124		break;
2125	}
2126	return rc;
2127}
2128
2129static ssize_t qedi_show_boot_ini_info(void *data, int type, char *buf)
2130{
2131	struct qedi_ctx *qedi = data;
2132	struct nvm_iscsi_initiator *initiator;
2133	int rc;
2134	struct nvm_iscsi_block *block;
2135
2136	block = qedi_get_nvram_block(qedi);
2137	if (!block)
2138		return 0;
2139
2140	initiator = &block->initiator;
2141
2142	switch (type) {
2143	case ISCSI_BOOT_INI_INITIATOR_NAME:
2144		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,
2145			     initiator->initiator_name.byte);
2146		break;
2147	default:
2148		rc = 0;
2149		break;
2150	}
2151	return rc;
2152}
2153
2154static umode_t qedi_ini_get_attr_visibility(void *data, int type)
2155{
2156	int rc;
2157
2158	switch (type) {
2159	case ISCSI_BOOT_INI_INITIATOR_NAME:
2160		rc = 0444;
2161		break;
2162	default:
2163		rc = 0;
2164		break;
2165	}
2166	return rc;
2167}
2168
2169static ssize_t
2170qedi_show_boot_tgt_info(struct qedi_ctx *qedi, int type,
2171			char *buf, enum qedi_nvm_tgts idx)
2172{
2173	int rc = 1;
2174	u32 ctrl_flags, ipv6_en, chap_en, mchap_en, ip_len;
2175	struct nvm_iscsi_block *block;
2176	char *chap_name, *chap_secret;
2177	char *mchap_name, *mchap_secret;
2178
2179	block = qedi_get_nvram_block(qedi);
2180	if (!block)
2181		goto exit_show_tgt_info;
2182
2183	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,
2184		  "Port:%d, tgt_idx:%d\n",
2185		  GET_FIELD2(block->id, NVM_ISCSI_CFG_BLK_MAPPED_PF_ID), idx);
2186
2187	ctrl_flags = block->target[idx].ctrl_flags &
2188		     NVM_ISCSI_CFG_TARGET_ENABLED;
2189
2190	if (!ctrl_flags) {
2191		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,
2192			  "Target disabled\n");
2193		goto exit_show_tgt_info;
2194	}
2195
2196	ipv6_en = block->generic.ctrl_flags &
2197		  NVM_ISCSI_CFG_GEN_IPV6_ENABLED;
2198	ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;
2199	chap_en = block->generic.ctrl_flags &
2200		  NVM_ISCSI_CFG_GEN_CHAP_ENABLED;
2201	chap_name = chap_en ? block->initiator.chap_name.byte : NULL;
2202	chap_secret = chap_en ? block->initiator.chap_password.byte : NULL;
2203
2204	mchap_en = block->generic.ctrl_flags &
2205		  NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED;
2206	mchap_name = mchap_en ? block->target[idx].chap_name.byte : NULL;
2207	mchap_secret = mchap_en ? block->target[idx].chap_password.byte : NULL;
2208
2209	switch (type) {
2210	case ISCSI_BOOT_TGT_NAME:
2211		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,
2212			     block->target[idx].target_name.byte);
2213		break;
2214	case ISCSI_BOOT_TGT_IP_ADDR:
2215		if (ipv6_en)
2216			rc = snprintf(buf, ip_len, "%pI6\n",
2217				      block->target[idx].ipv6_addr.byte);
2218		else
2219			rc = snprintf(buf, ip_len, "%pI4\n",
2220				      block->target[idx].ipv4_addr.byte);
2221		break;
2222	case ISCSI_BOOT_TGT_PORT:
2223		rc = snprintf(buf, 12, "%d\n",
2224			      GET_FIELD2(block->target[idx].generic_cont0,
2225					 NVM_ISCSI_CFG_TARGET_TCP_PORT));
2226		break;
2227	case ISCSI_BOOT_TGT_LUN:
2228		rc = snprintf(buf, 22, "%.*d\n",
2229			      block->target[idx].lun.value[1],
2230			      block->target[idx].lun.value[0]);
2231		break;
2232	case ISCSI_BOOT_TGT_CHAP_NAME:
2233		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
2234			     chap_name);
2235		break;
2236	case ISCSI_BOOT_TGT_CHAP_SECRET:
2237		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
2238			     chap_secret);
2239		break;
2240	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
2241		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
2242			     mchap_name);
2243		break;
2244	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
2245		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
2246			     mchap_secret);
2247		break;
2248	case ISCSI_BOOT_TGT_FLAGS:
2249		rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);
2250		break;
2251	case ISCSI_BOOT_TGT_NIC_ASSOC:
2252		rc = snprintf(buf, 3, "0\n");
2253		break;
2254	default:
2255		rc = 0;
2256		break;
2257	}
2258
2259exit_show_tgt_info:
2260	return rc;
2261}
2262
2263static ssize_t qedi_show_boot_tgt_pri_info(void *data, int type, char *buf)
2264{
2265	struct qedi_ctx *qedi = data;
2266
2267	return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_PRI);
2268}
2269
2270static ssize_t qedi_show_boot_tgt_sec_info(void *data, int type, char *buf)
2271{
2272	struct qedi_ctx *qedi = data;
2273
2274	return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_SEC);
2275}
2276
2277static umode_t qedi_tgt_get_attr_visibility(void *data, int type)
2278{
2279	int rc;
2280
2281	switch (type) {
2282	case ISCSI_BOOT_TGT_NAME:
2283	case ISCSI_BOOT_TGT_IP_ADDR:
2284	case ISCSI_BOOT_TGT_PORT:
2285	case ISCSI_BOOT_TGT_LUN:
2286	case ISCSI_BOOT_TGT_CHAP_NAME:
2287	case ISCSI_BOOT_TGT_CHAP_SECRET:
2288	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
2289	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
2290	case ISCSI_BOOT_TGT_NIC_ASSOC:
2291	case ISCSI_BOOT_TGT_FLAGS:
2292		rc = 0444;
2293		break;
2294	default:
2295		rc = 0;
2296		break;
2297	}
2298	return rc;
2299}
2300
2301static void qedi_boot_release(void *data)
2302{
2303	struct qedi_ctx *qedi = data;
2304
2305	scsi_host_put(qedi->shost);
2306}
2307
2308static int qedi_get_boot_info(struct qedi_ctx *qedi)
2309{
2310	int ret = 1;
2311
2312	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2313		  "Get NVM iSCSI CFG image\n");
2314	ret = qedi_ops->common->nvm_get_image(qedi->cdev,
2315					      QED_NVM_IMAGE_ISCSI_CFG,
2316					      (char *)qedi->iscsi_image,
2317					      sizeof(struct qedi_nvm_iscsi_image));
2318	if (ret)
2319		QEDI_ERR(&qedi->dbg_ctx,
2320			 "Could not get NVM image. ret = %d\n", ret);
2321
2322	return ret;
2323}
2324
2325static int qedi_setup_boot_info(struct qedi_ctx *qedi)
2326{
2327	struct iscsi_boot_kobj *boot_kobj;
2328
2329	if (qedi_get_boot_info(qedi))
2330		return -EPERM;
2331
2332	qedi->boot_kset = iscsi_boot_create_host_kset(qedi->shost->host_no);
2333	if (!qedi->boot_kset)
2334		goto kset_free;
2335
2336	if (!scsi_host_get(qedi->shost))
2337		goto kset_free;
2338
2339	boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 0, qedi,
2340					     qedi_show_boot_tgt_pri_info,
2341					     qedi_tgt_get_attr_visibility,
2342					     qedi_boot_release);
2343	if (!boot_kobj)
2344		goto put_host;
2345
2346	if (!scsi_host_get(qedi->shost))
2347		goto kset_free;
2348
2349	boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 1, qedi,
2350					     qedi_show_boot_tgt_sec_info,
2351					     qedi_tgt_get_attr_visibility,
2352					     qedi_boot_release);
2353	if (!boot_kobj)
2354		goto put_host;
2355
2356	if (!scsi_host_get(qedi->shost))
2357		goto kset_free;
2358
2359	boot_kobj = iscsi_boot_create_initiator(qedi->boot_kset, 0, qedi,
2360						qedi_show_boot_ini_info,
2361						qedi_ini_get_attr_visibility,
2362						qedi_boot_release);
2363	if (!boot_kobj)
2364		goto put_host;
2365
2366	if (!scsi_host_get(qedi->shost))
2367		goto kset_free;
2368
2369	boot_kobj = iscsi_boot_create_ethernet(qedi->boot_kset, 0, qedi,
2370					       qedi_show_boot_eth_info,
2371					       qedi_eth_get_attr_visibility,
2372					       qedi_boot_release);
2373	if (!boot_kobj)
2374		goto put_host;
2375
2376	return 0;
2377
2378put_host:
2379	scsi_host_put(qedi->shost);
2380kset_free:
2381	iscsi_boot_destroy_kset(qedi->boot_kset);
2382	return -ENOMEM;
2383}
2384
2385static pci_ers_result_t qedi_io_error_detected(struct pci_dev *pdev,
2386					       pci_channel_state_t state)
2387{
2388	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2389
2390	QEDI_ERR(&qedi->dbg_ctx, "%s: PCI error detected [%d]\n",
2391		 __func__, state);
2392
2393	if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags)) {
2394		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2395			  "Recovery already in progress.\n");
2396		return PCI_ERS_RESULT_NONE;
2397	}
2398
2399	qedi_ops->common->recovery_process(qedi->cdev);
2400
2401	return PCI_ERS_RESULT_CAN_RECOVER;
2402}
2403
2404static void __qedi_remove(struct pci_dev *pdev, int mode)
2405{
2406	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2407	int rval;
2408	u16 retry = 10;
2409
2410	if (mode == QEDI_MODE_NORMAL)
2411		iscsi_host_remove(qedi->shost, false);
2412	else if (mode == QEDI_MODE_SHUTDOWN)
2413		iscsi_host_remove(qedi->shost, true);
2414
2415	if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {
2416		if (qedi->tmf_thread) {
2417			destroy_workqueue(qedi->tmf_thread);
2418			qedi->tmf_thread = NULL;
2419		}
2420
2421		if (qedi->offload_thread) {
2422			destroy_workqueue(qedi->offload_thread);
2423			qedi->offload_thread = NULL;
2424		}
2425	}
2426
2427#ifdef CONFIG_DEBUG_FS
2428	qedi_dbg_host_exit(&qedi->dbg_ctx);
2429#endif
2430	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags))
2431		qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);
2432
2433	qedi_sync_free_irqs(qedi);
2434
2435	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2436		while (retry--) {
2437			rval = qedi_ops->stop(qedi->cdev);
2438			if (rval < 0)
2439				msleep(1000);
2440			else
2441				break;
2442		}
2443		qedi_ops->ll2->stop(qedi->cdev);
2444	}
2445
2446	cancel_delayed_work_sync(&qedi->recovery_work);
2447	cancel_delayed_work_sync(&qedi->board_disable_work);
2448
2449	qedi_free_iscsi_pf_param(qedi);
2450
2451	rval = qedi_ops->common->update_drv_state(qedi->cdev, false);
2452	if (rval)
2453		QEDI_ERR(&qedi->dbg_ctx, "Failed to send drv state to MFW\n");
2454
2455	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2456		qedi_ops->common->slowpath_stop(qedi->cdev);
2457		qedi_ops->common->remove(qedi->cdev);
2458	}
2459
2460	qedi_destroy_fp(qedi);
2461
2462	if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {
2463		qedi_release_cid_que(qedi);
2464		qedi_cm_free_mem(qedi);
2465		qedi_free_uio(qedi->udev);
2466		qedi_free_itt(qedi);
2467
2468		if (qedi->ll2_recv_thread) {
2469			kthread_stop(qedi->ll2_recv_thread);
2470			qedi->ll2_recv_thread = NULL;
2471		}
2472		qedi_ll2_free_skbs(qedi);
2473
2474		if (qedi->boot_kset)
2475			iscsi_boot_destroy_kset(qedi->boot_kset);
2476
2477		iscsi_host_free(qedi->shost);
2478	}
2479}
2480
2481static void qedi_board_disable_work(struct work_struct *work)
2482{
2483	struct qedi_ctx *qedi =
2484			container_of(work, struct qedi_ctx,
2485				     board_disable_work.work);
2486
2487	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2488		  "Fan failure, Unloading firmware context.\n");
2489
2490	if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
2491		return;
2492
2493	__qedi_remove(qedi->pdev, QEDI_MODE_NORMAL);
2494}
2495
2496static void qedi_shutdown(struct pci_dev *pdev)
2497{
2498	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2499
2500	QEDI_ERR(&qedi->dbg_ctx, "%s: Shutdown qedi\n", __func__);
2501	if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
2502		return;
2503	__qedi_remove(pdev, QEDI_MODE_SHUTDOWN);
2504}
2505
2506static int qedi_suspend(struct pci_dev *pdev, pm_message_t state)
2507{
2508	struct qedi_ctx *qedi;
2509
2510	if (!pdev) {
2511		QEDI_ERR(NULL, "pdev is NULL.\n");
2512		return -ENODEV;
2513	}
2514
2515	qedi = pci_get_drvdata(pdev);
2516
2517	QEDI_ERR(&qedi->dbg_ctx, "%s: Device does not support suspend operation\n", __func__);
2518
2519	return -EPERM;
2520}
2521
2522static int __qedi_probe(struct pci_dev *pdev, int mode)
2523{
2524	struct qedi_ctx *qedi;
2525	struct qed_ll2_params params;
2526	u8 dp_level = 0;
2527	bool is_vf = false;
2528	char host_buf[16];
2529	struct qed_link_params link_params;
2530	struct qed_slowpath_params sp_params;
2531	struct qed_probe_params qed_params;
2532	void *task_start, *task_end;
2533	int rc;
2534	u16 retry = 10;
2535
2536	if (mode != QEDI_MODE_RECOVERY) {
2537		qedi = qedi_host_alloc(pdev);
2538		if (!qedi) {
2539			rc = -ENOMEM;
2540			goto exit_probe;
2541		}
2542	} else {
2543		qedi = pci_get_drvdata(pdev);
2544	}
2545
2546retry_probe:
2547	if (mode == QEDI_MODE_RECOVERY)
2548		msleep(2000);
2549
2550	memset(&qed_params, 0, sizeof(qed_params));
2551	qed_params.protocol = QED_PROTOCOL_ISCSI;
2552	qed_params.dp_module = qedi_qed_debug;
2553	qed_params.dp_level = dp_level;
2554	qed_params.is_vf = is_vf;
2555	qedi->cdev = qedi_ops->common->probe(pdev, &qed_params);
2556	if (!qedi->cdev) {
2557		if (mode == QEDI_MODE_RECOVERY && retry) {
2558			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2559				  "Retry %d initialize hardware\n", retry);
2560			retry--;
2561			goto retry_probe;
2562		}
2563
2564		rc = -ENODEV;
2565		QEDI_ERR(&qedi->dbg_ctx, "Cannot initialize hardware\n");
2566		goto free_host;
2567	}
2568
2569	set_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags);
2570	set_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags);
2571	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2572
2573	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2574	if (rc)
2575		goto free_host;
2576
2577	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2578		  "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
2579		  qedi->dev_info.common.num_hwfns,
2580		  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
2581
2582	rc = qedi_set_iscsi_pf_param(qedi);
2583	if (rc) {
2584		rc = -ENOMEM;
2585		QEDI_ERR(&qedi->dbg_ctx,
2586			 "Set iSCSI pf param fail\n");
2587		goto free_host;
2588	}
2589
2590	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
2591
2592	rc = qedi_prepare_fp(qedi);
2593	if (rc) {
2594		QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath.\n");
2595		goto free_pf_params;
2596	}
2597
2598	/* Start the Slowpath-process */
2599	memset(&sp_params, 0, sizeof(struct qed_slowpath_params));
2600	sp_params.int_mode = QED_INT_MODE_MSIX;
2601	sp_params.drv_major = QEDI_DRIVER_MAJOR_VER;
2602	sp_params.drv_minor = QEDI_DRIVER_MINOR_VER;
2603	sp_params.drv_rev = QEDI_DRIVER_REV_VER;
2604	sp_params.drv_eng = QEDI_DRIVER_ENG_VER;
2605	strscpy(sp_params.name, "qedi iSCSI", QED_DRV_VER_STR_SIZE);
2606	rc = qedi_ops->common->slowpath_start(qedi->cdev, &sp_params);
2607	if (rc) {
2608		QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath\n");
2609		goto stop_hw;
2610	}
2611
2612	/* update_pf_params needs to be called before and after slowpath
2613	 * start
2614	 */
2615	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
2616
2617	rc = qedi_setup_int(qedi);
2618	if (rc)
2619		goto stop_iscsi_func;
2620
2621	qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);
2622
2623	/* Learn information crucial for qedi to progress */
2624	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2625	if (rc)
2626		goto stop_iscsi_func;
2627
2628	/* Record BDQ producer doorbell addresses */
2629	qedi->bdq_primary_prod = qedi->dev_info.primary_dbq_rq_addr;
2630	qedi->bdq_secondary_prod = qedi->dev_info.secondary_bdq_rq_addr;
2631	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2632		  "BDQ primary_prod=%p secondary_prod=%p.\n",
2633		  qedi->bdq_primary_prod,
2634		  qedi->bdq_secondary_prod);
2635
2636	/*
2637	 * We need to write the number of BDs in the BDQ we've preallocated so
2638	 * the f/w will do a prefetch and we'll get an unsolicited CQE when a
2639	 * packet arrives.
2640	 */
2641	qedi->bdq_prod_idx = QEDI_BDQ_NUM;
2642	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2643		  "Writing %d to primary and secondary BDQ doorbell registers.\n",
2644		  qedi->bdq_prod_idx);
2645	writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
2646	readw(qedi->bdq_primary_prod);
2647	writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
2648	readw(qedi->bdq_secondary_prod);
2649
2650	ether_addr_copy(qedi->mac, qedi->dev_info.common.hw_mac);
2651	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "MAC address is %pM.\n",
2652		  qedi->mac);
2653
2654	snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no);
2655	qedi_ops->common->set_name(qedi->cdev, host_buf);
2656
2657	qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi);
2658
2659	memset(&params, 0, sizeof(params));
2660	params.mtu = DEF_PATH_MTU + IPV6_HDR_LEN + TCP_HDR_LEN;
2661	qedi->ll2_mtu = DEF_PATH_MTU;
2662	params.drop_ttl0_packets = 0;
2663	params.rx_vlan_stripping = 1;
2664	ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);
2665
2666	if (mode != QEDI_MODE_RECOVERY) {
2667		/* set up rx path */
2668		INIT_LIST_HEAD(&qedi->ll2_skb_list);
2669		spin_lock_init(&qedi->ll2_lock);
2670		/* start qedi context */
2671		spin_lock_init(&qedi->hba_lock);
2672		spin_lock_init(&qedi->task_idx_lock);
2673		mutex_init(&qedi->stats_lock);
2674	}
2675	qedi_ops->ll2->register_cb_ops(qedi->cdev, &qedi_ll2_cb_ops, qedi);
2676	qedi_ops->ll2->start(qedi->cdev, &params);
2677
2678	if (mode != QEDI_MODE_RECOVERY) {
2679		qedi->ll2_recv_thread = kthread_run(qedi_ll2_recv_thread,
2680						    (void *)qedi,
2681						    "qedi_ll2_thread");
2682	}
2683
2684	rc = qedi_ops->start(qedi->cdev, &qedi->tasks,
2685			     qedi, qedi_iscsi_event_cb);
2686	if (rc) {
2687		rc = -ENODEV;
2688		QEDI_ERR(&qedi->dbg_ctx, "Cannot start iSCSI function\n");
2689		goto stop_slowpath;
2690	}
2691
2692	task_start = qedi_get_task_mem(&qedi->tasks, 0);
2693	task_end = qedi_get_task_mem(&qedi->tasks, MAX_TID_BLOCKS_ISCSI - 1);
2694	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2695		  "Task context start=%p, end=%p block_size=%u.\n",
2696		   task_start, task_end, qedi->tasks.size);
2697
2698	memset(&link_params, 0, sizeof(link_params));
2699	link_params.link_up = true;
2700	rc = qedi_ops->common->set_link(qedi->cdev, &link_params);
2701	if (rc) {
2702		QEDI_WARN(&qedi->dbg_ctx, "Link set up failed.\n");
2703		atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2704	}
2705
2706#ifdef CONFIG_DEBUG_FS
2707	qedi_dbg_host_init(&qedi->dbg_ctx, qedi_debugfs_ops,
2708			   qedi_dbg_fops);
2709#endif
2710	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2711		  "QLogic FastLinQ iSCSI Module qedi %s, FW %d.%d.%d.%d\n",
2712		  QEDI_MODULE_VERSION, FW_MAJOR_VERSION, FW_MINOR_VERSION,
2713		  FW_REVISION_VERSION, FW_ENGINEERING_VERSION);
2714
2715	if (mode == QEDI_MODE_NORMAL) {
2716		if (iscsi_host_add(qedi->shost, &pdev->dev)) {
2717			QEDI_ERR(&qedi->dbg_ctx,
2718				 "Could not add iscsi host\n");
2719			rc = -ENOMEM;
2720			goto remove_host;
2721		}
2722
2723		/* Allocate uio buffers */
2724		rc = qedi_alloc_uio_rings(qedi);
2725		if (rc) {
2726			QEDI_ERR(&qedi->dbg_ctx,
2727				 "UIO alloc ring failed err=%d\n", rc);
2728			goto remove_host;
2729		}
2730
2731		rc = qedi_init_uio(qedi);
2732		if (rc) {
2733			QEDI_ERR(&qedi->dbg_ctx,
2734				 "UIO init failed, err=%d\n", rc);
2735			goto free_uio;
2736		}
2737
2738		/* host the array on iscsi_conn */
2739		rc = qedi_setup_cid_que(qedi);
2740		if (rc) {
2741			QEDI_ERR(&qedi->dbg_ctx,
2742				 "Could not setup cid que\n");
2743			goto free_uio;
2744		}
2745
2746		rc = qedi_cm_alloc_mem(qedi);
2747		if (rc) {
2748			QEDI_ERR(&qedi->dbg_ctx,
2749				 "Could not alloc cm memory\n");
2750			goto free_cid_que;
2751		}
2752
2753		rc = qedi_alloc_itt(qedi);
2754		if (rc) {
2755			QEDI_ERR(&qedi->dbg_ctx,
2756				 "Could not alloc itt memory\n");
2757			goto free_cid_que;
2758		}
2759
2760		sprintf(host_buf, "host_%d", qedi->shost->host_no);
2761		qedi->tmf_thread =
2762			alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, host_buf);
2763		if (!qedi->tmf_thread) {
2764			QEDI_ERR(&qedi->dbg_ctx,
2765				 "Unable to start tmf thread!\n");
2766			rc = -ENODEV;
2767			goto free_cid_que;
2768		}
2769
2770		qedi->offload_thread = alloc_workqueue("qedi_ofld%d",
2771						       WQ_MEM_RECLAIM | WQ_PERCPU,
2772						       1, qedi->shost->host_no);
2773		if (!qedi->offload_thread) {
2774			QEDI_ERR(&qedi->dbg_ctx,
2775				 "Unable to start offload thread!\n");
2776			rc = -ENODEV;
2777			goto free_tmf_thread;
2778		}
2779
2780		INIT_DELAYED_WORK(&qedi->recovery_work, qedi_recovery_handler);
2781		INIT_DELAYED_WORK(&qedi->board_disable_work,
2782				  qedi_board_disable_work);
2783
2784		/* F/w needs 1st task context memory entry for performance */
2785		set_bit(QEDI_RESERVE_TASK_ID, qedi->task_idx_map);
2786		atomic_set(&qedi->num_offloads, 0);
2787
2788		if (qedi_setup_boot_info(qedi))
2789			QEDI_ERR(&qedi->dbg_ctx,
2790				 "No iSCSI boot target configured\n");
2791
2792		rc = qedi_ops->common->update_drv_state(qedi->cdev, true);
2793		if (rc)
2794			QEDI_ERR(&qedi->dbg_ctx,
2795				 "Failed to send drv state to MFW\n");
2796
2797	}
2798
2799	return 0;
2800
2801free_tmf_thread:
2802	destroy_workqueue(qedi->tmf_thread);
2803free_cid_que:
2804	qedi_release_cid_que(qedi);
2805free_uio:
2806	qedi_free_uio(qedi->udev);
2807remove_host:
2808#ifdef CONFIG_DEBUG_FS
2809	qedi_dbg_host_exit(&qedi->dbg_ctx);
2810#endif
2811	iscsi_host_remove(qedi->shost, false);
2812stop_iscsi_func:
2813	qedi_ops->stop(qedi->cdev);
2814stop_slowpath:
2815	qedi_ops->common->slowpath_stop(qedi->cdev);
2816stop_hw:
2817	qedi_ops->common->remove(qedi->cdev);
2818free_pf_params:
2819	qedi_free_iscsi_pf_param(qedi);
2820free_host:
2821	iscsi_host_free(qedi->shost);
2822exit_probe:
2823	return rc;
2824}
2825
2826static void qedi_mark_conn_recovery(struct iscsi_cls_session *cls_session)
2827{
2828	struct iscsi_session *session = cls_session->dd_data;
2829	struct iscsi_conn *conn = session->leadconn;
2830	struct qedi_conn *qedi_conn = conn->dd_data;
2831
2832	iscsi_conn_failure(qedi_conn->cls_conn->dd_data, ISCSI_ERR_CONN_FAILED);
2833}
2834
2835static void qedi_recovery_handler(struct work_struct *work)
2836{
2837	struct qedi_ctx *qedi =
2838			container_of(work, struct qedi_ctx, recovery_work.work);
2839
2840	iscsi_host_for_each_session(qedi->shost, qedi_mark_conn_recovery);
2841
2842	/* Call common_ops->recovery_prolog to allow the MFW to quiesce
2843	 * any PCI transactions.
2844	 */
2845	qedi_ops->common->recovery_prolog(qedi->cdev);
2846
2847	__qedi_remove(qedi->pdev, QEDI_MODE_RECOVERY);
2848	__qedi_probe(qedi->pdev, QEDI_MODE_RECOVERY);
2849	clear_bit(QEDI_IN_RECOVERY, &qedi->flags);
2850}
2851
2852static int qedi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2853{
2854	return __qedi_probe(pdev, QEDI_MODE_NORMAL);
2855}
2856
2857static void qedi_remove(struct pci_dev *pdev)
2858{
2859	__qedi_remove(pdev, QEDI_MODE_NORMAL);
2860}
2861
2862static const struct pci_device_id qedi_pci_tbl[] = {
2863	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x165E) },
2864	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x8084) },
2865	{ 0 },
2866};
2867MODULE_DEVICE_TABLE(pci, qedi_pci_tbl);
2868
2869static enum cpuhp_state qedi_cpuhp_state;
2870
2871static const struct pci_error_handlers qedi_err_handler = {
2872	.error_detected = qedi_io_error_detected,
2873};
2874
2875static struct pci_driver qedi_pci_driver = {
2876	.name = QEDI_MODULE_NAME,
2877	.id_table = qedi_pci_tbl,
2878	.probe = qedi_probe,
2879	.remove = qedi_remove,
2880	.shutdown = qedi_shutdown,
2881	.err_handler = &qedi_err_handler,
2882	.suspend = qedi_suspend,
2883};
2884
2885static int __init qedi_init(void)
2886{
2887	struct qedi_percpu_s *p;
2888	int cpu, rc = 0;
2889
2890	qedi_ops = qed_get_iscsi_ops();
2891	if (!qedi_ops) {
2892		QEDI_ERR(NULL, "Failed to get qed iSCSI operations\n");
2893		return -EINVAL;
2894	}
2895
2896#ifdef CONFIG_DEBUG_FS
2897	qedi_dbg_init("qedi");
2898#endif
2899
2900	qedi_scsi_transport = iscsi_register_transport(&qedi_iscsi_transport);
2901	if (!qedi_scsi_transport) {
2902		QEDI_ERR(NULL, "Could not register qedi transport");
2903		rc = -ENOMEM;
2904		goto exit_qedi_init_1;
2905	}
2906
2907	for_each_possible_cpu(cpu) {
2908		p = &per_cpu(qedi_percpu, cpu);
2909		INIT_LIST_HEAD(&p->work_list);
2910		spin_lock_init(&p->p_work_lock);
2911		p->iothread = NULL;
2912	}
2913
2914	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "scsi/qedi:online",
2915			       qedi_cpu_online, qedi_cpu_offline);
2916	if (rc < 0)
2917		goto exit_qedi_init_2;
2918	qedi_cpuhp_state = rc;
2919
2920	rc = pci_register_driver(&qedi_pci_driver);
2921	if (rc) {
2922		QEDI_ERR(NULL, "Failed to register driver\n");
2923		goto exit_qedi_hp;
2924	}
2925
2926	return 0;
2927
2928exit_qedi_hp:
2929	cpuhp_remove_state(qedi_cpuhp_state);
2930exit_qedi_init_2:
2931	iscsi_unregister_transport(&qedi_iscsi_transport);
2932exit_qedi_init_1:
2933#ifdef CONFIG_DEBUG_FS
2934	qedi_dbg_exit();
2935#endif
2936	qed_put_iscsi_ops();
2937	return rc;
2938}
2939
2940static void __exit qedi_cleanup(void)
2941{
2942	pci_unregister_driver(&qedi_pci_driver);
2943	cpuhp_remove_state(qedi_cpuhp_state);
2944	iscsi_unregister_transport(&qedi_iscsi_transport);
2945
2946#ifdef CONFIG_DEBUG_FS
2947	qedi_dbg_exit();
2948#endif
2949	qed_put_iscsi_ops();
2950}
2951
2952MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx iSCSI Module");
2953MODULE_LICENSE("GPL");
2954MODULE_AUTHOR("QLogic Corporation");
2955MODULE_VERSION(QEDI_MODULE_VERSION);
2956module_init(qedi_init);
2957module_exit(qedi_cleanup);