card_ddcb.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * IBM Accelerator Family 'GenWQE'
   4 *
   5 * (C) Copyright IBM Corp. 2013
   6 *
   7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
   8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
   9 * Author: Michael Jung <mijung@gmx.net>
  10 * Author: Michael Ruettger <michael@ibmra.de>
  11 */
  12
  13/*
  14 * Device Driver Control Block (DDCB) queue support. Definition of
  15 * interrupt handlers for queue support as well as triggering the
  16 * health monitor code in case of problems. The current hardware uses
  17 * an MSI interrupt which is shared between error handling and
  18 * functional code.
  19 */
  20
  21#include <linux/types.h>
  22#include <linux/sched.h>
  23#include <linux/wait.h>
  24#include <linux/pci.h>
  25#include <linux/string.h>
  26#include <linux/dma-mapping.h>
  27#include <linux/delay.h>
  28#include <linux/module.h>
  29#include <linux/interrupt.h>
  30#include <linux/crc-itu-t.h>
  31
  32#include "card_base.h"
  33#include "card_ddcb.h"
  34
  35/*
  36 * N: next DDCB, this is where the next DDCB will be put.
  37 * A: active DDCB, this is where the code will look for the next completion.
  38 * x: DDCB is enqueued, we are waiting for its completion.
  39
  40 * Situation (1): Empty queue
  41 *  +---+---+---+---+---+---+---+---+
  42 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  43 *  |   |   |   |   |   |   |   |   |
  44 *  +---+---+---+---+---+---+---+---+
  45 *           A/N
  46 *  enqueued_ddcbs = A - N = 2 - 2 = 0
  47 *
  48 * Situation (2): Wrapped, N > A
  49 *  +---+---+---+---+---+---+---+---+
  50 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  51 *  |   |   | x | x |   |   |   |   |
  52 *  +---+---+---+---+---+---+---+---+
  53 *            A       N
  54 *  enqueued_ddcbs = N - A = 4 - 2 = 2
  55 *
  56 * Situation (3): Queue wrapped, A > N
  57 *  +---+---+---+---+---+---+---+---+
  58 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  59 *  | x | x |   |   | x | x | x | x |
  60 *  +---+---+---+---+---+---+---+---+
  61 *            N       A
  62 *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
  63 *
  64 * Situation (4a): Queue full N > A
  65 *  +---+---+---+---+---+---+---+---+
  66 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  67 *  | x | x | x | x | x | x | x |   |
  68 *  +---+---+---+---+---+---+---+---+
  69 *    A                           N
  70 *
  71 *  enqueued_ddcbs = N - A = 7 - 0 = 7
  72 *
  73 * Situation (4a): Queue full A > N
  74 *  +---+---+---+---+---+---+---+---+
  75 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  76 *  | x | x | x |   | x | x | x | x |
  77 *  +---+---+---+---+---+---+---+---+
  78 *                N   A
  79 *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
  80 */
  81
  82static int queue_empty(struct ddcb_queue *queue)
  83{
  84	return queue->ddcb_next == queue->ddcb_act;
  85}
  86
  87static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
  88{
  89	if (queue->ddcb_next >= queue->ddcb_act)
  90		return queue->ddcb_next - queue->ddcb_act;
  91
  92	return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
  93}
  94
  95static int queue_free_ddcbs(struct ddcb_queue *queue)
  96{
  97	int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
  98
  99	if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
 100		return 0;
 101	}
 102	return free_ddcbs;
 103}
 104
 105/*
 106 * Use of the PRIV field in the DDCB for queue debugging:
 107 *
 108 * (1) Trying to get rid of a DDCB which saw a timeout:
 109 *     pddcb->priv[6] = 0xcc;   # cleared
 110 *
 111 * (2) Append a DDCB via NEXT bit:
 112 *     pddcb->priv[7] = 0xaa;	# appended
 113 *
 114 * (3) DDCB needed tapping:
 115 *     pddcb->priv[7] = 0xbb;   # tapped
 116 *
 117 * (4) DDCB marked as correctly finished:
 118 *     pddcb->priv[6] = 0xff;	# finished
 119 */
 120
 121static inline void ddcb_mark_tapped(struct ddcb *pddcb)
 122{
 123	pddcb->priv[7] = 0xbb;  /* tapped */
 124}
 125
 126static inline void ddcb_mark_appended(struct ddcb *pddcb)
 127{
 128	pddcb->priv[7] = 0xaa;	/* appended */
 129}
 130
 131static inline void ddcb_mark_cleared(struct ddcb *pddcb)
 132{
 133	pddcb->priv[6] = 0xcc; /* cleared */
 134}
 135
 136static inline void ddcb_mark_finished(struct ddcb *pddcb)
 137{
 138	pddcb->priv[6] = 0xff;	/* finished */
 139}
 140
 141static inline void ddcb_mark_unused(struct ddcb *pddcb)
 142{
 143	pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
 144}
 145
 146/**
 147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
 148 * @buff:       pointer to data buffer
 149 * @len:        length of data for calculation
 150 * @init:       initial crc (0xffff at start)
 151 *
 152 * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
 153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
 154 *          should result in a crc16 of 0x89c3
 155 *
 156 * Return: crc16 checksum in big endian format !
 157 */
 158static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
 159{
 160	return crc_itu_t(init, buff, len);
 161}
 162
 163static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
 164{
 165	int i;
 166	struct ddcb *pddcb;
 167	unsigned long flags;
 168	struct pci_dev *pci_dev = cd->pci_dev;
 169
 170	spin_lock_irqsave(&cd->print_lock, flags);
 171
 172	dev_info(&pci_dev->dev,
 173		 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
 174		 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
 175
 176	pddcb = queue->ddcb_vaddr;
 177	for (i = 0; i < queue->ddcb_max; i++) {
 178		dev_err(&pci_dev->dev,
 179			"  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
 180			i == queue->ddcb_act ? '>' : ' ',
 181			i,
 182			be16_to_cpu(pddcb->retc_16),
 183			be16_to_cpu(pddcb->seqnum_16),
 184			pddcb->hsi,
 185			pddcb->shi,
 186			be64_to_cpu(pddcb->priv_64),
 187			pddcb->cmd);
 188		pddcb++;
 189	}
 190	spin_unlock_irqrestore(&cd->print_lock, flags);
 191}
 192
 193struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
 194{
 195	struct ddcb_requ *req;
 196
 197	req = kzalloc(sizeof(*req), GFP_KERNEL);
 198	if (!req)
 199		return NULL;
 200
 201	return &req->cmd;
 202}
 203
 204void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
 205{
 206	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 207
 208	kfree(req);
 209}
 210
 211static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
 212{
 213	return req->req_state;
 214}
 215
 216static inline void ddcb_requ_set_state(struct ddcb_requ *req,
 217				       enum genwqe_requ_state new_state)
 218{
 219	req->req_state = new_state;
 220}
 221
 222static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
 223{
 224	return req->cmd.ddata_addr != 0x0;
 225}
 226
 227/**
 228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
 229 * @cd:          pointer to genwqe device descriptor
 230 * @req:         DDCB work request
 231 *
 232 * Status of ddcb_requ mirrors this hardware state, but is copied in
 233 * the ddcb_requ on interrupt/polling function. The lowlevel code
 234 * should check the hardware state directly, the higher level code
 235 * should check the copy.
 236 *
 237 * This function will also return true if the state of the queue is
 238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
 239 * shutdown case.
 240 */
 241static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
 242{
 243	return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
 244		(cd->card_state != GENWQE_CARD_USED);
 245}
 246
 247#define RET_DDCB_APPENDED 1
 248#define RET_DDCB_TAPPED   2
 249/**
 250 * enqueue_ddcb() - Enqueue a DDCB
 251 * @cd:         pointer to genwqe device descriptor
 252 * @queue:	queue this operation should be done on
 253 * @pddcb:      pointer to ddcb structure
 254 * @ddcb_no:    pointer to ddcb number being tapped
 255 *
 256 * Start execution of DDCB by tapping or append to queue via NEXT
 257 * bit. This is done by an atomic 'compare and swap' instruction and
 258 * checking SHI and HSI of the previous DDCB.
 259 *
 260 * This function must only be called with ddcb_lock held.
 261 *
 262 * Return: 1 if new DDCB is appended to previous
 263 *         2 if DDCB queue is tapped via register/simulation
 264 */
 265static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
 266			struct ddcb *pddcb, int ddcb_no)
 267{
 268	unsigned int try;
 269	int prev_no;
 270	struct ddcb *prev_ddcb;
 271	__be32 old, new, icrc_hsi_shi;
 272	u64 num;
 273
 274	/*
 275	 * For performance checks a Dispatch Timestamp can be put into
 276	 * DDCB It is supposed to use the SLU's free running counter,
 277	 * but this requires PCIe cycles.
 278	 */
 279	ddcb_mark_unused(pddcb);
 280
 281	/* check previous DDCB if already fetched */
 282	prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
 283	prev_ddcb = &queue->ddcb_vaddr[prev_no];
 284
 285	/*
 286	 * It might have happened that the HSI.FETCHED bit is
 287	 * set. Retry in this case. Therefore I expect maximum 2 times
 288	 * trying.
 289	 */
 290	ddcb_mark_appended(pddcb);
 291	for (try = 0; try < 2; try++) {
 292		old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
 293
 294		/* try to append via NEXT bit if prev DDCB is not completed */
 295		if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
 296			break;
 297
 298		new = (old | DDCB_NEXT_BE32);
 299
 300		wmb();		/* need to ensure write ordering */
 301		icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
 302
 303		if (icrc_hsi_shi == old)
 304			return RET_DDCB_APPENDED; /* appended to queue */
 305	}
 306
 307	/* Queue must be re-started by updating QUEUE_OFFSET */
 308	ddcb_mark_tapped(pddcb);
 309	num = (u64)ddcb_no << 8;
 310
 311	wmb();			/* need to ensure write ordering */
 312	__genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
 313
 314	return RET_DDCB_TAPPED;
 315}
 316
 317/**
 318 * copy_ddcb_results() - Copy output state from real DDCB to request
 319 * @req:        pointer to requested DDCB parameters
 320 * @ddcb_no:    pointer to ddcb number being tapped
 321 *
 322 * Copy DDCB ASV to request struct. There is no endian
 323 * conversion made, since data structure in ASV is still
 324 * unknown here.
 325 *
 326 * This is needed by:
 327 *   - genwqe_purge_ddcb()
 328 *   - genwqe_check_ddcb_queue()
 329 */
 330static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
 331{
 332	struct ddcb_queue *queue = req->queue;
 333	struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
 334
 335	memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
 336
 337	/* copy status flags of the variant part */
 338	req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
 339	req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
 340	req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
 341
 342	req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
 343	req->cmd.progress = be32_to_cpu(pddcb->progress_32);
 344	req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
 345
 346	if (ddcb_requ_collect_debug_data(req)) {
 347		int prev_no = (ddcb_no == 0) ?
 348			queue->ddcb_max - 1 : ddcb_no - 1;
 349		struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
 350
 351		memcpy(&req->debug_data.ddcb_finished, pddcb,
 352		       sizeof(req->debug_data.ddcb_finished));
 353		memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
 354		       sizeof(req->debug_data.ddcb_prev));
 355	}
 356}
 357
 358/**
 359 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests.
 360 * @cd:         pointer to genwqe device descriptor
 361 * @queue:	queue to be checked
 362 *
 363 * Return: Number of DDCBs which were finished
 364 */
 365static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
 366				   struct ddcb_queue *queue)
 367{
 368	unsigned long flags;
 369	int ddcbs_finished = 0;
 370	struct pci_dev *pci_dev = cd->pci_dev;
 371
 372	spin_lock_irqsave(&queue->ddcb_lock, flags);
 373
 374	/* FIXME avoid soft locking CPU */
 375	while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
 376
 377		struct ddcb *pddcb;
 378		struct ddcb_requ *req;
 379		u16 vcrc, vcrc_16, retc_16;
 380
 381		pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 382
 383		if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
 384		    0x00000000)
 385			goto go_home; /* not completed, continue waiting */
 386
 387		wmb();  /*  Add sync to decouple prev. read operations */
 388
 389		/* Note: DDCB could be purged */
 390		req = queue->ddcb_req[queue->ddcb_act];
 391		if (req == NULL) {
 392			/* this occurs if DDCB is purged, not an error */
 393			/* Move active DDCB further; Nothing to do anymore. */
 394			goto pick_next_one;
 395		}
 396
 397		/*
 398		 * HSI=0x44 (fetched and completed), but RETC is
 399		 * 0x101, or even worse 0x000.
 400		 *
 401		 * In case of seeing the queue in inconsistent state
 402		 * we read the errcnts and the queue status to provide
 403		 * a trigger for our PCIe analyzer stop capturing.
 404		 */
 405		retc_16 = be16_to_cpu(pddcb->retc_16);
 406		if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
 407			u64 errcnts, status;
 408			u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
 409
 410			errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
 411			status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 412
 413			dev_err(&pci_dev->dev,
 414				"[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
 415				__func__, be16_to_cpu(pddcb->seqnum_16),
 416				pddcb->hsi, retc_16, errcnts, status,
 417				queue->ddcb_daddr + ddcb_offs);
 418		}
 419
 420		copy_ddcb_results(req, queue->ddcb_act);
 421		queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
 422
 423		dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
 424		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 425
 426		ddcb_mark_finished(pddcb);
 427
 428		/* calculate CRC_16 to see if VCRC is correct */
 429		vcrc = genwqe_crc16(pddcb->asv,
 430				   VCRC_LENGTH(req->cmd.asv_length),
 431				   0xffff);
 432		vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
 433		if (vcrc != vcrc_16) {
 434			printk_ratelimited(KERN_ERR
 435				"%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
 436				GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 437				pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
 438				vcrc, vcrc_16);
 439		}
 440
 441		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 442		queue->ddcbs_completed++;
 443		queue->ddcbs_in_flight--;
 444
 445		/* wake up process waiting for this DDCB, and
 446                   processes on the busy queue */
 447		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
 448		wake_up_interruptible(&queue->busy_waitq);
 449
 450pick_next_one:
 451		queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
 452		ddcbs_finished++;
 453	}
 454
 455 go_home:
 456	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 457	return ddcbs_finished;
 458}
 459
 460/**
 461 * __genwqe_wait_ddcb(): Waits until DDCB is completed
 462 * @cd:         pointer to genwqe device descriptor
 463 * @req:        pointer to requsted DDCB parameters
 464 *
 465 * The Service Layer will update the RETC in DDCB when processing is
 466 * pending or done.
 467 *
 468 * Return: > 0 remaining jiffies, DDCB completed
 469 *           -ETIMEDOUT	when timeout
 470 *           -ERESTARTSYS when ^C
 471 *           -EINVAL when unknown error condition
 472 *
 473 * When an error is returned the called needs to ensure that
 474 * purge_ddcb() is being called to get the &req removed from the
 475 * queue.
 476 */
 477int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 478{
 479	int rc;
 480	unsigned int ddcb_no;
 481	struct ddcb_queue *queue;
 482	struct pci_dev *pci_dev = cd->pci_dev;
 483
 484	if (req == NULL)
 485		return -EINVAL;
 486
 487	queue = req->queue;
 488	if (queue == NULL)
 489		return -EINVAL;
 490
 491	ddcb_no = req->num;
 492	if (ddcb_no >= queue->ddcb_max)
 493		return -EINVAL;
 494
 495	rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
 496				ddcb_requ_finished(cd, req),
 497				GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
 498
 499	/*
 500	 * We need to distinguish 3 cases here:
 501	 *   1. rc == 0              timeout occurred
 502	 *   2. rc == -ERESTARTSYS   signal received
 503	 *   3. rc > 0               remaining jiffies condition is true
 504	 */
 505	if (rc == 0) {
 506		struct ddcb_queue *queue = req->queue;
 507		struct ddcb *pddcb;
 508
 509		/*
 510		 * Timeout may be caused by long task switching time.
 511		 * When timeout happens, check if the request has
 512		 * meanwhile completed.
 513		 */
 514		genwqe_check_ddcb_queue(cd, req->queue);
 515		if (ddcb_requ_finished(cd, req))
 516			return rc;
 517
 518		dev_err(&pci_dev->dev,
 519			"[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
 520			__func__, req->num, rc,	ddcb_requ_get_state(req),
 521			req);
 522		dev_err(&pci_dev->dev,
 523			"[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
 524			__genwqe_readq(cd, queue->IO_QUEUE_STATUS));
 525
 526		pddcb = &queue->ddcb_vaddr[req->num];
 527		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 528
 529		print_ddcb_info(cd, req->queue);
 530		return -ETIMEDOUT;
 531
 532	} else if (rc == -ERESTARTSYS) {
 533		return rc;
 534		/*
 535		 * EINTR:       Stops the application
 536		 * ERESTARTSYS: Restartable systemcall; called again
 537		 */
 538
 539	} else if (rc < 0) {
 540		dev_err(&pci_dev->dev,
 541			"[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
 542			__func__, req->num, rc, ddcb_requ_get_state(req));
 543		return -EINVAL;
 544	}
 545
 546	/* Severe error occured. Driver is forced to stop operation */
 547	if (cd->card_state != GENWQE_CARD_USED) {
 548		dev_err(&pci_dev->dev,
 549			"[%s] err: DDCB#%d forced to stop (rc=%d)\n",
 550			__func__, req->num, rc);
 551		return -EIO;
 552	}
 553	return rc;
 554}
 555
 556/**
 557 * get_next_ddcb() - Get next available DDCB
 558 * @cd:         pointer to genwqe device descriptor
 559 * @queue:      DDCB queue
 560 * @num:        internal DDCB number
 561 *
 562 * DDCB's content is completely cleared but presets for PRE and
 563 * SEQNUM. This function must only be called when ddcb_lock is held.
 564 *
 565 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
 566 */
 567static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
 568				  struct ddcb_queue *queue,
 569				  int *num)
 570{
 571	u64 *pu64;
 572	struct ddcb *pddcb;
 573
 574	if (queue_free_ddcbs(queue) == 0) /* queue is  full */
 575		return NULL;
 576
 577	/* find new ddcb */
 578	pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
 579
 580	/* if it is not completed, we are not allowed to use it */
 581	/* barrier(); */
 582	if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
 583		return NULL;
 584
 585	*num = queue->ddcb_next;	/* internal DDCB number */
 586	queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
 587
 588	/* clear important DDCB fields */
 589	pu64 = (u64 *)pddcb;
 590	pu64[0] = 0ULL;		/* offs 0x00 (ICRC,HSI,SHI,...) */
 591	pu64[1] = 0ULL;		/* offs 0x01 (ACFUNC,CMD...) */
 592
 593	/* destroy previous results in ASV */
 594	pu64[0x80/8] = 0ULL;	/* offs 0x80 (ASV + 0) */
 595	pu64[0x88/8] = 0ULL;	/* offs 0x88 (ASV + 0x08) */
 596	pu64[0x90/8] = 0ULL;	/* offs 0x90 (ASV + 0x10) */
 597	pu64[0x98/8] = 0ULL;	/* offs 0x98 (ASV + 0x18) */
 598	pu64[0xd0/8] = 0ULL;	/* offs 0xd0 (RETC,ATTN...) */
 599
 600	pddcb->pre = DDCB_PRESET_PRE; /* 128 */
 601	pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
 602	return pddcb;
 603}
 604
 605/**
 606 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
 607 * @cd:         genwqe device descriptor
 608 * @req:        DDCB request
 609 *
 610 * This will fail when the request was already FETCHED. In this case
 611 * we need to wait until it is finished. Else the DDCB can be
 612 * reused. This function also ensures that the request data structure
 613 * is removed from ddcb_req[].
 614 *
 615 * Do not forget to call this function when genwqe_wait_ddcb() fails,
 616 * such that the request gets really removed from ddcb_req[].
 617 *
 618 * Return: 0 success
 619 */
 620int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 621{
 622	struct ddcb *pddcb = NULL;
 623	unsigned int t;
 624	unsigned long flags;
 625	struct ddcb_queue *queue = req->queue;
 626	struct pci_dev *pci_dev = cd->pci_dev;
 627	u64 queue_status;
 628	__be32 icrc_hsi_shi = 0x0000;
 629	__be32 old, new;
 630
 631	/* unsigned long flags; */
 632	if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
 633		dev_err(&pci_dev->dev,
 634			"[%s] err: software timeout is not set!\n", __func__);
 635		return -EFAULT;
 636	}
 637
 638	pddcb = &queue->ddcb_vaddr[req->num];
 639
 640	for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
 641
 642		spin_lock_irqsave(&queue->ddcb_lock, flags);
 643
 644		/* Check if req was meanwhile finished */
 645		if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
 646			goto go_home;
 647
 648		/* try to set PURGE bit if FETCHED/COMPLETED are not set */
 649		old = pddcb->icrc_hsi_shi_32;	/* read SHI/HSI in BE32 */
 650		if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
 651
 652			new = (old | DDCB_PURGE_BE32);
 653			icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
 654					       old, new);
 655			if (icrc_hsi_shi == old)
 656				goto finish_ddcb;
 657		}
 658
 659		/* normal finish with HSI bit */
 660		barrier();
 661		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 662		if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
 663			goto finish_ddcb;
 664
 665		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 666
 667		/*
 668		 * Here the check_ddcb() function will most likely
 669		 * discover this DDCB to be finished some point in
 670		 * time. It will mark the req finished and free it up
 671		 * in the list.
 672		 */
 673
 674		copy_ddcb_results(req, req->num); /* for the failing case */
 675		msleep(100); /* sleep for 1/10 second and try again */
 676		continue;
 677
 678finish_ddcb:
 679		copy_ddcb_results(req, req->num);
 680		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 681		queue->ddcbs_in_flight--;
 682		queue->ddcb_req[req->num] = NULL; /* delete from array */
 683		ddcb_mark_cleared(pddcb);
 684
 685		/* Move active DDCB further; Nothing to do here anymore. */
 686
 687		/*
 688		 * We need to ensure that there is at least one free
 689		 * DDCB in the queue. To do that, we must update
 690		 * ddcb_act only if the COMPLETED bit is set for the
 691		 * DDCB we are working on else we treat that DDCB even
 692		 * if we PURGED it as occupied (hardware is supposed
 693		 * to set the COMPLETED bit yet!).
 694		 */
 695		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 696		if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
 697		    (queue->ddcb_act == req->num)) {
 698			queue->ddcb_act = ((queue->ddcb_act + 1) %
 699					   queue->ddcb_max);
 700		}
 701go_home:
 702		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 703		return 0;
 704	}
 705
 706	/*
 707	 * If the card is dead and the queue is forced to stop, we
 708	 * might see this in the queue status register.
 709	 */
 710	queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 711
 712	dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
 713	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 714
 715	dev_err(&pci_dev->dev,
 716		"[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
 717		__func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
 718		queue_status);
 719
 720	print_ddcb_info(cd, req->queue);
 721
 722	return -EFAULT;
 723}
 724
 725int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
 726{
 727	int len;
 728	struct pci_dev *pci_dev = cd->pci_dev;
 729
 730	if (d == NULL) {
 731		dev_err(&pci_dev->dev,
 732			"[%s] err: invalid memory for debug data!\n",
 733			__func__);
 734		return -EFAULT;
 735	}
 736
 737	len  = sizeof(d->driver_version);
 738	snprintf(d->driver_version, len, "%s", DRV_VERSION);
 739	d->slu_unitcfg = cd->slu_unitcfg;
 740	d->app_unitcfg = cd->app_unitcfg;
 741	return 0;
 742}
 743
 744/**
 745 * __genwqe_enqueue_ddcb() - Enqueue a DDCB
 746 * @cd:         pointer to genwqe device descriptor
 747 * @req:        pointer to DDCB execution request
 748 * @f_flags:    file mode: blocking, non-blocking
 749 *
 750 * Return: 0 if enqueuing succeeded
 751 *         -EIO if card is unusable/PCIe problems
 752 *         -EBUSY if enqueuing failed
 753 */
 754int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
 755			  unsigned int f_flags)
 756{
 757	struct ddcb *pddcb;
 758	unsigned long flags;
 759	struct ddcb_queue *queue;
 760	struct pci_dev *pci_dev = cd->pci_dev;
 761	u16 icrc;
 762
 763 retry:
 764	if (cd->card_state != GENWQE_CARD_USED) {
 765		printk_ratelimited(KERN_ERR
 766			"%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
 767			GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 768			__func__, req->num);
 769		return -EIO;
 770	}
 771
 772	queue = req->queue = &cd->queue;
 773
 774	/* FIXME circumvention to improve performance when no irq is
 775	 * there.
 776	 */
 777	if (GENWQE_POLLING_ENABLED)
 778		genwqe_check_ddcb_queue(cd, queue);
 779
 780	/*
 781	 * It must be ensured to process all DDCBs in successive
 782	 * order. Use a lock here in order to prevent nested DDCB
 783	 * enqueuing.
 784	 */
 785	spin_lock_irqsave(&queue->ddcb_lock, flags);
 786
 787	pddcb = get_next_ddcb(cd, queue, &req->num);	/* get ptr and num */
 788	if (pddcb == NULL) {
 789		int rc;
 790
 791		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 792
 793		if (f_flags & O_NONBLOCK) {
 794			queue->return_on_busy++;
 795			return -EBUSY;
 796		}
 797
 798		queue->wait_on_busy++;
 799		rc = wait_event_interruptible(queue->busy_waitq,
 800					      queue_free_ddcbs(queue) != 0);
 801		dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
 802			__func__, rc);
 803		if (rc == -ERESTARTSYS)
 804			return rc;  /* interrupted by a signal */
 805
 806		goto retry;
 807	}
 808
 809	if (queue->ddcb_req[req->num] != NULL) {
 810		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 811
 812		dev_err(&pci_dev->dev,
 813			"[%s] picked DDCB %d with req=%p still in use!!\n",
 814			__func__, req->num, req);
 815		return -EFAULT;
 816	}
 817	ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
 818	queue->ddcb_req[req->num] = req;
 819
 820	pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
 821	pddcb->cmd = req->cmd.cmd;
 822	pddcb->acfunc = req->cmd.acfunc;	/* functional unit */
 823
 824	/*
 825	 * We know that we can get retc 0x104 with CRC error, do not
 826	 * stop the queue in those cases for this command. XDIR = 1
 827	 * does not work for old SLU versions.
 828	 *
 829	 * Last bitstream with the old XDIR behavior had SLU_ID
 830	 * 0x34199.
 831	 */
 832	if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
 833		pddcb->xdir = 0x1;
 834	else
 835		pddcb->xdir = 0x0;
 836
 837
 838	pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
 839		      ((req->cmd.asv_length  / 8)));
 840	pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
 841
 842	/*
 843	 * If copying the whole DDCB_ASIV_LENGTH is impacting
 844	 * performance we need to change it to
 845	 * req->cmd.asiv_length. But simulation benefits from some
 846	 * non-architectured bits behind the architectured content.
 847	 *
 848	 * How much data is copied depends on the availability of the
 849	 * ATS field, which was introduced late. If the ATS field is
 850	 * supported ASIV is 8 bytes shorter than it used to be. Since
 851	 * the ATS field is copied too, the code should do exactly
 852	 * what it did before, but I wanted to make copying of the ATS
 853	 * field very explicit.
 854	 */
 855	if (genwqe_get_slu_id(cd) <= 0x2) {
 856		memcpy(&pddcb->__asiv[0],	/* destination */
 857		       &req->cmd.__asiv[0],	/* source */
 858		       DDCB_ASIV_LENGTH);	/* req->cmd.asiv_length */
 859	} else {
 860		pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
 861		memcpy(&pddcb->n.asiv[0],	/* destination */
 862			&req->cmd.asiv[0],	/* source */
 863			DDCB_ASIV_LENGTH_ATS);	/* req->cmd.asiv_length */
 864	}
 865
 866	pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
 867
 868	/*
 869	 * Calculate CRC_16 for corresponding range PSP(7:4). Include
 870	 * empty 4 bytes prior to the data.
 871	 */
 872	icrc = genwqe_crc16((const u8 *)pddcb,
 873			   ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
 874	pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
 875
 876	/* enable DDCB completion irq */
 877	if (!GENWQE_POLLING_ENABLED)
 878		pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
 879
 880	dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
 881	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 882
 883	if (ddcb_requ_collect_debug_data(req)) {
 884		/* use the kernel copy of debug data. copying back to
 885		   user buffer happens later */
 886
 887		genwqe_init_debug_data(cd, &req->debug_data);
 888		memcpy(&req->debug_data.ddcb_before, pddcb,
 889		       sizeof(req->debug_data.ddcb_before));
 890	}
 891
 892	enqueue_ddcb(cd, queue, pddcb, req->num);
 893	queue->ddcbs_in_flight++;
 894
 895	if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
 896		queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
 897
 898	ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
 899	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 900	wake_up_interruptible(&cd->queue_waitq);
 901
 902	return 0;
 903}
 904
 905/**
 906 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
 907 * @cd:         pointer to genwqe device descriptor
 908 * @cmd:        user provided DDCB command
 909 * @f_flags:    file mode: blocking, non-blocking
 910 */
 911int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
 912			      struct genwqe_ddcb_cmd *cmd,
 913			      unsigned int f_flags)
 914{
 915	int rc = 0;
 916	struct pci_dev *pci_dev = cd->pci_dev;
 917	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 918
 919	if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
 920		dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
 921			__func__, cmd->asiv_length);
 922		return -EINVAL;
 923	}
 924	if (cmd->asv_length > DDCB_ASV_LENGTH) {
 925		dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
 926			__func__, cmd->asv_length);
 927		return -EINVAL;
 928	}
 929	rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
 930	if (rc != 0)
 931		return rc;
 932
 933	rc = __genwqe_wait_ddcb(cd, req);
 934	if (rc < 0)		/* error or signal interrupt */
 935		goto err_exit;
 936
 937	if (ddcb_requ_collect_debug_data(req)) {
 938		if (copy_to_user((struct genwqe_debug_data __user *)
 939				 (unsigned long)cmd->ddata_addr,
 940				 &req->debug_data,
 941				 sizeof(struct genwqe_debug_data)))
 942			return -EFAULT;
 943	}
 944
 945	/*
 946	 * Higher values than 0x102 indicate completion with faults,
 947	 * lower values than 0x102 indicate processing faults. Note
 948	 * that DDCB might have been purged. E.g. Cntl+C.
 949	 */
 950	if (cmd->retc != DDCB_RETC_COMPLETE) {
 951		/* This might happen e.g. flash read, and needs to be
 952		   handled by the upper layer code. */
 953		rc = -EBADMSG;	/* not processed/error retc */
 954	}
 955
 956	return rc;
 957
 958 err_exit:
 959	__genwqe_purge_ddcb(cd, req);
 960
 961	if (ddcb_requ_collect_debug_data(req)) {
 962		if (copy_to_user((struct genwqe_debug_data __user *)
 963				 (unsigned long)cmd->ddata_addr,
 964				 &req->debug_data,
 965				 sizeof(struct genwqe_debug_data)))
 966			return -EFAULT;
 967	}
 968	return rc;
 969}
 970
 971/**
 972 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
 973 * @cd:         pointer to genwqe device descriptor
 974 *
 975 * We use this as condition for our wait-queue code.
 976 */
 977static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
 978{
 979	unsigned long flags;
 980	struct ddcb *pddcb;
 981	struct ddcb_queue *queue = &cd->queue;
 982
 983	spin_lock_irqsave(&queue->ddcb_lock, flags);
 984
 985	if (queue_empty(queue)) { /* empty queue */
 986		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 987		return 0;
 988	}
 989
 990	pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 991	if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
 992		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 993		return 1;
 994	}
 995
 996	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 997	return 0;
 998}
 999
1000/**
1001 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1002 * @cd:         pointer to genwqe device descriptor
1003 *
1004 * Keep track on the number of DDCBs which ware currently in the
1005 * queue. This is needed for statistics as well as condition if we want
1006 * to wait or better do polling in case of no interrupts available.
1007 */
1008int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1009{
1010	unsigned long flags;
1011	int ddcbs_in_flight = 0;
1012	struct ddcb_queue *queue = &cd->queue;
1013
1014	spin_lock_irqsave(&queue->ddcb_lock, flags);
1015	ddcbs_in_flight += queue->ddcbs_in_flight;
1016	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1017
1018	return ddcbs_in_flight;
1019}
1020
1021static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1022{
1023	int rc, i;
1024	struct ddcb *pddcb;
1025	u64 val64;
1026	unsigned int queue_size;
1027	struct pci_dev *pci_dev = cd->pci_dev;
1028
1029	if (GENWQE_DDCB_MAX < 2)
1030		return -EINVAL;
1031
1032	queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1033
1034	queue->ddcbs_in_flight = 0;  /* statistics */
1035	queue->ddcbs_max_in_flight = 0;
1036	queue->ddcbs_completed = 0;
1037	queue->return_on_busy = 0;
1038	queue->wait_on_busy = 0;
1039
1040	queue->ddcb_seq	  = 0x100; /* start sequence number */
1041	queue->ddcb_max	  = GENWQE_DDCB_MAX;
1042	queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1043						&queue->ddcb_daddr);
1044	if (queue->ddcb_vaddr == NULL) {
1045		dev_err(&pci_dev->dev,
1046			"[%s] **err: could not allocate DDCB **\n", __func__);
1047		return -ENOMEM;
1048	}
1049	queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1050				  GFP_KERNEL);
1051	if (!queue->ddcb_req) {
1052		rc = -ENOMEM;
1053		goto free_ddcbs;
1054	}
1055
1056	queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1057				     sizeof(wait_queue_head_t),
1058				     GFP_KERNEL);
1059	if (!queue->ddcb_waitqs) {
1060		rc = -ENOMEM;
1061		goto free_requs;
1062	}
1063
1064	for (i = 0; i < queue->ddcb_max; i++) {
1065		pddcb = &queue->ddcb_vaddr[i];		     /* DDCBs */
1066		pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1067		pddcb->retc_16 = cpu_to_be16(0xfff);
1068
1069		queue->ddcb_req[i] = NULL;		     /* requests */
1070		init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1071	}
1072
1073	queue->ddcb_act  = 0;
1074	queue->ddcb_next = 0;	/* queue is empty */
1075
1076	spin_lock_init(&queue->ddcb_lock);
1077	init_waitqueue_head(&queue->busy_waitq);
1078
1079	val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1080	__genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1081	__genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1082	__genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1083	__genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1084	return 0;
1085
1086 free_requs:
1087	kfree(queue->ddcb_req);
1088	queue->ddcb_req = NULL;
1089 free_ddcbs:
1090	__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1091				queue->ddcb_daddr);
1092	queue->ddcb_vaddr = NULL;
1093	queue->ddcb_daddr = 0ull;
1094	return rc;
1095
1096}
1097
1098static int ddcb_queue_initialized(struct ddcb_queue *queue)
1099{
1100	return queue->ddcb_vaddr != NULL;
1101}
1102
1103static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1104{
1105	unsigned int queue_size;
1106
1107	queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1108
1109	kfree(queue->ddcb_req);
1110	queue->ddcb_req = NULL;
1111
1112	if (queue->ddcb_vaddr) {
1113		__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1114					queue->ddcb_daddr);
1115		queue->ddcb_vaddr = NULL;
1116		queue->ddcb_daddr = 0ull;
1117	}
1118}
1119
1120static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1121{
1122	u64 gfir;
1123	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1124	struct pci_dev *pci_dev = cd->pci_dev;
1125
1126	/*
1127	 * In case of fatal FIR error the queue is stopped, such that
1128	 * we can safely check it without risking anything.
1129	 */
1130	cd->irqs_processed++;
1131	wake_up_interruptible(&cd->queue_waitq);
1132
1133	/*
1134	 * Checking for errors before kicking the queue might be
1135	 * safer, but slower for the good-case ... See above.
1136	 */
1137	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1138	if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1139	    !pci_channel_offline(pci_dev)) {
1140
1141		if (cd->use_platform_recovery) {
1142			/*
1143			 * Since we use raw accessors, EEH errors won't be
1144			 * detected by the platform until we do a non-raw
1145			 * MMIO or config space read
1146			 */
1147			readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1148
1149			/* Don't do anything if the PCI channel is frozen */
1150			if (pci_channel_offline(pci_dev))
1151				goto exit;
1152		}
1153
1154		wake_up_interruptible(&cd->health_waitq);
1155
1156		/*
1157		 * By default GFIRs causes recovery actions. This
1158		 * count is just for debug when recovery is masked.
1159		 */
1160		dev_err_ratelimited(&pci_dev->dev,
1161				    "[%s] GFIR=%016llx\n",
1162				    __func__, gfir);
1163	}
1164
1165 exit:
1166	return IRQ_HANDLED;
1167}
1168
1169static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1170{
1171	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1172
1173	cd->irqs_processed++;
1174	wake_up_interruptible(&cd->queue_waitq);
1175
1176	return IRQ_HANDLED;
1177}
1178
1179/**
1180 * genwqe_card_thread() - Work thread for the DDCB queue
1181 * @data:         pointer to genwqe device descriptor
1182 *
1183 * The idea is to check if there are DDCBs in processing. If there are
1184 * some finished DDCBs, we process them and wakeup the
1185 * requestors. Otherwise we give other processes time using
1186 * cond_resched().
1187 */
1188static int genwqe_card_thread(void *data)
1189{
1190	int should_stop = 0;
1191	struct genwqe_dev *cd = (struct genwqe_dev *)data;
1192
1193	while (!kthread_should_stop()) {
1194
1195		genwqe_check_ddcb_queue(cd, &cd->queue);
1196
1197		if (GENWQE_POLLING_ENABLED) {
1198			wait_event_interruptible_timeout(
1199				cd->queue_waitq,
1200				genwqe_ddcbs_in_flight(cd) ||
1201				(should_stop = kthread_should_stop()), 1);
1202		} else {
1203			wait_event_interruptible_timeout(
1204				cd->queue_waitq,
1205				genwqe_next_ddcb_ready(cd) ||
1206				(should_stop = kthread_should_stop()), HZ);
1207		}
1208		if (should_stop)
1209			break;
1210
1211		/*
1212		 * Avoid soft lockups on heavy loads; we do not want
1213		 * to disable our interrupts.
1214		 */
1215		cond_resched();
1216	}
1217	return 0;
1218}
1219
1220/**
1221 * genwqe_setup_service_layer() - Setup DDCB queue
1222 * @cd:         pointer to genwqe device descriptor
1223 *
1224 * Allocate DDCBs. Configure Service Layer Controller (SLC).
1225 *
1226 * Return: 0 success
1227 */
1228int genwqe_setup_service_layer(struct genwqe_dev *cd)
1229{
1230	int rc;
1231	struct ddcb_queue *queue;
1232	struct pci_dev *pci_dev = cd->pci_dev;
1233
1234	if (genwqe_is_privileged(cd)) {
1235		rc = genwqe_card_reset(cd);
1236		if (rc < 0) {
1237			dev_err(&pci_dev->dev,
1238				"[%s] err: reset failed.\n", __func__);
1239			return rc;
1240		}
1241		genwqe_read_softreset(cd);
1242	}
1243
1244	queue = &cd->queue;
1245	queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1246	queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1247	queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1248	queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1249	queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1250	queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1251	queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1252	queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1253	queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1254
1255	rc = setup_ddcb_queue(cd, queue);
1256	if (rc != 0) {
1257		rc = -ENODEV;
1258		goto err_out;
1259	}
1260
1261	init_waitqueue_head(&cd->queue_waitq);
1262	cd->card_thread = kthread_run(genwqe_card_thread, cd,
1263				      GENWQE_DEVNAME "%d_thread",
1264				      cd->card_idx);
1265	if (IS_ERR(cd->card_thread)) {
1266		rc = PTR_ERR(cd->card_thread);
1267		cd->card_thread = NULL;
1268		goto stop_free_queue;
1269	}
1270
1271	rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1272	if (rc)
1273		goto stop_kthread;
1274
1275	/*
1276	 * We must have all wait-queues initialized when we enable the
1277	 * interrupts. Otherwise we might crash if we get an early
1278	 * irq.
1279	 */
1280	init_waitqueue_head(&cd->health_waitq);
1281
1282	if (genwqe_is_privileged(cd)) {
1283		rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1284				 GENWQE_DEVNAME, cd);
1285	} else {
1286		rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1287				 GENWQE_DEVNAME, cd);
1288	}
1289	if (rc < 0) {
1290		dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1291		goto stop_irq_cap;
1292	}
1293
1294	cd->card_state = GENWQE_CARD_USED;
1295	return 0;
1296
1297 stop_irq_cap:
1298	genwqe_reset_interrupt_capability(cd);
1299 stop_kthread:
1300	kthread_stop(cd->card_thread);
1301	cd->card_thread = NULL;
1302 stop_free_queue:
1303	free_ddcb_queue(cd, queue);
1304 err_out:
1305	return rc;
1306}
1307
1308/**
1309 * queue_wake_up_all() - Handles fatal error case
1310 * @cd:         pointer to genwqe device descriptor
1311 *
1312 * The PCI device got unusable and we have to stop all pending
1313 * requests as fast as we can. The code after this must purge the
1314 * DDCBs in question and ensure that all mappings are freed.
1315 */
1316static int queue_wake_up_all(struct genwqe_dev *cd)
1317{
1318	unsigned int i;
1319	unsigned long flags;
1320	struct ddcb_queue *queue = &cd->queue;
1321
1322	spin_lock_irqsave(&queue->ddcb_lock, flags);
1323
1324	for (i = 0; i < queue->ddcb_max; i++)
1325		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1326
1327	wake_up_interruptible(&queue->busy_waitq);
1328	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1329
1330	return 0;
1331}
1332
1333/**
1334 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1335 * @cd:         pointer to genwqe device descriptor
1336 *
1337 * Relies on the pre-condition that there are no users of the card
1338 * device anymore e.g. with open file-descriptors.
1339 *
1340 * This function must be robust enough to be called twice.
1341 */
1342int genwqe_finish_queue(struct genwqe_dev *cd)
1343{
1344	int i, rc = 0, in_flight;
1345	int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1346	struct pci_dev *pci_dev = cd->pci_dev;
1347	struct ddcb_queue *queue = &cd->queue;
1348
1349	if (!ddcb_queue_initialized(queue))
1350		return 0;
1351
1352	/* Do not wipe out the error state. */
1353	if (cd->card_state == GENWQE_CARD_USED)
1354		cd->card_state = GENWQE_CARD_UNUSED;
1355
1356	/* Wake up all requests in the DDCB queue such that they
1357	   should be removed nicely. */
1358	queue_wake_up_all(cd);
1359
1360	/* We must wait to get rid of the DDCBs in flight */
1361	for (i = 0; i < waitmax; i++) {
1362		in_flight = genwqe_ddcbs_in_flight(cd);
1363
1364		if (in_flight == 0)
1365			break;
1366
1367		dev_dbg(&pci_dev->dev,
1368			"  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1369			i, waitmax, in_flight);
1370
1371		/*
1372		 * Severe severe error situation: The card itself has
1373		 * 16 DDCB queues, each queue has e.g. 32 entries,
1374		 * each DDBC has a hardware timeout of currently 250
1375		 * msec but the PFs have a hardware timeout of 8 sec
1376		 * ... so I take something large.
1377		 */
1378		msleep(1000);
1379	}
1380	if (i == waitmax) {
1381		dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1382			__func__);
1383		rc = -EIO;
1384	}
1385	return rc;
1386}
1387
1388/**
1389 * genwqe_release_service_layer() - Shutdown DDCB queue
1390 * @cd:       genwqe device descriptor
1391 *
1392 * This function must be robust enough to be called twice.
1393 */
1394int genwqe_release_service_layer(struct genwqe_dev *cd)
1395{
1396	struct pci_dev *pci_dev = cd->pci_dev;
1397
1398	if (!ddcb_queue_initialized(&cd->queue))
1399		return 1;
1400
1401	free_irq(pci_dev->irq, cd);
1402	genwqe_reset_interrupt_capability(cd);
1403
1404	if (cd->card_thread != NULL) {
1405		kthread_stop(cd->card_thread);
1406		cd->card_thread = NULL;
1407	}
1408
1409	free_ddcb_queue(cd, &cd->queue);
1410	return 0;
1411}