qdio_main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Linux for s390 qdio support, buffer handling, qdio API and module support.
   4 *
   5 * Copyright IBM Corp. 2000, 2008
   6 * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
   7 *	      Jan Glauber <jang@linux.vnet.ibm.com>
   8 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
   9 */
  10
  11#include <linux/export.h>
  12#include <linux/module.h>
  13#include <linux/init.h>
  14#include <linux/kernel.h>
  15#include <linux/kmemleak.h>
  16#include <linux/delay.h>
  17#include <linux/gfp.h>
  18#include <linux/io.h>
  19#include <linux/atomic.h>
  20#include <asm/debug.h>
  21#include <asm/qdio.h>
  22#include <asm/asm.h>
  23#include <asm/ipl.h>
  24
  25#include "cio.h"
  26#include "css.h"
  27#include "device.h"
  28#include "qdio.h"
  29#include "qdio_debug.h"
  30
  31MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
  32	"Jan Glauber <jang@linux.vnet.ibm.com>");
  33MODULE_DESCRIPTION("QDIO base support");
  34MODULE_LICENSE("GPL");
  35
  36static inline int do_siga_sync(unsigned long schid,
  37			       unsigned long out_mask, unsigned long in_mask,
  38			       unsigned int fc)
  39{
  40	int cc;
  41
  42	asm volatile(
  43		"	lgr	0,%[fc]\n"
  44		"	lgr	1,%[schid]\n"
  45		"	lgr	2,%[out]\n"
  46		"	lgr	3,%[in]\n"
  47		"	siga	0\n"
  48		CC_IPM(cc)
  49		: CC_OUT(cc, cc)
  50		: [fc] "d" (fc), [schid] "d" (schid),
  51		  [out] "d" (out_mask), [in] "d" (in_mask)
  52		: CC_CLOBBER_LIST("0", "1", "2", "3"));
  53	return CC_TRANSFORM(cc);
  54}
  55
  56static inline int do_siga_input(unsigned long schid, unsigned long mask,
  57				unsigned long fc)
  58{
  59	int cc;
  60
  61	asm volatile(
  62		"	lgr	0,%[fc]\n"
  63		"	lgr	1,%[schid]\n"
  64		"	lgr	2,%[mask]\n"
  65		"	siga	0\n"
  66		CC_IPM(cc)
  67		: CC_OUT(cc, cc)
  68		: [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask)
  69		: CC_CLOBBER_LIST("0", "1", "2"));
  70	return CC_TRANSFORM(cc);
  71}
  72
  73/**
  74 * do_siga_output - perform SIGA-w/wt function
  75 * @schid: subchannel id or in case of QEBSM the subchannel token
  76 * @mask: which output queues to process
  77 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
  78 * @fc: function code to perform
  79 * @aob: asynchronous operation block
  80 *
  81 * Returns condition code.
  82 * Note: For IQDC unicast queues only the highest priority queue is processed.
  83 */
  84static inline int do_siga_output(unsigned long schid, unsigned long mask,
  85				 unsigned int *bb, unsigned long fc,
  86				 dma64_t aob)
  87{
  88	int cc;
  89
  90	asm volatile(
  91		"	lgr	0,%[fc]\n"
  92		"	lgr	1,%[schid]\n"
  93		"	lgr	2,%[mask]\n"
  94		"	lgr	3,%[aob]\n"
  95		"	siga	0\n"
  96		"	lgr	%[fc],0\n"
  97		CC_IPM(cc)
  98		: CC_OUT(cc, cc), [fc] "+&d" (fc)
  99		: [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob)
 100		: CC_CLOBBER_LIST("0", "1", "2", "3"));
 101	*bb = fc >> 31;
 102	return CC_TRANSFORM(cc);
 103}
 104
 105/**
 106 * qdio_do_eqbs - extract buffer states for QEBSM
 107 * @q: queue to manipulate
 108 * @state: state of the extracted buffers
 109 * @start: buffer number to start at
 110 * @count: count of buffers to examine
 111 * @auto_ack: automatically acknowledge buffers
 112 *
 113 * Returns the number of successfully extracted equal buffer states.
 114 * Stops processing if a state is different from the last buffers state.
 115 */
 116static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
 117			int start, int count, int auto_ack)
 118{
 119	int tmp_count = count, tmp_start = start, nr = q->nr;
 120	unsigned int ccq = 0;
 121
 122	qperf_inc(q, eqbs);
 123
 124	if (!q->is_input_q)
 125		nr += q->irq_ptr->nr_input_qs;
 126again:
 127	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
 128		      auto_ack);
 129
 130	switch (ccq) {
 131	case 0:
 132	case 32:
 133		/* all done, or next buffer state different */
 134		return count - tmp_count;
 135	case 96:
 136		/* not all buffers processed */
 137		qperf_inc(q, eqbs_partial);
 138		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",
 139			tmp_count);
 140		return count - tmp_count;
 141	case 97:
 142		/* no buffer processed */
 143		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
 144		goto again;
 145	default:
 146		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
 147		DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
 148		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 149		q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
 150			   q->first_to_check, count, q->irq_ptr->int_parm);
 151		return 0;
 152	}
 153}
 154
 155/**
 156 * qdio_do_sqbs - set buffer states for QEBSM
 157 * @q: queue to manipulate
 158 * @state: new state of the buffers
 159 * @start: first buffer number to change
 160 * @count: how many buffers to change
 161 *
 162 * Returns the number of successfully changed buffers.
 163 * Does retrying until the specified count of buffer states is set or an
 164 * error occurs.
 165 */
 166static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
 167			int count)
 168{
 169	unsigned int ccq = 0;
 170	int tmp_count = count, tmp_start = start;
 171	int nr = q->nr;
 172
 173	qperf_inc(q, sqbs);
 174
 175	if (!q->is_input_q)
 176		nr += q->irq_ptr->nr_input_qs;
 177again:
 178	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
 179
 180	switch (ccq) {
 181	case 0:
 182	case 32:
 183		/* all done, or active buffer adapter-owned */
 184		WARN_ON_ONCE(tmp_count);
 185		return count - tmp_count;
 186	case 96:
 187		/* not all buffers processed */
 188		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
 189		qperf_inc(q, sqbs_partial);
 190		goto again;
 191	default:
 192		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
 193		DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
 194		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 195		q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
 196			   q->first_to_check, count, q->irq_ptr->int_parm);
 197		return 0;
 198	}
 199}
 200
 201/*
 202 * Returns number of examined buffers and their common state in *state.
 203 * Requested number of buffers-to-examine must be > 0.
 204 */
 205static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
 206				 unsigned char *state, unsigned int count,
 207				 int auto_ack)
 208{
 209	unsigned char __state = 0;
 210	int i = 1;
 211
 212	if (is_qebsm(q))
 213		return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
 214
 215	/* get initial state: */
 216	__state = q->slsb.val[bufnr];
 217
 218	/* Bail out early if there is no work on the queue: */
 219	if (__state & SLSB_OWNER_CU)
 220		goto out;
 221
 222	for (; i < count; i++) {
 223		bufnr = next_buf(bufnr);
 224
 225		/* stop if next state differs from initial state: */
 226		if (q->slsb.val[bufnr] != __state)
 227			break;
 228	}
 229
 230out:
 231	*state = __state;
 232	return i;
 233}
 234
 235static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
 236				unsigned char *state, int auto_ack)
 237{
 238	return get_buf_states(q, bufnr, state, 1, auto_ack);
 239}
 240
 241/* wrap-around safe setting of slsb states, returns number of changed buffers */
 242static inline int set_buf_states(struct qdio_q *q, int bufnr,
 243				 unsigned char state, int count)
 244{
 245	int i;
 246
 247	if (is_qebsm(q))
 248		return qdio_do_sqbs(q, state, bufnr, count);
 249
 250	/* Ensure that all preceding changes to the SBALs are visible: */
 251	mb();
 252
 253	for (i = 0; i < count; i++) {
 254		WRITE_ONCE(q->slsb.val[bufnr], state);
 255		bufnr = next_buf(bufnr);
 256	}
 257
 258	/* Make our SLSB changes visible: */
 259	mb();
 260
 261	return count;
 262}
 263
 264static inline int set_buf_state(struct qdio_q *q, int bufnr,
 265				unsigned char state)
 266{
 267	return set_buf_states(q, bufnr, state, 1);
 268}
 269
 270/* set slsb states to initial state */
 271static void qdio_init_buf_states(struct qdio_irq *irq_ptr)
 272{
 273	struct qdio_q *q;
 274	int i;
 275
 276	for_each_input_queue(irq_ptr, q, i)
 277		set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
 278			       QDIO_MAX_BUFFERS_PER_Q);
 279	for_each_output_queue(irq_ptr, q, i)
 280		set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
 281			       QDIO_MAX_BUFFERS_PER_Q);
 282}
 283
 284static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
 285			  unsigned int input)
 286{
 287	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 288	unsigned int fc = QDIO_SIGA_SYNC;
 289	int cc;
 290
 291	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
 292	qperf_inc(q, siga_sync);
 293
 294	if (is_qebsm(q)) {
 295		schid = q->irq_ptr->sch_token;
 296		fc |= QDIO_SIGA_QEBSM_FLAG;
 297	}
 298
 299	cc = do_siga_sync(schid, output, input, fc);
 300	if (unlikely(cc))
 301		DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
 302	return (cc) ? -EIO : 0;
 303}
 304
 305static inline int qdio_sync_input_queue(struct qdio_q *q)
 306{
 307	return qdio_siga_sync(q, 0, q->mask);
 308}
 309
 310static inline int qdio_sync_output_queue(struct qdio_q *q)
 311{
 312	return qdio_siga_sync(q, q->mask, 0);
 313}
 314
 315static inline int qdio_siga_sync_q(struct qdio_q *q)
 316{
 317	if (q->is_input_q)
 318		return qdio_sync_input_queue(q);
 319	else
 320		return qdio_sync_output_queue(q);
 321}
 322
 323static int qdio_siga_output(struct qdio_q *q, unsigned int count,
 324			    unsigned int *busy_bit, dma64_t aob)
 325{
 326	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 327	unsigned int fc = QDIO_SIGA_WRITE;
 328	u64 start_time = 0;
 329	int retries = 0, cc;
 330
 331	if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {
 332		if (count > 1)
 333			fc = QDIO_SIGA_WRITEM;
 334		else if (aob)
 335			fc = QDIO_SIGA_WRITEQ;
 336	}
 337
 338	if (is_qebsm(q)) {
 339		schid = q->irq_ptr->sch_token;
 340		fc |= QDIO_SIGA_QEBSM_FLAG;
 341	}
 342again:
 343	cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);
 344
 345	/* hipersocket busy condition */
 346	if (unlikely(*busy_bit)) {
 347		retries++;
 348
 349		if (!start_time) {
 350			start_time = get_tod_clock_fast();
 351			goto again;
 352		}
 353		if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE)
 354			goto again;
 355	}
 356	if (retries) {
 357		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,
 358			      "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);
 359		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);
 360	}
 361	return cc;
 362}
 363
 364static inline int qdio_siga_input(struct qdio_q *q)
 365{
 366	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
 367	unsigned int fc = QDIO_SIGA_READ;
 368	int cc;
 369
 370	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
 371	qperf_inc(q, siga_read);
 372
 373	if (is_qebsm(q)) {
 374		schid = q->irq_ptr->sch_token;
 375		fc |= QDIO_SIGA_QEBSM_FLAG;
 376	}
 377
 378	cc = do_siga_input(schid, q->mask, fc);
 379	if (unlikely(cc))
 380		DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
 381	return (cc) ? -EIO : 0;
 382}
 383
 384int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
 385			unsigned char *state)
 386{
 387	if (qdio_need_siga_sync(q->irq_ptr))
 388		qdio_siga_sync_q(q);
 389	return get_buf_state(q, bufnr, state, 0);
 390}
 391
 392static inline void qdio_stop_polling(struct qdio_q *q)
 393{
 394	if (!q->u.in.batch_count)
 395		return;
 396
 397	qperf_inc(q, stop_polling);
 398
 399	/* show the card that we are not polling anymore */
 400	set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,
 401		       q->u.in.batch_count);
 402	q->u.in.batch_count = 0;
 403}
 404
 405static inline void account_sbals(struct qdio_q *q, unsigned int count)
 406{
 407	q->q_stats.nr_sbal_total += count;
 408	q->q_stats.nr_sbals[ilog2(count)]++;
 409}
 410
 411static void process_buffer_error(struct qdio_q *q, unsigned int start,
 412				 int count)
 413{
 414	/* special handling for no target buffer empty */
 415	if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&
 416	    q->sbal[start]->element[15].sflags == 0x10) {
 417		qperf_inc(q, target_full);
 418		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);
 419		return;
 420	}
 421
 422	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
 423	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
 424	DBF_ERROR("FTC:%3d C:%3d", start, count);
 425	DBF_ERROR("F14:%2x F15:%2x",
 426		  q->sbal[start]->element[14].sflags,
 427		  q->sbal[start]->element[15].sflags);
 428}
 429
 430static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
 431				       int count, bool auto_ack)
 432{
 433	/* ACK the newest SBAL: */
 434	if (!auto_ack)
 435		set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);
 436
 437	if (!q->u.in.batch_count)
 438		q->u.in.batch_start = start;
 439	q->u.in.batch_count += count;
 440}
 441
 442static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start,
 443				       unsigned int *error)
 444{
 445	unsigned char state = 0;
 446	int count;
 447
 448	q->timestamp = get_tod_clock_fast();
 449
 450	count = atomic_read(&q->nr_buf_used);
 451	if (!count)
 452		return 0;
 453
 454	if (qdio_need_siga_sync(q->irq_ptr))
 455		qdio_sync_input_queue(q);
 456
 457	count = get_buf_states(q, start, &state, count, 1);
 458	if (!count)
 459		return 0;
 460
 461	switch (state) {
 462	case SLSB_P_INPUT_PRIMED:
 463		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
 464			      count);
 465
 466		inbound_handle_work(q, start, count, is_qebsm(q));
 467		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
 468			qperf_inc(q, inbound_queue_full);
 469		if (q->irq_ptr->perf_stat_enabled)
 470			account_sbals(q, count);
 471		return count;
 472	case SLSB_P_INPUT_ERROR:
 473		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
 474			      count);
 475
 476		*error = QDIO_ERROR_SLSB_STATE;
 477		process_buffer_error(q, start, count);
 478		inbound_handle_work(q, start, count, false);
 479		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
 480			qperf_inc(q, inbound_queue_full);
 481		if (q->irq_ptr->perf_stat_enabled)
 482			account_sbals_error(q, count);
 483		return count;
 484	case SLSB_CU_INPUT_EMPTY:
 485		if (q->irq_ptr->perf_stat_enabled)
 486			q->q_stats.nr_sbal_nop++;
 487		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",
 488			      q->nr, start);
 489		return 0;
 490	case SLSB_P_INPUT_NOT_INIT:
 491	case SLSB_P_INPUT_ACK:
 492		/* We should never see this state, throw a WARN: */
 493	default:
 494		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
 495			      "found state %#x at index %u on queue %u\n",
 496			      state, start, q->nr);
 497		return 0;
 498	}
 499}
 500
 501int qdio_inspect_input_queue(struct ccw_device *cdev, unsigned int nr,
 502			     unsigned int *bufnr, unsigned int *error)
 503{
 504	struct qdio_irq *irq = cdev->private->qdio_data;
 505	unsigned int start;
 506	struct qdio_q *q;
 507	int count;
 508
 509	if (!irq)
 510		return -ENODEV;
 511
 512	q = irq->input_qs[nr];
 513	start = q->first_to_check;
 514	*error = 0;
 515
 516	count = get_inbound_buffer_frontier(q, start, error);
 517	if (count == 0)
 518		return 0;
 519
 520	*bufnr = start;
 521	q->first_to_check = add_buf(start, count);
 522	return count;
 523}
 524EXPORT_SYMBOL_GPL(qdio_inspect_input_queue);
 525
 526static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)
 527{
 528	unsigned char state = 0;
 529
 530	if (!atomic_read(&q->nr_buf_used))
 531		return 1;
 532
 533	if (qdio_need_siga_sync(q->irq_ptr))
 534		qdio_sync_input_queue(q);
 535	get_buf_state(q, start, &state, 0);
 536
 537	if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
 538		/* more work coming */
 539		return 0;
 540
 541	return 1;
 542}
 543
 544static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start,
 545					unsigned int *error)
 546{
 547	unsigned char state = 0;
 548	int count;
 549
 550	q->timestamp = get_tod_clock_fast();
 551
 552	count = atomic_read(&q->nr_buf_used);
 553	if (!count)
 554		return 0;
 555
 556	if (qdio_need_siga_sync(q->irq_ptr))
 557		qdio_sync_output_queue(q);
 558
 559	count = get_buf_states(q, start, &state, count, 0);
 560	if (!count)
 561		return 0;
 562
 563	switch (state) {
 564	case SLSB_P_OUTPUT_PENDING:
 565		*error = QDIO_ERROR_SLSB_PENDING;
 566		fallthrough;
 567	case SLSB_P_OUTPUT_EMPTY:
 568		/* the adapter got it */
 569		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,
 570			"out empty:%1d %02x", q->nr, count);
 571
 572		atomic_sub(count, &q->nr_buf_used);
 573		if (q->irq_ptr->perf_stat_enabled)
 574			account_sbals(q, count);
 575		return count;
 576	case SLSB_P_OUTPUT_ERROR:
 577		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x",
 578			      q->nr, count);
 579
 580		*error = QDIO_ERROR_SLSB_STATE;
 581		process_buffer_error(q, start, count);
 582		atomic_sub(count, &q->nr_buf_used);
 583		if (q->irq_ptr->perf_stat_enabled)
 584			account_sbals_error(q, count);
 585		return count;
 586	case SLSB_CU_OUTPUT_PRIMED:
 587		/* the adapter has not fetched the output yet */
 588		if (q->irq_ptr->perf_stat_enabled)
 589			q->q_stats.nr_sbal_nop++;
 590		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
 591			      q->nr);
 592		return 0;
 593	case SLSB_P_OUTPUT_HALTED:
 594		return 0;
 595	case SLSB_P_OUTPUT_NOT_INIT:
 596		/* We should never see this state, throw a WARN: */
 597	default:
 598		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
 599			      "found state %#x at index %u on queue %u\n",
 600			      state, start, q->nr);
 601		return 0;
 602	}
 603}
 604
 605int qdio_inspect_output_queue(struct ccw_device *cdev, unsigned int nr,
 606			      unsigned int *bufnr, unsigned int *error)
 607{
 608	struct qdio_irq *irq = cdev->private->qdio_data;
 609	unsigned int start;
 610	struct qdio_q *q;
 611	int count;
 612
 613	if (!irq)
 614		return -ENODEV;
 615
 616	q = irq->output_qs[nr];
 617	start = q->first_to_check;
 618	*error = 0;
 619
 620	count = get_outbound_buffer_frontier(q, start, error);
 621	if (count == 0)
 622		return 0;
 623
 624	*bufnr = start;
 625	q->first_to_check = add_buf(start, count);
 626	return count;
 627}
 628EXPORT_SYMBOL_GPL(qdio_inspect_output_queue);
 629
 630static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,
 631				dma64_t aob)
 632{
 633	int retries = 0, cc;
 634	unsigned int busy_bit;
 635
 636	if (!qdio_need_siga_out(q->irq_ptr))
 637		return 0;
 638
 639	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
 640retry:
 641	qperf_inc(q, siga_write);
 642
 643	cc = qdio_siga_output(q, count, &busy_bit, aob);
 644	switch (cc) {
 645	case 0:
 646		break;
 647	case 2:
 648		if (busy_bit) {
 649			while (++retries < QDIO_BUSY_BIT_RETRIES) {
 650				mdelay(QDIO_BUSY_BIT_RETRY_DELAY);
 651				goto retry;
 652			}
 653			DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);
 654			cc = -EBUSY;
 655		} else {
 656			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
 657			cc = -ENOBUFS;
 658		}
 659		break;
 660	case 1:
 661	case 3:
 662		DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
 663		cc = -EIO;
 664		break;
 665	}
 666	if (retries) {
 667		DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);
 668		DBF_ERROR("count:%u", retries);
 669	}
 670	return cc;
 671}
 672
 673static inline void qdio_set_state(struct qdio_irq *irq_ptr,
 674				  enum qdio_irq_states state)
 675{
 676	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
 677
 678	irq_ptr->state = state;
 679	mb();
 680}
 681
 682static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
 683{
 684	if (irb->esw.esw0.erw.cons) {
 685		DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
 686		DBF_ERROR_HEX(irb, 64);
 687		DBF_ERROR_HEX(irb->ecw, 64);
 688	}
 689}
 690
 691/* PCI interrupt handler */
 692static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
 693{
 694	if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
 695		return;
 696
 697	qdio_deliver_irq(irq_ptr);
 698	irq_ptr->last_data_irq_time = get_lowcore()->int_clock;
 699}
 700
 701static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,
 702				       unsigned long intparm, int cstat,
 703				       int dstat)
 704{
 705	unsigned int first_to_check = 0;
 706
 707	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
 708	DBF_ERROR("intp :%lx", intparm);
 709	DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
 710
 711	/* zfcp wants this: */
 712	if (irq_ptr->nr_input_qs)
 713		first_to_check = irq_ptr->input_qs[0]->first_to_check;
 714
 715	irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0,
 716			       first_to_check, 0, irq_ptr->int_parm);
 717	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
 718	/*
 719	 * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen.
 720	 * Therefore we call the LGR detection function here.
 721	 */
 722	lgr_info_log();
 723}
 724
 725static int qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat,
 726				     int dstat, int dcc)
 727{
 728	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
 729
 730	if (cstat)
 731		goto error;
 732	if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
 733		goto error;
 734	if (dcc == 1)
 735		return -EAGAIN;
 736	if (!(dstat & DEV_STAT_DEV_END))
 737		goto error;
 738	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
 739	return 0;
 740
 741error:
 742	DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
 743	DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
 744	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 745	return -EIO;
 746}
 747
 748/* qdio interrupt handler */
 749void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
 750		      struct irb *irb)
 751{
 752	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 753	struct subchannel_id schid;
 754	int cstat, dstat, rc, dcc;
 755
 756	if (!intparm || !irq_ptr) {
 757		ccw_device_get_schid(cdev, &schid);
 758		DBF_ERROR("qint:%4x", schid.sch_no);
 759		return;
 760	}
 761
 762	if (irq_ptr->perf_stat_enabled)
 763		irq_ptr->perf_stat.qdio_int++;
 764
 765	if (IS_ERR(irb)) {
 766		DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
 767		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 768		wake_up(&cdev->private->wait_q);
 769		return;
 770	}
 771	qdio_irq_check_sense(irq_ptr, irb);
 772	cstat = irb->scsw.cmd.cstat;
 773	dstat = irb->scsw.cmd.dstat;
 774	dcc   = scsw_cmd_is_valid_cc(&irb->scsw) ? irb->scsw.cmd.cc : 0;
 775	rc    = 0;
 776
 777	switch (irq_ptr->state) {
 778	case QDIO_IRQ_STATE_INACTIVE:
 779		rc = qdio_establish_handle_irq(irq_ptr, cstat, dstat, dcc);
 780		break;
 781	case QDIO_IRQ_STATE_CLEANUP:
 782		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 783		break;
 784	case QDIO_IRQ_STATE_ESTABLISHED:
 785	case QDIO_IRQ_STATE_ACTIVE:
 786		if (cstat & SCHN_STAT_PCI) {
 787			qdio_int_handler_pci(irq_ptr);
 788			return;
 789		}
 790		if (cstat || dstat)
 791			qdio_handle_activate_check(irq_ptr, intparm, cstat,
 792						   dstat);
 793		else if (dcc == 1)
 794			rc = -EAGAIN;
 795		break;
 796	case QDIO_IRQ_STATE_STOPPED:
 797		break;
 798	default:
 799		WARN_ON_ONCE(1);
 800	}
 801
 802	if (rc == -EAGAIN) {
 803		DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qint retry");
 804		rc = ccw_device_start(cdev, irq_ptr->ccw, intparm, 0, 0);
 805		if (!rc)
 806			return;
 807		DBF_ERROR("%4x RETRY ERR", irq_ptr->schid.sch_no);
 808		DBF_ERROR("rc:%4x", rc);
 809		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
 810	}
 811
 812	wake_up(&cdev->private->wait_q);
 813}
 814
 815/**
 816 * qdio_get_ssqd_desc - get qdio subchannel description
 817 * @cdev: ccw device to get description for
 818 * @data: where to store the ssqd
 819 *
 820 * Returns 0 or an error code. The results of the chsc are stored in the
 821 * specified structure.
 822 */
 823int qdio_get_ssqd_desc(struct ccw_device *cdev,
 824		       struct qdio_ssqd_desc *data)
 825{
 826	struct subchannel_id schid;
 827
 828	if (!cdev || !cdev->private)
 829		return -EINVAL;
 830
 831	ccw_device_get_schid(cdev, &schid);
 832	DBF_EVENT("get ssqd:%4x", schid.sch_no);
 833	return qdio_setup_get_ssqd(NULL, &schid, data);
 834}
 835EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
 836
 837static int qdio_cancel_ccw(struct qdio_irq *irq, int how)
 838{
 839	struct ccw_device *cdev = irq->cdev;
 840	long timeout;
 841	int rc;
 842
 843	spin_lock_irq(get_ccwdev_lock(cdev));
 844	qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP);
 845	if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
 846		rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
 847	else
 848		/* default behaviour is halt */
 849		rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
 850	spin_unlock_irq(get_ccwdev_lock(cdev));
 851	if (rc) {
 852		DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no);
 853		DBF_ERROR("rc:%4d", rc);
 854		return rc;
 855	}
 856
 857	timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
 858						   irq->state == QDIO_IRQ_STATE_INACTIVE ||
 859						   irq->state == QDIO_IRQ_STATE_ERR,
 860						   10 * HZ);
 861	if (timeout <= 0)
 862		rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
 863
 864	return rc;
 865}
 866
 867/**
 868 * qdio_shutdown - shut down a qdio subchannel
 869 * @cdev: associated ccw device
 870 * @how: use halt or clear to shutdown
 871 */
 872int qdio_shutdown(struct ccw_device *cdev, int how)
 873{
 874	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 875	struct subchannel_id schid;
 876	int rc;
 877
 878	if (!irq_ptr)
 879		return -ENODEV;
 880
 881	WARN_ON_ONCE(irqs_disabled());
 882	ccw_device_get_schid(cdev, &schid);
 883	DBF_EVENT("qshutdown:%4x", schid.sch_no);
 884
 885	mutex_lock(&irq_ptr->setup_mutex);
 886	/*
 887	 * Subchannel was already shot down. We cannot prevent being called
 888	 * twice since cio may trigger a shutdown asynchronously.
 889	 */
 890	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
 891		mutex_unlock(&irq_ptr->setup_mutex);
 892		return 0;
 893	}
 894
 895	/*
 896	 * Indicate that the device is going down.
 897	 */
 898	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
 899
 900	qdio_shutdown_debug_entries(irq_ptr);
 901
 902	rc = qdio_cancel_ccw(irq_ptr, how);
 903	qdio_shutdown_thinint(irq_ptr);
 904	qdio_shutdown_irq(irq_ptr);
 905
 906	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 907	mutex_unlock(&irq_ptr->setup_mutex);
 908	if (rc)
 909		return rc;
 910	return 0;
 911}
 912EXPORT_SYMBOL_GPL(qdio_shutdown);
 913
 914/**
 915 * qdio_free - free data structures for a qdio subchannel
 916 * @cdev: associated ccw device
 917 */
 918int qdio_free(struct ccw_device *cdev)
 919{
 920	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 921	struct subchannel_id schid;
 922
 923	if (!irq_ptr)
 924		return -ENODEV;
 925
 926	ccw_device_get_schid(cdev, &schid);
 927	DBF_EVENT("qfree:%4x", schid.sch_no);
 928	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned");
 929	mutex_lock(&irq_ptr->setup_mutex);
 930
 931	irq_ptr->debug_area = NULL;
 932	cdev->private->qdio_data = NULL;
 933	mutex_unlock(&irq_ptr->setup_mutex);
 934
 935	qdio_free_queues(irq_ptr);
 936	free_page((unsigned long) irq_ptr->qdr);
 937	free_page(irq_ptr->chsc_page);
 938	kfree(irq_ptr->ccw);
 939	free_page((unsigned long) irq_ptr);
 940	return 0;
 941}
 942EXPORT_SYMBOL_GPL(qdio_free);
 943
 944/**
 945 * qdio_allocate - allocate qdio queues and associated data
 946 * @cdev: associated ccw device
 947 * @no_input_qs: allocate this number of Input Queues
 948 * @no_output_qs: allocate this number of Output Queues
 949 */
 950int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 951		  unsigned int no_output_qs)
 952{
 953	struct subchannel_id schid;
 954	struct qdio_irq *irq_ptr;
 955	int rc = -ENOMEM;
 956
 957	ccw_device_get_schid(cdev, &schid);
 958	DBF_EVENT("qallocate:%4x", schid.sch_no);
 959
 960	if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ ||
 961	    no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)
 962		return -EINVAL;
 963
 964	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL);
 965	if (!irq_ptr)
 966		return -ENOMEM;
 967
 968	irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA);
 969	if (!irq_ptr->ccw)
 970		goto err_ccw;
 971
 972	/* kmemleak doesn't scan the page-allocated irq_ptr: */
 973	kmemleak_not_leak(irq_ptr->ccw);
 974
 975	irq_ptr->cdev = cdev;
 976	mutex_init(&irq_ptr->setup_mutex);
 977	if (qdio_allocate_dbf(irq_ptr))
 978		goto err_dbf;
 979
 980	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,
 981		      no_output_qs);
 982
 983	/*
 984	 * Allocate a page for the chsc calls in qdio_establish.
 985	 * Must be pre-allocated since a zfcp recovery will call
 986	 * qdio_establish. In case of low memory and swap on a zfcp disk
 987	 * we may not be able to allocate memory otherwise.
 988	 */
 989	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
 990	if (!irq_ptr->chsc_page)
 991		goto err_chsc;
 992
 993	/* qdr is used in ccw1.cda which is u32 */
 994	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 995	if (!irq_ptr->qdr)
 996		goto err_qdr;
 997
 998	rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);
 999	if (rc)
1000		goto err_queues;
1001
1002	cdev->private->qdio_data = irq_ptr;
1003	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1004	return 0;
1005
1006err_queues:
1007	free_page((unsigned long) irq_ptr->qdr);
1008err_qdr:
1009	free_page(irq_ptr->chsc_page);
1010err_chsc:
1011err_dbf:
1012	kfree(irq_ptr->ccw);
1013err_ccw:
1014	free_page((unsigned long) irq_ptr);
1015	return rc;
1016}
1017EXPORT_SYMBOL_GPL(qdio_allocate);
1018
1019static void qdio_trace_init_data(struct qdio_irq *irq,
1020				 struct qdio_initialize *data)
1021{
1022	DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format);
1023	DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format);
1024	DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR);
1025	DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs,
1026		      data->no_output_qs);
1027	DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR);
1028	DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR);
1029	DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR);
1030	DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR);
1031	DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *),
1032		    DBF_ERR);
1033}
1034
1035/**
1036 * qdio_establish - establish queues on a qdio subchannel
1037 * @cdev: associated ccw device
1038 * @init_data: initialization data
1039 */
1040int qdio_establish(struct ccw_device *cdev,
1041		   struct qdio_initialize *init_data)
1042{
1043	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1044	struct subchannel_id schid;
1045	struct ciw *ciw;
1046	long timeout;
1047	int rc;
1048
1049	ccw_device_get_schid(cdev, &schid);
1050	DBF_EVENT("qestablish:%4x", schid.sch_no);
1051
1052	if (!irq_ptr)
1053		return -ENODEV;
1054
1055	if (init_data->no_input_qs > irq_ptr->max_input_qs ||
1056	    init_data->no_output_qs > irq_ptr->max_output_qs)
1057		return -EINVAL;
1058
1059	/* Needed as error_handler: */
1060	if (!init_data->input_handler)
1061		return -EINVAL;
1062
1063	if (init_data->no_output_qs && !init_data->output_handler)
1064		return -EINVAL;
1065
1066	if (!init_data->input_sbal_addr_array ||
1067	    !init_data->output_sbal_addr_array)
1068		return -EINVAL;
1069
1070	if (!init_data->irq_poll)
1071		return -EINVAL;
1072
1073	ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE);
1074	if (!ciw) {
1075		DBF_ERROR("%4x NO EQ", schid.sch_no);
1076		return -EIO;
1077	}
1078
1079	mutex_lock(&irq_ptr->setup_mutex);
1080	qdio_trace_init_data(irq_ptr, init_data);
1081	qdio_setup_irq(irq_ptr, init_data);
1082
1083	rc = qdio_establish_thinint(irq_ptr);
1084	if (rc)
1085		goto err_thinint;
1086
1087	/* establish q */
1088	irq_ptr->ccw->cmd_code = ciw->cmd;
1089	irq_ptr->ccw->flags = CCW_FLAG_SLI;
1090	irq_ptr->ccw->count = ciw->count;
1091	irq_ptr->ccw->cda = virt_to_dma32(irq_ptr->qdr);
1092
1093	spin_lock_irq(get_ccwdev_lock(cdev));
1094	ccw_device_set_options_mask(cdev, 0);
1095
1096	rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
1097	spin_unlock_irq(get_ccwdev_lock(cdev));
1098	if (rc) {
1099		DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1100		DBF_ERROR("rc:%4x", rc);
1101		goto err_ccw_start;
1102	}
1103
1104	timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
1105						   irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
1106						   irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
1107	if (timeout <= 0) {
1108		rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
1109		goto err_ccw_timeout;
1110	}
1111
1112	if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1113		rc = -EIO;
1114		goto err_ccw_error;
1115	}
1116
1117	qdio_setup_ssqd_info(irq_ptr);
1118
1119	/* qebsm is now setup if available, initialize buffer states */
1120	qdio_init_buf_states(irq_ptr);
1121
1122	mutex_unlock(&irq_ptr->setup_mutex);
1123	qdio_print_subchannel_info(irq_ptr);
1124	qdio_setup_debug_entries(irq_ptr);
1125	return 0;
1126
1127err_ccw_timeout:
1128	qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR);
1129err_ccw_error:
1130err_ccw_start:
1131	qdio_shutdown_thinint(irq_ptr);
1132err_thinint:
1133	qdio_shutdown_irq(irq_ptr);
1134	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1135	mutex_unlock(&irq_ptr->setup_mutex);
1136	return rc;
1137}
1138EXPORT_SYMBOL_GPL(qdio_establish);
1139
1140/**
1141 * qdio_activate - activate queues on a qdio subchannel
1142 * @cdev: associated cdev
1143 */
1144int qdio_activate(struct ccw_device *cdev)
1145{
1146	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1147	struct subchannel_id schid;
1148	struct ciw *ciw;
1149	int rc;
1150
1151	ccw_device_get_schid(cdev, &schid);
1152	DBF_EVENT("qactivate:%4x", schid.sch_no);
1153
1154	if (!irq_ptr)
1155		return -ENODEV;
1156
1157	ciw = ccw_device_get_ciw(cdev, CIW_TYPE_AQUEUE);
1158	if (!ciw) {
1159		DBF_ERROR("%4x NO AQ", schid.sch_no);
1160		return -EIO;
1161	}
1162
1163	mutex_lock(&irq_ptr->setup_mutex);
1164	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1165		rc = -EBUSY;
1166		goto out;
1167	}
1168
1169	irq_ptr->ccw->cmd_code = ciw->cmd;
1170	irq_ptr->ccw->flags = CCW_FLAG_SLI;
1171	irq_ptr->ccw->count = ciw->count;
1172	irq_ptr->ccw->cda = 0;
1173
1174	spin_lock_irq(get_ccwdev_lock(cdev));
1175	ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
1176
1177	rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE,
1178			      0, DOIO_DENY_PREFETCH);
1179	spin_unlock_irq(get_ccwdev_lock(cdev));
1180	if (rc) {
1181		DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
1182		DBF_ERROR("rc:%4x", rc);
1183		goto out;
1184	}
1185
1186	/* wait for subchannel to become active */
1187	msleep(5);
1188
1189	switch (irq_ptr->state) {
1190	case QDIO_IRQ_STATE_STOPPED:
1191	case QDIO_IRQ_STATE_ERR:
1192		rc = -EIO;
1193		break;
1194	default:
1195		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
1196		rc = 0;
1197	}
1198out:
1199	mutex_unlock(&irq_ptr->setup_mutex);
1200	return rc;
1201}
1202EXPORT_SYMBOL_GPL(qdio_activate);
1203
1204/**
1205 * handle_inbound - reset processed input buffers
1206 * @q: queue containing the buffers
1207 * @bufnr: first buffer to process
1208 * @count: how many buffers are emptied
1209 */
1210static int handle_inbound(struct qdio_q *q, int bufnr, int count)
1211{
1212	int overlap;
1213
1214	qperf_inc(q, inbound_call);
1215
1216	/* If any processed SBALs are returned to HW, adjust our tracking: */
1217	overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),
1218			     q->u.in.batch_count);
1219	if (overlap > 0) {
1220		q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);
1221		q->u.in.batch_count -= overlap;
1222	}
1223
1224	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1225	atomic_add(count, &q->nr_buf_used);
1226
1227	if (qdio_need_siga_in(q->irq_ptr))
1228		return qdio_siga_input(q);
1229
1230	return 0;
1231}
1232
1233/**
1234 * qdio_add_bufs_to_input_queue - process buffers on an Input Queue
1235 * @cdev: associated ccw_device for the qdio subchannel
1236 * @q_nr: queue number
1237 * @bufnr: buffer number
1238 * @count: how many buffers to process
1239 */
1240int qdio_add_bufs_to_input_queue(struct ccw_device *cdev, unsigned int q_nr,
1241				 unsigned int bufnr, unsigned int count)
1242{
1243	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1244
1245	if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1246		return -EINVAL;
1247
1248	if (!irq_ptr)
1249		return -ENODEV;
1250
1251	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count);
1252
1253	if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1254		return -EIO;
1255	if (!count)
1256		return 0;
1257
1258	return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count);
1259}
1260EXPORT_SYMBOL_GPL(qdio_add_bufs_to_input_queue);
1261
1262/**
1263 * handle_outbound - process filled outbound buffers
1264 * @q: queue containing the buffers
1265 * @bufnr: first buffer to process
1266 * @count: how many buffers are filled
1267 * @aob: asynchronous operation block
1268 */
1269static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count,
1270			   struct qaob *aob)
1271{
1272	unsigned char state = 0;
1273	int used, rc = 0;
1274
1275	qperf_inc(q, outbound_call);
1276
1277	count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1278	used = atomic_add_return(count, &q->nr_buf_used);
1279
1280	if (used == QDIO_MAX_BUFFERS_PER_Q)
1281		qperf_inc(q, outbound_queue_full);
1282
1283	if (queue_type(q) == QDIO_IQDIO_QFMT) {
1284		dma64_t phys_aob = aob ? virt_to_dma64(aob) : 0;
1285
1286		WARN_ON_ONCE(!IS_ALIGNED(dma64_to_u64(phys_aob), 256));
1287		rc = qdio_kick_outbound_q(q, count, phys_aob);
1288	} else if (qdio_need_siga_sync(q->irq_ptr)) {
1289		rc = qdio_sync_output_queue(q);
1290	} else if (count < QDIO_MAX_BUFFERS_PER_Q &&
1291		   get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 &&
1292		   state == SLSB_CU_OUTPUT_PRIMED) {
1293		/* The previous buffer is not processed yet, tack on. */
1294		qperf_inc(q, fast_requeue);
1295	} else {
1296		rc = qdio_kick_outbound_q(q, count, 0);
1297	}
1298
1299	return rc;
1300}
1301
1302/**
1303 * qdio_add_bufs_to_output_queue - process buffers on an Output Queue
1304 * @cdev: associated ccw_device for the qdio subchannel
1305 * @q_nr: queue number
1306 * @bufnr: buffer number
1307 * @count: how many buffers to process
1308 * @aob: asynchronous operation block
1309 */
1310int qdio_add_bufs_to_output_queue(struct ccw_device *cdev, unsigned int q_nr,
1311				  unsigned int bufnr, unsigned int count,
1312				  struct qaob *aob)
1313{
1314	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1315
1316	if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1317		return -EINVAL;
1318
1319	if (!irq_ptr)
1320		return -ENODEV;
1321
1322	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count);
1323
1324	if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1325		return -EIO;
1326	if (!count)
1327		return 0;
1328
1329	return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob);
1330}
1331EXPORT_SYMBOL_GPL(qdio_add_bufs_to_output_queue);
1332
1333/**
1334 * qdio_start_irq - enable interrupt processing for the device
1335 * @cdev: associated ccw_device for the qdio subchannel
1336 *
1337 * Return codes
1338 *   0 - success
1339 *   1 - irqs not started since new data is available
1340 */
1341int qdio_start_irq(struct ccw_device *cdev)
1342{
1343	struct qdio_q *q;
1344	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1345	unsigned int i;
1346
1347	if (!irq_ptr)
1348		return -ENODEV;
1349
1350	for_each_input_queue(irq_ptr, q, i)
1351		qdio_stop_polling(q);
1352
1353	clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state);
1354
1355	/*
1356	 * We need to check again to not lose initiative after
1357	 * resetting the ACK state.
1358	 */
1359	if (test_nonshared_ind(irq_ptr))
1360		goto rescan;
1361
1362	for_each_input_queue(irq_ptr, q, i) {
1363		if (!qdio_inbound_q_done(q, q->first_to_check))
1364			goto rescan;
1365	}
1366
1367	return 0;
1368
1369rescan:
1370	if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1371		return 0;
1372	else
1373		return 1;
1374
1375}
1376EXPORT_SYMBOL(qdio_start_irq);
1377
1378/**
1379 * qdio_stop_irq - disable interrupt processing for the device
1380 * @cdev: associated ccw_device for the qdio subchannel
1381 *
1382 * Return codes
1383 *   0 - interrupts were already disabled
1384 *   1 - interrupts successfully disabled
1385 */
1386int qdio_stop_irq(struct ccw_device *cdev)
1387{
1388	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1389
1390	if (!irq_ptr)
1391		return -ENODEV;
1392
1393	if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1394		return 0;
1395	else
1396		return 1;
1397}
1398EXPORT_SYMBOL(qdio_stop_irq);
1399
1400static int __init init_QDIO(void)
1401{
1402	int rc;
1403
1404	rc = qdio_debug_init();
1405	if (rc)
1406		return rc;
1407	rc = qdio_setup_init();
1408	if (rc)
1409		goto out_debug;
1410	rc = qdio_thinint_init();
1411	if (rc)
1412		goto out_cache;
1413	return 0;
1414
1415out_cache:
1416	qdio_setup_exit();
1417out_debug:
1418	qdio_debug_exit();
1419	return rc;
1420}
1421
1422static void __exit exit_QDIO(void)
1423{
1424	qdio_thinint_exit();
1425	qdio_setup_exit();
1426	qdio_debug_exit();
1427}
1428
1429module_init(init_QDIO);
1430module_exit(exit_QDIO);