airq.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *    Support for adapter interruptions
  4 *
  5 *    Copyright IBM Corp. 1999, 2007
  6 *    Author(s): Ingo Adlung <adlung@de.ibm.com>
  7 *		 Cornelia Huck <cornelia.huck@de.ibm.com>
  8 *		 Arnd Bergmann <arndb@de.ibm.com>
  9 *		 Peter Oberparleiter <peter.oberparleiter@de.ibm.com>
 10 */
 11
 12#include <linux/export.h>
 13#include <linux/init.h>
 14#include <linux/irq.h>
 15#include <linux/kernel_stat.h>
 16#include <linux/module.h>
 17#include <linux/mutex.h>
 18#include <linux/rculist.h>
 19#include <linux/slab.h>
 20#include <linux/dmapool.h>
 21
 22#include <asm/airq.h>
 23#include <asm/isc.h>
 24#include <asm/cio.h>
 25
 26#include "cio.h"
 27#include "cio_debug.h"
 28#include "ioasm.h"
 29
 30static DEFINE_SPINLOCK(airq_lists_lock);
 31static struct hlist_head airq_lists[MAX_ISC+1];
 32
 33static struct dma_pool *airq_iv_cache;
 34
 35/**
 36 * register_adapter_interrupt() - register adapter interrupt handler
 37 * @airq: pointer to adapter interrupt descriptor
 38 *
 39 * Returns 0 on success, or -EINVAL.
 40 */
 41int register_adapter_interrupt(struct airq_struct *airq)
 42{
 43	char dbf_txt[32];
 44
 45	if (!airq->handler || airq->isc > MAX_ISC)
 46		return -EINVAL;
 47	if (!airq->lsi_ptr) {
 48		airq->lsi_ptr = cio_dma_zalloc(1);
 49		if (!airq->lsi_ptr)
 50			return -ENOMEM;
 51		airq->flags |= AIRQ_PTR_ALLOCATED;
 52	}
 53	snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq);
 54	CIO_TRACE_EVENT(4, dbf_txt);
 55	isc_register(airq->isc);
 56	spin_lock(&airq_lists_lock);
 57	hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]);
 58	spin_unlock(&airq_lists_lock);
 59	return 0;
 60}
 61EXPORT_SYMBOL(register_adapter_interrupt);
 62
 63/**
 64 * unregister_adapter_interrupt - unregister adapter interrupt handler
 65 * @airq: pointer to adapter interrupt descriptor
 66 */
 67void unregister_adapter_interrupt(struct airq_struct *airq)
 68{
 69	char dbf_txt[32];
 70
 71	if (hlist_unhashed(&airq->list))
 72		return;
 73	snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq);
 74	CIO_TRACE_EVENT(4, dbf_txt);
 75	spin_lock(&airq_lists_lock);
 76	hlist_del_rcu(&airq->list);
 77	spin_unlock(&airq_lists_lock);
 78	synchronize_rcu();
 79	isc_unregister(airq->isc);
 80	if (airq->flags & AIRQ_PTR_ALLOCATED) {
 81		cio_dma_free(airq->lsi_ptr, 1);
 82		airq->lsi_ptr = NULL;
 83		airq->flags &= ~AIRQ_PTR_ALLOCATED;
 84	}
 85}
 86EXPORT_SYMBOL(unregister_adapter_interrupt);
 87
 88static irqreturn_t do_airq_interrupt(int irq, void *dummy)
 89{
 90	struct tpi_info *tpi_info;
 91	struct airq_struct *airq;
 92	struct hlist_head *head;
 93
 94	tpi_info = &get_irq_regs()->tpi_info;
 95	trace_s390_cio_adapter_int(tpi_info);
 96	head = &airq_lists[tpi_info->isc];
 97	rcu_read_lock();
 98	hlist_for_each_entry_rcu(airq, head, list)
 99		if (*airq->lsi_ptr != 0)
100			airq->handler(airq, tpi_info);
101	rcu_read_unlock();
102
103	return IRQ_HANDLED;
104}
105
106void __init init_airq_interrupts(void)
107{
108	irq_set_chip_and_handler(THIN_INTERRUPT,
109				 &dummy_irq_chip, handle_percpu_irq);
110	if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL))
111		panic("Failed to register AIO interrupt\n");
112}
113
114static inline unsigned long iv_size(unsigned long bits)
115{
116	return BITS_TO_LONGS(bits) * sizeof(unsigned long);
117}
118
119/**
120 * airq_iv_create - create an interrupt vector
121 * @bits: number of bits in the interrupt vector
122 * @flags: allocation flags
123 * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC
124 *
125 * Returns a pointer to an interrupt vector structure
126 */
127struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags,
128			       unsigned long *vec)
129{
130	struct airq_iv *iv;
131	unsigned long size;
132
133	iv = kzalloc(sizeof(*iv), GFP_KERNEL);
134	if (!iv)
135		goto out;
136	iv->bits = bits;
137	iv->flags = flags;
138	size = iv_size(bits);
139
140	if (flags & AIRQ_IV_CACHELINE) {
141		if ((cache_line_size() * BITS_PER_BYTE) < bits
142				|| !airq_iv_cache)
143			goto out_free;
144
145		iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL,
146					     &iv->vector_dma);
147		if (!iv->vector)
148			goto out_free;
149	} else if (flags & AIRQ_IV_GUESTVEC) {
150		iv->vector = vec;
151	} else {
152		iv->vector = cio_dma_zalloc(size);
153		if (!iv->vector)
154			goto out_free;
155	}
156	if (flags & AIRQ_IV_ALLOC) {
157		iv->avail = kmalloc(size, GFP_KERNEL);
158		if (!iv->avail)
159			goto out_free;
160		memset(iv->avail, 0xff, size);
161		iv->end = 0;
162	} else
163		iv->end = bits;
164	if (flags & AIRQ_IV_BITLOCK) {
165		iv->bitlock = kzalloc(size, GFP_KERNEL);
166		if (!iv->bitlock)
167			goto out_free;
168	}
169	if (flags & AIRQ_IV_PTR) {
170		size = bits * sizeof(unsigned long);
171		iv->ptr = kzalloc(size, GFP_KERNEL);
172		if (!iv->ptr)
173			goto out_free;
174	}
175	if (flags & AIRQ_IV_DATA) {
176		size = bits * sizeof(unsigned int);
177		iv->data = kzalloc(size, GFP_KERNEL);
178		if (!iv->data)
179			goto out_free;
180	}
181	spin_lock_init(&iv->lock);
182	return iv;
183
184out_free:
185	kfree(iv->ptr);
186	kfree(iv->bitlock);
187	kfree(iv->avail);
188	if (iv->flags & AIRQ_IV_CACHELINE && iv->vector)
189		dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
190	else if (!(iv->flags & AIRQ_IV_GUESTVEC))
191		cio_dma_free(iv->vector, size);
192	kfree(iv);
193out:
194	return NULL;
195}
196EXPORT_SYMBOL(airq_iv_create);
197
198/**
199 * airq_iv_release - release an interrupt vector
200 * @iv: pointer to interrupt vector structure
201 */
202void airq_iv_release(struct airq_iv *iv)
203{
204	kfree(iv->data);
205	kfree(iv->ptr);
206	kfree(iv->bitlock);
207	if (iv->flags & AIRQ_IV_CACHELINE)
208		dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
209	else if (!(iv->flags & AIRQ_IV_GUESTVEC))
210		cio_dma_free(iv->vector, iv_size(iv->bits));
211	kfree(iv->avail);
212	kfree(iv);
213}
214EXPORT_SYMBOL(airq_iv_release);
215
216/**
217 * airq_iv_alloc - allocate irq bits from an interrupt vector
218 * @iv: pointer to an interrupt vector structure
219 * @num: number of consecutive irq bits to allocate
220 *
221 * Returns the bit number of the first irq in the allocated block of irqs,
222 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been
223 * specified
224 */
225unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num)
226{
227	unsigned long bit, i, flags;
228
229	if (!iv->avail || num == 0)
230		return -1UL;
231	spin_lock_irqsave(&iv->lock, flags);
232	bit = find_first_bit_inv(iv->avail, iv->bits);
233	while (bit + num <= iv->bits) {
234		for (i = 1; i < num; i++)
235			if (!test_bit_inv(bit + i, iv->avail))
236				break;
237		if (i >= num) {
238			/* Found a suitable block of irqs */
239			for (i = 0; i < num; i++)
240				clear_bit_inv(bit + i, iv->avail);
241			if (bit + num >= iv->end)
242				iv->end = bit + num + 1;
243			break;
244		}
245		bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1);
246	}
247	if (bit + num > iv->bits)
248		bit = -1UL;
249	spin_unlock_irqrestore(&iv->lock, flags);
250	return bit;
251}
252EXPORT_SYMBOL(airq_iv_alloc);
253
254/**
255 * airq_iv_free - free irq bits of an interrupt vector
256 * @iv: pointer to interrupt vector structure
257 * @bit: number of the first irq bit to free
258 * @num: number of consecutive irq bits to free
259 */
260void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num)
261{
262	unsigned long i, flags;
263
264	if (!iv->avail || num == 0)
265		return;
266	spin_lock_irqsave(&iv->lock, flags);
267	for (i = 0; i < num; i++) {
268		/* Clear (possibly left over) interrupt bit */
269		clear_bit_inv(bit + i, iv->vector);
270		/* Make the bit positions available again */
271		set_bit_inv(bit + i, iv->avail);
272	}
273	if (bit + num >= iv->end) {
274		/* Find new end of bit-field */
275		while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail))
276			iv->end--;
277	}
278	spin_unlock_irqrestore(&iv->lock, flags);
279}
280EXPORT_SYMBOL(airq_iv_free);
281
282/**
283 * airq_iv_scan - scan interrupt vector for non-zero bits
284 * @iv: pointer to interrupt vector structure
285 * @start: bit number to start the search
286 * @end: bit number to end the search
287 *
288 * Returns the bit number of the next non-zero interrupt bit, or
289 * -1UL if the scan completed without finding any more any non-zero bits.
290 */
291unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
292			   unsigned long end)
293{
294	unsigned long bit;
295
296	/* Find non-zero bit starting from 'ivs->next'. */
297	bit = find_next_bit_inv(iv->vector, end, start);
298	if (bit >= end)
299		return -1UL;
300	clear_bit_inv(bit, iv->vector);
301	return bit;
302}
303EXPORT_SYMBOL(airq_iv_scan);
304
305int __init airq_init(void)
306{
307	airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(),
308					cache_line_size(),
309					cache_line_size(), PAGE_SIZE);
310	if (!airq_iv_cache)
311		return -ENOMEM;
312	return 0;
313}