irqbypass.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * IRQ offload/bypass manager
  4 *
  5 * Copyright (C) 2015 Red Hat, Inc.
  6 * Copyright (c) 2015 Linaro Ltd.
  7 *
  8 * Various virtualization hardware acceleration techniques allow bypassing or
  9 * offloading interrupts received from devices around the host kernel.  Posted
 10 * Interrupts on Intel VT-d systems can allow interrupts to be received
 11 * directly by a virtual machine.  ARM IRQ Forwarding allows forwarded physical
 12 * interrupts to be directly deactivated by the guest.  This manager allows
 13 * interrupt producers and consumers to find each other to enable this sort of
 14 * bypass.
 15 */
 16
 17#include <linux/irqbypass.h>
 18#include <linux/list.h>
 19#include <linux/module.h>
 20#include <linux/mutex.h>
 21
 22MODULE_LICENSE("GPL v2");
 23MODULE_DESCRIPTION("IRQ bypass manager utility module");
 24
 25static DEFINE_XARRAY(producers);
 26static DEFINE_XARRAY(consumers);
 27static DEFINE_MUTEX(lock);
 28
 29/* @lock must be held when calling connect */
 30static int __connect(struct irq_bypass_producer *prod,
 31		     struct irq_bypass_consumer *cons)
 32{
 33	int ret = 0;
 34
 35	if (prod->stop)
 36		prod->stop(prod);
 37	if (cons->stop)
 38		cons->stop(cons);
 39
 40	if (prod->add_consumer)
 41		ret = prod->add_consumer(prod, cons);
 42
 43	if (!ret) {
 44		ret = cons->add_producer(cons, prod);
 45		if (ret && prod->del_consumer)
 46			prod->del_consumer(prod, cons);
 47	}
 48
 49	if (cons->start)
 50		cons->start(cons);
 51	if (prod->start)
 52		prod->start(prod);
 53
 54	if (!ret) {
 55		prod->consumer = cons;
 56		cons->producer = prod;
 57	}
 58	return ret;
 59}
 60
 61/* @lock must be held when calling disconnect */
 62static void __disconnect(struct irq_bypass_producer *prod,
 63			 struct irq_bypass_consumer *cons)
 64{
 65	if (prod->stop)
 66		prod->stop(prod);
 67	if (cons->stop)
 68		cons->stop(cons);
 69
 70	cons->del_producer(cons, prod);
 71
 72	if (prod->del_consumer)
 73		prod->del_consumer(prod, cons);
 74
 75	if (cons->start)
 76		cons->start(cons);
 77	if (prod->start)
 78		prod->start(prod);
 79
 80	prod->consumer = NULL;
 81	cons->producer = NULL;
 82}
 83
 84/**
 85 * irq_bypass_register_producer - register IRQ bypass producer
 86 * @producer: pointer to producer structure
 87 * @eventfd: pointer to the eventfd context associated with the producer
 88 * @irq: Linux IRQ number of the underlying producer device
 89 *
 90 * Add the provided IRQ producer to the set of producers and connect with the
 91 * consumer with a matching eventfd, if one exists.
 92 */
 93int irq_bypass_register_producer(struct irq_bypass_producer *producer,
 94				 struct eventfd_ctx *eventfd, int irq)
 95{
 96	unsigned long index = (unsigned long)eventfd;
 97	struct irq_bypass_consumer *consumer;
 98	int ret;
 99
100	if (WARN_ON_ONCE(producer->eventfd))
101		return -EINVAL;
102
103	producer->irq = irq;
104
105	guard(mutex)(&lock);
106
107	ret = xa_insert(&producers, index, producer, GFP_KERNEL);
108	if (ret)
109		return ret;
110
111	consumer = xa_load(&consumers, index);
112	if (consumer) {
113		ret = __connect(producer, consumer);
114		if (ret) {
115			WARN_ON_ONCE(xa_erase(&producers, index) != producer);
116			return ret;
117		}
118	}
119
120	producer->eventfd = eventfd;
121	return 0;
122}
123EXPORT_SYMBOL_GPL(irq_bypass_register_producer);
124
125/**
126 * irq_bypass_unregister_producer - unregister IRQ bypass producer
127 * @producer: pointer to producer structure
128 *
129 * Remove a previously registered IRQ producer (note, it's safe to call this
130 * even if registration was unsuccessful).  Disconnect from the associated
131 * consumer, if one exists.
132 */
133void irq_bypass_unregister_producer(struct irq_bypass_producer *producer)
134{
135	unsigned long index = (unsigned long)producer->eventfd;
136
137	if (!producer->eventfd)
138		return;
139
140	guard(mutex)(&lock);
141
142	if (producer->consumer)
143		__disconnect(producer, producer->consumer);
144
145	WARN_ON_ONCE(xa_erase(&producers, index) != producer);
146	producer->eventfd = NULL;
147}
148EXPORT_SYMBOL_GPL(irq_bypass_unregister_producer);
149
150/**
151 * irq_bypass_register_consumer - register IRQ bypass consumer
152 * @consumer: pointer to consumer structure
153 * @eventfd: pointer to the eventfd context associated with the consumer
154 *
155 * Add the provided IRQ consumer to the set of consumers and connect with the
156 * producer with a matching eventfd, if one exists.
157 */
158int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer,
159				 struct eventfd_ctx *eventfd)
160{
161	unsigned long index = (unsigned long)eventfd;
162	struct irq_bypass_producer *producer;
163	int ret;
164
165	if (WARN_ON_ONCE(consumer->eventfd))
166		return -EINVAL;
167
168	if (!consumer->add_producer || !consumer->del_producer)
169		return -EINVAL;
170
171	guard(mutex)(&lock);
172
173	ret = xa_insert(&consumers, index, consumer, GFP_KERNEL);
174	if (ret)
175		return ret;
176
177	producer = xa_load(&producers, index);
178	if (producer) {
179		ret = __connect(producer, consumer);
180		if (ret) {
181			WARN_ON_ONCE(xa_erase(&consumers, index) != consumer);
182			return ret;
183		}
184	}
185
186	consumer->eventfd = eventfd;
187	return 0;
188}
189EXPORT_SYMBOL_GPL(irq_bypass_register_consumer);
190
191/**
192 * irq_bypass_unregister_consumer - unregister IRQ bypass consumer
193 * @consumer: pointer to consumer structure
194 *
195 * Remove a previously registered IRQ consumer (note, it's safe to call this
196 * even if registration was unsuccessful).  Disconnect from the associated
197 * producer, if one exists.
198 */
199void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer)
200{
201	unsigned long index = (unsigned long)consumer->eventfd;
202
203	if (!consumer->eventfd)
204		return;
205
206	guard(mutex)(&lock);
207
208	if (consumer->producer)
209		__disconnect(consumer->producer, consumer);
210
211	WARN_ON_ONCE(xa_erase(&consumers, index) != consumer);
212	consumer->eventfd = NULL;
213}
214EXPORT_SYMBOL_GPL(irq_bypass_unregister_consumer);