api.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * PCI MSI/MSI-X — Exported APIs for device drivers
  4 *
  5 * Copyright (C) 2003-2004 Intel
  6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
  7 * Copyright (C) 2016 Christoph Hellwig.
  8 * Copyright (C) 2022 Linutronix GmbH
  9 */
 10
 11#include <linux/export.h>
 12#include <linux/irq.h>
 13
 14#include "msi.h"
 15
 16/**
 17 * pci_enable_msi() - Enable MSI interrupt mode on device
 18 * @dev: the PCI device to operate on
 19 *
 20 * Legacy device driver API to enable MSI interrupts mode on device and
 21 * allocate a single interrupt vector. On success, the allocated vector
 22 * Linux IRQ will be saved at @dev->irq. The driver must invoke
 23 * pci_disable_msi() on cleanup.
 24 *
 25 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
 26 * pair should, in general, be used instead.
 27 *
 28 * Return: 0 on success, errno otherwise
 29 */
 30int pci_enable_msi(struct pci_dev *dev)
 31{
 32	int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
 33	if (rc < 0)
 34		return rc;
 35	return 0;
 36}
 37EXPORT_SYMBOL(pci_enable_msi);
 38
 39/**
 40 * pci_disable_msi() - Disable MSI interrupt mode on device
 41 * @dev: the PCI device to operate on
 42 *
 43 * Legacy device driver API to disable MSI interrupt mode on device,
 44 * free earlier allocated interrupt vectors, and restore INTx emulation.
 45 * The PCI device Linux IRQ (@dev->irq) is restored to its default
 46 * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi().
 47 *
 48 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
 49 * pair should, in general, be used instead.
 50 */
 51void pci_disable_msi(struct pci_dev *dev)
 52{
 53	if (!pci_msi_enabled() || !dev || !dev->msi_enabled)
 54		return;
 55
 56	guard(msi_descs_lock)(&dev->dev);
 57	pci_msi_shutdown(dev);
 58	pci_free_msi_irqs(dev);
 59}
 60EXPORT_SYMBOL(pci_disable_msi);
 61
 62/**
 63 * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device
 64 * @dev: the PCI device to operate on
 65 *
 66 * Return: number of MSI-X interrupt vectors available on this device
 67 * (i.e., the device's MSI-X capability structure "table size"), -EINVAL
 68 * if the device is not MSI-X capable, other errnos otherwise.
 69 */
 70int pci_msix_vec_count(struct pci_dev *dev)
 71{
 72	u16 control;
 73
 74	if (!dev->msix_cap)
 75		return -EINVAL;
 76
 77	pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
 78	return msix_table_size(control);
 79}
 80EXPORT_SYMBOL(pci_msix_vec_count);
 81
 82/**
 83 * pci_enable_msix_range() - Enable MSI-X interrupt mode on device
 84 * @dev:     the PCI device to operate on
 85 * @entries: input/output parameter, array of MSI-X configuration entries
 86 * @minvec:  minimum required number of MSI-X vectors
 87 * @maxvec:  maximum desired number of MSI-X vectors
 88 *
 89 * Legacy device driver API to enable MSI-X interrupt mode on device and
 90 * configure its MSI-X capability structure as appropriate.  The passed
 91 * @entries array must have each of its members "entry" field set to a
 92 * desired (valid) MSI-X vector number, where the range of valid MSI-X
 93 * vector numbers can be queried through pci_msix_vec_count().  If
 94 * successful, the driver must invoke pci_disable_msix() on cleanup.
 95 *
 96 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
 97 * pair should, in general, be used instead.
 98 *
 99 * Return: number of MSI-X vectors allocated (which might be smaller
100 * than @maxvecs), where Linux IRQ numbers for such allocated vectors
101 * are saved back in the @entries array elements' "vector" field. Return
102 * -ENOSPC if less than @minvecs interrupt vectors are available.
103 * Return -EINVAL if one of the passed @entries members "entry" field
104 * was invalid or a duplicate, or if plain MSI interrupts mode was
105 * earlier enabled on device. Return other errnos otherwise.
106 */
107int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
108			  int minvec, int maxvec)
109{
110	return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
111}
112EXPORT_SYMBOL(pci_enable_msix_range);
113
114/**
115 * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling
116 *			    MSI-X is supported
117 *
118 * @dev:	PCI device to operate on
119 *
120 * Return: True if supported, false otherwise
121 */
122bool pci_msix_can_alloc_dyn(struct pci_dev *dev)
123{
124	if (!dev->msix_cap)
125		return false;
126
127	return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY);
128}
129EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);
130
131/**
132 * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X
133 *			   at a given MSI-X vector index or any free vector index
134 *
135 * @dev:	PCI device to operate on
136 * @index:	Index to allocate. If @index == MSI_ANY_INDEX this allocates
137 *		the next free index in the MSI-X table
138 * @affdesc:	Optional pointer to an affinity descriptor structure. NULL otherwise
139 *
140 * Return: A struct msi_map
141 *
142 *	On success msi_map::index contains the allocated index (>= 0) and
143 *	msi_map::virq contains the allocated Linux interrupt number (> 0).
144 *
145 *	On fail msi_map::index contains the error code and msi_map::virq
146 *	is set to 0.
147 */
148struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
149				     const struct irq_affinity_desc *affdesc)
150{
151	struct msi_map map = { .index = -ENOTSUPP };
152
153	if (!dev->msix_enabled)
154		return map;
155
156	if (!pci_msix_can_alloc_dyn(dev))
157		return map;
158
159	return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL);
160}
161EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);
162
163/**
164 * pci_msix_free_irq - Free an interrupt on a PCI/MSI-X interrupt domain
165 *
166 * @dev:	The PCI device to operate on
167 * @map:	A struct msi_map describing the interrupt to free
168 *
169 * Undo an interrupt vector allocation. Does not disable MSI-X.
170 */
171void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)
172{
173	if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
174		return;
175	if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))
176		return;
177	msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index);
178}
179EXPORT_SYMBOL_GPL(pci_msix_free_irq);
180
181/**
182 * pci_disable_msix() - Disable MSI-X interrupt mode on device
183 * @dev: the PCI device to operate on
184 *
185 * Legacy device driver API to disable MSI-X interrupt mode on device,
186 * free earlier-allocated interrupt vectors, and restore INTx.
187 * The PCI device Linux IRQ (@dev->irq) is restored to its default pin
188 * assertion IRQ. This is the cleanup pair of pci_enable_msix_range().
189 *
190 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
191 * pair should, in general, be used instead.
192 */
193void pci_disable_msix(struct pci_dev *dev)
194{
195	if (!pci_msi_enabled() || !dev || !dev->msix_enabled)
196		return;
197
198	guard(msi_descs_lock)(&dev->dev);
199	pci_msix_shutdown(dev);
200	pci_free_msi_irqs(dev);
201}
202EXPORT_SYMBOL(pci_disable_msix);
203
204/**
205 * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors
206 * @dev:      the PCI device to operate on
207 * @min_vecs: minimum required number of vectors (must be >= 1)
208 * @max_vecs: maximum desired number of vectors
209 * @flags:    One or more of:
210 *
211 *            * %PCI_IRQ_MSIX      Allow trying MSI-X vector allocations
212 *            * %PCI_IRQ_MSI       Allow trying MSI vector allocations
213 *
214 *            * %PCI_IRQ_INTX      Allow trying INTx interrupts, if and
215 *              only if @min_vecs == 1
216 *
217 *            * %PCI_IRQ_AFFINITY  Auto-manage IRQs affinity by spreading
218 *              the vectors around available CPUs
219 *
220 * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq
221 * vector allocation has a higher precedence over plain MSI, which has a
222 * higher precedence over legacy INTx emulation.
223 *
224 * Upon a successful allocation, the caller should use pci_irq_vector()
225 * to get the Linux IRQ number to be passed to request_threaded_irq().
226 * The driver must call pci_free_irq_vectors() on cleanup.
227 *
228 * Return: number of allocated vectors (which might be smaller than
229 * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are
230 * available, other errnos otherwise.
231 */
232int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
233			  unsigned int max_vecs, unsigned int flags)
234{
235	return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs,
236					      flags, NULL);
237}
238EXPORT_SYMBOL(pci_alloc_irq_vectors);
239
240/**
241 * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt
242 *                                    vectors with affinity requirements
243 * @dev:      the PCI device to operate on
244 * @min_vecs: minimum required number of vectors (must be >= 1)
245 * @max_vecs: maximum desired number of vectors
246 * @flags:    allocation flags, as in pci_alloc_irq_vectors()
247 * @affd:     affinity requirements (can be %NULL).
248 *
249 * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter.
250 * Check that function docs, and &struct irq_affinity, for more details.
251 */
252int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
253				   unsigned int max_vecs, unsigned int flags,
254				   struct irq_affinity *affd)
255{
256	struct irq_affinity msi_default_affd = {0};
257	int nvecs = -ENOSPC;
258
259	if (flags & PCI_IRQ_AFFINITY) {
260		if (!affd)
261			affd = &msi_default_affd;
262	} else {
263		if (WARN_ON(affd))
264			affd = NULL;
265	}
266
267	if (flags & PCI_IRQ_MSIX) {
268		nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
269						affd, flags);
270		if (nvecs > 0)
271			return nvecs;
272	}
273
274	if (flags & PCI_IRQ_MSI) {
275		nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
276		if (nvecs > 0)
277			return nvecs;
278	}
279
280	/* use INTx IRQ if allowed */
281	if (flags & PCI_IRQ_INTX) {
282		if (min_vecs == 1 && dev->irq) {
283			/*
284			 * Invoke the affinity spreading logic to ensure that
285			 * the device driver can adjust queue configuration
286			 * for the single interrupt case.
287			 */
288			if (affd)
289				irq_create_affinity_masks(1, affd);
290			pci_intx(dev, 1);
291			return 1;
292		}
293	}
294
295	return nvecs;
296}
297EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
298
299/**
300 * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector
301 * @dev: the PCI device to operate on
302 * @nr:  device-relative interrupt vector index (0-based); has different
303 *       meanings, depending on interrupt mode:
304 *
305 *         * MSI-X     the index in the MSI-X vector table
306 *         * MSI       the index of the enabled MSI vectors
307 *         * INTx      must be 0
308 *
309 * Return: the Linux IRQ number, or -EINVAL if @nr is out of range
310 */
311int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
312{
313	unsigned int irq;
314
315	if (!dev->msi_enabled && !dev->msix_enabled)
316		return !nr ? dev->irq : -EINVAL;
317
318	irq = msi_get_virq(&dev->dev, nr);
319	return irq ? irq : -EINVAL;
320}
321EXPORT_SYMBOL(pci_irq_vector);
322
323/**
324 * pci_irq_get_affinity() - Get a device interrupt vector affinity
325 * @dev: the PCI device to operate on
326 * @nr:  device-relative interrupt vector index (0-based); has different
327 *       meanings, depending on interrupt mode:
328 *
329 *         * MSI-X     the index in the MSI-X vector table
330 *         * MSI       the index of the enabled MSI vectors
331 *         * INTx      must be 0
332 *
333 * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if
334 * the MSI(-X) vector was allocated without explicit affinity
335 * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or
336 * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a
337 * generic set of CPU IDs representing all possible CPUs available
338 * during system boot if the device is in legacy INTx mode.
339 */
340const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
341{
342	int idx, irq = pci_irq_vector(dev, nr);
343	struct msi_desc *desc;
344
345	if (WARN_ON_ONCE(irq <= 0))
346		return NULL;
347
348	desc = irq_get_msi_desc(irq);
349	/* Non-MSI does not have the information handy */
350	if (!desc)
351		return cpu_possible_mask;
352
353	/* MSI[X] interrupts can be allocated without affinity descriptor */
354	if (!desc->affinity)
355		return NULL;
356
357	/*
358	 * MSI has a mask array in the descriptor.
359	 * MSI-X has a single mask.
360	 */
361	idx = dev->msi_enabled ? nr : 0;
362	return &desc->affinity[idx].mask;
363}
364EXPORT_SYMBOL(pci_irq_get_affinity);
365
366/**
367 * pci_free_irq_vectors() - Free previously allocated IRQs for a device
368 * @dev: the PCI device to operate on
369 *
370 * Undo the interrupt vector allocations and possible device MSI/MSI-X
371 * enablement earlier done through pci_alloc_irq_vectors_affinity() or
372 * pci_alloc_irq_vectors().
373 */
374void pci_free_irq_vectors(struct pci_dev *dev)
375{
376	pci_disable_msix(dev);
377	pci_disable_msi(dev);
378}
379EXPORT_SYMBOL(pci_free_irq_vectors);
380
381/**
382 * pci_restore_msi_state() - Restore cached MSI(-X) state on device
383 * @dev: the PCI device to operate on
384 *
385 * Write the Linux-cached MSI(-X) state back on device. This is
386 * typically useful upon system resume, or after an error-recovery PCI
387 * adapter reset.
388 */
389void pci_restore_msi_state(struct pci_dev *dev)
390{
391	__pci_restore_msi_state(dev);
392	__pci_restore_msix_state(dev);
393}
394EXPORT_SYMBOL_GPL(pci_restore_msi_state);
395
396/**
397 * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide?
398 *
399 * Return: true if MSI has not been globally disabled through ACPI FADT,
400 * PCI bridge quirks, or the "pci=nomsi" kernel command-line option.
401 */
402bool pci_msi_enabled(void)
403{
404	return pci_msi_enable;
405}
406EXPORT_SYMBOL(pci_msi_enabled);