irq-gic-v2m.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * ARM GIC v2m MSI(-X) support
  4 * Support for Message Signaled Interrupts for systems that
  5 * implement ARM Generic Interrupt Controller: GICv2m.
  6 *
  7 * Copyright (C) 2014 Advanced Micro Devices, Inc.
  8 * Authors: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  9 *	    Harish Kasiviswanathan <harish.kasiviswanathan@amd.com>
 10 *	    Brandon Anderson <brandon.anderson@amd.com>
 11 */
 12
 13#define pr_fmt(fmt) "GICv2m: " fmt
 14
 15#include <linux/acpi.h>
 16#include <linux/iommu.h>
 17#include <linux/irq.h>
 18#include <linux/irqdomain.h>
 19#include <linux/kernel.h>
 20#include <linux/pci.h>
 21#include <linux/msi.h>
 22#include <linux/of_address.h>
 23#include <linux/of_pci.h>
 24#include <linux/slab.h>
 25#include <linux/spinlock.h>
 26#include <linux/irqchip/arm-gic.h>
 27#include <linux/irqchip/arm-gic-common.h>
 28
 29#include <linux/irqchip/irq-msi-lib.h>
 30
 31/*
 32* MSI_TYPER:
 33*     [31:26] Reserved
 34*     [25:16] lowest SPI assigned to MSI
 35*     [15:10] Reserved
 36*     [9:0]   Numer of SPIs assigned to MSI
 37*/
 38#define V2M_MSI_TYPER		       0x008
 39#define V2M_MSI_TYPER_BASE_SHIFT       16
 40#define V2M_MSI_TYPER_BASE_MASK	       0x3FF
 41#define V2M_MSI_TYPER_NUM_MASK	       0x3FF
 42#define V2M_MSI_SETSPI_NS	       0x040
 43#define V2M_MIN_SPI		       32
 44#define V2M_MAX_SPI		       1019
 45#define V2M_MSI_IIDR		       0xFCC
 46
 47#define V2M_MSI_TYPER_BASE_SPI(x)      \
 48	       (((x) >> V2M_MSI_TYPER_BASE_SHIFT) & V2M_MSI_TYPER_BASE_MASK)
 49
 50#define V2M_MSI_TYPER_NUM_SPI(x)       ((x) & V2M_MSI_TYPER_NUM_MASK)
 51
 52/* APM X-Gene with GICv2m MSI_IIDR register value */
 53#define XGENE_GICV2M_MSI_IIDR		0x06000170
 54
 55/* Broadcom NS2 GICv2m MSI_IIDR register value */
 56#define BCM_NS2_GICV2M_MSI_IIDR		0x0000013f
 57
 58/* List of flags for specific v2m implementation */
 59#define GICV2M_NEEDS_SPI_OFFSET		0x00000001
 60#define GICV2M_GRAVITON_ADDRESS_ONLY	0x00000002
 61
 62static LIST_HEAD(v2m_nodes);
 63static DEFINE_SPINLOCK(v2m_lock);
 64
 65struct v2m_data {
 66	struct list_head entry;
 67	struct fwnode_handle *fwnode;
 68	struct resource res;	/* GICv2m resource */
 69	void __iomem *base;	/* GICv2m virt address */
 70	u32 spi_start;		/* The SPI number that MSIs start */
 71	u32 nr_spis;		/* The number of SPIs for MSIs */
 72	u32 spi_offset;		/* offset to be subtracted from SPI number */
 73	unsigned long *bm;	/* MSI vector bitmap */
 74	u32 flags;		/* v2m flags for specific implementation */
 75};
 76
 77static phys_addr_t gicv2m_get_msi_addr(struct v2m_data *v2m, int hwirq)
 78{
 79	if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)
 80		return v2m->res.start | ((hwirq - 32) << 3);
 81	else
 82		return v2m->res.start + V2M_MSI_SETSPI_NS;
 83}
 84
 85static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 86{
 87	struct v2m_data *v2m = irq_data_get_irq_chip_data(data);
 88	phys_addr_t addr = gicv2m_get_msi_addr(v2m, data->hwirq);
 89
 90	if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)
 91		msg->data = 0;
 92	else
 93		msg->data = data->hwirq;
 94	if (v2m->flags & GICV2M_NEEDS_SPI_OFFSET)
 95		msg->data -= v2m->spi_offset;
 96
 97	msi_msg_set_addr(irq_data_get_msi_desc(data), msg, addr);
 98}
 99
100static struct irq_chip gicv2m_irq_chip = {
101	.name			= "GICv2m",
102	.irq_mask		= irq_chip_mask_parent,
103	.irq_unmask		= irq_chip_unmask_parent,
104	.irq_eoi		= irq_chip_eoi_parent,
105	.irq_set_affinity	= irq_chip_set_affinity_parent,
106	.irq_compose_msi_msg	= gicv2m_compose_msi_msg,
107};
108
109static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain,
110				       unsigned int virq,
111				       irq_hw_number_t hwirq)
112{
113	struct irq_fwspec fwspec;
114	struct irq_data *d;
115	int err;
116
117	if (is_of_node(domain->parent->fwnode)) {
118		fwspec.fwnode = domain->parent->fwnode;
119		fwspec.param_count = 3;
120		fwspec.param[0] = 0;
121		fwspec.param[1] = hwirq - 32;
122		fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
123	} else if (is_fwnode_irqchip(domain->parent->fwnode)) {
124		fwspec.fwnode = domain->parent->fwnode;
125		fwspec.param_count = 2;
126		fwspec.param[0] = hwirq;
127		fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
128	} else {
129		return -EINVAL;
130	}
131
132	err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
133	if (err)
134		return err;
135
136	/* Configure the interrupt line to be edge */
137	d = irq_domain_get_irq_data(domain->parent, virq);
138	d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING);
139	return 0;
140}
141
142static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq,
143			       int nr_irqs)
144{
145	spin_lock(&v2m_lock);
146	bitmap_release_region(v2m->bm, hwirq - v2m->spi_start,
147			      get_count_order(nr_irqs));
148	spin_unlock(&v2m_lock);
149}
150
151static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
152				   unsigned int nr_irqs, void *args)
153{
154	msi_alloc_info_t *info = args;
155	struct v2m_data *v2m = NULL, *tmp;
156	int hwirq, i, err = 0;
157	unsigned long offset;
158	unsigned long align_mask = nr_irqs - 1;
159
160	spin_lock(&v2m_lock);
161	list_for_each_entry(tmp, &v2m_nodes, entry) {
162		unsigned long align_off = tmp->spi_start - (tmp->spi_start & ~align_mask);
163
164		offset = bitmap_find_next_zero_area_off(tmp->bm, tmp->nr_spis, 0,
165							nr_irqs, align_mask, align_off);
166		if (offset < tmp->nr_spis) {
167			v2m = tmp;
168			bitmap_set(v2m->bm, offset, nr_irqs);
169			break;
170		}
171	}
172	spin_unlock(&v2m_lock);
173
174	if (!v2m)
175		return -ENOSPC;
176
177	hwirq = v2m->spi_start + offset;
178
179	err = iommu_dma_prepare_msi(info->desc,
180				    gicv2m_get_msi_addr(v2m, hwirq));
181	if (err)
182		return err;
183
184	for (i = 0; i < nr_irqs; i++) {
185		err = gicv2m_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
186		if (err)
187			goto fail;
188
189		irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
190					      &gicv2m_irq_chip, v2m);
191	}
192
193	return 0;
194
195fail:
196	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
197	gicv2m_unalloc_msi(v2m, hwirq, nr_irqs);
198	return err;
199}
200
201static void gicv2m_irq_domain_free(struct irq_domain *domain,
202				   unsigned int virq, unsigned int nr_irqs)
203{
204	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
205	struct v2m_data *v2m = irq_data_get_irq_chip_data(d);
206
207	gicv2m_unalloc_msi(v2m, d->hwirq, nr_irqs);
208	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
209}
210
211static const struct irq_domain_ops gicv2m_domain_ops = {
212	.select			= msi_lib_irq_domain_select,
213	.alloc			= gicv2m_irq_domain_alloc,
214	.free			= gicv2m_irq_domain_free,
215};
216
217static bool is_msi_spi_valid(u32 base, u32 num)
218{
219	if (base < V2M_MIN_SPI) {
220		pr_err("Invalid MSI base SPI (base:%u)\n", base);
221		return false;
222	}
223
224	if ((num == 0) || (base + num > V2M_MAX_SPI)) {
225		pr_err("Number of SPIs (%u) exceed maximum (%u)\n",
226		       num, V2M_MAX_SPI - V2M_MIN_SPI + 1);
227		return false;
228	}
229
230	return true;
231}
232
233static void __init gicv2m_teardown(void)
234{
235	struct v2m_data *v2m, *tmp;
236
237	list_for_each_entry_safe(v2m, tmp, &v2m_nodes, entry) {
238		list_del(&v2m->entry);
239		bitmap_free(v2m->bm);
240		iounmap(v2m->base);
241		of_node_put(to_of_node(v2m->fwnode));
242		if (is_fwnode_irqchip(v2m->fwnode))
243			irq_domain_free_fwnode(v2m->fwnode);
244		kfree(v2m);
245	}
246}
247
248
249#define GICV2M_MSI_FLAGS_REQUIRED  (MSI_FLAG_USE_DEF_DOM_OPS |		\
250				    MSI_FLAG_USE_DEF_CHIP_OPS |		\
251				    MSI_FLAG_PCI_MSI_MASK_PARENT)
252
253#define GICV2M_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK |	\
254				    MSI_FLAG_PCI_MSIX      |	\
255				    MSI_FLAG_MULTI_PCI_MSI)
256
257static struct msi_parent_ops gicv2m_msi_parent_ops = {
258	.supported_flags	= GICV2M_MSI_FLAGS_SUPPORTED,
259	.required_flags		= GICV2M_MSI_FLAGS_REQUIRED,
260	.chip_flags		= MSI_CHIP_FLAG_SET_EOI,
261	.bus_select_token	= DOMAIN_BUS_NEXUS,
262	.bus_select_mask	= MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
263	.prefix			= "GICv2m-",
264	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
265};
266
267static __init int gicv2m_allocate_domains(struct irq_domain *parent)
268{
269	struct irq_domain_info info = {
270		.ops		= &gicv2m_domain_ops,
271		.parent		= parent,
272	};
273	struct v2m_data *v2m;
274
275	v2m = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry);
276	if (!v2m)
277		return 0;
278
279	info.host_data = v2m;
280	info.fwnode = v2m->fwnode;
281
282	if (!msi_create_parent_irq_domain(&info, &gicv2m_msi_parent_ops)) {
283		pr_err("Failed to create GICv2m domain\n");
284		return -ENOMEM;
285	}
286	return 0;
287}
288
289static int __init gicv2m_init_one(struct fwnode_handle *fwnode,
290				  u32 spi_start, u32 nr_spis,
291				  struct resource *res, u32 flags)
292{
293	int ret;
294	struct v2m_data *v2m;
295
296	v2m = kzalloc(sizeof(struct v2m_data), GFP_KERNEL);
297	if (!v2m)
298		return -ENOMEM;
299
300	INIT_LIST_HEAD(&v2m->entry);
301	v2m->fwnode = fwnode;
302	v2m->flags = flags;
303
304	memcpy(&v2m->res, res, sizeof(struct resource));
305
306	v2m->base = ioremap(v2m->res.start, resource_size(&v2m->res));
307	if (!v2m->base) {
308		pr_err("Failed to map GICv2m resource\n");
309		ret = -ENOMEM;
310		goto err_free_v2m;
311	}
312
313	if (spi_start && nr_spis) {
314		v2m->spi_start = spi_start;
315		v2m->nr_spis = nr_spis;
316	} else {
317		u32 typer;
318
319		/* Graviton should always have explicit spi_start/nr_spis */
320		if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) {
321			ret = -EINVAL;
322			goto err_iounmap;
323		}
324		typer = readl_relaxed(v2m->base + V2M_MSI_TYPER);
325
326		v2m->spi_start = V2M_MSI_TYPER_BASE_SPI(typer);
327		v2m->nr_spis = V2M_MSI_TYPER_NUM_SPI(typer);
328	}
329
330	if (!is_msi_spi_valid(v2m->spi_start, v2m->nr_spis)) {
331		ret = -EINVAL;
332		goto err_iounmap;
333	}
334
335	/*
336	 * APM X-Gene GICv2m implementation has an erratum where
337	 * the MSI data needs to be the offset from the spi_start
338	 * in order to trigger the correct MSI interrupt. This is
339	 * different from the standard GICv2m implementation where
340	 * the MSI data is the absolute value within the range from
341	 * spi_start to (spi_start + num_spis).
342	 *
343	 * Broadcom NS2 GICv2m implementation has an erratum where the MSI data
344	 * is 'spi_number - 32'
345	 *
346	 * Reading that register fails on the Graviton implementation
347	 */
348	if (!(v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)) {
349		switch (readl_relaxed(v2m->base + V2M_MSI_IIDR)) {
350		case XGENE_GICV2M_MSI_IIDR:
351			v2m->flags |= GICV2M_NEEDS_SPI_OFFSET;
352			v2m->spi_offset = v2m->spi_start;
353			break;
354		case BCM_NS2_GICV2M_MSI_IIDR:
355			v2m->flags |= GICV2M_NEEDS_SPI_OFFSET;
356			v2m->spi_offset = 32;
357			break;
358		}
359	}
360	v2m->bm = bitmap_zalloc(v2m->nr_spis, GFP_KERNEL);
361	if (!v2m->bm) {
362		ret = -ENOMEM;
363		goto err_iounmap;
364	}
365
366	list_add_tail(&v2m->entry, &v2m_nodes);
367
368	pr_info("range%pR, SPI[%d:%d]\n", res,
369		v2m->spi_start, (v2m->spi_start + v2m->nr_spis - 1));
370	return 0;
371
372err_iounmap:
373	iounmap(v2m->base);
374err_free_v2m:
375	kfree(v2m);
376	return ret;
377}
378
379static __initconst struct of_device_id gicv2m_device_id[] = {
380	{	.compatible	= "arm,gic-v2m-frame",	},
381	{},
382};
383
384static int __init gicv2m_of_init(struct fwnode_handle *parent_handle,
385				 struct irq_domain *parent)
386{
387	int ret = 0;
388	struct device_node *node = to_of_node(parent_handle);
389	struct device_node *child;
390
391	for (child = of_find_matching_node(node, gicv2m_device_id); child;
392	     child = of_find_matching_node(child, gicv2m_device_id)) {
393		u32 spi_start = 0, nr_spis = 0;
394		struct resource res;
395
396		if (!of_property_read_bool(child, "msi-controller"))
397			continue;
398
399		ret = of_address_to_resource(child, 0, &res);
400		if (ret) {
401			pr_err("Failed to allocate v2m resource.\n");
402			break;
403		}
404
405		if (!of_property_read_u32(child, "arm,msi-base-spi",
406					  &spi_start) &&
407		    !of_property_read_u32(child, "arm,msi-num-spis", &nr_spis))
408			pr_info("DT overriding V2M MSI_TYPER (base:%u, num:%u)\n",
409				spi_start, nr_spis);
410
411		ret = gicv2m_init_one(&child->fwnode, spi_start, nr_spis,
412				      &res, 0);
413		if (ret)
414			break;
415	}
416
417	if (ret && child)
418		of_node_put(child);
419	if (!ret)
420		ret = gicv2m_allocate_domains(parent);
421	if (ret)
422		gicv2m_teardown();
423	return ret;
424}
425
426#ifdef CONFIG_ACPI
427static int acpi_num_msi;
428
429static struct fwnode_handle *gicv2m_get_fwnode(struct device *dev)
430{
431	struct v2m_data *data;
432
433	if (WARN_ON(acpi_num_msi <= 0))
434		return NULL;
435
436	/* We only return the fwnode of the first MSI frame. */
437	data = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry);
438	if (!data)
439		return NULL;
440
441	return data->fwnode;
442}
443
444static __init bool acpi_check_amazon_graviton_quirks(void)
445{
446	static struct acpi_table_madt *madt;
447	acpi_status status;
448	bool rc = false;
449
450#define ACPI_AMZN_OEM_ID		"AMAZON"
451
452	status = acpi_get_table(ACPI_SIG_MADT, 0,
453				(struct acpi_table_header **)&madt);
454
455	if (ACPI_FAILURE(status) || !madt)
456		return rc;
457	rc = !memcmp(madt->header.oem_id, ACPI_AMZN_OEM_ID, ACPI_OEM_ID_SIZE);
458	acpi_put_table((struct acpi_table_header *)madt);
459
460	return rc;
461}
462
463static int __init
464acpi_parse_madt_msi(union acpi_subtable_headers *header,
465		    const unsigned long end)
466{
467	int ret;
468	struct resource res;
469	u32 spi_start = 0, nr_spis = 0;
470	struct acpi_madt_generic_msi_frame *m;
471	struct fwnode_handle *fwnode;
472	u32 flags = 0;
473
474	m = (struct acpi_madt_generic_msi_frame *)header;
475	if (BAD_MADT_ENTRY(m, end))
476		return -EINVAL;
477
478	res.start = m->base_address;
479	res.end = m->base_address + SZ_4K - 1;
480	res.flags = IORESOURCE_MEM;
481
482	if (acpi_check_amazon_graviton_quirks()) {
483		pr_info("applying Amazon Graviton quirk\n");
484		res.end = res.start + SZ_8K - 1;
485		flags |= GICV2M_GRAVITON_ADDRESS_ONLY;
486		gicv2m_msi_parent_ops.supported_flags &= ~MSI_FLAG_MULTI_PCI_MSI;
487	}
488
489	if (m->flags & ACPI_MADT_OVERRIDE_SPI_VALUES) {
490		spi_start = m->spi_base;
491		nr_spis = m->spi_count;
492
493		pr_info("ACPI overriding V2M MSI_TYPER (base:%u, num:%u)\n",
494			spi_start, nr_spis);
495	}
496
497	fwnode = irq_domain_alloc_fwnode(&res.start);
498	if (!fwnode) {
499		pr_err("Unable to allocate GICv2m domain token\n");
500		return -EINVAL;
501	}
502
503	ret = gicv2m_init_one(fwnode, spi_start, nr_spis, &res, flags);
504	if (ret)
505		irq_domain_free_fwnode(fwnode);
506
507	return ret;
508}
509
510static int __init gicv2m_acpi_init(struct irq_domain *parent)
511{
512	int ret;
513
514	if (acpi_num_msi > 0)
515		return 0;
516
517	acpi_num_msi = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_MSI_FRAME,
518				      acpi_parse_madt_msi, 0);
519
520	if (acpi_num_msi <= 0)
521		goto err_out;
522
523	ret = gicv2m_allocate_domains(parent);
524	if (ret)
525		goto err_out;
526
527	pci_msi_register_fwnode_provider(&gicv2m_get_fwnode);
528
529	return 0;
530
531err_out:
532	gicv2m_teardown();
533	return -EINVAL;
534}
535#else /* CONFIG_ACPI */
536static int __init gicv2m_acpi_init(struct irq_domain *parent)
537{
538	return -EINVAL;
539}
540#endif /* CONFIG_ACPI */
541
542int __init gicv2m_init(struct fwnode_handle *parent_handle,
543		       struct irq_domain *parent)
544{
545	if (is_of_node(parent_handle))
546		return gicv2m_of_init(parent_handle, parent);
547
548	return gicv2m_acpi_init(parent);
549}