irq-riscv-imsic-early.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  4 * Copyright (C) 2022 Ventana Micro Systems Inc.
  5 */
  6
  7#define pr_fmt(fmt) "riscv-imsic: " fmt
  8#include <linux/acpi.h>
  9#include <linux/cpu.h>
 10#include <linux/cpu_pm.h>
 11#include <linux/export.h>
 12#include <linux/interrupt.h>
 13#include <linux/init.h>
 14#include <linux/io.h>
 15#include <linux/irq.h>
 16#include <linux/irqchip.h>
 17#include <linux/irqchip/chained_irq.h>
 18#include <linux/irqchip/riscv-imsic.h>
 19#include <linux/module.h>
 20#include <linux/pci.h>
 21#include <linux/spinlock.h>
 22#include <linux/smp.h>
 23
 24#include "irq-riscv-imsic-state.h"
 25
 26static int imsic_parent_irq;
 27bool imsic_noipi __ro_after_init;
 28
 29static int __init imsic_noipi_cfg(char *buf)
 30{
 31	imsic_noipi = true;
 32	return 0;
 33}
 34early_param("irqchip.riscv_imsic_noipi", imsic_noipi_cfg);
 35
 36#ifdef CONFIG_SMP
 37static void imsic_ipi_send(unsigned int cpu)
 38{
 39	struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu);
 40
 41	writel(IMSIC_IPI_ID, local->msi_va);
 42}
 43
 44static void imsic_ipi_starting_cpu(void)
 45{
 46	if (imsic_noipi)
 47		return;
 48
 49	/* Enable IPIs for current CPU. */
 50	__imsic_id_set_enable(IMSIC_IPI_ID);
 51}
 52
 53static void imsic_ipi_dying_cpu(void)
 54{
 55	if (imsic_noipi)
 56		return;
 57
 58	/* Disable IPIs for current CPU. */
 59	__imsic_id_clear_enable(IMSIC_IPI_ID);
 60}
 61
 62static int __init imsic_ipi_domain_init(void)
 63{
 64	int virq;
 65
 66	if (imsic_noipi)
 67		return 0;
 68
 69	/* Create IMSIC IPI multiplexing */
 70	virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send);
 71	if (virq <= 0)
 72		return virq < 0 ? virq : -ENOMEM;
 73
 74	/* Set vIRQ range */
 75	riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI);
 76
 77	/* Announce that IMSIC is providing IPIs */
 78	pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID);
 79
 80	return 0;
 81}
 82#else
 83static void imsic_ipi_starting_cpu(void) { }
 84static void imsic_ipi_dying_cpu(void) { }
 85static int __init imsic_ipi_domain_init(void) { return 0; }
 86#endif
 87
 88/*
 89 * To handle an interrupt, we read the TOPEI CSR and write zero in one
 90 * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to
 91 * Linux interrupt number and let Linux IRQ subsystem handle it.
 92 */
 93static void imsic_handle_irq(struct irq_desc *desc)
 94{
 95	struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv);
 96	struct irq_chip *chip = irq_desc_get_chip(desc);
 97	unsigned long local_id;
 98
 99	/*
100	 * Process pending local synchronization instead of waiting
101	 * for per-CPU local timer to expire.
102	 */
103	imsic_local_sync_all(false);
104
105	chained_irq_enter(chip, desc);
106
107	while ((local_id = csr_swap(CSR_TOPEI, 0))) {
108		local_id >>= TOPEI_ID_SHIFT;
109
110		if (!imsic_noipi && local_id == IMSIC_IPI_ID) {
111			if (IS_ENABLED(CONFIG_SMP))
112				ipi_mux_process();
113			continue;
114		}
115
116		if (unlikely(local_id > imsic->global.nr_ids)) {
117			pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id);
118			continue;
119		}
120
121		generic_handle_irq(lpriv->vectors[local_id].irq);
122	}
123
124	chained_irq_exit(chip, desc);
125}
126
127static void imsic_hw_states_init(void)
128{
129	/* Setup IPIs */
130	imsic_ipi_starting_cpu();
131
132	/*
133	 * Interrupts identities might have been enabled/disabled while
134	 * this CPU was not running so sync-up local enable/disable state.
135	 */
136	imsic_local_sync_all(true);
137
138	/* Enable local interrupt delivery */
139	imsic_local_delivery(true);
140}
141
142static int imsic_starting_cpu(unsigned int cpu)
143{
144	/* Mark per-CPU IMSIC state as online */
145	imsic_state_online();
146
147	/* Enable per-CPU parent interrupt */
148	enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
149
150	/* Initialize the IMSIC registers to enable the interrupt delivery */
151	imsic_hw_states_init();
152
153	return 0;
154}
155
156static int imsic_dying_cpu(unsigned int cpu)
157{
158	/* Cleanup IPIs */
159	imsic_ipi_dying_cpu();
160
161	/* Mark per-CPU IMSIC state as offline */
162	imsic_state_offline();
163
164	return 0;
165}
166
167static int imsic_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v)
168{
169	switch (cmd) {
170	case CPU_PM_EXIT:
171		/* Initialize the IMSIC registers to enable the interrupt delivery */
172		imsic_hw_states_init();
173		break;
174	}
175
176	return NOTIFY_OK;
177}
178
179static struct notifier_block imsic_pm_notifier_block = {
180	.notifier_call = imsic_pm_notifier,
181};
182
183static int __init imsic_early_probe(struct fwnode_handle *fwnode)
184{
185	struct irq_domain *domain;
186	int rc;
187
188	/* Find parent domain and register chained handler */
189	domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
190	if (!domain) {
191		pr_err("%pfwP: Failed to find INTC domain\n", fwnode);
192		return -ENOENT;
193	}
194	imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
195	if (!imsic_parent_irq) {
196		pr_err("%pfwP: Failed to create INTC mapping\n", fwnode);
197		return -ENOENT;
198	}
199
200	/* Initialize IPI domain */
201	rc = imsic_ipi_domain_init();
202	if (rc) {
203		pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode);
204		return rc;
205	}
206
207	/* Setup chained handler to the parent domain interrupt */
208	irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq);
209
210	/*
211	 * Setup cpuhp state (must be done after setting imsic_parent_irq)
212	 *
213	 * Don't disable per-CPU IMSIC file when CPU goes offline
214	 * because this affects IPI and the masking/unmasking of
215	 * virtual IPIs is done via generic IPI-Mux
216	 */
217	cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting",
218			  imsic_starting_cpu, imsic_dying_cpu);
219
220	return cpu_pm_register_notifier(&imsic_pm_notifier_block);
221}
222
223static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent)
224{
225	struct fwnode_handle *fwnode = &node->fwnode;
226	int rc;
227
228	/* Setup IMSIC state */
229	rc = imsic_setup_state(fwnode, NULL);
230	if (rc) {
231		pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc);
232		return rc;
233	}
234
235	/* Do early setup of IPIs */
236	rc = imsic_early_probe(fwnode);
237	if (rc)
238		return rc;
239
240	/* Ensure that OF platform device gets probed */
241	of_node_clear_flag(node, OF_POPULATED);
242	return 0;
243}
244
245IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init);
246
247#ifdef CONFIG_ACPI
248
249static struct fwnode_handle *imsic_acpi_fwnode;
250
251struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev)
252{
253	return imsic_acpi_fwnode;
254}
255EXPORT_SYMBOL_GPL(imsic_acpi_get_fwnode);
256
257static int __init imsic_early_acpi_init(union acpi_subtable_headers *header,
258					const unsigned long end)
259{
260	struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header;
261	int rc;
262
263	imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic");
264	if (!imsic_acpi_fwnode) {
265		pr_err("unable to allocate IMSIC FW node\n");
266		return -ENOMEM;
267	}
268
269	/* Setup IMSIC state */
270	rc = imsic_setup_state(imsic_acpi_fwnode, imsic);
271	if (rc) {
272		pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc);
273		return rc;
274	}
275
276	/* Do early setup of IMSIC state and IPIs */
277	rc = imsic_early_probe(imsic_acpi_fwnode);
278	if (rc) {
279		irq_domain_free_fwnode(imsic_acpi_fwnode);
280		imsic_acpi_fwnode = NULL;
281		return rc;
282	}
283
284	rc = imsic_platform_acpi_probe(imsic_acpi_fwnode);
285
286#ifdef CONFIG_PCI
287	if (!rc)
288		pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode);
289#endif
290
291	if (rc)
292		pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n",
293		       imsic_acpi_fwnode, rc);
294
295	/*
296	 * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can
297	 * continue to work. So, no need to return failure. This is similar to
298	 * DT where IPI works but MSI probe fails for some reason.
299	 */
300	return 0;
301}
302
303IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL,
304		     1, imsic_early_acpi_init);
305#endif