irq-riscv-imsic-platform.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  4 * Copyright (C) 2022 Ventana Micro Systems Inc.
  5 */
  6
  7#define pr_fmt(fmt) "riscv-imsic: " fmt
  8#include <linux/acpi.h>
  9#include <linux/bitmap.h>
 10#include <linux/cpu.h>
 11#include <linux/interrupt.h>
 12#include <linux/io.h>
 13#include <linux/irq.h>
 14#include <linux/irqchip.h>
 15#include <linux/irqdomain.h>
 16#include <linux/module.h>
 17#include <linux/msi.h>
 18#include <linux/pci.h>
 19#include <linux/platform_device.h>
 20#include <linux/spinlock.h>
 21#include <linux/smp.h>
 22
 23#include <linux/irqchip/irq-msi-lib.h>
 24#include "irq-riscv-imsic-state.h"
 25
 26static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
 27				phys_addr_t *out_msi_pa)
 28{
 29	struct imsic_global_config *global;
 30	struct imsic_local_config *local;
 31
 32	global = &imsic->global;
 33	local = per_cpu_ptr(global->local, cpu);
 34
 35	if (BIT(global->guest_index_bits) <= guest_index)
 36		return false;
 37
 38	if (out_msi_pa)
 39		*out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);
 40
 41	return true;
 42}
 43
 44static void imsic_irq_mask(struct irq_data *d)
 45{
 46	imsic_vector_mask(irq_data_get_irq_chip_data(d));
 47}
 48
 49static void imsic_irq_unmask(struct irq_data *d)
 50{
 51	imsic_vector_unmask(irq_data_get_irq_chip_data(d));
 52}
 53
 54static int imsic_irq_retrigger(struct irq_data *d)
 55{
 56	struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
 57	struct imsic_local_config *local;
 58
 59	if (WARN_ON(!vec))
 60		return -ENOENT;
 61
 62	local = per_cpu_ptr(imsic->global.local, vec->cpu);
 63	writel_relaxed(vec->local_id, local->msi_va);
 64	return 0;
 65}
 66
 67static void imsic_irq_ack(struct irq_data *d)
 68{
 69	irq_move_irq(d);
 70}
 71
 72static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
 73{
 74	phys_addr_t msi_addr;
 75
 76	if (WARN_ON(!vec))
 77		return;
 78
 79	if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
 80		return;
 81
 82	msg->address_hi = upper_32_bits(msi_addr);
 83	msg->address_lo = lower_32_bits(msi_addr);
 84	msg->data = vec->local_id;
 85}
 86
 87static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
 88{
 89	imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
 90}
 91
 92#ifdef CONFIG_SMP
 93static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
 94{
 95	struct msi_msg msg = { };
 96
 97	imsic_irq_compose_vector_msg(vec, &msg);
 98	irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
 99}
100
101static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
102				  bool force)
103{
104	struct imsic_vector *old_vec, *new_vec;
105	struct imsic_vector tmp_vec;
106
107	/*
108	 * Requirements for the downstream irqdomains (or devices):
109	 *
110	 * 1) Downstream irqdomains (or devices) with atomic MSI update can
111	 *    happily do imsic_irq_set_affinity() in the process-context on
112	 *    any CPU so the irqchip of such irqdomains must not set the
113	 *    IRQCHIP_MOVE_DEFERRED flag.
114	 *
115	 * 2) Downstream irqdomains (or devices) with non-atomic MSI update
116	 *    must use imsic_irq_set_affinity() in nterrupt-context upon
117	 *    the next device interrupt so the irqchip of such irqdomains
118	 *    must set the IRQCHIP_MOVE_DEFERRED flag.
119	 */
120
121	old_vec = irq_data_get_irq_chip_data(d);
122	if (WARN_ON(!old_vec))
123		return -ENOENT;
124
125	/* If old vector cpu belongs to the target cpumask then do nothing */
126	if (cpumask_test_cpu(old_vec->cpu, mask_val))
127		return IRQ_SET_MASK_OK_DONE;
128
129	/* If move is already in-flight then return failure */
130	if (imsic_vector_get_move(old_vec))
131		return -EBUSY;
132
133	/* Get a new vector on the desired set of CPUs */
134	new_vec = imsic_vector_alloc(old_vec->irq, mask_val);
135	if (!new_vec)
136		return -ENOSPC;
137
138	/*
139	 * Device having non-atomic MSI update might see an intermediate
140	 * state when changing target IMSIC vector from one CPU to another.
141	 *
142	 * To avoid losing interrupt to such intermediate state, do the
143	 * following (just like x86 APIC):
144	 *
145	 * 1) First write a temporary IMSIC vector to the device which
146	 * has MSI address same as the old IMSIC vector but MSI data
147	 * matches the new IMSIC vector.
148	 *
149	 * 2) Next write the new IMSIC vector to the device.
150	 *
151	 * Based on the above, __imsic_local_sync() must check pending
152	 * status of both old MSI data and new MSI data on the old CPU.
153	 */
154	if (!irq_can_move_in_process_context(d) &&
155	    new_vec->local_id != old_vec->local_id) {
156		/* Setup temporary vector */
157		tmp_vec.cpu = old_vec->cpu;
158		tmp_vec.local_id = new_vec->local_id;
159
160		/* Point device to the temporary vector */
161		imsic_msi_update_msg(irq_get_irq_data(d->irq), &tmp_vec);
162	}
163
164	/* Point device to the new vector */
165	imsic_msi_update_msg(irq_get_irq_data(d->irq), new_vec);
166
167	/* Update irq descriptors with the new vector */
168	d->chip_data = new_vec;
169
170	/* Update effective affinity */
171	irq_data_update_effective_affinity(d, cpumask_of(new_vec->cpu));
172
173	/* Move state of the old vector to the new vector */
174	imsic_vector_move(old_vec, new_vec);
175
176	return IRQ_SET_MASK_OK_DONE;
177}
178
179static void imsic_irq_force_complete_move(struct irq_data *d)
180{
181	struct imsic_vector *mvec, *vec = irq_data_get_irq_chip_data(d);
182	unsigned int cpu = smp_processor_id();
183
184	if (WARN_ON(!vec))
185		return;
186
187	/* Do nothing if there is no in-flight move */
188	mvec = imsic_vector_get_move(vec);
189	if (!mvec)
190		return;
191
192	/* Do nothing if the old IMSIC vector does not belong to current CPU */
193	if (mvec->cpu != cpu)
194		return;
195
196	/*
197	 * The best we can do is force cleanup the old IMSIC vector.
198	 *
199	 * The challenges over here are same as x86 vector domain so
200	 * refer to the comments in irq_force_complete_move() function
201	 * implemented at arch/x86/kernel/apic/vector.c.
202	 */
203
204	/* Force cleanup in-flight move */
205	pr_info("IRQ fixup: irq %d move in progress, old vector cpu %d local_id %d\n",
206		d->irq, mvec->cpu, mvec->local_id);
207	imsic_vector_force_move_cleanup(vec);
208}
209#endif
210
211static struct irq_chip imsic_irq_base_chip = {
212	.name				= "IMSIC",
213	.irq_mask			= imsic_irq_mask,
214	.irq_unmask			= imsic_irq_unmask,
215#ifdef CONFIG_SMP
216	.irq_set_affinity		= imsic_irq_set_affinity,
217	.irq_force_complete_move	= imsic_irq_force_complete_move,
218#endif
219	.irq_retrigger			= imsic_irq_retrigger,
220	.irq_ack			= imsic_irq_ack,
221	.irq_compose_msi_msg		= imsic_irq_compose_msg,
222	.flags				= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
223};
224
225static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
226				  unsigned int nr_irqs, void *args)
227{
228	struct imsic_vector *vec;
229
230	/* Multi-MSI is not supported yet. */
231	if (nr_irqs > 1)
232		return -EOPNOTSUPP;
233
234	vec = imsic_vector_alloc(virq, cpu_online_mask);
235	if (!vec)
236		return -ENOSPC;
237
238	irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
239			    handle_edge_irq, NULL, NULL);
240	irq_set_noprobe(virq);
241	irq_set_affinity(virq, cpu_online_mask);
242	irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));
243
244	return 0;
245}
246
247static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
248				  unsigned int nr_irqs)
249{
250	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
251
252	imsic_vector_free(irq_data_get_irq_chip_data(d));
253	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
254}
255
256#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
257static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
258				 struct irq_data *irqd, int ind)
259{
260	if (!irqd) {
261		imsic_vector_debug_show_summary(m, ind);
262		return;
263	}
264
265	imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind);
266}
267#endif
268
269static const struct irq_domain_ops imsic_base_domain_ops = {
270	.alloc		= imsic_irq_domain_alloc,
271	.free		= imsic_irq_domain_free,
272	.select		= msi_lib_irq_domain_select,
273#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
274	.debug_show	= imsic_irq_debug_show,
275#endif
276};
277
278static bool imsic_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
279				    struct irq_domain *real_parent, struct msi_domain_info *info)
280{
281	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
282		return false;
283
284	switch (info->bus_token) {
285	case DOMAIN_BUS_PCI_DEVICE_MSI:
286	case DOMAIN_BUS_PCI_DEVICE_MSIX:
287		info->chip->flags |= IRQCHIP_MOVE_DEFERRED;
288		break;
289	default:
290		break;
291	}
292
293	return true;
294}
295
296static const struct msi_parent_ops imsic_msi_parent_ops = {
297	.supported_flags	= MSI_GENERIC_FLAGS_MASK |
298				  MSI_FLAG_PCI_MSIX,
299	.required_flags		= MSI_FLAG_USE_DEF_DOM_OPS |
300				  MSI_FLAG_USE_DEF_CHIP_OPS |
301				  MSI_FLAG_PCI_MSI_MASK_PARENT,
302	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
303	.bus_select_token	= DOMAIN_BUS_NEXUS,
304	.bus_select_mask	= MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
305	.init_dev_msi_info	= imsic_init_dev_msi_info,
306};
307
308int imsic_irqdomain_init(void)
309{
310	struct irq_domain_info info = {
311		.ops		= &imsic_base_domain_ops,
312		.host_data	= imsic,
313	};
314	struct imsic_global_config *global;
315
316	if (!imsic || !imsic->fwnode) {
317		pr_err("early driver not probed\n");
318		return -ENODEV;
319	}
320
321	if (imsic->base_domain) {
322		pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
323		return -ENODEV;
324	}
325
326	/* Create Base IRQ domain */
327	info.fwnode = imsic->fwnode,
328	imsic->base_domain = msi_create_parent_irq_domain(&info, &imsic_msi_parent_ops);
329	if (!imsic->base_domain) {
330		pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
331		return -ENOMEM;
332	}
333
334	global = &imsic->global;
335	pr_info("%pfwP:  hart-index-bits: %d,  guest-index-bits: %d\n",
336		imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
337	pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
338		imsic->fwnode, global->group_index_bits, global->group_index_shift);
339	pr_info("%pfwP: per-CPU IDs %d at base address %pa\n",
340		imsic->fwnode, global->nr_ids, &global->base_addr);
341	pr_info("%pfwP: total %d interrupts available\n",
342		imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));
343
344	return 0;
345}
346
347static int imsic_platform_probe_common(struct fwnode_handle *fwnode)
348{
349	if (imsic && imsic->fwnode != fwnode) {
350		pr_err("%pfwP: fwnode mismatch\n", fwnode);
351		return -ENODEV;
352	}
353
354	return imsic_irqdomain_init();
355}
356
357static int imsic_platform_dt_probe(struct platform_device *pdev)
358{
359	return imsic_platform_probe_common(pdev->dev.fwnode);
360}
361
362#ifdef CONFIG_ACPI
363
364/*
365 *  On ACPI based systems, PCI enumeration happens early during boot in
366 *  acpi_scan_init(). PCI enumeration expects MSI domain setup before
367 *  it calls pci_set_msi_domain(). Hence, unlike in DT where
368 *  imsic-platform drive probe happens late during boot, ACPI based
369 *  systems need to setup the MSI domain early.
370 */
371int imsic_platform_acpi_probe(struct fwnode_handle *fwnode)
372{
373	return imsic_platform_probe_common(fwnode);
374}
375
376#endif
377
378static const struct of_device_id imsic_platform_match[] = {
379	{ .compatible = "riscv,imsics" },
380	{}
381};
382
383static struct platform_driver imsic_platform_driver = {
384	.driver = {
385		.name		= "riscv-imsic",
386		.of_match_table	= imsic_platform_match,
387	},
388	.probe = imsic_platform_dt_probe,
389};
390builtin_platform_driver(imsic_platform_driver);