pci-xgene-msi.c

  1// SPDX-License-Identifier: GPL-2.0+
  2/*
  3 * APM X-Gene MSI Driver
  4 *
  5 * Copyright (c) 2014, Applied Micro Circuits Corporation
  6 * Author: Tanmay Inamdar <tinamdar@apm.com>
  7 *	   Duc Dang <dhdang@apm.com>
  8 */
  9#include <linux/bitfield.h>
 10#include <linux/cpu.h>
 11#include <linux/interrupt.h>
 12#include <linux/irqdomain.h>
 13#include <linux/module.h>
 14#include <linux/msi.h>
 15#include <linux/irqchip/chained_irq.h>
 16#include <linux/irqchip/irq-msi-lib.h>
 17#include <linux/pci.h>
 18#include <linux/platform_device.h>
 19#include <linux/of_pci.h>
 20
 21#define MSI_IR0			0x000000
 22#define MSI_INT0		0x800000
 23#define IDX_PER_GROUP		8
 24#define IRQS_PER_IDX		16
 25#define NR_HW_IRQS		16
 26#define NR_MSI_BITS		(IDX_PER_GROUP * IRQS_PER_IDX * NR_HW_IRQS)
 27#define NR_MSI_VEC		(NR_MSI_BITS / num_possible_cpus())
 28
 29#define MSI_GROUP_MASK		GENMASK(22, 19)
 30#define MSI_INDEX_MASK		GENMASK(18, 16)
 31#define MSI_INTR_MASK		GENMASK(19, 16)
 32
 33#define MSInRx_HWIRQ_MASK	GENMASK(6, 4)
 34#define DATA_HWIRQ_MASK		GENMASK(3, 0)
 35
 36struct xgene_msi {
 37	struct irq_domain	*inner_domain;
 38	u64			msi_addr;
 39	void __iomem		*msi_regs;
 40	unsigned long		*bitmap;
 41	struct mutex		bitmap_lock;
 42	unsigned int		gic_irq[NR_HW_IRQS];
 43};
 44
 45/* Global data */
 46static struct xgene_msi *xgene_msi_ctrl;
 47
 48/*
 49 * X-Gene v1 has 16 frames of MSI termination registers MSInIRx, where n is
 50 * frame number (0..15), x is index of registers in each frame (0..7).  Each
 51 * 32b register is at the beginning of a 64kB region, each frame occupying
 52 * 512kB (and the whole thing 8MB of PA space).
 53 *
 54 * Each register supports 16 MSI vectors (0..15) to generate interrupts. A
 55 * write to the MSInIRx from the PCI side generates an interrupt. A read
 56 * from the MSInRx on the CPU side returns a bitmap of the pending MSIs in
 57 * the lower 16 bits. A side effect of this read is that all pending
 58 * interrupts are acknowledged and cleared).
 59 *
 60 * Additionally, each MSI termination frame has 1 MSIINTn register (n is
 61 * 0..15) to indicate the MSI pending status caused by any of its 8
 62 * termination registers, reported as a bitmap in the lower 8 bits. Each 32b
 63 * register is at the beginning of a 64kB region (and overall occupying an
 64 * extra 1MB).
 65 *
 66 * There is one GIC IRQ assigned for each MSI termination frame, 16 in
 67 * total.
 68 *
 69 * The register layout is as follows:
 70 * MSI0IR0			base_addr
 71 * MSI0IR1			base_addr +  0x10000
 72 * ...				...
 73 * MSI0IR6			base_addr +  0x60000
 74 * MSI0IR7			base_addr +  0x70000
 75 * MSI1IR0			base_addr +  0x80000
 76 * MSI1IR1			base_addr +  0x90000
 77 * ...				...
 78 * MSI1IR7			base_addr +  0xF0000
 79 * MSI2IR0			base_addr + 0x100000
 80 * ...				...
 81 * MSIFIR0			base_addr + 0x780000
 82 * MSIFIR1			base_addr + 0x790000
 83 * ...				...
 84 * MSIFIR7			base_addr + 0x7F0000
 85 * MSIINT0			base_addr + 0x800000
 86 * MSIINT1			base_addr + 0x810000
 87 * ...				...
 88 * MSIINTF			base_addr + 0x8F0000
 89 */
 90
 91/* MSInIRx read helper */
 92static u32 xgene_msi_ir_read(struct xgene_msi *msi, u32 msi_grp, u32 msir_idx)
 93{
 94	return readl_relaxed(msi->msi_regs + MSI_IR0 +
 95			     (FIELD_PREP(MSI_GROUP_MASK, msi_grp) |
 96			      FIELD_PREP(MSI_INDEX_MASK, msir_idx)));
 97}
 98
 99/* MSIINTn read helper */
100static u32 xgene_msi_int_read(struct xgene_msi *msi, u32 msi_grp)
101{
102	return readl_relaxed(msi->msi_regs + MSI_INT0 +
103			     FIELD_PREP(MSI_INTR_MASK, msi_grp));
104}
105
106/*
107 * In order to allow an MSI to be moved from one CPU to another without
108 * having to repaint both the address and the data (which cannot be done
109 * atomically), we statically partitions the MSI frames between CPUs. Given
110 * that XGene-1 has 8 CPUs, each CPU gets two frames assigned to it
111 *
112 * We adopt the convention that when an MSI is moved, it is configured to
113 * target the same register number in the congruent frame assigned to the
114 * new target CPU. This reserves a given MSI across all CPUs, and reduces
115 * the MSI capacity from 2048 to 256.
116 *
117 * Effectively, this amounts to:
118 * - hwirq[7]::cpu[2:0] is the target frame number (n in MSInIRx)
119 * - hwirq[6:4] is the register index in any given frame (x in MSInIRx)
120 * - hwirq[3:0] is the MSI data
121 */
122static irq_hw_number_t compute_hwirq(u8 frame, u8 index, u8 data)
123{
124	return (FIELD_PREP(BIT(7), FIELD_GET(BIT(3), frame))	|
125		FIELD_PREP(MSInRx_HWIRQ_MASK, index)		|
126		FIELD_PREP(DATA_HWIRQ_MASK, data));
127}
128
129static void xgene_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
130{
131	struct xgene_msi *msi = irq_data_get_irq_chip_data(data);
132	u64 target_addr;
133	u32 frame, msir;
134	int cpu;
135
136	cpu	= cpumask_first(irq_data_get_effective_affinity_mask(data));
137	msir	= FIELD_GET(MSInRx_HWIRQ_MASK, data->hwirq);
138	frame	= FIELD_PREP(BIT(3), FIELD_GET(BIT(7), data->hwirq)) | cpu;
139
140	target_addr = msi->msi_addr;
141	target_addr += (FIELD_PREP(MSI_GROUP_MASK, frame) |
142			FIELD_PREP(MSI_INTR_MASK, msir));
143
144	msg->address_hi = upper_32_bits(target_addr);
145	msg->address_lo = lower_32_bits(target_addr);
146	msg->data = FIELD_GET(DATA_HWIRQ_MASK, data->hwirq);
147}
148
149static int xgene_msi_set_affinity(struct irq_data *irqdata,
150				  const struct cpumask *mask, bool force)
151{
152	int target_cpu = cpumask_first(mask);
153
154	irq_data_update_effective_affinity(irqdata, cpumask_of(target_cpu));
155
156	/* Force the core code to regenerate the message */
157	return IRQ_SET_MASK_OK;
158}
159
160static struct irq_chip xgene_msi_bottom_irq_chip = {
161	.name			= "MSI",
162	.irq_set_affinity       = xgene_msi_set_affinity,
163	.irq_compose_msi_msg	= xgene_compose_msi_msg,
164};
165
166static int xgene_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
167				  unsigned int nr_irqs, void *args)
168{
169	struct xgene_msi *msi = domain->host_data;
170	irq_hw_number_t hwirq;
171
172	mutex_lock(&msi->bitmap_lock);
173
174	hwirq = find_first_zero_bit(msi->bitmap, NR_MSI_VEC);
175	if (hwirq < NR_MSI_VEC)
176		set_bit(hwirq, msi->bitmap);
177
178	mutex_unlock(&msi->bitmap_lock);
179
180	if (hwirq >= NR_MSI_VEC)
181		return -ENOSPC;
182
183	irq_domain_set_info(domain, virq, hwirq,
184			    &xgene_msi_bottom_irq_chip, domain->host_data,
185			    handle_simple_irq, NULL, NULL);
186	irqd_set_resend_when_in_progress(irq_get_irq_data(virq));
187
188	return 0;
189}
190
191static void xgene_irq_domain_free(struct irq_domain *domain,
192				  unsigned int virq, unsigned int nr_irqs)
193{
194	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
195	struct xgene_msi *msi = irq_data_get_irq_chip_data(d);
196
197	mutex_lock(&msi->bitmap_lock);
198
199	clear_bit(d->hwirq, msi->bitmap);
200
201	mutex_unlock(&msi->bitmap_lock);
202
203	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
204}
205
206static const struct irq_domain_ops xgene_msi_domain_ops = {
207	.alloc  = xgene_irq_domain_alloc,
208	.free   = xgene_irq_domain_free,
209};
210
211static const struct msi_parent_ops xgene_msi_parent_ops = {
212	.supported_flags	= (MSI_GENERIC_FLAGS_MASK	|
213				   MSI_FLAG_PCI_MSIX),
214	.required_flags		= (MSI_FLAG_USE_DEF_DOM_OPS	|
215				   MSI_FLAG_USE_DEF_CHIP_OPS),
216	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
217	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
218};
219
220static int xgene_allocate_domains(struct device_node *node,
221				  struct xgene_msi *msi)
222{
223	struct irq_domain_info info = {
224		.fwnode		= of_fwnode_handle(node),
225		.ops		= &xgene_msi_domain_ops,
226		.size		= NR_MSI_VEC,
227		.host_data	= msi,
228	};
229
230	msi->inner_domain = msi_create_parent_irq_domain(&info, &xgene_msi_parent_ops);
231	return msi->inner_domain ? 0 : -ENOMEM;
232}
233
234static int xgene_msi_init_allocator(struct device *dev)
235{
236	xgene_msi_ctrl->bitmap = devm_bitmap_zalloc(dev, NR_MSI_VEC, GFP_KERNEL);
237	if (!xgene_msi_ctrl->bitmap)
238		return -ENOMEM;
239
240	mutex_init(&xgene_msi_ctrl->bitmap_lock);
241
242	return 0;
243}
244
245static void xgene_msi_isr(struct irq_desc *desc)
246{
247	unsigned int *irqp = irq_desc_get_handler_data(desc);
248	struct irq_chip *chip = irq_desc_get_chip(desc);
249	struct xgene_msi *xgene_msi = xgene_msi_ctrl;
250	unsigned long grp_pending;
251	int msir_idx;
252	u32 msi_grp;
253
254	chained_irq_enter(chip, desc);
255
256	msi_grp = irqp - xgene_msi->gic_irq;
257
258	grp_pending = xgene_msi_int_read(xgene_msi, msi_grp);
259
260	for_each_set_bit(msir_idx, &grp_pending, IDX_PER_GROUP) {
261		unsigned long msir;
262		int intr_idx;
263
264		msir = xgene_msi_ir_read(xgene_msi, msi_grp, msir_idx);
265
266		for_each_set_bit(intr_idx, &msir, IRQS_PER_IDX) {
267			irq_hw_number_t hwirq;
268			int ret;
269
270			hwirq = compute_hwirq(msi_grp, msir_idx, intr_idx);
271			ret = generic_handle_domain_irq(xgene_msi->inner_domain,
272							hwirq);
273			WARN_ON_ONCE(ret);
274		}
275	}
276
277	chained_irq_exit(chip, desc);
278}
279
280static void xgene_msi_remove(struct platform_device *pdev)
281{
282	for (int i = 0; i < NR_HW_IRQS; i++) {
283		unsigned int irq = xgene_msi_ctrl->gic_irq[i];
284		if (!irq)
285			continue;
286		irq_set_chained_handler_and_data(irq, NULL, NULL);
287	}
288
289	if (xgene_msi_ctrl->inner_domain)
290		irq_domain_remove(xgene_msi_ctrl->inner_domain);
291}
292
293static int xgene_msi_handler_setup(struct platform_device *pdev)
294{
295	struct xgene_msi *xgene_msi = xgene_msi_ctrl;
296	int i;
297
298	for (i = 0; i < NR_HW_IRQS; i++) {
299		u32 msi_val;
300		int irq, err;
301
302		/*
303		 * MSInIRx registers are read-to-clear; before registering
304		 * interrupt handlers, read all of them to clear spurious
305		 * interrupts that may occur before the driver is probed.
306		 */
307		for (int msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++)
308			xgene_msi_ir_read(xgene_msi, i, msi_idx);
309
310		/* Read MSIINTn to confirm */
311		msi_val = xgene_msi_int_read(xgene_msi, i);
312		if (msi_val) {
313			dev_err(&pdev->dev, "Failed to clear spurious IRQ\n");
314			return -EINVAL;
315		}
316
317		irq = platform_get_irq(pdev, i);
318		if (irq < 0)
319			return irq;
320
321		xgene_msi->gic_irq[i] = irq;
322
323		/*
324		 * Statically allocate MSI GIC IRQs to each CPU core.
325		 * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated
326		 * to each core.
327		 */
328		irq_set_status_flags(irq, IRQ_NO_BALANCING);
329		err = irq_set_affinity(irq, cpumask_of(i % num_possible_cpus()));
330		if (err) {
331			pr_err("failed to set affinity for GIC IRQ");
332			return err;
333		}
334
335		irq_set_chained_handler_and_data(irq, xgene_msi_isr,
336						 &xgene_msi_ctrl->gic_irq[i]);
337	}
338
339	return 0;
340}
341
342static const struct of_device_id xgene_msi_match_table[] = {
343	{.compatible = "apm,xgene1-msi"},
344	{},
345};
346
347static int xgene_msi_probe(struct platform_device *pdev)
348{
349	struct resource *res;
350	struct xgene_msi *xgene_msi;
351	int rc;
352
353	xgene_msi_ctrl = devm_kzalloc(&pdev->dev, sizeof(*xgene_msi_ctrl),
354				      GFP_KERNEL);
355	if (!xgene_msi_ctrl)
356		return -ENOMEM;
357
358	xgene_msi = xgene_msi_ctrl;
359
360	xgene_msi->msi_regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
361	if (IS_ERR(xgene_msi->msi_regs)) {
362		rc = PTR_ERR(xgene_msi->msi_regs);
363		goto error;
364	}
365	xgene_msi->msi_addr = res->start;
366
367	rc = xgene_msi_init_allocator(&pdev->dev);
368	if (rc) {
369		dev_err(&pdev->dev, "Error allocating MSI bitmap\n");
370		goto error;
371	}
372
373	rc = xgene_allocate_domains(dev_of_node(&pdev->dev), xgene_msi);
374	if (rc) {
375		dev_err(&pdev->dev, "Failed to allocate MSI domain\n");
376		goto error;
377	}
378
379	rc = xgene_msi_handler_setup(pdev);
380	if (rc)
381		goto error;
382
383	dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n");
384
385	return 0;
386error:
387	xgene_msi_remove(pdev);
388	return rc;
389}
390
391static struct platform_driver xgene_msi_driver = {
392	.driver = {
393		.name = "xgene-msi",
394		.of_match_table = xgene_msi_match_table,
395	},
396	.probe = xgene_msi_probe,
397	.remove = xgene_msi_remove,
398};
399builtin_platform_driver(xgene_msi_driver);