irq-gic-v5-irs.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
  4 */
  5
  6#define pr_fmt(fmt)	"GICv5 IRS: " fmt
  7
  8#include <linux/kmemleak.h>
  9#include <linux/log2.h>
 10#include <linux/of.h>
 11#include <linux/of_address.h>
 12
 13#include <linux/irqchip.h>
 14#include <linux/irqchip/arm-gic-v5.h>
 15
 16/*
 17 * Hardcoded ID_BITS limit for systems supporting only a 1-level IST
 18 * table. Systems supporting only a 1-level IST table aren't expected
 19 * to require more than 2^12 LPIs. Tweak as required.
 20 */
 21#define LPI_ID_BITS_LINEAR		12
 22
 23#define IRS_FLAGS_NON_COHERENT		BIT(0)
 24
 25static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
 26static LIST_HEAD(irs_nodes);
 27
 28static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data,
 29			     const u32 reg_offset)
 30{
 31	return readl_relaxed(irs_data->irs_base + reg_offset);
 32}
 33
 34static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data,
 35			       const u32 val, const u32 reg_offset)
 36{
 37	writel_relaxed(val, irs_data->irs_base + reg_offset);
 38}
 39
 40static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data,
 41			     const u32 reg_offset)
 42{
 43	return readq_relaxed(irs_data->irs_base + reg_offset);
 44}
 45
 46static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
 47			       const u64 val, const u32 reg_offset)
 48{
 49	writeq_relaxed(val, irs_data->irs_base + reg_offset);
 50}
 51
 52/*
 53 * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
 54 * provides the memory barriers (through MMIO accessors)
 55 * required to synchronize CPU and GIC access to IST memory.
 56 */
 57static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data)
 58{
 59	return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR,
 60					GICV5_IRS_IST_STATUSR_IDLE, NULL);
 61}
 62
 63static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data,
 64					    unsigned int lpi_id_bits,
 65					    unsigned int istsz)
 66{
 67	size_t l2istsz;
 68	u32 n, cfgr;
 69	void *ist;
 70	u64 baser;
 71	int ret;
 72
 73	/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
 74	n = max(5, lpi_id_bits + 1 + istsz);
 75
 76	l2istsz = BIT(n + 1);
 77	/*
 78	 * Check memory requirements. For a linear IST we cap the
 79	 * number of ID bits to a value that should never exceed
 80	 * kmalloc interface memory allocation limits, so this
 81	 * check is really belt and braces.
 82	 */
 83	if (l2istsz > KMALLOC_MAX_SIZE) {
 84		u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz;
 85
 86		pr_warn("Limiting LPI ID bits from %u to %u\n",
 87			lpi_id_bits, lpi_id_cap);
 88		lpi_id_bits = lpi_id_cap;
 89		l2istsz = KMALLOC_MAX_SIZE;
 90	}
 91
 92	ist = kzalloc(l2istsz, GFP_KERNEL);
 93	if (!ist)
 94		return -ENOMEM;
 95
 96	if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
 97		dcache_clean_inval_poc((unsigned long)ist,
 98				       (unsigned long)ist + l2istsz);
 99	else
100		dsb(ishst);
101
102	cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
103			  GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR)	|
104	       FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz)	|
105	       FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ,
106			  GICV5_IRS_IST_CFGR_L2SZ_4K)		|
107	       FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
108	irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
109
110	gicv5_global_data.ist.l2 = false;
111
112	baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
113		FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
114	irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
115
116	ret = gicv5_irs_ist_synchronise(irs_data);
117	if (ret) {
118		kfree(ist);
119		return ret;
120	}
121	kmemleak_ignore(ist);
122
123	return 0;
124}
125
126static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data,
127					       unsigned int lpi_id_bits,
128					       unsigned int istsz,
129					       unsigned int l2sz)
130{
131	__le64 *l1ist;
132	u32 cfgr, n;
133	size_t l1sz;
134	u64 baser;
135	int ret;
136
137	/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
138	n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2);
139
140	l1sz = BIT(n + 1);
141
142	l1ist = kzalloc(l1sz, GFP_KERNEL);
143	if (!l1ist)
144		return -ENOMEM;
145
146	if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
147		dcache_clean_inval_poc((unsigned long)l1ist,
148				       (unsigned long)l1ist + l1sz);
149	else
150		dsb(ishst);
151
152	cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
153			  GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL)	|
154	       FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz)		|
155	       FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz)		|
156	       FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
157	irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
158
159	/*
160	 * The L2SZ determine bits required at L2 level. Number of bytes
161	 * required by metadata is reported through istsz - the number of bits
162	 * covered by L2 entries scales accordingly.
163	 */
164	gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1);
165	gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz);
166	gicv5_global_data.ist.l1ist_addr = l1ist;
167	gicv5_global_data.ist.l2 = true;
168
169	baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
170		FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
171	irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
172
173	ret = gicv5_irs_ist_synchronise(irs_data);
174	if (ret) {
175		kfree(l1ist);
176		return ret;
177	}
178
179	return 0;
180}
181
182/*
183 * Alloc L2 IST entries on demand.
184 *
185 * Locking/serialization is guaranteed by irqdomain core code by
186 * taking the hierarchical domain struct irq_domain.root->mutex.
187 */
188int gicv5_irs_iste_alloc(const u32 lpi)
189{
190	struct gicv5_irs_chip_data *irs_data;
191	unsigned int index;
192	u32 l2istr, l2bits;
193	__le64 *l1ist;
194	size_t l2size;
195	void *l2ist;
196	int ret;
197
198	if (!gicv5_global_data.ist.l2)
199		return 0;
200
201	irs_data = per_cpu(per_cpu_irs_data, smp_processor_id());
202	if (!irs_data)
203		return -ENOENT;
204
205	l2size = gicv5_global_data.ist.l2_size;
206	l2bits = gicv5_global_data.ist.l2_bits;
207	l1ist = gicv5_global_data.ist.l1ist_addr;
208	index = lpi >> l2bits;
209
210	if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index])))
211		return 0;
212
213	l2ist = kzalloc(l2size, GFP_KERNEL);
214	if (!l2ist)
215		return -ENOMEM;
216
217	l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK);
218
219	if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
220		dcache_clean_inval_poc((unsigned long)l2ist,
221				       (unsigned long)l2ist + l2size);
222		dcache_clean_poc((unsigned long)(l1ist + index),
223				 (unsigned long)(l1ist + index) + sizeof(*l1ist));
224	} else {
225		dsb(ishst);
226	}
227
228	l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi);
229	irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR);
230
231	ret = gicv5_irs_ist_synchronise(irs_data);
232	if (ret) {
233		l1ist[index] = 0;
234		kfree(l2ist);
235		return ret;
236	}
237	kmemleak_ignore(l2ist);
238
239	/*
240	 * Make sure we invalidate the cache line pulled before the IRS
241	 * had a chance to update the L1 entry and mark it valid.
242	 */
243	if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
244		/*
245		 * gicv5_irs_ist_synchronise() includes memory
246		 * barriers (MMIO accessors) required to guarantee that the
247		 * following dcache invalidation is not executed before the
248		 * IST mapping operation has completed.
249		 */
250		dcache_inval_poc((unsigned long)(l1ist + index),
251				 (unsigned long)(l1ist + index) + sizeof(*l1ist));
252	}
253
254	return 0;
255}
256
257/*
258 * Try to match the L2 IST size to the pagesize, and if this is not possible
259 * pick the smallest supported L2 size in order to minimise the requirement for
260 * physically contiguous blocks of memory as page-sized allocations are
261 * guaranteed to be physically contiguous, and are by definition the easiest to
262 * find.
263 *
264 * Fall back to the smallest supported size (in the event that the pagesize
265 * itself is not supported) again serves to make it easier to find physically
266 * contiguous blocks of memory.
267 */
268static unsigned int gicv5_irs_l2_sz(u32 idr2)
269{
270	switch (PAGE_SIZE) {
271	case SZ_64K:
272		if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
273			return GICV5_IRS_IST_CFGR_L2SZ_64K;
274		fallthrough;
275	case SZ_4K:
276		if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
277			return GICV5_IRS_IST_CFGR_L2SZ_4K;
278		fallthrough;
279	case SZ_16K:
280		if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
281			return GICV5_IRS_IST_CFGR_L2SZ_16K;
282		break;
283	}
284
285	if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
286		return GICV5_IRS_IST_CFGR_L2SZ_4K;
287
288	return GICV5_IRS_IST_CFGR_L2SZ_64K;
289}
290
291static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
292{
293	u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz;
294	u32 l2_iste_sz_split, idr2;
295	bool two_levels, istmd;
296	u64 baser;
297	int ret;
298
299	baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER);
300	if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) {
301		pr_err("IST is marked as valid already; cannot allocate\n");
302		return -EPERM;
303	}
304
305	idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
306	two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2);
307
308	idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2);
309	idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2);
310
311	/*
312	 * For two level tables we are always supporting the maximum allowed
313	 * number of IDs.
314	 *
315	 * For 1-level tables, we should support a number of bits that
316	 * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest
317	 * the level 1-table gets too large and its memory allocation
318	 * may fail.
319	 */
320	if (two_levels) {
321		lpi_id_bits = idr2_id_bits;
322	} else {
323		lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits);
324		lpi_id_bits = min(lpi_id_bits, idr2_id_bits);
325	}
326
327	/*
328	 * Cap the ID bits according to the CPUIF supported ID bits
329	 */
330	lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
331
332	if (two_levels)
333		l2sz = gicv5_irs_l2_sz(idr2);
334
335	istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
336
337	l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4;
338
339	if (istmd) {
340		l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2);
341
342		if (lpi_id_bits < l2_iste_sz_split)
343			l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8;
344		else
345			l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16;
346	}
347
348	/*
349	 * Follow GICv5 specification recommendation to opt in for two
350	 * level tables (ref: 10.2.1.14 IRS_IST_CFGR).
351	 */
352	if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) {
353		ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits,
354						   l2_iste_sz, l2sz);
355	} else {
356		ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits,
357						l2_iste_sz);
358	}
359	if (ret)
360		return ret;
361
362	gicv5_init_lpis(BIT(lpi_id_bits));
363
364	return 0;
365}
366
367struct iaffid_entry {
368	u16	iaffid;
369	bool	valid;
370};
371
372static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid);
373
374int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid)
375{
376	if (!per_cpu(cpu_iaffid, cpuid).valid) {
377		pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
378		return -ENODEV;
379	}
380
381	*iaffid = per_cpu(cpu_iaffid, cpuid).iaffid;
382
383	return 0;
384}
385
386struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id)
387{
388	struct gicv5_irs_chip_data *irs_data;
389	u32 min, max;
390
391	list_for_each_entry(irs_data, &irs_nodes, entry) {
392		if (!irs_data->spi_range)
393			continue;
394
395		min = irs_data->spi_min;
396		max = irs_data->spi_min + irs_data->spi_range - 1;
397		if (spi_id >= min && spi_id <= max)
398			return irs_data;
399	}
400
401	return NULL;
402}
403
404static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data)
405{
406	u32 statusr;
407	int ret;
408
409	ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR,
410				       GICV5_IRS_SPI_STATUSR_IDLE, &statusr);
411	if (ret)
412		return ret;
413
414	return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO;
415}
416
417static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data,
418				     bool selr)
419{
420	bool valid = true;
421	u32 statusr;
422	int ret;
423
424	ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR,
425				       GICV5_IRS_PE_STATUSR_IDLE, &statusr);
426	if (ret)
427		return ret;
428
429	if (selr)
430		valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr);
431
432	return valid ? 0 : -EIO;
433}
434
435static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data)
436{
437	return gicv5_irs_wait_for_irs_pe(irs_data, true);
438}
439
440static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data)
441{
442	return gicv5_irs_wait_for_irs_pe(irs_data, false);
443}
444
445int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type)
446{
447	struct gicv5_irs_chip_data *irs_data = d->chip_data;
448	u32 selr, cfgr;
449	bool level;
450	int ret;
451
452	/*
453	 * There is no distinction between HIGH/LOW for level IRQs
454	 * and RISING/FALLING for edge IRQs in the architecture,
455	 * hence consider them equivalent.
456	 */
457	switch (type) {
458	case IRQ_TYPE_EDGE_RISING:
459	case IRQ_TYPE_EDGE_FALLING:
460		level = false;
461		break;
462	case IRQ_TYPE_LEVEL_HIGH:
463	case IRQ_TYPE_LEVEL_LOW:
464		level = true;
465		break;
466	default:
467		return -EINVAL;
468	}
469
470	guard(raw_spinlock)(&irs_data->spi_config_lock);
471
472	selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq);
473	irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR);
474	ret = gicv5_irs_wait_for_spi_op(irs_data);
475	if (ret)
476		return ret;
477
478	cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level);
479	irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR);
480
481	return gicv5_irs_wait_for_spi_op(irs_data);
482}
483
484static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data)
485{
486	return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0,
487					GICV5_IRS_CR0_IDLE, NULL);
488}
489
490void gicv5_irs_syncr(void)
491{
492	struct gicv5_irs_chip_data *irs_data;
493	u32 syncr;
494
495	irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry);
496	if (WARN_ON_ONCE(!irs_data))
497		return;
498
499	syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1);
500	irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR);
501
502	gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR,
503			  GICV5_IRS_SYNC_STATUSR_IDLE);
504}
505
506int gicv5_irs_register_cpu(int cpuid)
507{
508	struct gicv5_irs_chip_data *irs_data;
509	u32 selr, cr0;
510	u16 iaffid;
511	int ret;
512
513	ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
514	if (ret) {
515		pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
516		return ret;
517	}
518
519	irs_data = per_cpu(per_cpu_irs_data, cpuid);
520	if (!irs_data) {
521		pr_err("No IRS associated with CPU %u\n", cpuid);
522		return -ENXIO;
523	}
524
525	selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid);
526	irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR);
527
528	ret = gicv5_irs_wait_for_pe_selr(irs_data);
529	if (ret) {
530		pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid);
531		return -ENXIO;
532	}
533
534	cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1);
535	irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0);
536
537	ret = gicv5_irs_wait_for_pe_cr0(irs_data);
538	if (ret)
539		return ret;
540
541	pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid);
542
543	return 0;
544}
545
546static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data,
547					void __iomem *irs_base,
548					struct fwnode_handle *handle)
549{
550	struct device_node *np = to_of_node(handle);
551	u32 cr0, cr1;
552
553	irs_data->fwnode = handle;
554	irs_data->irs_base = irs_base;
555
556	if (of_property_read_bool(np, "dma-noncoherent")) {
557		/*
558		 * A non-coherent IRS implies that some cache levels cannot be
559		 * used coherently by the cores and GIC. Our only option is to mark
560		 * memory attributes for the GIC as non-cacheable; by default,
561		 * non-cacheable memory attributes imply outer-shareable
562		 * shareability, the value written into IRS_CR1_SH is ignored.
563		 */
564		cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC)	|
565			FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC)	|
566			FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC)	|
567			FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC)	|
568			FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC)	|
569			FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC)	|
570			FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC)	|
571			FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC)	|
572			FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE)		|
573			FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE);
574		irs_data->flags |= IRS_FLAGS_NON_COHERENT;
575	} else {
576		cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC)	|
577			FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC)	|
578			FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC)	|
579			FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC)	|
580			FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC)	|
581			FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC)	|
582			FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC)	|
583			FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC)	|
584			FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE)		|
585			FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE)		|
586			FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE);
587	}
588
589	irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1);
590
591	cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1);
592	irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0);
593	gicv5_irs_wait_for_idle(irs_data);
594}
595
596static int __init gicv5_irs_of_init_affinity(struct device_node *node,
597					     struct gicv5_irs_chip_data *irs_data,
598					     u8 iaffid_bits)
599{
600	/*
601	 * Detect IAFFID<->CPU mappings from the device tree and
602	 * record IRS<->CPU topology information.
603	 */
604	u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0);
605	int ret, i, ncpus, niaffids;
606
607	ncpus = of_count_phandle_with_args(node, "cpus", NULL);
608	if (ncpus < 0)
609		return -EINVAL;
610
611	niaffids = of_property_count_elems_of_size(node, "arm,iaffids",
612						   sizeof(u16));
613	if (niaffids != ncpus)
614		return -EINVAL;
615
616	u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL);
617	if (!iaffids)
618		return -ENOMEM;
619
620	ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids);
621	if (ret)
622		return ret;
623
624	for (i = 0; i < ncpus; i++) {
625		struct device_node *cpu_node;
626		int cpu;
627
628		cpu_node = of_parse_phandle(node, "cpus", i);
629		if (!cpu_node) {
630			pr_warn(FW_BUG "Erroneous CPU node phandle\n");
631			continue;
632		}
633
634		cpu = of_cpu_node_to_id(cpu_node);
635		of_node_put(cpu_node);
636		if (cpu < 0)
637			continue;
638
639		if (iaffids[i] & ~iaffid_mask) {
640			pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n",
641				cpu, iaffids[i]);
642			continue;
643		}
644
645		per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i];
646		per_cpu(cpu_iaffid, cpu).valid = true;
647
648		/* We also know that the CPU is connected to this IRS */
649		per_cpu(per_cpu_irs_data, cpu) = irs_data;
650	}
651
652	return ret;
653}
654
655static void irs_setup_pri_bits(u32 idr1)
656{
657	switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) {
658	case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS:
659		gicv5_global_data.irs_pri_bits = 1;
660		break;
661	case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS:
662		gicv5_global_data.irs_pri_bits = 2;
663		break;
664	case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS:
665		gicv5_global_data.irs_pri_bits = 3;
666		break;
667	case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS:
668		gicv5_global_data.irs_pri_bits = 4;
669		break;
670	case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS:
671		gicv5_global_data.irs_pri_bits = 5;
672		break;
673	default:
674		pr_warn("Detected wrong IDR priority bits value 0x%lx\n",
675			FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1));
676		gicv5_global_data.irs_pri_bits = 1;
677		break;
678	}
679}
680
681static int __init gicv5_irs_init(struct device_node *node)
682{
683	struct gicv5_irs_chip_data *irs_data;
684	void __iomem *irs_base;
685	u32 idr, spi_count;
686	u8 iaffid_bits;
687	int ret;
688
689	irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL);
690	if (!irs_data)
691		return -ENOMEM;
692
693	raw_spin_lock_init(&irs_data->spi_config_lock);
694
695	ret = of_property_match_string(node, "reg-names", "ns-config");
696	if (ret < 0) {
697		pr_err("%pOF: ns-config reg-name not present\n", node);
698		goto out_err;
699	}
700
701	irs_base = of_io_request_and_map(node, ret, of_node_full_name(node));
702	if (IS_ERR(irs_base)) {
703		pr_err("%pOF: unable to map GICv5 IRS registers\n", node);
704		ret = PTR_ERR(irs_base);
705		goto out_err;
706	}
707
708	gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode);
709
710	idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
711	iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1;
712
713	ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits);
714	if (ret) {
715		pr_err("Failed to parse CPU IAFFIDs from the device tree!\n");
716		goto out_iomem;
717	}
718
719	idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
720	if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr),
721		 "LPI support not available - no IPIs, can't proceed\n")) {
722		ret = -ENODEV;
723		goto out_iomem;
724	}
725
726	idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7);
727	irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr);
728
729	idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6);
730	irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr);
731
732	if (irs_data->spi_range) {
733		pr_info("%s detected SPI range [%u-%u]\n",
734						of_node_full_name(node),
735						irs_data->spi_min,
736						irs_data->spi_min +
737						irs_data->spi_range - 1);
738	}
739
740	/*
741	 * Do the global setting only on the first IRS.
742	 * Global properties (iaffid_bits, global spi count) are guaranteed to
743	 * be consistent across IRSes by the architecture.
744	 */
745	if (list_empty(&irs_nodes)) {
746
747		idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
748		irs_setup_pri_bits(idr);
749
750		idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5);
751
752		spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr);
753		gicv5_global_data.global_spi_count = spi_count;
754
755		gicv5_init_lpi_domain();
756
757		pr_debug("Detected %u SPIs globally\n", spi_count);
758	}
759
760	list_add_tail(&irs_data->entry, &irs_nodes);
761
762	return 0;
763
764out_iomem:
765	iounmap(irs_base);
766out_err:
767	kfree(irs_data);
768	return ret;
769}
770
771void __init gicv5_irs_remove(void)
772{
773	struct gicv5_irs_chip_data *irs_data, *tmp_data;
774
775	gicv5_free_lpi_domain();
776	gicv5_deinit_lpis();
777
778	list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) {
779		iounmap(irs_data->irs_base);
780		list_del(&irs_data->entry);
781		kfree(irs_data);
782	}
783}
784
785int __init gicv5_irs_enable(void)
786{
787	struct gicv5_irs_chip_data *irs_data;
788	int ret;
789
790	irs_data = list_first_entry_or_null(&irs_nodes,
791					    struct gicv5_irs_chip_data, entry);
792	if (!irs_data)
793		return -ENODEV;
794
795	ret = gicv5_irs_init_ist(irs_data);
796	if (ret) {
797		pr_err("Failed to init IST\n");
798		return ret;
799	}
800
801	return 0;
802}
803
804void __init gicv5_irs_its_probe(void)
805{
806	struct gicv5_irs_chip_data *irs_data;
807
808	list_for_each_entry(irs_data, &irs_nodes, entry)
809		gicv5_its_of_probe(to_of_node(irs_data->fwnode));
810}
811
812int __init gicv5_irs_of_probe(struct device_node *parent)
813{
814	struct device_node *np;
815	int ret;
816
817	for_each_available_child_of_node(parent, np) {
818		if (!of_device_is_compatible(np, "arm,gic-v5-irs"))
819			continue;
820
821		ret = gicv5_irs_init(np);
822		if (ret)
823			pr_err("Failed to init IRS %s\n", np->full_name);
824	}
825
826	return list_empty(&irs_nodes) ? -ENODEV : 0;
827}