irq-gic-v5-its.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
   4 */
   5
   6#define pr_fmt(fmt)	"GICv5 ITS: " fmt
   7
   8#include <linux/bitmap.h>
   9#include <linux/iommu.h>
  10#include <linux/init.h>
  11#include <linux/kernel.h>
  12#include <linux/msi.h>
  13#include <linux/of.h>
  14#include <linux/of_address.h>
  15#include <linux/of_irq.h>
  16#include <linux/slab.h>
  17
  18#include <linux/irqchip.h>
  19#include <linux/irqchip/arm-gic-v5.h>
  20#include <linux/irqchip/irq-msi-lib.h>
  21
  22#include "irq-gic-its-msi-parent.h"
  23
  24#define ITS_FLAGS_NON_COHERENT		BIT(0)
  25
  26struct gicv5_its_chip_data {
  27	struct	xarray			its_devices;
  28	struct	mutex			dev_alloc_lock;
  29	struct	fwnode_handle		*fwnode;
  30	struct gicv5_its_devtab_cfg	devtab_cfgr;
  31	void	__iomem			*its_base;
  32	u32				flags;
  33	unsigned int			msi_domain_flags;
  34};
  35
  36struct gicv5_its_dev {
  37	struct gicv5_its_chip_data	*its_node;
  38	struct gicv5_its_itt_cfg	itt_cfg;
  39	unsigned long			*event_map;
  40	u32				device_id;
  41	u32				num_events;
  42	phys_addr_t			its_trans_phys_base;
  43};
  44
  45static u32 its_readl_relaxed(struct gicv5_its_chip_data *its_node, const u32 reg_offset)
  46{
  47	return readl_relaxed(its_node->its_base + reg_offset);
  48}
  49
  50static void its_writel_relaxed(struct gicv5_its_chip_data *its_node, const u32 val,
  51			       const u32 reg_offset)
  52{
  53	writel_relaxed(val, its_node->its_base + reg_offset);
  54}
  55
  56static void its_writeq_relaxed(struct gicv5_its_chip_data *its_node, const u64 val,
  57			       const u32 reg_offset)
  58{
  59	writeq_relaxed(val, its_node->its_base + reg_offset);
  60}
  61
  62static void gicv5_its_dcache_clean(struct gicv5_its_chip_data *its, void *start,
  63				   size_t sz)
  64{
  65	void *end = start + sz;
  66
  67	if (its->flags & ITS_FLAGS_NON_COHERENT)
  68		dcache_clean_inval_poc((unsigned long)start, (unsigned long)end);
  69	else
  70		dsb(ishst);
  71}
  72
  73static void its_write_table_entry(struct gicv5_its_chip_data *its, __le64 *entry,
  74				  u64 val)
  75{
  76	WRITE_ONCE(*entry, cpu_to_le64(val));
  77	gicv5_its_dcache_clean(its, entry, sizeof(*entry));
  78}
  79
  80#define devtab_cfgr_field(its, f)	\
  81	FIELD_GET(GICV5_ITS_DT_CFGR_##f, (its)->devtab_cfgr.cfgr)
  82
  83static int gicv5_its_cache_sync(struct gicv5_its_chip_data *its)
  84{
  85	return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_STATUSR,
  86					GICV5_ITS_STATUSR_IDLE, NULL);
  87}
  88
  89static void gicv5_its_syncr(struct gicv5_its_chip_data *its,
  90			    struct gicv5_its_dev *its_dev)
  91{
  92	u64 syncr;
  93
  94	syncr = FIELD_PREP(GICV5_ITS_SYNCR_SYNC, 1) |
  95		FIELD_PREP(GICV5_ITS_SYNCR_DEVICEID, its_dev->device_id);
  96
  97	its_writeq_relaxed(its, syncr, GICV5_ITS_SYNCR);
  98
  99	gicv5_wait_for_op(its->its_base, GICV5_ITS_SYNC_STATUSR, GICV5_ITS_SYNC_STATUSR_IDLE);
 100}
 101
 102/* Number of bits required for each L2 {device/interrupt translation} table size */
 103#define ITS_L2SZ_64K_L2_BITS	13
 104#define ITS_L2SZ_16K_L2_BITS	11
 105#define ITS_L2SZ_4K_L2_BITS	9
 106
 107static unsigned int gicv5_its_l2sz_to_l2_bits(unsigned int sz)
 108{
 109	switch (sz) {
 110	case GICV5_ITS_DT_ITT_CFGR_L2SZ_64k:
 111		return ITS_L2SZ_64K_L2_BITS;
 112	case GICV5_ITS_DT_ITT_CFGR_L2SZ_16k:
 113		return ITS_L2SZ_16K_L2_BITS;
 114	case GICV5_ITS_DT_ITT_CFGR_L2SZ_4k:
 115	default:
 116		return ITS_L2SZ_4K_L2_BITS;
 117	}
 118}
 119
 120static int gicv5_its_itt_cache_inv(struct gicv5_its_chip_data *its, u32 device_id,
 121				   u16 event_id)
 122{
 123	u32 eventr, eidr;
 124	u64 didr;
 125
 126	didr = FIELD_PREP(GICV5_ITS_DIDR_DEVICEID, device_id);
 127	eidr = FIELD_PREP(GICV5_ITS_EIDR_EVENTID, event_id);
 128	eventr = FIELD_PREP(GICV5_ITS_INV_EVENTR_I, 0x1);
 129
 130	its_writeq_relaxed(its, didr, GICV5_ITS_DIDR);
 131	its_writel_relaxed(its, eidr, GICV5_ITS_EIDR);
 132	its_writel_relaxed(its, eventr, GICV5_ITS_INV_EVENTR);
 133
 134	return gicv5_its_cache_sync(its);
 135}
 136
 137static void gicv5_its_free_itt_linear(struct gicv5_its_dev *its_dev)
 138{
 139	kfree(its_dev->itt_cfg.linear.itt);
 140}
 141
 142static void gicv5_its_free_itt_two_level(struct gicv5_its_dev *its_dev)
 143{
 144	unsigned int i, num_ents = its_dev->itt_cfg.l2.num_l1_ents;
 145
 146	for (i = 0; i < num_ents; i++)
 147		kfree(its_dev->itt_cfg.l2.l2ptrs[i]);
 148
 149	kfree(its_dev->itt_cfg.l2.l2ptrs);
 150	kfree(its_dev->itt_cfg.l2.l1itt);
 151}
 152
 153static void gicv5_its_free_itt(struct gicv5_its_dev *its_dev)
 154{
 155	if (!its_dev->itt_cfg.l2itt)
 156		gicv5_its_free_itt_linear(its_dev);
 157	else
 158		gicv5_its_free_itt_two_level(its_dev);
 159}
 160
 161static int gicv5_its_create_itt_linear(struct gicv5_its_chip_data *its,
 162				       struct gicv5_its_dev *its_dev,
 163				       unsigned int event_id_bits)
 164{
 165	unsigned int num_ents = BIT(event_id_bits);
 166	__le64 *itt;
 167
 168	itt = kcalloc(num_ents, sizeof(*itt), GFP_KERNEL);
 169	if (!itt)
 170		return -ENOMEM;
 171
 172	its_dev->itt_cfg.linear.itt = itt;
 173	its_dev->itt_cfg.linear.num_ents = num_ents;
 174	its_dev->itt_cfg.l2itt = false;
 175	its_dev->itt_cfg.event_id_bits = event_id_bits;
 176
 177	gicv5_its_dcache_clean(its, itt, num_ents * sizeof(*itt));
 178
 179	return 0;
 180}
 181
 182/*
 183 * Allocate a two-level ITT. All ITT entries are allocated in one go, unlike
 184 * with the device table. Span may be used to limit the second level table
 185 * size, where possible.
 186 */
 187static int gicv5_its_create_itt_two_level(struct gicv5_its_chip_data *its,
 188					  struct gicv5_its_dev *its_dev,
 189					  unsigned int event_id_bits,
 190					  unsigned int itt_l2sz,
 191					  unsigned int num_events)
 192{
 193	unsigned int l1_bits, l2_bits, span, events_per_l2_table;
 194	unsigned int complete_tables, final_span, num_ents;
 195	__le64 *itt_l1, *itt_l2, **l2ptrs;
 196	int i, ret;
 197	u64 val;
 198
 199	ret = gicv5_its_l2sz_to_l2_bits(itt_l2sz);
 200	if (ret >= event_id_bits) {
 201		pr_debug("Incorrect l2sz (0x%x) for %u EventID bits. Cannot allocate ITT\n",
 202			 itt_l2sz, event_id_bits);
 203		return -EINVAL;
 204	}
 205
 206	l2_bits = ret;
 207
 208	l1_bits = event_id_bits - l2_bits;
 209
 210	num_ents = BIT(l1_bits);
 211
 212	itt_l1 = kcalloc(num_ents, sizeof(*itt_l1), GFP_KERNEL);
 213	if (!itt_l1)
 214		return -ENOMEM;
 215
 216	l2ptrs = kcalloc(num_ents, sizeof(*l2ptrs), GFP_KERNEL);
 217	if (!l2ptrs) {
 218		kfree(itt_l1);
 219		return -ENOMEM;
 220	}
 221
 222	its_dev->itt_cfg.l2.l2ptrs = l2ptrs;
 223
 224	its_dev->itt_cfg.l2.l2sz = itt_l2sz;
 225	its_dev->itt_cfg.l2.l1itt = itt_l1;
 226	its_dev->itt_cfg.l2.num_l1_ents = num_ents;
 227	its_dev->itt_cfg.l2itt = true;
 228	its_dev->itt_cfg.event_id_bits = event_id_bits;
 229
 230	/*
 231	 * Need to determine how many entries there are per L2 - this is based
 232	 * on the number of bits in the table.
 233	 */
 234	events_per_l2_table = BIT(l2_bits);
 235	complete_tables = num_events / events_per_l2_table;
 236	final_span = order_base_2(num_events % events_per_l2_table);
 237
 238	for (i = 0; i < num_ents; i++) {
 239		size_t l2sz;
 240
 241		span = i == complete_tables ? final_span : l2_bits;
 242
 243		itt_l2 = kcalloc(BIT(span), sizeof(*itt_l2), GFP_KERNEL);
 244		if (!itt_l2) {
 245			ret = -ENOMEM;
 246			goto out_free;
 247		}
 248
 249		its_dev->itt_cfg.l2.l2ptrs[i] = itt_l2;
 250
 251		l2sz = BIT(span) * sizeof(*itt_l2);
 252
 253		gicv5_its_dcache_clean(its, itt_l2, l2sz);
 254
 255		val = (virt_to_phys(itt_l2) & GICV5_ITTL1E_L2_ADDR_MASK) |
 256		       FIELD_PREP(GICV5_ITTL1E_SPAN, span)		 |
 257		       FIELD_PREP(GICV5_ITTL1E_VALID, 0x1);
 258
 259		WRITE_ONCE(itt_l1[i], cpu_to_le64(val));
 260	}
 261
 262	gicv5_its_dcache_clean(its, itt_l1, num_ents * sizeof(*itt_l1));
 263
 264	return 0;
 265
 266out_free:
 267	for (i = i - 1; i >= 0; i--)
 268		kfree(its_dev->itt_cfg.l2.l2ptrs[i]);
 269
 270	kfree(its_dev->itt_cfg.l2.l2ptrs);
 271	kfree(itt_l1);
 272	return ret;
 273}
 274
 275/*
 276 * Function to check whether the device table or ITT table support
 277 * a two-level table and if so depending on the number of id_bits
 278 * requested, determine whether a two-level table is required.
 279 *
 280 * Return the 2-level size value if a two level table is deemed
 281 * necessary.
 282 */
 283static bool gicv5_its_l2sz_two_level(bool devtab, u32 its_idr1, u8 id_bits, u8 *sz)
 284{
 285	unsigned int l2_bits, l2_sz;
 286
 287	if (devtab && !FIELD_GET(GICV5_ITS_IDR1_DT_LEVELS, its_idr1))
 288		return false;
 289
 290	if (!devtab && !FIELD_GET(GICV5_ITS_IDR1_ITT_LEVELS, its_idr1))
 291		return false;
 292
 293	/*
 294	 * Pick an L2 size that matches the pagesize; if a match
 295	 * is not found, go for the smallest supported l2 size granule.
 296	 *
 297	 * This ensures that we will always be able to allocate
 298	 * contiguous memory at L2.
 299	 */
 300	switch (PAGE_SIZE) {
 301	case SZ_64K:
 302		if (GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(its_idr1)) {
 303			l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_64k;
 304			break;
 305		}
 306		fallthrough;
 307	case SZ_4K:
 308		if (GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(its_idr1)) {
 309			l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k;
 310			break;
 311		}
 312		fallthrough;
 313	case SZ_16K:
 314		if (GICV5_ITS_IDR1_L2SZ_SUPPORT_16KB(its_idr1)) {
 315			l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_16k;
 316			break;
 317		}
 318		if (GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(its_idr1)) {
 319			l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k;
 320			break;
 321		}
 322		if (GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(its_idr1)) {
 323			l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_64k;
 324			break;
 325		}
 326
 327		l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k;
 328		break;
 329	}
 330
 331	l2_bits = gicv5_its_l2sz_to_l2_bits(l2_sz);
 332
 333	if (l2_bits > id_bits)
 334		return false;
 335
 336	*sz = l2_sz;
 337
 338	return true;
 339}
 340
 341static __le64 *gicv5_its_device_get_itte_ref(struct gicv5_its_dev *its_dev,
 342					     u16 event_id)
 343{
 344	unsigned int l1_idx, l2_idx, l2_bits;
 345	__le64 *l2_itt;
 346
 347	if (!its_dev->itt_cfg.l2itt) {
 348		__le64 *itt = its_dev->itt_cfg.linear.itt;
 349
 350		return &itt[event_id];
 351	}
 352
 353	l2_bits = gicv5_its_l2sz_to_l2_bits(its_dev->itt_cfg.l2.l2sz);
 354	l1_idx = event_id >> l2_bits;
 355	l2_idx = event_id & GENMASK(l2_bits - 1, 0);
 356	l2_itt = its_dev->itt_cfg.l2.l2ptrs[l1_idx];
 357
 358	return &l2_itt[l2_idx];
 359}
 360
 361static int gicv5_its_device_cache_inv(struct gicv5_its_chip_data *its,
 362				      struct gicv5_its_dev *its_dev)
 363{
 364	u32 devicer;
 365	u64 didr;
 366
 367	didr = FIELD_PREP(GICV5_ITS_DIDR_DEVICEID, its_dev->device_id);
 368	devicer = FIELD_PREP(GICV5_ITS_INV_DEVICER_I, 0x1)	|
 369		  FIELD_PREP(GICV5_ITS_INV_DEVICER_EVENTID_BITS,
 370			     its_dev->itt_cfg.event_id_bits)	|
 371		  FIELD_PREP(GICV5_ITS_INV_DEVICER_L1, 0x0);
 372	its_writeq_relaxed(its, didr, GICV5_ITS_DIDR);
 373	its_writel_relaxed(its, devicer, GICV5_ITS_INV_DEVICER);
 374
 375	return gicv5_its_cache_sync(its);
 376}
 377
 378/*
 379 * Allocate a level 2 device table entry, update L1 parent to reference it.
 380 * Only used for 2-level device tables, and it is called on demand.
 381 */
 382static int gicv5_its_alloc_l2_devtab(struct gicv5_its_chip_data *its,
 383				     unsigned int l1_index)
 384{
 385	__le64 *l2devtab, *l1devtab = its->devtab_cfgr.l2.l1devtab;
 386	u8 span, l2sz, l2_bits;
 387	u64 l1dte;
 388
 389	if (FIELD_GET(GICV5_DTL1E_VALID, le64_to_cpu(l1devtab[l1_index])))
 390		return 0;
 391
 392	span = FIELD_GET(GICV5_DTL1E_SPAN, le64_to_cpu(l1devtab[l1_index]));
 393	l2sz = devtab_cfgr_field(its, L2SZ);
 394
 395	l2_bits = gicv5_its_l2sz_to_l2_bits(l2sz);
 396
 397	/*
 398	 * Span allows us to create a smaller L2 device table.
 399	 * If it is too large, use the number of allowed L2 bits.
 400	 */
 401	if (span > l2_bits)
 402		span = l2_bits;
 403
 404	l2devtab = kcalloc(BIT(span), sizeof(*l2devtab), GFP_KERNEL);
 405	if (!l2devtab)
 406		return -ENOMEM;
 407
 408	its->devtab_cfgr.l2.l2ptrs[l1_index] = l2devtab;
 409
 410	l1dte = FIELD_PREP(GICV5_DTL1E_SPAN, span)			|
 411		(virt_to_phys(l2devtab) & GICV5_DTL1E_L2_ADDR_MASK)	|
 412		FIELD_PREP(GICV5_DTL1E_VALID, 0x1);
 413	its_write_table_entry(its, &l1devtab[l1_index], l1dte);
 414
 415	return 0;
 416}
 417
 418static __le64 *gicv5_its_devtab_get_dte_ref(struct gicv5_its_chip_data *its,
 419					    u32 device_id, bool alloc)
 420{
 421	u8 str = devtab_cfgr_field(its, STRUCTURE);
 422	unsigned int l2sz, l2_bits, l1_idx, l2_idx;
 423	__le64 *l2devtab;
 424	int ret;
 425
 426	if (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) {
 427		l2devtab = its->devtab_cfgr.linear.devtab;
 428		return &l2devtab[device_id];
 429	}
 430
 431	l2sz = devtab_cfgr_field(its, L2SZ);
 432	l2_bits = gicv5_its_l2sz_to_l2_bits(l2sz);
 433	l1_idx = device_id >> l2_bits;
 434	l2_idx = device_id & GENMASK(l2_bits - 1, 0);
 435
 436	if (alloc) {
 437		/*
 438		 * Allocate a new L2 device table here before
 439		 * continuing. We make the assumption that the span in
 440		 * the L1 table has been set correctly, and blindly use
 441		 * that value.
 442		 */
 443		ret = gicv5_its_alloc_l2_devtab(its, l1_idx);
 444		if (ret)
 445			return NULL;
 446	}
 447
 448	l2devtab = its->devtab_cfgr.l2.l2ptrs[l1_idx];
 449	return &l2devtab[l2_idx];
 450}
 451
 452/*
 453 * Register a new device in the device table. Allocate an ITT and
 454 * program the L2DTE entry according to the ITT structure that
 455 * was chosen.
 456 */
 457static int gicv5_its_device_register(struct gicv5_its_chip_data *its,
 458				     struct gicv5_its_dev *its_dev)
 459{
 460	u8 event_id_bits, device_id_bits, itt_struct, itt_l2sz;
 461	phys_addr_t itt_phys_base;
 462	bool two_level_itt;
 463	u32 idr1, idr2;
 464	__le64 *dte;
 465	u64 val;
 466	int ret;
 467
 468	device_id_bits = devtab_cfgr_field(its, DEVICEID_BITS);
 469
 470	if (its_dev->device_id >= BIT(device_id_bits)) {
 471		pr_err("Supplied DeviceID (%u) outside of Device Table range (%u)!",
 472		       its_dev->device_id, (u32)GENMASK(device_id_bits - 1, 0));
 473		return -EINVAL;
 474	}
 475
 476	dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, true);
 477	if (!dte)
 478		return -ENOMEM;
 479
 480	if (FIELD_GET(GICV5_DTL2E_VALID, le64_to_cpu(*dte)))
 481		return -EBUSY;
 482
 483	/*
 484	 * Determine how many bits we need, validate those against the max.
 485	 * Based on these, determine if we should go for a 1- or 2-level ITT.
 486	 */
 487	event_id_bits = order_base_2(its_dev->num_events);
 488
 489	idr2 = its_readl_relaxed(its, GICV5_ITS_IDR2);
 490
 491	if (event_id_bits > FIELD_GET(GICV5_ITS_IDR2_EVENTID_BITS, idr2)) {
 492		pr_err("Required EventID bits (%u) larger than supported bits (%u)!",
 493		       event_id_bits,
 494		       (u8)FIELD_GET(GICV5_ITS_IDR2_EVENTID_BITS, idr2));
 495		return -EINVAL;
 496	}
 497
 498	idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1);
 499
 500	/*
 501	 * L2 ITT size is programmed into the L2DTE regardless of
 502	 * whether a two-level or linear ITT is built, init it.
 503	 */
 504	itt_l2sz = 0;
 505
 506	two_level_itt = gicv5_its_l2sz_two_level(false, idr1, event_id_bits,
 507						  &itt_l2sz);
 508	if (two_level_itt)
 509		ret = gicv5_its_create_itt_two_level(its, its_dev, event_id_bits,
 510						     itt_l2sz,
 511						     its_dev->num_events);
 512	else
 513		ret = gicv5_its_create_itt_linear(its, its_dev, event_id_bits);
 514	if (ret)
 515		return ret;
 516
 517	itt_phys_base = two_level_itt ? virt_to_phys(its_dev->itt_cfg.l2.l1itt) :
 518					virt_to_phys(its_dev->itt_cfg.linear.itt);
 519
 520	itt_struct = two_level_itt ? GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL :
 521				     GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR;
 522
 523	val = FIELD_PREP(GICV5_DTL2E_EVENT_ID_BITS, event_id_bits)	|
 524	      FIELD_PREP(GICV5_DTL2E_ITT_STRUCTURE, itt_struct)		|
 525	      (itt_phys_base & GICV5_DTL2E_ITT_ADDR_MASK)		|
 526	      FIELD_PREP(GICV5_DTL2E_ITT_L2SZ, itt_l2sz)		|
 527	      FIELD_PREP(GICV5_DTL2E_VALID, 0x1);
 528
 529	its_write_table_entry(its, dte, val);
 530
 531	ret = gicv5_its_device_cache_inv(its, its_dev);
 532	if (ret) {
 533		its_write_table_entry(its, dte, 0);
 534		gicv5_its_free_itt(its_dev);
 535		return ret;
 536	}
 537
 538	return 0;
 539}
 540
 541/*
 542 * Unregister a device in the device table. Lookup the device by ID, free the
 543 * corresponding ITT, mark the device as invalid in the device table.
 544 */
 545static int gicv5_its_device_unregister(struct gicv5_its_chip_data *its,
 546				       struct gicv5_its_dev *its_dev)
 547{
 548	__le64 *dte;
 549
 550	dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, false);
 551
 552	if (!FIELD_GET(GICV5_DTL2E_VALID, le64_to_cpu(*dte))) {
 553		pr_debug("Device table entry for DeviceID 0x%x is not valid. Nothing to clean up!",
 554			 its_dev->device_id);
 555		return -EINVAL;
 556	}
 557
 558	/* Zero everything - make it clear that this is an invalid entry */
 559	its_write_table_entry(its, dte, 0);
 560
 561	gicv5_its_free_itt(its_dev);
 562
 563	return gicv5_its_device_cache_inv(its, its_dev);
 564}
 565
 566/*
 567 * Allocate a 1-level device table. All entries are allocated, but marked
 568 * invalid.
 569 */
 570static int gicv5_its_alloc_devtab_linear(struct gicv5_its_chip_data *its,
 571					u8 device_id_bits)
 572{
 573	__le64 *devtab;
 574	size_t sz;
 575	u64 baser;
 576	u32 cfgr;
 577
 578	/*
 579	 * We expect a GICv5 implementation requiring a large number of
 580	 * deviceID bits to support a 2-level device table. If that's not
 581	 * the case, cap the number of deviceIDs supported according to the
 582	 * kmalloc limits so that the system can chug along with a linear
 583	 * device table.
 584	 */
 585	sz = BIT_ULL(device_id_bits) * sizeof(*devtab);
 586	if (sz > KMALLOC_MAX_SIZE) {
 587		u8 device_id_cap = ilog2(KMALLOC_MAX_SIZE/sizeof(*devtab));
 588
 589		pr_warn("Limiting device ID bits from %u to %u\n",
 590			device_id_bits, device_id_cap);
 591		device_id_bits = device_id_cap;
 592	}
 593
 594	devtab = kcalloc(BIT(device_id_bits), sizeof(*devtab), GFP_KERNEL);
 595	if (!devtab)
 596		return -ENOMEM;
 597
 598	gicv5_its_dcache_clean(its, devtab, sz);
 599
 600	cfgr = FIELD_PREP(GICV5_ITS_DT_CFGR_STRUCTURE,
 601			  GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR)	|
 602	       FIELD_PREP(GICV5_ITS_DT_CFGR_L2SZ, 0)			|
 603	       FIELD_PREP(GICV5_ITS_DT_CFGR_DEVICEID_BITS, device_id_bits);
 604	its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR);
 605
 606	baser = virt_to_phys(devtab) & GICV5_ITS_DT_BASER_ADDR_MASK;
 607	its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER);
 608
 609	its->devtab_cfgr.cfgr = cfgr;
 610	its->devtab_cfgr.linear.devtab = devtab;
 611
 612	return 0;
 613}
 614
 615/*
 616 * Allocate a 2-level device table. L2 entries are not allocated,
 617 * they are allocated on-demand.
 618 */
 619static int gicv5_its_alloc_devtab_two_level(struct gicv5_its_chip_data *its,
 620					    u8 device_id_bits,
 621					    u8 devtab_l2sz)
 622{
 623	unsigned int l1_bits, l2_bits, i;
 624	__le64 *l1devtab, **l2ptrs;
 625	size_t l1_sz;
 626	u64 baser;
 627	u32 cfgr;
 628
 629	l2_bits = gicv5_its_l2sz_to_l2_bits(devtab_l2sz);
 630
 631	l1_bits = device_id_bits - l2_bits;
 632	l1_sz = BIT(l1_bits) * sizeof(*l1devtab);
 633	/*
 634	 * With 2-level device table support it is highly unlikely
 635	 * that we are not able to allocate the required amount of
 636	 * device table memory to cover deviceID space; cap the
 637	 * deviceID space if we encounter such set-up.
 638	 * If this ever becomes a problem we could revisit the policy
 639	 * behind level 2 size selection to reduce level-1 deviceID bits.
 640	 */
 641	if (l1_sz > KMALLOC_MAX_SIZE) {
 642		l1_bits = ilog2(KMALLOC_MAX_SIZE/sizeof(*l1devtab));
 643
 644		pr_warn("Limiting device ID bits from %u to %u\n",
 645			device_id_bits, l1_bits + l2_bits);
 646		device_id_bits = l1_bits + l2_bits;
 647		l1_sz = KMALLOC_MAX_SIZE;
 648	}
 649
 650	l1devtab = kcalloc(BIT(l1_bits), sizeof(*l1devtab), GFP_KERNEL);
 651	if (!l1devtab)
 652		return -ENOMEM;
 653
 654	l2ptrs = kcalloc(BIT(l1_bits), sizeof(*l2ptrs), GFP_KERNEL);
 655	if (!l2ptrs) {
 656		kfree(l1devtab);
 657		return -ENOMEM;
 658	}
 659
 660	for (i = 0; i < BIT(l1_bits); i++)
 661		l1devtab[i] = cpu_to_le64(FIELD_PREP(GICV5_DTL1E_SPAN, l2_bits));
 662
 663	gicv5_its_dcache_clean(its, l1devtab, l1_sz);
 664
 665	cfgr = FIELD_PREP(GICV5_ITS_DT_CFGR_STRUCTURE,
 666			  GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL)	|
 667	       FIELD_PREP(GICV5_ITS_DT_CFGR_L2SZ, devtab_l2sz)		|
 668	       FIELD_PREP(GICV5_ITS_DT_CFGR_DEVICEID_BITS, device_id_bits);
 669	its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR);
 670
 671	baser = virt_to_phys(l1devtab) & GICV5_ITS_DT_BASER_ADDR_MASK;
 672	its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER);
 673
 674	its->devtab_cfgr.cfgr = cfgr;
 675	its->devtab_cfgr.l2.l1devtab = l1devtab;
 676	its->devtab_cfgr.l2.l2ptrs = l2ptrs;
 677
 678	return 0;
 679}
 680
 681/*
 682 * Initialise the device table as either 1- or 2-level depending on what is
 683 * supported by the hardware.
 684 */
 685static int gicv5_its_init_devtab(struct gicv5_its_chip_data *its)
 686{
 687	u8 device_id_bits, devtab_l2sz;
 688	bool two_level_devtab;
 689	u32 idr1;
 690
 691	idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1);
 692
 693	device_id_bits = FIELD_GET(GICV5_ITS_IDR1_DEVICEID_BITS, idr1);
 694	two_level_devtab = gicv5_its_l2sz_two_level(true, idr1, device_id_bits,
 695						    &devtab_l2sz);
 696	if (two_level_devtab)
 697		return gicv5_its_alloc_devtab_two_level(its, device_id_bits,
 698						       devtab_l2sz);
 699	else
 700		return gicv5_its_alloc_devtab_linear(its, device_id_bits);
 701}
 702
 703static void gicv5_its_deinit_devtab(struct gicv5_its_chip_data *its)
 704{
 705	u8 str = devtab_cfgr_field(its, STRUCTURE);
 706
 707	if (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) {
 708		kfree(its->devtab_cfgr.linear.devtab);
 709	} else {
 710		kfree(its->devtab_cfgr.l2.l1devtab);
 711		kfree(its->devtab_cfgr.l2.l2ptrs);
 712	}
 713}
 714
 715static void gicv5_its_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
 716{
 717	struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d);
 718	u64 addr = its_dev->its_trans_phys_base;
 719
 720	msg->data = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
 721	msi_msg_set_addr(irq_data_get_msi_desc(d), msg, addr);
 722}
 723
 724static const struct irq_chip gicv5_its_irq_chip = {
 725	.name			= "GICv5-ITS-MSI",
 726	.irq_mask		= irq_chip_mask_parent,
 727	.irq_unmask		= irq_chip_unmask_parent,
 728	.irq_eoi		= irq_chip_eoi_parent,
 729	.irq_set_affinity	= irq_chip_set_affinity_parent,
 730	.irq_get_irqchip_state	= irq_chip_get_parent_state,
 731	.irq_set_irqchip_state	= irq_chip_set_parent_state,
 732	.irq_compose_msi_msg	= gicv5_its_compose_msi_msg,
 733};
 734
 735static struct gicv5_its_dev *gicv5_its_find_device(struct gicv5_its_chip_data *its,
 736						   u32 device_id)
 737{
 738	struct gicv5_its_dev *dev = xa_load(&its->its_devices, device_id);
 739
 740	return dev ? dev : ERR_PTR(-ENODEV);
 741}
 742
 743static struct gicv5_its_dev *gicv5_its_alloc_device(struct gicv5_its_chip_data *its, int nvec,
 744						    u32 dev_id)
 745{
 746	struct gicv5_its_dev *its_dev;
 747	void *entry;
 748	int ret;
 749
 750	its_dev = gicv5_its_find_device(its, dev_id);
 751	if (!IS_ERR(its_dev)) {
 752		pr_err("A device with this DeviceID (0x%x) has already been registered.\n",
 753		       dev_id);
 754
 755		return ERR_PTR(-EBUSY);
 756	}
 757
 758	its_dev = kzalloc(sizeof(*its_dev), GFP_KERNEL);
 759	if (!its_dev)
 760		return ERR_PTR(-ENOMEM);
 761
 762	its_dev->device_id = dev_id;
 763	its_dev->num_events = nvec;
 764
 765	ret = gicv5_its_device_register(its, its_dev);
 766	if (ret) {
 767		pr_err("Failed to register the device\n");
 768		goto out_dev_free;
 769	}
 770
 771	its_dev->its_node = its;
 772
 773	its_dev->event_map = (unsigned long *)bitmap_zalloc(its_dev->num_events, GFP_KERNEL);
 774	if (!its_dev->event_map) {
 775		ret = -ENOMEM;
 776		goto out_unregister;
 777	}
 778
 779	entry = xa_store(&its->its_devices, dev_id, its_dev, GFP_KERNEL);
 780	if (xa_is_err(entry)) {
 781		ret = xa_err(entry);
 782		goto out_bitmap_free;
 783	}
 784
 785	return its_dev;
 786
 787out_bitmap_free:
 788	bitmap_free(its_dev->event_map);
 789out_unregister:
 790	gicv5_its_device_unregister(its, its_dev);
 791out_dev_free:
 792	kfree(its_dev);
 793	return ERR_PTR(ret);
 794}
 795
 796static int gicv5_its_msi_prepare(struct irq_domain *domain, struct device *dev,
 797				 int nvec, msi_alloc_info_t *info)
 798{
 799	u32 dev_id = info->scratchpad[0].ul;
 800	struct msi_domain_info *msi_info;
 801	struct gicv5_its_chip_data *its;
 802	struct gicv5_its_dev *its_dev;
 803
 804	msi_info = msi_get_domain_info(domain);
 805	its = msi_info->data;
 806
 807	guard(mutex)(&its->dev_alloc_lock);
 808
 809	its_dev = gicv5_its_alloc_device(its, nvec, dev_id);
 810	if (IS_ERR(its_dev))
 811		return PTR_ERR(its_dev);
 812
 813	its_dev->its_trans_phys_base = info->scratchpad[1].ul;
 814	info->scratchpad[0].ptr = its_dev;
 815
 816	return 0;
 817}
 818
 819static void gicv5_its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info)
 820{
 821	struct gicv5_its_dev *its_dev = info->scratchpad[0].ptr;
 822	struct msi_domain_info *msi_info;
 823	struct gicv5_its_chip_data *its;
 824
 825	msi_info = msi_get_domain_info(domain);
 826	its = msi_info->data;
 827
 828	guard(mutex)(&its->dev_alloc_lock);
 829
 830	if (WARN_ON_ONCE(!bitmap_empty(its_dev->event_map, its_dev->num_events)))
 831		return;
 832
 833	xa_erase(&its->its_devices, its_dev->device_id);
 834	bitmap_free(its_dev->event_map);
 835	gicv5_its_device_unregister(its, its_dev);
 836	kfree(its_dev);
 837}
 838
 839static struct msi_domain_ops gicv5_its_msi_domain_ops = {
 840	.msi_prepare	= gicv5_its_msi_prepare,
 841	.msi_teardown	= gicv5_its_msi_teardown,
 842};
 843
 844static int gicv5_its_map_event(struct gicv5_its_dev *its_dev, u16 event_id, u32 lpi)
 845{
 846	struct gicv5_its_chip_data *its = its_dev->its_node;
 847	u64 itt_entry;
 848	__le64 *itte;
 849
 850	itte = gicv5_its_device_get_itte_ref(its_dev, event_id);
 851
 852	if (FIELD_GET(GICV5_ITTL2E_VALID, le64_to_cpu(*itte)))
 853		return -EEXIST;
 854
 855	itt_entry = FIELD_PREP(GICV5_ITTL2E_LPI_ID, lpi) |
 856		    FIELD_PREP(GICV5_ITTL2E_VALID, 0x1);
 857
 858	its_write_table_entry(its, itte, itt_entry);
 859
 860	gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id);
 861
 862	return 0;
 863}
 864
 865static void gicv5_its_unmap_event(struct gicv5_its_dev *its_dev, u16 event_id)
 866{
 867	struct gicv5_its_chip_data *its = its_dev->its_node;
 868	u64 itte_val;
 869	__le64 *itte;
 870
 871	itte = gicv5_its_device_get_itte_ref(its_dev, event_id);
 872
 873	itte_val = le64_to_cpu(*itte);
 874	itte_val &= ~GICV5_ITTL2E_VALID;
 875
 876	its_write_table_entry(its, itte, itte_val);
 877
 878	gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id);
 879}
 880
 881static int gicv5_its_alloc_eventid(struct gicv5_its_dev *its_dev, msi_alloc_info_t *info,
 882				   unsigned int nr_irqs, u32 *eventid)
 883{
 884	int event_id_base;
 885
 886	if (!(info->flags & MSI_ALLOC_FLAGS_FIXED_MSG_DATA)) {
 887		event_id_base = bitmap_find_free_region(its_dev->event_map,
 888							its_dev->num_events,
 889							get_count_order(nr_irqs));
 890		if (event_id_base < 0)
 891			return event_id_base;
 892	} else {
 893		/*
 894		 * We want to have a fixed EventID mapped for hardcoded
 895		 * message data allocations.
 896		 */
 897		if (WARN_ON_ONCE(nr_irqs != 1))
 898			return -EINVAL;
 899
 900		event_id_base = info->hwirq;
 901
 902		if (event_id_base >= its_dev->num_events) {
 903			pr_err("EventID ouside of ITT range; cannot allocate an ITT entry!\n");
 904
 905			return -EINVAL;
 906		}
 907
 908		if (test_and_set_bit(event_id_base, its_dev->event_map)) {
 909			pr_warn("Can't reserve event_id bitmap\n");
 910			return -EINVAL;
 911
 912		}
 913	}
 914
 915	*eventid = event_id_base;
 916
 917	return 0;
 918}
 919
 920static void gicv5_its_free_eventid(struct gicv5_its_dev *its_dev, u32 event_id_base,
 921				   unsigned int nr_irqs)
 922{
 923	bitmap_release_region(its_dev->event_map, event_id_base,
 924			      get_count_order(nr_irqs));
 925}
 926
 927static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 928				      unsigned int nr_irqs, void *arg)
 929{
 930	u32 device_id, event_id_base, lpi;
 931	struct gicv5_its_dev *its_dev;
 932	msi_alloc_info_t *info = arg;
 933	irq_hw_number_t hwirq;
 934	struct irq_data *irqd;
 935	int ret, i;
 936
 937	its_dev = info->scratchpad[0].ptr;
 938
 939	ret = gicv5_its_alloc_eventid(its_dev, info, nr_irqs, &event_id_base);
 940	if (ret)
 941		return ret;
 942
 943	ret = iommu_dma_prepare_msi(info->desc, its_dev->its_trans_phys_base);
 944	if (ret)
 945		goto out_eventid;
 946
 947	device_id = its_dev->device_id;
 948
 949	for (i = 0; i < nr_irqs; i++) {
 950		ret = gicv5_alloc_lpi();
 951		if (ret < 0) {
 952			pr_debug("Failed to find free LPI!\n");
 953			goto out_free_irqs;
 954		}
 955		lpi = ret;
 956
 957		ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi);
 958		if (ret) {
 959			gicv5_free_lpi(lpi);
 960			goto out_free_irqs;
 961		}
 962
 963		/*
 964		 * Store eventid and deviceid into the hwirq for later use.
 965		 *
 966		 *	hwirq  = event_id << 32 | device_id
 967		 */
 968		hwirq = FIELD_PREP(GICV5_ITS_HWIRQ_DEVICE_ID, device_id) |
 969			FIELD_PREP(GICV5_ITS_HWIRQ_EVENT_ID, (u64)event_id_base + i);
 970		irq_domain_set_info(domain, virq + i, hwirq,
 971				    &gicv5_its_irq_chip, its_dev,
 972				    handle_fasteoi_irq, NULL, NULL);
 973
 974		irqd = irq_get_irq_data(virq + i);
 975		irqd_set_single_target(irqd);
 976		irqd_set_affinity_on_activate(irqd);
 977	}
 978
 979	return 0;
 980
 981out_free_irqs:
 982	while (--i >= 0) {
 983		irqd = irq_domain_get_irq_data(domain, virq + i);
 984		gicv5_free_lpi(irqd->parent_data->hwirq);
 985		irq_domain_reset_irq_data(irqd);
 986		irq_domain_free_irqs_parent(domain, virq + i, 1);
 987	}
 988out_eventid:
 989	gicv5_its_free_eventid(its_dev, event_id_base, nr_irqs);
 990	return ret;
 991}
 992
 993static void gicv5_its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
 994				      unsigned int nr_irqs)
 995{
 996	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 997	struct gicv5_its_chip_data *its;
 998	struct gicv5_its_dev *its_dev;
 999	u16 event_id_base;
1000	unsigned int i;
1001
1002	its_dev = irq_data_get_irq_chip_data(d);
1003	its = its_dev->its_node;
1004
1005	event_id_base = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
1006
1007	bitmap_release_region(its_dev->event_map, event_id_base,
1008			      get_count_order(nr_irqs));
1009
1010	/*  Hierarchically free irq data */
1011	for (i = 0; i < nr_irqs; i++) {
1012		d = irq_domain_get_irq_data(domain, virq + i);
1013
1014		gicv5_free_lpi(d->parent_data->hwirq);
1015		irq_domain_reset_irq_data(d);
1016		irq_domain_free_irqs_parent(domain, virq + i, 1);
1017	}
1018
1019	gicv5_its_syncr(its, its_dev);
1020	gicv5_irs_syncr();
1021}
1022
1023static int gicv5_its_irq_domain_activate(struct irq_domain *domain, struct irq_data *d,
1024					 bool reserve)
1025{
1026	struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d);
1027	u16 event_id;
1028	u32 lpi;
1029
1030	event_id = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
1031	lpi = d->parent_data->hwirq;
1032
1033	return gicv5_its_map_event(its_dev, event_id, lpi);
1034}
1035
1036static void gicv5_its_irq_domain_deactivate(struct irq_domain *domain,
1037					    struct irq_data *d)
1038{
1039	struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d);
1040	u16 event_id;
1041
1042	event_id = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq);
1043
1044	gicv5_its_unmap_event(its_dev, event_id);
1045}
1046
1047static const struct irq_domain_ops gicv5_its_irq_domain_ops = {
1048	.alloc		= gicv5_its_irq_domain_alloc,
1049	.free		= gicv5_its_irq_domain_free,
1050	.activate	= gicv5_its_irq_domain_activate,
1051	.deactivate	= gicv5_its_irq_domain_deactivate,
1052	.select		= msi_lib_irq_domain_select,
1053};
1054
1055static int gicv5_its_write_cr0(struct gicv5_its_chip_data *its, bool enable)
1056{
1057	u32 cr0 = FIELD_PREP(GICV5_ITS_CR0_ITSEN, enable);
1058
1059	its_writel_relaxed(its, cr0, GICV5_ITS_CR0);
1060	return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_CR0,
1061					GICV5_ITS_CR0_IDLE, NULL);
1062}
1063
1064static int gicv5_its_enable(struct gicv5_its_chip_data *its)
1065{
1066	return gicv5_its_write_cr0(its, true);
1067}
1068
1069static int gicv5_its_disable(struct gicv5_its_chip_data *its)
1070{
1071	return gicv5_its_write_cr0(its, false);
1072}
1073
1074static void gicv5_its_print_info(struct gicv5_its_chip_data *its_node)
1075{
1076	bool devtab_linear;
1077	u8 device_id_bits;
1078	u8 str;
1079
1080	device_id_bits = devtab_cfgr_field(its_node, DEVICEID_BITS);
1081
1082	str = devtab_cfgr_field(its_node, STRUCTURE);
1083	devtab_linear = (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR);
1084
1085	pr_info("ITS %s enabled using %s device table device_id_bits %u\n",
1086		fwnode_get_name(its_node->fwnode),
1087		devtab_linear ? "linear" : "2-level",
1088		device_id_bits);
1089}
1090
1091static int gicv5_its_init_domain(struct gicv5_its_chip_data *its, struct irq_domain *parent)
1092{
1093	struct irq_domain_info dom_info = {
1094		.fwnode		= its->fwnode,
1095		.ops		= &gicv5_its_irq_domain_ops,
1096		.domain_flags	= its->msi_domain_flags,
1097		.parent		= parent,
1098	};
1099	struct msi_domain_info *info;
1100
1101	info = kzalloc(sizeof(*info), GFP_KERNEL);
1102	if (!info)
1103		return -ENOMEM;
1104
1105	info->ops = &gicv5_its_msi_domain_ops;
1106	info->data = its;
1107	dom_info.host_data = info;
1108
1109	if (!msi_create_parent_irq_domain(&dom_info, &gic_v5_its_msi_parent_ops)) {
1110		kfree(info);
1111		return -ENOMEM;
1112	}
1113
1114	return 0;
1115}
1116
1117static int __init gicv5_its_init_bases(void __iomem *its_base, struct fwnode_handle *handle,
1118				       struct irq_domain *parent_domain)
1119{
1120	struct device_node *np = to_of_node(handle);
1121	struct gicv5_its_chip_data *its_node;
1122	u32 cr0, cr1;
1123	bool enabled;
1124	int ret;
1125
1126	its_node = kzalloc(sizeof(*its_node), GFP_KERNEL);
1127	if (!its_node)
1128		return -ENOMEM;
1129
1130	mutex_init(&its_node->dev_alloc_lock);
1131	xa_init(&its_node->its_devices);
1132	its_node->fwnode = handle;
1133	its_node->its_base = its_base;
1134	its_node->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI |
1135				     IRQ_DOMAIN_FLAG_FWNODE_PARENT;
1136
1137	cr0 = its_readl_relaxed(its_node, GICV5_ITS_CR0);
1138	enabled = FIELD_GET(GICV5_ITS_CR0_ITSEN, cr0);
1139	if (WARN(enabled, "ITS %s enabled, disabling it before proceeding\n", np->full_name)) {
1140		ret = gicv5_its_disable(its_node);
1141		if (ret)
1142			goto out_free_node;
1143	}
1144
1145	if (of_property_read_bool(np, "dma-noncoherent")) {
1146		/*
1147		 * A non-coherent ITS implies that some cache levels cannot be
1148		 * used coherently by the cores and GIC. Our only option is to mark
1149		 * memory attributes for the GIC as non-cacheable; by default,
1150		 * non-cacheable memory attributes imply outer-shareable
1151		 * shareability, the value written into ITS_CR1_SH is ignored.
1152		 */
1153		cr1 = FIELD_PREP(GICV5_ITS_CR1_ITT_RA, GICV5_NO_READ_ALLOC)	|
1154		      FIELD_PREP(GICV5_ITS_CR1_DT_RA, GICV5_NO_READ_ALLOC)	|
1155		      FIELD_PREP(GICV5_ITS_CR1_IC, GICV5_NON_CACHE)		|
1156		      FIELD_PREP(GICV5_ITS_CR1_OC, GICV5_NON_CACHE);
1157		its_node->flags |= ITS_FLAGS_NON_COHERENT;
1158	} else {
1159		cr1 = FIELD_PREP(GICV5_ITS_CR1_ITT_RA, GICV5_READ_ALLOC)	|
1160		      FIELD_PREP(GICV5_ITS_CR1_DT_RA, GICV5_READ_ALLOC)		|
1161		      FIELD_PREP(GICV5_ITS_CR1_IC, GICV5_WB_CACHE)		|
1162		      FIELD_PREP(GICV5_ITS_CR1_OC, GICV5_WB_CACHE)		|
1163		      FIELD_PREP(GICV5_ITS_CR1_SH, GICV5_INNER_SHARE);
1164	}
1165
1166	its_writel_relaxed(its_node, cr1, GICV5_ITS_CR1);
1167
1168	ret = gicv5_its_init_devtab(its_node);
1169	if (ret)
1170		goto out_free_node;
1171
1172	ret = gicv5_its_enable(its_node);
1173	if (ret)
1174		goto out_free_devtab;
1175
1176	ret = gicv5_its_init_domain(its_node, parent_domain);
1177	if (ret)
1178		goto out_disable_its;
1179
1180	gicv5_its_print_info(its_node);
1181
1182	return 0;
1183
1184out_disable_its:
1185	gicv5_its_disable(its_node);
1186out_free_devtab:
1187	gicv5_its_deinit_devtab(its_node);
1188out_free_node:
1189	kfree(its_node);
1190	return ret;
1191}
1192
1193static int __init gicv5_its_init(struct device_node *node)
1194{
1195	void __iomem *its_base;
1196	int ret, idx;
1197
1198	idx = of_property_match_string(node, "reg-names", "ns-config");
1199	if (idx < 0) {
1200		pr_err("%pOF: ns-config reg-name not present\n", node);
1201		return -ENODEV;
1202	}
1203
1204	its_base = of_io_request_and_map(node, idx, of_node_full_name(node));
1205	if (IS_ERR(its_base)) {
1206		pr_err("%pOF: unable to map GICv5 ITS_CONFIG_FRAME\n", node);
1207		return PTR_ERR(its_base);
1208	}
1209
1210	ret = gicv5_its_init_bases(its_base, of_fwnode_handle(node),
1211				   gicv5_global_data.lpi_domain);
1212	if (ret)
1213		goto out_unmap;
1214
1215	return 0;
1216
1217out_unmap:
1218	iounmap(its_base);
1219	return ret;
1220}
1221
1222void __init gicv5_its_of_probe(struct device_node *parent)
1223{
1224	struct device_node *np;
1225
1226	for_each_available_child_of_node(parent, np) {
1227		if (!of_device_is_compatible(np, "arm,gic-v5-its"))
1228			continue;
1229
1230		if (gicv5_its_init(np))
1231			pr_err("Failed to init ITS %s\n", np->full_name);
1232	}
1233}