irq-gic-v3-its.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4 * Author: Marc Zyngier <marc.zyngier@arm.com>
   5 */
   6
   7#include <linux/acpi.h>
   8#include <linux/acpi_iort.h>
   9#include <linux/bitfield.h>
  10#include <linux/bitmap.h>
  11#include <linux/cpu.h>
  12#include <linux/crash_dump.h>
  13#include <linux/delay.h>
  14#include <linux/efi.h>
  15#include <linux/genalloc.h>
  16#include <linux/interrupt.h>
  17#include <linux/iommu.h>
  18#include <linux/iopoll.h>
  19#include <linux/irqdomain.h>
  20#include <linux/list.h>
  21#include <linux/log2.h>
  22#include <linux/mem_encrypt.h>
  23#include <linux/memblock.h>
  24#include <linux/mm.h>
  25#include <linux/msi.h>
  26#include <linux/of.h>
  27#include <linux/of_address.h>
  28#include <linux/of_irq.h>
  29#include <linux/of_pci.h>
  30#include <linux/of_platform.h>
  31#include <linux/percpu.h>
  32#include <linux/set_memory.h>
  33#include <linux/slab.h>
  34#include <linux/syscore_ops.h>
  35
  36#include <linux/irqchip.h>
  37#include <linux/irqchip/arm-gic-v3.h>
  38#include <linux/irqchip/arm-gic-v4.h>
  39
  40#include <asm/cputype.h>
  41#include <asm/exception.h>
  42
  43#include "irq-gic-common.h"
  44#include "irq-gic-its-msi-parent.h"
  45#include <linux/irqchip/irq-msi-lib.h>
  46
  47#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING		(1ULL << 0)
  48#define ITS_FLAGS_WORKAROUND_CAVIUM_22375	(1ULL << 1)
  49#define ITS_FLAGS_WORKAROUND_CAVIUM_23144	(1ULL << 2)
  50#define ITS_FLAGS_FORCE_NON_SHAREABLE		(1ULL << 3)
  51#define ITS_FLAGS_WORKAROUND_HISILICON_162100801	(1ULL << 4)
  52
  53#define RD_LOCAL_LPI_ENABLED                    BIT(0)
  54#define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
  55#define RD_LOCAL_MEMRESERVE_DONE                BIT(2)
  56
  57static u32 lpi_id_bits;
  58
  59/*
  60 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  61 * deal with (one configuration byte per interrupt). PENDBASE has to
  62 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  63 */
  64#define LPI_NRBITS		lpi_id_bits
  65#define LPI_PROPBASE_SZ		ALIGN(BIT(LPI_NRBITS), SZ_64K)
  66#define LPI_PENDBASE_SZ		ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  67
  68static u8 __ro_after_init lpi_prop_prio;
  69static struct its_node *find_4_1_its(void);
  70
  71/*
  72 * Collection structure - just an ID, and a redistributor address to
  73 * ping. We use one per CPU as a bag of interrupts assigned to this
  74 * CPU.
  75 */
  76struct its_collection {
  77	u64			target_address;
  78	u16			col_id;
  79};
  80
  81/*
  82 * The ITS_BASER structure - contains memory information, cached
  83 * value of BASER register configuration and ITS page size.
  84 */
  85struct its_baser {
  86	void		*base;
  87	u64		val;
  88	u32		order;
  89	u32		psz;
  90};
  91
  92struct its_device;
  93
  94/*
  95 * The ITS structure - contains most of the infrastructure, with the
  96 * top-level MSI domain, the command queue, the collections, and the
  97 * list of devices writing to it.
  98 *
  99 * dev_alloc_lock has to be taken for device allocations, while the
 100 * spinlock must be taken to parse data structures such as the device
 101 * list.
 102 */
 103struct its_node {
 104	raw_spinlock_t		lock;
 105	struct mutex		dev_alloc_lock;
 106	struct list_head	entry;
 107	void __iomem		*base;
 108	void __iomem		*sgir_base;
 109	phys_addr_t		phys_base;
 110	struct its_cmd_block	*cmd_base;
 111	struct its_cmd_block	*cmd_write;
 112	struct its_baser	tables[GITS_BASER_NR_REGS];
 113	struct its_collection	*collections;
 114	struct fwnode_handle	*fwnode_handle;
 115	u64			(*get_msi_base)(struct its_device *its_dev);
 116	u64			typer;
 117	u64			cbaser_save;
 118	u32			ctlr_save;
 119	u32			mpidr;
 120	struct list_head	its_device_list;
 121	u64			flags;
 122	unsigned long		list_nr;
 123	int			numa_node;
 124	unsigned int		msi_domain_flags;
 125	u32			pre_its_base; /* for Socionext Synquacer */
 126	int			vlpi_redist_offset;
 127};
 128
 129static DEFINE_PER_CPU(struct its_node *, local_4_1_its);
 130
 131#define is_v4(its)		(!!((its)->typer & GITS_TYPER_VLPIS))
 132#define is_v4_1(its)		(!!((its)->typer & GITS_TYPER_VMAPP))
 133#define device_ids(its)		(FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
 134
 135#define ITS_ITT_ALIGN		SZ_256
 136
 137/* The maximum number of VPEID bits supported by VLPI commands */
 138#define ITS_MAX_VPEID_BITS						\
 139	({								\
 140		int nvpeid = 16;					\
 141		if (gic_rdists->has_rvpeid &&				\
 142		    gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)		\
 143			nvpeid = 1 + (gic_rdists->gicd_typer2 &		\
 144				      GICD_TYPER2_VID);			\
 145									\
 146		nvpeid;							\
 147	})
 148#define ITS_MAX_VPEID		(1 << (ITS_MAX_VPEID_BITS))
 149
 150/* Convert page order to size in bytes */
 151#define PAGE_ORDER_TO_SIZE(o)	(PAGE_SIZE << (o))
 152
 153struct event_lpi_map {
 154	unsigned long		*lpi_map;
 155	u16			*col_map;
 156	irq_hw_number_t		lpi_base;
 157	int			nr_lpis;
 158	raw_spinlock_t		vlpi_lock;
 159	struct its_vm		*vm;
 160	struct its_vlpi_map	*vlpi_maps;
 161	int			nr_vlpis;
 162};
 163
 164/*
 165 * The ITS view of a device - belongs to an ITS, owns an interrupt
 166 * translation table, and a list of interrupts.  If it some of its
 167 * LPIs are injected into a guest (GICv4), the event_map.vm field
 168 * indicates which one.
 169 */
 170struct its_device {
 171	struct list_head	entry;
 172	struct its_node		*its;
 173	struct event_lpi_map	event_map;
 174	void			*itt;
 175	u32			itt_sz;
 176	u32			nr_ites;
 177	u32			device_id;
 178	bool			shared;
 179};
 180
 181static struct {
 182	raw_spinlock_t		lock;
 183	struct its_device	*dev;
 184	struct its_vpe		**vpes;
 185	int			next_victim;
 186} vpe_proxy;
 187
 188struct cpu_lpi_count {
 189	atomic_t	managed;
 190	atomic_t	unmanaged;
 191};
 192
 193static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
 194
 195static LIST_HEAD(its_nodes);
 196static DEFINE_RAW_SPINLOCK(its_lock);
 197static struct rdists *gic_rdists;
 198static struct irq_domain *its_parent;
 199
 200static unsigned long its_list_map;
 201static u16 vmovp_seq_num;
 202static DEFINE_RAW_SPINLOCK(vmovp_lock);
 203
 204static DEFINE_IDA(its_vpeid_ida);
 205
 206#define gic_data_rdist()		(raw_cpu_ptr(gic_rdists->rdist))
 207#define gic_data_rdist_cpu(cpu)		(per_cpu_ptr(gic_rdists->rdist, cpu))
 208#define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
 209#define gic_data_rdist_vlpi_base()	(gic_data_rdist_rd_base() + SZ_128K)
 210
 211static gfp_t gfp_flags_quirk;
 212
 213static struct page *its_alloc_pages_node(int node, gfp_t gfp,
 214					 unsigned int order)
 215{
 216	struct page *page;
 217	int ret = 0;
 218
 219	page = alloc_pages_node(node, gfp | gfp_flags_quirk, order);
 220
 221	if (!page)
 222		return NULL;
 223
 224	ret = set_memory_decrypted((unsigned long)page_address(page),
 225				   1 << order);
 226	/*
 227	 * If set_memory_decrypted() fails then we don't know what state the
 228	 * page is in, so we can't free it. Instead we leak it.
 229	 * set_memory_decrypted() will already have WARNed.
 230	 */
 231	if (ret)
 232		return NULL;
 233
 234	return page;
 235}
 236
 237static struct page *its_alloc_pages(gfp_t gfp, unsigned int order)
 238{
 239	return its_alloc_pages_node(NUMA_NO_NODE, gfp, order);
 240}
 241
 242static void its_free_pages(void *addr, unsigned int order)
 243{
 244	/*
 245	 * If the memory cannot be encrypted again then we must leak the pages.
 246	 * set_memory_encrypted() will already have WARNed.
 247	 */
 248	if (set_memory_encrypted((unsigned long)addr, 1 << order))
 249		return;
 250	free_pages((unsigned long)addr, order);
 251}
 252
 253static struct gen_pool *itt_pool;
 254
 255static void *itt_alloc_pool(int node, int size)
 256{
 257	unsigned long addr;
 258	struct page *page;
 259
 260	if (size >= PAGE_SIZE) {
 261		page = its_alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size));
 262
 263		return page ? page_address(page) : NULL;
 264	}
 265
 266	do {
 267		addr = gen_pool_alloc(itt_pool, size);
 268		if (addr)
 269			break;
 270
 271		page = its_alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
 272		if (!page)
 273			break;
 274
 275		gen_pool_add(itt_pool, (unsigned long)page_address(page), PAGE_SIZE, node);
 276	} while (!addr);
 277
 278	return (void *)addr;
 279}
 280
 281static void itt_free_pool(void *addr, int size)
 282{
 283	if (!addr)
 284		return;
 285
 286	if (size >= PAGE_SIZE) {
 287		its_free_pages(addr, get_order(size));
 288		return;
 289	}
 290
 291	gen_pool_free(itt_pool, (unsigned long)addr, size);
 292}
 293
 294/*
 295 * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
 296 * always have vSGIs mapped.
 297 */
 298static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
 299{
 300	return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
 301}
 302
 303static bool rdists_support_shareable(void)
 304{
 305	return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE);
 306}
 307
 308static u16 get_its_list(struct its_vm *vm)
 309{
 310	struct its_node *its;
 311	unsigned long its_list = 0;
 312
 313	list_for_each_entry(its, &its_nodes, entry) {
 314		if (!is_v4(its))
 315			continue;
 316
 317		if (require_its_list_vmovp(vm, its))
 318			__set_bit(its->list_nr, &its_list);
 319	}
 320
 321	return (u16)its_list;
 322}
 323
 324static inline u32 its_get_event_id(struct irq_data *d)
 325{
 326	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 327	return d->hwirq - its_dev->event_map.lpi_base;
 328}
 329
 330static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 331					       u32 event)
 332{
 333	struct its_node *its = its_dev->its;
 334
 335	return its->collections + its_dev->event_map.col_map[event];
 336}
 337
 338static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
 339					       u32 event)
 340{
 341	if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
 342		return NULL;
 343
 344	return &its_dev->event_map.vlpi_maps[event];
 345}
 346
 347static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
 348{
 349	if (irqd_is_forwarded_to_vcpu(d)) {
 350		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 351		u32 event = its_get_event_id(d);
 352
 353		return dev_event_to_vlpi_map(its_dev, event);
 354	}
 355
 356	return NULL;
 357}
 358
 359static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
 360{
 361	raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
 362	return vpe->col_idx;
 363}
 364
 365static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
 366{
 367	raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
 368}
 369
 370static struct irq_chip its_vpe_irq_chip;
 371
 372static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
 373{
 374	struct its_vpe *vpe = NULL;
 375	int cpu;
 376
 377	if (d->chip == &its_vpe_irq_chip) {
 378		vpe = irq_data_get_irq_chip_data(d);
 379	} else {
 380		struct its_vlpi_map *map = get_vlpi_map(d);
 381		if (map)
 382			vpe = map->vpe;
 383	}
 384
 385	if (vpe) {
 386		cpu = vpe_to_cpuid_lock(vpe, flags);
 387	} else {
 388		/* Physical LPIs are already locked via the irq_desc lock */
 389		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 390		cpu = its_dev->event_map.col_map[its_get_event_id(d)];
 391		/* Keep GCC quiet... */
 392		*flags = 0;
 393	}
 394
 395	return cpu;
 396}
 397
 398static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
 399{
 400	struct its_vpe *vpe = NULL;
 401
 402	if (d->chip == &its_vpe_irq_chip) {
 403		vpe = irq_data_get_irq_chip_data(d);
 404	} else {
 405		struct its_vlpi_map *map = get_vlpi_map(d);
 406		if (map)
 407			vpe = map->vpe;
 408	}
 409
 410	if (vpe)
 411		vpe_to_cpuid_unlock(vpe, flags);
 412}
 413
 414static struct its_collection *valid_col(struct its_collection *col)
 415{
 416	if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
 417		return NULL;
 418
 419	return col;
 420}
 421
 422static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
 423{
 424	if (valid_col(its->collections + vpe->col_idx))
 425		return vpe;
 426
 427	return NULL;
 428}
 429
 430/*
 431 * ITS command descriptors - parameters to be encoded in a command
 432 * block.
 433 */
 434struct its_cmd_desc {
 435	union {
 436		struct {
 437			struct its_device *dev;
 438			u32 event_id;
 439		} its_inv_cmd;
 440
 441		struct {
 442			struct its_device *dev;
 443			u32 event_id;
 444		} its_clear_cmd;
 445
 446		struct {
 447			struct its_device *dev;
 448			u32 event_id;
 449		} its_int_cmd;
 450
 451		struct {
 452			struct its_device *dev;
 453			int valid;
 454		} its_mapd_cmd;
 455
 456		struct {
 457			struct its_collection *col;
 458			int valid;
 459		} its_mapc_cmd;
 460
 461		struct {
 462			struct its_device *dev;
 463			u32 phys_id;
 464			u32 event_id;
 465		} its_mapti_cmd;
 466
 467		struct {
 468			struct its_device *dev;
 469			struct its_collection *col;
 470			u32 event_id;
 471		} its_movi_cmd;
 472
 473		struct {
 474			struct its_device *dev;
 475			u32 event_id;
 476		} its_discard_cmd;
 477
 478		struct {
 479			struct its_collection *col;
 480		} its_invall_cmd;
 481
 482		struct {
 483			struct its_vpe *vpe;
 484		} its_vinvall_cmd;
 485
 486		struct {
 487			struct its_vpe *vpe;
 488			struct its_collection *col;
 489			bool valid;
 490		} its_vmapp_cmd;
 491
 492		struct {
 493			struct its_vpe *vpe;
 494			struct its_device *dev;
 495			u32 virt_id;
 496			u32 event_id;
 497			bool db_enabled;
 498		} its_vmapti_cmd;
 499
 500		struct {
 501			struct its_vpe *vpe;
 502			struct its_device *dev;
 503			u32 event_id;
 504			bool db_enabled;
 505		} its_vmovi_cmd;
 506
 507		struct {
 508			struct its_vpe *vpe;
 509			struct its_collection *col;
 510			u16 seq_num;
 511			u16 its_list;
 512		} its_vmovp_cmd;
 513
 514		struct {
 515			struct its_vpe *vpe;
 516		} its_invdb_cmd;
 517
 518		struct {
 519			struct its_vpe *vpe;
 520			u8 sgi;
 521			u8 priority;
 522			bool enable;
 523			bool group;
 524			bool clear;
 525		} its_vsgi_cmd;
 526	};
 527};
 528
 529/*
 530 * The ITS command block, which is what the ITS actually parses.
 531 */
 532struct its_cmd_block {
 533	union {
 534		u64	raw_cmd[4];
 535		__le64	raw_cmd_le[4];
 536	};
 537};
 538
 539#define ITS_CMD_QUEUE_SZ		SZ_64K
 540#define ITS_CMD_QUEUE_NR_ENTRIES	(ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 541
 542typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
 543						    struct its_cmd_block *,
 544						    struct its_cmd_desc *);
 545
 546typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
 547					      struct its_cmd_block *,
 548					      struct its_cmd_desc *);
 549
 550static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 551{
 552	u64 mask = GENMASK_ULL(h, l);
 553	*raw_cmd &= ~mask;
 554	*raw_cmd |= (val << l) & mask;
 555}
 556
 557static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 558{
 559	its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
 560}
 561
 562static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 563{
 564	its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
 565}
 566
 567static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 568{
 569	its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
 570}
 571
 572static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 573{
 574	its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
 575}
 576
 577static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 578{
 579	its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
 580}
 581
 582static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 583{
 584	its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
 585}
 586
 587static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 588{
 589	its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
 590}
 591
 592static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 593{
 594	its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
 595}
 596
 597static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 598{
 599	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 600}
 601
 602static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
 603{
 604	its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
 605}
 606
 607static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
 608{
 609	its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
 610}
 611
 612static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
 613{
 614	its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
 615}
 616
 617static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
 618{
 619	its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
 620}
 621
 622static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
 623{
 624	its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
 625}
 626
 627static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
 628{
 629	its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
 630}
 631
 632static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
 633{
 634	its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
 635}
 636
 637static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
 638{
 639	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
 640}
 641
 642static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
 643{
 644	its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
 645}
 646
 647static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
 648{
 649	its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
 650}
 651
 652static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
 653{
 654	its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
 655}
 656
 657static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
 658					u32 vpe_db_lpi)
 659{
 660	its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
 661}
 662
 663static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
 664					u32 vpe_db_lpi)
 665{
 666	its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
 667}
 668
 669static void its_encode_db(struct its_cmd_block *cmd, bool db)
 670{
 671	its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
 672}
 673
 674static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
 675{
 676	its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
 677}
 678
 679static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
 680{
 681	its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
 682}
 683
 684static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
 685{
 686	its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
 687}
 688
 689static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
 690{
 691	its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
 692}
 693
 694static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
 695{
 696	its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
 697}
 698
 699static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 700{
 701	/* Let's fixup BE commands */
 702	cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
 703	cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
 704	cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
 705	cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
 706}
 707
 708static struct its_collection *its_build_mapd_cmd(struct its_node *its,
 709						 struct its_cmd_block *cmd,
 710						 struct its_cmd_desc *desc)
 711{
 712	phys_addr_t itt_addr;
 713	u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 714
 715	itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 716
 717	its_encode_cmd(cmd, GITS_CMD_MAPD);
 718	its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 719	its_encode_size(cmd, size - 1);
 720	its_encode_itt(cmd, itt_addr);
 721	its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 722
 723	its_fixup_cmd(cmd);
 724
 725	return NULL;
 726}
 727
 728static struct its_collection *its_build_mapc_cmd(struct its_node *its,
 729						 struct its_cmd_block *cmd,
 730						 struct its_cmd_desc *desc)
 731{
 732	its_encode_cmd(cmd, GITS_CMD_MAPC);
 733	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 734	its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 735	its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 736
 737	its_fixup_cmd(cmd);
 738
 739	return desc->its_mapc_cmd.col;
 740}
 741
 742static struct its_collection *its_build_mapti_cmd(struct its_node *its,
 743						  struct its_cmd_block *cmd,
 744						  struct its_cmd_desc *desc)
 745{
 746	struct its_collection *col;
 747
 748	col = dev_event_to_col(desc->its_mapti_cmd.dev,
 749			       desc->its_mapti_cmd.event_id);
 750
 751	its_encode_cmd(cmd, GITS_CMD_MAPTI);
 752	its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
 753	its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
 754	its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
 755	its_encode_collection(cmd, col->col_id);
 756
 757	its_fixup_cmd(cmd);
 758
 759	return valid_col(col);
 760}
 761
 762static struct its_collection *its_build_movi_cmd(struct its_node *its,
 763						 struct its_cmd_block *cmd,
 764						 struct its_cmd_desc *desc)
 765{
 766	struct its_collection *col;
 767
 768	col = dev_event_to_col(desc->its_movi_cmd.dev,
 769			       desc->its_movi_cmd.event_id);
 770
 771	its_encode_cmd(cmd, GITS_CMD_MOVI);
 772	its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 773	its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 774	its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 775
 776	its_fixup_cmd(cmd);
 777
 778	return valid_col(col);
 779}
 780
 781static struct its_collection *its_build_discard_cmd(struct its_node *its,
 782						    struct its_cmd_block *cmd,
 783						    struct its_cmd_desc *desc)
 784{
 785	struct its_collection *col;
 786
 787	col = dev_event_to_col(desc->its_discard_cmd.dev,
 788			       desc->its_discard_cmd.event_id);
 789
 790	its_encode_cmd(cmd, GITS_CMD_DISCARD);
 791	its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 792	its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 793
 794	its_fixup_cmd(cmd);
 795
 796	return valid_col(col);
 797}
 798
 799static struct its_collection *its_build_inv_cmd(struct its_node *its,
 800						struct its_cmd_block *cmd,
 801						struct its_cmd_desc *desc)
 802{
 803	struct its_collection *col;
 804
 805	col = dev_event_to_col(desc->its_inv_cmd.dev,
 806			       desc->its_inv_cmd.event_id);
 807
 808	its_encode_cmd(cmd, GITS_CMD_INV);
 809	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 810	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 811
 812	its_fixup_cmd(cmd);
 813
 814	return valid_col(col);
 815}
 816
 817static struct its_collection *its_build_int_cmd(struct its_node *its,
 818						struct its_cmd_block *cmd,
 819						struct its_cmd_desc *desc)
 820{
 821	struct its_collection *col;
 822
 823	col = dev_event_to_col(desc->its_int_cmd.dev,
 824			       desc->its_int_cmd.event_id);
 825
 826	its_encode_cmd(cmd, GITS_CMD_INT);
 827	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 828	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 829
 830	its_fixup_cmd(cmd);
 831
 832	return valid_col(col);
 833}
 834
 835static struct its_collection *its_build_clear_cmd(struct its_node *its,
 836						  struct its_cmd_block *cmd,
 837						  struct its_cmd_desc *desc)
 838{
 839	struct its_collection *col;
 840
 841	col = dev_event_to_col(desc->its_clear_cmd.dev,
 842			       desc->its_clear_cmd.event_id);
 843
 844	its_encode_cmd(cmd, GITS_CMD_CLEAR);
 845	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 846	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 847
 848	its_fixup_cmd(cmd);
 849
 850	return valid_col(col);
 851}
 852
 853static struct its_collection *its_build_invall_cmd(struct its_node *its,
 854						   struct its_cmd_block *cmd,
 855						   struct its_cmd_desc *desc)
 856{
 857	its_encode_cmd(cmd, GITS_CMD_INVALL);
 858	its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
 859
 860	its_fixup_cmd(cmd);
 861
 862	return desc->its_invall_cmd.col;
 863}
 864
 865static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
 866					     struct its_cmd_block *cmd,
 867					     struct its_cmd_desc *desc)
 868{
 869	its_encode_cmd(cmd, GITS_CMD_VINVALL);
 870	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
 871
 872	its_fixup_cmd(cmd);
 873
 874	return valid_vpe(its, desc->its_vinvall_cmd.vpe);
 875}
 876
 877static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
 878					   struct its_cmd_block *cmd,
 879					   struct its_cmd_desc *desc)
 880{
 881	struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe);
 882	phys_addr_t vpt_addr, vconf_addr;
 883	u64 target;
 884	bool alloc;
 885
 886	its_encode_cmd(cmd, GITS_CMD_VMAPP);
 887	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
 888	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
 889
 890	if (!desc->its_vmapp_cmd.valid) {
 891		alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
 892		if (is_v4_1(its)) {
 893			its_encode_alloc(cmd, alloc);
 894			/*
 895			 * Unmapping a VPE is self-synchronizing on GICv4.1,
 896			 * no need to issue a VSYNC.
 897			 */
 898			vpe = NULL;
 899		}
 900
 901		goto out;
 902	}
 903
 904	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
 905	target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
 906
 907	its_encode_target(cmd, target);
 908	its_encode_vpt_addr(cmd, vpt_addr);
 909	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
 910
 911	alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
 912
 913	if (!is_v4_1(its))
 914		goto out;
 915
 916	vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
 917
 918	its_encode_alloc(cmd, alloc);
 919
 920	/*
 921	 * GICv4.1 provides a way to get the VLPI state, which needs the vPE
 922	 * to be unmapped first, and in this case, we may remap the vPE
 923	 * back while the VPT is not empty. So we can't assume that the
 924	 * VPT is empty on map. This is why we never advertise PTZ.
 925	 */
 926	its_encode_ptz(cmd, false);
 927	its_encode_vconf_addr(cmd, vconf_addr);
 928	its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
 929
 930out:
 931	its_fixup_cmd(cmd);
 932
 933	return vpe;
 934}
 935
 936static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
 937					    struct its_cmd_block *cmd,
 938					    struct its_cmd_desc *desc)
 939{
 940	u32 db;
 941
 942	if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
 943		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
 944	else
 945		db = 1023;
 946
 947	its_encode_cmd(cmd, GITS_CMD_VMAPTI);
 948	its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
 949	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
 950	its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
 951	its_encode_db_phys_id(cmd, db);
 952	its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
 953
 954	its_fixup_cmd(cmd);
 955
 956	return valid_vpe(its, desc->its_vmapti_cmd.vpe);
 957}
 958
 959static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
 960					   struct its_cmd_block *cmd,
 961					   struct its_cmd_desc *desc)
 962{
 963	u32 db;
 964
 965	if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
 966		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
 967	else
 968		db = 1023;
 969
 970	its_encode_cmd(cmd, GITS_CMD_VMOVI);
 971	its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
 972	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
 973	its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
 974	its_encode_db_phys_id(cmd, db);
 975	its_encode_db_valid(cmd, true);
 976
 977	its_fixup_cmd(cmd);
 978
 979	return valid_vpe(its, desc->its_vmovi_cmd.vpe);
 980}
 981
 982static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
 983					   struct its_cmd_block *cmd,
 984					   struct its_cmd_desc *desc)
 985{
 986	u64 target;
 987
 988	target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
 989	its_encode_cmd(cmd, GITS_CMD_VMOVP);
 990	its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
 991	its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
 992	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
 993	its_encode_target(cmd, target);
 994
 995	if (is_v4_1(its)) {
 996		its_encode_db(cmd, true);
 997		its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
 998	}
 999
1000	its_fixup_cmd(cmd);
1001
1002	return valid_vpe(its, desc->its_vmovp_cmd.vpe);
1003}
1004
1005static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
1006					  struct its_cmd_block *cmd,
1007					  struct its_cmd_desc *desc)
1008{
1009	struct its_vlpi_map *map;
1010
1011	map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
1012				    desc->its_inv_cmd.event_id);
1013
1014	its_encode_cmd(cmd, GITS_CMD_INV);
1015	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
1016	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
1017
1018	its_fixup_cmd(cmd);
1019
1020	return valid_vpe(its, map->vpe);
1021}
1022
1023static struct its_vpe *its_build_vint_cmd(struct its_node *its,
1024					  struct its_cmd_block *cmd,
1025					  struct its_cmd_desc *desc)
1026{
1027	struct its_vlpi_map *map;
1028
1029	map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
1030				    desc->its_int_cmd.event_id);
1031
1032	its_encode_cmd(cmd, GITS_CMD_INT);
1033	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
1034	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
1035
1036	its_fixup_cmd(cmd);
1037
1038	return valid_vpe(its, map->vpe);
1039}
1040
1041static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
1042					    struct its_cmd_block *cmd,
1043					    struct its_cmd_desc *desc)
1044{
1045	struct its_vlpi_map *map;
1046
1047	map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
1048				    desc->its_clear_cmd.event_id);
1049
1050	its_encode_cmd(cmd, GITS_CMD_CLEAR);
1051	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
1052	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
1053
1054	its_fixup_cmd(cmd);
1055
1056	return valid_vpe(its, map->vpe);
1057}
1058
1059static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
1060					   struct its_cmd_block *cmd,
1061					   struct its_cmd_desc *desc)
1062{
1063	if (WARN_ON(!is_v4_1(its)))
1064		return NULL;
1065
1066	its_encode_cmd(cmd, GITS_CMD_INVDB);
1067	its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
1068
1069	its_fixup_cmd(cmd);
1070
1071	return valid_vpe(its, desc->its_invdb_cmd.vpe);
1072}
1073
1074static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
1075					  struct its_cmd_block *cmd,
1076					  struct its_cmd_desc *desc)
1077{
1078	if (WARN_ON(!is_v4_1(its)))
1079		return NULL;
1080
1081	its_encode_cmd(cmd, GITS_CMD_VSGI);
1082	its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
1083	its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
1084	its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
1085	its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
1086	its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
1087	its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
1088
1089	its_fixup_cmd(cmd);
1090
1091	return valid_vpe(its, desc->its_vsgi_cmd.vpe);
1092}
1093
1094static u64 its_cmd_ptr_to_offset(struct its_node *its,
1095				 struct its_cmd_block *ptr)
1096{
1097	return (ptr - its->cmd_base) * sizeof(*ptr);
1098}
1099
1100static int its_queue_full(struct its_node *its)
1101{
1102	int widx;
1103	int ridx;
1104
1105	widx = its->cmd_write - its->cmd_base;
1106	ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1107
1108	/* This is incredibly unlikely to happen, unless the ITS locks up. */
1109	if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1110		return 1;
1111
1112	return 0;
1113}
1114
1115static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1116{
1117	struct its_cmd_block *cmd;
1118	u32 count = 1000000;	/* 1s! */
1119
1120	while (its_queue_full(its)) {
1121		count--;
1122		if (!count) {
1123			pr_err_ratelimited("ITS queue not draining\n");
1124			return NULL;
1125		}
1126		cpu_relax();
1127		udelay(1);
1128	}
1129
1130	cmd = its->cmd_write++;
1131
1132	/* Handle queue wrapping */
1133	if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1134		its->cmd_write = its->cmd_base;
1135
1136	/* Clear command  */
1137	cmd->raw_cmd[0] = 0;
1138	cmd->raw_cmd[1] = 0;
1139	cmd->raw_cmd[2] = 0;
1140	cmd->raw_cmd[3] = 0;
1141
1142	return cmd;
1143}
1144
1145static struct its_cmd_block *its_post_commands(struct its_node *its)
1146{
1147	u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1148
1149	writel_relaxed(wr, its->base + GITS_CWRITER);
1150
1151	return its->cmd_write;
1152}
1153
1154static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1155{
1156	/*
1157	 * Make sure the commands written to memory are observable by
1158	 * the ITS.
1159	 */
1160	if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1161		gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1162	else
1163		dsb(ishst);
1164}
1165
1166static int its_wait_for_range_completion(struct its_node *its,
1167					 u64	prev_idx,
1168					 struct its_cmd_block *to)
1169{
1170	u64 rd_idx, to_idx, linear_idx;
1171	u32 count = 1000000;	/* 1s! */
1172
1173	/* Linearize to_idx if the command set has wrapped around */
1174	to_idx = its_cmd_ptr_to_offset(its, to);
1175	if (to_idx < prev_idx)
1176		to_idx += ITS_CMD_QUEUE_SZ;
1177
1178	linear_idx = prev_idx;
1179
1180	while (1) {
1181		s64 delta;
1182
1183		rd_idx = readl_relaxed(its->base + GITS_CREADR);
1184
1185		/*
1186		 * Compute the read pointer progress, taking the
1187		 * potential wrap-around into account.
1188		 */
1189		delta = rd_idx - prev_idx;
1190		if (rd_idx < prev_idx)
1191			delta += ITS_CMD_QUEUE_SZ;
1192
1193		linear_idx += delta;
1194		if (linear_idx >= to_idx)
1195			break;
1196
1197		count--;
1198		if (!count) {
1199			pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1200					   to_idx, linear_idx);
1201			return -1;
1202		}
1203		prev_idx = rd_idx;
1204		cpu_relax();
1205		udelay(1);
1206	}
1207
1208	return 0;
1209}
1210
1211/* Warning, macro hell follows */
1212#define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)	\
1213void name(struct its_node *its,						\
1214	  buildtype builder,						\
1215	  struct its_cmd_desc *desc)					\
1216{									\
1217	struct its_cmd_block *cmd, *sync_cmd, *next_cmd;		\
1218	synctype *sync_obj;						\
1219	unsigned long flags;						\
1220	u64 rd_idx;							\
1221									\
1222	raw_spin_lock_irqsave(&its->lock, flags);			\
1223									\
1224	cmd = its_allocate_entry(its);					\
1225	if (!cmd) {		/* We're soooooo screewed... */		\
1226		raw_spin_unlock_irqrestore(&its->lock, flags);		\
1227		return;							\
1228	}								\
1229	sync_obj = builder(its, cmd, desc);				\
1230	its_flush_cmd(its, cmd);					\
1231									\
1232	if (sync_obj) {							\
1233		sync_cmd = its_allocate_entry(its);			\
1234		if (!sync_cmd)						\
1235			goto post;					\
1236									\
1237		buildfn(its, sync_cmd, sync_obj);			\
1238		its_flush_cmd(its, sync_cmd);				\
1239	}								\
1240									\
1241post:									\
1242	rd_idx = readl_relaxed(its->base + GITS_CREADR);		\
1243	next_cmd = its_post_commands(its);				\
1244	raw_spin_unlock_irqrestore(&its->lock, flags);			\
1245									\
1246	if (its_wait_for_range_completion(its, rd_idx, next_cmd))	\
1247		pr_err_ratelimited("ITS cmd %ps failed\n", builder);	\
1248}
1249
1250static void its_build_sync_cmd(struct its_node *its,
1251			       struct its_cmd_block *sync_cmd,
1252			       struct its_collection *sync_col)
1253{
1254	its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1255	its_encode_target(sync_cmd, sync_col->target_address);
1256
1257	its_fixup_cmd(sync_cmd);
1258}
1259
1260static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1261			     struct its_collection, its_build_sync_cmd)
1262
1263static void its_build_vsync_cmd(struct its_node *its,
1264				struct its_cmd_block *sync_cmd,
1265				struct its_vpe *sync_vpe)
1266{
1267	its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1268	its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1269
1270	its_fixup_cmd(sync_cmd);
1271}
1272
1273static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1274			     struct its_vpe, its_build_vsync_cmd)
1275
1276static void its_send_int(struct its_device *dev, u32 event_id)
1277{
1278	struct its_cmd_desc desc;
1279
1280	desc.its_int_cmd.dev = dev;
1281	desc.its_int_cmd.event_id = event_id;
1282
1283	its_send_single_command(dev->its, its_build_int_cmd, &desc);
1284}
1285
1286static void its_send_clear(struct its_device *dev, u32 event_id)
1287{
1288	struct its_cmd_desc desc;
1289
1290	desc.its_clear_cmd.dev = dev;
1291	desc.its_clear_cmd.event_id = event_id;
1292
1293	its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1294}
1295
1296static void its_send_inv(struct its_device *dev, u32 event_id)
1297{
1298	struct its_cmd_desc desc;
1299
1300	desc.its_inv_cmd.dev = dev;
1301	desc.its_inv_cmd.event_id = event_id;
1302
1303	its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1304}
1305
1306static void its_send_mapd(struct its_device *dev, int valid)
1307{
1308	struct its_cmd_desc desc;
1309
1310	desc.its_mapd_cmd.dev = dev;
1311	desc.its_mapd_cmd.valid = !!valid;
1312
1313	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1314}
1315
1316static void its_send_mapc(struct its_node *its, struct its_collection *col,
1317			  int valid)
1318{
1319	struct its_cmd_desc desc;
1320
1321	desc.its_mapc_cmd.col = col;
1322	desc.its_mapc_cmd.valid = !!valid;
1323
1324	its_send_single_command(its, its_build_mapc_cmd, &desc);
1325}
1326
1327static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1328{
1329	struct its_cmd_desc desc;
1330
1331	desc.its_mapti_cmd.dev = dev;
1332	desc.its_mapti_cmd.phys_id = irq_id;
1333	desc.its_mapti_cmd.event_id = id;
1334
1335	its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1336}
1337
1338static void its_send_movi(struct its_device *dev,
1339			  struct its_collection *col, u32 id)
1340{
1341	struct its_cmd_desc desc;
1342
1343	desc.its_movi_cmd.dev = dev;
1344	desc.its_movi_cmd.col = col;
1345	desc.its_movi_cmd.event_id = id;
1346
1347	its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1348}
1349
1350static void its_send_discard(struct its_device *dev, u32 id)
1351{
1352	struct its_cmd_desc desc;
1353
1354	desc.its_discard_cmd.dev = dev;
1355	desc.its_discard_cmd.event_id = id;
1356
1357	its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1358}
1359
1360static void its_send_invall(struct its_node *its, struct its_collection *col)
1361{
1362	struct its_cmd_desc desc;
1363
1364	desc.its_invall_cmd.col = col;
1365
1366	its_send_single_command(its, its_build_invall_cmd, &desc);
1367}
1368
1369static void its_send_vmapti(struct its_device *dev, u32 id)
1370{
1371	struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1372	struct its_cmd_desc desc;
1373
1374	desc.its_vmapti_cmd.vpe = map->vpe;
1375	desc.its_vmapti_cmd.dev = dev;
1376	desc.its_vmapti_cmd.virt_id = map->vintid;
1377	desc.its_vmapti_cmd.event_id = id;
1378	desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1379
1380	its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1381}
1382
1383static void its_send_vmovi(struct its_device *dev, u32 id)
1384{
1385	struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1386	struct its_cmd_desc desc;
1387
1388	desc.its_vmovi_cmd.vpe = map->vpe;
1389	desc.its_vmovi_cmd.dev = dev;
1390	desc.its_vmovi_cmd.event_id = id;
1391	desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1392
1393	its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1394}
1395
1396static void its_send_vmapp(struct its_node *its,
1397			   struct its_vpe *vpe, bool valid)
1398{
1399	struct its_cmd_desc desc;
1400
1401	desc.its_vmapp_cmd.vpe = vpe;
1402	desc.its_vmapp_cmd.valid = valid;
1403	desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1404
1405	its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1406}
1407
1408static void its_send_vmovp(struct its_vpe *vpe)
1409{
1410	struct its_cmd_desc desc = {};
1411	struct its_node *its;
1412	int col_id = vpe->col_idx;
1413
1414	desc.its_vmovp_cmd.vpe = vpe;
1415
1416	if (!its_list_map) {
1417		its = list_first_entry(&its_nodes, struct its_node, entry);
1418		desc.its_vmovp_cmd.col = &its->collections[col_id];
1419		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1420		return;
1421	}
1422
1423	/*
1424	 * Yet another marvel of the architecture. If using the
1425	 * its_list "feature", we need to make sure that all ITSs
1426	 * receive all VMOVP commands in the same order. The only way
1427	 * to guarantee this is to make vmovp a serialization point.
1428	 *
1429	 * Wall <-- Head.
1430	 */
1431	guard(raw_spinlock)(&vmovp_lock);
1432	desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1433	desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1434
1435	/* Emit VMOVPs */
1436	list_for_each_entry(its, &its_nodes, entry) {
1437		if (!is_v4(its))
1438			continue;
1439
1440		if (!require_its_list_vmovp(vpe->its_vm, its))
1441			continue;
1442
1443		desc.its_vmovp_cmd.col = &its->collections[col_id];
1444		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1445	}
1446}
1447
1448static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1449{
1450	struct its_cmd_desc desc;
1451
1452	desc.its_vinvall_cmd.vpe = vpe;
1453	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1454}
1455
1456static void its_send_vinv(struct its_device *dev, u32 event_id)
1457{
1458	struct its_cmd_desc desc;
1459
1460	/*
1461	 * There is no real VINV command. This is just a normal INV,
1462	 * with a VSYNC instead of a SYNC.
1463	 */
1464	desc.its_inv_cmd.dev = dev;
1465	desc.its_inv_cmd.event_id = event_id;
1466
1467	its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1468}
1469
1470static void its_send_vint(struct its_device *dev, u32 event_id)
1471{
1472	struct its_cmd_desc desc;
1473
1474	/*
1475	 * There is no real VINT command. This is just a normal INT,
1476	 * with a VSYNC instead of a SYNC.
1477	 */
1478	desc.its_int_cmd.dev = dev;
1479	desc.its_int_cmd.event_id = event_id;
1480
1481	its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1482}
1483
1484static void its_send_vclear(struct its_device *dev, u32 event_id)
1485{
1486	struct its_cmd_desc desc;
1487
1488	/*
1489	 * There is no real VCLEAR command. This is just a normal CLEAR,
1490	 * with a VSYNC instead of a SYNC.
1491	 */
1492	desc.its_clear_cmd.dev = dev;
1493	desc.its_clear_cmd.event_id = event_id;
1494
1495	its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1496}
1497
1498static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1499{
1500	struct its_cmd_desc desc;
1501
1502	desc.its_invdb_cmd.vpe = vpe;
1503	its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1504}
1505
1506/*
1507 * irqchip functions - assumes MSI, mostly.
1508 */
1509static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1510{
1511	struct its_vlpi_map *map = get_vlpi_map(d);
1512	irq_hw_number_t hwirq;
1513	void *va;
1514	u8 *cfg;
1515
1516	if (map) {
1517		va = page_address(map->vm->vprop_page);
1518		hwirq = map->vintid;
1519
1520		/* Remember the updated property */
1521		map->properties &= ~clr;
1522		map->properties |= set | LPI_PROP_GROUP1;
1523	} else {
1524		va = gic_rdists->prop_table_va;
1525		hwirq = d->hwirq;
1526	}
1527
1528	cfg = va + hwirq - 8192;
1529	*cfg &= ~clr;
1530	*cfg |= set | LPI_PROP_GROUP1;
1531
1532	/*
1533	 * Make the above write visible to the redistributors.
1534	 * And yes, we're flushing exactly: One. Single. Byte.
1535	 * Humpf...
1536	 */
1537	if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1538		gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1539	else
1540		dsb(ishst);
1541}
1542
1543static void wait_for_syncr(void __iomem *rdbase)
1544{
1545	while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1546		cpu_relax();
1547}
1548
1549static void __direct_lpi_inv(struct irq_data *d, u64 val)
1550{
1551	void __iomem *rdbase;
1552	unsigned long flags;
1553	int cpu;
1554
1555	/* Target the redistributor this LPI is currently routed to */
1556	cpu = irq_to_cpuid_lock(d, &flags);
1557	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1558
1559	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1560	gic_write_lpir(val, rdbase + GICR_INVLPIR);
1561	wait_for_syncr(rdbase);
1562
1563	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1564	irq_to_cpuid_unlock(d, flags);
1565}
1566
1567static void direct_lpi_inv(struct irq_data *d)
1568{
1569	struct its_vlpi_map *map = get_vlpi_map(d);
1570	u64 val;
1571
1572	if (map) {
1573		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1574
1575		WARN_ON(!is_v4_1(its_dev->its));
1576
1577		val  = GICR_INVLPIR_V;
1578		val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1579		val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1580	} else {
1581		val = d->hwirq;
1582	}
1583
1584	__direct_lpi_inv(d, val);
1585}
1586
1587static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1588{
1589	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1590
1591	lpi_write_config(d, clr, set);
1592	if (gic_rdists->has_direct_lpi &&
1593	    (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1594		direct_lpi_inv(d);
1595	else if (!irqd_is_forwarded_to_vcpu(d))
1596		its_send_inv(its_dev, its_get_event_id(d));
1597	else
1598		its_send_vinv(its_dev, its_get_event_id(d));
1599}
1600
1601static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1602{
1603	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1604	u32 event = its_get_event_id(d);
1605	struct its_vlpi_map *map;
1606
1607	/*
1608	 * GICv4.1 does away with the per-LPI nonsense, nothing to do
1609	 * here.
1610	 */
1611	if (is_v4_1(its_dev->its))
1612		return;
1613
1614	map = dev_event_to_vlpi_map(its_dev, event);
1615
1616	if (map->db_enabled == enable)
1617		return;
1618
1619	map->db_enabled = enable;
1620
1621	/*
1622	 * More fun with the architecture:
1623	 *
1624	 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1625	 * value or to 1023, depending on the enable bit. But that
1626	 * would be issuing a mapping for an /existing/ DevID+EventID
1627	 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1628	 * to the /same/ vPE, using this opportunity to adjust the
1629	 * doorbell. Mouahahahaha. We loves it, Precious.
1630	 */
1631	its_send_vmovi(its_dev, event);
1632}
1633
1634static void its_mask_irq(struct irq_data *d)
1635{
1636	if (irqd_is_forwarded_to_vcpu(d))
1637		its_vlpi_set_doorbell(d, false);
1638
1639	lpi_update_config(d, LPI_PROP_ENABLED, 0);
1640}
1641
1642static void its_unmask_irq(struct irq_data *d)
1643{
1644	if (irqd_is_forwarded_to_vcpu(d))
1645		its_vlpi_set_doorbell(d, true);
1646
1647	lpi_update_config(d, 0, LPI_PROP_ENABLED);
1648}
1649
1650static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1651{
1652	if (irqd_affinity_is_managed(d))
1653		return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1654
1655	return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1656}
1657
1658static void its_inc_lpi_count(struct irq_data *d, int cpu)
1659{
1660	if (irqd_affinity_is_managed(d))
1661		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1662	else
1663		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1664}
1665
1666static void its_dec_lpi_count(struct irq_data *d, int cpu)
1667{
1668	if (irqd_affinity_is_managed(d))
1669		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1670	else
1671		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1672}
1673
1674static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1675					      const struct cpumask *cpu_mask)
1676{
1677	unsigned int cpu = nr_cpu_ids, tmp;
1678	int count = S32_MAX;
1679
1680	for_each_cpu(tmp, cpu_mask) {
1681		int this_count = its_read_lpi_count(d, tmp);
1682		if (this_count < count) {
1683			cpu = tmp;
1684		        count = this_count;
1685		}
1686	}
1687
1688	return cpu;
1689}
1690
1691/*
1692 * As suggested by Thomas Gleixner in:
1693 * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1694 */
1695static int its_select_cpu(struct irq_data *d,
1696			  const struct cpumask *aff_mask)
1697{
1698	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1699	static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1700	static struct cpumask __tmpmask;
1701	struct cpumask *tmpmask;
1702	unsigned long flags;
1703	int cpu, node;
1704	node = its_dev->its->numa_node;
1705	tmpmask = &__tmpmask;
1706
1707	raw_spin_lock_irqsave(&tmpmask_lock, flags);
1708
1709	if (!irqd_affinity_is_managed(d)) {
1710		/* First try the NUMA node */
1711		if (node != NUMA_NO_NODE) {
1712			/*
1713			 * Try the intersection of the affinity mask and the
1714			 * node mask (and the online mask, just to be safe).
1715			 */
1716			cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1717			cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1718
1719			/*
1720			 * Ideally, we would check if the mask is empty, and
1721			 * try again on the full node here.
1722			 *
1723			 * But it turns out that the way ACPI describes the
1724			 * affinity for ITSs only deals about memory, and
1725			 * not target CPUs, so it cannot describe a single
1726			 * ITS placed next to two NUMA nodes.
1727			 *
1728			 * Instead, just fallback on the online mask. This
1729			 * diverges from Thomas' suggestion above.
1730			 */
1731			cpu = cpumask_pick_least_loaded(d, tmpmask);
1732			if (cpu < nr_cpu_ids)
1733				goto out;
1734
1735			/* If we can't cross sockets, give up */
1736			if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1737				goto out;
1738
1739			/* If the above failed, expand the search */
1740		}
1741
1742		/* Try the intersection of the affinity and online masks */
1743		cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1744
1745		/* If that doesn't fly, the online mask is the last resort */
1746		if (cpumask_empty(tmpmask))
1747			cpumask_copy(tmpmask, cpu_online_mask);
1748
1749		cpu = cpumask_pick_least_loaded(d, tmpmask);
1750	} else {
1751		cpumask_copy(tmpmask, aff_mask);
1752
1753		/* If we cannot cross sockets, limit the search to that node */
1754		if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1755		    node != NUMA_NO_NODE)
1756			cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1757
1758		cpu = cpumask_pick_least_loaded(d, tmpmask);
1759	}
1760out:
1761	raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
1762
1763	pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1764	return cpu;
1765}
1766
1767static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1768			    bool force)
1769{
1770	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1771	struct its_collection *target_col;
1772	u32 id = its_get_event_id(d);
1773	int cpu, prev_cpu;
1774
1775	/* A forwarded interrupt should use irq_set_vcpu_affinity */
1776	if (irqd_is_forwarded_to_vcpu(d))
1777		return -EINVAL;
1778
1779	prev_cpu = its_dev->event_map.col_map[id];
1780	its_dec_lpi_count(d, prev_cpu);
1781
1782	if (!force)
1783		cpu = its_select_cpu(d, mask_val);
1784	else
1785		cpu = cpumask_pick_least_loaded(d, mask_val);
1786
1787	if (cpu < 0 || cpu >= nr_cpu_ids)
1788		goto err;
1789
1790	/* don't set the affinity when the target cpu is same as current one */
1791	if (cpu != prev_cpu) {
1792		target_col = &its_dev->its->collections[cpu];
1793		its_send_movi(its_dev, target_col, id);
1794		its_dev->event_map.col_map[id] = cpu;
1795		irq_data_update_effective_affinity(d, cpumask_of(cpu));
1796	}
1797
1798	its_inc_lpi_count(d, cpu);
1799
1800	return IRQ_SET_MASK_OK_DONE;
1801
1802err:
1803	its_inc_lpi_count(d, prev_cpu);
1804	return -EINVAL;
1805}
1806
1807static u64 its_irq_get_msi_base(struct its_device *its_dev)
1808{
1809	struct its_node *its = its_dev->its;
1810
1811	return its->phys_base + GITS_TRANSLATER;
1812}
1813
1814static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1815{
1816	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1817
1818	msg->data = its_get_event_id(d);
1819	msi_msg_set_addr(irq_data_get_msi_desc(d), msg,
1820			 its_dev->its->get_msi_base(its_dev));
1821}
1822
1823static int its_irq_set_irqchip_state(struct irq_data *d,
1824				     enum irqchip_irq_state which,
1825				     bool state)
1826{
1827	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1828	u32 event = its_get_event_id(d);
1829
1830	if (which != IRQCHIP_STATE_PENDING)
1831		return -EINVAL;
1832
1833	if (irqd_is_forwarded_to_vcpu(d)) {
1834		if (state)
1835			its_send_vint(its_dev, event);
1836		else
1837			its_send_vclear(its_dev, event);
1838	} else {
1839		if (state)
1840			its_send_int(its_dev, event);
1841		else
1842			its_send_clear(its_dev, event);
1843	}
1844
1845	return 0;
1846}
1847
1848static int its_irq_retrigger(struct irq_data *d)
1849{
1850	return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1851}
1852
1853/*
1854 * Two favourable cases:
1855 *
1856 * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1857 *     for vSGI delivery
1858 *
1859 * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1860 *     and we're better off mapping all VPEs always
1861 *
1862 * If neither (a) nor (b) is true, then we map vPEs on demand.
1863 *
1864 */
1865static bool gic_requires_eager_mapping(void)
1866{
1867	if (!its_list_map || gic_rdists->has_rvpeid)
1868		return true;
1869
1870	return false;
1871}
1872
1873static void its_map_vm(struct its_node *its, struct its_vm *vm)
1874{
1875	if (gic_requires_eager_mapping())
1876		return;
1877
1878	guard(raw_spinlock_irqsave)(&vm->vmapp_lock);
1879
1880	/*
1881	 * If the VM wasn't mapped yet, iterate over the vpes and get
1882	 * them mapped now.
1883	 */
1884	vm->vlpi_count[its->list_nr]++;
1885
1886	if (vm->vlpi_count[its->list_nr] == 1) {
1887		int i;
1888
1889		for (i = 0; i < vm->nr_vpes; i++) {
1890			struct its_vpe *vpe = vm->vpes[i];
1891
1892			scoped_guard(raw_spinlock, &vpe->vpe_lock)
1893				its_send_vmapp(its, vpe, true);
1894
1895			its_send_vinvall(its, vpe);
1896		}
1897	}
1898}
1899
1900static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1901{
1902	/* Not using the ITS list? Everything is always mapped. */
1903	if (gic_requires_eager_mapping())
1904		return;
1905
1906	guard(raw_spinlock_irqsave)(&vm->vmapp_lock);
1907
1908	if (!--vm->vlpi_count[its->list_nr]) {
1909		int i;
1910
1911		for (i = 0; i < vm->nr_vpes; i++) {
1912			guard(raw_spinlock)(&vm->vpes[i]->vpe_lock);
1913			its_send_vmapp(its, vm->vpes[i], false);
1914		}
1915	}
1916}
1917
1918static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1919{
1920	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1921	u32 event = its_get_event_id(d);
1922
1923	if (!info->map)
1924		return -EINVAL;
1925
1926	if (!its_dev->event_map.vm) {
1927		struct its_vlpi_map *maps;
1928
1929		maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1930			       GFP_ATOMIC);
1931		if (!maps)
1932			return -ENOMEM;
1933
1934		its_dev->event_map.vm = info->map->vm;
1935		its_dev->event_map.vlpi_maps = maps;
1936	} else if (its_dev->event_map.vm != info->map->vm) {
1937		return -EINVAL;
1938	}
1939
1940	/* Get our private copy of the mapping information */
1941	its_dev->event_map.vlpi_maps[event] = *info->map;
1942
1943	if (irqd_is_forwarded_to_vcpu(d)) {
1944		/* Already mapped, move it around */
1945		its_send_vmovi(its_dev, event);
1946	} else {
1947		/* Ensure all the VPEs are mapped on this ITS */
1948		its_map_vm(its_dev->its, info->map->vm);
1949
1950		/*
1951		 * Flag the interrupt as forwarded so that we can
1952		 * start poking the virtual property table.
1953		 */
1954		irqd_set_forwarded_to_vcpu(d);
1955
1956		/* Write out the property to the prop table */
1957		lpi_write_config(d, 0xff, info->map->properties);
1958
1959		/* Drop the physical mapping */
1960		its_send_discard(its_dev, event);
1961
1962		/* and install the virtual one */
1963		its_send_vmapti(its_dev, event);
1964
1965		/* Increment the number of VLPIs */
1966		its_dev->event_map.nr_vlpis++;
1967	}
1968
1969	return 0;
1970}
1971
1972static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1973{
1974	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1975	struct its_vlpi_map *map;
1976
1977	map = get_vlpi_map(d);
1978
1979	if (!its_dev->event_map.vm || !map)
1980		return -EINVAL;
1981
1982	/* Copy our mapping information to the incoming request */
1983	*info->map = *map;
1984
1985	return 0;
1986}
1987
1988static int its_vlpi_unmap(struct irq_data *d)
1989{
1990	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1991	u32 event = its_get_event_id(d);
1992
1993	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1994		return -EINVAL;
1995
1996	/* Drop the virtual mapping */
1997	its_send_discard(its_dev, event);
1998
1999	/* and restore the physical one */
2000	irqd_clr_forwarded_to_vcpu(d);
2001	its_send_mapti(its_dev, d->hwirq, event);
2002	lpi_update_config(d, 0xff, (lpi_prop_prio |
2003				    LPI_PROP_ENABLED |
2004				    LPI_PROP_GROUP1));
2005
2006	/* Potentially unmap the VM from this ITS */
2007	its_unmap_vm(its_dev->its, its_dev->event_map.vm);
2008
2009	/*
2010	 * Drop the refcount and make the device available again if
2011	 * this was the last VLPI.
2012	 */
2013	if (!--its_dev->event_map.nr_vlpis) {
2014		its_dev->event_map.vm = NULL;
2015		kfree(its_dev->event_map.vlpi_maps);
2016	}
2017
2018	return 0;
2019}
2020
2021static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
2022{
2023	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2024
2025	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
2026		return -EINVAL;
2027
2028	if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
2029		lpi_update_config(d, 0xff, info->config);
2030	else
2031		lpi_write_config(d, 0xff, info->config);
2032	its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
2033
2034	return 0;
2035}
2036
2037static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
2038{
2039	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2040	struct its_cmd_info *info = vcpu_info;
2041
2042	/* Need a v4 ITS */
2043	if (!is_v4(its_dev->its))
2044		return -EINVAL;
2045
2046	guard(raw_spinlock)(&its_dev->event_map.vlpi_lock);
2047
2048	/* Unmap request? */
2049	if (!info)
2050		return its_vlpi_unmap(d);
2051
2052	switch (info->cmd_type) {
2053	case MAP_VLPI:
2054		return its_vlpi_map(d, info);
2055
2056	case GET_VLPI:
2057		return its_vlpi_get(d, info);
2058
2059	case PROP_UPDATE_VLPI:
2060	case PROP_UPDATE_AND_INV_VLPI:
2061		return its_vlpi_prop_update(d, info);
2062
2063	default:
2064		return -EINVAL;
2065	}
2066}
2067
2068static struct irq_chip its_irq_chip = {
2069	.name			= "ITS",
2070	.irq_mask		= its_mask_irq,
2071	.irq_unmask		= its_unmask_irq,
2072	.irq_eoi		= irq_chip_eoi_parent,
2073	.irq_set_affinity	= its_set_affinity,
2074	.irq_compose_msi_msg	= its_irq_compose_msi_msg,
2075	.irq_set_irqchip_state	= its_irq_set_irqchip_state,
2076	.irq_retrigger		= its_irq_retrigger,
2077	.irq_set_vcpu_affinity	= its_irq_set_vcpu_affinity,
2078};
2079
2080
2081/*
2082 * How we allocate LPIs:
2083 *
2084 * lpi_range_list contains ranges of LPIs that are to available to
2085 * allocate from. To allocate LPIs, just pick the first range that
2086 * fits the required allocation, and reduce it by the required
2087 * amount. Once empty, remove the range from the list.
2088 *
2089 * To free a range of LPIs, add a free range to the list, sort it and
2090 * merge the result if the new range happens to be adjacent to an
2091 * already free block.
2092 *
2093 * The consequence of the above is that allocation is cost is low, but
2094 * freeing is expensive. We assumes that freeing rarely occurs.
2095 */
2096#define ITS_MAX_LPI_NRBITS	16 /* 64K LPIs */
2097
2098static DEFINE_MUTEX(lpi_range_lock);
2099static LIST_HEAD(lpi_range_list);
2100
2101struct lpi_range {
2102	struct list_head	entry;
2103	u32			base_id;
2104	u32			span;
2105};
2106
2107static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2108{
2109	struct lpi_range *range;
2110
2111	range = kmalloc(sizeof(*range), GFP_KERNEL);
2112	if (range) {
2113		range->base_id = base;
2114		range->span = span;
2115	}
2116
2117	return range;
2118}
2119
2120static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2121{
2122	struct lpi_range *range, *tmp;
2123	int err = -ENOSPC;
2124
2125	mutex_lock(&lpi_range_lock);
2126
2127	list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2128		if (range->span >= nr_lpis) {
2129			*base = range->base_id;
2130			range->base_id += nr_lpis;
2131			range->span -= nr_lpis;
2132
2133			if (range->span == 0) {
2134				list_del(&range->entry);
2135				kfree(range);
2136			}
2137
2138			err = 0;
2139			break;
2140		}
2141	}
2142
2143	mutex_unlock(&lpi_range_lock);
2144
2145	pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2146	return err;
2147}
2148
2149static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2150{
2151	if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2152		return;
2153	if (a->base_id + a->span != b->base_id)
2154		return;
2155	b->base_id = a->base_id;
2156	b->span += a->span;
2157	list_del(&a->entry);
2158	kfree(a);
2159}
2160
2161static int free_lpi_range(u32 base, u32 nr_lpis)
2162{
2163	struct lpi_range *new, *old;
2164
2165	new = mk_lpi_range(base, nr_lpis);
2166	if (!new)
2167		return -ENOMEM;
2168
2169	mutex_lock(&lpi_range_lock);
2170
2171	list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2172		if (old->base_id < base)
2173			break;
2174	}
2175	/*
2176	 * old is the last element with ->base_id smaller than base,
2177	 * so new goes right after it. If there are no elements with
2178	 * ->base_id smaller than base, &old->entry ends up pointing
2179	 * at the head of the list, and inserting new it the start of
2180	 * the list is the right thing to do in that case as well.
2181	 */
2182	list_add(&new->entry, &old->entry);
2183	/*
2184	 * Now check if we can merge with the preceding and/or
2185	 * following ranges.
2186	 */
2187	merge_lpi_ranges(old, new);
2188	merge_lpi_ranges(new, list_next_entry(new, entry));
2189
2190	mutex_unlock(&lpi_range_lock);
2191	return 0;
2192}
2193
2194static int __init its_lpi_init(u32 id_bits)
2195{
2196	u32 lpis = (1UL << id_bits) - 8192;
2197	u32 numlpis;
2198	int err;
2199
2200	numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2201
2202	if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2203		lpis = numlpis;
2204		pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2205			lpis);
2206	}
2207
2208	/*
2209	 * Initializing the allocator is just the same as freeing the
2210	 * full range of LPIs.
2211	 */
2212	err = free_lpi_range(8192, lpis);
2213	pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2214	return err;
2215}
2216
2217static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2218{
2219	unsigned long *bitmap = NULL;
2220	int err = 0;
2221
2222	do {
2223		err = alloc_lpi_range(nr_irqs, base);
2224		if (!err)
2225			break;
2226
2227		nr_irqs /= 2;
2228	} while (nr_irqs > 0);
2229
2230	if (!nr_irqs)
2231		err = -ENOSPC;
2232
2233	if (err)
2234		goto out;
2235
2236	bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2237	if (!bitmap)
2238		goto out;
2239
2240	*nr_ids = nr_irqs;
2241
2242out:
2243	if (!bitmap)
2244		*base = *nr_ids = 0;
2245
2246	return bitmap;
2247}
2248
2249static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2250{
2251	WARN_ON(free_lpi_range(base, nr_ids));
2252	bitmap_free(bitmap);
2253}
2254
2255static void gic_reset_prop_table(void *va)
2256{
2257	/* Regular IRQ priority, Group-1, disabled */
2258	memset(va, lpi_prop_prio | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2259
2260	/* Make sure the GIC will observe the written configuration */
2261	gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2262}
2263
2264static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2265{
2266	struct page *prop_page;
2267
2268	prop_page = its_alloc_pages(gfp_flags,
2269				    get_order(LPI_PROPBASE_SZ));
2270	if (!prop_page)
2271		return NULL;
2272
2273	gic_reset_prop_table(page_address(prop_page));
2274
2275	return prop_page;
2276}
2277
2278static void its_free_prop_table(struct page *prop_page)
2279{
2280	its_free_pages(page_address(prop_page), get_order(LPI_PROPBASE_SZ));
2281}
2282
2283static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2284{
2285	phys_addr_t start, end, addr_end;
2286	u64 i;
2287
2288	/*
2289	 * We don't bother checking for a kdump kernel as by
2290	 * construction, the LPI tables are out of this kernel's
2291	 * memory map.
2292	 */
2293	if (is_kdump_kernel())
2294		return true;
2295
2296	addr_end = addr + size - 1;
2297
2298	for_each_reserved_mem_range(i, &start, &end) {
2299		if (addr >= start && addr_end <= end)
2300			return true;
2301	}
2302
2303	/* Not found, not a good sign... */
2304	pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2305		&addr, &addr_end);
2306	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2307	return false;
2308}
2309
2310static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2311{
2312	if (efi_enabled(EFI_CONFIG_TABLES))
2313		return efi_mem_reserve_persistent(addr, size);
2314
2315	return 0;
2316}
2317
2318static int __init its_setup_lpi_prop_table(void)
2319{
2320	if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2321		u64 val;
2322
2323		val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2324		lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2325
2326		gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2327		gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2328						     LPI_PROPBASE_SZ,
2329						     MEMREMAP_WB);
2330		gic_reset_prop_table(gic_rdists->prop_table_va);
2331	} else {
2332		struct page *page;
2333
2334		lpi_id_bits = min_t(u32,
2335				    GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2336				    ITS_MAX_LPI_NRBITS);
2337		page = its_allocate_prop_table(GFP_NOWAIT);
2338		if (!page) {
2339			pr_err("Failed to allocate PROPBASE\n");
2340			return -ENOMEM;
2341		}
2342
2343		gic_rdists->prop_table_pa = page_to_phys(page);
2344		gic_rdists->prop_table_va = page_address(page);
2345		WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2346					  LPI_PROPBASE_SZ));
2347	}
2348
2349	pr_info("GICv3: using LPI property table @%pa\n",
2350		&gic_rdists->prop_table_pa);
2351
2352	return its_lpi_init(lpi_id_bits);
2353}
2354
2355static const char *its_base_type_string[] = {
2356	[GITS_BASER_TYPE_DEVICE]	= "Devices",
2357	[GITS_BASER_TYPE_VCPU]		= "Virtual CPUs",
2358	[GITS_BASER_TYPE_RESERVED3]	= "Reserved (3)",
2359	[GITS_BASER_TYPE_COLLECTION]	= "Interrupt Collections",
2360	[GITS_BASER_TYPE_RESERVED5] 	= "Reserved (5)",
2361	[GITS_BASER_TYPE_RESERVED6] 	= "Reserved (6)",
2362	[GITS_BASER_TYPE_RESERVED7] 	= "Reserved (7)",
2363};
2364
2365static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2366{
2367	u32 idx = baser - its->tables;
2368
2369	return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2370}
2371
2372static void its_write_baser(struct its_node *its, struct its_baser *baser,
2373			    u64 val)
2374{
2375	u32 idx = baser - its->tables;
2376
2377	gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2378	baser->val = its_read_baser(its, baser);
2379}
2380
2381static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2382			   u64 cache, u64 shr, u32 order, bool indirect)
2383{
2384	u64 val = its_read_baser(its, baser);
2385	u64 esz = GITS_BASER_ENTRY_SIZE(val);
2386	u64 type = GITS_BASER_TYPE(val);
2387	u64 baser_phys, tmp;
2388	u32 alloc_pages, psz;
2389	struct page *page;
2390	void *base;
2391
2392	psz = baser->psz;
2393	alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2394	if (alloc_pages > GITS_BASER_PAGES_MAX) {
2395		pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2396			&its->phys_base, its_base_type_string[type],
2397			alloc_pages, GITS_BASER_PAGES_MAX);
2398		alloc_pages = GITS_BASER_PAGES_MAX;
2399		order = get_order(GITS_BASER_PAGES_MAX * psz);
2400	}
2401
2402	page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2403	if (!page)
2404		return -ENOMEM;
2405
2406	base = (void *)page_address(page);
2407	baser_phys = virt_to_phys(base);
2408
2409	/* Check if the physical address of the memory is above 48bits */
2410	if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2411
2412		/* 52bit PA is supported only when PageSize=64K */
2413		if (psz != SZ_64K) {
2414			pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2415			its_free_pages(base, order);
2416			return -ENXIO;
2417		}
2418
2419		/* Convert 52bit PA to 48bit field */
2420		baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2421	}
2422
2423retry_baser:
2424	val = (baser_phys					 |
2425		(type << GITS_BASER_TYPE_SHIFT)			 |
2426		((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)	 |
2427		((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)	 |
2428		cache						 |
2429		shr						 |
2430		GITS_BASER_VALID);
2431
2432	val |=	indirect ? GITS_BASER_INDIRECT : 0x0;
2433
2434	switch (psz) {
2435	case SZ_4K:
2436		val |= GITS_BASER_PAGE_SIZE_4K;
2437		break;
2438	case SZ_16K:
2439		val |= GITS_BASER_PAGE_SIZE_16K;
2440		break;
2441	case SZ_64K:
2442		val |= GITS_BASER_PAGE_SIZE_64K;
2443		break;
2444	}
2445
2446	if (!shr)
2447		gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2448
2449	its_write_baser(its, baser, val);
2450	tmp = baser->val;
2451
2452	if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2453		/*
2454		 * Shareability didn't stick. Just use
2455		 * whatever the read reported, which is likely
2456		 * to be the only thing this redistributor
2457		 * supports. If that's zero, make it
2458		 * non-cacheable as well.
2459		 */
2460		shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2461		if (!shr)
2462			cache = GITS_BASER_nC;
2463
2464		goto retry_baser;
2465	}
2466
2467	if (val != tmp) {
2468		pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2469		       &its->phys_base, its_base_type_string[type],
2470		       val, tmp);
2471		its_free_pages(base, order);
2472		return -ENXIO;
2473	}
2474
2475	baser->order = order;
2476	baser->base = base;
2477	baser->psz = psz;
2478	tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2479
2480	pr_info("ITS@%pa: allocated %d %s @%llx (%s, esz %d, psz %dK, shr %d)\n",
2481		&its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2482		its_base_type_string[type],
2483		(u64)virt_to_phys(base),
2484		indirect ? "indirect" : "flat", (int)esz,
2485		psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2486
2487	return 0;
2488}
2489
2490static bool its_parse_indirect_baser(struct its_node *its,
2491				     struct its_baser *baser,
2492				     u32 *order, u32 ids)
2493{
2494	u64 tmp = its_read_baser(its, baser);
2495	u64 type = GITS_BASER_TYPE(tmp);
2496	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2497	u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2498	u32 new_order = *order;
2499	u32 psz = baser->psz;
2500	bool indirect = false;
2501
2502	/* No need to enable Indirection if memory requirement < (psz*2)bytes */
2503	if ((esz << ids) > (psz * 2)) {
2504		/*
2505		 * Find out whether hw supports a single or two-level table by
2506		 * table by reading bit at offset '62' after writing '1' to it.
2507		 */
2508		its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2509		indirect = !!(baser->val & GITS_BASER_INDIRECT);
2510
2511		if (indirect) {
2512			/*
2513			 * The size of the lvl2 table is equal to ITS page size
2514			 * which is 'psz'. For computing lvl1 table size,
2515			 * subtract ID bits that sparse lvl2 table from 'ids'
2516			 * which is reported by ITS hardware times lvl1 table
2517			 * entry size.
2518			 */
2519			ids -= ilog2(psz / (int)esz);
2520			esz = GITS_LVL1_ENTRY_SIZE;
2521		}
2522	}
2523
2524	/*
2525	 * Allocate as many entries as required to fit the
2526	 * range of device IDs that the ITS can grok... The ID
2527	 * space being incredibly sparse, this results in a
2528	 * massive waste of memory if two-level device table
2529	 * feature is not supported by hardware.
2530	 */
2531	new_order = max_t(u32, get_order(esz << ids), new_order);
2532	if (new_order > MAX_PAGE_ORDER) {
2533		new_order = MAX_PAGE_ORDER;
2534		ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2535		pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2536			&its->phys_base, its_base_type_string[type],
2537			device_ids(its), ids);
2538	}
2539
2540	*order = new_order;
2541
2542	return indirect;
2543}
2544
2545static u32 compute_common_aff(u64 val)
2546{
2547	u32 aff, clpiaff;
2548
2549	aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2550	clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2551
2552	return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2553}
2554
2555static u32 compute_its_aff(struct its_node *its)
2556{
2557	u64 val;
2558	u32 svpet;
2559
2560	/*
2561	 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2562	 * the resulting affinity. We then use that to see if this match
2563	 * our own affinity.
2564	 */
2565	svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2566	val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2567	val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2568	return compute_common_aff(val);
2569}
2570
2571static struct its_node *find_sibling_its(struct its_node *cur_its)
2572{
2573	struct its_node *its;
2574	u32 aff;
2575
2576	if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2577		return NULL;
2578
2579	aff = compute_its_aff(cur_its);
2580
2581	list_for_each_entry(its, &its_nodes, entry) {
2582		u64 baser;
2583
2584		if (!is_v4_1(its) || its == cur_its)
2585			continue;
2586
2587		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2588			continue;
2589
2590		if (aff != compute_its_aff(its))
2591			continue;
2592
2593		/* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2594		baser = its->tables[2].val;
2595		if (!(baser & GITS_BASER_VALID))
2596			continue;
2597
2598		return its;
2599	}
2600
2601	return NULL;
2602}
2603
2604static void its_free_tables(struct its_node *its)
2605{
2606	int i;
2607
2608	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2609		if (its->tables[i].base) {
2610			its_free_pages(its->tables[i].base, its->tables[i].order);
2611			its->tables[i].base = NULL;
2612		}
2613	}
2614}
2615
2616static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2617{
2618	u64 psz = SZ_64K;
2619
2620	while (psz) {
2621		u64 val, gpsz;
2622
2623		val = its_read_baser(its, baser);
2624		val &= ~GITS_BASER_PAGE_SIZE_MASK;
2625
2626		switch (psz) {
2627		case SZ_64K:
2628			gpsz = GITS_BASER_PAGE_SIZE_64K;
2629			break;
2630		case SZ_16K:
2631			gpsz = GITS_BASER_PAGE_SIZE_16K;
2632			break;
2633		case SZ_4K:
2634		default:
2635			gpsz = GITS_BASER_PAGE_SIZE_4K;
2636			break;
2637		}
2638
2639		gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2640
2641		val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2642		its_write_baser(its, baser, val);
2643
2644		if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2645			break;
2646
2647		switch (psz) {
2648		case SZ_64K:
2649			psz = SZ_16K;
2650			break;
2651		case SZ_16K:
2652			psz = SZ_4K;
2653			break;
2654		case SZ_4K:
2655		default:
2656			return -1;
2657		}
2658	}
2659
2660	baser->psz = psz;
2661	return 0;
2662}
2663
2664static int its_alloc_tables(struct its_node *its)
2665{
2666	u64 shr = GITS_BASER_InnerShareable;
2667	u64 cache = GITS_BASER_RaWaWb;
2668	int err, i;
2669
2670	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2671		/* erratum 24313: ignore memory access type */
2672		cache = GITS_BASER_nCnB;
2673
2674	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {
2675		cache = GITS_BASER_nC;
2676		shr = 0;
2677	}
2678
2679	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2680		struct its_baser *baser = its->tables + i;
2681		u64 val = its_read_baser(its, baser);
2682		u64 type = GITS_BASER_TYPE(val);
2683		bool indirect = false;
2684		u32 order;
2685
2686		if (type == GITS_BASER_TYPE_NONE)
2687			continue;
2688
2689		if (its_probe_baser_psz(its, baser)) {
2690			its_free_tables(its);
2691			return -ENXIO;
2692		}
2693
2694		order = get_order(baser->psz);
2695
2696		switch (type) {
2697		case GITS_BASER_TYPE_DEVICE:
2698			indirect = its_parse_indirect_baser(its, baser, &order,
2699							    device_ids(its));
2700			break;
2701
2702		case GITS_BASER_TYPE_VCPU:
2703			if (is_v4_1(its)) {
2704				struct its_node *sibling;
2705
2706				WARN_ON(i != 2);
2707				if ((sibling = find_sibling_its(its))) {
2708					*baser = sibling->tables[2];
2709					its_write_baser(its, baser, baser->val);
2710					continue;
2711				}
2712			}
2713
2714			indirect = its_parse_indirect_baser(its, baser, &order,
2715							    ITS_MAX_VPEID_BITS);
2716			break;
2717		}
2718
2719		err = its_setup_baser(its, baser, cache, shr, order, indirect);
2720		if (err < 0) {
2721			its_free_tables(its);
2722			return err;
2723		}
2724
2725		/* Update settings which will be used for next BASERn */
2726		cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2727		shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2728	}
2729
2730	return 0;
2731}
2732
2733static u64 inherit_vpe_l1_table_from_its(void)
2734{
2735	struct its_node *its;
2736	u64 val;
2737	u32 aff;
2738
2739	val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2740	aff = compute_common_aff(val);
2741
2742	list_for_each_entry(its, &its_nodes, entry) {
2743		u64 baser, addr;
2744
2745		if (!is_v4_1(its))
2746			continue;
2747
2748		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2749			continue;
2750
2751		if (aff != compute_its_aff(its))
2752			continue;
2753
2754		/* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2755		baser = its->tables[2].val;
2756		if (!(baser & GITS_BASER_VALID))
2757			continue;
2758
2759		/* We have a winner! */
2760		gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2761
2762		val  = GICR_VPROPBASER_4_1_VALID;
2763		if (baser & GITS_BASER_INDIRECT)
2764			val |= GICR_VPROPBASER_4_1_INDIRECT;
2765		val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2766				  FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2767		switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2768		case GIC_PAGE_SIZE_64K:
2769			addr = GITS_BASER_ADDR_48_to_52(baser);
2770			break;
2771		default:
2772			addr = baser & GENMASK_ULL(47, 12);
2773			break;
2774		}
2775		val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2776		if (rdists_support_shareable()) {
2777			val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2778					  FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2779			val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2780					  FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2781		}
2782		val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2783
2784		*this_cpu_ptr(&local_4_1_its) = its;
2785		return val;
2786	}
2787
2788	return 0;
2789}
2790
2791static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2792{
2793	u32 aff;
2794	u64 val;
2795	int cpu;
2796
2797	val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2798	aff = compute_common_aff(val);
2799
2800	for_each_possible_cpu(cpu) {
2801		void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2802
2803		if (!base || cpu == smp_processor_id())
2804			continue;
2805
2806		val = gic_read_typer(base + GICR_TYPER);
2807		if (aff != compute_common_aff(val))
2808			continue;
2809
2810		/*
2811		 * At this point, we have a victim. This particular CPU
2812		 * has already booted, and has an affinity that matches
2813		 * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2814		 * Make sure we don't write the Z bit in that case.
2815		 */
2816		val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2817		val &= ~GICR_VPROPBASER_4_1_Z;
2818
2819		gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2820		*mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2821
2822		*this_cpu_ptr(&local_4_1_its) = *per_cpu_ptr(&local_4_1_its, cpu);
2823		return val;
2824	}
2825
2826	return 0;
2827}
2828
2829static bool allocate_vpe_l2_table(int cpu, u32 id)
2830{
2831	void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2832	unsigned int psz, esz, idx, npg, gpsz;
2833	u64 val;
2834	struct page *page;
2835	__le64 *table;
2836
2837	if (!gic_rdists->has_rvpeid)
2838		return true;
2839
2840	/* Skip non-present CPUs */
2841	if (!base)
2842		return true;
2843
2844	val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2845
2846	esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2847	gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2848	npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2849
2850	switch (gpsz) {
2851	default:
2852		WARN_ON(1);
2853		fallthrough;
2854	case GIC_PAGE_SIZE_4K:
2855		psz = SZ_4K;
2856		break;
2857	case GIC_PAGE_SIZE_16K:
2858		psz = SZ_16K;
2859		break;
2860	case GIC_PAGE_SIZE_64K:
2861		psz = SZ_64K;
2862		break;
2863	}
2864
2865	/* Don't allow vpe_id that exceeds single, flat table limit */
2866	if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2867		return (id < (npg * psz / (esz * SZ_8)));
2868
2869	/* Compute 1st level table index & check if that exceeds table limit */
2870	idx = id >> ilog2(psz / (esz * SZ_8));
2871	if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2872		return false;
2873
2874	table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2875
2876	/* Allocate memory for 2nd level table */
2877	if (!table[idx]) {
2878		page = its_alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2879		if (!page)
2880			return false;
2881
2882		/* Flush Lvl2 table to PoC if hw doesn't support coherency */
2883		if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2884			gic_flush_dcache_to_poc(page_address(page), psz);
2885
2886		table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2887
2888		/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2889		if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2890			gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2891
2892		/* Ensure updated table contents are visible to RD hardware */
2893		dsb(sy);
2894	}
2895
2896	return true;
2897}
2898
2899static int allocate_vpe_l1_table(void)
2900{
2901	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2902	u64 val, gpsz, npg, pa;
2903	unsigned int psz = SZ_64K;
2904	unsigned int np, epp, esz;
2905	struct page *page;
2906
2907	if (!gic_rdists->has_rvpeid)
2908		return 0;
2909
2910	/*
2911	 * if VPENDBASER.Valid is set, disable any previously programmed
2912	 * VPE by setting PendingLast while clearing Valid. This has the
2913	 * effect of making sure no doorbell will be generated and we can
2914	 * then safely clear VPROPBASER.Valid.
2915	 */
2916	if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2917		gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2918				      vlpi_base + GICR_VPENDBASER);
2919
2920	/*
2921	 * If we can inherit the configuration from another RD, let's do
2922	 * so. Otherwise, we have to go through the allocation process. We
2923	 * assume that all RDs have the exact same requirements, as
2924	 * nothing will work otherwise.
2925	 */
2926	val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2927	if (val & GICR_VPROPBASER_4_1_VALID)
2928		goto out;
2929
2930	gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2931	if (!gic_data_rdist()->vpe_table_mask)
2932		return -ENOMEM;
2933
2934	val = inherit_vpe_l1_table_from_its();
2935	if (val & GICR_VPROPBASER_4_1_VALID)
2936		goto out;
2937
2938	/* First probe the page size */
2939	val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2940	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2941	val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2942	gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2943	esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2944
2945	switch (gpsz) {
2946	default:
2947		gpsz = GIC_PAGE_SIZE_4K;
2948		fallthrough;
2949	case GIC_PAGE_SIZE_4K:
2950		psz = SZ_4K;
2951		break;
2952	case GIC_PAGE_SIZE_16K:
2953		psz = SZ_16K;
2954		break;
2955	case GIC_PAGE_SIZE_64K:
2956		psz = SZ_64K;
2957		break;
2958	}
2959
2960	/*
2961	 * Start populating the register from scratch, including RO fields
2962	 * (which we want to print in debug cases...)
2963	 */
2964	val = 0;
2965	val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2966	val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2967
2968	/* How many entries per GIC page? */
2969	esz++;
2970	epp = psz / (esz * SZ_8);
2971
2972	/*
2973	 * If we need more than just a single L1 page, flag the table
2974	 * as indirect and compute the number of required L1 pages.
2975	 */
2976	if (epp < ITS_MAX_VPEID) {
2977		int nl2;
2978
2979		val |= GICR_VPROPBASER_4_1_INDIRECT;
2980
2981		/* Number of L2 pages required to cover the VPEID space */
2982		nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2983
2984		/* Number of L1 pages to point to the L2 pages */
2985		npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2986	} else {
2987		npg = 1;
2988	}
2989
2990	val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2991
2992	/* Right, that's the number of CPU pages we need for L1 */
2993	np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2994
2995	pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2996		 np, npg, psz, epp, esz);
2997	page = its_alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2998	if (!page)
2999		return -ENOMEM;
3000
3001	gic_data_rdist()->vpe_l1_base = page_address(page);
3002	pa = virt_to_phys(page_address(page));
3003	WARN_ON(!IS_ALIGNED(pa, psz));
3004
3005	val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
3006	if (rdists_support_shareable()) {
3007		val |= GICR_VPROPBASER_RaWb;
3008		val |= GICR_VPROPBASER_InnerShareable;
3009	}
3010	val |= GICR_VPROPBASER_4_1_Z;
3011	val |= GICR_VPROPBASER_4_1_VALID;
3012
3013out:
3014	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3015	cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
3016
3017	pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
3018		 smp_processor_id(), val,
3019		 cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
3020
3021	return 0;
3022}
3023
3024static int its_alloc_collections(struct its_node *its)
3025{
3026	int i;
3027
3028	its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
3029				   GFP_KERNEL);
3030	if (!its->collections)
3031		return -ENOMEM;
3032
3033	for (i = 0; i < nr_cpu_ids; i++)
3034		its->collections[i].target_address = ~0ULL;
3035
3036	return 0;
3037}
3038
3039static struct page *its_allocate_pending_table(gfp_t gfp_flags)
3040{
3041	struct page *pend_page;
3042
3043	pend_page = its_alloc_pages(gfp_flags | __GFP_ZERO, get_order(LPI_PENDBASE_SZ));
3044	if (!pend_page)
3045		return NULL;
3046
3047	/* Make sure the GIC will observe the zero-ed page */
3048	gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
3049
3050	return pend_page;
3051}
3052
3053static void its_free_pending_table(struct page *pt)
3054{
3055	its_free_pages(page_address(pt), get_order(LPI_PENDBASE_SZ));
3056}
3057
3058/*
3059 * Booting with kdump and LPIs enabled is generally fine. Any other
3060 * case is wrong in the absence of firmware/EFI support.
3061 */
3062static bool enabled_lpis_allowed(void)
3063{
3064	phys_addr_t addr;
3065	u64 val;
3066
3067	/* Check whether the property table is in a reserved region */
3068	val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
3069	addr = val & GENMASK_ULL(51, 12);
3070
3071	return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
3072}
3073
3074static int __init allocate_lpi_tables(void)
3075{
3076	u64 val;
3077	int err, cpu;
3078
3079	/*
3080	 * If LPIs are enabled while we run this from the boot CPU,
3081	 * flag the RD tables as pre-allocated if the stars do align.
3082	 */
3083	val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3084	if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3085		gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3086				      RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3087		pr_info("GICv3: Using preallocated redistributor tables\n");
3088	}
3089
3090	err = its_setup_lpi_prop_table();
3091	if (err)
3092		return err;
3093
3094	/*
3095	 * We allocate all the pending tables anyway, as we may have a
3096	 * mix of RDs that have had LPIs enabled, and some that
3097	 * don't. We'll free the unused ones as each CPU comes online.
3098	 */
3099	for_each_possible_cpu(cpu) {
3100		struct page *pend_page;
3101
3102		pend_page = its_allocate_pending_table(GFP_NOWAIT);
3103		if (!pend_page) {
3104			pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3105			return -ENOMEM;
3106		}
3107
3108		gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3109	}
3110
3111	return 0;
3112}
3113
3114static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
3115{
3116	u32 count = 1000000;	/* 1s! */
3117	bool clean;
3118	u64 val;
3119
3120	do {
3121		val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3122		clean = !(val & GICR_VPENDBASER_Dirty);
3123		if (!clean) {
3124			count--;
3125			cpu_relax();
3126			udelay(1);
3127		}
3128	} while (!clean && count);
3129
3130	if (unlikely(!clean))
3131		pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3132
3133	return val;
3134}
3135
3136static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3137{
3138	u64 val;
3139
3140	/* Make sure we wait until the RD is done with the initial scan */
3141	val = read_vpend_dirty_clear(vlpi_base);
3142	val &= ~GICR_VPENDBASER_Valid;
3143	val &= ~clr;
3144	val |= set;
3145	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3146
3147	val = read_vpend_dirty_clear(vlpi_base);
3148	if (unlikely(val & GICR_VPENDBASER_Dirty))
3149		val |= GICR_VPENDBASER_PendingLast;
3150
3151	return val;
3152}
3153
3154static void its_cpu_init_lpis(void)
3155{
3156	void __iomem *rbase = gic_data_rdist_rd_base();
3157	struct page *pend_page;
3158	phys_addr_t paddr;
3159	u64 val, tmp;
3160
3161	if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
3162		return;
3163
3164	val = readl_relaxed(rbase + GICR_CTLR);
3165	if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3166	    (val & GICR_CTLR_ENABLE_LPIS)) {
3167		/*
3168		 * Check that we get the same property table on all
3169		 * RDs. If we don't, this is hopeless.
3170		 */
3171		paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3172		paddr &= GENMASK_ULL(51, 12);
3173		if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3174			add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3175
3176		paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3177		paddr &= GENMASK_ULL(51, 16);
3178
3179		WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3180		gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
3181
3182		goto out;
3183	}
3184
3185	pend_page = gic_data_rdist()->pend_page;
3186	paddr = page_to_phys(pend_page);
3187
3188	/* set PROPBASE */
3189	val = (gic_rdists->prop_table_pa |
3190	       GICR_PROPBASER_InnerShareable |
3191	       GICR_PROPBASER_RaWaWb |
3192	       ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3193
3194	gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3195	tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3196
3197	if (!rdists_support_shareable())
3198		tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3199
3200	if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3201		if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3202			/*
3203			 * The HW reports non-shareable, we must
3204			 * remove the cacheability attributes as
3205			 * well.
3206			 */
3207			val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3208				 GICR_PROPBASER_CACHEABILITY_MASK);
3209			val |= GICR_PROPBASER_nC;
3210			gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3211		}
3212		pr_info_once("GIC: using cache flushing for LPI property table\n");
3213		gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3214	}
3215
3216	/* set PENDBASE */
3217	val = (page_to_phys(pend_page) |
3218	       GICR_PENDBASER_InnerShareable |
3219	       GICR_PENDBASER_RaWaWb);
3220
3221	gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3222	tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3223
3224	if (!rdists_support_shareable())
3225		tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3226
3227	if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3228		/*
3229		 * The HW reports non-shareable, we must remove the
3230		 * cacheability attributes as well.
3231		 */
3232		val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3233			 GICR_PENDBASER_CACHEABILITY_MASK);
3234		val |= GICR_PENDBASER_nC;
3235		gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3236	}
3237
3238	/* Enable LPIs */
3239	val = readl_relaxed(rbase + GICR_CTLR);
3240	val |= GICR_CTLR_ENABLE_LPIS;
3241	writel_relaxed(val, rbase + GICR_CTLR);
3242
3243out:
3244	if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3245		void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3246
3247		/*
3248		 * It's possible for CPU to receive VLPIs before it is
3249		 * scheduled as a vPE, especially for the first CPU, and the
3250		 * VLPI with INTID larger than 2^(IDbits+1) will be considered
3251		 * as out of range and dropped by GIC.
3252		 * So we initialize IDbits to known value to avoid VLPI drop.
3253		 */
3254		val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3255		pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3256			smp_processor_id(), val);
3257		gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3258
3259		/*
3260		 * Also clear Valid bit of GICR_VPENDBASER, in case some
3261		 * ancient programming gets left in and has possibility of
3262		 * corrupting memory.
3263		 */
3264		val = its_clear_vpend_valid(vlpi_base, 0, 0);
3265	}
3266
3267	if (allocate_vpe_l1_table()) {
3268		/*
3269		 * If the allocation has failed, we're in massive trouble.
3270		 * Disable direct injection, and pray that no VM was
3271		 * already running...
3272		 */
3273		gic_rdists->has_rvpeid = false;
3274		gic_rdists->has_vlpis = false;
3275	}
3276
3277	/* Make sure the GIC has seen the above */
3278	dsb(sy);
3279	gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
3280	pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3281		smp_processor_id(),
3282		gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3283		"reserved" : "allocated",
3284		&paddr);
3285}
3286
3287static void its_cpu_init_collection(struct its_node *its)
3288{
3289	int cpu = smp_processor_id();
3290	u64 target;
3291
3292	/* avoid cross node collections and its mapping */
3293	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3294		struct device_node *cpu_node;
3295
3296		cpu_node = of_get_cpu_node(cpu, NULL);
3297		if (its->numa_node != NUMA_NO_NODE &&
3298			its->numa_node != of_node_to_nid(cpu_node))
3299			return;
3300	}
3301
3302	/*
3303	 * We now have to bind each collection to its target
3304	 * redistributor.
3305	 */
3306	if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3307		/*
3308		 * This ITS wants the physical address of the
3309		 * redistributor.
3310		 */
3311		target = gic_data_rdist()->phys_base;
3312	} else {
3313		/* This ITS wants a linear CPU number. */
3314		target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3315		target = GICR_TYPER_CPU_NUMBER(target) << 16;
3316	}
3317
3318	/* Perform collection mapping */
3319	its->collections[cpu].target_address = target;
3320	its->collections[cpu].col_id = cpu;
3321
3322	its_send_mapc(its, &its->collections[cpu], 1);
3323	its_send_invall(its, &its->collections[cpu]);
3324}
3325
3326static void its_cpu_init_collections(void)
3327{
3328	struct its_node *its;
3329
3330	raw_spin_lock(&its_lock);
3331
3332	list_for_each_entry(its, &its_nodes, entry)
3333		its_cpu_init_collection(its);
3334
3335	raw_spin_unlock(&its_lock);
3336}
3337
3338static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3339{
3340	struct its_device *its_dev = NULL, *tmp;
3341	unsigned long flags;
3342
3343	raw_spin_lock_irqsave(&its->lock, flags);
3344
3345	list_for_each_entry(tmp, &its->its_device_list, entry) {
3346		if (tmp->device_id == dev_id) {
3347			its_dev = tmp;
3348			break;
3349		}
3350	}
3351
3352	raw_spin_unlock_irqrestore(&its->lock, flags);
3353
3354	return its_dev;
3355}
3356
3357static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3358{
3359	int i;
3360
3361	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3362		if (GITS_BASER_TYPE(its->tables[i].val) == type)
3363			return &its->tables[i];
3364	}
3365
3366	return NULL;
3367}
3368
3369static bool its_alloc_table_entry(struct its_node *its,
3370				  struct its_baser *baser, u32 id)
3371{
3372	struct page *page;
3373	u32 esz, idx;
3374	__le64 *table;
3375
3376	/* Don't allow device id that exceeds single, flat table limit */
3377	esz = GITS_BASER_ENTRY_SIZE(baser->val);
3378	if (!(baser->val & GITS_BASER_INDIRECT))
3379		return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3380
3381	/* Compute 1st level table index & check if that exceeds table limit */
3382	idx = id >> ilog2(baser->psz / esz);
3383	if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3384		return false;
3385
3386	table = baser->base;
3387
3388	/* Allocate memory for 2nd level table */
3389	if (!table[idx]) {
3390		page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3391					    get_order(baser->psz));
3392		if (!page)
3393			return false;
3394
3395		/* Flush Lvl2 table to PoC if hw doesn't support coherency */
3396		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3397			gic_flush_dcache_to_poc(page_address(page), baser->psz);
3398
3399		table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3400
3401		/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3402		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3403			gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3404
3405		/* Ensure updated table contents are visible to ITS hardware */
3406		dsb(sy);
3407	}
3408
3409	return true;
3410}
3411
3412static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3413{
3414	struct its_baser *baser;
3415
3416	baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3417
3418	/* Don't allow device id that exceeds ITS hardware limit */
3419	if (!baser)
3420		return (ilog2(dev_id) < device_ids(its));
3421
3422	return its_alloc_table_entry(its, baser, dev_id);
3423}
3424
3425static bool its_alloc_vpe_table(u32 vpe_id)
3426{
3427	struct its_node *its;
3428	int cpu;
3429
3430	/*
3431	 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3432	 * could try and only do it on ITSs corresponding to devices
3433	 * that have interrupts targeted at this VPE, but the
3434	 * complexity becomes crazy (and you have tons of memory
3435	 * anyway, right?).
3436	 */
3437	list_for_each_entry(its, &its_nodes, entry) {
3438		struct its_baser *baser;
3439
3440		if (!is_v4(its))
3441			continue;
3442
3443		baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3444		if (!baser)
3445			return false;
3446
3447		if (!its_alloc_table_entry(its, baser, vpe_id))
3448			return false;
3449	}
3450
3451	/* Non v4.1? No need to iterate RDs and go back early. */
3452	if (!gic_rdists->has_rvpeid)
3453		return true;
3454
3455	/*
3456	 * Make sure the L2 tables are allocated for all copies of
3457	 * the L1 table on *all* v4.1 RDs.
3458	 */
3459	for_each_possible_cpu(cpu) {
3460		if (!allocate_vpe_l2_table(cpu, vpe_id))
3461			return false;
3462	}
3463
3464	return true;
3465}
3466
3467static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3468					    int nvecs, bool alloc_lpis)
3469{
3470	struct its_device *dev;
3471	unsigned long *lpi_map = NULL;
3472	unsigned long flags;
3473	u16 *col_map = NULL;
3474	void *itt;
3475	int lpi_base;
3476	int nr_lpis;
3477	int nr_ites;
3478	int id_bits;
3479	int sz;
3480
3481	if (!its_alloc_device_table(its, dev_id))
3482		return NULL;
3483
3484	if (WARN_ON(!is_power_of_2(nvecs)))
3485		nvecs = roundup_pow_of_two(nvecs);
3486
3487	/*
3488	 * Even if the device wants a single LPI, the ITT must be
3489	 * sized as a power of two (and you need at least one bit...).
3490	 * Also honor the ITS's own EID limit.
3491	 */
3492	id_bits = FIELD_GET(GITS_TYPER_IDBITS, its->typer) + 1;
3493	nvecs = min_t(unsigned int, nvecs, BIT(id_bits));
3494	nr_ites = max(2, nvecs);
3495	sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3496	sz = max(sz, ITS_ITT_ALIGN);
3497
3498	itt = itt_alloc_pool(its->numa_node, sz);
3499
3500	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3501
3502	if (alloc_lpis) {
3503		lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3504		if (lpi_map)
3505			col_map = kcalloc(nr_lpis, sizeof(*col_map),
3506					  GFP_KERNEL);
3507	} else {
3508		col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3509		nr_lpis = 0;
3510		lpi_base = 0;
3511	}
3512
3513	if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
3514		kfree(dev);
3515		itt_free_pool(itt, sz);
3516		bitmap_free(lpi_map);
3517		kfree(col_map);
3518		return NULL;
3519	}
3520
3521	gic_flush_dcache_to_poc(itt, sz);
3522
3523	dev->its = its;
3524	dev->itt = itt;
3525	dev->itt_sz = sz;
3526	dev->nr_ites = nr_ites;
3527	dev->event_map.lpi_map = lpi_map;
3528	dev->event_map.col_map = col_map;
3529	dev->event_map.lpi_base = lpi_base;
3530	dev->event_map.nr_lpis = nr_lpis;
3531	raw_spin_lock_init(&dev->event_map.vlpi_lock);
3532	dev->device_id = dev_id;
3533	INIT_LIST_HEAD(&dev->entry);
3534
3535	raw_spin_lock_irqsave(&its->lock, flags);
3536	list_add(&dev->entry, &its->its_device_list);
3537	raw_spin_unlock_irqrestore(&its->lock, flags);
3538
3539	/* Map device to its ITT */
3540	its_send_mapd(dev, 1);
3541
3542	return dev;
3543}
3544
3545static void its_free_device(struct its_device *its_dev)
3546{
3547	unsigned long flags;
3548
3549	raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3550	list_del(&its_dev->entry);
3551	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3552	kfree(its_dev->event_map.col_map);
3553	itt_free_pool(its_dev->itt, its_dev->itt_sz);
3554	kfree(its_dev);
3555}
3556
3557static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3558{
3559	int idx;
3560
3561	/* Find a free LPI region in lpi_map and allocate them. */
3562	idx = bitmap_find_free_region(dev->event_map.lpi_map,
3563				      dev->event_map.nr_lpis,
3564				      get_count_order(nvecs));
3565	if (idx < 0)
3566		return -ENOSPC;
3567
3568	*hwirq = dev->event_map.lpi_base + idx;
3569
3570	return 0;
3571}
3572
3573static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3574			   int nvec, msi_alloc_info_t *info)
3575{
3576	struct its_node *its;
3577	struct its_device *its_dev;
3578	struct msi_domain_info *msi_info;
3579	u32 dev_id;
3580	int err = 0;
3581
3582	/*
3583	 * We ignore "dev" entirely, and rely on the dev_id that has
3584	 * been passed via the scratchpad. This limits this domain's
3585	 * usefulness to upper layers that definitely know that they
3586	 * are built on top of the ITS.
3587	 */
3588	dev_id = info->scratchpad[0].ul;
3589
3590	msi_info = msi_get_domain_info(domain);
3591	its = msi_info->data;
3592
3593	if (!gic_rdists->has_direct_lpi &&
3594	    vpe_proxy.dev &&
3595	    vpe_proxy.dev->its == its &&
3596	    dev_id == vpe_proxy.dev->device_id) {
3597		/* Bad luck. Get yourself a better implementation */
3598		WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3599			  dev_id);
3600		return -EINVAL;
3601	}
3602
3603	mutex_lock(&its->dev_alloc_lock);
3604	its_dev = its_find_device(its, dev_id);
3605	if (its_dev) {
3606		/*
3607		 * We already have seen this ID, probably through
3608		 * another alias (PCI bridge of some sort). No need to
3609		 * create the device.
3610		 */
3611		its_dev->shared = true;
3612		pr_debug("Reusing ITT for devID %x\n", dev_id);
3613		goto out;
3614	}
3615
3616	its_dev = its_create_device(its, dev_id, nvec, true);
3617	if (!its_dev) {
3618		err = -ENOMEM;
3619		goto out;
3620	}
3621
3622	if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3623		its_dev->shared = true;
3624
3625	pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3626out:
3627	mutex_unlock(&its->dev_alloc_lock);
3628	info->scratchpad[0].ptr = its_dev;
3629	return err;
3630}
3631
3632static void its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info)
3633{
3634	struct its_device *its_dev = info->scratchpad[0].ptr;
3635
3636	guard(mutex)(&its_dev->its->dev_alloc_lock);
3637
3638	/* If the device is shared, keep everything around */
3639	if (its_dev->shared)
3640		return;
3641
3642	/* LPIs should have been already unmapped at this stage */
3643	if (WARN_ON_ONCE(!bitmap_empty(its_dev->event_map.lpi_map,
3644				       its_dev->event_map.nr_lpis)))
3645		return;
3646
3647	its_lpi_free(its_dev->event_map.lpi_map,
3648		     its_dev->event_map.lpi_base,
3649		     its_dev->event_map.nr_lpis);
3650
3651	/* Unmap device/itt, and get rid of the tracking */
3652	its_send_mapd(its_dev, 0);
3653	its_free_device(its_dev);
3654}
3655
3656static struct msi_domain_ops its_msi_domain_ops = {
3657	.msi_prepare	= its_msi_prepare,
3658	.msi_teardown	= its_msi_teardown,
3659};
3660
3661static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3662				    unsigned int virq,
3663				    irq_hw_number_t hwirq)
3664{
3665	struct irq_fwspec fwspec;
3666
3667	if (irq_domain_get_of_node(domain->parent)) {
3668		fwspec.fwnode = domain->parent->fwnode;
3669		fwspec.param_count = 3;
3670		fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3671		fwspec.param[1] = hwirq;
3672		fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3673	} else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3674		fwspec.fwnode = domain->parent->fwnode;
3675		fwspec.param_count = 2;
3676		fwspec.param[0] = hwirq;
3677		fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3678	} else {
3679		return -EINVAL;
3680	}
3681
3682	return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3683}
3684
3685static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3686				unsigned int nr_irqs, void *args)
3687{
3688	msi_alloc_info_t *info = args;
3689	struct its_device *its_dev = info->scratchpad[0].ptr;
3690	struct its_node *its = its_dev->its;
3691	struct irq_data *irqd;
3692	irq_hw_number_t hwirq;
3693	int err;
3694	int i;
3695
3696	err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3697	if (err)
3698		return err;
3699
3700	err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3701	if (err)
3702		return err;
3703
3704	for (i = 0; i < nr_irqs; i++) {
3705		err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3706		if (err)
3707			return err;
3708
3709		irq_domain_set_hwirq_and_chip(domain, virq + i,
3710					      hwirq + i, &its_irq_chip, its_dev);
3711		irqd = irq_get_irq_data(virq + i);
3712		irqd_set_single_target(irqd);
3713		irqd_set_affinity_on_activate(irqd);
3714		irqd_set_resend_when_in_progress(irqd);
3715		pr_debug("ID:%d pID:%d vID:%d\n",
3716			 (int)(hwirq + i - its_dev->event_map.lpi_base),
3717			 (int)(hwirq + i), virq + i);
3718	}
3719
3720	return 0;
3721}
3722
3723static int its_irq_domain_activate(struct irq_domain *domain,
3724				   struct irq_data *d, bool reserve)
3725{
3726	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3727	u32 event = its_get_event_id(d);
3728	int cpu;
3729
3730	cpu = its_select_cpu(d, cpu_online_mask);
3731	if (cpu < 0 || cpu >= nr_cpu_ids)
3732		return -EINVAL;
3733
3734	its_inc_lpi_count(d, cpu);
3735	its_dev->event_map.col_map[event] = cpu;
3736	irq_data_update_effective_affinity(d, cpumask_of(cpu));
3737
3738	/* Map the GIC IRQ and event to the device */
3739	its_send_mapti(its_dev, d->hwirq, event);
3740	return 0;
3741}
3742
3743static void its_irq_domain_deactivate(struct irq_domain *domain,
3744				      struct irq_data *d)
3745{
3746	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3747	u32 event = its_get_event_id(d);
3748
3749	its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3750	/* Stop the delivery of interrupts */
3751	its_send_discard(its_dev, event);
3752}
3753
3754static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3755				unsigned int nr_irqs)
3756{
3757	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3758	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3759	int i;
3760
3761	bitmap_release_region(its_dev->event_map.lpi_map,
3762			      its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3763			      get_count_order(nr_irqs));
3764
3765	for (i = 0; i < nr_irqs; i++) {
3766		struct irq_data *data = irq_domain_get_irq_data(domain,
3767								virq + i);
3768		/* Nuke the entry in the domain */
3769		irq_domain_reset_irq_data(data);
3770	}
3771
3772	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3773}
3774
3775static const struct irq_domain_ops its_domain_ops = {
3776	.select			= msi_lib_irq_domain_select,
3777	.alloc			= its_irq_domain_alloc,
3778	.free			= its_irq_domain_free,
3779	.activate		= its_irq_domain_activate,
3780	.deactivate		= its_irq_domain_deactivate,
3781};
3782
3783/*
3784 * This is insane.
3785 *
3786 * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3787 * likely), the only way to perform an invalidate is to use a fake
3788 * device to issue an INV command, implying that the LPI has first
3789 * been mapped to some event on that device. Since this is not exactly
3790 * cheap, we try to keep that mapping around as long as possible, and
3791 * only issue an UNMAP if we're short on available slots.
3792 *
3793 * Broken by design(tm).
3794 *
3795 * GICv4.1, on the other hand, mandates that we're able to invalidate
3796 * by writing to a MMIO register. It doesn't implement the whole of
3797 * DirectLPI, but that's good enough. And most of the time, we don't
3798 * even have to invalidate anything, as the redistributor can be told
3799 * whether to generate a doorbell or not (we thus leave it enabled,
3800 * always).
3801 */
3802static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3803{
3804	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3805	if (gic_rdists->has_rvpeid)
3806		return;
3807
3808	/* Already unmapped? */
3809	if (vpe->vpe_proxy_event == -1)
3810		return;
3811
3812	its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3813	vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3814
3815	/*
3816	 * We don't track empty slots at all, so let's move the
3817	 * next_victim pointer if we can quickly reuse that slot
3818	 * instead of nuking an existing entry. Not clear that this is
3819	 * always a win though, and this might just generate a ripple
3820	 * effect... Let's just hope VPEs don't migrate too often.
3821	 */
3822	if (vpe_proxy.vpes[vpe_proxy.next_victim])
3823		vpe_proxy.next_victim = vpe->vpe_proxy_event;
3824
3825	vpe->vpe_proxy_event = -1;
3826}
3827
3828static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3829{
3830	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3831	if (gic_rdists->has_rvpeid)
3832		return;
3833
3834	if (!gic_rdists->has_direct_lpi) {
3835		unsigned long flags;
3836
3837		raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3838		its_vpe_db_proxy_unmap_locked(vpe);
3839		raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3840	}
3841}
3842
3843static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3844{
3845	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3846	if (gic_rdists->has_rvpeid)
3847		return;
3848
3849	/* Already mapped? */
3850	if (vpe->vpe_proxy_event != -1)
3851		return;
3852
3853	/* This slot was already allocated. Kick the other VPE out. */
3854	if (vpe_proxy.vpes[vpe_proxy.next_victim])
3855		its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3856
3857	/* Map the new VPE instead */
3858	vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3859	vpe->vpe_proxy_event = vpe_proxy.next_victim;
3860	vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3861
3862	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3863	its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3864}
3865
3866static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3867{
3868	unsigned long flags;
3869	struct its_collection *target_col;
3870
3871	/* GICv4.1 doesn't use a proxy, so nothing to do here */
3872	if (gic_rdists->has_rvpeid)
3873		return;
3874
3875	if (gic_rdists->has_direct_lpi) {
3876		void __iomem *rdbase;
3877
3878		rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3879		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3880		wait_for_syncr(rdbase);
3881
3882		return;
3883	}
3884
3885	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3886
3887	its_vpe_db_proxy_map_locked(vpe);
3888
3889	target_col = &vpe_proxy.dev->its->collections[to];
3890	its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3891	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3892
3893	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3894}
3895
3896static void its_vpe_4_1_invall_locked(int cpu, struct its_vpe *vpe)
3897{
3898	void __iomem *rdbase;
3899	u64 val;
3900
3901	val  = GICR_INVALLR_V;
3902	val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
3903
3904	guard(raw_spinlock)(&gic_data_rdist_cpu(cpu)->rd_lock);
3905	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
3906	gic_write_lpir(val, rdbase + GICR_INVALLR);
3907	wait_for_syncr(rdbase);
3908}
3909
3910static int its_vpe_set_affinity(struct irq_data *d,
3911				const struct cpumask *mask_val,
3912				bool force)
3913{
3914	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3915	unsigned int from, cpu = nr_cpu_ids;
3916	struct cpumask *table_mask;
3917	struct its_node *its;
3918	unsigned long flags;
3919
3920	/*
3921	 * Check if we're racing against a VPE being destroyed, for
3922	 * which we don't want to allow a VMOVP.
3923	 */
3924	if (!atomic_read(&vpe->vmapp_count)) {
3925		if (gic_requires_eager_mapping())
3926			return -EINVAL;
3927
3928		/*
3929		 * If we lazily map the VPEs, this isn't an error and
3930		 * we can exit cleanly.
3931		 */
3932		cpu = cpumask_first(mask_val);
3933		irq_data_update_effective_affinity(d, cpumask_of(cpu));
3934		return IRQ_SET_MASK_OK_DONE;
3935	}
3936
3937	/*
3938	 * Changing affinity is mega expensive, so let's be as lazy as
3939	 * we can and only do it if we really have to. Also, if mapped
3940	 * into the proxy device, we need to move the doorbell
3941	 * interrupt to its new location.
3942	 *
3943	 * Another thing is that changing the affinity of a vPE affects
3944	 * *other interrupts* such as all the vLPIs that are routed to
3945	 * this vPE. This means that the irq_desc lock is not enough to
3946	 * protect us, and that we must ensure nobody samples vpe->col_idx
3947	 * during the update, hence the lock below which must also be
3948	 * taken on any vLPI handling path that evaluates vpe->col_idx.
3949	 *
3950	 * Finally, we must protect ourselves against concurrent updates of
3951	 * the mapping state on this VM should the ITS list be in use (see
3952	 * the shortcut in its_send_vmovp() otherewise).
3953	 */
3954	if (its_list_map)
3955		raw_spin_lock(&vpe->its_vm->vmapp_lock);
3956
3957	from = vpe_to_cpuid_lock(vpe, &flags);
3958	table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;
3959
3960	/*
3961	 * If we are offered another CPU in the same GICv4.1 ITS
3962	 * affinity, pick this one. Otherwise, any CPU will do.
3963	 */
3964	if (table_mask)
3965		cpu = cpumask_any_and(mask_val, table_mask);
3966	if (cpu < nr_cpu_ids) {
3967		if (cpumask_test_cpu(from, mask_val) &&
3968		    cpumask_test_cpu(from, table_mask))
3969			cpu = from;
3970	} else {
3971		cpu = cpumask_first(mask_val);
3972	}
3973
3974	if (from == cpu)
3975		goto out;
3976
3977	vpe->col_idx = cpu;
3978
3979	its_send_vmovp(vpe);
3980
3981	its = find_4_1_its();
3982	if (its && its->flags & ITS_FLAGS_WORKAROUND_HISILICON_162100801)
3983		its_vpe_4_1_invall_locked(cpu, vpe);
3984
3985	its_vpe_db_proxy_move(vpe, from, cpu);
3986
3987out:
3988	irq_data_update_effective_affinity(d, cpumask_of(cpu));
3989	vpe_to_cpuid_unlock(vpe, flags);
3990
3991	if (its_list_map)
3992		raw_spin_unlock(&vpe->its_vm->vmapp_lock);
3993
3994	return IRQ_SET_MASK_OK_DONE;
3995}
3996
3997static void its_wait_vpt_parse_complete(void)
3998{
3999	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4000	u64 val;
4001
4002	if (!gic_rdists->has_vpend_valid_dirty)
4003		return;
4004
4005	WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
4006						       val,
4007						       !(val & GICR_VPENDBASER_Dirty),
4008						       1, 500));
4009}
4010
4011static void its_vpe_schedule(struct its_vpe *vpe)
4012{
4013	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4014	u64 val;
4015
4016	/* Schedule the VPE */
4017	val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
4018		GENMASK_ULL(51, 12);
4019	val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
4020	if (rdists_support_shareable()) {
4021		val |= GICR_VPROPBASER_RaWb;
4022		val |= GICR_VPROPBASER_InnerShareable;
4023	}
4024	gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
4025
4026	val  = virt_to_phys(page_address(vpe->vpt_page)) &
4027		GENMASK_ULL(51, 16);
4028	if (rdists_support_shareable()) {
4029		val |= GICR_VPENDBASER_RaWaWb;
4030		val |= GICR_VPENDBASER_InnerShareable;
4031	}
4032	/*
4033	 * There is no good way of finding out if the pending table is
4034	 * empty as we can race against the doorbell interrupt very
4035	 * easily. So in the end, vpe->pending_last is only an
4036	 * indication that the vcpu has something pending, not one
4037	 * that the pending table is empty. A good implementation
4038	 * would be able to read its coarse map pretty quickly anyway,
4039	 * making this a tolerable issue.
4040	 */
4041	val |= GICR_VPENDBASER_PendingLast;
4042	val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
4043	val |= GICR_VPENDBASER_Valid;
4044	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4045}
4046
4047static void its_vpe_deschedule(struct its_vpe *vpe)
4048{
4049	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4050	u64 val;
4051
4052	val = its_clear_vpend_valid(vlpi_base, 0, 0);
4053
4054	vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
4055	vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4056}
4057
4058static void its_vpe_invall(struct its_vpe *vpe)
4059{
4060	struct its_node *its;
4061
4062	guard(raw_spinlock_irqsave)(&vpe->its_vm->vmapp_lock);
4063
4064	list_for_each_entry(its, &its_nodes, entry) {
4065		if (!is_v4(its))
4066			continue;
4067
4068		if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
4069			continue;
4070
4071		/*
4072		 * Sending a VINVALL to a single ITS is enough, as all
4073		 * we need is to reach the redistributors.
4074		 */
4075		its_send_vinvall(its, vpe);
4076		return;
4077	}
4078}
4079
4080static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4081{
4082	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4083	struct its_cmd_info *info = vcpu_info;
4084
4085	switch (info->cmd_type) {
4086	case SCHEDULE_VPE:
4087		its_vpe_schedule(vpe);
4088		return 0;
4089
4090	case DESCHEDULE_VPE:
4091		its_vpe_deschedule(vpe);
4092		return 0;
4093
4094	case COMMIT_VPE:
4095		its_wait_vpt_parse_complete();
4096		return 0;
4097
4098	case INVALL_VPE:
4099		its_vpe_invall(vpe);
4100		return 0;
4101
4102	default:
4103		return -EINVAL;
4104	}
4105}
4106
4107static void its_vpe_send_cmd(struct its_vpe *vpe,
4108			     void (*cmd)(struct its_device *, u32))
4109{
4110	unsigned long flags;
4111
4112	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
4113
4114	its_vpe_db_proxy_map_locked(vpe);
4115	cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
4116
4117	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
4118}
4119
4120static void its_vpe_send_inv(struct irq_data *d)
4121{
4122	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4123
4124	if (gic_rdists->has_direct_lpi)
4125		__direct_lpi_inv(d, d->parent_data->hwirq);
4126	else
4127		its_vpe_send_cmd(vpe, its_send_inv);
4128}
4129
4130static void its_vpe_mask_irq(struct irq_data *d)
4131{
4132	/*
4133	 * We need to unmask the LPI, which is described by the parent
4134	 * irq_data. Instead of calling into the parent (which won't
4135	 * exactly do the right thing, let's simply use the
4136	 * parent_data pointer. Yes, I'm naughty.
4137	 */
4138	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4139	its_vpe_send_inv(d);
4140}
4141
4142static void its_vpe_unmask_irq(struct irq_data *d)
4143{
4144	/* Same hack as above... */
4145	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4146	its_vpe_send_inv(d);
4147}
4148
4149static int its_vpe_set_irqchip_state(struct irq_data *d,
4150				     enum irqchip_irq_state which,
4151				     bool state)
4152{
4153	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4154
4155	if (which != IRQCHIP_STATE_PENDING)
4156		return -EINVAL;
4157
4158	if (gic_rdists->has_direct_lpi) {
4159		void __iomem *rdbase;
4160
4161		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4162		if (state) {
4163			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4164		} else {
4165			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
4166			wait_for_syncr(rdbase);
4167		}
4168	} else {
4169		if (state)
4170			its_vpe_send_cmd(vpe, its_send_int);
4171		else
4172			its_vpe_send_cmd(vpe, its_send_clear);
4173	}
4174
4175	return 0;
4176}
4177
4178static int its_vpe_retrigger(struct irq_data *d)
4179{
4180	return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4181}
4182
4183static struct irq_chip its_vpe_irq_chip = {
4184	.name			= "GICv4-vpe",
4185	.irq_mask		= its_vpe_mask_irq,
4186	.irq_unmask		= its_vpe_unmask_irq,
4187	.irq_eoi		= irq_chip_eoi_parent,
4188	.irq_set_affinity	= its_vpe_set_affinity,
4189	.irq_retrigger		= its_vpe_retrigger,
4190	.irq_set_irqchip_state	= its_vpe_set_irqchip_state,
4191	.irq_set_vcpu_affinity	= its_vpe_set_vcpu_affinity,
4192};
4193
4194static struct its_node *find_4_1_its(void)
4195{
4196	struct its_node *its = *this_cpu_ptr(&local_4_1_its);
4197
4198	if (!its) {
4199		list_for_each_entry(its, &its_nodes, entry) {
4200			if (is_v4_1(its))
4201				return its;
4202		}
4203
4204		/* Oops? */
4205		its = NULL;
4206	}
4207
4208	return its;
4209}
4210
4211static void its_vpe_4_1_send_inv(struct irq_data *d)
4212{
4213	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4214	struct its_node *its;
4215
4216	/*
4217	 * GICv4.1 wants doorbells to be invalidated using the
4218	 * INVDB command in order to be broadcast to all RDs. Send
4219	 * it to the first valid ITS, and let the HW do its magic.
4220	 */
4221	its = find_4_1_its();
4222	if (its)
4223		its_send_invdb(its, vpe);
4224}
4225
4226static void its_vpe_4_1_mask_irq(struct irq_data *d)
4227{
4228	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4229	its_vpe_4_1_send_inv(d);
4230}
4231
4232static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4233{
4234	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4235	its_vpe_4_1_send_inv(d);
4236}
4237
4238static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4239				 struct its_cmd_info *info)
4240{
4241	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4242	u64 val = 0;
4243
4244	/* Schedule the VPE */
4245	val |= GICR_VPENDBASER_Valid;
4246	val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4247	val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4248	val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4249
4250	gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4251}
4252
4253static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4254				   struct its_cmd_info *info)
4255{
4256	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4257	u64 val;
4258
4259	if (info->req_db) {
4260		unsigned long flags;
4261
4262		/*
4263		 * vPE is going to block: make the vPE non-resident with
4264		 * PendingLast clear and DB set. The GIC guarantees that if
4265		 * we read-back PendingLast clear, then a doorbell will be
4266		 * delivered when an interrupt comes.
4267		 *
4268		 * Note the locking to deal with the concurrent update of
4269		 * pending_last from the doorbell interrupt handler that can
4270		 * run concurrently.
4271		 */
4272		raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4273		val = its_clear_vpend_valid(vlpi_base,
4274					    GICR_VPENDBASER_PendingLast,
4275					    GICR_VPENDBASER_4_1_DB);
4276		vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4277		raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4278	} else {
4279		/*
4280		 * We're not blocking, so just make the vPE non-resident
4281		 * with PendingLast set, indicating that we'll be back.
4282		 */
4283		val = its_clear_vpend_valid(vlpi_base,
4284					    0,
4285					    GICR_VPENDBASER_PendingLast);
4286		vpe->pending_last = true;
4287	}
4288}
4289
4290static void its_vpe_4_1_invall(struct its_vpe *vpe)
4291{
4292	unsigned long flags;
4293	int cpu;
4294
4295	/* Target the redistributor this vPE is currently known on */
4296	cpu = vpe_to_cpuid_lock(vpe, &flags);
4297	its_vpe_4_1_invall_locked(cpu, vpe);
4298	vpe_to_cpuid_unlock(vpe, flags);
4299}
4300
4301static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4302{
4303	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4304	struct its_cmd_info *info = vcpu_info;
4305
4306	switch (info->cmd_type) {
4307	case SCHEDULE_VPE:
4308		its_vpe_4_1_schedule(vpe, info);
4309		return 0;
4310
4311	case DESCHEDULE_VPE:
4312		its_vpe_4_1_deschedule(vpe, info);
4313		return 0;
4314
4315	case COMMIT_VPE:
4316		its_wait_vpt_parse_complete();
4317		return 0;
4318
4319	case INVALL_VPE:
4320		its_vpe_4_1_invall(vpe);
4321		return 0;
4322
4323	default:
4324		return -EINVAL;
4325	}
4326}
4327
4328static struct irq_chip its_vpe_4_1_irq_chip = {
4329	.name			= "GICv4.1-vpe",
4330	.irq_mask		= its_vpe_4_1_mask_irq,
4331	.irq_unmask		= its_vpe_4_1_unmask_irq,
4332	.irq_eoi		= irq_chip_eoi_parent,
4333	.irq_set_affinity	= its_vpe_set_affinity,
4334	.irq_set_vcpu_affinity	= its_vpe_4_1_set_vcpu_affinity,
4335};
4336
4337static void its_configure_sgi(struct irq_data *d, bool clear)
4338{
4339	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4340	struct its_cmd_desc desc;
4341
4342	desc.its_vsgi_cmd.vpe = vpe;
4343	desc.its_vsgi_cmd.sgi = d->hwirq;
4344	desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4345	desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4346	desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4347	desc.its_vsgi_cmd.clear = clear;
4348
4349	/*
4350	 * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4351	 * destination VPE is mapped there. Since we map them eagerly at
4352	 * activation time, we're pretty sure the first GICv4.1 ITS will do.
4353	 */
4354	its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4355}
4356
4357static void its_sgi_mask_irq(struct irq_data *d)
4358{
4359	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4360
4361	vpe->sgi_config[d->hwirq].enabled = false;
4362	its_configure_sgi(d, false);
4363}
4364
4365static void its_sgi_unmask_irq(struct irq_data *d)
4366{
4367	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4368
4369	vpe->sgi_config[d->hwirq].enabled = true;
4370	its_configure_sgi(d, false);
4371}
4372
4373static int its_sgi_set_affinity(struct irq_data *d,
4374				const struct cpumask *mask_val,
4375				bool force)
4376{
4377	/*
4378	 * There is no notion of affinity for virtual SGIs, at least
4379	 * not on the host (since they can only be targeting a vPE).
4380	 * Tell the kernel we've done whatever it asked for.
4381	 */
4382	irq_data_update_effective_affinity(d, mask_val);
4383	return IRQ_SET_MASK_OK;
4384}
4385
4386static int its_sgi_set_irqchip_state(struct irq_data *d,
4387				     enum irqchip_irq_state which,
4388				     bool state)
4389{
4390	if (which != IRQCHIP_STATE_PENDING)
4391		return -EINVAL;
4392
4393	if (state) {
4394		struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4395		struct its_node *its = find_4_1_its();
4396		u64 val;
4397
4398		val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4399		val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4400		writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4401	} else {
4402		its_configure_sgi(d, true);
4403	}
4404
4405	return 0;
4406}
4407
4408static int its_sgi_get_irqchip_state(struct irq_data *d,
4409				     enum irqchip_irq_state which, bool *val)
4410{
4411	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4412	void __iomem *base;
4413	unsigned long flags;
4414	u32 count = 1000000;	/* 1s! */
4415	u32 status;
4416	int cpu;
4417
4418	if (which != IRQCHIP_STATE_PENDING)
4419		return -EINVAL;
4420
4421	/*
4422	 * Locking galore! We can race against two different events:
4423	 *
4424	 * - Concurrent vPE affinity change: we must make sure it cannot
4425	 *   happen, or we'll talk to the wrong redistributor. This is
4426	 *   identical to what happens with vLPIs.
4427	 *
4428	 * - Concurrent VSGIPENDR access: As it involves accessing two
4429	 *   MMIO registers, this must be made atomic one way or another.
4430	 */
4431	cpu = vpe_to_cpuid_lock(vpe, &flags);
4432	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4433	base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4434	writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4435	do {
4436		status = readl_relaxed(base + GICR_VSGIPENDR);
4437		if (!(status & GICR_VSGIPENDR_BUSY))
4438			goto out;
4439
4440		count--;
4441		if (!count) {
4442			pr_err_ratelimited("Unable to get SGI status\n");
4443			goto out;
4444		}
4445		cpu_relax();
4446		udelay(1);
4447	} while (count);
4448
4449out:
4450	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4451	vpe_to_cpuid_unlock(vpe, flags);
4452
4453	if (!count)
4454		return -ENXIO;
4455
4456	*val = !!(status & (1 << d->hwirq));
4457
4458	return 0;
4459}
4460
4461static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4462{
4463	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4464	struct its_cmd_info *info = vcpu_info;
4465
4466	switch (info->cmd_type) {
4467	case PROP_UPDATE_VSGI:
4468		vpe->sgi_config[d->hwirq].priority = info->priority;
4469		vpe->sgi_config[d->hwirq].group = info->group;
4470		its_configure_sgi(d, false);
4471		return 0;
4472
4473	default:
4474		return -EINVAL;
4475	}
4476}
4477
4478static struct irq_chip its_sgi_irq_chip = {
4479	.name			= "GICv4.1-sgi",
4480	.irq_mask		= its_sgi_mask_irq,
4481	.irq_unmask		= its_sgi_unmask_irq,
4482	.irq_set_affinity	= its_sgi_set_affinity,
4483	.irq_set_irqchip_state	= its_sgi_set_irqchip_state,
4484	.irq_get_irqchip_state	= its_sgi_get_irqchip_state,
4485	.irq_set_vcpu_affinity	= its_sgi_set_vcpu_affinity,
4486};
4487
4488static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4489				    unsigned int virq, unsigned int nr_irqs,
4490				    void *args)
4491{
4492	struct its_vpe *vpe = args;
4493	int i;
4494
4495	/* Yes, we do want 16 SGIs */
4496	WARN_ON(nr_irqs != 16);
4497
4498	for (i = 0; i < 16; i++) {
4499		vpe->sgi_config[i].priority = 0;
4500		vpe->sgi_config[i].enabled = false;
4501		vpe->sgi_config[i].group = false;
4502
4503		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4504					      &its_sgi_irq_chip, vpe);
4505		irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4506	}
4507
4508	return 0;
4509}
4510
4511static void its_sgi_irq_domain_free(struct irq_domain *domain,
4512				    unsigned int virq,
4513				    unsigned int nr_irqs)
4514{
4515	/* Nothing to do */
4516}
4517
4518static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4519				       struct irq_data *d, bool reserve)
4520{
4521	/* Write out the initial SGI configuration */
4522	its_configure_sgi(d, false);
4523	return 0;
4524}
4525
4526static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4527					  struct irq_data *d)
4528{
4529	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4530
4531	/*
4532	 * The VSGI command is awkward:
4533	 *
4534	 * - To change the configuration, CLEAR must be set to false,
4535	 *   leaving the pending bit unchanged.
4536	 * - To clear the pending bit, CLEAR must be set to true, leaving
4537	 *   the configuration unchanged.
4538	 *
4539	 * You just can't do both at once, hence the two commands below.
4540	 */
4541	vpe->sgi_config[d->hwirq].enabled = false;
4542	its_configure_sgi(d, false);
4543	its_configure_sgi(d, true);
4544}
4545
4546static const struct irq_domain_ops its_sgi_domain_ops = {
4547	.alloc		= its_sgi_irq_domain_alloc,
4548	.free		= its_sgi_irq_domain_free,
4549	.activate	= its_sgi_irq_domain_activate,
4550	.deactivate	= its_sgi_irq_domain_deactivate,
4551};
4552
4553static int its_vpe_id_alloc(void)
4554{
4555	return ida_alloc_max(&its_vpeid_ida, ITS_MAX_VPEID - 1, GFP_KERNEL);
4556}
4557
4558static void its_vpe_id_free(u16 id)
4559{
4560	ida_free(&its_vpeid_ida, id);
4561}
4562
4563static int its_vpe_init(struct its_vpe *vpe)
4564{
4565	struct page *vpt_page;
4566	int vpe_id;
4567
4568	/* Allocate vpe_id */
4569	vpe_id = its_vpe_id_alloc();
4570	if (vpe_id < 0)
4571		return vpe_id;
4572
4573	/* Allocate VPT */
4574	vpt_page = its_allocate_pending_table(GFP_KERNEL);
4575	if (!vpt_page) {
4576		its_vpe_id_free(vpe_id);
4577		return -ENOMEM;
4578	}
4579
4580	if (!its_alloc_vpe_table(vpe_id)) {
4581		its_vpe_id_free(vpe_id);
4582		its_free_pending_table(vpt_page);
4583		return -ENOMEM;
4584	}
4585
4586	raw_spin_lock_init(&vpe->vpe_lock);
4587	vpe->vpe_id = vpe_id;
4588	vpe->vpt_page = vpt_page;
4589	atomic_set(&vpe->vmapp_count, 0);
4590	if (!gic_rdists->has_rvpeid)
4591		vpe->vpe_proxy_event = -1;
4592
4593	return 0;
4594}
4595
4596static void its_vpe_teardown(struct its_vpe *vpe)
4597{
4598	its_vpe_db_proxy_unmap(vpe);
4599	its_vpe_id_free(vpe->vpe_id);
4600	its_free_pending_table(vpe->vpt_page);
4601}
4602
4603static void its_vpe_irq_domain_free(struct irq_domain *domain,
4604				    unsigned int virq,
4605				    unsigned int nr_irqs)
4606{
4607	struct its_vm *vm = domain->host_data;
4608	int i;
4609
4610	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4611
4612	for (i = 0; i < nr_irqs; i++) {
4613		struct irq_data *data = irq_domain_get_irq_data(domain,
4614								virq + i);
4615		struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4616
4617		BUG_ON(vm != vpe->its_vm);
4618
4619		clear_bit(data->hwirq, vm->db_bitmap);
4620		its_vpe_teardown(vpe);
4621		irq_domain_reset_irq_data(data);
4622	}
4623
4624	if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4625		its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4626		its_free_prop_table(vm->vprop_page);
4627	}
4628}
4629
4630static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4631				    unsigned int nr_irqs, void *args)
4632{
4633	struct irq_chip *irqchip = &its_vpe_irq_chip;
4634	struct its_vm *vm = args;
4635	unsigned long *bitmap;
4636	struct page *vprop_page;
4637	int base, nr_ids, i, err = 0;
4638
4639	bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4640	if (!bitmap)
4641		return -ENOMEM;
4642
4643	if (nr_ids < nr_irqs) {
4644		its_lpi_free(bitmap, base, nr_ids);
4645		return -ENOMEM;
4646	}
4647
4648	vprop_page = its_allocate_prop_table(GFP_KERNEL);
4649	if (!vprop_page) {
4650		its_lpi_free(bitmap, base, nr_ids);
4651		return -ENOMEM;
4652	}
4653
4654	vm->db_bitmap = bitmap;
4655	vm->db_lpi_base = base;
4656	vm->nr_db_lpis = nr_ids;
4657	vm->vprop_page = vprop_page;
4658	raw_spin_lock_init(&vm->vmapp_lock);
4659
4660	if (gic_rdists->has_rvpeid)
4661		irqchip = &its_vpe_4_1_irq_chip;
4662
4663	for (i = 0; i < nr_irqs; i++) {
4664		vm->vpes[i]->vpe_db_lpi = base + i;
4665		err = its_vpe_init(vm->vpes[i]);
4666		if (err)
4667			break;
4668		err = its_irq_gic_domain_alloc(domain, virq + i,
4669					       vm->vpes[i]->vpe_db_lpi);
4670		if (err)
4671			break;
4672		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4673					      irqchip, vm->vpes[i]);
4674		set_bit(i, bitmap);
4675		irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
4676	}
4677
4678	if (err)
4679		its_vpe_irq_domain_free(domain, virq, i);
4680
4681	return err;
4682}
4683
4684static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4685				       struct irq_data *d, bool reserve)
4686{
4687	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4688	struct its_node *its;
4689
4690	/* Map the VPE to the first possible CPU */
4691	vpe->col_idx = cpumask_first(cpu_online_mask);
4692	irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4693
4694	/*
4695	 * If we use the list map, we issue VMAPP on demand... Unless
4696	 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4697	 * so that VSGIs can work.
4698	 */
4699	if (!gic_requires_eager_mapping())
4700		return 0;
4701
4702	list_for_each_entry(its, &its_nodes, entry) {
4703		if (!is_v4(its))
4704			continue;
4705
4706		its_send_vmapp(its, vpe, true);
4707		its_send_vinvall(its, vpe);
4708	}
4709
4710	return 0;
4711}
4712
4713static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4714					  struct irq_data *d)
4715{
4716	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4717	struct its_node *its;
4718
4719	/*
4720	 * If we use the list map on GICv4.0, we unmap the VPE once no
4721	 * VLPIs are associated with the VM.
4722	 */
4723	if (!gic_requires_eager_mapping())
4724		return;
4725
4726	list_for_each_entry(its, &its_nodes, entry) {
4727		if (!is_v4(its))
4728			continue;
4729
4730		its_send_vmapp(its, vpe, false);
4731	}
4732
4733	/*
4734	 * There may be a direct read to the VPT after unmapping the
4735	 * vPE, to guarantee the validity of this, we make the VPT
4736	 * memory coherent with the CPU caches here.
4737	 */
4738	if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4739		gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4740					LPI_PENDBASE_SZ);
4741}
4742
4743static const struct irq_domain_ops its_vpe_domain_ops = {
4744	.alloc			= its_vpe_irq_domain_alloc,
4745	.free			= its_vpe_irq_domain_free,
4746	.activate		= its_vpe_irq_domain_activate,
4747	.deactivate		= its_vpe_irq_domain_deactivate,
4748};
4749
4750static int its_force_quiescent(void __iomem *base)
4751{
4752	u32 count = 1000000;	/* 1s */
4753	u32 val;
4754
4755	val = readl_relaxed(base + GITS_CTLR);
4756	/*
4757	 * GIC architecture specification requires the ITS to be both
4758	 * disabled and quiescent for writes to GITS_BASER<n> or
4759	 * GITS_CBASER to not have UNPREDICTABLE results.
4760	 */
4761	if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4762		return 0;
4763
4764	/* Disable the generation of all interrupts to this ITS */
4765	val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4766	writel_relaxed(val, base + GITS_CTLR);
4767
4768	/* Poll GITS_CTLR and wait until ITS becomes quiescent */
4769	while (1) {
4770		val = readl_relaxed(base + GITS_CTLR);
4771		if (val & GITS_CTLR_QUIESCENT)
4772			return 0;
4773
4774		count--;
4775		if (!count)
4776			return -EBUSY;
4777
4778		cpu_relax();
4779		udelay(1);
4780	}
4781}
4782
4783static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4784{
4785	struct its_node *its = data;
4786
4787	/* erratum 22375: only alloc 8MB table size (20 bits) */
4788	its->typer &= ~GITS_TYPER_DEVBITS;
4789	its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4790	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4791
4792	return true;
4793}
4794
4795static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4796{
4797	struct its_node *its = data;
4798
4799	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4800
4801	return true;
4802}
4803
4804static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4805{
4806	struct its_node *its = data;
4807
4808	/* On QDF2400, the size of the ITE is 16Bytes */
4809	its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4810	its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4811
4812	return true;
4813}
4814
4815static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4816{
4817	struct its_node *its = its_dev->its;
4818
4819	/*
4820	 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4821	 * which maps 32-bit writes targeted at a separate window of
4822	 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4823	 * with device ID taken from bits [device_id_bits + 1:2] of
4824	 * the window offset.
4825	 */
4826	return its->pre_its_base + (its_dev->device_id << 2);
4827}
4828
4829static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4830{
4831	struct its_node *its = data;
4832	u32 pre_its_window[2];
4833	u32 ids;
4834
4835	if (!fwnode_property_read_u32_array(its->fwnode_handle,
4836					   "socionext,synquacer-pre-its",
4837					   pre_its_window,
4838					   ARRAY_SIZE(pre_its_window))) {
4839
4840		its->pre_its_base = pre_its_window[0];
4841		its->get_msi_base = its_irq_get_msi_base_pre_its;
4842
4843		ids = ilog2(pre_its_window[1]) - 2;
4844		if (device_ids(its) > ids) {
4845			its->typer &= ~GITS_TYPER_DEVBITS;
4846			its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4847		}
4848
4849		/* the pre-ITS breaks isolation, so disable MSI remapping */
4850		its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
4851		return true;
4852	}
4853	return false;
4854}
4855
4856static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4857{
4858	struct its_node *its = data;
4859
4860	/*
4861	 * Hip07 insists on using the wrong address for the VLPI
4862	 * page. Trick it into doing the right thing...
4863	 */
4864	its->vlpi_redist_offset = SZ_128K;
4865	return true;
4866}
4867
4868static bool __maybe_unused its_enable_rk3588001(void *data)
4869{
4870	struct its_node *its = data;
4871
4872	if (!of_machine_is_compatible("rockchip,rk3588") &&
4873	    !of_machine_is_compatible("rockchip,rk3588s"))
4874		return false;
4875
4876	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4877	gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4878
4879	return true;
4880}
4881
4882static bool its_set_non_coherent(void *data)
4883{
4884	struct its_node *its = data;
4885
4886	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4887	return true;
4888}
4889
4890static bool __maybe_unused its_enable_quirk_hip09_162100801(void *data)
4891{
4892	struct its_node *its = data;
4893
4894	its->flags |= ITS_FLAGS_WORKAROUND_HISILICON_162100801;
4895	return true;
4896}
4897
4898static bool __maybe_unused its_enable_rk3568002(void *data)
4899{
4900	if (!of_machine_is_compatible("rockchip,rk3566") &&
4901	    !of_machine_is_compatible("rockchip,rk3568"))
4902		return false;
4903
4904	gfp_flags_quirk |= GFP_DMA32;
4905
4906	return true;
4907}
4908
4909static const struct gic_quirk its_quirks[] = {
4910#ifdef CONFIG_CAVIUM_ERRATUM_22375
4911	{
4912		.desc	= "ITS: Cavium errata 22375, 24313",
4913		.iidr	= 0xa100034c,	/* ThunderX pass 1.x */
4914		.mask	= 0xffff0fff,
4915		.init	= its_enable_quirk_cavium_22375,
4916	},
4917#endif
4918#ifdef CONFIG_CAVIUM_ERRATUM_23144
4919	{
4920		.desc	= "ITS: Cavium erratum 23144",
4921		.iidr	= 0xa100034c,	/* ThunderX pass 1.x */
4922		.mask	= 0xffff0fff,
4923		.init	= its_enable_quirk_cavium_23144,
4924	},
4925#endif
4926#ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4927	{
4928		.desc	= "ITS: QDF2400 erratum 0065",
4929		.iidr	= 0x00001070, /* QDF2400 ITS rev 1.x */
4930		.mask	= 0xffffffff,
4931		.init	= its_enable_quirk_qdf2400_e0065,
4932	},
4933#endif
4934#ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4935	{
4936		/*
4937		 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4938		 * implementation, but with a 'pre-ITS' added that requires
4939		 * special handling in software.
4940		 */
4941		.desc	= "ITS: Socionext Synquacer pre-ITS",
4942		.iidr	= 0x0001143b,
4943		.mask	= 0xffffffff,
4944		.init	= its_enable_quirk_socionext_synquacer,
4945	},
4946#endif
4947#ifdef CONFIG_HISILICON_ERRATUM_161600802
4948	{
4949		.desc	= "ITS: Hip07 erratum 161600802",
4950		.iidr	= 0x00000004,
4951		.mask	= 0xffffffff,
4952		.init	= its_enable_quirk_hip07_161600802,
4953	},
4954#endif
4955#ifdef CONFIG_HISILICON_ERRATUM_162100801
4956	{
4957		.desc	= "ITS: Hip09 erratum 162100801",
4958		.iidr	= 0x00051736,
4959		.mask	= 0xffffffff,
4960		.init	= its_enable_quirk_hip09_162100801,
4961	},
4962#endif
4963#ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4964	{
4965		.desc   = "ITS: Rockchip erratum RK3588001",
4966		.iidr   = 0x0201743b,
4967		.mask   = 0xffffffff,
4968		.init   = its_enable_rk3588001,
4969	},
4970#endif
4971	{
4972		.desc   = "ITS: non-coherent attribute",
4973		.property = "dma-noncoherent",
4974		.init   = its_set_non_coherent,
4975	},
4976#ifdef CONFIG_ROCKCHIP_ERRATUM_3568002
4977	{
4978		.desc   = "ITS: Rockchip erratum RK3568002",
4979		.iidr   = 0x0201743b,
4980		.mask   = 0xffffffff,
4981		.init   = its_enable_rk3568002,
4982	},
4983#endif
4984	{
4985	}
4986};
4987
4988static void its_enable_quirks(struct its_node *its)
4989{
4990	u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4991
4992	gic_enable_quirks(iidr, its_quirks, its);
4993
4994	if (is_of_node(its->fwnode_handle))
4995		gic_enable_of_quirks(to_of_node(its->fwnode_handle),
4996				     its_quirks, its);
4997}
4998
4999static int its_save_disable(void *data)
5000{
5001	struct its_node *its;
5002	int err = 0;
5003
5004	raw_spin_lock(&its_lock);
5005	list_for_each_entry(its, &its_nodes, entry) {
5006		void __iomem *base;
5007
5008		base = its->base;
5009		its->ctlr_save = readl_relaxed(base + GITS_CTLR);
5010		err = its_force_quiescent(base);
5011		if (err) {
5012			pr_err("ITS@%pa: failed to quiesce: %d\n",
5013			       &its->phys_base, err);
5014			writel_relaxed(its->ctlr_save, base + GITS_CTLR);
5015			goto err;
5016		}
5017
5018		its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
5019	}
5020
5021err:
5022	if (err) {
5023		list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
5024			void __iomem *base;
5025
5026			base = its->base;
5027			writel_relaxed(its->ctlr_save, base + GITS_CTLR);
5028		}
5029	}
5030	raw_spin_unlock(&its_lock);
5031
5032	return err;
5033}
5034
5035static void its_restore_enable(void *data)
5036{
5037	struct its_node *its;
5038	int ret;
5039
5040	raw_spin_lock(&its_lock);
5041	list_for_each_entry(its, &its_nodes, entry) {
5042		void __iomem *base;
5043		int i;
5044
5045		base = its->base;
5046
5047		/*
5048		 * Make sure that the ITS is disabled. If it fails to quiesce,
5049		 * don't restore it since writing to CBASER or BASER<n>
5050		 * registers is undefined according to the GIC v3 ITS
5051		 * Specification.
5052		 *
5053		 * Firmware resuming with the ITS enabled is terminally broken.
5054		 */
5055		WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
5056		ret = its_force_quiescent(base);
5057		if (ret) {
5058			pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
5059			       &its->phys_base, ret);
5060			continue;
5061		}
5062
5063		gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
5064
5065		/*
5066		 * Writing CBASER resets CREADR to 0, so make CWRITER and
5067		 * cmd_write line up with it.
5068		 */
5069		its->cmd_write = its->cmd_base;
5070		gits_write_cwriter(0, base + GITS_CWRITER);
5071
5072		/* Restore GITS_BASER from the value cache. */
5073		for (i = 0; i < GITS_BASER_NR_REGS; i++) {
5074			struct its_baser *baser = &its->tables[i];
5075
5076			if (!(baser->val & GITS_BASER_VALID))
5077				continue;
5078
5079			its_write_baser(its, baser, baser->val);
5080		}
5081		writel_relaxed(its->ctlr_save, base + GITS_CTLR);
5082
5083		/*
5084		 * Reinit the collection if it's stored in the ITS. This is
5085		 * indicated by the col_id being less than the HCC field.
5086		 * CID < HCC as specified in the GIC v3 Documentation.
5087		 */
5088		if (its->collections[smp_processor_id()].col_id <
5089		    GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
5090			its_cpu_init_collection(its);
5091	}
5092	raw_spin_unlock(&its_lock);
5093}
5094
5095static const struct syscore_ops its_syscore_ops = {
5096	.suspend = its_save_disable,
5097	.resume = its_restore_enable,
5098};
5099
5100static struct syscore its_syscore = {
5101	.ops = &its_syscore_ops,
5102};
5103
5104static void __init __iomem *its_map_one(struct resource *res, int *err)
5105{
5106	void __iomem *its_base;
5107	u32 val;
5108
5109	its_base = ioremap(res->start, SZ_64K);
5110	if (!its_base) {
5111		pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
5112		*err = -ENOMEM;
5113		return NULL;
5114	}
5115
5116	val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
5117	if (val != 0x30 && val != 0x40) {
5118		pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
5119		*err = -ENODEV;
5120		goto out_unmap;
5121	}
5122
5123	*err = its_force_quiescent(its_base);
5124	if (*err) {
5125		pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
5126		goto out_unmap;
5127	}
5128
5129	return its_base;
5130
5131out_unmap:
5132	iounmap(its_base);
5133	return NULL;
5134}
5135
5136static int its_init_domain(struct its_node *its)
5137{
5138	struct irq_domain_info dom_info = {
5139		.fwnode		= its->fwnode_handle,
5140		.ops		= &its_domain_ops,
5141		.domain_flags	= its->msi_domain_flags,
5142		.parent		= its_parent,
5143	};
5144	struct msi_domain_info *info;
5145
5146	info = kzalloc(sizeof(*info), GFP_KERNEL);
5147	if (!info)
5148		return -ENOMEM;
5149
5150	info->ops = &its_msi_domain_ops;
5151	info->data = its;
5152	dom_info.host_data = info;
5153
5154	if (!msi_create_parent_irq_domain(&dom_info, &gic_v3_its_msi_parent_ops)) {
5155		kfree(info);
5156		return -ENOMEM;
5157	}
5158	return 0;
5159}
5160
5161static int its_init_vpe_domain(void)
5162{
5163	struct its_node *its;
5164	u32 devid;
5165	int entries;
5166
5167	if (gic_rdists->has_direct_lpi) {
5168		pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5169		return 0;
5170	}
5171
5172	/* Any ITS will do, even if not v4 */
5173	its = list_first_entry(&its_nodes, struct its_node, entry);
5174
5175	entries = roundup_pow_of_two(nr_cpu_ids);
5176	vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
5177				 GFP_KERNEL);
5178	if (!vpe_proxy.vpes)
5179		return -ENOMEM;
5180
5181	/* Use the last possible DevID */
5182	devid = GENMASK(device_ids(its) - 1, 0);
5183	vpe_proxy.dev = its_create_device(its, devid, entries, false);
5184	if (!vpe_proxy.dev) {
5185		kfree(vpe_proxy.vpes);
5186		pr_err("ITS: Can't allocate GICv4 proxy device\n");
5187		return -ENOMEM;
5188	}
5189
5190	BUG_ON(entries > vpe_proxy.dev->nr_ites);
5191
5192	raw_spin_lock_init(&vpe_proxy.lock);
5193	vpe_proxy.next_victim = 0;
5194	pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5195		devid, vpe_proxy.dev->nr_ites);
5196
5197	return 0;
5198}
5199
5200static int __init its_compute_its_list_map(struct its_node *its)
5201{
5202	int its_number;
5203	u32 ctlr;
5204
5205	/*
5206	 * This is assumed to be done early enough that we're
5207	 * guaranteed to be single-threaded, hence no
5208	 * locking. Should this change, we should address
5209	 * this.
5210	 */
5211	its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5212	if (its_number >= GICv4_ITS_LIST_MAX) {
5213		pr_err("ITS@%pa: No ITSList entry available!\n",
5214		       &its->phys_base);
5215		return -EINVAL;
5216	}
5217
5218	ctlr = readl_relaxed(its->base + GITS_CTLR);
5219	ctlr &= ~GITS_CTLR_ITS_NUMBER;
5220	ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
5221	writel_relaxed(ctlr, its->base + GITS_CTLR);
5222	ctlr = readl_relaxed(its->base + GITS_CTLR);
5223	if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
5224		its_number = ctlr & GITS_CTLR_ITS_NUMBER;
5225		its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
5226	}
5227
5228	if (test_and_set_bit(its_number, &its_list_map)) {
5229		pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
5230		       &its->phys_base, its_number);
5231		return -EINVAL;
5232	}
5233
5234	return its_number;
5235}
5236
5237static int __init its_probe_one(struct its_node *its)
5238{
5239	u64 baser, tmp;
5240	struct page *page;
5241	u32 ctlr;
5242	int err;
5243
5244	its_enable_quirks(its);
5245
5246	if (is_v4(its)) {
5247		if (!(its->typer & GITS_TYPER_VMOVP)) {
5248			err = its_compute_its_list_map(its);
5249			if (err < 0)
5250				goto out;
5251
5252			its->list_nr = err;
5253
5254			pr_info("ITS@%pa: Using ITS number %d\n",
5255				&its->phys_base, err);
5256		} else {
5257			pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
5258		}
5259
5260		if (is_v4_1(its)) {
5261			u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
5262
5263			its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
5264			if (!its->sgir_base) {
5265				err = -ENOMEM;
5266				goto out;
5267			}
5268
5269			its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
5270
5271			pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5272				&its->phys_base, its->mpidr, svpet);
5273		}
5274	}
5275
5276	page = its_alloc_pages_node(its->numa_node,
5277				    GFP_KERNEL | __GFP_ZERO,
5278				    get_order(ITS_CMD_QUEUE_SZ));
5279	if (!page) {
5280		err = -ENOMEM;
5281		goto out_unmap_sgir;
5282	}
5283	its->cmd_base = (void *)page_address(page);
5284	its->cmd_write = its->cmd_base;
5285
5286	err = its_alloc_tables(its);
5287	if (err)
5288		goto out_free_cmd;
5289
5290	err = its_alloc_collections(its);
5291	if (err)
5292		goto out_free_tables;
5293
5294	baser = (virt_to_phys(its->cmd_base)	|
5295		 GITS_CBASER_RaWaWb		|
5296		 GITS_CBASER_InnerShareable	|
5297		 (ITS_CMD_QUEUE_SZ / SZ_4K - 1)	|
5298		 GITS_CBASER_VALID);
5299
5300	gits_write_cbaser(baser, its->base + GITS_CBASER);
5301	tmp = gits_read_cbaser(its->base + GITS_CBASER);
5302
5303	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5304		tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5305
5306	if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5307		if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5308			/*
5309			 * The HW reports non-shareable, we must
5310			 * remove the cacheability attributes as
5311			 * well.
5312			 */
5313			baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5314				   GITS_CBASER_CACHEABILITY_MASK);
5315			baser |= GITS_CBASER_nC;
5316			gits_write_cbaser(baser, its->base + GITS_CBASER);
5317		}
5318		pr_info("ITS: using cache flushing for cmd queue\n");
5319		its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5320	}
5321
5322	gits_write_cwriter(0, its->base + GITS_CWRITER);
5323	ctlr = readl_relaxed(its->base + GITS_CTLR);
5324	ctlr |= GITS_CTLR_ENABLE;
5325	if (is_v4(its))
5326		ctlr |= GITS_CTLR_ImDe;
5327	writel_relaxed(ctlr, its->base + GITS_CTLR);
5328
5329	err = its_init_domain(its);
5330	if (err)
5331		goto out_free_tables;
5332
5333	raw_spin_lock(&its_lock);
5334	list_add(&its->entry, &its_nodes);
5335	raw_spin_unlock(&its_lock);
5336
5337	return 0;
5338
5339out_free_tables:
5340	its_free_tables(its);
5341out_free_cmd:
5342	its_free_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5343out_unmap_sgir:
5344	if (its->sgir_base)
5345		iounmap(its->sgir_base);
5346out:
5347	pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
5348	return err;
5349}
5350
5351static bool gic_rdists_supports_plpis(void)
5352{
5353	return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5354}
5355
5356static int redist_disable_lpis(void)
5357{
5358	void __iomem *rbase = gic_data_rdist_rd_base();
5359	u64 timeout = USEC_PER_SEC;
5360	u64 val;
5361
5362	if (!gic_rdists_supports_plpis()) {
5363		pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5364		return -ENXIO;
5365	}
5366
5367	val = readl_relaxed(rbase + GICR_CTLR);
5368	if (!(val & GICR_CTLR_ENABLE_LPIS))
5369		return 0;
5370
5371	/*
5372	 * If coming via a CPU hotplug event, we don't need to disable
5373	 * LPIs before trying to re-enable them. They are already
5374	 * configured and all is well in the world.
5375	 *
5376	 * If running with preallocated tables, there is nothing to do.
5377	 */
5378	if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
5379	    (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5380		return 0;
5381
5382	/*
5383	 * From that point on, we only try to do some damage control.
5384	 */
5385	pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5386		smp_processor_id());
5387	add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5388
5389	/* Disable LPIs */
5390	val &= ~GICR_CTLR_ENABLE_LPIS;
5391	writel_relaxed(val, rbase + GICR_CTLR);
5392
5393	/* Make sure any change to GICR_CTLR is observable by the GIC */
5394	dsb(sy);
5395
5396	/*
5397	 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5398	 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5399	 * Error out if we time out waiting for RWP to clear.
5400	 */
5401	while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5402		if (!timeout) {
5403			pr_err("CPU%d: Timeout while disabling LPIs\n",
5404			       smp_processor_id());
5405			return -ETIMEDOUT;
5406		}
5407		udelay(1);
5408		timeout--;
5409	}
5410
5411	/*
5412	 * After it has been written to 1, it is IMPLEMENTATION
5413	 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5414	 * cleared to 0. Error out if clearing the bit failed.
5415	 */
5416	if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5417		pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5418		return -EBUSY;
5419	}
5420
5421	return 0;
5422}
5423
5424int its_cpu_init(void)
5425{
5426	if (!list_empty(&its_nodes)) {
5427		int ret;
5428
5429		ret = redist_disable_lpis();
5430		if (ret)
5431			return ret;
5432
5433		its_cpu_init_lpis();
5434		its_cpu_init_collections();
5435	}
5436
5437	return 0;
5438}
5439
5440static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5441{
5442	cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5443	gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5444}
5445
5446static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5447		    rdist_memreserve_cpuhp_cleanup_workfn);
5448
5449static int its_cpu_memreserve_lpi(unsigned int cpu)
5450{
5451	struct page *pend_page;
5452	int ret = 0;
5453
5454	/* This gets to run exactly once per CPU */
5455	if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5456		return 0;
5457
5458	pend_page = gic_data_rdist()->pend_page;
5459	if (WARN_ON(!pend_page)) {
5460		ret = -ENOMEM;
5461		goto out;
5462	}
5463	/*
5464	 * If the pending table was pre-programmed, free the memory we
5465	 * preemptively allocated. Otherwise, reserve that memory for
5466	 * later kexecs.
5467	 */
5468	if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5469		its_free_pending_table(pend_page);
5470		gic_data_rdist()->pend_page = NULL;
5471	} else {
5472		phys_addr_t paddr = page_to_phys(pend_page);
5473		WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5474	}
5475
5476out:
5477	/* Last CPU being brought up gets to issue the cleanup */
5478	if (!IS_ENABLED(CONFIG_SMP) ||
5479	    cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
5480		schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5481
5482	gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5483	return ret;
5484}
5485
5486/* Mark all the BASER registers as invalid before they get reprogrammed */
5487static int __init its_reset_one(struct resource *res)
5488{
5489	void __iomem *its_base;
5490	int err, i;
5491
5492	its_base = its_map_one(res, &err);
5493	if (!its_base)
5494		return err;
5495
5496	for (i = 0; i < GITS_BASER_NR_REGS; i++)
5497		gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5498
5499	iounmap(its_base);
5500	return 0;
5501}
5502
5503static const struct of_device_id its_device_id[] = {
5504	{	.compatible	= "arm,gic-v3-its",	},
5505	{},
5506};
5507
5508static struct its_node __init *its_node_init(struct resource *res,
5509					     struct fwnode_handle *handle, int numa_node)
5510{
5511	void __iomem *its_base;
5512	struct its_node *its;
5513	int err;
5514
5515	its_base = its_map_one(res, &err);
5516	if (!its_base)
5517		return NULL;
5518
5519	pr_info("ITS %pR\n", res);
5520
5521	its = kzalloc(sizeof(*its), GFP_KERNEL);
5522	if (!its)
5523		goto out_unmap;
5524
5525	raw_spin_lock_init(&its->lock);
5526	mutex_init(&its->dev_alloc_lock);
5527	INIT_LIST_HEAD(&its->entry);
5528	INIT_LIST_HEAD(&its->its_device_list);
5529
5530	its->typer = gic_read_typer(its_base + GITS_TYPER);
5531	its->base = its_base;
5532	its->phys_base = res->start;
5533	its->get_msi_base = its_irq_get_msi_base;
5534	its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI | IRQ_DOMAIN_FLAG_MSI_IMMUTABLE;
5535
5536	its->numa_node = numa_node;
5537	its->fwnode_handle = handle;
5538
5539	return its;
5540
5541out_unmap:
5542	iounmap(its_base);
5543	return NULL;
5544}
5545
5546static void its_node_destroy(struct its_node *its)
5547{
5548	iounmap(its->base);
5549	kfree(its);
5550}
5551
5552static int __init its_of_probe(struct device_node *node)
5553{
5554	struct device_node *np;
5555	struct resource res;
5556	int err;
5557
5558	/*
5559	 * Make sure *all* the ITS are reset before we probe any, as
5560	 * they may be sharing memory. If any of the ITS fails to
5561	 * reset, don't even try to go any further, as this could
5562	 * result in something even worse.
5563	 */
5564	for (np = of_find_matching_node(node, its_device_id); np;
5565	     np = of_find_matching_node(np, its_device_id)) {
5566		if (!of_device_is_available(np) ||
5567		    !of_property_read_bool(np, "msi-controller") ||
5568		    of_address_to_resource(np, 0, &res))
5569			continue;
5570
5571		err = its_reset_one(&res);
5572		if (err)
5573			return err;
5574	}
5575
5576	for (np = of_find_matching_node(node, its_device_id); np;
5577	     np = of_find_matching_node(np, its_device_id)) {
5578		struct its_node *its;
5579
5580		if (!of_device_is_available(np))
5581			continue;
5582		if (!of_property_read_bool(np, "msi-controller")) {
5583			pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5584				np);
5585			continue;
5586		}
5587
5588		if (of_address_to_resource(np, 0, &res)) {
5589			pr_warn("%pOF: no regs?\n", np);
5590			continue;
5591		}
5592
5593
5594		its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
5595		if (!its)
5596			return -ENOMEM;
5597
5598		err = its_probe_one(its);
5599		if (err)  {
5600			its_node_destroy(its);
5601			return err;
5602		}
5603	}
5604	return 0;
5605}
5606
5607#ifdef CONFIG_ACPI
5608
5609#define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5610
5611#ifdef CONFIG_ACPI_NUMA
5612struct its_srat_map {
5613	/* numa node id */
5614	u32	numa_node;
5615	/* GIC ITS ID */
5616	u32	its_id;
5617};
5618
5619static struct its_srat_map *its_srat_maps __initdata;
5620static int its_in_srat __initdata;
5621
5622static int __init acpi_get_its_numa_node(u32 its_id)
5623{
5624	int i;
5625
5626	for (i = 0; i < its_in_srat; i++) {
5627		if (its_id == its_srat_maps[i].its_id)
5628			return its_srat_maps[i].numa_node;
5629	}
5630	return NUMA_NO_NODE;
5631}
5632
5633static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5634					  const unsigned long end)
5635{
5636	return 0;
5637}
5638
5639static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5640			 const unsigned long end)
5641{
5642	int node;
5643	struct acpi_srat_gic_its_affinity *its_affinity;
5644
5645	its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5646	if (!its_affinity)
5647		return -EINVAL;
5648
5649	if (its_affinity->header.length < sizeof(*its_affinity)) {
5650		pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5651			its_affinity->header.length);
5652		return -EINVAL;
5653	}
5654
5655	/*
5656	 * Note that in theory a new proximity node could be created by this
5657	 * entry as it is an SRAT resource allocation structure.
5658	 * We do not currently support doing so.
5659	 */
5660	node = pxm_to_node(its_affinity->proximity_domain);
5661
5662	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5663		pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5664		return 0;
5665	}
5666
5667	its_srat_maps[its_in_srat].numa_node = node;
5668	its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5669	its_in_srat++;
5670	pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5671		its_affinity->proximity_domain, its_affinity->its_id, node);
5672
5673	return 0;
5674}
5675
5676static void __init acpi_table_parse_srat_its(void)
5677{
5678	int count;
5679
5680	count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5681			sizeof(struct acpi_table_srat),
5682			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5683			gic_acpi_match_srat_its, 0);
5684	if (count <= 0)
5685		return;
5686
5687	its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5688				      GFP_KERNEL);
5689	if (!its_srat_maps)
5690		return;
5691
5692	acpi_table_parse_entries(ACPI_SIG_SRAT,
5693			sizeof(struct acpi_table_srat),
5694			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5695			gic_acpi_parse_srat_its, 0);
5696}
5697
5698/* free the its_srat_maps after ITS probing */
5699static void __init acpi_its_srat_maps_free(void)
5700{
5701	kfree(its_srat_maps);
5702}
5703#else
5704static void __init acpi_table_parse_srat_its(void)	{ }
5705static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
5706static void __init acpi_its_srat_maps_free(void) { }
5707#endif
5708
5709static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5710					  const unsigned long end)
5711{
5712	struct acpi_madt_generic_translator *its_entry;
5713	struct fwnode_handle *dom_handle;
5714	struct its_node *its;
5715	struct resource res;
5716	int err;
5717
5718	its_entry = (struct acpi_madt_generic_translator *)header;
5719	memset(&res, 0, sizeof(res));
5720	res.start = its_entry->base_address;
5721	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5722	res.flags = IORESOURCE_MEM;
5723
5724	dom_handle = irq_domain_alloc_fwnode(&res.start);
5725	if (!dom_handle) {
5726		pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5727		       &res.start);
5728		return -ENOMEM;
5729	}
5730
5731	err = iort_register_domain_token(its_entry->translation_id, res.start,
5732					 dom_handle);
5733	if (err) {
5734		pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5735		       &res.start, its_entry->translation_id);
5736		goto dom_err;
5737	}
5738
5739	its = its_node_init(&res, dom_handle,
5740			    acpi_get_its_numa_node(its_entry->translation_id));
5741	if (!its) {
5742		err = -ENOMEM;
5743		goto node_err;
5744	}
5745
5746	if (acpi_get_madt_revision() >= 7 &&
5747	    (its_entry->flags & ACPI_MADT_ITS_NON_COHERENT))
5748		its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
5749
5750	err = its_probe_one(its);
5751	if (!err)
5752		return 0;
5753
5754node_err:
5755	iort_deregister_domain_token(its_entry->translation_id);
5756dom_err:
5757	irq_domain_free_fwnode(dom_handle);
5758	return err;
5759}
5760
5761static int __init its_acpi_reset(union acpi_subtable_headers *header,
5762				 const unsigned long end)
5763{
5764	struct acpi_madt_generic_translator *its_entry;
5765	struct resource res;
5766
5767	its_entry = (struct acpi_madt_generic_translator *)header;
5768	res = (struct resource) {
5769		.start	= its_entry->base_address,
5770		.end	= its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5771		.flags	= IORESOURCE_MEM,
5772	};
5773
5774	return its_reset_one(&res);
5775}
5776
5777static void __init its_acpi_probe(void)
5778{
5779	acpi_table_parse_srat_its();
5780	/*
5781	 * Make sure *all* the ITS are reset before we probe any, as
5782	 * they may be sharing memory. If any of the ITS fails to
5783	 * reset, don't even try to go any further, as this could
5784	 * result in something even worse.
5785	 */
5786	if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5787				  its_acpi_reset, 0) > 0)
5788		acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5789				      gic_acpi_parse_madt_its, 0);
5790	acpi_its_srat_maps_free();
5791}
5792#else
5793static void __init its_acpi_probe(void) { }
5794#endif
5795
5796int __init its_lpi_memreserve_init(void)
5797{
5798	int state;
5799
5800	if (!efi_enabled(EFI_CONFIG_TABLES))
5801		return 0;
5802
5803	if (list_empty(&its_nodes))
5804		return 0;
5805
5806	gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5807	state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5808				  "irqchip/arm/gicv3/memreserve:online",
5809				  its_cpu_memreserve_lpi,
5810				  NULL);
5811	if (state < 0)
5812		return state;
5813
5814	gic_rdists->cpuhp_memreserve_state = state;
5815
5816	return 0;
5817}
5818
5819int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5820		    struct irq_domain *parent_domain, u8 irq_prio)
5821{
5822	struct device_node *of_node;
5823	struct its_node *its;
5824	bool has_v4 = false;
5825	bool has_v4_1 = false;
5826	int err;
5827
5828	itt_pool = gen_pool_create(get_order(ITS_ITT_ALIGN), -1);
5829	if (!itt_pool)
5830		return -ENOMEM;
5831
5832	gic_rdists = rdists;
5833
5834	lpi_prop_prio = irq_prio;
5835	its_parent = parent_domain;
5836	of_node = to_of_node(handle);
5837	if (of_node)
5838		its_of_probe(of_node);
5839	else
5840		its_acpi_probe();
5841
5842	if (list_empty(&its_nodes)) {
5843		pr_warn("ITS: No ITS available, not enabling LPIs\n");
5844		return -ENXIO;
5845	}
5846
5847	err = allocate_lpi_tables();
5848	if (err)
5849		return err;
5850
5851	list_for_each_entry(its, &its_nodes, entry) {
5852		has_v4 |= is_v4(its);
5853		has_v4_1 |= is_v4_1(its);
5854	}
5855
5856	/* Don't bother with inconsistent systems */
5857	if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5858		rdists->has_rvpeid = false;
5859
5860	if (has_v4 & rdists->has_vlpis) {
5861		const struct irq_domain_ops *sgi_ops;
5862
5863		if (has_v4_1)
5864			sgi_ops = &its_sgi_domain_ops;
5865		else
5866			sgi_ops = NULL;
5867
5868		if (its_init_vpe_domain() ||
5869		    its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5870			rdists->has_vlpis = false;
5871			pr_err("ITS: Disabling GICv4 support\n");
5872		}
5873	}
5874
5875	register_syscore(&its_syscore);
5876
5877	return 0;
5878}