vgic-mmio-v3.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * VGICv3 MMIO handling functions
   4 */
   5
   6#include <linux/bitfield.h>
   7#include <linux/irqchip/arm-gic-v3.h>
   8#include <linux/kvm.h>
   9#include <linux/kvm_host.h>
  10#include <linux/interrupt.h>
  11#include <kvm/iodev.h>
  12#include <kvm/arm_vgic.h>
  13
  14#include <asm/kvm_emulate.h>
  15#include <asm/kvm_arm.h>
  16#include <asm/kvm_mmu.h>
  17
  18#include "vgic.h"
  19#include "vgic-mmio.h"
  20
  21/* extract @num bytes at @offset bytes offset in data */
  22unsigned long extract_bytes(u64 data, unsigned int offset,
  23			    unsigned int num)
  24{
  25	return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0);
  26}
  27
  28/* allows updates of any half of a 64-bit register (or the whole thing) */
  29u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
  30		     unsigned long val)
  31{
  32	int lower = (offset & 4) * 8;
  33	int upper = lower + 8 * len - 1;
  34
  35	reg &= ~GENMASK_ULL(upper, lower);
  36	val &= GENMASK_ULL(len * 8 - 1, 0);
  37
  38	return reg | ((u64)val << lower);
  39}
  40
  41bool vgic_has_its(struct kvm *kvm)
  42{
  43	struct vgic_dist *dist = &kvm->arch.vgic;
  44
  45	if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
  46		return false;
  47
  48	return dist->has_its;
  49}
  50
  51bool vgic_supports_direct_msis(struct kvm *kvm)
  52{
  53	/*
  54	 * Deliberately conflate vLPI and vSGI support on GICv4.1 hardware,
  55	 * indirectly allowing userspace to control whether or not vPEs are
  56	 * allocated for the VM.
  57	 */
  58	if (system_supports_direct_sgis() && !vgic_supports_direct_sgis(kvm))
  59		return false;
  60
  61	return kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm);
  62}
  63
  64bool system_supports_direct_sgis(void)
  65{
  66	return kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi();
  67}
  68
  69bool vgic_supports_direct_sgis(struct kvm *kvm)
  70{
  71	return kvm->arch.vgic.nassgicap;
  72}
  73
  74/*
  75 * The Revision field in the IIDR have the following meanings:
  76 *
  77 * Revision 2: Interrupt groups are guest-configurable and signaled using
  78 * 	       their configured groups.
  79 */
  80
  81static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
  82					    gpa_t addr, unsigned int len)
  83{
  84	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
  85	u32 value = 0;
  86
  87	switch (addr & 0x0c) {
  88	case GICD_CTLR:
  89		if (vgic->enabled)
  90			value |= GICD_CTLR_ENABLE_SS_G1;
  91		value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
  92		if (vgic->nassgireq)
  93			value |= GICD_CTLR_nASSGIreq;
  94		break;
  95	case GICD_TYPER:
  96		value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS;
  97		value = (value >> 5) - 1;
  98		if (vgic_has_its(vcpu->kvm)) {
  99			value |= (INTERRUPT_ID_BITS_ITS - 1) << 19;
 100			value |= GICD_TYPER_LPIS;
 101		} else {
 102			value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19;
 103		}
 104		break;
 105	case GICD_TYPER2:
 106		if (vgic_supports_direct_sgis(vcpu->kvm))
 107			value = GICD_TYPER2_nASSGIcap;
 108		break;
 109	case GICD_IIDR:
 110		value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) |
 111			(vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) |
 112			(IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT);
 113		break;
 114	default:
 115		return 0;
 116	}
 117
 118	return value;
 119}
 120
 121static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
 122				    gpa_t addr, unsigned int len,
 123				    unsigned long val)
 124{
 125	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 126
 127	switch (addr & 0x0c) {
 128	case GICD_CTLR: {
 129		bool was_enabled, is_hwsgi;
 130
 131		mutex_lock(&vcpu->kvm->arch.config_lock);
 132
 133		was_enabled = dist->enabled;
 134		is_hwsgi = dist->nassgireq;
 135
 136		dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
 137
 138		/* Not a GICv4.1? No HW SGIs */
 139		if (!vgic_supports_direct_sgis(vcpu->kvm))
 140			val &= ~GICD_CTLR_nASSGIreq;
 141
 142		/* Dist stays enabled? nASSGIreq is RO */
 143		if (was_enabled && dist->enabled) {
 144			val &= ~GICD_CTLR_nASSGIreq;
 145			val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi);
 146		}
 147
 148		/* Switching HW SGIs? */
 149		dist->nassgireq = val & GICD_CTLR_nASSGIreq;
 150		if (is_hwsgi != dist->nassgireq)
 151			vgic_v4_configure_vsgis(vcpu->kvm);
 152
 153		if (vgic_supports_direct_sgis(vcpu->kvm) &&
 154		    was_enabled != dist->enabled)
 155			kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
 156		else if (!was_enabled && dist->enabled)
 157			vgic_kick_vcpus(vcpu->kvm);
 158
 159		mutex_unlock(&vcpu->kvm->arch.config_lock);
 160		break;
 161	}
 162	case GICD_TYPER:
 163	case GICD_TYPER2:
 164	case GICD_IIDR:
 165		/* This is at best for documentation purposes... */
 166		return;
 167	}
 168}
 169
 170static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
 171					   gpa_t addr, unsigned int len,
 172					   unsigned long val)
 173{
 174	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 175	u32 reg;
 176
 177	switch (addr & 0x0c) {
 178	case GICD_TYPER2:
 179		reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
 180
 181		if (reg == val)
 182			return 0;
 183		if (vgic_initialized(vcpu->kvm))
 184			return -EBUSY;
 185		if ((reg ^ val) & ~GICD_TYPER2_nASSGIcap)
 186			return -EINVAL;
 187		if (!system_supports_direct_sgis() && val)
 188			return -EINVAL;
 189
 190		dist->nassgicap = val & GICD_TYPER2_nASSGIcap;
 191		return 0;
 192	case GICD_IIDR:
 193		reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
 194		if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
 195			return -EINVAL;
 196
 197		reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
 198		switch (reg) {
 199		case KVM_VGIC_IMP_REV_2:
 200		case KVM_VGIC_IMP_REV_3:
 201			dist->implementation_rev = reg;
 202			return 0;
 203		default:
 204			return -EINVAL;
 205		}
 206	case GICD_CTLR:
 207		/* Not a GICv4.1? No HW SGIs */
 208		if (!vgic_supports_direct_sgis(vcpu->kvm))
 209			val &= ~GICD_CTLR_nASSGIreq;
 210
 211		dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
 212		dist->nassgireq = val & GICD_CTLR_nASSGIreq;
 213		return 0;
 214	}
 215
 216	vgic_mmio_write_v3_misc(vcpu, addr, len, val);
 217	return 0;
 218}
 219
 220static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu,
 221					    gpa_t addr, unsigned int len)
 222{
 223	int intid = VGIC_ADDR_TO_INTID(addr, 64);
 224	struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, intid);
 225	unsigned long ret = 0;
 226
 227	if (!irq)
 228		return 0;
 229
 230	/* The upper word is RAZ for us. */
 231	if (!(addr & 4))
 232		ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len);
 233
 234	vgic_put_irq(vcpu->kvm, irq);
 235	return ret;
 236}
 237
 238static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
 239				    gpa_t addr, unsigned int len,
 240				    unsigned long val)
 241{
 242	int intid = VGIC_ADDR_TO_INTID(addr, 64);
 243	struct vgic_irq *irq;
 244	unsigned long flags;
 245
 246	/* The upper word is WI for us since we don't implement Aff3. */
 247	if (addr & 4)
 248		return;
 249
 250	irq = vgic_get_irq(vcpu->kvm, intid);
 251
 252	if (!irq)
 253		return;
 254
 255	raw_spin_lock_irqsave(&irq->irq_lock, flags);
 256
 257	/* We only care about and preserve Aff0, Aff1 and Aff2. */
 258	irq->mpidr = val & GENMASK(23, 0);
 259	irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
 260
 261	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 262	vgic_put_irq(vcpu->kvm, irq);
 263}
 264
 265bool vgic_lpis_enabled(struct kvm_vcpu *vcpu)
 266{
 267	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 268
 269	return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS;
 270}
 271
 272static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
 273					     gpa_t addr, unsigned int len)
 274{
 275	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 276	unsigned long val;
 277
 278	val = atomic_read(&vgic_cpu->ctlr);
 279	if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3)
 280		val |= GICR_CTLR_IR | GICR_CTLR_CES;
 281
 282	return val;
 283}
 284
 285static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
 286				     gpa_t addr, unsigned int len,
 287				     unsigned long val)
 288{
 289	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 290	u32 ctlr;
 291
 292	if (!vgic_has_its(vcpu->kvm))
 293		return;
 294
 295	if (!(val & GICR_CTLR_ENABLE_LPIS)) {
 296		/*
 297		 * Don't disable if RWP is set, as there already an
 298		 * ongoing disable. Funky guest...
 299		 */
 300		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr,
 301					      GICR_CTLR_ENABLE_LPIS,
 302					      GICR_CTLR_RWP);
 303		if (ctlr != GICR_CTLR_ENABLE_LPIS)
 304			return;
 305
 306		vgic_flush_pending_lpis(vcpu);
 307		vgic_its_invalidate_all_caches(vcpu->kvm);
 308		atomic_set_release(&vgic_cpu->ctlr, 0);
 309	} else {
 310		ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
 311					      GICR_CTLR_ENABLE_LPIS);
 312		if (ctlr != 0)
 313			return;
 314
 315		vgic_enable_lpis(vcpu);
 316	}
 317}
 318
 319static bool vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu *vcpu)
 320{
 321	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
 322	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 323	struct vgic_redist_region *iter, *rdreg = vgic_cpu->rdreg;
 324
 325	if (!rdreg)
 326		return false;
 327
 328	if (vgic_cpu->rdreg_index < rdreg->free_index - 1) {
 329		return false;
 330	} else if (rdreg->count && vgic_cpu->rdreg_index == (rdreg->count - 1)) {
 331		struct list_head *rd_regions = &vgic->rd_regions;
 332		gpa_t end = rdreg->base + rdreg->count * KVM_VGIC_V3_REDIST_SIZE;
 333
 334		/*
 335		 * the rdist is the last one of the redist region,
 336		 * check whether there is no other contiguous rdist region
 337		 */
 338		list_for_each_entry(iter, rd_regions, list) {
 339			if (iter->base == end && iter->free_index > 0)
 340				return false;
 341		}
 342	}
 343	return true;
 344}
 345
 346static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
 347					      gpa_t addr, unsigned int len)
 348{
 349	unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
 350	int target_vcpu_id = vcpu->vcpu_id;
 351	u64 value;
 352
 353	value = (u64)(mpidr & GENMASK(23, 0)) << 32;
 354	value |= ((target_vcpu_id & 0xffff) << 8);
 355
 356	if (vgic_has_its(vcpu->kvm))
 357		value |= GICR_TYPER_PLPIS;
 358
 359	if (vgic_mmio_vcpu_rdist_is_last(vcpu))
 360		value |= GICR_TYPER_LAST;
 361
 362	return extract_bytes(value, addr & 7, len);
 363}
 364
 365static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu,
 366					     gpa_t addr, unsigned int len)
 367{
 368	return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
 369}
 370
 371static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu,
 372					      gpa_t addr, unsigned int len)
 373{
 374	switch (addr & 0xffff) {
 375	case GICD_PIDR2:
 376		/* report a GICv3 compliant implementation */
 377		return 0x3b;
 378	}
 379
 380	return 0;
 381}
 382
 383static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu,
 384					 gpa_t addr, unsigned int len,
 385					 unsigned long val)
 386{
 387	int ret;
 388
 389	ret = vgic_uaccess_write_spending(vcpu, addr, len, val);
 390	if (ret)
 391		return ret;
 392
 393	return vgic_uaccess_write_cpending(vcpu, addr, len, ~val);
 394}
 395
 396/* We want to avoid outer shareable. */
 397u64 vgic_sanitise_shareability(u64 field)
 398{
 399	switch (field) {
 400	case GIC_BASER_OuterShareable:
 401		return GIC_BASER_InnerShareable;
 402	default:
 403		return field;
 404	}
 405}
 406
 407/* Avoid any inner non-cacheable mapping. */
 408u64 vgic_sanitise_inner_cacheability(u64 field)
 409{
 410	switch (field) {
 411	case GIC_BASER_CACHE_nCnB:
 412	case GIC_BASER_CACHE_nC:
 413		return GIC_BASER_CACHE_RaWb;
 414	default:
 415		return field;
 416	}
 417}
 418
 419/* Non-cacheable or same-as-inner are OK. */
 420u64 vgic_sanitise_outer_cacheability(u64 field)
 421{
 422	switch (field) {
 423	case GIC_BASER_CACHE_SameAsInner:
 424	case GIC_BASER_CACHE_nC:
 425		return field;
 426	default:
 427		return GIC_BASER_CACHE_SameAsInner;
 428	}
 429}
 430
 431u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift,
 432			u64 (*sanitise_fn)(u64))
 433{
 434	u64 field = (reg & field_mask) >> field_shift;
 435
 436	field = sanitise_fn(field) << field_shift;
 437	return (reg & ~field_mask) | field;
 438}
 439
 440#define PROPBASER_RES0_MASK						\
 441	(GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5))
 442#define PENDBASER_RES0_MASK						\
 443	(BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) |	\
 444	 GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0))
 445
 446static u64 vgic_sanitise_pendbaser(u64 reg)
 447{
 448	reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK,
 449				  GICR_PENDBASER_SHAREABILITY_SHIFT,
 450				  vgic_sanitise_shareability);
 451	reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK,
 452				  GICR_PENDBASER_INNER_CACHEABILITY_SHIFT,
 453				  vgic_sanitise_inner_cacheability);
 454	reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK,
 455				  GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT,
 456				  vgic_sanitise_outer_cacheability);
 457
 458	reg &= ~PENDBASER_RES0_MASK;
 459
 460	return reg;
 461}
 462
 463static u64 vgic_sanitise_propbaser(u64 reg)
 464{
 465	reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK,
 466				  GICR_PROPBASER_SHAREABILITY_SHIFT,
 467				  vgic_sanitise_shareability);
 468	reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK,
 469				  GICR_PROPBASER_INNER_CACHEABILITY_SHIFT,
 470				  vgic_sanitise_inner_cacheability);
 471	reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK,
 472				  GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT,
 473				  vgic_sanitise_outer_cacheability);
 474
 475	reg &= ~PROPBASER_RES0_MASK;
 476	return reg;
 477}
 478
 479static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu,
 480					     gpa_t addr, unsigned int len)
 481{
 482	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 483
 484	return extract_bytes(dist->propbaser, addr & 7, len);
 485}
 486
 487static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu,
 488				     gpa_t addr, unsigned int len,
 489				     unsigned long val)
 490{
 491	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 492	u64 old_propbaser, propbaser;
 493
 494	/* Storing a value with LPIs already enabled is undefined */
 495	if (vgic_lpis_enabled(vcpu))
 496		return;
 497
 498	do {
 499		old_propbaser = READ_ONCE(dist->propbaser);
 500		propbaser = old_propbaser;
 501		propbaser = update_64bit_reg(propbaser, addr & 4, len, val);
 502		propbaser = vgic_sanitise_propbaser(propbaser);
 503	} while (cmpxchg64(&dist->propbaser, old_propbaser,
 504			   propbaser) != old_propbaser);
 505}
 506
 507static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu,
 508					     gpa_t addr, unsigned int len)
 509{
 510	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 511	u64 value = vgic_cpu->pendbaser;
 512
 513	value &= ~GICR_PENDBASER_PTZ;
 514
 515	return extract_bytes(value, addr & 7, len);
 516}
 517
 518static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
 519				     gpa_t addr, unsigned int len,
 520				     unsigned long val)
 521{
 522	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 523	u64 old_pendbaser, pendbaser;
 524
 525	/* Storing a value with LPIs already enabled is undefined */
 526	if (vgic_lpis_enabled(vcpu))
 527		return;
 528
 529	do {
 530		old_pendbaser = READ_ONCE(vgic_cpu->pendbaser);
 531		pendbaser = old_pendbaser;
 532		pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val);
 533		pendbaser = vgic_sanitise_pendbaser(pendbaser);
 534	} while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser,
 535			   pendbaser) != old_pendbaser);
 536}
 537
 538static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,
 539					 gpa_t addr, unsigned int len)
 540{
 541	return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);
 542}
 543
 544static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)
 545{
 546	if (busy) {
 547		atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);
 548		smp_mb__after_atomic();
 549	} else {
 550		smp_mb__before_atomic();
 551		atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);
 552	}
 553}
 554
 555static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,
 556				   gpa_t addr, unsigned int len,
 557				   unsigned long val)
 558{
 559	struct vgic_irq *irq;
 560	u32 intid;
 561
 562	/*
 563	 * If the guest wrote only to the upper 32bit part of the
 564	 * register, drop the write on the floor, as it is only for
 565	 * vPEs (which we don't support for obvious reasons).
 566	 *
 567	 * Also discard the access if LPIs are not enabled.
 568	 */
 569	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
 570		return;
 571
 572	intid = lower_32_bits(val);
 573	if (intid < VGIC_MIN_LPI)
 574		return;
 575
 576	vgic_set_rdist_busy(vcpu, true);
 577
 578	irq = vgic_get_irq(vcpu->kvm, intid);
 579	if (irq) {
 580		vgic_its_inv_lpi(vcpu->kvm, irq);
 581		vgic_put_irq(vcpu->kvm, irq);
 582	}
 583
 584	vgic_set_rdist_busy(vcpu, false);
 585}
 586
 587static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,
 588				   gpa_t addr, unsigned int len,
 589				   unsigned long val)
 590{
 591	/* See vgic_mmio_write_invlpi() for the early return rationale */
 592	if ((addr & 4) || !vgic_lpis_enabled(vcpu))
 593		return;
 594
 595	vgic_set_rdist_busy(vcpu, true);
 596	vgic_its_invall(vcpu);
 597	vgic_set_rdist_busy(vcpu, false);
 598}
 599
 600/*
 601 * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
 602 * redistributors, while SPIs are covered by registers in the distributor
 603 * block. Trying to set private IRQs in this block gets ignored.
 604 * We take some special care here to fix the calculation of the register
 605 * offset.
 606 */
 607#define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \
 608	{								\
 609		.reg_offset = off,					\
 610		.bits_per_irq = bpi,					\
 611		.len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8,		\
 612		.access_flags = acc,					\
 613		.read = vgic_mmio_read_raz,				\
 614		.write = vgic_mmio_write_wi,				\
 615	}, {								\
 616		.reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8,	\
 617		.bits_per_irq = bpi,					\
 618		.len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8,	\
 619		.access_flags = acc,					\
 620		.read = rd,						\
 621		.write = wr,						\
 622		.uaccess_read = ur,					\
 623		.uaccess_write = uw,					\
 624	}
 625
 626static const struct vgic_register_region vgic_v3_dist_registers[] = {
 627	REGISTER_DESC_WITH_LENGTH_UACCESS(GICD_CTLR,
 628		vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc,
 629		NULL, vgic_mmio_uaccess_write_v3_misc,
 630		16, VGIC_ACCESS_32bit),
 631	REGISTER_DESC_WITH_LENGTH(GICD_STATUSR,
 632		vgic_mmio_read_rao, vgic_mmio_write_wi, 4,
 633		VGIC_ACCESS_32bit),
 634	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR,
 635		vgic_mmio_read_group, vgic_mmio_write_group, NULL, NULL, 1,
 636		VGIC_ACCESS_32bit),
 637	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER,
 638		vgic_mmio_read_enable, vgic_mmio_write_senable,
 639		NULL, vgic_uaccess_write_senable, 1,
 640		VGIC_ACCESS_32bit),
 641	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER,
 642		vgic_mmio_read_enable, vgic_mmio_write_cenable,
 643	       NULL, vgic_uaccess_write_cenable, 1,
 644		VGIC_ACCESS_32bit),
 645	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR,
 646		vgic_mmio_read_pending, vgic_mmio_write_spending,
 647		vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1,
 648		VGIC_ACCESS_32bit),
 649	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR,
 650		vgic_mmio_read_pending, vgic_mmio_write_cpending,
 651		vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 1,
 652		VGIC_ACCESS_32bit),
 653	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER,
 654		vgic_mmio_read_active, vgic_mmio_write_sactive,
 655		vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1,
 656		VGIC_ACCESS_32bit),
 657	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER,
 658		vgic_mmio_read_active, vgic_mmio_write_cactive,
 659		vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive,
 660		1, VGIC_ACCESS_32bit),
 661	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR,
 662		vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL,
 663		8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
 664	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR,
 665		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8,
 666		VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
 667	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR,
 668		vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2,
 669		VGIC_ACCESS_32bit),
 670	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR,
 671		vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1,
 672		VGIC_ACCESS_32bit),
 673	REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER,
 674		vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64,
 675		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
 676	REGISTER_DESC_WITH_LENGTH(GICD_IDREGS,
 677		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
 678		VGIC_ACCESS_32bit),
 679};
 680
 681static const struct vgic_register_region vgic_v3_rd_registers[] = {
 682	/* RD_base registers */
 683	REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
 684		vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
 685		VGIC_ACCESS_32bit),
 686	REGISTER_DESC_WITH_LENGTH(GICR_STATUSR,
 687		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
 688		VGIC_ACCESS_32bit),
 689	REGISTER_DESC_WITH_LENGTH(GICR_IIDR,
 690		vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4,
 691		VGIC_ACCESS_32bit),
 692	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_TYPER,
 693		vgic_mmio_read_v3r_typer, vgic_mmio_write_wi,
 694		NULL, vgic_mmio_uaccess_write_wi, 8,
 695		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
 696	REGISTER_DESC_WITH_LENGTH(GICR_WAKER,
 697		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
 698		VGIC_ACCESS_32bit),
 699	REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER,
 700		vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8,
 701		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
 702	REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
 703		vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
 704		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
 705	REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,
 706		vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,
 707		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
 708	REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,
 709		vgic_mmio_read_raz, vgic_mmio_write_invall, 8,
 710		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
 711	REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,
 712		vgic_mmio_read_sync, vgic_mmio_write_wi, 4,
 713		VGIC_ACCESS_32bit),
 714	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
 715		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
 716		VGIC_ACCESS_32bit),
 717	/* SGI_base registers */
 718	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
 719		vgic_mmio_read_group, vgic_mmio_write_group, 4,
 720		VGIC_ACCESS_32bit),
 721	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISENABLER0,
 722		vgic_mmio_read_enable, vgic_mmio_write_senable,
 723		NULL, vgic_uaccess_write_senable, 4,
 724		VGIC_ACCESS_32bit),
 725	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICENABLER0,
 726		vgic_mmio_read_enable, vgic_mmio_write_cenable,
 727		NULL, vgic_uaccess_write_cenable, 4,
 728		VGIC_ACCESS_32bit),
 729	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
 730		vgic_mmio_read_pending, vgic_mmio_write_spending,
 731		vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
 732		VGIC_ACCESS_32bit),
 733	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
 734		vgic_mmio_read_pending, vgic_mmio_write_cpending,
 735		vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
 736		VGIC_ACCESS_32bit),
 737	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
 738		vgic_mmio_read_active, vgic_mmio_write_sactive,
 739		vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 4,
 740		VGIC_ACCESS_32bit),
 741	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
 742		vgic_mmio_read_active, vgic_mmio_write_cactive,
 743		vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 4,
 744		VGIC_ACCESS_32bit),
 745	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
 746		vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
 747		VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
 748	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
 749		vgic_mmio_read_config, vgic_mmio_write_config, 8,
 750		VGIC_ACCESS_32bit),
 751	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
 752		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
 753		VGIC_ACCESS_32bit),
 754	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
 755		vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
 756		VGIC_ACCESS_32bit),
 757};
 758
 759unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev)
 760{
 761	dev->regions = vgic_v3_dist_registers;
 762	dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
 763
 764	kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
 765
 766	return SZ_64K;
 767}
 768
 769/**
 770 * vgic_register_redist_iodev - register a single redist iodev
 771 * @vcpu:    The VCPU to which the redistributor belongs
 772 *
 773 * Register a KVM iodev for this VCPU's redistributor using the address
 774 * provided.
 775 *
 776 * Return 0 on success, -ERRNO otherwise.
 777 */
 778int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
 779{
 780	struct kvm *kvm = vcpu->kvm;
 781	struct vgic_dist *vgic = &kvm->arch.vgic;
 782	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
 783	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
 784	struct vgic_redist_region *rdreg;
 785	gpa_t rd_base;
 786	int ret = 0;
 787
 788	lockdep_assert_held(&kvm->slots_lock);
 789	mutex_lock(&kvm->arch.config_lock);
 790
 791	if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
 792		goto out_unlock;
 793
 794	/*
 795	 * We may be creating VCPUs before having set the base address for the
 796	 * redistributor region, in which case we will come back to this
 797	 * function for all VCPUs when the base address is set.  Just return
 798	 * without doing any work for now.
 799	 */
 800	rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions);
 801	if (!rdreg)
 802		goto out_unlock;
 803
 804	if (!vgic_v3_check_base(kvm)) {
 805		ret = -EINVAL;
 806		goto out_unlock;
 807	}
 808
 809	vgic_cpu->rdreg = rdreg;
 810	vgic_cpu->rdreg_index = rdreg->free_index;
 811
 812	rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
 813
 814	kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
 815	rd_dev->base_addr = rd_base;
 816	rd_dev->iodev_type = IODEV_REDIST;
 817	rd_dev->regions = vgic_v3_rd_registers;
 818	rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
 819	rd_dev->redist_vcpu = vcpu;
 820
 821	mutex_unlock(&kvm->arch.config_lock);
 822
 823	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
 824				      2 * SZ_64K, &rd_dev->dev);
 825	if (ret)
 826		return ret;
 827
 828	/* Protected by slots_lock */
 829	rdreg->free_index++;
 830	return 0;
 831
 832out_unlock:
 833	mutex_unlock(&kvm->arch.config_lock);
 834	return ret;
 835}
 836
 837void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
 838{
 839	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
 840
 841	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
 842}
 843
 844static int vgic_register_all_redist_iodevs(struct kvm *kvm)
 845{
 846	struct kvm_vcpu *vcpu;
 847	unsigned long c;
 848	int ret = 0;
 849
 850	lockdep_assert_held(&kvm->slots_lock);
 851
 852	kvm_for_each_vcpu(c, vcpu, kvm) {
 853		ret = vgic_register_redist_iodev(vcpu);
 854		if (ret)
 855			break;
 856	}
 857
 858	if (ret) {
 859		/* The current c failed, so iterate over the previous ones. */
 860		int i;
 861
 862		for (i = 0; i < c; i++) {
 863			vcpu = kvm_get_vcpu(kvm, i);
 864			vgic_unregister_redist_iodev(vcpu);
 865		}
 866	}
 867
 868	return ret;
 869}
 870
 871/**
 872 * vgic_v3_alloc_redist_region - Allocate a new redistributor region
 873 *
 874 * Performs various checks before inserting the rdist region in the list.
 875 * Those tests depend on whether the size of the rdist region is known
 876 * (ie. count != 0). The list is sorted by rdist region index.
 877 *
 878 * @kvm: kvm handle
 879 * @index: redist region index
 880 * @base: base of the new rdist region
 881 * @count: number of redistributors the region is made of (0 in the old style
 882 * single region, whose size is induced from the number of vcpus)
 883 *
 884 * Return 0 on success, < 0 otherwise
 885 */
 886static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index,
 887				       gpa_t base, uint32_t count)
 888{
 889	struct vgic_dist *d = &kvm->arch.vgic;
 890	struct vgic_redist_region *rdreg;
 891	struct list_head *rd_regions = &d->rd_regions;
 892	int nr_vcpus = atomic_read(&kvm->online_vcpus);
 893	size_t size = count ? count * KVM_VGIC_V3_REDIST_SIZE
 894			    : nr_vcpus * KVM_VGIC_V3_REDIST_SIZE;
 895	int ret;
 896
 897	/* cross the end of memory ? */
 898	if (base + size < base)
 899		return -EINVAL;
 900
 901	if (list_empty(rd_regions)) {
 902		if (index != 0)
 903			return -EINVAL;
 904	} else {
 905		rdreg = list_last_entry(rd_regions,
 906					struct vgic_redist_region, list);
 907
 908		/* Don't mix single region and discrete redist regions */
 909		if (!count && rdreg->count)
 910			return -EINVAL;
 911
 912		if (!count)
 913			return -EEXIST;
 914
 915		if (index != rdreg->index + 1)
 916			return -EINVAL;
 917	}
 918
 919	/*
 920	 * For legacy single-region redistributor regions (!count),
 921	 * check that the redistributor region does not overlap with the
 922	 * distributor's address space.
 923	 */
 924	if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&
 925		vgic_dist_overlap(kvm, base, size))
 926		return -EINVAL;
 927
 928	/* collision with any other rdist region? */
 929	if (vgic_v3_rdist_overlap(kvm, base, size))
 930		return -EINVAL;
 931
 932	rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL_ACCOUNT);
 933	if (!rdreg)
 934		return -ENOMEM;
 935
 936	rdreg->base = VGIC_ADDR_UNDEF;
 937
 938	ret = vgic_check_iorange(kvm, rdreg->base, base, SZ_64K, size);
 939	if (ret)
 940		goto free;
 941
 942	rdreg->base = base;
 943	rdreg->count = count;
 944	rdreg->free_index = 0;
 945	rdreg->index = index;
 946
 947	list_add_tail(&rdreg->list, rd_regions);
 948	return 0;
 949free:
 950	kfree(rdreg);
 951	return ret;
 952}
 953
 954void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdreg)
 955{
 956	struct kvm_vcpu *vcpu;
 957	unsigned long c;
 958
 959	lockdep_assert_held(&kvm->arch.config_lock);
 960
 961	/* Garbage collect the region */
 962	kvm_for_each_vcpu(c, vcpu, kvm) {
 963		if (vcpu->arch.vgic_cpu.rdreg == rdreg)
 964			vcpu->arch.vgic_cpu.rdreg = NULL;
 965	}
 966
 967	list_del(&rdreg->list);
 968	kfree(rdreg);
 969}
 970
 971int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
 972{
 973	int ret;
 974
 975	mutex_lock(&kvm->arch.config_lock);
 976	ret = vgic_v3_alloc_redist_region(kvm, index, addr, count);
 977	mutex_unlock(&kvm->arch.config_lock);
 978	if (ret)
 979		return ret;
 980
 981	/*
 982	 * Register iodevs for each existing VCPU.  Adding more VCPUs
 983	 * afterwards will register the iodevs when needed.
 984	 */
 985	ret = vgic_register_all_redist_iodevs(kvm);
 986	if (ret) {
 987		struct vgic_redist_region *rdreg;
 988
 989		mutex_lock(&kvm->arch.config_lock);
 990		rdreg = vgic_v3_rdist_region_from_index(kvm, index);
 991		vgic_v3_free_redist_region(kvm, rdreg);
 992		mutex_unlock(&kvm->arch.config_lock);
 993		return ret;
 994	}
 995
 996	return 0;
 997}
 998
 999int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
1000{
1001	const struct vgic_register_region *region;
1002	struct vgic_io_device iodev;
1003	struct vgic_reg_attr reg_attr;
1004	struct kvm_vcpu *vcpu;
1005	gpa_t addr;
1006	int ret;
1007
1008	ret = vgic_v3_parse_attr(dev, attr, &reg_attr);
1009	if (ret)
1010		return ret;
1011
1012	vcpu = reg_attr.vcpu;
1013	addr = reg_attr.addr;
1014
1015	switch (attr->group) {
1016	case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
1017		iodev.regions = vgic_v3_dist_registers;
1018		iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
1019		iodev.base_addr = 0;
1020		break;
1021	case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
1022		iodev.regions = vgic_v3_rd_registers;
1023		iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
1024		iodev.base_addr = 0;
1025		break;
1026	}
1027	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
1028		return vgic_v3_has_cpu_sysregs_attr(vcpu, attr);
1029	default:
1030		return -ENXIO;
1031	}
1032
1033	/* We only support aligned 32-bit accesses. */
1034	if (addr & 3)
1035		return -ENXIO;
1036
1037	region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32));
1038	if (!region)
1039		return -ENXIO;
1040
1041	return 0;
1042}
1043
1044/*
1045 * The ICC_SGI* registers encode the affinity differently from the MPIDR,
1046 * so provide a wrapper to use the existing defines to isolate a certain
1047 * affinity level.
1048 */
1049#define SGI_AFFINITY_LEVEL(reg, level) \
1050	((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
1051	>> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
1052
1053static void vgic_v3_queue_sgi(struct kvm_vcpu *vcpu, u32 sgi, bool allow_group1)
1054{
1055	struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, sgi);
1056	unsigned long flags;
1057
1058	raw_spin_lock_irqsave(&irq->irq_lock, flags);
1059
1060	/*
1061	 * An access targeting Group0 SGIs can only generate
1062	 * those, while an access targeting Group1 SGIs can
1063	 * generate interrupts of either group.
1064	 */
1065	if (!irq->group || allow_group1) {
1066		if (!irq->hw) {
1067			irq->pending_latch = true;
1068			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
1069		} else {
1070			/* HW SGI? Ask the GIC to inject it */
1071			int err;
1072			err = irq_set_irqchip_state(irq->host_irq,
1073						    IRQCHIP_STATE_PENDING,
1074						    true);
1075			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
1076			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1077		}
1078	} else {
1079		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1080	}
1081
1082	vgic_put_irq(vcpu->kvm, irq);
1083}
1084
1085/**
1086 * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
1087 * @vcpu: The VCPU requesting a SGI
1088 * @reg: The value written into ICC_{ASGI1,SGI0,SGI1}R by that VCPU
1089 * @allow_group1: Does the sysreg access allow generation of G1 SGIs
1090 *
1091 * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
1092 * This will trap in sys_regs.c and call this function.
1093 * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
1094 * target processors as well as a bitmask of 16 Aff0 CPUs.
1095 *
1096 * If the interrupt routing mode bit is not set, we iterate over the Aff0
1097 * bits and signal the VCPUs matching the provided Aff{3,2,1}.
1098 *
1099 * If this bit is set, we signal all, but not the calling VCPU.
1100 */
1101void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1)
1102{
1103	struct kvm *kvm = vcpu->kvm;
1104	struct kvm_vcpu *c_vcpu;
1105	unsigned long target_cpus;
1106	u64 mpidr;
1107	u32 sgi, aff0;
1108	unsigned long c;
1109
1110	sgi = FIELD_GET(ICC_SGI1R_SGI_ID_MASK, reg);
1111
1112	/* Broadcast */
1113	if (unlikely(reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT))) {
1114		kvm_for_each_vcpu(c, c_vcpu, kvm) {
1115			/* Don't signal the calling VCPU */
1116			if (c_vcpu == vcpu)
1117				continue;
1118
1119			vgic_v3_queue_sgi(c_vcpu, sgi, allow_group1);
1120		}
1121
1122		return;
1123	}
1124
1125	/* We iterate over affinities to find the corresponding vcpus */
1126	mpidr = SGI_AFFINITY_LEVEL(reg, 3);
1127	mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
1128	mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
1129	target_cpus = FIELD_GET(ICC_SGI1R_TARGET_LIST_MASK, reg);
1130
1131	for_each_set_bit(aff0, &target_cpus, hweight_long(ICC_SGI1R_TARGET_LIST_MASK)) {
1132		c_vcpu = kvm_mpidr_to_vcpu(kvm, mpidr | aff0);
1133		if (c_vcpu)
1134			vgic_v3_queue_sgi(c_vcpu, sgi, allow_group1);
1135	}
1136}
1137
1138int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1139			 int offset, u32 *val)
1140{
1141	struct vgic_io_device dev = {
1142		.regions = vgic_v3_dist_registers,
1143		.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers),
1144	};
1145
1146	return vgic_uaccess(vcpu, &dev, is_write, offset, val);
1147}
1148
1149int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1150			   int offset, u32 *val)
1151{
1152	struct vgic_io_device rd_dev = {
1153		.regions = vgic_v3_rd_registers,
1154		.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
1155	};
1156
1157	return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
1158}
1159
1160int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1161				    u32 intid, u32 *val)
1162{
1163	if (intid % 32)
1164		return -EINVAL;
1165
1166	if (is_write)
1167		vgic_write_irq_line_level_info(vcpu, intid, *val);
1168	else
1169		*val = vgic_read_irq_line_level_info(vcpu, intid);
1170
1171	return 0;
1172}