vgic-v3-sr.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2012-2015 - ARM Ltd
   4 * Author: Marc Zyngier <marc.zyngier@arm.com>
   5 */
   6
   7#include <hyp/adjust_pc.h>
   8
   9#include <linux/compiler.h>
  10#include <linux/irqchip/arm-gic-v3.h>
  11#include <linux/kvm_host.h>
  12
  13#include <asm/kvm_emulate.h>
  14#include <asm/kvm_hyp.h>
  15#include <asm/kvm_mmu.h>
  16
  17#include "../../vgic/vgic.h"
  18
  19#define vtr_to_max_lr_idx(v)		((v) & 0xf)
  20#define vtr_to_nr_pre_bits(v)		((((u32)(v) >> 26) & 7) + 1)
  21#define vtr_to_nr_apr_regs(v)		(1 << (vtr_to_nr_pre_bits(v) - 5))
  22
  23u64 __gic_v3_get_lr(unsigned int lr)
  24{
  25	switch (lr & 0xf) {
  26	case 0:
  27		return read_gicreg(ICH_LR0_EL2);
  28	case 1:
  29		return read_gicreg(ICH_LR1_EL2);
  30	case 2:
  31		return read_gicreg(ICH_LR2_EL2);
  32	case 3:
  33		return read_gicreg(ICH_LR3_EL2);
  34	case 4:
  35		return read_gicreg(ICH_LR4_EL2);
  36	case 5:
  37		return read_gicreg(ICH_LR5_EL2);
  38	case 6:
  39		return read_gicreg(ICH_LR6_EL2);
  40	case 7:
  41		return read_gicreg(ICH_LR7_EL2);
  42	case 8:
  43		return read_gicreg(ICH_LR8_EL2);
  44	case 9:
  45		return read_gicreg(ICH_LR9_EL2);
  46	case 10:
  47		return read_gicreg(ICH_LR10_EL2);
  48	case 11:
  49		return read_gicreg(ICH_LR11_EL2);
  50	case 12:
  51		return read_gicreg(ICH_LR12_EL2);
  52	case 13:
  53		return read_gicreg(ICH_LR13_EL2);
  54	case 14:
  55		return read_gicreg(ICH_LR14_EL2);
  56	case 15:
  57		return read_gicreg(ICH_LR15_EL2);
  58	}
  59
  60	unreachable();
  61}
  62
  63void __gic_v3_set_lr(u64 val, int lr)
  64{
  65	switch (lr & 0xf) {
  66	case 0:
  67		write_gicreg(val, ICH_LR0_EL2);
  68		break;
  69	case 1:
  70		write_gicreg(val, ICH_LR1_EL2);
  71		break;
  72	case 2:
  73		write_gicreg(val, ICH_LR2_EL2);
  74		break;
  75	case 3:
  76		write_gicreg(val, ICH_LR3_EL2);
  77		break;
  78	case 4:
  79		write_gicreg(val, ICH_LR4_EL2);
  80		break;
  81	case 5:
  82		write_gicreg(val, ICH_LR5_EL2);
  83		break;
  84	case 6:
  85		write_gicreg(val, ICH_LR6_EL2);
  86		break;
  87	case 7:
  88		write_gicreg(val, ICH_LR7_EL2);
  89		break;
  90	case 8:
  91		write_gicreg(val, ICH_LR8_EL2);
  92		break;
  93	case 9:
  94		write_gicreg(val, ICH_LR9_EL2);
  95		break;
  96	case 10:
  97		write_gicreg(val, ICH_LR10_EL2);
  98		break;
  99	case 11:
 100		write_gicreg(val, ICH_LR11_EL2);
 101		break;
 102	case 12:
 103		write_gicreg(val, ICH_LR12_EL2);
 104		break;
 105	case 13:
 106		write_gicreg(val, ICH_LR13_EL2);
 107		break;
 108	case 14:
 109		write_gicreg(val, ICH_LR14_EL2);
 110		break;
 111	case 15:
 112		write_gicreg(val, ICH_LR15_EL2);
 113		break;
 114	}
 115}
 116
 117static void __vgic_v3_write_ap0rn(u32 val, int n)
 118{
 119	switch (n) {
 120	case 0:
 121		write_gicreg(val, ICH_AP0R0_EL2);
 122		break;
 123	case 1:
 124		write_gicreg(val, ICH_AP0R1_EL2);
 125		break;
 126	case 2:
 127		write_gicreg(val, ICH_AP0R2_EL2);
 128		break;
 129	case 3:
 130		write_gicreg(val, ICH_AP0R3_EL2);
 131		break;
 132	}
 133}
 134
 135static void __vgic_v3_write_ap1rn(u32 val, int n)
 136{
 137	switch (n) {
 138	case 0:
 139		write_gicreg(val, ICH_AP1R0_EL2);
 140		break;
 141	case 1:
 142		write_gicreg(val, ICH_AP1R1_EL2);
 143		break;
 144	case 2:
 145		write_gicreg(val, ICH_AP1R2_EL2);
 146		break;
 147	case 3:
 148		write_gicreg(val, ICH_AP1R3_EL2);
 149		break;
 150	}
 151}
 152
 153static u32 __vgic_v3_read_ap0rn(int n)
 154{
 155	u32 val;
 156
 157	switch (n) {
 158	case 0:
 159		val = read_gicreg(ICH_AP0R0_EL2);
 160		break;
 161	case 1:
 162		val = read_gicreg(ICH_AP0R1_EL2);
 163		break;
 164	case 2:
 165		val = read_gicreg(ICH_AP0R2_EL2);
 166		break;
 167	case 3:
 168		val = read_gicreg(ICH_AP0R3_EL2);
 169		break;
 170	default:
 171		unreachable();
 172	}
 173
 174	return val;
 175}
 176
 177static u32 __vgic_v3_read_ap1rn(int n)
 178{
 179	u32 val;
 180
 181	switch (n) {
 182	case 0:
 183		val = read_gicreg(ICH_AP1R0_EL2);
 184		break;
 185	case 1:
 186		val = read_gicreg(ICH_AP1R1_EL2);
 187		break;
 188	case 2:
 189		val = read_gicreg(ICH_AP1R2_EL2);
 190		break;
 191	case 3:
 192		val = read_gicreg(ICH_AP1R3_EL2);
 193		break;
 194	default:
 195		unreachable();
 196	}
 197
 198	return val;
 199}
 200
 201static u64 compute_ich_hcr(struct vgic_v3_cpu_if *cpu_if)
 202{
 203	return cpu_if->vgic_hcr | vgic_ich_hcr_trap_bits();
 204}
 205
 206void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
 207{
 208	u64 used_lrs = cpu_if->used_lrs;
 209
 210	/*
 211	 * Make sure stores to the GIC via the memory mapped interface
 212	 * are now visible to the system register interface when reading the
 213	 * LRs, and when reading back the VMCR on non-VHE systems.
 214	 */
 215	if (used_lrs || !has_vhe()) {
 216		if (!cpu_if->vgic_sre) {
 217			dsb(sy);
 218			isb();
 219		}
 220	}
 221
 222	if (used_lrs) {
 223		int i;
 224		u32 elrsr;
 225
 226		elrsr = read_gicreg(ICH_ELRSR_EL2);
 227
 228		for (i = 0; i < used_lrs; i++) {
 229			if (elrsr & (1 << i))
 230				cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
 231			else
 232				cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
 233
 234			__gic_v3_set_lr(0, i);
 235		}
 236	}
 237
 238	cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
 239
 240	if (cpu_if->vgic_hcr & ICH_HCR_EL2_LRENPIE) {
 241		u64 val = read_gicreg(ICH_HCR_EL2);
 242		cpu_if->vgic_hcr &= ~ICH_HCR_EL2_EOIcount;
 243		cpu_if->vgic_hcr |= val & ICH_HCR_EL2_EOIcount;
 244	}
 245
 246	write_gicreg(0, ICH_HCR_EL2);
 247
 248	/*
 249	 * Hack alert: On NV, this results in a trap so that the above write
 250	 * actually takes effect... No synchronisation is necessary, as we
 251	 * only care about the effects when this traps.
 252	 */
 253	read_gicreg(ICH_MISR_EL2);
 254}
 255
 256void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
 257{
 258	u64 used_lrs = cpu_if->used_lrs;
 259	int i;
 260
 261	write_gicreg(compute_ich_hcr(cpu_if), ICH_HCR_EL2);
 262
 263	for (i = 0; i < used_lrs; i++)
 264		__gic_v3_set_lr(cpu_if->vgic_lr[i], i);
 265
 266	/*
 267	 * Ensure that writes to the LRs, and on non-VHE systems ensure that
 268	 * the write to the VMCR in __vgic_v3_activate_traps(), will have
 269	 * reached the (re)distributors. This ensure the guest will read the
 270	 * correct values from the memory-mapped interface.
 271	 */
 272	if (used_lrs || !has_vhe()) {
 273		if (!cpu_if->vgic_sre) {
 274			isb();
 275			dsb(sy);
 276		}
 277	}
 278}
 279
 280void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
 281{
 282	/*
 283	 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
 284	 * Group0 interrupt (as generated in GICv2 mode) to be
 285	 * delivered as a FIQ to the guest, with potentially fatal
 286	 * consequences. So we must make sure that ICC_SRE_EL1 has
 287	 * been actually programmed with the value we want before
 288	 * starting to mess with the rest of the GIC, and VMCR_EL2 in
 289	 * particular.  This logic must be called before
 290	 * __vgic_v3_restore_state().
 291	 *
 292	 * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is
 293	 * provisioned at all. In order to prevent illegal accesses to the
 294	 * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1
 295	 * so that the trap bits can take effect. Yes, we *loves* the GIC.
 296	 */
 297	if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) {
 298		write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1);
 299		isb();
 300	} else if (!cpu_if->vgic_sre) {
 301		write_gicreg(0, ICC_SRE_EL1);
 302		isb();
 303		write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
 304
 305
 306		if (has_vhe()) {
 307			/*
 308			 * Ensure that the write to the VMCR will have reached
 309			 * the (re)distributors. This ensure the guest will
 310			 * read the correct values from the memory-mapped
 311			 * interface.
 312			 */
 313			isb();
 314			dsb(sy);
 315		}
 316	}
 317
 318	/* Only disable SRE if the host implements the GICv2 interface */
 319	if (static_branch_unlikely(&vgic_v3_has_v2_compat)) {
 320		/*
 321		 * Prevent the guest from touching the ICC_SRE_EL1 system
 322		 * register. Note that this may not have any effect, as
 323		 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
 324		 */
 325		write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
 326			     ICC_SRE_EL2);
 327	}
 328
 329	/*
 330	 * If we need to trap system registers, we must write ICH_HCR_EL2
 331	 * anyway, even if no interrupts are being injected. Note that this
 332	 * also applies if we don't expect any system register access (no
 333	 * vgic at all). In any case, no need to provide MI configuration.
 334	 */
 335	if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 336	    cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
 337		write_gicreg(vgic_ich_hcr_trap_bits() | ICH_HCR_EL2_En, ICH_HCR_EL2);
 338}
 339
 340void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
 341{
 342	u64 val;
 343
 344	/* Only restore SRE if the host implements the GICv2 interface */
 345	if (static_branch_unlikely(&vgic_v3_has_v2_compat)) {
 346		val = read_gicreg(ICC_SRE_EL2);
 347		write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
 348
 349		if (!cpu_if->vgic_sre) {
 350			/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
 351			isb();
 352			write_gicreg(1, ICC_SRE_EL1);
 353		}
 354	}
 355
 356	/*
 357	 * If we were trapping system registers, we enabled the VGIC even if
 358	 * no interrupts were being injected, and we disable it again here.
 359	 */
 360	if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
 361	    cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
 362		write_gicreg(0, ICH_HCR_EL2);
 363}
 364
 365void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
 366{
 367	u64 val;
 368	u32 nr_pre_bits;
 369
 370	val = read_gicreg(ICH_VTR_EL2);
 371	nr_pre_bits = vtr_to_nr_pre_bits(val);
 372
 373	switch (nr_pre_bits) {
 374	case 7:
 375		cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
 376		cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
 377		fallthrough;
 378	case 6:
 379		cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
 380		fallthrough;
 381	default:
 382		cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
 383	}
 384
 385	switch (nr_pre_bits) {
 386	case 7:
 387		cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
 388		cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
 389		fallthrough;
 390	case 6:
 391		cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
 392		fallthrough;
 393	default:
 394		cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
 395	}
 396}
 397
 398static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
 399{
 400	u64 val;
 401	u32 nr_pre_bits;
 402
 403	val = read_gicreg(ICH_VTR_EL2);
 404	nr_pre_bits = vtr_to_nr_pre_bits(val);
 405
 406	switch (nr_pre_bits) {
 407	case 7:
 408		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
 409		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
 410		fallthrough;
 411	case 6:
 412		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
 413		fallthrough;
 414	default:
 415		__vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
 416	}
 417
 418	switch (nr_pre_bits) {
 419	case 7:
 420		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
 421		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
 422		fallthrough;
 423	case 6:
 424		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
 425		fallthrough;
 426	default:
 427		__vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
 428	}
 429}
 430
 431void __vgic_v3_init_lrs(void)
 432{
 433	int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
 434	int i;
 435
 436	for (i = 0; i <= max_lr_idx; i++)
 437		__gic_v3_set_lr(0, i);
 438}
 439
 440/*
 441 * Return the GIC CPU configuration:
 442 * - [31:0]  ICH_VTR_EL2
 443 * - [62:32] RES0
 444 * - [63]    MMIO (GICv2) capable
 445 */
 446u64 __vgic_v3_get_gic_config(void)
 447{
 448	u64 val, sre;
 449	unsigned long flags = 0;
 450
 451	/*
 452	 * In compat mode, we cannot access ICC_SRE_EL1 at any EL
 453	 * other than EL1 itself; just return the
 454	 * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
 455	 * system, so we first check if we have GICv5 support.
 456	 */
 457	if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
 458		return read_gicreg(ICH_VTR_EL2);
 459
 460	sre = read_gicreg(ICC_SRE_EL1);
 461	/*
 462	 * To check whether we have a MMIO-based (GICv2 compatible)
 463	 * CPU interface, we need to disable the system register
 464	 * view.
 465	 *
 466	 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
 467	 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
 468	 * interrupt overrides must be set. You've got to love this.
 469	 *
 470	 * As we always run VHE with HCR_xMO set, no extra xMO
 471	 * manipulation is required in that case.
 472	 *
 473	 * To safely disable SRE, we have to prevent any interrupt
 474	 * from firing (which would be deadly). This only makes sense
 475	 * on VHE, as interrupts are already masked for nVHE as part
 476	 * of the exception entry to EL2.
 477	 */
 478	if (has_vhe()) {
 479		flags = local_daif_save();
 480	} else {
 481		sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
 482		isb();
 483	}
 484
 485	write_gicreg(0, ICC_SRE_EL1);
 486	isb();
 487
 488	val = read_gicreg(ICC_SRE_EL1);
 489
 490	write_gicreg(sre, ICC_SRE_EL1);
 491	isb();
 492
 493	if (has_vhe()) {
 494		local_daif_restore(flags);
 495	} else {
 496		sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
 497		isb();
 498	}
 499
 500	val  = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
 501	val |= read_gicreg(ICH_VTR_EL2);
 502
 503	return val;
 504}
 505
 506static void __vgic_v3_compat_mode_enable(void)
 507{
 508	if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
 509		return;
 510
 511	sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
 512	/* Wait for V3 to become enabled */
 513	isb();
 514}
 515
 516static u64 __vgic_v3_read_vmcr(void)
 517{
 518	return read_gicreg(ICH_VMCR_EL2);
 519}
 520
 521static void __vgic_v3_write_vmcr(u32 vmcr)
 522{
 523	write_gicreg(vmcr, ICH_VMCR_EL2);
 524}
 525
 526void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
 527{
 528	__vgic_v3_compat_mode_enable();
 529
 530	/*
 531	 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
 532	 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
 533	 * VMCR_EL2 save/restore in the world switch.
 534	 */
 535	if (cpu_if->vgic_sre)
 536		__vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
 537	__vgic_v3_restore_aprs(cpu_if);
 538}
 539
 540static int __vgic_v3_bpr_min(void)
 541{
 542	/* See Pseudocode for VPriorityGroup */
 543	return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
 544}
 545
 546static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
 547{
 548	u64 esr = kvm_vcpu_get_esr(vcpu);
 549	u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 550
 551	return crm != 8;
 552}
 553
 554#define GICv3_IDLE_PRIORITY	0xff
 555
 556static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
 557					 u64 *lr_val)
 558{
 559	unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
 560	u8 priority = GICv3_IDLE_PRIORITY;
 561	int i, lr = -1;
 562
 563	for (i = 0; i < used_lrs; i++) {
 564		u64 val = __gic_v3_get_lr(i);
 565		u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 566
 567		/* Not pending in the state? */
 568		if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
 569			continue;
 570
 571		/* Group-0 interrupt, but Group-0 disabled? */
 572		if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
 573			continue;
 574
 575		/* Group-1 interrupt, but Group-1 disabled? */
 576		if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
 577			continue;
 578
 579		/* Not the highest priority? */
 580		if (lr_prio >= priority)
 581			continue;
 582
 583		/* This is a candidate */
 584		priority = lr_prio;
 585		*lr_val = val;
 586		lr = i;
 587	}
 588
 589	if (lr == -1)
 590		*lr_val = ICC_IAR1_EL1_SPURIOUS;
 591
 592	return lr;
 593}
 594
 595static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
 596				    u64 *lr_val)
 597{
 598	unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
 599	int i;
 600
 601	for (i = 0; i < used_lrs; i++) {
 602		u64 val = __gic_v3_get_lr(i);
 603
 604		if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
 605		    (val & ICH_LR_ACTIVE_BIT)) {
 606			*lr_val = val;
 607			return i;
 608		}
 609	}
 610
 611	*lr_val = ICC_IAR1_EL1_SPURIOUS;
 612	return -1;
 613}
 614
 615static int __vgic_v3_get_highest_active_priority(void)
 616{
 617	u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 618	u32 hap = 0;
 619	int i;
 620
 621	for (i = 0; i < nr_apr_regs; i++) {
 622		u32 val;
 623
 624		/*
 625		 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
 626		 * contain the active priority levels for this VCPU
 627		 * for the maximum number of supported priority
 628		 * levels, and we return the full priority level only
 629		 * if the BPR is programmed to its minimum, otherwise
 630		 * we return a combination of the priority level and
 631		 * subpriority, as determined by the setting of the
 632		 * BPR, but without the full subpriority.
 633		 */
 634		val  = __vgic_v3_read_ap0rn(i);
 635		val |= __vgic_v3_read_ap1rn(i);
 636		if (!val) {
 637			hap += 32;
 638			continue;
 639		}
 640
 641		return (hap + __ffs(val)) << __vgic_v3_bpr_min();
 642	}
 643
 644	return GICv3_IDLE_PRIORITY;
 645}
 646
 647static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
 648{
 649	return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
 650}
 651
 652static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
 653{
 654	unsigned int bpr;
 655
 656	if (vmcr & ICH_VMCR_CBPR_MASK) {
 657		bpr = __vgic_v3_get_bpr0(vmcr);
 658		if (bpr < 7)
 659			bpr++;
 660	} else {
 661		bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
 662	}
 663
 664	return bpr;
 665}
 666
 667/*
 668 * Convert a priority to a preemption level, taking the relevant BPR
 669 * into account by zeroing the sub-priority bits.
 670 */
 671static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
 672{
 673	unsigned int bpr;
 674
 675	if (!grp)
 676		bpr = __vgic_v3_get_bpr0(vmcr) + 1;
 677	else
 678		bpr = __vgic_v3_get_bpr1(vmcr);
 679
 680	return pri & (GENMASK(7, 0) << bpr);
 681}
 682
 683/*
 684 * The priority value is independent of any of the BPR values, so we
 685 * normalize it using the minimal BPR value. This guarantees that no
 686 * matter what the guest does with its BPR, we can always set/get the
 687 * same value of a priority.
 688 */
 689static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
 690{
 691	u8 pre, ap;
 692	u32 val;
 693	int apr;
 694
 695	pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
 696	ap = pre >> __vgic_v3_bpr_min();
 697	apr = ap / 32;
 698
 699	if (!grp) {
 700		val = __vgic_v3_read_ap0rn(apr);
 701		__vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
 702	} else {
 703		val = __vgic_v3_read_ap1rn(apr);
 704		__vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
 705	}
 706}
 707
 708static int __vgic_v3_clear_highest_active_priority(void)
 709{
 710	u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
 711	u32 hap = 0;
 712	int i;
 713
 714	for (i = 0; i < nr_apr_regs; i++) {
 715		u32 ap0, ap1;
 716		int c0, c1;
 717
 718		ap0 = __vgic_v3_read_ap0rn(i);
 719		ap1 = __vgic_v3_read_ap1rn(i);
 720		if (!ap0 && !ap1) {
 721			hap += 32;
 722			continue;
 723		}
 724
 725		c0 = ap0 ? __ffs(ap0) : 32;
 726		c1 = ap1 ? __ffs(ap1) : 32;
 727
 728		/* Always clear the LSB, which is the highest priority */
 729		if (c0 < c1) {
 730			ap0 &= ~BIT(c0);
 731			__vgic_v3_write_ap0rn(ap0, i);
 732			hap += c0;
 733		} else {
 734			ap1 &= ~BIT(c1);
 735			__vgic_v3_write_ap1rn(ap1, i);
 736			hap += c1;
 737		}
 738
 739		/* Rescale to 8 bits of priority */
 740		return hap << __vgic_v3_bpr_min();
 741	}
 742
 743	return GICv3_IDLE_PRIORITY;
 744}
 745
 746static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 747{
 748	u64 lr_val;
 749	u8 lr_prio, pmr;
 750	int lr, grp;
 751
 752	grp = __vgic_v3_get_group(vcpu);
 753
 754	lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
 755	if (lr < 0)
 756		goto spurious;
 757
 758	if (grp != !!(lr_val & ICH_LR_GROUP))
 759		goto spurious;
 760
 761	pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
 762	lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 763	if (pmr <= lr_prio)
 764		goto spurious;
 765
 766	if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
 767		goto spurious;
 768
 769	lr_val &= ~ICH_LR_STATE;
 770	lr_val |= ICH_LR_ACTIVE_BIT;
 771	__gic_v3_set_lr(lr_val, lr);
 772	__vgic_v3_set_active_priority(lr_prio, vmcr, grp);
 773	vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
 774	return;
 775
 776spurious:
 777	vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
 778}
 779
 780static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
 781{
 782	lr_val &= ~ICH_LR_ACTIVE_BIT;
 783	if (lr_val & ICH_LR_HW) {
 784		u32 pid;
 785
 786		pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
 787		gic_write_dir(pid);
 788	}
 789
 790	__gic_v3_set_lr(lr_val, lr);
 791}
 792
 793static void __vgic_v3_bump_eoicount(void)
 794{
 795	u32 hcr;
 796
 797	hcr = read_gicreg(ICH_HCR_EL2);
 798	hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT;
 799	write_gicreg(hcr, ICH_HCR_EL2);
 800}
 801
 802static int ___vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 803{
 804	u32 vid = vcpu_get_reg(vcpu, rt);
 805	u64 lr_val;
 806	int lr;
 807
 808	/* EOImode == 0, nothing to be done here */
 809	if (!(vmcr & ICH_VMCR_EOIM_MASK))
 810		return 1;
 811
 812	/* No deactivate to be performed on an LPI */
 813	if (vid >= VGIC_MIN_LPI)
 814		return 1;
 815
 816	lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 817	if (lr != -1) {
 818		__vgic_v3_clear_active_lr(lr, lr_val);
 819		return 1;
 820	}
 821
 822	return 0;
 823}
 824
 825static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 826{
 827	if (!___vgic_v3_write_dir(vcpu, vmcr, rt))
 828		__vgic_v3_bump_eoicount();
 829}
 830
 831static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 832{
 833	u32 vid = vcpu_get_reg(vcpu, rt);
 834	u64 lr_val;
 835	u8 lr_prio, act_prio;
 836	int lr, grp;
 837
 838	grp = __vgic_v3_get_group(vcpu);
 839
 840	/* Drop priority in any case */
 841	act_prio = __vgic_v3_clear_highest_active_priority();
 842
 843	lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
 844	if (lr == -1) {
 845		/* Do not bump EOIcount for LPIs that aren't in the LRs */
 846		if (!(vid >= VGIC_MIN_LPI))
 847			__vgic_v3_bump_eoicount();
 848		return;
 849	}
 850
 851	/* EOImode == 1 and not an LPI, nothing to be done here */
 852	if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
 853		return;
 854
 855	lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
 856
 857	/* If priorities or group do not match, the guest has fscked-up. */
 858	if (grp != !!(lr_val & ICH_LR_GROUP) ||
 859	    __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
 860		return;
 861
 862	/* Let's now perform the deactivation */
 863	__vgic_v3_clear_active_lr(lr, lr_val);
 864}
 865
 866static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 867{
 868	vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
 869}
 870
 871static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 872{
 873	vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
 874}
 875
 876static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 877{
 878	u64 val = vcpu_get_reg(vcpu, rt);
 879
 880	if (val & 1)
 881		vmcr |= ICH_VMCR_ENG0_MASK;
 882	else
 883		vmcr &= ~ICH_VMCR_ENG0_MASK;
 884
 885	__vgic_v3_write_vmcr(vmcr);
 886}
 887
 888static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 889{
 890	u64 val = vcpu_get_reg(vcpu, rt);
 891
 892	if (val & 1)
 893		vmcr |= ICH_VMCR_ENG1_MASK;
 894	else
 895		vmcr &= ~ICH_VMCR_ENG1_MASK;
 896
 897	__vgic_v3_write_vmcr(vmcr);
 898}
 899
 900static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 901{
 902	vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
 903}
 904
 905static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 906{
 907	vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
 908}
 909
 910static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 911{
 912	u64 val = vcpu_get_reg(vcpu, rt);
 913	u8 bpr_min = __vgic_v3_bpr_min() - 1;
 914
 915	/* Enforce BPR limiting */
 916	if (val < bpr_min)
 917		val = bpr_min;
 918
 919	val <<= ICH_VMCR_BPR0_SHIFT;
 920	val &= ICH_VMCR_BPR0_MASK;
 921	vmcr &= ~ICH_VMCR_BPR0_MASK;
 922	vmcr |= val;
 923
 924	__vgic_v3_write_vmcr(vmcr);
 925}
 926
 927static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 928{
 929	u64 val = vcpu_get_reg(vcpu, rt);
 930	u8 bpr_min = __vgic_v3_bpr_min();
 931
 932	if (vmcr & ICH_VMCR_CBPR_MASK)
 933		return;
 934
 935	/* Enforce BPR limiting */
 936	if (val < bpr_min)
 937		val = bpr_min;
 938
 939	val <<= ICH_VMCR_BPR1_SHIFT;
 940	val &= ICH_VMCR_BPR1_MASK;
 941	vmcr &= ~ICH_VMCR_BPR1_MASK;
 942	vmcr |= val;
 943
 944	__vgic_v3_write_vmcr(vmcr);
 945}
 946
 947static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 948{
 949	u32 val;
 950
 951	if (!__vgic_v3_get_group(vcpu))
 952		val = __vgic_v3_read_ap0rn(n);
 953	else
 954		val = __vgic_v3_read_ap1rn(n);
 955
 956	vcpu_set_reg(vcpu, rt, val);
 957}
 958
 959static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
 960{
 961	u32 val = vcpu_get_reg(vcpu, rt);
 962
 963	if (!__vgic_v3_get_group(vcpu))
 964		__vgic_v3_write_ap0rn(val, n);
 965	else
 966		__vgic_v3_write_ap1rn(val, n);
 967}
 968
 969static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
 970					    u32 vmcr, int rt)
 971{
 972	__vgic_v3_read_apxrn(vcpu, rt, 0);
 973}
 974
 975static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
 976					    u32 vmcr, int rt)
 977{
 978	__vgic_v3_read_apxrn(vcpu, rt, 1);
 979}
 980
 981static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 982{
 983	__vgic_v3_read_apxrn(vcpu, rt, 2);
 984}
 985
 986static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 987{
 988	__vgic_v3_read_apxrn(vcpu, rt, 3);
 989}
 990
 991static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 992{
 993	__vgic_v3_write_apxrn(vcpu, rt, 0);
 994}
 995
 996static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
 997{
 998	__vgic_v3_write_apxrn(vcpu, rt, 1);
 999}
1000
1001static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1002{
1003	__vgic_v3_write_apxrn(vcpu, rt, 2);
1004}
1005
1006static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1007{
1008	__vgic_v3_write_apxrn(vcpu, rt, 3);
1009}
1010
1011static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1012{
1013	u64 lr_val;
1014	int lr, lr_grp, grp;
1015
1016	grp = __vgic_v3_get_group(vcpu);
1017
1018	lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
1019	if (lr == -1)
1020		goto spurious;
1021
1022	lr_grp = !!(lr_val & ICH_LR_GROUP);
1023	if (lr_grp != grp)
1024		lr_val = ICC_IAR1_EL1_SPURIOUS;
1025
1026spurious:
1027	vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
1028}
1029
1030static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1031{
1032	vmcr &= ICH_VMCR_PMR_MASK;
1033	vmcr >>= ICH_VMCR_PMR_SHIFT;
1034	vcpu_set_reg(vcpu, rt, vmcr);
1035}
1036
1037static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1038{
1039	u32 val = vcpu_get_reg(vcpu, rt);
1040
1041	val <<= ICH_VMCR_PMR_SHIFT;
1042	val &= ICH_VMCR_PMR_MASK;
1043	vmcr &= ~ICH_VMCR_PMR_MASK;
1044	vmcr |= val;
1045
1046	write_gicreg(vmcr, ICH_VMCR_EL2);
1047}
1048
1049static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1050{
1051	u32 val = __vgic_v3_get_highest_active_priority();
1052	vcpu_set_reg(vcpu, rt, val);
1053}
1054
1055static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1056{
1057	u32 vtr, val;
1058
1059	vtr = read_gicreg(ICH_VTR_EL2);
1060	/* PRIbits */
1061	val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1062	/* IDbits */
1063	val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1064	/* A3V */
1065	val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1066	/* EOImode */
1067	val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1068	/* CBPR */
1069	val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1070
1071	vcpu_set_reg(vcpu, rt, val);
1072}
1073
1074static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1075{
1076	u32 val = vcpu_get_reg(vcpu, rt);
1077
1078	if (val & ICC_CTLR_EL1_CBPR_MASK)
1079		vmcr |= ICH_VMCR_CBPR_MASK;
1080	else
1081		vmcr &= ~ICH_VMCR_CBPR_MASK;
1082
1083	if (val & ICC_CTLR_EL1_EOImode_MASK)
1084		vmcr |= ICH_VMCR_EOIM_MASK;
1085	else
1086		vmcr &= ~ICH_VMCR_EOIM_MASK;
1087
1088	write_gicreg(vmcr, ICH_VMCR_EL2);
1089}
1090
1091static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
1092					    u32 sysreg, bool is_read)
1093{
1094	u64 ich_hcr;
1095
1096	if (!is_nested_ctxt(vcpu))
1097		return false;
1098
1099	ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
1100
1101	switch (sysreg) {
1102	case SYS_ICC_IGRPEN0_EL1:
1103		if (is_read &&
1104		    (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1105			return true;
1106
1107		if (!is_read &&
1108		    (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1109			return true;
1110
1111		fallthrough;
1112
1113	case SYS_ICC_AP0Rn_EL1(0):
1114	case SYS_ICC_AP0Rn_EL1(1):
1115	case SYS_ICC_AP0Rn_EL1(2):
1116	case SYS_ICC_AP0Rn_EL1(3):
1117	case SYS_ICC_BPR0_EL1:
1118	case SYS_ICC_EOIR0_EL1:
1119	case SYS_ICC_HPPIR0_EL1:
1120	case SYS_ICC_IAR0_EL1:
1121		return ich_hcr & ICH_HCR_EL2_TALL0;
1122
1123	case SYS_ICC_IGRPEN1_EL1:
1124		if (is_read &&
1125		    (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1126			return true;
1127
1128		if (!is_read &&
1129		    (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1130			return true;
1131
1132		fallthrough;
1133
1134	case SYS_ICC_AP1Rn_EL1(0):
1135	case SYS_ICC_AP1Rn_EL1(1):
1136	case SYS_ICC_AP1Rn_EL1(2):
1137	case SYS_ICC_AP1Rn_EL1(3):
1138	case SYS_ICC_BPR1_EL1:
1139	case SYS_ICC_EOIR1_EL1:
1140	case SYS_ICC_HPPIR1_EL1:
1141	case SYS_ICC_IAR1_EL1:
1142		return ich_hcr & ICH_HCR_EL2_TALL1;
1143
1144	case SYS_ICC_DIR_EL1:
1145		if (ich_hcr & ICH_HCR_EL2_TDIR)
1146			return true;
1147
1148		fallthrough;
1149
1150	case SYS_ICC_RPR_EL1:
1151	case SYS_ICC_CTLR_EL1:
1152	case SYS_ICC_PMR_EL1:
1153		return ich_hcr & ICH_HCR_EL2_TC;
1154
1155	default:
1156		return false;
1157	}
1158}
1159
1160int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1161{
1162	int rt;
1163	u64 esr;
1164	u32 vmcr;
1165	void (*fn)(struct kvm_vcpu *, u32, int);
1166	bool is_read;
1167	u32 sysreg;
1168
1169	if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
1170		return 0;
1171
1172	esr = kvm_vcpu_get_esr(vcpu);
1173	if (vcpu_mode_is_32bit(vcpu)) {
1174		if (!kvm_condition_valid(vcpu)) {
1175			__kvm_skip_instr(vcpu);
1176			return 1;
1177		}
1178
1179		sysreg = esr_cp15_to_sysreg(esr);
1180	} else {
1181		sysreg = esr_sys64_to_sysreg(esr);
1182	}
1183
1184	is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1185
1186	if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read))
1187		return 0;
1188
1189	switch (sysreg) {
1190	case SYS_ICC_IAR0_EL1:
1191	case SYS_ICC_IAR1_EL1:
1192		if (unlikely(!is_read))
1193			return 0;
1194		fn = __vgic_v3_read_iar;
1195		break;
1196	case SYS_ICC_EOIR0_EL1:
1197	case SYS_ICC_EOIR1_EL1:
1198		if (unlikely(is_read))
1199			return 0;
1200		fn = __vgic_v3_write_eoir;
1201		break;
1202	case SYS_ICC_IGRPEN1_EL1:
1203		if (is_read)
1204			fn = __vgic_v3_read_igrpen1;
1205		else
1206			fn = __vgic_v3_write_igrpen1;
1207		break;
1208	case SYS_ICC_BPR1_EL1:
1209		if (is_read)
1210			fn = __vgic_v3_read_bpr1;
1211		else
1212			fn = __vgic_v3_write_bpr1;
1213		break;
1214	case SYS_ICC_AP0Rn_EL1(0):
1215	case SYS_ICC_AP1Rn_EL1(0):
1216		if (is_read)
1217			fn = __vgic_v3_read_apxr0;
1218		else
1219			fn = __vgic_v3_write_apxr0;
1220		break;
1221	case SYS_ICC_AP0Rn_EL1(1):
1222	case SYS_ICC_AP1Rn_EL1(1):
1223		if (is_read)
1224			fn = __vgic_v3_read_apxr1;
1225		else
1226			fn = __vgic_v3_write_apxr1;
1227		break;
1228	case SYS_ICC_AP0Rn_EL1(2):
1229	case SYS_ICC_AP1Rn_EL1(2):
1230		if (is_read)
1231			fn = __vgic_v3_read_apxr2;
1232		else
1233			fn = __vgic_v3_write_apxr2;
1234		break;
1235	case SYS_ICC_AP0Rn_EL1(3):
1236	case SYS_ICC_AP1Rn_EL1(3):
1237		if (is_read)
1238			fn = __vgic_v3_read_apxr3;
1239		else
1240			fn = __vgic_v3_write_apxr3;
1241		break;
1242	case SYS_ICC_HPPIR0_EL1:
1243	case SYS_ICC_HPPIR1_EL1:
1244		if (unlikely(!is_read))
1245			return 0;
1246		fn = __vgic_v3_read_hppir;
1247		break;
1248	case SYS_ICC_IGRPEN0_EL1:
1249		if (is_read)
1250			fn = __vgic_v3_read_igrpen0;
1251		else
1252			fn = __vgic_v3_write_igrpen0;
1253		break;
1254	case SYS_ICC_BPR0_EL1:
1255		if (is_read)
1256			fn = __vgic_v3_read_bpr0;
1257		else
1258			fn = __vgic_v3_write_bpr0;
1259		break;
1260	case SYS_ICC_DIR_EL1:
1261		if (unlikely(is_read))
1262			return 0;
1263		/*
1264		 * Full exit if required to handle overflow deactivation,
1265		 * unless we can emulate it in the LRs (likely the majority
1266		 * of the cases).
1267		 */
1268		if (vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr & ICH_HCR_EL2_TDIR) {
1269			int ret;
1270
1271			ret = ___vgic_v3_write_dir(vcpu, __vgic_v3_read_vmcr(),
1272						   kvm_vcpu_sys_get_rt(vcpu));
1273			if (ret)
1274				__kvm_skip_instr(vcpu);
1275
1276			return ret;
1277		}
1278		fn = __vgic_v3_write_dir;
1279		break;
1280	case SYS_ICC_RPR_EL1:
1281		if (unlikely(!is_read))
1282			return 0;
1283		fn = __vgic_v3_read_rpr;
1284		break;
1285	case SYS_ICC_CTLR_EL1:
1286		if (is_read)
1287			fn = __vgic_v3_read_ctlr;
1288		else
1289			fn = __vgic_v3_write_ctlr;
1290		break;
1291	case SYS_ICC_PMR_EL1:
1292		if (is_read)
1293			fn = __vgic_v3_read_pmr;
1294		else
1295			fn = __vgic_v3_write_pmr;
1296		break;
1297	default:
1298		return 0;
1299	}
1300
1301	vmcr = __vgic_v3_read_vmcr();
1302	rt = kvm_vcpu_sys_get_rt(vcpu);
1303	fn(vcpu, vmcr, rt);
1304
1305	__kvm_skip_instr(vcpu);
1306
1307	return 1;
1308}