vgic-mmio.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * VGIC MMIO handling functions
   4 */
   5
   6#include <linux/bitops.h>
   7#include <linux/bsearch.h>
   8#include <linux/interrupt.h>
   9#include <linux/irq.h>
  10#include <linux/kvm.h>
  11#include <linux/kvm_host.h>
  12#include <kvm/iodev.h>
  13#include <kvm/arm_arch_timer.h>
  14#include <kvm/arm_vgic.h>
  15
  16#include "vgic.h"
  17#include "vgic-mmio.h"
  18
  19unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu,
  20				 gpa_t addr, unsigned int len)
  21{
  22	return 0;
  23}
  24
  25unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu,
  26				 gpa_t addr, unsigned int len)
  27{
  28	return -1UL;
  29}
  30
  31void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
  32			unsigned int len, unsigned long val)
  33{
  34	/* Ignore */
  35}
  36
  37int vgic_mmio_uaccess_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
  38			       unsigned int len, unsigned long val)
  39{
  40	/* Ignore */
  41	return 0;
  42}
  43
  44unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu,
  45				   gpa_t addr, unsigned int len)
  46{
  47	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  48	u32 value = 0;
  49	int i;
  50
  51	/* Loop over all IRQs affected by this read */
  52	for (i = 0; i < len * 8; i++) {
  53		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
  54
  55		if (irq->group)
  56			value |= BIT(i);
  57
  58		vgic_put_irq(vcpu->kvm, irq);
  59	}
  60
  61	return value;
  62}
  63
  64static void vgic_update_vsgi(struct vgic_irq *irq)
  65{
  66	WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));
  67}
  68
  69void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr,
  70			   unsigned int len, unsigned long val)
  71{
  72	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
  73	int i;
  74	unsigned long flags;
  75
  76	for (i = 0; i < len * 8; i++) {
  77		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
  78
  79		raw_spin_lock_irqsave(&irq->irq_lock, flags);
  80		irq->group = !!(val & BIT(i));
  81		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
  82			vgic_update_vsgi(irq);
  83			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
  84		} else {
  85			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
  86		}
  87
  88		vgic_put_irq(vcpu->kvm, irq);
  89	}
  90}
  91
  92/*
  93 * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value
  94 * of the enabled bit, so there is only one function for both here.
  95 */
  96unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu,
  97				    gpa_t addr, unsigned int len)
  98{
  99	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 100	u32 value = 0;
 101	int i;
 102
 103	/* Loop over all IRQs affected by this read */
 104	for (i = 0; i < len * 8; i++) {
 105		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 106
 107		if (irq->enabled)
 108			value |= (1U << i);
 109
 110		vgic_put_irq(vcpu->kvm, irq);
 111	}
 112
 113	return value;
 114}
 115
 116void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
 117			     gpa_t addr, unsigned int len,
 118			     unsigned long val)
 119{
 120	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 121	int i;
 122	unsigned long flags;
 123
 124	for_each_set_bit(i, &val, len * 8) {
 125		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 126
 127		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 128		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 129			if (!irq->enabled) {
 130				struct irq_data *data;
 131
 132				irq->enabled = true;
 133				data = &irq_to_desc(irq->host_irq)->irq_data;
 134				while (irqd_irq_disabled(data))
 135					enable_irq(irq->host_irq);
 136			}
 137
 138			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 139			vgic_put_irq(vcpu->kvm, irq);
 140
 141			continue;
 142		} else if (vgic_irq_is_mapped_level(irq)) {
 143			bool was_high = irq->line_level;
 144
 145			/*
 146			 * We need to update the state of the interrupt because
 147			 * the guest might have changed the state of the device
 148			 * while the interrupt was disabled at the VGIC level.
 149			 */
 150			irq->line_level = vgic_get_phys_line_level(irq);
 151			/*
 152			 * Deactivate the physical interrupt so the GIC will let
 153			 * us know when it is asserted again.
 154			 */
 155			if (!irq->active && was_high && !irq->line_level)
 156				vgic_irq_set_phys_active(irq, false);
 157		}
 158		irq->enabled = true;
 159		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 160
 161		vgic_put_irq(vcpu->kvm, irq);
 162	}
 163}
 164
 165void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
 166			     gpa_t addr, unsigned int len,
 167			     unsigned long val)
 168{
 169	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 170	int i;
 171	unsigned long flags;
 172
 173	for_each_set_bit(i, &val, len * 8) {
 174		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 175
 176		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 177		if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled)
 178			disable_irq_nosync(irq->host_irq);
 179
 180		irq->enabled = false;
 181
 182		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 183		vgic_put_irq(vcpu->kvm, irq);
 184	}
 185}
 186
 187int vgic_uaccess_write_senable(struct kvm_vcpu *vcpu,
 188			       gpa_t addr, unsigned int len,
 189			       unsigned long val)
 190{
 191	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 192	int i;
 193	unsigned long flags;
 194
 195	for_each_set_bit(i, &val, len * 8) {
 196		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 197
 198		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 199		irq->enabled = true;
 200		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 201
 202		vgic_put_irq(vcpu->kvm, irq);
 203	}
 204
 205	return 0;
 206}
 207
 208int vgic_uaccess_write_cenable(struct kvm_vcpu *vcpu,
 209			       gpa_t addr, unsigned int len,
 210			       unsigned long val)
 211{
 212	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 213	int i;
 214	unsigned long flags;
 215
 216	for_each_set_bit(i, &val, len * 8) {
 217		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 218
 219		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 220		irq->enabled = false;
 221		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 222
 223		vgic_put_irq(vcpu->kvm, irq);
 224	}
 225
 226	return 0;
 227}
 228
 229static unsigned long __read_pending(struct kvm_vcpu *vcpu,
 230				    gpa_t addr, unsigned int len,
 231				    bool is_user)
 232{
 233	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 234	u32 value = 0;
 235	int i;
 236
 237	/* Loop over all IRQs affected by this read */
 238	for (i = 0; i < len * 8; i++) {
 239		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 240		unsigned long flags;
 241		bool val;
 242
 243		/*
 244		 * When used from userspace with a GICv3 model:
 245		 *
 246		 * Pending state of interrupt is latched in pending_latch
 247		 * variable.  Userspace will save and restore pending state
 248		 * and line_level separately.
 249		 * Refer to Documentation/virt/kvm/devices/arm-vgic-v3.rst
 250		 * for handling of ISPENDR and ICPENDR.
 251		 */
 252		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 253		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 254			int err;
 255
 256			val = false;
 257			err = irq_get_irqchip_state(irq->host_irq,
 258						    IRQCHIP_STATE_PENDING,
 259						    &val);
 260			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
 261		} else if (!is_user && vgic_irq_is_mapped_level(irq)) {
 262			val = vgic_get_phys_line_level(irq);
 263		} else {
 264			switch (vcpu->kvm->arch.vgic.vgic_model) {
 265			case KVM_DEV_TYPE_ARM_VGIC_V3:
 266				if (is_user) {
 267					val = irq->pending_latch;
 268					break;
 269				}
 270				fallthrough;
 271			default:
 272				val = irq_is_pending(irq);
 273				break;
 274			}
 275		}
 276
 277		value |= ((u32)val << i);
 278		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 279
 280		vgic_put_irq(vcpu->kvm, irq);
 281	}
 282
 283	return value;
 284}
 285
 286unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
 287				     gpa_t addr, unsigned int len)
 288{
 289	return __read_pending(vcpu, addr, len, false);
 290}
 291
 292unsigned long vgic_uaccess_read_pending(struct kvm_vcpu *vcpu,
 293					gpa_t addr, unsigned int len)
 294{
 295	return __read_pending(vcpu, addr, len, true);
 296}
 297
 298static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
 299{
 300	return (vgic_irq_is_sgi(irq->intid) &&
 301		vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2);
 302}
 303
 304static void __set_pending(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len,
 305			  unsigned long val, bool is_user)
 306{
 307	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 308	int i;
 309	unsigned long flags;
 310
 311	for_each_set_bit(i, &val, len * 8) {
 312		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 313
 314		/* GICD_ISPENDR0 SGI bits are WI when written from the guest. */
 315		if (is_vgic_v2_sgi(vcpu, irq) && !is_user) {
 316			vgic_put_irq(vcpu->kvm, irq);
 317			continue;
 318		}
 319
 320		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 321
 322		/*
 323		 * GICv2 SGIs are terribly broken. We can't restore
 324		 * the source of the interrupt, so just pick the vcpu
 325		 * itself as the source...
 326		 */
 327		if (is_vgic_v2_sgi(vcpu, irq))
 328			irq->source |= BIT(vcpu->vcpu_id);
 329
 330		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 331			/* HW SGI? Ask the GIC to inject it */
 332			int err;
 333			err = irq_set_irqchip_state(irq->host_irq,
 334						    IRQCHIP_STATE_PENDING,
 335						    true);
 336			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
 337
 338			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 339			vgic_put_irq(vcpu->kvm, irq);
 340
 341			continue;
 342		}
 343
 344		irq->pending_latch = true;
 345		if (irq->hw && !is_user)
 346			vgic_irq_set_phys_active(irq, true);
 347
 348		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 349		vgic_put_irq(vcpu->kvm, irq);
 350	}
 351}
 352
 353void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
 354			      gpa_t addr, unsigned int len,
 355			      unsigned long val)
 356{
 357	__set_pending(vcpu, addr, len, val, false);
 358}
 359
 360int vgic_uaccess_write_spending(struct kvm_vcpu *vcpu,
 361				gpa_t addr, unsigned int len,
 362				unsigned long val)
 363{
 364	__set_pending(vcpu, addr, len, val, true);
 365	return 0;
 366}
 367
 368/* Must be called with irq->irq_lock held */
 369static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
 370{
 371	irq->pending_latch = false;
 372
 373	/*
 374	 * We don't want the guest to effectively mask the physical
 375	 * interrupt by doing a write to SPENDR followed by a write to
 376	 * CPENDR for HW interrupts, so we clear the active state on
 377	 * the physical side if the virtual interrupt is not active.
 378	 * This may lead to taking an additional interrupt on the
 379	 * host, but that should not be a problem as the worst that
 380	 * can happen is an additional vgic injection.  We also clear
 381	 * the pending state to maintain proper semantics for edge HW
 382	 * interrupts.
 383	 */
 384	vgic_irq_set_phys_pending(irq, false);
 385	if (!irq->active)
 386		vgic_irq_set_phys_active(irq, false);
 387}
 388
 389static void __clear_pending(struct kvm_vcpu *vcpu,
 390			    gpa_t addr, unsigned int len,
 391			    unsigned long val, bool is_user)
 392{
 393	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 394	int i;
 395	unsigned long flags;
 396
 397	for_each_set_bit(i, &val, len * 8) {
 398		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 399
 400		/* GICD_ICPENDR0 SGI bits are WI when written from the guest. */
 401		if (is_vgic_v2_sgi(vcpu, irq) && !is_user) {
 402			vgic_put_irq(vcpu->kvm, irq);
 403			continue;
 404		}
 405
 406		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 407
 408		/*
 409		 * More fun with GICv2 SGIs! If we're clearing one of them
 410		 * from userspace, which source vcpu to clear? Let's not
 411		 * even think of it, and blow the whole set.
 412		 */
 413		if (is_vgic_v2_sgi(vcpu, irq))
 414			irq->source = 0;
 415
 416		if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 417			/* HW SGI? Ask the GIC to clear its pending bit */
 418			int err;
 419			err = irq_set_irqchip_state(irq->host_irq,
 420						    IRQCHIP_STATE_PENDING,
 421						    false);
 422			WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
 423
 424			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 425			vgic_put_irq(vcpu->kvm, irq);
 426
 427			continue;
 428		}
 429
 430		if (irq->hw && !is_user)
 431			vgic_hw_irq_cpending(vcpu, irq);
 432		else
 433			irq->pending_latch = false;
 434
 435		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 436		vgic_put_irq(vcpu->kvm, irq);
 437	}
 438}
 439
 440void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
 441			      gpa_t addr, unsigned int len,
 442			      unsigned long val)
 443{
 444	__clear_pending(vcpu, addr, len, val, false);
 445}
 446
 447int vgic_uaccess_write_cpending(struct kvm_vcpu *vcpu,
 448				gpa_t addr, unsigned int len,
 449				unsigned long val)
 450{
 451	__clear_pending(vcpu, addr, len, val, true);
 452	return 0;
 453}
 454
 455/*
 456 * If we are fiddling with an IRQ's active state, we have to make sure the IRQ
 457 * is not queued on some running VCPU's LRs, because then the change to the
 458 * active state can be overwritten when the VCPU's state is synced coming back
 459 * from the guest.
 460 *
 461 * For shared interrupts as well as GICv3 private interrupts accessed from the
 462 * non-owning CPU, we have to stop all the VCPUs because interrupts can be
 463 * migrated while we don't hold the IRQ locks and we don't want to be chasing
 464 * moving targets.
 465 *
 466 * For GICv2 private interrupts we don't have to do anything because
 467 * userspace accesses to the VGIC state already require all VCPUs to be
 468 * stopped, and only the VCPU itself can modify its private interrupts
 469 * active state, which guarantees that the VCPU is not running.
 470 */
 471static void vgic_access_active_prepare(struct kvm_vcpu *vcpu, u32 intid)
 472{
 473	if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
 474	     vcpu != kvm_get_running_vcpu()) ||
 475	    intid >= VGIC_NR_PRIVATE_IRQS)
 476		kvm_arm_halt_guest(vcpu->kvm);
 477}
 478
 479/* See vgic_access_active_prepare */
 480static void vgic_access_active_finish(struct kvm_vcpu *vcpu, u32 intid)
 481{
 482	if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
 483	     vcpu != kvm_get_running_vcpu()) ||
 484	    intid >= VGIC_NR_PRIVATE_IRQS)
 485		kvm_arm_resume_guest(vcpu->kvm);
 486}
 487
 488static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu,
 489					     gpa_t addr, unsigned int len)
 490{
 491	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 492	u32 value = 0;
 493	int i;
 494
 495	/* Loop over all IRQs affected by this read */
 496	for (i = 0; i < len * 8; i++) {
 497		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 498
 499		/*
 500		 * Even for HW interrupts, don't evaluate the HW state as
 501		 * all the guest is interested in is the virtual state.
 502		 */
 503		if (irq->active)
 504			value |= (1U << i);
 505
 506		vgic_put_irq(vcpu->kvm, irq);
 507	}
 508
 509	return value;
 510}
 511
 512unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu,
 513				    gpa_t addr, unsigned int len)
 514{
 515	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 516	u32 val;
 517
 518	mutex_lock(&vcpu->kvm->arch.config_lock);
 519	vgic_access_active_prepare(vcpu, intid);
 520
 521	val = __vgic_mmio_read_active(vcpu, addr, len);
 522
 523	vgic_access_active_finish(vcpu, intid);
 524	mutex_unlock(&vcpu->kvm->arch.config_lock);
 525
 526	return val;
 527}
 528
 529unsigned long vgic_uaccess_read_active(struct kvm_vcpu *vcpu,
 530				    gpa_t addr, unsigned int len)
 531{
 532	return __vgic_mmio_read_active(vcpu, addr, len);
 533}
 534
 535/* Must be called with irq->irq_lock held */
 536static void vgic_hw_irq_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
 537				      bool active, bool is_uaccess)
 538{
 539	if (is_uaccess)
 540		return;
 541
 542	irq->active = active;
 543	vgic_irq_set_phys_active(irq, active);
 544}
 545
 546static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
 547				    bool active)
 548{
 549	unsigned long flags;
 550	struct kvm_vcpu *requester_vcpu = kvm_get_running_vcpu();
 551
 552	raw_spin_lock_irqsave(&irq->irq_lock, flags);
 553
 554	if (irq->hw && !vgic_irq_is_sgi(irq->intid)) {
 555		vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);
 556	} else if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
 557		/*
 558		 * GICv4.1 VSGI feature doesn't track an active state,
 559		 * so let's not kid ourselves, there is nothing we can
 560		 * do here.
 561		 */
 562		irq->active = false;
 563	} else {
 564		u32 model = vcpu->kvm->arch.vgic.vgic_model;
 565		u8 active_source;
 566
 567		irq->active = active;
 568
 569		/*
 570		 * The GICv2 architecture indicates that the source CPUID for
 571		 * an SGI should be provided during an EOI which implies that
 572		 * the active state is stored somewhere, but at the same time
 573		 * this state is not architecturally exposed anywhere and we
 574		 * have no way of knowing the right source.
 575		 *
 576		 * This may lead to a VCPU not being able to receive
 577		 * additional instances of a particular SGI after migration
 578		 * for a GICv2 VM on some GIC implementations.  Oh well.
 579		 */
 580		active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0;
 581
 582		if (model == KVM_DEV_TYPE_ARM_VGIC_V2 &&
 583		    active && vgic_irq_is_sgi(irq->intid))
 584			irq->active_source = active_source;
 585	}
 586
 587	if (irq->active)
 588		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 589	else
 590		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 591}
 592
 593static void __vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
 594				      gpa_t addr, unsigned int len,
 595				      unsigned long val)
 596{
 597	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 598	int i;
 599
 600	for_each_set_bit(i, &val, len * 8) {
 601		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 602		vgic_mmio_change_active(vcpu, irq, false);
 603		vgic_put_irq(vcpu->kvm, irq);
 604	}
 605}
 606
 607void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
 608			     gpa_t addr, unsigned int len,
 609			     unsigned long val)
 610{
 611	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 612
 613	mutex_lock(&vcpu->kvm->arch.config_lock);
 614	vgic_access_active_prepare(vcpu, intid);
 615
 616	__vgic_mmio_write_cactive(vcpu, addr, len, val);
 617
 618	vgic_access_active_finish(vcpu, intid);
 619	mutex_unlock(&vcpu->kvm->arch.config_lock);
 620}
 621
 622int vgic_mmio_uaccess_write_cactive(struct kvm_vcpu *vcpu,
 623				     gpa_t addr, unsigned int len,
 624				     unsigned long val)
 625{
 626	__vgic_mmio_write_cactive(vcpu, addr, len, val);
 627	return 0;
 628}
 629
 630static void __vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
 631				      gpa_t addr, unsigned int len,
 632				      unsigned long val)
 633{
 634	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 635	int i;
 636
 637	for_each_set_bit(i, &val, len * 8) {
 638		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 639		vgic_mmio_change_active(vcpu, irq, true);
 640		vgic_put_irq(vcpu->kvm, irq);
 641	}
 642}
 643
 644void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
 645			     gpa_t addr, unsigned int len,
 646			     unsigned long val)
 647{
 648	u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
 649
 650	mutex_lock(&vcpu->kvm->arch.config_lock);
 651	vgic_access_active_prepare(vcpu, intid);
 652
 653	__vgic_mmio_write_sactive(vcpu, addr, len, val);
 654
 655	vgic_access_active_finish(vcpu, intid);
 656	mutex_unlock(&vcpu->kvm->arch.config_lock);
 657}
 658
 659int vgic_mmio_uaccess_write_sactive(struct kvm_vcpu *vcpu,
 660				     gpa_t addr, unsigned int len,
 661				     unsigned long val)
 662{
 663	__vgic_mmio_write_sactive(vcpu, addr, len, val);
 664	return 0;
 665}
 666
 667unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
 668				      gpa_t addr, unsigned int len)
 669{
 670	u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
 671	int i;
 672	u64 val = 0;
 673
 674	for (i = 0; i < len; i++) {
 675		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 676
 677		val |= (u64)irq->priority << (i * 8);
 678
 679		vgic_put_irq(vcpu->kvm, irq);
 680	}
 681
 682	return val;
 683}
 684
 685/*
 686 * We currently don't handle changing the priority of an interrupt that
 687 * is already pending on a VCPU. If there is a need for this, we would
 688 * need to make this VCPU exit and re-evaluate the priorities, potentially
 689 * leading to this interrupt getting presented now to the guest (if it has
 690 * been masked by the priority mask before).
 691 */
 692void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
 693			      gpa_t addr, unsigned int len,
 694			      unsigned long val)
 695{
 696	u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
 697	int i;
 698	unsigned long flags;
 699
 700	for (i = 0; i < len; i++) {
 701		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 702
 703		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 704		/* Narrow the priority range to what we actually support */
 705		irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
 706		if (irq->hw && vgic_irq_is_sgi(irq->intid))
 707			vgic_update_vsgi(irq);
 708		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 709
 710		vgic_put_irq(vcpu->kvm, irq);
 711	}
 712}
 713
 714unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu,
 715				    gpa_t addr, unsigned int len)
 716{
 717	u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
 718	u32 value = 0;
 719	int i;
 720
 721	for (i = 0; i < len * 4; i++) {
 722		struct vgic_irq *irq = vgic_get_vcpu_irq(vcpu, intid + i);
 723
 724		if (irq->config == VGIC_CONFIG_EDGE)
 725			value |= (2U << (i * 2));
 726
 727		vgic_put_irq(vcpu->kvm, irq);
 728	}
 729
 730	return value;
 731}
 732
 733void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
 734			    gpa_t addr, unsigned int len,
 735			    unsigned long val)
 736{
 737	u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
 738	int i;
 739	unsigned long flags;
 740
 741	for (i = 0; i < len * 4; i++) {
 742		struct vgic_irq *irq;
 743
 744		/*
 745		 * The configuration cannot be changed for SGIs in general,
 746		 * for PPIs this is IMPLEMENTATION DEFINED. The arch timer
 747		 * code relies on PPIs being level triggered, so we also
 748		 * make them read-only here.
 749		 */
 750		if (intid + i < VGIC_NR_PRIVATE_IRQS)
 751			continue;
 752
 753		irq = vgic_get_irq(vcpu->kvm, intid + i);
 754		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 755
 756		if (test_bit(i * 2 + 1, &val))
 757			irq->config = VGIC_CONFIG_EDGE;
 758		else
 759			irq->config = VGIC_CONFIG_LEVEL;
 760
 761		raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 762		vgic_put_irq(vcpu->kvm, irq);
 763	}
 764}
 765
 766u32 vgic_read_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid)
 767{
 768	int i;
 769	u32 val = 0;
 770	int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 771
 772	for (i = 0; i < 32; i++) {
 773		struct vgic_irq *irq;
 774
 775		if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs)
 776			continue;
 777
 778		irq = vgic_get_vcpu_irq(vcpu, intid + i);
 779		if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level)
 780			val |= (1U << i);
 781
 782		vgic_put_irq(vcpu->kvm, irq);
 783	}
 784
 785	return val;
 786}
 787
 788void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid,
 789				    const u32 val)
 790{
 791	int i;
 792	int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 793	unsigned long flags;
 794
 795	for (i = 0; i < 32; i++) {
 796		struct vgic_irq *irq;
 797		bool new_level;
 798
 799		if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs)
 800			continue;
 801
 802		irq = vgic_get_vcpu_irq(vcpu, intid + i);
 803
 804		/*
 805		 * Line level is set irrespective of irq type
 806		 * (level or edge) to avoid dependency that VM should
 807		 * restore irq config before line level.
 808		 */
 809		new_level = !!(val & (1U << i));
 810		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 811		irq->line_level = new_level;
 812		if (new_level)
 813			vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 814		else
 815			raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 816
 817		vgic_put_irq(vcpu->kvm, irq);
 818	}
 819}
 820
 821static int match_region(const void *key, const void *elt)
 822{
 823	const unsigned int offset = (unsigned long)key;
 824	const struct vgic_register_region *region = elt;
 825
 826	if (offset < region->reg_offset)
 827		return -1;
 828
 829	if (offset >= region->reg_offset + region->len)
 830		return 1;
 831
 832	return 0;
 833}
 834
 835const struct vgic_register_region *
 836vgic_find_mmio_region(const struct vgic_register_region *regions,
 837		      int nr_regions, unsigned int offset)
 838{
 839	return bsearch((void *)(uintptr_t)offset, regions, nr_regions,
 840		       sizeof(regions[0]), match_region);
 841}
 842
 843void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 844{
 845	if (kvm_vgic_global_state.type == VGIC_V2)
 846		vgic_v2_set_vmcr(vcpu, vmcr);
 847	else
 848		vgic_v3_set_vmcr(vcpu, vmcr);
 849}
 850
 851void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
 852{
 853	if (kvm_vgic_global_state.type == VGIC_V2)
 854		vgic_v2_get_vmcr(vcpu, vmcr);
 855	else
 856		vgic_v3_get_vmcr(vcpu, vmcr);
 857}
 858
 859/*
 860 * kvm_mmio_read_buf() returns a value in a format where it can be converted
 861 * to a byte array and be directly observed as the guest wanted it to appear
 862 * in memory if it had done the store itself, which is LE for the GIC, as the
 863 * guest knows the GIC is always LE.
 864 *
 865 * We convert this value to the CPUs native format to deal with it as a data
 866 * value.
 867 */
 868unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len)
 869{
 870	unsigned long data = kvm_mmio_read_buf(val, len);
 871
 872	switch (len) {
 873	case 1:
 874		return data;
 875	case 2:
 876		return le16_to_cpu(data);
 877	case 4:
 878		return le32_to_cpu(data);
 879	default:
 880		return le64_to_cpu(data);
 881	}
 882}
 883
 884/*
 885 * kvm_mmio_write_buf() expects a value in a format such that if converted to
 886 * a byte array it is observed as the guest would see it if it could perform
 887 * the load directly.  Since the GIC is LE, and the guest knows this, the
 888 * guest expects a value in little endian format.
 889 *
 890 * We convert the data value from the CPUs native format to LE so that the
 891 * value is returned in the proper format.
 892 */
 893void vgic_data_host_to_mmio_bus(void *buf, unsigned int len,
 894				unsigned long data)
 895{
 896	switch (len) {
 897	case 1:
 898		break;
 899	case 2:
 900		data = cpu_to_le16(data);
 901		break;
 902	case 4:
 903		data = cpu_to_le32(data);
 904		break;
 905	default:
 906		data = cpu_to_le64(data);
 907	}
 908
 909	kvm_mmio_write_buf(buf, len, data);
 910}
 911
 912static
 913struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev)
 914{
 915	return container_of(dev, struct vgic_io_device, dev);
 916}
 917
 918static bool check_region(const struct kvm *kvm,
 919			 const struct vgic_register_region *region,
 920			 gpa_t addr, int len)
 921{
 922	int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
 923
 924	switch (len) {
 925	case sizeof(u8):
 926		flags = VGIC_ACCESS_8bit;
 927		break;
 928	case sizeof(u32):
 929		flags = VGIC_ACCESS_32bit;
 930		break;
 931	case sizeof(u64):
 932		flags = VGIC_ACCESS_64bit;
 933		break;
 934	default:
 935		return false;
 936	}
 937
 938	if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {
 939		if (!region->bits_per_irq)
 940			return true;
 941
 942		/* Do we access a non-allocated IRQ? */
 943		return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;
 944	}
 945
 946	return false;
 947}
 948
 949const struct vgic_register_region *
 950vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 951		     gpa_t addr, int len)
 952{
 953	const struct vgic_register_region *region;
 954
 955	region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
 956				       addr - iodev->base_addr);
 957	if (!region || !check_region(vcpu->kvm, region, addr, len))
 958		return NULL;
 959
 960	return region;
 961}
 962
 963static int vgic_uaccess_read(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 964			     gpa_t addr, u32 *val)
 965{
 966	const struct vgic_register_region *region;
 967	struct kvm_vcpu *r_vcpu;
 968
 969	region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
 970	if (!region) {
 971		*val = 0;
 972		return 0;
 973	}
 974
 975	r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
 976	if (region->uaccess_read)
 977		*val = region->uaccess_read(r_vcpu, addr, sizeof(u32));
 978	else
 979		*val = region->read(r_vcpu, addr, sizeof(u32));
 980
 981	return 0;
 982}
 983
 984static int vgic_uaccess_write(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
 985			      gpa_t addr, const u32 *val)
 986{
 987	const struct vgic_register_region *region;
 988	struct kvm_vcpu *r_vcpu;
 989
 990	region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
 991	if (!region)
 992		return 0;
 993
 994	r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
 995	if (region->uaccess_write)
 996		return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val);
 997
 998	region->write(r_vcpu, addr, sizeof(u32), *val);
 999	return 0;
1000}
1001
1002/*
1003 * Userland access to VGIC registers.
1004 */
1005int vgic_uaccess(struct kvm_vcpu *vcpu, struct vgic_io_device *dev,
1006		 bool is_write, int offset, u32 *val)
1007{
1008	if (is_write)
1009		return vgic_uaccess_write(vcpu, dev, offset, val);
1010	else
1011		return vgic_uaccess_read(vcpu, dev, offset, val);
1012}
1013
1014static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
1015			      gpa_t addr, int len, void *val)
1016{
1017	struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
1018	const struct vgic_register_region *region;
1019	unsigned long data = 0;
1020
1021	region = vgic_get_mmio_region(vcpu, iodev, addr, len);
1022	if (!region) {
1023		memset(val, 0, len);
1024		return 0;
1025	}
1026
1027	switch (iodev->iodev_type) {
1028	case IODEV_CPUIF:
1029		data = region->read(vcpu, addr, len);
1030		break;
1031	case IODEV_DIST:
1032		data = region->read(vcpu, addr, len);
1033		break;
1034	case IODEV_REDIST:
1035		data = region->read(iodev->redist_vcpu, addr, len);
1036		break;
1037	case IODEV_ITS:
1038		data = region->its_read(vcpu->kvm, iodev->its, addr, len);
1039		break;
1040	}
1041
1042	vgic_data_host_to_mmio_bus(val, len, data);
1043	return 0;
1044}
1045
1046static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
1047			       gpa_t addr, int len, const void *val)
1048{
1049	struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
1050	const struct vgic_register_region *region;
1051	unsigned long data = vgic_data_mmio_bus_to_host(val, len);
1052
1053	region = vgic_get_mmio_region(vcpu, iodev, addr, len);
1054	if (!region)
1055		return 0;
1056
1057	switch (iodev->iodev_type) {
1058	case IODEV_CPUIF:
1059		region->write(vcpu, addr, len, data);
1060		break;
1061	case IODEV_DIST:
1062		region->write(vcpu, addr, len, data);
1063		break;
1064	case IODEV_REDIST:
1065		region->write(iodev->redist_vcpu, addr, len, data);
1066		break;
1067	case IODEV_ITS:
1068		region->its_write(vcpu->kvm, iodev->its, addr, len, data);
1069		break;
1070	}
1071
1072	return 0;
1073}
1074
1075const struct kvm_io_device_ops kvm_io_gic_ops = {
1076	.read = dispatch_mmio_read,
1077	.write = dispatch_mmio_write,
1078};
1079
1080int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
1081			     enum vgic_type type)
1082{
1083	struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;
1084	unsigned int len;
1085
1086	switch (type) {
1087	case VGIC_V2:
1088		len = vgic_v2_init_dist_iodev(io_device);
1089		break;
1090	case VGIC_V3:
1091		len = vgic_v3_init_dist_iodev(io_device);
1092		break;
1093	default:
1094		BUG();
1095	}
1096
1097	io_device->base_addr = dist_base_address;
1098	io_device->iodev_type = IODEV_DIST;
1099	io_device->redist_vcpu = NULL;
1100
1101	return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address,
1102				       len, &io_device->dev);
1103}