vgic-its.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * GICv3 ITS emulation
   4 *
   5 * Copyright (C) 2015,2016 ARM Ltd.
   6 * Author: Andre Przywara <andre.przywara@arm.com>
   7 */
   8
   9#include <linux/cpu.h>
  10#include <linux/kvm.h>
  11#include <linux/kvm_host.h>
  12#include <linux/interrupt.h>
  13#include <linux/list.h>
  14#include <linux/uaccess.h>
  15#include <linux/list_sort.h>
  16
  17#include <linux/irqchip/arm-gic-v3.h>
  18
  19#include <asm/kvm_emulate.h>
  20#include <asm/kvm_arm.h>
  21#include <asm/kvm_mmu.h>
  22
  23#include "vgic.h"
  24#include "vgic-mmio.h"
  25
  26static struct kvm_device_ops kvm_arm_vgic_its_ops;
  27
  28static int vgic_its_save_tables_v0(struct vgic_its *its);
  29static int vgic_its_restore_tables_v0(struct vgic_its *its);
  30static int vgic_its_commit_v0(struct vgic_its *its);
  31static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
  32			     struct kvm_vcpu *filter_vcpu, bool needs_inv);
  33
  34#define vgic_its_read_entry_lock(i, g, valp, t)				\
  35	({								\
  36		int __sz = vgic_its_get_abi(i)->t##_esz;		\
  37		struct kvm *__k = (i)->dev->kvm;			\
  38		int __ret;						\
  39									\
  40		BUILD_BUG_ON(NR_ITS_ABIS == 1 &&			\
  41			     sizeof(*(valp)) != ABI_0_ESZ);		\
  42		if (NR_ITS_ABIS > 1 &&					\
  43		    KVM_BUG_ON(__sz != sizeof(*(valp)), __k))		\
  44			__ret = -EINVAL;				\
  45		else							\
  46			__ret = kvm_read_guest_lock(__k, (g),		\
  47						    valp, __sz);	\
  48		__ret;							\
  49	})
  50
  51#define vgic_its_write_entry_lock(i, g, val, t)				\
  52	({								\
  53		int __sz = vgic_its_get_abi(i)->t##_esz;		\
  54		struct kvm *__k = (i)->dev->kvm;			\
  55		typeof(val) __v = (val);				\
  56		int __ret;						\
  57									\
  58		BUILD_BUG_ON(NR_ITS_ABIS == 1 &&			\
  59			     sizeof(__v) != ABI_0_ESZ);			\
  60		if (NR_ITS_ABIS > 1 &&					\
  61		    KVM_BUG_ON(__sz != sizeof(__v), __k))		\
  62			__ret = -EINVAL;				\
  63		else							\
  64			__ret = vgic_write_guest_lock(__k, (g),		\
  65						      &__v, __sz);	\
  66		__ret;							\
  67	})
  68
  69/*
  70 * Creates a new (reference to a) struct vgic_irq for a given LPI.
  71 * If this LPI is already mapped on another ITS, we increase its refcount
  72 * and return a pointer to the existing structure.
  73 * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
  74 * This function returns a pointer to the _unlocked_ structure.
  75 */
  76static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
  77				     struct kvm_vcpu *vcpu)
  78{
  79	struct vgic_dist *dist = &kvm->arch.vgic;
  80	struct vgic_irq *irq = vgic_get_irq(kvm, intid), *oldirq;
  81	unsigned long flags;
  82	int ret;
  83
  84	/* In this case there is no put, since we keep the reference. */
  85	if (irq)
  86		return irq;
  87
  88	irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT);
  89	if (!irq)
  90		return ERR_PTR(-ENOMEM);
  91
  92	ret = xa_reserve_irq(&dist->lpi_xa, intid, GFP_KERNEL_ACCOUNT);
  93	if (ret) {
  94		kfree(irq);
  95		return ERR_PTR(ret);
  96	}
  97
  98	INIT_LIST_HEAD(&irq->ap_list);
  99	raw_spin_lock_init(&irq->irq_lock);
 100
 101	irq->config = VGIC_CONFIG_EDGE;
 102	refcount_set(&irq->refcount, 1);
 103	irq->intid = intid;
 104	irq->target_vcpu = vcpu;
 105	irq->group = 1;
 106
 107	xa_lock_irqsave(&dist->lpi_xa, flags);
 108
 109	/*
 110	 * There could be a race with another vgic_add_lpi(), so we need to
 111	 * check that we don't add a second list entry with the same LPI.
 112	 */
 113	oldirq = xa_load(&dist->lpi_xa, intid);
 114	if (vgic_try_get_irq_ref(oldirq)) {
 115		/* Someone was faster with adding this LPI, lets use that. */
 116		kfree(irq);
 117		irq = oldirq;
 118	} else {
 119		ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0));
 120	}
 121
 122	xa_unlock_irqrestore(&dist->lpi_xa, flags);
 123
 124	if (ret) {
 125		xa_release(&dist->lpi_xa, intid);
 126		kfree(irq);
 127
 128		return ERR_PTR(ret);
 129	}
 130
 131	/*
 132	 * We "cache" the configuration table entries in our struct vgic_irq's.
 133	 * However we only have those structs for mapped IRQs, so we read in
 134	 * the respective config data from memory here upon mapping the LPI.
 135	 *
 136	 * Should any of these fail, behave as if we couldn't create the LPI
 137	 * by dropping the refcount and returning the error.
 138	 */
 139	ret = update_lpi_config(kvm, irq, NULL, false);
 140	if (ret) {
 141		vgic_put_irq(kvm, irq);
 142		return ERR_PTR(ret);
 143	}
 144
 145	ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
 146	if (ret) {
 147		vgic_put_irq(kvm, irq);
 148		return ERR_PTR(ret);
 149	}
 150
 151	return irq;
 152}
 153
 154/**
 155 * struct vgic_its_abi - ITS abi ops and settings
 156 * @cte_esz: collection table entry size
 157 * @dte_esz: device table entry size
 158 * @ite_esz: interrupt translation table entry size
 159 * @save_tables: save the ITS tables into guest RAM
 160 * @restore_tables: restore the ITS internal structs from tables
 161 *  stored in guest RAM
 162 * @commit: initialize the registers which expose the ABI settings,
 163 *  especially the entry sizes
 164 */
 165struct vgic_its_abi {
 166	int cte_esz;
 167	int dte_esz;
 168	int ite_esz;
 169	int (*save_tables)(struct vgic_its *its);
 170	int (*restore_tables)(struct vgic_its *its);
 171	int (*commit)(struct vgic_its *its);
 172};
 173
 174#define ABI_0_ESZ	8
 175#define ESZ_MAX		ABI_0_ESZ
 176
 177static const struct vgic_its_abi its_table_abi_versions[] = {
 178	[0] = {
 179	 .cte_esz = ABI_0_ESZ,
 180	 .dte_esz = ABI_0_ESZ,
 181	 .ite_esz = ABI_0_ESZ,
 182	 .save_tables = vgic_its_save_tables_v0,
 183	 .restore_tables = vgic_its_restore_tables_v0,
 184	 .commit = vgic_its_commit_v0,
 185	},
 186};
 187
 188#define NR_ITS_ABIS	ARRAY_SIZE(its_table_abi_versions)
 189
 190inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
 191{
 192	return &its_table_abi_versions[its->abi_rev];
 193}
 194
 195static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
 196{
 197	const struct vgic_its_abi *abi;
 198
 199	its->abi_rev = rev;
 200	abi = vgic_its_get_abi(its);
 201	return abi->commit(its);
 202}
 203
 204/*
 205 * Find and returns a device in the device table for an ITS.
 206 * Must be called with the its_lock mutex held.
 207 */
 208static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
 209{
 210	struct its_device *device;
 211
 212	list_for_each_entry(device, &its->device_list, dev_list)
 213		if (device_id == device->device_id)
 214			return device;
 215
 216	return NULL;
 217}
 218
 219/*
 220 * Find and returns an interrupt translation table entry (ITTE) for a given
 221 * Device ID/Event ID pair on an ITS.
 222 * Must be called with the its_lock mutex held.
 223 */
 224static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
 225				  u32 event_id)
 226{
 227	struct its_device *device;
 228	struct its_ite *ite;
 229
 230	device = find_its_device(its, device_id);
 231	if (device == NULL)
 232		return NULL;
 233
 234	list_for_each_entry(ite, &device->itt_head, ite_list)
 235		if (ite->event_id == event_id)
 236			return ite;
 237
 238	return NULL;
 239}
 240
 241/* To be used as an iterator this macro misses the enclosing parentheses */
 242#define for_each_lpi_its(dev, ite, its) \
 243	list_for_each_entry(dev, &(its)->device_list, dev_list) \
 244		list_for_each_entry(ite, &(dev)->itt_head, ite_list)
 245
 246#define GIC_LPI_OFFSET 8192
 247
 248#define VITS_TYPER_IDBITS		16
 249#define VITS_MAX_EVENTID		(BIT(VITS_TYPER_IDBITS) - 1)
 250#define VITS_TYPER_DEVBITS		16
 251#define VITS_MAX_DEVID			(BIT(VITS_TYPER_DEVBITS) - 1)
 252#define VITS_DTE_MAX_DEVID_OFFSET	(BIT(14) - 1)
 253#define VITS_ITE_MAX_EVENTID_OFFSET	(BIT(16) - 1)
 254
 255/*
 256 * Finds and returns a collection in the ITS collection table.
 257 * Must be called with the its_lock mutex held.
 258 */
 259static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
 260{
 261	struct its_collection *collection;
 262
 263	list_for_each_entry(collection, &its->collection_list, coll_list) {
 264		if (coll_id == collection->collection_id)
 265			return collection;
 266	}
 267
 268	return NULL;
 269}
 270
 271#define LPI_PROP_ENABLE_BIT(p)	((p) & LPI_PROP_ENABLED)
 272#define LPI_PROP_PRIORITY(p)	((p) & 0xfc)
 273
 274/*
 275 * Reads the configuration data for a given LPI from guest memory and
 276 * updates the fields in struct vgic_irq.
 277 * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
 278 * VCPU. Unconditionally applies if filter_vcpu is NULL.
 279 */
 280static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 281			     struct kvm_vcpu *filter_vcpu, bool needs_inv)
 282{
 283	u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
 284	u8 prop;
 285	int ret;
 286	unsigned long flags;
 287
 288	ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
 289				  &prop, 1);
 290
 291	if (ret)
 292		return ret;
 293
 294	raw_spin_lock_irqsave(&irq->irq_lock, flags);
 295
 296	if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
 297		irq->priority = LPI_PROP_PRIORITY(prop);
 298		irq->enabled = LPI_PROP_ENABLE_BIT(prop);
 299
 300		if (!irq->hw) {
 301			vgic_queue_irq_unlock(kvm, irq, flags);
 302			return 0;
 303		}
 304	}
 305
 306	if (irq->hw)
 307		ret = its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
 308
 309	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 310	return ret;
 311}
 312
 313static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
 314{
 315	struct its_vlpi_map map;
 316	int ret;
 317
 318	guard(raw_spinlock_irqsave)(&irq->irq_lock);
 319	irq->target_vcpu = vcpu;
 320
 321	if (!irq->hw)
 322		return 0;
 323
 324	ret = its_get_vlpi(irq->host_irq, &map);
 325	if (ret)
 326		return ret;
 327
 328	if (map.vpe)
 329		atomic_dec(&map.vpe->vlpi_count);
 330
 331	map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
 332	atomic_inc(&map.vpe->vlpi_count);
 333	return its_map_vlpi(irq->host_irq, &map);
 334}
 335
 336static struct kvm_vcpu *collection_to_vcpu(struct kvm *kvm,
 337					   struct its_collection *col)
 338{
 339	return kvm_get_vcpu_by_id(kvm, col->target_addr);
 340}
 341
 342/*
 343 * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
 344 * is targeting) to the VGIC's view, which deals with target VCPUs.
 345 * Needs to be called whenever either the collection for a LPIs has
 346 * changed or the collection itself got retargeted.
 347 */
 348static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 349{
 350	struct kvm_vcpu *vcpu;
 351
 352	if (!its_is_collection_mapped(ite->collection))
 353		return;
 354
 355	vcpu = collection_to_vcpu(kvm, ite->collection);
 356	update_affinity(ite->irq, vcpu);
 357}
 358
 359/*
 360 * Updates the target VCPU for every LPI targeting this collection.
 361 * Must be called with the its_lock mutex held.
 362 */
 363static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
 364				       struct its_collection *coll)
 365{
 366	struct its_device *device;
 367	struct its_ite *ite;
 368
 369	for_each_lpi_its(device, ite, its) {
 370		if (ite->collection != coll)
 371			continue;
 372
 373		update_affinity_ite(kvm, ite);
 374	}
 375}
 376
 377static u32 max_lpis_propbaser(u64 propbaser)
 378{
 379	int nr_idbits = (propbaser & 0x1f) + 1;
 380
 381	return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
 382}
 383
 384/*
 385 * Sync the pending table pending bit of LPIs targeting @vcpu
 386 * with our own data structures. This relies on the LPI being
 387 * mapped before.
 388 */
 389static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
 390{
 391	gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
 392	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 393	unsigned long intid, flags;
 394	struct vgic_irq *irq;
 395	int last_byte_offset = -1;
 396	int ret = 0;
 397	u8 pendmask;
 398
 399	xa_for_each(&dist->lpi_xa, intid, irq) {
 400		int byte_offset, bit_nr;
 401
 402		byte_offset = intid / BITS_PER_BYTE;
 403		bit_nr = intid % BITS_PER_BYTE;
 404
 405		/*
 406		 * For contiguously allocated LPIs chances are we just read
 407		 * this very same byte in the last iteration. Reuse that.
 408		 */
 409		if (byte_offset != last_byte_offset) {
 410			ret = kvm_read_guest_lock(vcpu->kvm,
 411						  pendbase + byte_offset,
 412						  &pendmask, 1);
 413			if (ret)
 414				return ret;
 415
 416			last_byte_offset = byte_offset;
 417		}
 418
 419		irq = vgic_get_irq(vcpu->kvm, intid);
 420		if (!irq)
 421			continue;
 422
 423		raw_spin_lock_irqsave(&irq->irq_lock, flags);
 424		if (irq->target_vcpu == vcpu)
 425			irq->pending_latch = pendmask & (1U << bit_nr);
 426		vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 427		vgic_put_irq(vcpu->kvm, irq);
 428	}
 429
 430	return ret;
 431}
 432
 433static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
 434					      struct vgic_its *its,
 435					      gpa_t addr, unsigned int len)
 436{
 437	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 438	u64 reg = GITS_TYPER_PLPIS;
 439
 440	/*
 441	 * We use linear CPU numbers for redistributor addressing,
 442	 * so GITS_TYPER.PTA is 0.
 443	 * Also we force all PROPBASER registers to be the same, so
 444	 * CommonLPIAff is 0 as well.
 445	 * To avoid memory waste in the guest, we keep the number of IDBits and
 446	 * DevBits low - as least for the time being.
 447	 */
 448	reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
 449	reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
 450	reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
 451
 452	return extract_bytes(reg, addr & 7, len);
 453}
 454
 455static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
 456					     struct vgic_its *its,
 457					     gpa_t addr, unsigned int len)
 458{
 459	u32 val;
 460
 461	val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
 462	val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
 463	return val;
 464}
 465
 466static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
 467					    struct vgic_its *its,
 468					    gpa_t addr, unsigned int len,
 469					    unsigned long val)
 470{
 471	u32 rev = GITS_IIDR_REV(val);
 472
 473	if (rev >= NR_ITS_ABIS)
 474		return -EINVAL;
 475	return vgic_its_set_abi(its, rev);
 476}
 477
 478static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
 479					       struct vgic_its *its,
 480					       gpa_t addr, unsigned int len)
 481{
 482	switch (addr & 0xffff) {
 483	case GITS_PIDR0:
 484		return 0x92;	/* part number, bits[7:0] */
 485	case GITS_PIDR1:
 486		return 0xb4;	/* part number, bits[11:8] */
 487	case GITS_PIDR2:
 488		return GIC_PIDR2_ARCH_GICv3 | 0x0b;
 489	case GITS_PIDR4:
 490		return 0x40;	/* This is a 64K software visible page */
 491	/* The following are the ID registers for (any) GIC. */
 492	case GITS_CIDR0:
 493		return 0x0d;
 494	case GITS_CIDR1:
 495		return 0xf0;
 496	case GITS_CIDR2:
 497		return 0x05;
 498	case GITS_CIDR3:
 499		return 0xb1;
 500	}
 501
 502	return 0;
 503}
 504
 505static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db)
 506{
 507	struct kvm_io_device *kvm_io_dev;
 508	struct vgic_io_device *iodev;
 509
 510	kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db);
 511	if (!kvm_io_dev)
 512		return ERR_PTR(-EINVAL);
 513
 514	if (kvm_io_dev->ops != &kvm_io_gic_ops)
 515		return ERR_PTR(-EINVAL);
 516
 517	iodev = container_of(kvm_io_dev, struct vgic_io_device, dev);
 518	if (iodev->iodev_type != IODEV_ITS)
 519		return ERR_PTR(-EINVAL);
 520
 521	return iodev->its;
 522}
 523
 524static unsigned long vgic_its_cache_key(u32 devid, u32 eventid)
 525{
 526	return (((unsigned long)devid) << VITS_TYPER_IDBITS) | eventid;
 527
 528}
 529
 530static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db,
 531					     u32 devid, u32 eventid)
 532{
 533	unsigned long cache_key = vgic_its_cache_key(devid, eventid);
 534	struct vgic_its *its;
 535	struct vgic_irq *irq;
 536
 537	if (devid > VITS_MAX_DEVID || eventid > VITS_MAX_EVENTID)
 538		return NULL;
 539
 540	its = __vgic_doorbell_to_its(kvm, db);
 541	if (IS_ERR(its))
 542		return NULL;
 543
 544	rcu_read_lock();
 545
 546	irq = xa_load(&its->translation_cache, cache_key);
 547	if (!vgic_try_get_irq_ref(irq))
 548		irq = NULL;
 549
 550	rcu_read_unlock();
 551
 552	return irq;
 553}
 554
 555static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
 556				       u32 devid, u32 eventid,
 557				       struct vgic_irq *irq)
 558{
 559	unsigned long cache_key = vgic_its_cache_key(devid, eventid);
 560	struct vgic_irq *old;
 561
 562	/* Do not cache a directly injected interrupt */
 563	if (irq->hw)
 564		return;
 565
 566	/*
 567	 * The irq refcount is guaranteed to be nonzero while holding the
 568	 * its_lock, as the ITE (and the reference it holds) cannot be freed.
 569	 */
 570	lockdep_assert_held(&its->its_lock);
 571	vgic_get_irq_ref(irq);
 572
 573	old = xa_store(&its->translation_cache, cache_key, irq, GFP_KERNEL_ACCOUNT);
 574
 575	/*
 576	 * Put the reference taken on @irq if the store fails. Intentionally do
 577	 * not return the error as the translation cache is best effort.
 578	 */
 579	if (xa_is_err(old)) {
 580		vgic_put_irq(kvm, irq);
 581		return;
 582	}
 583
 584	/*
 585	 * We could have raced with another CPU caching the same
 586	 * translation behind our back, ensure we don't leak a
 587	 * reference if that is the case.
 588	 */
 589	if (old)
 590		vgic_put_irq(kvm, old);
 591}
 592
 593static void vgic_its_invalidate_cache(struct vgic_its *its)
 594{
 595	struct kvm *kvm = its->dev->kvm;
 596	struct vgic_irq *irq;
 597	unsigned long idx;
 598
 599	xa_for_each(&its->translation_cache, idx, irq) {
 600		xa_erase(&its->translation_cache, idx);
 601		vgic_put_irq(kvm, irq);
 602	}
 603}
 604
 605void vgic_its_invalidate_all_caches(struct kvm *kvm)
 606{
 607	struct kvm_device *dev;
 608	struct vgic_its *its;
 609
 610	rcu_read_lock();
 611
 612	list_for_each_entry_rcu(dev, &kvm->devices, vm_node) {
 613		if (dev->ops != &kvm_arm_vgic_its_ops)
 614			continue;
 615
 616		its = dev->private;
 617		vgic_its_invalidate_cache(its);
 618	}
 619
 620	rcu_read_unlock();
 621}
 622
 623int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
 624			 u32 devid, u32 eventid, struct vgic_irq **irq)
 625{
 626	struct kvm_vcpu *vcpu;
 627	struct its_ite *ite;
 628
 629	if (!its->enabled)
 630		return -EBUSY;
 631
 632	ite = find_ite(its, devid, eventid);
 633	if (!ite || !its_is_collection_mapped(ite->collection))
 634		return E_ITS_INT_UNMAPPED_INTERRUPT;
 635
 636	vcpu = collection_to_vcpu(kvm, ite->collection);
 637	if (!vcpu)
 638		return E_ITS_INT_UNMAPPED_INTERRUPT;
 639
 640	if (!vgic_lpis_enabled(vcpu))
 641		return -EBUSY;
 642
 643	vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
 644
 645	*irq = ite->irq;
 646	return 0;
 647}
 648
 649struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi)
 650{
 651	u64 address;
 652
 653	if (!vgic_has_its(kvm))
 654		return ERR_PTR(-ENODEV);
 655
 656	if (!(msi->flags & KVM_MSI_VALID_DEVID))
 657		return ERR_PTR(-EINVAL);
 658
 659	address = (u64)msi->address_hi << 32 | msi->address_lo;
 660
 661	return __vgic_doorbell_to_its(kvm, address);
 662}
 663
 664/*
 665 * Find the target VCPU and the LPI number for a given devid/eventid pair
 666 * and make this IRQ pending, possibly injecting it.
 667 * Must be called with the its_lock mutex held.
 668 * Returns 0 on success, a positive error value for any ITS mapping
 669 * related errors and negative error values for generic errors.
 670 */
 671static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
 672				u32 devid, u32 eventid)
 673{
 674	struct vgic_irq *irq = NULL;
 675	unsigned long flags;
 676	int err;
 677
 678	err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
 679	if (err)
 680		return err;
 681
 682	if (irq->hw)
 683		return irq_set_irqchip_state(irq->host_irq,
 684					     IRQCHIP_STATE_PENDING, true);
 685
 686	raw_spin_lock_irqsave(&irq->irq_lock, flags);
 687	irq->pending_latch = true;
 688	vgic_queue_irq_unlock(kvm, irq, flags);
 689
 690	return 0;
 691}
 692
 693int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi)
 694{
 695	struct vgic_irq *irq;
 696	unsigned long flags;
 697	phys_addr_t db;
 698
 699	db = (u64)msi->address_hi << 32 | msi->address_lo;
 700	irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data);
 701	if (!irq)
 702		return -EWOULDBLOCK;
 703
 704	raw_spin_lock_irqsave(&irq->irq_lock, flags);
 705	irq->pending_latch = true;
 706	vgic_queue_irq_unlock(kvm, irq, flags);
 707	vgic_put_irq(kvm, irq);
 708
 709	return 0;
 710}
 711
 712/*
 713 * Queries the KVM IO bus framework to get the ITS pointer from the given
 714 * doorbell address.
 715 * We then call vgic_its_trigger_msi() with the decoded data.
 716 * According to the KVM_SIGNAL_MSI API description returns 1 on success.
 717 */
 718int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
 719{
 720	struct vgic_its *its;
 721	int ret;
 722
 723	if (!vgic_its_inject_cached_translation(kvm, msi))
 724		return 1;
 725
 726	its = vgic_msi_to_its(kvm, msi);
 727	if (IS_ERR(its))
 728		return PTR_ERR(its);
 729
 730	mutex_lock(&its->its_lock);
 731	ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
 732	mutex_unlock(&its->its_lock);
 733
 734	if (ret < 0)
 735		return ret;
 736
 737	/*
 738	 * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
 739	 * if the guest has blocked the MSI. So we map any LPI mapping
 740	 * related error to that.
 741	 */
 742	if (ret)
 743		return 0;
 744	else
 745		return 1;
 746}
 747
 748/* Requires the its_lock to be held. */
 749static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 750{
 751	struct vgic_irq *irq = ite->irq;
 752	list_del(&ite->ite_list);
 753
 754	/* This put matches the get in vgic_add_lpi. */
 755	if (irq) {
 756		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
 757			if (irq->hw)
 758				its_unmap_vlpi(ite->irq->host_irq);
 759
 760			irq->hw = false;
 761		}
 762
 763		vgic_put_irq(kvm, ite->irq);
 764	}
 765
 766	kfree(ite);
 767}
 768
 769static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 770{
 771	return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
 772}
 773
 774#define its_cmd_get_command(cmd)	its_cmd_mask_field(cmd, 0,  0,  8)
 775#define its_cmd_get_deviceid(cmd)	its_cmd_mask_field(cmd, 0, 32, 32)
 776#define its_cmd_get_size(cmd)		(its_cmd_mask_field(cmd, 1,  0,  5) + 1)
 777#define its_cmd_get_id(cmd)		its_cmd_mask_field(cmd, 1,  0, 32)
 778#define its_cmd_get_physical_id(cmd)	its_cmd_mask_field(cmd, 1, 32, 32)
 779#define its_cmd_get_collection(cmd)	its_cmd_mask_field(cmd, 2,  0, 16)
 780#define its_cmd_get_ittaddr(cmd)	(its_cmd_mask_field(cmd, 2,  8, 44) << 8)
 781#define its_cmd_get_target_addr(cmd)	its_cmd_mask_field(cmd, 2, 16, 32)
 782#define its_cmd_get_validbit(cmd)	its_cmd_mask_field(cmd, 2, 63,  1)
 783
 784/*
 785 * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
 786 * Must be called with the its_lock mutex held.
 787 */
 788static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 789				       u64 *its_cmd)
 790{
 791	u32 device_id = its_cmd_get_deviceid(its_cmd);
 792	u32 event_id = its_cmd_get_id(its_cmd);
 793	struct its_ite *ite;
 794
 795	ite = find_ite(its, device_id, event_id);
 796	if (ite && its_is_collection_mapped(ite->collection)) {
 797		struct its_device *device = find_its_device(its, device_id);
 798		int ite_esz = vgic_its_get_abi(its)->ite_esz;
 799		gpa_t gpa = device->itt_addr + ite->event_id * ite_esz;
 800		/*
 801		 * Though the spec talks about removing the pending state, we
 802		 * don't bother here since we clear the ITTE anyway and the
 803		 * pending state is a property of the ITTE struct.
 804		 */
 805		vgic_its_invalidate_cache(its);
 806
 807		its_free_ite(kvm, ite);
 808
 809		return vgic_its_write_entry_lock(its, gpa, 0ULL, ite);
 810	}
 811
 812	return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
 813}
 814
 815/*
 816 * The MOVI command moves an ITTE to a different collection.
 817 * Must be called with the its_lock mutex held.
 818 */
 819static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 820				    u64 *its_cmd)
 821{
 822	u32 device_id = its_cmd_get_deviceid(its_cmd);
 823	u32 event_id = its_cmd_get_id(its_cmd);
 824	u32 coll_id = its_cmd_get_collection(its_cmd);
 825	struct kvm_vcpu *vcpu;
 826	struct its_ite *ite;
 827	struct its_collection *collection;
 828
 829	ite = find_ite(its, device_id, event_id);
 830	if (!ite)
 831		return E_ITS_MOVI_UNMAPPED_INTERRUPT;
 832
 833	if (!its_is_collection_mapped(ite->collection))
 834		return E_ITS_MOVI_UNMAPPED_COLLECTION;
 835
 836	collection = find_collection(its, coll_id);
 837	if (!its_is_collection_mapped(collection))
 838		return E_ITS_MOVI_UNMAPPED_COLLECTION;
 839
 840	ite->collection = collection;
 841	vcpu = collection_to_vcpu(kvm, collection);
 842
 843	vgic_its_invalidate_cache(its);
 844
 845	return update_affinity(ite->irq, vcpu);
 846}
 847
 848static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)
 849{
 850	gfn_t gfn = gpa >> PAGE_SHIFT;
 851	int idx;
 852	bool ret;
 853
 854	idx = srcu_read_lock(&its->dev->kvm->srcu);
 855	ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
 856	srcu_read_unlock(&its->dev->kvm->srcu, idx);
 857	return ret;
 858}
 859
 860/*
 861 * Check whether an ID can be stored into the corresponding guest table.
 862 * For a direct table this is pretty easy, but gets a bit nasty for
 863 * indirect tables. We check whether the resulting guest physical address
 864 * is actually valid (covered by a memslot and guest accessible).
 865 * For this we have to read the respective first level entry.
 866 */
 867static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
 868			      gpa_t *eaddr)
 869{
 870	int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 871	u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
 872	phys_addr_t base = GITS_BASER_ADDR_48_to_52(baser);
 873	int esz = GITS_BASER_ENTRY_SIZE(baser);
 874	int index;
 875
 876	switch (type) {
 877	case GITS_BASER_TYPE_DEVICE:
 878		if (id > VITS_MAX_DEVID)
 879			return false;
 880		break;
 881	case GITS_BASER_TYPE_COLLECTION:
 882		/* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
 883		if (id >= BIT_ULL(16))
 884			return false;
 885		break;
 886	default:
 887		return false;
 888	}
 889
 890	if (!(baser & GITS_BASER_INDIRECT)) {
 891		phys_addr_t addr;
 892
 893		if (id >= (l1_tbl_size / esz))
 894			return false;
 895
 896		addr = base + id * esz;
 897
 898		if (eaddr)
 899			*eaddr = addr;
 900
 901		return __is_visible_gfn_locked(its, addr);
 902	}
 903
 904	/* calculate and check the index into the 1st level */
 905	index = id / (SZ_64K / esz);
 906	if (index >= (l1_tbl_size / sizeof(u64)))
 907		return false;
 908
 909	/* Each 1st level entry is represented by a 64-bit value. */
 910	if (kvm_read_guest_lock(its->dev->kvm,
 911			   base + index * sizeof(indirect_ptr),
 912			   &indirect_ptr, sizeof(indirect_ptr)))
 913		return false;
 914
 915	indirect_ptr = le64_to_cpu(indirect_ptr);
 916
 917	/* check the valid bit of the first level entry */
 918	if (!(indirect_ptr & BIT_ULL(63)))
 919		return false;
 920
 921	/* Mask the guest physical address and calculate the frame number. */
 922	indirect_ptr &= GENMASK_ULL(51, 16);
 923
 924	/* Find the address of the actual entry */
 925	index = id % (SZ_64K / esz);
 926	indirect_ptr += index * esz;
 927
 928	if (eaddr)
 929		*eaddr = indirect_ptr;
 930
 931	return __is_visible_gfn_locked(its, indirect_ptr);
 932}
 933
 934/*
 935 * Check whether an event ID can be stored in the corresponding Interrupt
 936 * Translation Table, which starts at device->itt_addr.
 937 */
 938static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,
 939		u32 event_id)
 940{
 941	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 942	int ite_esz = abi->ite_esz;
 943	gpa_t gpa;
 944
 945	/* max table size is: BIT_ULL(device->num_eventid_bits) * ite_esz */
 946	if (event_id >= BIT_ULL(device->num_eventid_bits))
 947		return false;
 948
 949	gpa = device->itt_addr + event_id * ite_esz;
 950	return __is_visible_gfn_locked(its, gpa);
 951}
 952
 953/*
 954 * Add a new collection into the ITS collection table.
 955 * Returns 0 on success, and a negative error value for generic errors.
 956 */
 957static int vgic_its_alloc_collection(struct vgic_its *its,
 958				     struct its_collection **colp,
 959				     u32 coll_id)
 960{
 961	struct its_collection *collection;
 962
 963	collection = kzalloc(sizeof(*collection), GFP_KERNEL_ACCOUNT);
 964	if (!collection)
 965		return -ENOMEM;
 966
 967	collection->collection_id = coll_id;
 968	collection->target_addr = COLLECTION_NOT_MAPPED;
 969
 970	list_add_tail(&collection->coll_list, &its->collection_list);
 971	*colp = collection;
 972
 973	return 0;
 974}
 975
 976static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
 977{
 978	struct its_collection *collection;
 979	struct its_device *device;
 980	struct its_ite *ite;
 981
 982	/*
 983	 * Clearing the mapping for that collection ID removes the
 984	 * entry from the list. If there wasn't any before, we can
 985	 * go home early.
 986	 */
 987	collection = find_collection(its, coll_id);
 988	if (!collection)
 989		return;
 990
 991	for_each_lpi_its(device, ite, its)
 992		if (ite->collection &&
 993		    ite->collection->collection_id == coll_id)
 994			ite->collection = NULL;
 995
 996	list_del(&collection->coll_list);
 997	kfree(collection);
 998}
 999
1000/* Must be called with its_lock mutex held */
1001static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
1002					  struct its_collection *collection,
1003					  u32 event_id)
1004{
1005	struct its_ite *ite;
1006
1007	ite = kzalloc(sizeof(*ite), GFP_KERNEL_ACCOUNT);
1008	if (!ite)
1009		return ERR_PTR(-ENOMEM);
1010
1011	ite->event_id	= event_id;
1012	ite->collection = collection;
1013
1014	list_add_tail(&ite->ite_list, &device->itt_head);
1015	return ite;
1016}
1017
1018/*
1019 * The MAPTI and MAPI commands map LPIs to ITTEs.
1020 * Must be called with its_lock mutex held.
1021 */
1022static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
1023				    u64 *its_cmd)
1024{
1025	u32 device_id = its_cmd_get_deviceid(its_cmd);
1026	u32 event_id = its_cmd_get_id(its_cmd);
1027	u32 coll_id = its_cmd_get_collection(its_cmd);
1028	struct its_ite *ite;
1029	struct kvm_vcpu *vcpu = NULL;
1030	struct its_device *device;
1031	struct its_collection *collection, *new_coll = NULL;
1032	struct vgic_irq *irq;
1033	int lpi_nr;
1034
1035	device = find_its_device(its, device_id);
1036	if (!device)
1037		return E_ITS_MAPTI_UNMAPPED_DEVICE;
1038
1039	if (!vgic_its_check_event_id(its, device, event_id))
1040		return E_ITS_MAPTI_ID_OOR;
1041
1042	if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
1043		lpi_nr = its_cmd_get_physical_id(its_cmd);
1044	else
1045		lpi_nr = event_id;
1046	if (lpi_nr < GIC_LPI_OFFSET ||
1047	    lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
1048		return E_ITS_MAPTI_PHYSICALID_OOR;
1049
1050	/* If there is an existing mapping, behavior is UNPREDICTABLE. */
1051	if (find_ite(its, device_id, event_id))
1052		return 0;
1053
1054	collection = find_collection(its, coll_id);
1055	if (!collection) {
1056		int ret;
1057
1058		if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
1059			return E_ITS_MAPC_COLLECTION_OOR;
1060
1061		ret = vgic_its_alloc_collection(its, &collection, coll_id);
1062		if (ret)
1063			return ret;
1064		new_coll = collection;
1065	}
1066
1067	ite = vgic_its_alloc_ite(device, collection, event_id);
1068	if (IS_ERR(ite)) {
1069		if (new_coll)
1070			vgic_its_free_collection(its, coll_id);
1071		return PTR_ERR(ite);
1072	}
1073
1074	if (its_is_collection_mapped(collection))
1075		vcpu = collection_to_vcpu(kvm, collection);
1076
1077	irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
1078	if (IS_ERR(irq)) {
1079		if (new_coll)
1080			vgic_its_free_collection(its, coll_id);
1081		its_free_ite(kvm, ite);
1082		return PTR_ERR(irq);
1083	}
1084	ite->irq = irq;
1085
1086	return 0;
1087}
1088
1089/* Requires the its_lock to be held. */
1090static void vgic_its_free_device(struct kvm *kvm, struct vgic_its *its,
1091				 struct its_device *device)
1092{
1093	struct its_ite *ite, *temp;
1094
1095	/*
1096	 * The spec says that unmapping a device with still valid
1097	 * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
1098	 * since we cannot leave the memory unreferenced.
1099	 */
1100	list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
1101		its_free_ite(kvm, ite);
1102
1103	vgic_its_invalidate_cache(its);
1104
1105	list_del(&device->dev_list);
1106	kfree(device);
1107}
1108
1109/* its lock must be held */
1110static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
1111{
1112	struct its_device *cur, *temp;
1113
1114	list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
1115		vgic_its_free_device(kvm, its, cur);
1116}
1117
1118/* its lock must be held */
1119static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
1120{
1121	struct its_collection *cur, *temp;
1122
1123	list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
1124		vgic_its_free_collection(its, cur->collection_id);
1125}
1126
1127/* Must be called with its_lock mutex held */
1128static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
1129						u32 device_id, gpa_t itt_addr,
1130						u8 num_eventid_bits)
1131{
1132	struct its_device *device;
1133
1134	device = kzalloc(sizeof(*device), GFP_KERNEL_ACCOUNT);
1135	if (!device)
1136		return ERR_PTR(-ENOMEM);
1137
1138	device->device_id = device_id;
1139	device->itt_addr = itt_addr;
1140	device->num_eventid_bits = num_eventid_bits;
1141	INIT_LIST_HEAD(&device->itt_head);
1142
1143	list_add_tail(&device->dev_list, &its->device_list);
1144	return device;
1145}
1146
1147/*
1148 * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
1149 * Must be called with the its_lock mutex held.
1150 */
1151static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1152				    u64 *its_cmd)
1153{
1154	u32 device_id = its_cmd_get_deviceid(its_cmd);
1155	bool valid = its_cmd_get_validbit(its_cmd);
1156	u8 num_eventid_bits = its_cmd_get_size(its_cmd);
1157	gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
1158	struct its_device *device;
1159	gpa_t gpa;
1160
1161	if (!vgic_its_check_id(its, its->baser_device_table, device_id, &gpa))
1162		return E_ITS_MAPD_DEVICE_OOR;
1163
1164	if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
1165		return E_ITS_MAPD_ITTSIZE_OOR;
1166
1167	device = find_its_device(its, device_id);
1168
1169	/*
1170	 * The spec says that calling MAPD on an already mapped device
1171	 * invalidates all cached data for this device. We implement this
1172	 * by removing the mapping and re-establishing it.
1173	 */
1174	if (device)
1175		vgic_its_free_device(kvm, its, device);
1176
1177	/*
1178	 * The spec does not say whether unmapping a not-mapped device
1179	 * is an error, so we are done in any case.
1180	 */
1181	if (!valid)
1182		return vgic_its_write_entry_lock(its, gpa, 0ULL, dte);
1183
1184	device = vgic_its_alloc_device(its, device_id, itt_addr,
1185				       num_eventid_bits);
1186
1187	return PTR_ERR_OR_ZERO(device);
1188}
1189
1190/*
1191 * The MAPC command maps collection IDs to redistributors.
1192 * Must be called with the its_lock mutex held.
1193 */
1194static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1195				    u64 *its_cmd)
1196{
1197	u16 coll_id;
1198	struct its_collection *collection;
1199	bool valid;
1200
1201	valid = its_cmd_get_validbit(its_cmd);
1202	coll_id = its_cmd_get_collection(its_cmd);
1203
1204	if (!valid) {
1205		vgic_its_free_collection(its, coll_id);
1206		vgic_its_invalidate_cache(its);
1207	} else {
1208		struct kvm_vcpu *vcpu;
1209
1210		vcpu = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
1211		if (!vcpu)
1212			return E_ITS_MAPC_PROCNUM_OOR;
1213
1214		collection = find_collection(its, coll_id);
1215
1216		if (!collection) {
1217			int ret;
1218
1219			if (!vgic_its_check_id(its, its->baser_coll_table,
1220						coll_id, NULL))
1221				return E_ITS_MAPC_COLLECTION_OOR;
1222
1223			ret = vgic_its_alloc_collection(its, &collection,
1224							coll_id);
1225			if (ret)
1226				return ret;
1227			collection->target_addr = vcpu->vcpu_id;
1228		} else {
1229			collection->target_addr = vcpu->vcpu_id;
1230			update_affinity_collection(kvm, its, collection);
1231		}
1232	}
1233
1234	return 0;
1235}
1236
1237/*
1238 * The CLEAR command removes the pending state for a particular LPI.
1239 * Must be called with the its_lock mutex held.
1240 */
1241static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1242				     u64 *its_cmd)
1243{
1244	u32 device_id = its_cmd_get_deviceid(its_cmd);
1245	u32 event_id = its_cmd_get_id(its_cmd);
1246	struct its_ite *ite;
1247
1248
1249	ite = find_ite(its, device_id, event_id);
1250	if (!ite)
1251		return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1252
1253	ite->irq->pending_latch = false;
1254
1255	if (ite->irq->hw)
1256		return irq_set_irqchip_state(ite->irq->host_irq,
1257					     IRQCHIP_STATE_PENDING, false);
1258
1259	return 0;
1260}
1261
1262int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq)
1263{
1264	return update_lpi_config(kvm, irq, NULL, true);
1265}
1266
1267/*
1268 * The INV command syncs the configuration bits from the memory table.
1269 * Must be called with the its_lock mutex held.
1270 */
1271static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1272				   u64 *its_cmd)
1273{
1274	u32 device_id = its_cmd_get_deviceid(its_cmd);
1275	u32 event_id = its_cmd_get_id(its_cmd);
1276	struct its_ite *ite;
1277
1278
1279	ite = find_ite(its, device_id, event_id);
1280	if (!ite)
1281		return E_ITS_INV_UNMAPPED_INTERRUPT;
1282
1283	return vgic_its_inv_lpi(kvm, ite->irq);
1284}
1285
1286/**
1287 * vgic_its_invall - invalidate all LPIs targeting a given vcpu
1288 * @vcpu: the vcpu for which the RD is targeted by an invalidation
1289 *
1290 * Contrary to the INVALL command, this targets a RD instead of a
1291 * collection, and we don't need to hold the its_lock, since no ITS is
1292 * involved here.
1293 */
1294int vgic_its_invall(struct kvm_vcpu *vcpu)
1295{
1296	struct kvm *kvm = vcpu->kvm;
1297	struct vgic_dist *dist = &kvm->arch.vgic;
1298	struct vgic_irq *irq;
1299	unsigned long intid;
1300
1301	xa_for_each(&dist->lpi_xa, intid, irq) {
1302		irq = vgic_get_irq(kvm, intid);
1303		if (!irq)
1304			continue;
1305
1306		update_lpi_config(kvm, irq, vcpu, false);
1307		vgic_put_irq(kvm, irq);
1308	}
1309
1310	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1311		its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
1312
1313	return 0;
1314}
1315
1316/*
1317 * The INVALL command requests flushing of all IRQ data in this collection.
1318 * Find the VCPU mapped to that collection, then iterate over the VM's list
1319 * of mapped LPIs and update the configuration for each IRQ which targets
1320 * the specified vcpu. The configuration will be read from the in-memory
1321 * configuration table.
1322 * Must be called with the its_lock mutex held.
1323 */
1324static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1325				      u64 *its_cmd)
1326{
1327	u32 coll_id = its_cmd_get_collection(its_cmd);
1328	struct its_collection *collection;
1329	struct kvm_vcpu *vcpu;
1330
1331	collection = find_collection(its, coll_id);
1332	if (!its_is_collection_mapped(collection))
1333		return E_ITS_INVALL_UNMAPPED_COLLECTION;
1334
1335	vcpu = collection_to_vcpu(kvm, collection);
1336	vgic_its_invall(vcpu);
1337
1338	return 0;
1339}
1340
1341/*
1342 * The MOVALL command moves the pending state of all IRQs targeting one
1343 * redistributor to another. We don't hold the pending state in the VCPUs,
1344 * but in the IRQs instead, so there is really not much to do for us here.
1345 * However the spec says that no IRQ must target the old redistributor
1346 * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1347 * This command affects all LPIs in the system that target that redistributor.
1348 */
1349static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1350				      u64 *its_cmd)
1351{
1352	struct vgic_dist *dist = &kvm->arch.vgic;
1353	struct kvm_vcpu *vcpu1, *vcpu2;
1354	struct vgic_irq *irq;
1355	unsigned long intid;
1356
1357	/* We advertise GITS_TYPER.PTA==0, making the address the vcpu ID */
1358	vcpu1 = kvm_get_vcpu_by_id(kvm, its_cmd_get_target_addr(its_cmd));
1359	vcpu2 = kvm_get_vcpu_by_id(kvm, its_cmd_mask_field(its_cmd, 3, 16, 32));
1360
1361	if (!vcpu1 || !vcpu2)
1362		return E_ITS_MOVALL_PROCNUM_OOR;
1363
1364	if (vcpu1 == vcpu2)
1365		return 0;
1366
1367	xa_for_each(&dist->lpi_xa, intid, irq) {
1368		irq = vgic_get_irq(kvm, intid);
1369		if (!irq)
1370			continue;
1371
1372		update_affinity(irq, vcpu2);
1373
1374		vgic_put_irq(kvm, irq);
1375	}
1376
1377	vgic_its_invalidate_cache(its);
1378
1379	return 0;
1380}
1381
1382/*
1383 * The INT command injects the LPI associated with that DevID/EvID pair.
1384 * Must be called with the its_lock mutex held.
1385 */
1386static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1387				   u64 *its_cmd)
1388{
1389	u32 msi_data = its_cmd_get_id(its_cmd);
1390	u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1391
1392	return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1393}
1394
1395/*
1396 * This function is called with the its_cmd lock held, but the ITS data
1397 * structure lock dropped.
1398 */
1399static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1400				   u64 *its_cmd)
1401{
1402	int ret = -ENODEV;
1403
1404	mutex_lock(&its->its_lock);
1405	switch (its_cmd_get_command(its_cmd)) {
1406	case GITS_CMD_MAPD:
1407		ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1408		break;
1409	case GITS_CMD_MAPC:
1410		ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1411		break;
1412	case GITS_CMD_MAPI:
1413		ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1414		break;
1415	case GITS_CMD_MAPTI:
1416		ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1417		break;
1418	case GITS_CMD_MOVI:
1419		ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1420		break;
1421	case GITS_CMD_DISCARD:
1422		ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1423		break;
1424	case GITS_CMD_CLEAR:
1425		ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1426		break;
1427	case GITS_CMD_MOVALL:
1428		ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1429		break;
1430	case GITS_CMD_INT:
1431		ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1432		break;
1433	case GITS_CMD_INV:
1434		ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1435		break;
1436	case GITS_CMD_INVALL:
1437		ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1438		break;
1439	case GITS_CMD_SYNC:
1440		/* we ignore this command: we are in sync all of the time */
1441		ret = 0;
1442		break;
1443	}
1444	mutex_unlock(&its->its_lock);
1445
1446	return ret;
1447}
1448
1449static u64 vgic_sanitise_its_baser(u64 reg)
1450{
1451	reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1452				  GITS_BASER_SHAREABILITY_SHIFT,
1453				  vgic_sanitise_shareability);
1454	reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1455				  GITS_BASER_INNER_CACHEABILITY_SHIFT,
1456				  vgic_sanitise_inner_cacheability);
1457	reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1458				  GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1459				  vgic_sanitise_outer_cacheability);
1460
1461	/* We support only one (ITS) page size: 64K */
1462	reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1463
1464	return reg;
1465}
1466
1467static u64 vgic_sanitise_its_cbaser(u64 reg)
1468{
1469	reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1470				  GITS_CBASER_SHAREABILITY_SHIFT,
1471				  vgic_sanitise_shareability);
1472	reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1473				  GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1474				  vgic_sanitise_inner_cacheability);
1475	reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1476				  GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1477				  vgic_sanitise_outer_cacheability);
1478
1479	/* Sanitise the physical address to be 64k aligned. */
1480	reg &= ~GENMASK_ULL(15, 12);
1481
1482	return reg;
1483}
1484
1485static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1486					       struct vgic_its *its,
1487					       gpa_t addr, unsigned int len)
1488{
1489	return extract_bytes(its->cbaser, addr & 7, len);
1490}
1491
1492static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1493				       gpa_t addr, unsigned int len,
1494				       unsigned long val)
1495{
1496	/* When GITS_CTLR.Enable is 1, this register is RO. */
1497	if (its->enabled)
1498		return;
1499
1500	mutex_lock(&its->cmd_lock);
1501	its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1502	its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1503	its->creadr = 0;
1504	/*
1505	 * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1506	 * it to CREADR to make sure we start with an empty command buffer.
1507	 */
1508	its->cwriter = its->creadr;
1509	mutex_unlock(&its->cmd_lock);
1510}
1511
1512#define ITS_CMD_BUFFER_SIZE(baser)	((((baser) & 0xff) + 1) << 12)
1513#define ITS_CMD_SIZE			32
1514#define ITS_CMD_OFFSET(reg)		((reg) & GENMASK(19, 5))
1515
1516/* Must be called with the cmd_lock held. */
1517static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1518{
1519	gpa_t cbaser;
1520	u64 cmd_buf[4];
1521
1522	/* Commands are only processed when the ITS is enabled. */
1523	if (!its->enabled)
1524		return;
1525
1526	cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1527
1528	while (its->cwriter != its->creadr) {
1529		int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1530					      cmd_buf, ITS_CMD_SIZE);
1531		/*
1532		 * If kvm_read_guest() fails, this could be due to the guest
1533		 * programming a bogus value in CBASER or something else going
1534		 * wrong from which we cannot easily recover.
1535		 * According to section 6.3.2 in the GICv3 spec we can just
1536		 * ignore that command then.
1537		 */
1538		if (!ret)
1539			vgic_its_handle_command(kvm, its, cmd_buf);
1540
1541		its->creadr += ITS_CMD_SIZE;
1542		if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1543			its->creadr = 0;
1544	}
1545}
1546
1547/*
1548 * By writing to CWRITER the guest announces new commands to be processed.
1549 * To avoid any races in the first place, we take the its_cmd lock, which
1550 * protects our ring buffer variables, so that there is only one user
1551 * per ITS handling commands at a given time.
1552 */
1553static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1554					gpa_t addr, unsigned int len,
1555					unsigned long val)
1556{
1557	u64 reg;
1558
1559	if (!its)
1560		return;
1561
1562	mutex_lock(&its->cmd_lock);
1563
1564	reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1565	reg = ITS_CMD_OFFSET(reg);
1566	if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1567		mutex_unlock(&its->cmd_lock);
1568		return;
1569	}
1570	its->cwriter = reg;
1571
1572	vgic_its_process_commands(kvm, its);
1573
1574	mutex_unlock(&its->cmd_lock);
1575}
1576
1577static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1578						struct vgic_its *its,
1579						gpa_t addr, unsigned int len)
1580{
1581	return extract_bytes(its->cwriter, addr & 0x7, len);
1582}
1583
1584static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1585					       struct vgic_its *its,
1586					       gpa_t addr, unsigned int len)
1587{
1588	return extract_bytes(its->creadr, addr & 0x7, len);
1589}
1590
1591static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1592					      struct vgic_its *its,
1593					      gpa_t addr, unsigned int len,
1594					      unsigned long val)
1595{
1596	u32 cmd_offset;
1597	int ret = 0;
1598
1599	mutex_lock(&its->cmd_lock);
1600
1601	if (its->enabled) {
1602		ret = -EBUSY;
1603		goto out;
1604	}
1605
1606	cmd_offset = ITS_CMD_OFFSET(val);
1607	if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1608		ret = -EINVAL;
1609		goto out;
1610	}
1611
1612	its->creadr = cmd_offset;
1613out:
1614	mutex_unlock(&its->cmd_lock);
1615	return ret;
1616}
1617
1618#define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1619static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1620					      struct vgic_its *its,
1621					      gpa_t addr, unsigned int len)
1622{
1623	u64 reg;
1624
1625	switch (BASER_INDEX(addr)) {
1626	case 0:
1627		reg = its->baser_device_table;
1628		break;
1629	case 1:
1630		reg = its->baser_coll_table;
1631		break;
1632	default:
1633		reg = 0;
1634		break;
1635	}
1636
1637	return extract_bytes(reg, addr & 7, len);
1638}
1639
1640#define GITS_BASER_RO_MASK	(GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1641static void vgic_mmio_write_its_baser(struct kvm *kvm,
1642				      struct vgic_its *its,
1643				      gpa_t addr, unsigned int len,
1644				      unsigned long val)
1645{
1646	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1647	u64 entry_size, table_type;
1648	u64 reg, *regptr, clearbits = 0;
1649
1650	/* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1651	if (its->enabled)
1652		return;
1653
1654	switch (BASER_INDEX(addr)) {
1655	case 0:
1656		regptr = &its->baser_device_table;
1657		entry_size = abi->dte_esz;
1658		table_type = GITS_BASER_TYPE_DEVICE;
1659		break;
1660	case 1:
1661		regptr = &its->baser_coll_table;
1662		entry_size = abi->cte_esz;
1663		table_type = GITS_BASER_TYPE_COLLECTION;
1664		clearbits = GITS_BASER_INDIRECT;
1665		break;
1666	default:
1667		return;
1668	}
1669
1670	reg = update_64bit_reg(*regptr, addr & 7, len, val);
1671	reg &= ~GITS_BASER_RO_MASK;
1672	reg &= ~clearbits;
1673
1674	reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1675	reg |= table_type << GITS_BASER_TYPE_SHIFT;
1676	reg = vgic_sanitise_its_baser(reg);
1677
1678	*regptr = reg;
1679
1680	if (!(reg & GITS_BASER_VALID)) {
1681		/* Take the its_lock to prevent a race with a save/restore */
1682		mutex_lock(&its->its_lock);
1683		switch (table_type) {
1684		case GITS_BASER_TYPE_DEVICE:
1685			vgic_its_free_device_list(kvm, its);
1686			break;
1687		case GITS_BASER_TYPE_COLLECTION:
1688			vgic_its_free_collection_list(kvm, its);
1689			break;
1690		}
1691		mutex_unlock(&its->its_lock);
1692	}
1693}
1694
1695static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1696					     struct vgic_its *its,
1697					     gpa_t addr, unsigned int len)
1698{
1699	u32 reg = 0;
1700
1701	mutex_lock(&its->cmd_lock);
1702	if (its->creadr == its->cwriter)
1703		reg |= GITS_CTLR_QUIESCENT;
1704	if (its->enabled)
1705		reg |= GITS_CTLR_ENABLE;
1706	mutex_unlock(&its->cmd_lock);
1707
1708	return reg;
1709}
1710
1711static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1712				     gpa_t addr, unsigned int len,
1713				     unsigned long val)
1714{
1715	mutex_lock(&its->cmd_lock);
1716
1717	/*
1718	 * It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1719	 * device/collection BASER are invalid
1720	 */
1721	if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1722		(!(its->baser_device_table & GITS_BASER_VALID) ||
1723		 !(its->baser_coll_table & GITS_BASER_VALID) ||
1724		 !(its->cbaser & GITS_CBASER_VALID)))
1725		goto out;
1726
1727	its->enabled = !!(val & GITS_CTLR_ENABLE);
1728	if (!its->enabled)
1729		vgic_its_invalidate_cache(its);
1730
1731	/*
1732	 * Try to process any pending commands. This function bails out early
1733	 * if the ITS is disabled or no commands have been queued.
1734	 */
1735	vgic_its_process_commands(kvm, its);
1736
1737out:
1738	mutex_unlock(&its->cmd_lock);
1739}
1740
1741#define REGISTER_ITS_DESC(off, rd, wr, length, acc)		\
1742{								\
1743	.reg_offset = off,					\
1744	.len = length,						\
1745	.access_flags = acc,					\
1746	.its_read = rd,						\
1747	.its_write = wr,					\
1748}
1749
1750#define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1751{								\
1752	.reg_offset = off,					\
1753	.len = length,						\
1754	.access_flags = acc,					\
1755	.its_read = rd,						\
1756	.its_write = wr,					\
1757	.uaccess_its_write = uwr,				\
1758}
1759
1760static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1761			      gpa_t addr, unsigned int len, unsigned long val)
1762{
1763	/* Ignore */
1764}
1765
1766static struct vgic_register_region its_registers[] = {
1767	REGISTER_ITS_DESC(GITS_CTLR,
1768		vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1769		VGIC_ACCESS_32bit),
1770	REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1771		vgic_mmio_read_its_iidr, its_mmio_write_wi,
1772		vgic_mmio_uaccess_write_its_iidr, 4,
1773		VGIC_ACCESS_32bit),
1774	REGISTER_ITS_DESC(GITS_TYPER,
1775		vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1776		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1777	REGISTER_ITS_DESC(GITS_CBASER,
1778		vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1779		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1780	REGISTER_ITS_DESC(GITS_CWRITER,
1781		vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1782		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1783	REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1784		vgic_mmio_read_its_creadr, its_mmio_write_wi,
1785		vgic_mmio_uaccess_write_its_creadr, 8,
1786		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1787	REGISTER_ITS_DESC(GITS_BASER,
1788		vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1789		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1790	REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1791		vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1792		VGIC_ACCESS_32bit),
1793};
1794
1795/* This is called on setting the LPI enable bit in the redistributor. */
1796void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1797{
1798	if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1799		its_sync_lpi_pending_table(vcpu);
1800}
1801
1802static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1803				   u64 addr)
1804{
1805	struct vgic_io_device *iodev = &its->iodev;
1806	int ret;
1807
1808	mutex_lock(&kvm->slots_lock);
1809	if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1810		ret = -EBUSY;
1811		goto out;
1812	}
1813
1814	its->vgic_its_base = addr;
1815	iodev->regions = its_registers;
1816	iodev->nr_regions = ARRAY_SIZE(its_registers);
1817	kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1818
1819	iodev->base_addr = its->vgic_its_base;
1820	iodev->iodev_type = IODEV_ITS;
1821	iodev->its = its;
1822	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1823				      KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1824out:
1825	mutex_unlock(&kvm->slots_lock);
1826
1827	return ret;
1828}
1829
1830#define INITIAL_BASER_VALUE						  \
1831	(GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)		| \
1832	 GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)		| \
1833	 GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)		| \
1834	 GITS_BASER_PAGE_SIZE_64K)
1835
1836#define INITIAL_PROPBASER_VALUE						  \
1837	(GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)		| \
1838	 GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)	| \
1839	 GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1840
1841static int vgic_its_create(struct kvm_device *dev, u32 type)
1842{
1843	int ret;
1844	struct vgic_its *its;
1845
1846	if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1847		return -ENODEV;
1848
1849	its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);
1850	if (!its)
1851		return -ENOMEM;
1852
1853	mutex_lock(&dev->kvm->arch.config_lock);
1854
1855	if (vgic_initialized(dev->kvm)) {
1856		ret = vgic_v4_init(dev->kvm);
1857		if (ret < 0) {
1858			mutex_unlock(&dev->kvm->arch.config_lock);
1859			kfree(its);
1860			return ret;
1861		}
1862	}
1863
1864	mutex_init(&its->its_lock);
1865	mutex_init(&its->cmd_lock);
1866
1867	/* Yep, even more trickery for lock ordering... */
1868#ifdef CONFIG_LOCKDEP
1869	mutex_lock(&its->cmd_lock);
1870	mutex_lock(&its->its_lock);
1871	mutex_unlock(&its->its_lock);
1872	mutex_unlock(&its->cmd_lock);
1873#endif
1874
1875	its->vgic_its_base = VGIC_ADDR_UNDEF;
1876
1877	INIT_LIST_HEAD(&its->device_list);
1878	INIT_LIST_HEAD(&its->collection_list);
1879	xa_init(&its->translation_cache);
1880
1881	dev->kvm->arch.vgic.msis_require_devid = true;
1882	dev->kvm->arch.vgic.has_its = true;
1883	its->enabled = false;
1884	its->dev = dev;
1885
1886	its->baser_device_table = INITIAL_BASER_VALUE			|
1887		((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1888	its->baser_coll_table = INITIAL_BASER_VALUE |
1889		((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1890	dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1891
1892	dev->private = its;
1893
1894	ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1895
1896	mutex_unlock(&dev->kvm->arch.config_lock);
1897
1898	return ret;
1899}
1900
1901static void vgic_its_destroy(struct kvm_device *kvm_dev)
1902{
1903	struct kvm *kvm = kvm_dev->kvm;
1904	struct vgic_its *its = kvm_dev->private;
1905
1906	mutex_lock(&its->its_lock);
1907
1908	vgic_its_debug_destroy(kvm_dev);
1909
1910	vgic_its_free_device_list(kvm, its);
1911	vgic_its_free_collection_list(kvm, its);
1912	vgic_its_invalidate_cache(its);
1913	xa_destroy(&its->translation_cache);
1914
1915	mutex_unlock(&its->its_lock);
1916	kfree(its);
1917	kfree(kvm_dev);/* alloc by kvm_ioctl_create_device, free by .destroy */
1918}
1919
1920static int vgic_its_has_attr_regs(struct kvm_device *dev,
1921				  struct kvm_device_attr *attr)
1922{
1923	const struct vgic_register_region *region;
1924	gpa_t offset = attr->attr;
1925	int align;
1926
1927	align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1928
1929	if (offset & align)
1930		return -EINVAL;
1931
1932	region = vgic_find_mmio_region(its_registers,
1933				       ARRAY_SIZE(its_registers),
1934				       offset);
1935	if (!region)
1936		return -ENXIO;
1937
1938	return 0;
1939}
1940
1941static int vgic_its_attr_regs_access(struct kvm_device *dev,
1942				     struct kvm_device_attr *attr,
1943				     u64 *reg, bool is_write)
1944{
1945	const struct vgic_register_region *region;
1946	struct vgic_its *its;
1947	gpa_t addr, offset;
1948	unsigned int len;
1949	int align, ret = 0;
1950
1951	its = dev->private;
1952	offset = attr->attr;
1953
1954	/*
1955	 * Although the spec supports upper/lower 32-bit accesses to
1956	 * 64-bit ITS registers, the userspace ABI requires 64-bit
1957	 * accesses to all 64-bit wide registers. We therefore only
1958	 * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1959	 * registers
1960	 */
1961	if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1962		align = 0x3;
1963	else
1964		align = 0x7;
1965
1966	if (offset & align)
1967		return -EINVAL;
1968
1969	mutex_lock(&dev->kvm->lock);
1970
1971	if (kvm_trylock_all_vcpus(dev->kvm)) {
1972		mutex_unlock(&dev->kvm->lock);
1973		return -EBUSY;
1974	}
1975
1976	mutex_lock(&dev->kvm->arch.config_lock);
1977
1978	if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1979		ret = -ENXIO;
1980		goto out;
1981	}
1982
1983	region = vgic_find_mmio_region(its_registers,
1984				       ARRAY_SIZE(its_registers),
1985				       offset);
1986	if (!region) {
1987		ret = -ENXIO;
1988		goto out;
1989	}
1990
1991	addr = its->vgic_its_base + offset;
1992
1993	len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1994
1995	if (is_write) {
1996		if (region->uaccess_its_write)
1997			ret = region->uaccess_its_write(dev->kvm, its, addr,
1998							len, *reg);
1999		else
2000			region->its_write(dev->kvm, its, addr, len, *reg);
2001	} else {
2002		*reg = region->its_read(dev->kvm, its, addr, len);
2003	}
2004out:
2005	mutex_unlock(&dev->kvm->arch.config_lock);
2006	kvm_unlock_all_vcpus(dev->kvm);
2007	mutex_unlock(&dev->kvm->lock);
2008	return ret;
2009}
2010
2011static u32 compute_next_devid_offset(struct list_head *h,
2012				     struct its_device *dev)
2013{
2014	struct its_device *next;
2015	u32 next_offset;
2016
2017	if (list_is_last(&dev->dev_list, h))
2018		return 0;
2019	next = list_next_entry(dev, dev_list);
2020	next_offset = next->device_id - dev->device_id;
2021
2022	return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
2023}
2024
2025static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
2026{
2027	struct its_ite *next;
2028	u32 next_offset;
2029
2030	if (list_is_last(&ite->ite_list, h))
2031		return 0;
2032	next = list_next_entry(ite, ite_list);
2033	next_offset = next->event_id - ite->event_id;
2034
2035	return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
2036}
2037
2038/**
2039 * typedef entry_fn_t - Callback called on a table entry restore path
2040 * @its: its handle
2041 * @id: id of the entry
2042 * @entry: pointer to the entry
2043 * @opaque: pointer to an opaque data
2044 *
2045 * Return: < 0 on error, 0 if last element was identified, id offset to next
2046 * element otherwise
2047 */
2048typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2049			  void *opaque);
2050
2051/**
2052 * scan_its_table - Scan a contiguous table in guest RAM and applies a function
2053 * to each entry
2054 *
2055 * @its: its handle
2056 * @base: base gpa of the table
2057 * @size: size of the table in bytes
2058 * @esz: entry size in bytes
2059 * @start_id: the ID of the first entry in the table
2060 * (non zero for 2d level tables)
2061 * @fn: function to apply on each entry
2062 * @opaque: pointer to opaque data
2063 *
2064 * Return: < 0 on error, 0 if last element was identified, 1 otherwise
2065 * (the last element may not be found on second level tables)
2066 */
2067static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
2068			  int start_id, entry_fn_t fn, void *opaque)
2069{
2070	struct kvm *kvm = its->dev->kvm;
2071	unsigned long len = size;
2072	int id = start_id;
2073	gpa_t gpa = base;
2074	char entry[ESZ_MAX];
2075	int ret;
2076
2077	memset(entry, 0, esz);
2078
2079	while (true) {
2080		int next_offset;
2081		size_t byte_offset;
2082
2083		ret = kvm_read_guest_lock(kvm, gpa, entry, esz);
2084		if (ret)
2085			return ret;
2086
2087		next_offset = fn(its, id, entry, opaque);
2088		if (next_offset <= 0)
2089			return next_offset;
2090
2091		byte_offset = next_offset * esz;
2092		if (byte_offset >= len)
2093			break;
2094
2095		id += next_offset;
2096		gpa += byte_offset;
2097		len -= byte_offset;
2098	}
2099	return 1;
2100}
2101
2102/*
2103 * vgic_its_save_ite - Save an interrupt translation entry at @gpa
2104 */
2105static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
2106			      struct its_ite *ite, gpa_t gpa)
2107{
2108	u32 next_offset;
2109	u64 val;
2110
2111	next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
2112	val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
2113	       ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
2114		ite->collection->collection_id;
2115	val = cpu_to_le64(val);
2116
2117	return vgic_its_write_entry_lock(its, gpa, val, ite);
2118}
2119
2120/**
2121 * vgic_its_restore_ite - restore an interrupt translation entry
2122 *
2123 * @its: its handle
2124 * @event_id: id used for indexing
2125 * @ptr: pointer to the ITE entry
2126 * @opaque: pointer to the its_device
2127 */
2128static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
2129				void *ptr, void *opaque)
2130{
2131	struct its_device *dev = opaque;
2132	struct its_collection *collection;
2133	struct kvm *kvm = its->dev->kvm;
2134	struct kvm_vcpu *vcpu = NULL;
2135	u64 val;
2136	u64 *p = (u64 *)ptr;
2137	struct vgic_irq *irq;
2138	u32 coll_id, lpi_id;
2139	struct its_ite *ite;
2140	u32 offset;
2141
2142	val = *p;
2143
2144	val = le64_to_cpu(val);
2145
2146	coll_id = val & KVM_ITS_ITE_ICID_MASK;
2147	lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
2148
2149	if (!lpi_id)
2150		return 1; /* invalid entry, no choice but to scan next entry */
2151
2152	if (lpi_id < VGIC_MIN_LPI)
2153		return -EINVAL;
2154
2155	offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
2156	if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
2157		return -EINVAL;
2158
2159	collection = find_collection(its, coll_id);
2160	if (!collection)
2161		return -EINVAL;
2162
2163	if (!vgic_its_check_event_id(its, dev, event_id))
2164		return -EINVAL;
2165
2166	ite = vgic_its_alloc_ite(dev, collection, event_id);
2167	if (IS_ERR(ite))
2168		return PTR_ERR(ite);
2169
2170	if (its_is_collection_mapped(collection))
2171		vcpu = kvm_get_vcpu_by_id(kvm, collection->target_addr);
2172
2173	irq = vgic_add_lpi(kvm, lpi_id, vcpu);
2174	if (IS_ERR(irq)) {
2175		its_free_ite(kvm, ite);
2176		return PTR_ERR(irq);
2177	}
2178	ite->irq = irq;
2179
2180	return offset;
2181}
2182
2183static int vgic_its_ite_cmp(void *priv, const struct list_head *a,
2184			    const struct list_head *b)
2185{
2186	struct its_ite *itea = container_of(a, struct its_ite, ite_list);
2187	struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
2188
2189	if (itea->event_id < iteb->event_id)
2190		return -1;
2191	else
2192		return 1;
2193}
2194
2195static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
2196{
2197	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2198	gpa_t base = device->itt_addr;
2199	struct its_ite *ite;
2200	int ret;
2201	int ite_esz = abi->ite_esz;
2202
2203	list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
2204
2205	list_for_each_entry(ite, &device->itt_head, ite_list) {
2206		gpa_t gpa = base + ite->event_id * ite_esz;
2207
2208		/*
2209		 * If an LPI carries the HW bit, this means that this
2210		 * interrupt is controlled by GICv4, and we do not
2211		 * have direct access to that state without GICv4.1.
2212		 * Let's simply fail the save operation...
2213		 */
2214		if (ite->irq->hw && !kvm_vgic_global_state.has_gicv4_1)
2215			return -EACCES;
2216
2217		ret = vgic_its_save_ite(its, device, ite, gpa);
2218		if (ret)
2219			return ret;
2220	}
2221	return 0;
2222}
2223
2224/**
2225 * vgic_its_restore_itt - restore the ITT of a device
2226 *
2227 * @its: its handle
2228 * @dev: device handle
2229 *
2230 * Return 0 on success, < 0 on error
2231 */
2232static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2233{
2234	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2235	gpa_t base = dev->itt_addr;
2236	int ret;
2237	int ite_esz = abi->ite_esz;
2238	size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
2239
2240	ret = scan_its_table(its, base, max_size, ite_esz, 0,
2241			     vgic_its_restore_ite, dev);
2242
2243	/* scan_its_table returns +1 if all ITEs are invalid */
2244	if (ret > 0)
2245		ret = 0;
2246
2247	return ret;
2248}
2249
2250/**
2251 * vgic_its_save_dte - Save a device table entry at a given GPA
2252 *
2253 * @its: ITS handle
2254 * @dev: ITS device
2255 * @ptr: GPA
2256 */
2257static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2258			     gpa_t ptr)
2259{
2260	u64 val, itt_addr_field;
2261	u32 next_offset;
2262
2263	itt_addr_field = dev->itt_addr >> 8;
2264	next_offset = compute_next_devid_offset(&its->device_list, dev);
2265	val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2266	       ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2267	       (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2268		(dev->num_eventid_bits - 1));
2269	val = cpu_to_le64(val);
2270
2271	return vgic_its_write_entry_lock(its, ptr, val, dte);
2272}
2273
2274/**
2275 * vgic_its_restore_dte - restore a device table entry
2276 *
2277 * @its: its handle
2278 * @id: device id the DTE corresponds to
2279 * @ptr: kernel VA where the 8 byte DTE is located
2280 * @opaque: unused
2281 *
2282 * Return: < 0 on error, 0 if the dte is the last one, id offset to the
2283 * next dte otherwise
2284 */
2285static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2286				void *ptr, void *opaque)
2287{
2288	struct its_device *dev;
2289	u64 baser = its->baser_device_table;
2290	gpa_t itt_addr;
2291	u8 num_eventid_bits;
2292	u64 entry = *(u64 *)ptr;
2293	bool valid;
2294	u32 offset;
2295	int ret;
2296
2297	entry = le64_to_cpu(entry);
2298
2299	valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2300	num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2301	itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2302			>> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2303
2304	if (!valid)
2305		return 1;
2306
2307	/* dte entry is valid */
2308	offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2309
2310	if (!vgic_its_check_id(its, baser, id, NULL))
2311		return -EINVAL;
2312
2313	dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2314	if (IS_ERR(dev))
2315		return PTR_ERR(dev);
2316
2317	ret = vgic_its_restore_itt(its, dev);
2318	if (ret) {
2319		vgic_its_free_device(its->dev->kvm, its, dev);
2320		return ret;
2321	}
2322
2323	return offset;
2324}
2325
2326static int vgic_its_device_cmp(void *priv, const struct list_head *a,
2327			       const struct list_head *b)
2328{
2329	struct its_device *deva = container_of(a, struct its_device, dev_list);
2330	struct its_device *devb = container_of(b, struct its_device, dev_list);
2331
2332	if (deva->device_id < devb->device_id)
2333		return -1;
2334	else
2335		return 1;
2336}
2337
2338/*
2339 * vgic_its_save_device_tables - Save the device table and all ITT
2340 * into guest RAM
2341 *
2342 * L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2343 * returns the GPA of the device entry
2344 */
2345static int vgic_its_save_device_tables(struct vgic_its *its)
2346{
2347	u64 baser = its->baser_device_table;
2348	struct its_device *dev;
2349
2350	if (!(baser & GITS_BASER_VALID))
2351		return 0;
2352
2353	list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2354
2355	list_for_each_entry(dev, &its->device_list, dev_list) {
2356		int ret;
2357		gpa_t eaddr;
2358
2359		if (!vgic_its_check_id(its, baser,
2360				       dev->device_id, &eaddr))
2361			return -EINVAL;
2362
2363		ret = vgic_its_save_itt(its, dev);
2364		if (ret)
2365			return ret;
2366
2367		ret = vgic_its_save_dte(its, dev, eaddr);
2368		if (ret)
2369			return ret;
2370	}
2371	return 0;
2372}
2373
2374/**
2375 * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2376 *
2377 * @its: its handle
2378 * @id: index of the entry in the L1 table
2379 * @addr: kernel VA
2380 * @opaque: unused
2381 *
2382 * L1 table entries are scanned by steps of 1 entry
2383 * Return < 0 if error, 0 if last dte was found when scanning the L2
2384 * table, +1 otherwise (meaning next L1 entry must be scanned)
2385 */
2386static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2387			 void *opaque)
2388{
2389	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2390	int l2_start_id = id * (SZ_64K / abi->dte_esz);
2391	u64 entry = *(u64 *)addr;
2392	int dte_esz = abi->dte_esz;
2393	gpa_t gpa;
2394	int ret;
2395
2396	entry = le64_to_cpu(entry);
2397
2398	if (!(entry & KVM_ITS_L1E_VALID_MASK))
2399		return 1;
2400
2401	gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2402
2403	ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2404			     l2_start_id, vgic_its_restore_dte, NULL);
2405
2406	return ret;
2407}
2408
2409/*
2410 * vgic_its_restore_device_tables - Restore the device table and all ITT
2411 * from guest RAM to internal data structs
2412 */
2413static int vgic_its_restore_device_tables(struct vgic_its *its)
2414{
2415	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2416	u64 baser = its->baser_device_table;
2417	int l1_esz, ret;
2418	int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2419	gpa_t l1_gpa;
2420
2421	if (!(baser & GITS_BASER_VALID))
2422		return 0;
2423
2424	l1_gpa = GITS_BASER_ADDR_48_to_52(baser);
2425
2426	if (baser & GITS_BASER_INDIRECT) {
2427		l1_esz = GITS_LVL1_ENTRY_SIZE;
2428		ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2429				     handle_l1_dte, NULL);
2430	} else {
2431		l1_esz = abi->dte_esz;
2432		ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2433				     vgic_its_restore_dte, NULL);
2434	}
2435
2436	/* scan_its_table returns +1 if all entries are invalid */
2437	if (ret > 0)
2438		ret = 0;
2439
2440	if (ret < 0)
2441		vgic_its_free_device_list(its->dev->kvm, its);
2442
2443	return ret;
2444}
2445
2446static int vgic_its_save_cte(struct vgic_its *its,
2447			     struct its_collection *collection,
2448			     gpa_t gpa)
2449{
2450	u64 val;
2451
2452	val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2453	       ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2454	       collection->collection_id);
2455	val = cpu_to_le64(val);
2456
2457	return vgic_its_write_entry_lock(its, gpa, val, cte);
2458}
2459
2460/*
2461 * Restore a collection entry into the ITS collection table.
2462 * Return +1 on success, 0 if the entry was invalid (which should be
2463 * interpreted as end-of-table), and a negative error value for generic errors.
2464 */
2465static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa)
2466{
2467	struct its_collection *collection;
2468	struct kvm *kvm = its->dev->kvm;
2469	u32 target_addr, coll_id;
2470	u64 val;
2471	int ret;
2472
2473	ret = vgic_its_read_entry_lock(its, gpa, &val, cte);
2474	if (ret)
2475		return ret;
2476	val = le64_to_cpu(val);
2477	if (!(val & KVM_ITS_CTE_VALID_MASK))
2478		return 0;
2479
2480	target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2481	coll_id = val & KVM_ITS_CTE_ICID_MASK;
2482
2483	if (target_addr != COLLECTION_NOT_MAPPED &&
2484	    !kvm_get_vcpu_by_id(kvm, target_addr))
2485		return -EINVAL;
2486
2487	collection = find_collection(its, coll_id);
2488	if (collection)
2489		return -EEXIST;
2490
2491	if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
2492		return -EINVAL;
2493
2494	ret = vgic_its_alloc_collection(its, &collection, coll_id);
2495	if (ret)
2496		return ret;
2497	collection->target_addr = target_addr;
2498	return 1;
2499}
2500
2501/*
2502 * vgic_its_save_collection_table - Save the collection table into
2503 * guest RAM
2504 */
2505static int vgic_its_save_collection_table(struct vgic_its *its)
2506{
2507	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2508	u64 baser = its->baser_coll_table;
2509	gpa_t gpa = GITS_BASER_ADDR_48_to_52(baser);
2510	struct its_collection *collection;
2511	size_t max_size, filled = 0;
2512	int ret, cte_esz = abi->cte_esz;
2513
2514	if (!(baser & GITS_BASER_VALID))
2515		return 0;
2516
2517	max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2518
2519	list_for_each_entry(collection, &its->collection_list, coll_list) {
2520		ret = vgic_its_save_cte(its, collection, gpa);
2521		if (ret)
2522			return ret;
2523		gpa += cte_esz;
2524		filled += cte_esz;
2525	}
2526
2527	if (filled == max_size)
2528		return 0;
2529
2530	/*
2531	 * table is not fully filled, add a last dummy element
2532	 * with valid bit unset
2533	 */
2534	return vgic_its_write_entry_lock(its, gpa, 0ULL, cte);
2535}
2536
2537/*
2538 * vgic_its_restore_collection_table - reads the collection table
2539 * in guest memory and restores the ITS internal state. Requires the
2540 * BASER registers to be restored before.
2541 */
2542static int vgic_its_restore_collection_table(struct vgic_its *its)
2543{
2544	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2545	u64 baser = its->baser_coll_table;
2546	int cte_esz = abi->cte_esz;
2547	size_t max_size, read = 0;
2548	gpa_t gpa;
2549	int ret;
2550
2551	if (!(baser & GITS_BASER_VALID))
2552		return 0;
2553
2554	gpa = GITS_BASER_ADDR_48_to_52(baser);
2555
2556	max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2557
2558	while (read < max_size) {
2559		ret = vgic_its_restore_cte(its, gpa);
2560		if (ret <= 0)
2561			break;
2562		gpa += cte_esz;
2563		read += cte_esz;
2564	}
2565
2566	if (ret > 0)
2567		return 0;
2568
2569	if (ret < 0)
2570		vgic_its_free_collection_list(its->dev->kvm, its);
2571
2572	return ret;
2573}
2574
2575/*
2576 * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2577 * according to v0 ABI
2578 */
2579static int vgic_its_save_tables_v0(struct vgic_its *its)
2580{
2581	int ret;
2582
2583	ret = vgic_its_save_device_tables(its);
2584	if (ret)
2585		return ret;
2586
2587	return vgic_its_save_collection_table(its);
2588}
2589
2590/*
2591 * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2592 * to internal data structs according to V0 ABI
2593 *
2594 */
2595static int vgic_its_restore_tables_v0(struct vgic_its *its)
2596{
2597	int ret;
2598
2599	ret = vgic_its_restore_collection_table(its);
2600	if (ret)
2601		return ret;
2602
2603	ret = vgic_its_restore_device_tables(its);
2604	if (ret)
2605		vgic_its_free_collection_list(its->dev->kvm, its);
2606	return ret;
2607}
2608
2609static int vgic_its_commit_v0(struct vgic_its *its)
2610{
2611	const struct vgic_its_abi *abi;
2612
2613	abi = vgic_its_get_abi(its);
2614	its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2615	its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2616
2617	its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2618					<< GITS_BASER_ENTRY_SIZE_SHIFT);
2619
2620	its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2621					<< GITS_BASER_ENTRY_SIZE_SHIFT);
2622	return 0;
2623}
2624
2625static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2626{
2627	/* We need to keep the ABI specific field values */
2628	its->baser_coll_table &= ~GITS_BASER_VALID;
2629	its->baser_device_table &= ~GITS_BASER_VALID;
2630	its->cbaser = 0;
2631	its->creadr = 0;
2632	its->cwriter = 0;
2633	its->enabled = 0;
2634	vgic_its_free_device_list(kvm, its);
2635	vgic_its_free_collection_list(kvm, its);
2636}
2637
2638static int vgic_its_has_attr(struct kvm_device *dev,
2639			     struct kvm_device_attr *attr)
2640{
2641	switch (attr->group) {
2642	case KVM_DEV_ARM_VGIC_GRP_ADDR:
2643		switch (attr->attr) {
2644		case KVM_VGIC_ITS_ADDR_TYPE:
2645			return 0;
2646		}
2647		break;
2648	case KVM_DEV_ARM_VGIC_GRP_CTRL:
2649		switch (attr->attr) {
2650		case KVM_DEV_ARM_VGIC_CTRL_INIT:
2651			return 0;
2652		case KVM_DEV_ARM_ITS_CTRL_RESET:
2653			return 0;
2654		case KVM_DEV_ARM_ITS_SAVE_TABLES:
2655			return 0;
2656		case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2657			return 0;
2658		}
2659		break;
2660	case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2661		return vgic_its_has_attr_regs(dev, attr);
2662	}
2663	return -ENXIO;
2664}
2665
2666static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2667{
2668	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2669	int ret = 0;
2670
2671	if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
2672		return 0;
2673
2674	mutex_lock(&kvm->lock);
2675
2676	if (kvm_trylock_all_vcpus(kvm)) {
2677		mutex_unlock(&kvm->lock);
2678		return -EBUSY;
2679	}
2680
2681	mutex_lock(&kvm->arch.config_lock);
2682	mutex_lock(&its->its_lock);
2683
2684	switch (attr) {
2685	case KVM_DEV_ARM_ITS_CTRL_RESET:
2686		vgic_its_reset(kvm, its);
2687		break;
2688	case KVM_DEV_ARM_ITS_SAVE_TABLES:
2689		ret = abi->save_tables(its);
2690		break;
2691	case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2692		ret = abi->restore_tables(its);
2693		break;
2694	default:
2695		ret = -ENXIO;
2696		break;
2697	}
2698
2699	mutex_unlock(&its->its_lock);
2700	mutex_unlock(&kvm->arch.config_lock);
2701	kvm_unlock_all_vcpus(kvm);
2702	mutex_unlock(&kvm->lock);
2703	return ret;
2704}
2705
2706/*
2707 * kvm_arch_allow_write_without_running_vcpu - allow writing guest memory
2708 * without the running VCPU when dirty ring is enabled.
2709 *
2710 * The running VCPU is required to track dirty guest pages when dirty ring
2711 * is enabled. Otherwise, the backup bitmap should be used to track the
2712 * dirty guest pages. When vgic/its tables are being saved, the backup
2713 * bitmap is used to track the dirty guest pages due to the missed running
2714 * VCPU in the period.
2715 */
2716bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
2717{
2718	struct vgic_dist *dist = &kvm->arch.vgic;
2719
2720	return dist->table_write_in_progress;
2721}
2722
2723static int vgic_its_set_attr(struct kvm_device *dev,
2724			     struct kvm_device_attr *attr)
2725{
2726	struct vgic_its *its = dev->private;
2727	int ret;
2728
2729	switch (attr->group) {
2730	case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2731		u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2732		unsigned long type = (unsigned long)attr->attr;
2733		u64 addr;
2734
2735		if (type != KVM_VGIC_ITS_ADDR_TYPE)
2736			return -ENODEV;
2737
2738		if (copy_from_user(&addr, uaddr, sizeof(addr)))
2739			return -EFAULT;
2740
2741		ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
2742					 addr, SZ_64K, KVM_VGIC_V3_ITS_SIZE);
2743		if (ret)
2744			return ret;
2745
2746		ret = vgic_register_its_iodev(dev->kvm, its, addr);
2747		if (ret)
2748			return ret;
2749
2750		return vgic_its_debug_init(dev);
2751
2752	}
2753	case KVM_DEV_ARM_VGIC_GRP_CTRL:
2754		return vgic_its_ctrl(dev->kvm, its, attr->attr);
2755	case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2756		u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2757		u64 reg;
2758
2759		if (get_user(reg, uaddr))
2760			return -EFAULT;
2761
2762		return vgic_its_attr_regs_access(dev, attr, &reg, true);
2763	}
2764	}
2765	return -ENXIO;
2766}
2767
2768static int vgic_its_get_attr(struct kvm_device *dev,
2769			     struct kvm_device_attr *attr)
2770{
2771	switch (attr->group) {
2772	case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2773		struct vgic_its *its = dev->private;
2774		u64 addr = its->vgic_its_base;
2775		u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2776		unsigned long type = (unsigned long)attr->attr;
2777
2778		if (type != KVM_VGIC_ITS_ADDR_TYPE)
2779			return -ENODEV;
2780
2781		if (copy_to_user(uaddr, &addr, sizeof(addr)))
2782			return -EFAULT;
2783		break;
2784	}
2785	case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2786		u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2787		u64 reg;
2788		int ret;
2789
2790		ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
2791		if (ret)
2792			return ret;
2793		return put_user(reg, uaddr);
2794	}
2795	default:
2796		return -ENXIO;
2797	}
2798
2799	return 0;
2800}
2801
2802static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2803	.name = "kvm-arm-vgic-its",
2804	.create = vgic_its_create,
2805	.destroy = vgic_its_destroy,
2806	.set_attr = vgic_its_set_attr,
2807	.get_attr = vgic_its_get_attr,
2808	.has_attr = vgic_its_has_attr,
2809};
2810
2811int kvm_vgic_register_its_device(void)
2812{
2813	return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2814				       KVM_DEV_TYPE_ARM_VGIC_ITS);
2815}