guest.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2012,2013 - ARM Ltd
   4 * Author: Marc Zyngier <marc.zyngier@arm.com>
   5 *
   6 * Derived from arch/arm/kvm/guest.c:
   7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
   8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
   9 */
  10
  11#include <linux/bits.h>
  12#include <linux/errno.h>
  13#include <linux/err.h>
  14#include <linux/nospec.h>
  15#include <linux/kvm_host.h>
  16#include <linux/module.h>
  17#include <linux/stddef.h>
  18#include <linux/string.h>
  19#include <linux/vmalloc.h>
  20#include <linux/fs.h>
  21#include <kvm/arm_hypercalls.h>
  22#include <asm/cputype.h>
  23#include <linux/uaccess.h>
  24#include <asm/fpsimd.h>
  25#include <asm/kvm.h>
  26#include <asm/kvm_emulate.h>
  27#include <asm/kvm_nested.h>
  28#include <asm/sigcontext.h>
  29
  30#include "trace.h"
  31
  32const struct kvm_stats_desc kvm_vm_stats_desc[] = {
  33	KVM_GENERIC_VM_STATS()
  34};
  35
  36const struct kvm_stats_header kvm_vm_stats_header = {
  37	.name_size = KVM_STATS_NAME_SIZE,
  38	.num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
  39	.id_offset =  sizeof(struct kvm_stats_header),
  40	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  41	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  42		       sizeof(kvm_vm_stats_desc),
  43};
  44
  45const struct kvm_stats_desc kvm_vcpu_stats_desc[] = {
  46	KVM_GENERIC_VCPU_STATS(),
  47	STATS_DESC_COUNTER(VCPU, hvc_exit_stat),
  48	STATS_DESC_COUNTER(VCPU, wfe_exit_stat),
  49	STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
  50	STATS_DESC_COUNTER(VCPU, mmio_exit_user),
  51	STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
  52	STATS_DESC_COUNTER(VCPU, signal_exits),
  53	STATS_DESC_COUNTER(VCPU, exits)
  54};
  55
  56const struct kvm_stats_header kvm_vcpu_stats_header = {
  57	.name_size = KVM_STATS_NAME_SIZE,
  58	.num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
  59	.id_offset = sizeof(struct kvm_stats_header),
  60	.desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  61	.data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  62		       sizeof(kvm_vcpu_stats_desc),
  63};
  64
  65static bool core_reg_offset_is_vreg(u64 off)
  66{
  67	return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) &&
  68		off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr);
  69}
  70
  71static u64 core_reg_offset_from_id(u64 id)
  72{
  73	return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
  74}
  75
  76static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off)
  77{
  78	int size;
  79
  80	switch (off) {
  81	case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
  82	     KVM_REG_ARM_CORE_REG(regs.regs[30]):
  83	case KVM_REG_ARM_CORE_REG(regs.sp):
  84	case KVM_REG_ARM_CORE_REG(regs.pc):
  85	case KVM_REG_ARM_CORE_REG(regs.pstate):
  86	case KVM_REG_ARM_CORE_REG(sp_el1):
  87	case KVM_REG_ARM_CORE_REG(elr_el1):
  88	case KVM_REG_ARM_CORE_REG(spsr[0]) ...
  89	     KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
  90		size = sizeof(__u64);
  91		break;
  92
  93	case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
  94	     KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
  95		size = sizeof(__uint128_t);
  96		break;
  97
  98	case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
  99	case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
 100		size = sizeof(__u32);
 101		break;
 102
 103	default:
 104		return -EINVAL;
 105	}
 106
 107	if (!IS_ALIGNED(off, size / sizeof(__u32)))
 108		return -EINVAL;
 109
 110	/*
 111	 * The KVM_REG_ARM64_SVE regs must be used instead of
 112	 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on
 113	 * SVE-enabled vcpus:
 114	 */
 115	if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off))
 116		return -EINVAL;
 117
 118	return size;
 119}
 120
 121static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 122{
 123	u64 off = core_reg_offset_from_id(reg->id);
 124	int size = core_reg_size_from_offset(vcpu, off);
 125
 126	if (size < 0)
 127		return NULL;
 128
 129	if (KVM_REG_SIZE(reg->id) != size)
 130		return NULL;
 131
 132	switch (off) {
 133	case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
 134	     KVM_REG_ARM_CORE_REG(regs.regs[30]):
 135		off -= KVM_REG_ARM_CORE_REG(regs.regs[0]);
 136		off /= 2;
 137		return &vcpu->arch.ctxt.regs.regs[off];
 138
 139	case KVM_REG_ARM_CORE_REG(regs.sp):
 140		return &vcpu->arch.ctxt.regs.sp;
 141
 142	case KVM_REG_ARM_CORE_REG(regs.pc):
 143		return &vcpu->arch.ctxt.regs.pc;
 144
 145	case KVM_REG_ARM_CORE_REG(regs.pstate):
 146		return &vcpu->arch.ctxt.regs.pstate;
 147
 148	case KVM_REG_ARM_CORE_REG(sp_el1):
 149		return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1);
 150
 151	case KVM_REG_ARM_CORE_REG(elr_el1):
 152		return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1);
 153
 154	case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]):
 155		return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1);
 156
 157	case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]):
 158		return &vcpu->arch.ctxt.spsr_abt;
 159
 160	case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]):
 161		return &vcpu->arch.ctxt.spsr_und;
 162
 163	case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]):
 164		return &vcpu->arch.ctxt.spsr_irq;
 165
 166	case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]):
 167		return &vcpu->arch.ctxt.spsr_fiq;
 168
 169	case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
 170	     KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
 171		off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]);
 172		off /= 4;
 173		return &vcpu->arch.ctxt.fp_regs.vregs[off];
 174
 175	case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
 176		return &vcpu->arch.ctxt.fp_regs.fpsr;
 177
 178	case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
 179		return &vcpu->arch.ctxt.fp_regs.fpcr;
 180
 181	default:
 182		return NULL;
 183	}
 184}
 185
 186static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 187{
 188	/*
 189	 * Because the kvm_regs structure is a mix of 32, 64 and
 190	 * 128bit fields, we index it as if it was a 32bit
 191	 * array. Hence below, nr_regs is the number of entries, and
 192	 * off the index in the "array".
 193	 */
 194	__u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
 195	int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
 196	void *addr;
 197	u32 off;
 198
 199	/* Our ID is an index into the kvm_regs struct. */
 200	off = core_reg_offset_from_id(reg->id);
 201	if (off >= nr_regs ||
 202	    (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
 203		return -ENOENT;
 204
 205	addr = core_reg_addr(vcpu, reg);
 206	if (!addr)
 207		return -EINVAL;
 208
 209	if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id)))
 210		return -EFAULT;
 211
 212	return 0;
 213}
 214
 215static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 216{
 217	__u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
 218	int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
 219	__uint128_t tmp;
 220	void *valp = &tmp, *addr;
 221	u64 off;
 222	int err = 0;
 223
 224	/* Our ID is an index into the kvm_regs struct. */
 225	off = core_reg_offset_from_id(reg->id);
 226	if (off >= nr_regs ||
 227	    (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
 228		return -ENOENT;
 229
 230	addr = core_reg_addr(vcpu, reg);
 231	if (!addr)
 232		return -EINVAL;
 233
 234	if (KVM_REG_SIZE(reg->id) > sizeof(tmp))
 235		return -EINVAL;
 236
 237	if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) {
 238		err = -EFAULT;
 239		goto out;
 240	}
 241
 242	if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) {
 243		u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK;
 244		switch (mode) {
 245		case PSR_AA32_MODE_USR:
 246			if (!kvm_supports_32bit_el0())
 247				return -EINVAL;
 248			break;
 249		case PSR_AA32_MODE_FIQ:
 250		case PSR_AA32_MODE_IRQ:
 251		case PSR_AA32_MODE_SVC:
 252		case PSR_AA32_MODE_ABT:
 253		case PSR_AA32_MODE_UND:
 254		case PSR_AA32_MODE_SYS:
 255			if (!vcpu_el1_is_32bit(vcpu))
 256				return -EINVAL;
 257			break;
 258		case PSR_MODE_EL2h:
 259		case PSR_MODE_EL2t:
 260			if (!vcpu_has_nv(vcpu))
 261				return -EINVAL;
 262			fallthrough;
 263		case PSR_MODE_EL0t:
 264		case PSR_MODE_EL1t:
 265		case PSR_MODE_EL1h:
 266			if (vcpu_el1_is_32bit(vcpu))
 267				return -EINVAL;
 268			break;
 269		default:
 270			err = -EINVAL;
 271			goto out;
 272		}
 273	}
 274
 275	memcpy(addr, valp, KVM_REG_SIZE(reg->id));
 276
 277	if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) {
 278		int i, nr_reg;
 279
 280		switch (*vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK) {
 281		/*
 282		 * Either we are dealing with user mode, and only the
 283		 * first 15 registers (+ PC) must be narrowed to 32bit.
 284		 * AArch32 r0-r14 conveniently map to AArch64 x0-x14.
 285		 */
 286		case PSR_AA32_MODE_USR:
 287		case PSR_AA32_MODE_SYS:
 288			nr_reg = 15;
 289			break;
 290
 291		/*
 292		 * Otherwise, this is a privileged mode, and *all* the
 293		 * registers must be narrowed to 32bit.
 294		 */
 295		default:
 296			nr_reg = 31;
 297			break;
 298		}
 299
 300		for (i = 0; i < nr_reg; i++)
 301			vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i));
 302
 303		*vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu);
 304	}
 305out:
 306	return err;
 307}
 308
 309#define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64)
 310#define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64)
 311#define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq)))
 312
 313static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 314{
 315	unsigned int max_vq, vq;
 316	u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
 317
 318	if (!vcpu_has_sve(vcpu))
 319		return -ENOENT;
 320
 321	if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl)))
 322		return -EINVAL;
 323
 324	memset(vqs, 0, sizeof(vqs));
 325
 326	max_vq = vcpu_sve_max_vq(vcpu);
 327	for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
 328		if (sve_vq_available(vq))
 329			vqs[vq_word(vq)] |= vq_mask(vq);
 330
 331	if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs)))
 332		return -EFAULT;
 333
 334	return 0;
 335}
 336
 337static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 338{
 339	unsigned int max_vq, vq;
 340	u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
 341
 342	if (!vcpu_has_sve(vcpu))
 343		return -ENOENT;
 344
 345	if (kvm_arm_vcpu_sve_finalized(vcpu))
 346		return -EPERM; /* too late! */
 347
 348	if (WARN_ON(vcpu->arch.sve_state))
 349		return -EINVAL;
 350
 351	if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs)))
 352		return -EFAULT;
 353
 354	max_vq = 0;
 355	for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq)
 356		if (vq_present(vqs, vq))
 357			max_vq = vq;
 358
 359	if (max_vq > sve_vq_from_vl(kvm_sve_max_vl))
 360		return -EINVAL;
 361
 362	/*
 363	 * Vector lengths supported by the host can't currently be
 364	 * hidden from the guest individually: instead we can only set a
 365	 * maximum via ZCR_EL2.LEN.  So, make sure the available vector
 366	 * lengths match the set requested exactly up to the requested
 367	 * maximum:
 368	 */
 369	for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
 370		if (vq_present(vqs, vq) != sve_vq_available(vq))
 371			return -EINVAL;
 372
 373	/* Can't run with no vector lengths at all: */
 374	if (max_vq < SVE_VQ_MIN)
 375		return -EINVAL;
 376
 377	/* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */
 378	vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq);
 379
 380	return 0;
 381}
 382
 383#define SVE_REG_SLICE_SHIFT	0
 384#define SVE_REG_SLICE_BITS	5
 385#define SVE_REG_ID_SHIFT	(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS)
 386#define SVE_REG_ID_BITS		5
 387
 388#define SVE_REG_SLICE_MASK					\
 389	GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1,	\
 390		SVE_REG_SLICE_SHIFT)
 391#define SVE_REG_ID_MASK							\
 392	GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT)
 393
 394#define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS)
 395
 396#define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0))
 397#define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0))
 398
 399/*
 400 * Number of register slices required to cover each whole SVE register.
 401 * NOTE: Only the first slice every exists, for now.
 402 * If you are tempted to modify this, you must also rework sve_reg_to_region()
 403 * to match:
 404 */
 405#define vcpu_sve_slices(vcpu) 1
 406
 407/* Bounds of a single SVE register slice within vcpu->arch.sve_state */
 408struct sve_state_reg_region {
 409	unsigned int koffset;	/* offset into sve_state in kernel memory */
 410	unsigned int klen;	/* length in kernel memory */
 411	unsigned int upad;	/* extra trailing padding in user memory */
 412};
 413
 414/*
 415 * Validate SVE register ID and get sanitised bounds for user/kernel SVE
 416 * register copy
 417 */
 418static int sve_reg_to_region(struct sve_state_reg_region *region,
 419			     struct kvm_vcpu *vcpu,
 420			     const struct kvm_one_reg *reg)
 421{
 422	/* reg ID ranges for Z- registers */
 423	const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0);
 424	const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1,
 425						       SVE_NUM_SLICES - 1);
 426
 427	/* reg ID ranges for P- registers and FFR (which are contiguous) */
 428	const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0);
 429	const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1);
 430
 431	unsigned int vq;
 432	unsigned int reg_num;
 433
 434	unsigned int reqoffset, reqlen; /* User-requested offset and length */
 435	unsigned int maxlen; /* Maximum permitted length */
 436
 437	size_t sve_state_size;
 438
 439	const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1,
 440							SVE_NUM_SLICES - 1);
 441
 442	/* Verify that the P-regs and FFR really do have contiguous IDs: */
 443	BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1);
 444
 445	/* Verify that we match the UAPI header: */
 446	BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES);
 447
 448	reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
 449
 450	if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) {
 451		if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
 452			return -ENOENT;
 453
 454		vq = vcpu_sve_max_vq(vcpu);
 455
 456		reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) -
 457				SVE_SIG_REGS_OFFSET;
 458		reqlen = KVM_SVE_ZREG_SIZE;
 459		maxlen = SVE_SIG_ZREG_SIZE(vq);
 460	} else if (reg->id >= preg_id_min && reg->id <= preg_id_max) {
 461		if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
 462			return -ENOENT;
 463
 464		vq = vcpu_sve_max_vq(vcpu);
 465
 466		reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) -
 467				SVE_SIG_REGS_OFFSET;
 468		reqlen = KVM_SVE_PREG_SIZE;
 469		maxlen = SVE_SIG_PREG_SIZE(vq);
 470	} else {
 471		return -EINVAL;
 472	}
 473
 474	sve_state_size = vcpu_sve_state_size(vcpu);
 475	if (WARN_ON(!sve_state_size))
 476		return -EINVAL;
 477
 478	region->koffset = array_index_nospec(reqoffset, sve_state_size);
 479	region->klen = min(maxlen, reqlen);
 480	region->upad = reqlen - region->klen;
 481
 482	return 0;
 483}
 484
 485static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 486{
 487	int ret;
 488	struct sve_state_reg_region region;
 489	char __user *uptr = (char __user *)reg->addr;
 490
 491	/* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
 492	if (reg->id == KVM_REG_ARM64_SVE_VLS)
 493		return get_sve_vls(vcpu, reg);
 494
 495	/* Try to interpret reg ID as an architectural SVE register... */
 496	ret = sve_reg_to_region(&region, vcpu, reg);
 497	if (ret)
 498		return ret;
 499
 500	if (!kvm_arm_vcpu_sve_finalized(vcpu))
 501		return -EPERM;
 502
 503	if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset,
 504			 region.klen) ||
 505	    clear_user(uptr + region.klen, region.upad))
 506		return -EFAULT;
 507
 508	return 0;
 509}
 510
 511static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 512{
 513	int ret;
 514	struct sve_state_reg_region region;
 515	const char __user *uptr = (const char __user *)reg->addr;
 516
 517	/* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
 518	if (reg->id == KVM_REG_ARM64_SVE_VLS)
 519		return set_sve_vls(vcpu, reg);
 520
 521	/* Try to interpret reg ID as an architectural SVE register... */
 522	ret = sve_reg_to_region(&region, vcpu, reg);
 523	if (ret)
 524		return ret;
 525
 526	if (!kvm_arm_vcpu_sve_finalized(vcpu))
 527		return -EPERM;
 528
 529	if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr,
 530			   region.klen))
 531		return -EFAULT;
 532
 533	return 0;
 534}
 535
 536int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 537{
 538	return -EINVAL;
 539}
 540
 541int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 542{
 543	return -EINVAL;
 544}
 545
 546static int copy_core_reg_indices(const struct kvm_vcpu *vcpu,
 547				 u64 __user *uindices)
 548{
 549	unsigned int i;
 550	int n = 0;
 551
 552	for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {
 553		u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i;
 554		int size = core_reg_size_from_offset(vcpu, i);
 555
 556		if (size < 0)
 557			continue;
 558
 559		switch (size) {
 560		case sizeof(__u32):
 561			reg |= KVM_REG_SIZE_U32;
 562			break;
 563
 564		case sizeof(__u64):
 565			reg |= KVM_REG_SIZE_U64;
 566			break;
 567
 568		case sizeof(__uint128_t):
 569			reg |= KVM_REG_SIZE_U128;
 570			break;
 571
 572		default:
 573			WARN_ON(1);
 574			continue;
 575		}
 576
 577		if (uindices) {
 578			if (put_user(reg, uindices))
 579				return -EFAULT;
 580			uindices++;
 581		}
 582
 583		n++;
 584	}
 585
 586	return n;
 587}
 588
 589static unsigned long num_core_regs(const struct kvm_vcpu *vcpu)
 590{
 591	return copy_core_reg_indices(vcpu, NULL);
 592}
 593
 594static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
 595{
 596	const unsigned int slices = vcpu_sve_slices(vcpu);
 597
 598	if (!vcpu_has_sve(vcpu))
 599		return 0;
 600
 601	/* Policed by KVM_GET_REG_LIST: */
 602	WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
 603
 604	return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */)
 605		+ 1; /* KVM_REG_ARM64_SVE_VLS */
 606}
 607
 608static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu,
 609				u64 __user *uindices)
 610{
 611	const unsigned int slices = vcpu_sve_slices(vcpu);
 612	u64 reg;
 613	unsigned int i, n;
 614	int num_regs = 0;
 615
 616	if (!vcpu_has_sve(vcpu))
 617		return 0;
 618
 619	/* Policed by KVM_GET_REG_LIST: */
 620	WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
 621
 622	/*
 623	 * Enumerate this first, so that userspace can save/restore in
 624	 * the order reported by KVM_GET_REG_LIST:
 625	 */
 626	reg = KVM_REG_ARM64_SVE_VLS;
 627	if (put_user(reg, uindices++))
 628		return -EFAULT;
 629	++num_regs;
 630
 631	for (i = 0; i < slices; i++) {
 632		for (n = 0; n < SVE_NUM_ZREGS; n++) {
 633			reg = KVM_REG_ARM64_SVE_ZREG(n, i);
 634			if (put_user(reg, uindices++))
 635				return -EFAULT;
 636			num_regs++;
 637		}
 638
 639		for (n = 0; n < SVE_NUM_PREGS; n++) {
 640			reg = KVM_REG_ARM64_SVE_PREG(n, i);
 641			if (put_user(reg, uindices++))
 642				return -EFAULT;
 643			num_regs++;
 644		}
 645
 646		reg = KVM_REG_ARM64_SVE_FFR(i);
 647		if (put_user(reg, uindices++))
 648			return -EFAULT;
 649		num_regs++;
 650	}
 651
 652	return num_regs;
 653}
 654
 655/**
 656 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
 657 * @vcpu: the vCPU pointer
 658 *
 659 * This is for all registers.
 660 */
 661unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
 662{
 663	unsigned long res = 0;
 664
 665	res += num_core_regs(vcpu);
 666	res += num_sve_regs(vcpu);
 667	res += kvm_arm_num_sys_reg_descs(vcpu);
 668	res += kvm_arm_get_fw_num_regs(vcpu);
 669
 670	return res;
 671}
 672
 673/**
 674 * kvm_arm_copy_reg_indices - get indices of all registers.
 675 * @vcpu: the vCPU pointer
 676 * @uindices: register list to copy
 677 *
 678 * We do core registers right here, then we append system regs.
 679 */
 680int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 681{
 682	int ret;
 683
 684	ret = copy_core_reg_indices(vcpu, uindices);
 685	if (ret < 0)
 686		return ret;
 687	uindices += ret;
 688
 689	ret = copy_sve_reg_indices(vcpu, uindices);
 690	if (ret < 0)
 691		return ret;
 692	uindices += ret;
 693
 694	ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices);
 695	if (ret < 0)
 696		return ret;
 697	uindices += kvm_arm_get_fw_num_regs(vcpu);
 698
 699	return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
 700}
 701
 702int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 703{
 704	/* We currently use nothing arch-specific in upper 32 bits */
 705	if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
 706		return -EINVAL;
 707
 708	switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
 709	case KVM_REG_ARM_CORE:	return get_core_reg(vcpu, reg);
 710	case KVM_REG_ARM_FW:
 711	case KVM_REG_ARM_FW_FEAT_BMAP:
 712		return kvm_arm_get_fw_reg(vcpu, reg);
 713	case KVM_REG_ARM64_SVE:	return get_sve_reg(vcpu, reg);
 714	}
 715
 716	return kvm_arm_sys_reg_get_reg(vcpu, reg);
 717}
 718
 719int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 720{
 721	/* We currently use nothing arch-specific in upper 32 bits */
 722	if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
 723		return -EINVAL;
 724
 725	switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
 726	case KVM_REG_ARM_CORE:	return set_core_reg(vcpu, reg);
 727	case KVM_REG_ARM_FW:
 728	case KVM_REG_ARM_FW_FEAT_BMAP:
 729		return kvm_arm_set_fw_reg(vcpu, reg);
 730	case KVM_REG_ARM64_SVE:	return set_sve_reg(vcpu, reg);
 731	}
 732
 733	return kvm_arm_sys_reg_set_reg(vcpu, reg);
 734}
 735
 736int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 737				  struct kvm_sregs *sregs)
 738{
 739	return -EINVAL;
 740}
 741
 742int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 743				  struct kvm_sregs *sregs)
 744{
 745	return -EINVAL;
 746}
 747
 748int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
 749			      struct kvm_vcpu_events *events)
 750{
 751	events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
 752	events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) ||
 753					   vcpu_get_flag(vcpu, NESTED_SERROR_PENDING);
 754
 755	if (events->exception.serror_pending && events->exception.serror_has_esr)
 756		events->exception.serror_esr = vcpu_get_vsesr(vcpu);
 757
 758	/*
 759	 * We never return a pending ext_dabt here because we deliver it to
 760	 * the virtual CPU directly when setting the event and it's no longer
 761	 * 'pending' at this point.
 762	 */
 763
 764	return 0;
 765}
 766
 767static void commit_pending_events(struct kvm_vcpu *vcpu)
 768{
 769	if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
 770		return;
 771
 772	/*
 773	 * Reset the MMIO emulation state to avoid stepping PC after emulating
 774	 * the exception entry.
 775	 */
 776	vcpu->mmio_needed = false;
 777	kvm_call_hyp(__kvm_adjust_pc, vcpu);
 778}
 779
 780int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
 781			      struct kvm_vcpu_events *events)
 782{
 783	bool serror_pending = events->exception.serror_pending;
 784	bool has_esr = events->exception.serror_has_esr;
 785	bool ext_dabt_pending = events->exception.ext_dabt_pending;
 786	u64 esr = events->exception.serror_esr;
 787	int ret = 0;
 788
 789	/*
 790	 * Immediately commit the pending SEA to the vCPU's architectural
 791	 * state which is necessary since we do not return a pending SEA
 792	 * to userspace via KVM_GET_VCPU_EVENTS.
 793	 */
 794	if (ext_dabt_pending) {
 795		ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
 796		commit_pending_events(vcpu);
 797	}
 798
 799	if (ret < 0)
 800		return ret;
 801
 802	if (!serror_pending)
 803		return 0;
 804
 805	if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && has_esr)
 806		return -EINVAL;
 807
 808	if (has_esr && (esr & ~ESR_ELx_ISS_MASK))
 809		return -EINVAL;
 810
 811	if (has_esr)
 812		ret = kvm_inject_serror_esr(vcpu, esr);
 813	else
 814		ret = kvm_inject_serror(vcpu);
 815
 816	/*
 817	 * We could've decided that the SError is due for immediate software
 818	 * injection; commit the exception in case userspace decides it wants
 819	 * to inject more exceptions for some strange reason.
 820	 */
 821	commit_pending_events(vcpu);
 822	return (ret < 0) ? ret : 0;
 823}
 824
 825u32 __attribute_const__ kvm_target_cpu(void)
 826{
 827	unsigned long implementor = read_cpuid_implementor();
 828	unsigned long part_number = read_cpuid_part_number();
 829
 830	switch (implementor) {
 831	case ARM_CPU_IMP_ARM:
 832		switch (part_number) {
 833		case ARM_CPU_PART_AEM_V8:
 834			return KVM_ARM_TARGET_AEM_V8;
 835		case ARM_CPU_PART_FOUNDATION:
 836			return KVM_ARM_TARGET_FOUNDATION_V8;
 837		case ARM_CPU_PART_CORTEX_A53:
 838			return KVM_ARM_TARGET_CORTEX_A53;
 839		case ARM_CPU_PART_CORTEX_A57:
 840			return KVM_ARM_TARGET_CORTEX_A57;
 841		}
 842		break;
 843	case ARM_CPU_IMP_APM:
 844		switch (part_number) {
 845		case APM_CPU_PART_XGENE:
 846			return KVM_ARM_TARGET_XGENE_POTENZA;
 847		}
 848		break;
 849	}
 850
 851	/* Return a default generic target */
 852	return KVM_ARM_TARGET_GENERIC_V8;
 853}
 854
 855int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 856{
 857	return -EINVAL;
 858}
 859
 860int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 861{
 862	return -EINVAL;
 863}
 864
 865int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 866				  struct kvm_translation *tr)
 867{
 868	return -EINVAL;
 869}
 870
 871/**
 872 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging
 873 * @vcpu: the vCPU pointer
 874 * @dbg: the ioctl data buffer
 875 *
 876 * This sets up and enables the VM for guest debugging. Userspace
 877 * passes in a control flag to enable different debug types and
 878 * potentially other architecture specific information in the rest of
 879 * the structure.
 880 */
 881int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 882					struct kvm_guest_debug *dbg)
 883{
 884	trace_kvm_set_guest_debug(vcpu, dbg->control);
 885
 886	if (dbg->control & ~KVM_GUESTDBG_VALID_MASK)
 887		return -EINVAL;
 888
 889	if (!(dbg->control & KVM_GUESTDBG_ENABLE)) {
 890		vcpu->guest_debug = 0;
 891		vcpu_clear_flag(vcpu, HOST_SS_ACTIVE_PENDING);
 892		return 0;
 893	}
 894
 895	vcpu->guest_debug = dbg->control;
 896
 897	/* Hardware assisted Break and Watch points */
 898	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW)
 899		vcpu->arch.external_debug_state = dbg->arch;
 900
 901	return 0;
 902}
 903
 904int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
 905			       struct kvm_device_attr *attr)
 906{
 907	int ret;
 908
 909	switch (attr->group) {
 910	case KVM_ARM_VCPU_PMU_V3_CTRL:
 911		mutex_lock(&vcpu->kvm->arch.config_lock);
 912		ret = kvm_arm_pmu_v3_set_attr(vcpu, attr);
 913		mutex_unlock(&vcpu->kvm->arch.config_lock);
 914		break;
 915	case KVM_ARM_VCPU_TIMER_CTRL:
 916		ret = kvm_arm_timer_set_attr(vcpu, attr);
 917		break;
 918	case KVM_ARM_VCPU_PVTIME_CTRL:
 919		ret = kvm_arm_pvtime_set_attr(vcpu, attr);
 920		break;
 921	default:
 922		ret = -ENXIO;
 923		break;
 924	}
 925
 926	return ret;
 927}
 928
 929int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
 930			       struct kvm_device_attr *attr)
 931{
 932	int ret;
 933
 934	switch (attr->group) {
 935	case KVM_ARM_VCPU_PMU_V3_CTRL:
 936		ret = kvm_arm_pmu_v3_get_attr(vcpu, attr);
 937		break;
 938	case KVM_ARM_VCPU_TIMER_CTRL:
 939		ret = kvm_arm_timer_get_attr(vcpu, attr);
 940		break;
 941	case KVM_ARM_VCPU_PVTIME_CTRL:
 942		ret = kvm_arm_pvtime_get_attr(vcpu, attr);
 943		break;
 944	default:
 945		ret = -ENXIO;
 946		break;
 947	}
 948
 949	return ret;
 950}
 951
 952int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
 953			       struct kvm_device_attr *attr)
 954{
 955	int ret;
 956
 957	switch (attr->group) {
 958	case KVM_ARM_VCPU_PMU_V3_CTRL:
 959		ret = kvm_arm_pmu_v3_has_attr(vcpu, attr);
 960		break;
 961	case KVM_ARM_VCPU_TIMER_CTRL:
 962		ret = kvm_arm_timer_has_attr(vcpu, attr);
 963		break;
 964	case KVM_ARM_VCPU_PVTIME_CTRL:
 965		ret = kvm_arm_pvtime_has_attr(vcpu, attr);
 966		break;
 967	default:
 968		ret = -ENXIO;
 969		break;
 970	}
 971
 972	return ret;
 973}
 974
 975int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
 976			       struct kvm_arm_copy_mte_tags *copy_tags)
 977{
 978	gpa_t guest_ipa = copy_tags->guest_ipa;
 979	size_t length = copy_tags->length;
 980	void __user *tags = copy_tags->addr;
 981	gpa_t gfn;
 982	bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST);
 983	int ret = 0;
 984
 985	if (!kvm_has_mte(kvm))
 986		return -EINVAL;
 987
 988	if (copy_tags->reserved[0] || copy_tags->reserved[1])
 989		return -EINVAL;
 990
 991	if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST)
 992		return -EINVAL;
 993
 994	if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK)
 995		return -EINVAL;
 996
 997	/* Lengths above INT_MAX cannot be represented in the return value */
 998	if (length > INT_MAX)
 999		return -EINVAL;
1000
1001	gfn = gpa_to_gfn(guest_ipa);
1002
1003	mutex_lock(&kvm->slots_lock);
1004
1005	if (write && atomic_read(&kvm->nr_memslots_dirty_logging)) {
1006		ret = -EBUSY;
1007		goto out;
1008	}
1009
1010	while (length > 0) {
1011		struct page *page = __gfn_to_page(kvm, gfn, write);
1012		void *maddr;
1013		unsigned long num_tags;
1014		struct folio *folio;
1015
1016		if (!page) {
1017			ret = -EFAULT;
1018			goto out;
1019		}
1020
1021		if (!pfn_to_online_page(page_to_pfn(page))) {
1022			/* Reject ZONE_DEVICE memory */
1023			kvm_release_page_unused(page);
1024			ret = -EFAULT;
1025			goto out;
1026		}
1027		folio = page_folio(page);
1028		maddr = page_address(page);
1029
1030		if (!write) {
1031			if ((folio_test_hugetlb(folio) &&
1032			     folio_test_hugetlb_mte_tagged(folio)) ||
1033			     page_mte_tagged(page))
1034				num_tags = mte_copy_tags_to_user(tags, maddr,
1035							MTE_GRANULES_PER_PAGE);
1036			else
1037				/* No tags in memory, so write zeros */
1038				num_tags = MTE_GRANULES_PER_PAGE -
1039					clear_user(tags, MTE_GRANULES_PER_PAGE);
1040			kvm_release_page_clean(page);
1041		} else {
1042			/*
1043			 * Only locking to serialise with a concurrent
1044			 * __set_ptes() in the VMM but still overriding the
1045			 * tags, hence ignoring the return value.
1046			 */
1047			if (folio_test_hugetlb(folio))
1048				folio_try_hugetlb_mte_tagging(folio);
1049			else
1050				try_page_mte_tagging(page);
1051			num_tags = mte_copy_tags_from_user(maddr, tags,
1052							MTE_GRANULES_PER_PAGE);
1053
1054			/* uaccess failed, don't leave stale tags */
1055			if (num_tags != MTE_GRANULES_PER_PAGE)
1056				mte_clear_page_tags(maddr);
1057			if (folio_test_hugetlb(folio))
1058				folio_set_hugetlb_mte_tagged(folio);
1059			else
1060				set_page_mte_tagged(page);
1061
1062			kvm_release_page_dirty(page);
1063		}
1064
1065		if (num_tags != MTE_GRANULES_PER_PAGE) {
1066			ret = -EFAULT;
1067			goto out;
1068		}
1069
1070		gfn++;
1071		tags += num_tags;
1072		length -= PAGE_SIZE;
1073	}
1074
1075out:
1076	mutex_unlock(&kvm->slots_lock);
1077	/* If some data has been copied report the number of bytes copied */
1078	if (length != copy_tags->length)
1079		return copy_tags->length - length;
1080	return ret;
1081}