vcpu_onereg.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
   4 * Copyright (C) 2023 Ventana Micro Systems Inc.
   5 *
   6 * Authors:
   7 *	Anup Patel <apatel@ventanamicro.com>
   8 */
   9
  10#include <linux/bitops.h>
  11#include <linux/errno.h>
  12#include <linux/err.h>
  13#include <linux/uaccess.h>
  14#include <linux/kvm_host.h>
  15#include <asm/cacheflush.h>
  16#include <asm/cpufeature.h>
  17#include <asm/kvm_vcpu_vector.h>
  18#include <asm/pgtable.h>
  19#include <asm/vector.h>
  20
  21#define KVM_RISCV_BASE_ISA_MASK		GENMASK(25, 0)
  22
  23#define KVM_ISA_EXT_ARR(ext)		\
  24[KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext
  25
  26/* Mapping between KVM ISA Extension ID & guest ISA extension ID */
  27static const unsigned long kvm_isa_ext_arr[] = {
  28	/* Single letter extensions (alphabetically sorted) */
  29	[KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a,
  30	[KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c,
  31	[KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d,
  32	[KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f,
  33	[KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h,
  34	[KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i,
  35	[KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
  36	[KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,
  37	/* Multi letter extensions (alphabetically sorted) */
  38	KVM_ISA_EXT_ARR(SMNPM),
  39	KVM_ISA_EXT_ARR(SMSTATEEN),
  40	KVM_ISA_EXT_ARR(SSAIA),
  41	KVM_ISA_EXT_ARR(SSCOFPMF),
  42	KVM_ISA_EXT_ARR(SSNPM),
  43	KVM_ISA_EXT_ARR(SSTC),
  44	KVM_ISA_EXT_ARR(SVADE),
  45	KVM_ISA_EXT_ARR(SVADU),
  46	KVM_ISA_EXT_ARR(SVINVAL),
  47	KVM_ISA_EXT_ARR(SVNAPOT),
  48	KVM_ISA_EXT_ARR(SVPBMT),
  49	KVM_ISA_EXT_ARR(SVVPTC),
  50	KVM_ISA_EXT_ARR(ZAAMO),
  51	KVM_ISA_EXT_ARR(ZABHA),
  52	KVM_ISA_EXT_ARR(ZACAS),
  53	KVM_ISA_EXT_ARR(ZALRSC),
  54	KVM_ISA_EXT_ARR(ZAWRS),
  55	KVM_ISA_EXT_ARR(ZBA),
  56	KVM_ISA_EXT_ARR(ZBB),
  57	KVM_ISA_EXT_ARR(ZBC),
  58	KVM_ISA_EXT_ARR(ZBKB),
  59	KVM_ISA_EXT_ARR(ZBKC),
  60	KVM_ISA_EXT_ARR(ZBKX),
  61	KVM_ISA_EXT_ARR(ZBS),
  62	KVM_ISA_EXT_ARR(ZCA),
  63	KVM_ISA_EXT_ARR(ZCB),
  64	KVM_ISA_EXT_ARR(ZCD),
  65	KVM_ISA_EXT_ARR(ZCF),
  66	KVM_ISA_EXT_ARR(ZCMOP),
  67	KVM_ISA_EXT_ARR(ZFA),
  68	KVM_ISA_EXT_ARR(ZFBFMIN),
  69	KVM_ISA_EXT_ARR(ZFH),
  70	KVM_ISA_EXT_ARR(ZFHMIN),
  71	KVM_ISA_EXT_ARR(ZICBOM),
  72	KVM_ISA_EXT_ARR(ZICBOP),
  73	KVM_ISA_EXT_ARR(ZICBOZ),
  74	KVM_ISA_EXT_ARR(ZICCRSE),
  75	KVM_ISA_EXT_ARR(ZICNTR),
  76	KVM_ISA_EXT_ARR(ZICOND),
  77	KVM_ISA_EXT_ARR(ZICSR),
  78	KVM_ISA_EXT_ARR(ZIFENCEI),
  79	KVM_ISA_EXT_ARR(ZIHINTNTL),
  80	KVM_ISA_EXT_ARR(ZIHINTPAUSE),
  81	KVM_ISA_EXT_ARR(ZIHPM),
  82	KVM_ISA_EXT_ARR(ZIMOP),
  83	KVM_ISA_EXT_ARR(ZKND),
  84	KVM_ISA_EXT_ARR(ZKNE),
  85	KVM_ISA_EXT_ARR(ZKNH),
  86	KVM_ISA_EXT_ARR(ZKR),
  87	KVM_ISA_EXT_ARR(ZKSED),
  88	KVM_ISA_EXT_ARR(ZKSH),
  89	KVM_ISA_EXT_ARR(ZKT),
  90	KVM_ISA_EXT_ARR(ZTSO),
  91	KVM_ISA_EXT_ARR(ZVBB),
  92	KVM_ISA_EXT_ARR(ZVBC),
  93	KVM_ISA_EXT_ARR(ZVFBFMIN),
  94	KVM_ISA_EXT_ARR(ZVFBFWMA),
  95	KVM_ISA_EXT_ARR(ZVFH),
  96	KVM_ISA_EXT_ARR(ZVFHMIN),
  97	KVM_ISA_EXT_ARR(ZVKB),
  98	KVM_ISA_EXT_ARR(ZVKG),
  99	KVM_ISA_EXT_ARR(ZVKNED),
 100	KVM_ISA_EXT_ARR(ZVKNHA),
 101	KVM_ISA_EXT_ARR(ZVKNHB),
 102	KVM_ISA_EXT_ARR(ZVKSED),
 103	KVM_ISA_EXT_ARR(ZVKSH),
 104	KVM_ISA_EXT_ARR(ZVKT),
 105};
 106
 107static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext)
 108{
 109	unsigned long i;
 110
 111	for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
 112		if (kvm_isa_ext_arr[i] == base_ext)
 113			return i;
 114	}
 115
 116	return KVM_RISCV_ISA_EXT_MAX;
 117}
 118
 119static int kvm_riscv_vcpu_isa_check_host(unsigned long kvm_ext, unsigned long *guest_ext)
 120{
 121	unsigned long host_ext;
 122
 123	if (kvm_ext >= KVM_RISCV_ISA_EXT_MAX ||
 124	    kvm_ext >= ARRAY_SIZE(kvm_isa_ext_arr))
 125		return -ENOENT;
 126
 127	*guest_ext = kvm_isa_ext_arr[kvm_ext];
 128	switch (*guest_ext) {
 129	case RISCV_ISA_EXT_SMNPM:
 130		/*
 131		 * Pointer masking effective in (H)S-mode is provided by the
 132		 * Smnpm extension, so that extension is reported to the guest,
 133		 * even though the CSR bits for configuring VS-mode pointer
 134		 * masking on the host side are part of the Ssnpm extension.
 135		 */
 136		host_ext = RISCV_ISA_EXT_SSNPM;
 137		break;
 138	default:
 139		host_ext = *guest_ext;
 140		break;
 141	}
 142
 143	if (!__riscv_isa_extension_available(NULL, host_ext))
 144		return -ENOENT;
 145
 146	return 0;
 147}
 148
 149static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext)
 150{
 151	switch (ext) {
 152	case KVM_RISCV_ISA_EXT_H:
 153		return false;
 154	case KVM_RISCV_ISA_EXT_SSCOFPMF:
 155		/* Sscofpmf depends on interrupt filtering defined in ssaia */
 156		return __riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSAIA);
 157	case KVM_RISCV_ISA_EXT_SVADU:
 158		/*
 159		 * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero.
 160		 * Guest OS can use Svadu only when host OS enable Svadu.
 161		 */
 162		return arch_has_hw_pte_young();
 163	case KVM_RISCV_ISA_EXT_V:
 164		return riscv_v_vstate_ctrl_user_allowed();
 165	default:
 166		break;
 167	}
 168
 169	return true;
 170}
 171
 172static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext)
 173{
 174	switch (ext) {
 175	/* Extensions which don't have any mechanism to disable */
 176	case KVM_RISCV_ISA_EXT_A:
 177	case KVM_RISCV_ISA_EXT_C:
 178	case KVM_RISCV_ISA_EXT_I:
 179	case KVM_RISCV_ISA_EXT_M:
 180	/* There is not architectural config bit to disable sscofpmf completely */
 181	case KVM_RISCV_ISA_EXT_SSCOFPMF:
 182	case KVM_RISCV_ISA_EXT_SSNPM:
 183	case KVM_RISCV_ISA_EXT_SSTC:
 184	case KVM_RISCV_ISA_EXT_SVINVAL:
 185	case KVM_RISCV_ISA_EXT_SVNAPOT:
 186	case KVM_RISCV_ISA_EXT_SVVPTC:
 187	case KVM_RISCV_ISA_EXT_ZAAMO:
 188	case KVM_RISCV_ISA_EXT_ZABHA:
 189	case KVM_RISCV_ISA_EXT_ZACAS:
 190	case KVM_RISCV_ISA_EXT_ZALRSC:
 191	case KVM_RISCV_ISA_EXT_ZAWRS:
 192	case KVM_RISCV_ISA_EXT_ZBA:
 193	case KVM_RISCV_ISA_EXT_ZBB:
 194	case KVM_RISCV_ISA_EXT_ZBC:
 195	case KVM_RISCV_ISA_EXT_ZBKB:
 196	case KVM_RISCV_ISA_EXT_ZBKC:
 197	case KVM_RISCV_ISA_EXT_ZBKX:
 198	case KVM_RISCV_ISA_EXT_ZBS:
 199	case KVM_RISCV_ISA_EXT_ZCA:
 200	case KVM_RISCV_ISA_EXT_ZCB:
 201	case KVM_RISCV_ISA_EXT_ZCD:
 202	case KVM_RISCV_ISA_EXT_ZCF:
 203	case KVM_RISCV_ISA_EXT_ZCMOP:
 204	case KVM_RISCV_ISA_EXT_ZFA:
 205	case KVM_RISCV_ISA_EXT_ZFBFMIN:
 206	case KVM_RISCV_ISA_EXT_ZFH:
 207	case KVM_RISCV_ISA_EXT_ZFHMIN:
 208	case KVM_RISCV_ISA_EXT_ZICBOP:
 209	case KVM_RISCV_ISA_EXT_ZICCRSE:
 210	case KVM_RISCV_ISA_EXT_ZICNTR:
 211	case KVM_RISCV_ISA_EXT_ZICOND:
 212	case KVM_RISCV_ISA_EXT_ZICSR:
 213	case KVM_RISCV_ISA_EXT_ZIFENCEI:
 214	case KVM_RISCV_ISA_EXT_ZIHINTNTL:
 215	case KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
 216	case KVM_RISCV_ISA_EXT_ZIHPM:
 217	case KVM_RISCV_ISA_EXT_ZIMOP:
 218	case KVM_RISCV_ISA_EXT_ZKND:
 219	case KVM_RISCV_ISA_EXT_ZKNE:
 220	case KVM_RISCV_ISA_EXT_ZKNH:
 221	case KVM_RISCV_ISA_EXT_ZKR:
 222	case KVM_RISCV_ISA_EXT_ZKSED:
 223	case KVM_RISCV_ISA_EXT_ZKSH:
 224	case KVM_RISCV_ISA_EXT_ZKT:
 225	case KVM_RISCV_ISA_EXT_ZTSO:
 226	case KVM_RISCV_ISA_EXT_ZVBB:
 227	case KVM_RISCV_ISA_EXT_ZVBC:
 228	case KVM_RISCV_ISA_EXT_ZVFBFMIN:
 229	case KVM_RISCV_ISA_EXT_ZVFBFWMA:
 230	case KVM_RISCV_ISA_EXT_ZVFH:
 231	case KVM_RISCV_ISA_EXT_ZVFHMIN:
 232	case KVM_RISCV_ISA_EXT_ZVKB:
 233	case KVM_RISCV_ISA_EXT_ZVKG:
 234	case KVM_RISCV_ISA_EXT_ZVKNED:
 235	case KVM_RISCV_ISA_EXT_ZVKNHA:
 236	case KVM_RISCV_ISA_EXT_ZVKNHB:
 237	case KVM_RISCV_ISA_EXT_ZVKSED:
 238	case KVM_RISCV_ISA_EXT_ZVKSH:
 239	case KVM_RISCV_ISA_EXT_ZVKT:
 240		return false;
 241	/* Extensions which can be disabled using Smstateen */
 242	case KVM_RISCV_ISA_EXT_SSAIA:
 243		return riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN);
 244	case KVM_RISCV_ISA_EXT_SVADE:
 245		/*
 246		 * The henvcfg.ADUE is read-only zero if menvcfg.ADUE is zero.
 247		 * Svade can't be disabled unless we support Svadu.
 248		 */
 249		return arch_has_hw_pte_young();
 250	default:
 251		break;
 252	}
 253
 254	return true;
 255}
 256
 257void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu)
 258{
 259	unsigned long guest_ext, i;
 260
 261	for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) {
 262		if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext))
 263			continue;
 264		if (kvm_riscv_vcpu_isa_enable_allowed(i))
 265			set_bit(guest_ext, vcpu->arch.isa);
 266	}
 267}
 268
 269static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu,
 270					 const struct kvm_one_reg *reg)
 271{
 272	unsigned long __user *uaddr =
 273			(unsigned long __user *)(unsigned long)reg->addr;
 274	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 275					    KVM_REG_SIZE_MASK |
 276					    KVM_REG_RISCV_CONFIG);
 277	unsigned long reg_val;
 278
 279	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 280		return -EINVAL;
 281
 282	switch (reg_num) {
 283	case KVM_REG_RISCV_CONFIG_REG(isa):
 284		reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK;
 285		break;
 286	case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
 287		if (!riscv_isa_extension_available(NULL, ZICBOM))
 288			return -ENOENT;
 289		reg_val = riscv_cbom_block_size;
 290		break;
 291	case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size):
 292		if (!riscv_isa_extension_available(NULL, ZICBOZ))
 293			return -ENOENT;
 294		reg_val = riscv_cboz_block_size;
 295		break;
 296	case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size):
 297		if (!riscv_isa_extension_available(NULL, ZICBOP))
 298			return -ENOENT;
 299		reg_val = riscv_cbop_block_size;
 300		break;
 301	case KVM_REG_RISCV_CONFIG_REG(mvendorid):
 302		reg_val = vcpu->arch.mvendorid;
 303		break;
 304	case KVM_REG_RISCV_CONFIG_REG(marchid):
 305		reg_val = vcpu->arch.marchid;
 306		break;
 307	case KVM_REG_RISCV_CONFIG_REG(mimpid):
 308		reg_val = vcpu->arch.mimpid;
 309		break;
 310	case KVM_REG_RISCV_CONFIG_REG(satp_mode):
 311		reg_val = satp_mode >> SATP_MODE_SHIFT;
 312		break;
 313	default:
 314		return -ENOENT;
 315	}
 316
 317	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 318		return -EFAULT;
 319
 320	return 0;
 321}
 322
 323static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu,
 324					 const struct kvm_one_reg *reg)
 325{
 326	unsigned long __user *uaddr =
 327			(unsigned long __user *)(unsigned long)reg->addr;
 328	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 329					    KVM_REG_SIZE_MASK |
 330					    KVM_REG_RISCV_CONFIG);
 331	unsigned long i, isa_ext, reg_val;
 332
 333	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 334		return -EINVAL;
 335
 336	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 337		return -EFAULT;
 338
 339	switch (reg_num) {
 340	case KVM_REG_RISCV_CONFIG_REG(isa):
 341		/*
 342		 * This ONE REG interface is only defined for
 343		 * single letter extensions.
 344		 */
 345		if (fls(reg_val) >= RISCV_ISA_EXT_BASE)
 346			return -EINVAL;
 347
 348		/*
 349		 * Return early (i.e. do nothing) if reg_val is the same
 350		 * value retrievable via kvm_riscv_vcpu_get_reg_config().
 351		 */
 352		if (reg_val == (vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK))
 353			break;
 354
 355		if (!vcpu->arch.ran_atleast_once) {
 356			/* Ignore the enable/disable request for certain extensions */
 357			for (i = 0; i < RISCV_ISA_EXT_BASE; i++) {
 358				isa_ext = kvm_riscv_vcpu_base2isa_ext(i);
 359				if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) {
 360					reg_val &= ~BIT(i);
 361					continue;
 362				}
 363				if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext))
 364					if (reg_val & BIT(i))
 365						reg_val &= ~BIT(i);
 366				if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext))
 367					if (!(reg_val & BIT(i)))
 368						reg_val |= BIT(i);
 369			}
 370			reg_val &= riscv_isa_extension_base(NULL);
 371			/* Do not modify anything beyond single letter extensions */
 372			reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) |
 373				  (reg_val & KVM_RISCV_BASE_ISA_MASK);
 374			vcpu->arch.isa[0] = reg_val;
 375			kvm_riscv_vcpu_fp_reset(vcpu);
 376		} else {
 377			return -EBUSY;
 378		}
 379		break;
 380	case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
 381		if (!riscv_isa_extension_available(NULL, ZICBOM))
 382			return -ENOENT;
 383		if (reg_val != riscv_cbom_block_size)
 384			return -EINVAL;
 385		break;
 386	case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size):
 387		if (!riscv_isa_extension_available(NULL, ZICBOZ))
 388			return -ENOENT;
 389		if (reg_val != riscv_cboz_block_size)
 390			return -EINVAL;
 391		break;
 392	case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size):
 393		if (!riscv_isa_extension_available(NULL, ZICBOP))
 394			return -ENOENT;
 395		if (reg_val != riscv_cbop_block_size)
 396			return -EINVAL;
 397		break;
 398	case KVM_REG_RISCV_CONFIG_REG(mvendorid):
 399		if (reg_val == vcpu->arch.mvendorid)
 400			break;
 401		if (!vcpu->arch.ran_atleast_once)
 402			vcpu->arch.mvendorid = reg_val;
 403		else
 404			return -EBUSY;
 405		break;
 406	case KVM_REG_RISCV_CONFIG_REG(marchid):
 407		if (reg_val == vcpu->arch.marchid)
 408			break;
 409		if (!vcpu->arch.ran_atleast_once)
 410			vcpu->arch.marchid = reg_val;
 411		else
 412			return -EBUSY;
 413		break;
 414	case KVM_REG_RISCV_CONFIG_REG(mimpid):
 415		if (reg_val == vcpu->arch.mimpid)
 416			break;
 417		if (!vcpu->arch.ran_atleast_once)
 418			vcpu->arch.mimpid = reg_val;
 419		else
 420			return -EBUSY;
 421		break;
 422	case KVM_REG_RISCV_CONFIG_REG(satp_mode):
 423		if (reg_val != (satp_mode >> SATP_MODE_SHIFT))
 424			return -EINVAL;
 425		break;
 426	default:
 427		return -ENOENT;
 428	}
 429
 430	return 0;
 431}
 432
 433static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu,
 434				       const struct kvm_one_reg *reg)
 435{
 436	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 437	unsigned long __user *uaddr =
 438			(unsigned long __user *)(unsigned long)reg->addr;
 439	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 440					    KVM_REG_SIZE_MASK |
 441					    KVM_REG_RISCV_CORE);
 442	unsigned long reg_val;
 443
 444	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 445		return -EINVAL;
 446	if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
 447		return -ENOENT;
 448
 449	if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
 450		reg_val = cntx->sepc;
 451	else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
 452		 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
 453		reg_val = ((unsigned long *)cntx)[reg_num];
 454	else if (reg_num == KVM_REG_RISCV_CORE_REG(mode))
 455		reg_val = (cntx->sstatus & SR_SPP) ?
 456				KVM_RISCV_MODE_S : KVM_RISCV_MODE_U;
 457	else
 458		return -ENOENT;
 459
 460	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 461		return -EFAULT;
 462
 463	return 0;
 464}
 465
 466static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu,
 467				       const struct kvm_one_reg *reg)
 468{
 469	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 470	unsigned long __user *uaddr =
 471			(unsigned long __user *)(unsigned long)reg->addr;
 472	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 473					    KVM_REG_SIZE_MASK |
 474					    KVM_REG_RISCV_CORE);
 475	unsigned long reg_val;
 476
 477	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 478		return -EINVAL;
 479	if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
 480		return -ENOENT;
 481
 482	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 483		return -EFAULT;
 484
 485	if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
 486		cntx->sepc = reg_val;
 487	else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
 488		 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
 489		((unsigned long *)cntx)[reg_num] = reg_val;
 490	else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) {
 491		if (reg_val == KVM_RISCV_MODE_S)
 492			cntx->sstatus |= SR_SPP;
 493		else
 494			cntx->sstatus &= ~SR_SPP;
 495	} else
 496		return -ENOENT;
 497
 498	return 0;
 499}
 500
 501static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu,
 502					  unsigned long reg_num,
 503					  unsigned long *out_val)
 504{
 505	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 506
 507	if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
 508		return -ENOENT;
 509
 510	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 511		kvm_riscv_vcpu_flush_interrupts(vcpu);
 512		*out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK;
 513		*out_val |= csr->hvip & ~IRQ_LOCAL_MASK;
 514	} else
 515		*out_val = ((unsigned long *)csr)[reg_num];
 516
 517	return 0;
 518}
 519
 520static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
 521					  unsigned long reg_num,
 522					  unsigned long reg_val)
 523{
 524	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 525
 526	if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
 527		return -ENOENT;
 528
 529	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 530		reg_val &= VSIP_VALID_MASK;
 531		reg_val <<= VSIP_TO_HVIP_SHIFT;
 532	}
 533
 534	((unsigned long *)csr)[reg_num] = reg_val;
 535
 536	if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
 537		WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0);
 538
 539	return 0;
 540}
 541
 542static inline int kvm_riscv_vcpu_smstateen_set_csr(struct kvm_vcpu *vcpu,
 543						   unsigned long reg_num,
 544						   unsigned long reg_val)
 545{
 546	struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr;
 547
 548	if (reg_num >= sizeof(struct kvm_riscv_smstateen_csr) /
 549		sizeof(unsigned long))
 550		return -EINVAL;
 551
 552	((unsigned long *)csr)[reg_num] = reg_val;
 553	return 0;
 554}
 555
 556static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu,
 557					    unsigned long reg_num,
 558					    unsigned long *out_val)
 559{
 560	struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr;
 561
 562	if (reg_num >= sizeof(struct kvm_riscv_smstateen_csr) /
 563		sizeof(unsigned long))
 564		return -EINVAL;
 565
 566	*out_val = ((unsigned long *)csr)[reg_num];
 567	return 0;
 568}
 569
 570static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu,
 571				      const struct kvm_one_reg *reg)
 572{
 573	int rc;
 574	unsigned long __user *uaddr =
 575			(unsigned long __user *)(unsigned long)reg->addr;
 576	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 577					    KVM_REG_SIZE_MASK |
 578					    KVM_REG_RISCV_CSR);
 579	unsigned long reg_val, reg_subtype;
 580
 581	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 582		return -EINVAL;
 583
 584	reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK;
 585	reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK;
 586	switch (reg_subtype) {
 587	case KVM_REG_RISCV_CSR_GENERAL:
 588		rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, &reg_val);
 589		break;
 590	case KVM_REG_RISCV_CSR_AIA:
 591		rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, &reg_val);
 592		break;
 593	case KVM_REG_RISCV_CSR_SMSTATEEN:
 594		rc = -EINVAL;
 595		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN))
 596			rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num,
 597							      &reg_val);
 598		break;
 599	default:
 600		rc = -ENOENT;
 601		break;
 602	}
 603	if (rc)
 604		return rc;
 605
 606	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 607		return -EFAULT;
 608
 609	return 0;
 610}
 611
 612static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu,
 613				      const struct kvm_one_reg *reg)
 614{
 615	int rc;
 616	unsigned long __user *uaddr =
 617			(unsigned long __user *)(unsigned long)reg->addr;
 618	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 619					    KVM_REG_SIZE_MASK |
 620					    KVM_REG_RISCV_CSR);
 621	unsigned long reg_val, reg_subtype;
 622
 623	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 624		return -EINVAL;
 625
 626	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 627		return -EFAULT;
 628
 629	reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK;
 630	reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK;
 631	switch (reg_subtype) {
 632	case KVM_REG_RISCV_CSR_GENERAL:
 633		rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val);
 634		break;
 635	case KVM_REG_RISCV_CSR_AIA:
 636		rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val);
 637		break;
 638	case KVM_REG_RISCV_CSR_SMSTATEEN:
 639		rc = -EINVAL;
 640		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN))
 641			rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num,
 642							      reg_val);
 643		break;
 644	default:
 645		rc = -ENOENT;
 646		break;
 647	}
 648	if (rc)
 649		return rc;
 650
 651	return 0;
 652}
 653
 654static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu,
 655					 unsigned long reg_num,
 656					 unsigned long *reg_val)
 657{
 658	unsigned long guest_ext;
 659	int ret;
 660
 661	ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext);
 662	if (ret)
 663		return ret;
 664
 665	*reg_val = 0;
 666	if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext))
 667		*reg_val = 1; /* Mark the given extension as available */
 668
 669	return 0;
 670}
 671
 672static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu,
 673					 unsigned long reg_num,
 674					 unsigned long reg_val)
 675{
 676	unsigned long guest_ext;
 677	int ret;
 678
 679	ret = kvm_riscv_vcpu_isa_check_host(reg_num, &guest_ext);
 680	if (ret)
 681		return ret;
 682
 683	if (reg_val == test_bit(guest_ext, vcpu->arch.isa))
 684		return 0;
 685
 686	if (!vcpu->arch.ran_atleast_once) {
 687		/*
 688		 * All multi-letter extension and a few single letter
 689		 * extension can be disabled
 690		 */
 691		if (reg_val == 1 &&
 692		    kvm_riscv_vcpu_isa_enable_allowed(reg_num))
 693			set_bit(guest_ext, vcpu->arch.isa);
 694		else if (!reg_val &&
 695			 kvm_riscv_vcpu_isa_disable_allowed(reg_num))
 696			clear_bit(guest_ext, vcpu->arch.isa);
 697		else
 698			return -EINVAL;
 699		kvm_riscv_vcpu_fp_reset(vcpu);
 700	} else {
 701		return -EBUSY;
 702	}
 703
 704	return 0;
 705}
 706
 707static int riscv_vcpu_get_isa_ext_multi(struct kvm_vcpu *vcpu,
 708					unsigned long reg_num,
 709					unsigned long *reg_val)
 710{
 711	unsigned long i, ext_id, ext_val;
 712
 713	if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST)
 714		return -ENOENT;
 715
 716	for (i = 0; i < BITS_PER_LONG; i++) {
 717		ext_id = i + reg_num * BITS_PER_LONG;
 718		if (ext_id >= KVM_RISCV_ISA_EXT_MAX)
 719			break;
 720
 721		ext_val = 0;
 722		riscv_vcpu_get_isa_ext_single(vcpu, ext_id, &ext_val);
 723		if (ext_val)
 724			*reg_val |= KVM_REG_RISCV_ISA_MULTI_MASK(ext_id);
 725	}
 726
 727	return 0;
 728}
 729
 730static int riscv_vcpu_set_isa_ext_multi(struct kvm_vcpu *vcpu,
 731					unsigned long reg_num,
 732					unsigned long reg_val, bool enable)
 733{
 734	unsigned long i, ext_id;
 735
 736	if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST)
 737		return -ENOENT;
 738
 739	for_each_set_bit(i, &reg_val, BITS_PER_LONG) {
 740		ext_id = i + reg_num * BITS_PER_LONG;
 741		if (ext_id >= KVM_RISCV_ISA_EXT_MAX)
 742			break;
 743
 744		riscv_vcpu_set_isa_ext_single(vcpu, ext_id, enable);
 745	}
 746
 747	return 0;
 748}
 749
 750static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu,
 751					  const struct kvm_one_reg *reg)
 752{
 753	int rc;
 754	unsigned long __user *uaddr =
 755			(unsigned long __user *)(unsigned long)reg->addr;
 756	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 757					    KVM_REG_SIZE_MASK |
 758					    KVM_REG_RISCV_ISA_EXT);
 759	unsigned long reg_val, reg_subtype;
 760
 761	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 762		return -EINVAL;
 763
 764	reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK;
 765	reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK;
 766
 767	reg_val = 0;
 768	switch (reg_subtype) {
 769	case KVM_REG_RISCV_ISA_SINGLE:
 770		rc = riscv_vcpu_get_isa_ext_single(vcpu, reg_num, &reg_val);
 771		break;
 772	case KVM_REG_RISCV_ISA_MULTI_EN:
 773	case KVM_REG_RISCV_ISA_MULTI_DIS:
 774		rc = riscv_vcpu_get_isa_ext_multi(vcpu, reg_num, &reg_val);
 775		if (!rc && reg_subtype == KVM_REG_RISCV_ISA_MULTI_DIS)
 776			reg_val = ~reg_val;
 777		break;
 778	default:
 779		rc = -ENOENT;
 780	}
 781	if (rc)
 782		return rc;
 783
 784	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 785		return -EFAULT;
 786
 787	return 0;
 788}
 789
 790static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu,
 791					  const struct kvm_one_reg *reg)
 792{
 793	unsigned long __user *uaddr =
 794			(unsigned long __user *)(unsigned long)reg->addr;
 795	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 796					    KVM_REG_SIZE_MASK |
 797					    KVM_REG_RISCV_ISA_EXT);
 798	unsigned long reg_val, reg_subtype;
 799
 800	if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 801		return -EINVAL;
 802
 803	reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK;
 804	reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK;
 805
 806	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 807		return -EFAULT;
 808
 809	switch (reg_subtype) {
 810	case KVM_REG_RISCV_ISA_SINGLE:
 811		return riscv_vcpu_set_isa_ext_single(vcpu, reg_num, reg_val);
 812	case KVM_REG_RISCV_ISA_MULTI_EN:
 813		return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, true);
 814	case KVM_REG_RISCV_ISA_MULTI_DIS:
 815		return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, false);
 816	default:
 817		return -ENOENT;
 818	}
 819
 820	return 0;
 821}
 822
 823static int copy_config_reg_indices(const struct kvm_vcpu *vcpu,
 824				u64 __user *uindices)
 825{
 826	int n = 0;
 827
 828	for (int i = 0; i < sizeof(struct kvm_riscv_config)/sizeof(unsigned long);
 829		 i++) {
 830		u64 size;
 831		u64 reg;
 832
 833		/*
 834		 * Avoid reporting config reg if the corresponding extension
 835		 * was not available.
 836		 */
 837		if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) &&
 838			!riscv_isa_extension_available(NULL, ZICBOM))
 839			continue;
 840		else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) &&
 841			!riscv_isa_extension_available(NULL, ZICBOZ))
 842			continue;
 843		else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) &&
 844			!riscv_isa_extension_available(NULL, ZICBOP))
 845			continue;
 846
 847		size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
 848		reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i;
 849
 850		if (uindices) {
 851			if (put_user(reg, uindices))
 852				return -EFAULT;
 853			uindices++;
 854		}
 855
 856		n++;
 857	}
 858
 859	return n;
 860}
 861
 862static unsigned long num_config_regs(const struct kvm_vcpu *vcpu)
 863{
 864	return copy_config_reg_indices(vcpu, NULL);
 865}
 866
 867static inline unsigned long num_core_regs(void)
 868{
 869	return sizeof(struct kvm_riscv_core) / sizeof(unsigned long);
 870}
 871
 872static int copy_core_reg_indices(u64 __user *uindices)
 873{
 874	int n = num_core_regs();
 875
 876	for (int i = 0; i < n; i++) {
 877		u64 size = IS_ENABLED(CONFIG_32BIT) ?
 878			   KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
 879		u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CORE | i;
 880
 881		if (uindices) {
 882			if (put_user(reg, uindices))
 883				return -EFAULT;
 884			uindices++;
 885		}
 886	}
 887
 888	return n;
 889}
 890
 891static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu)
 892{
 893	unsigned long n = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long);
 894
 895	if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA))
 896		n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long);
 897	if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN))
 898		n += sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long);
 899
 900	return n;
 901}
 902
 903static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu,
 904				u64 __user *uindices)
 905{
 906	int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long);
 907	int n2 = 0, n3 = 0;
 908
 909	/* copy general csr regs */
 910	for (int i = 0; i < n1; i++) {
 911		u64 size = IS_ENABLED(CONFIG_32BIT) ?
 912			   KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
 913		u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR |
 914				  KVM_REG_RISCV_CSR_GENERAL | i;
 915
 916		if (uindices) {
 917			if (put_user(reg, uindices))
 918				return -EFAULT;
 919			uindices++;
 920		}
 921	}
 922
 923	/* copy AIA csr regs */
 924	if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) {
 925		n2 = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long);
 926
 927		for (int i = 0; i < n2; i++) {
 928			u64 size = IS_ENABLED(CONFIG_32BIT) ?
 929				   KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
 930			u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR |
 931					  KVM_REG_RISCV_CSR_AIA | i;
 932
 933			if (uindices) {
 934				if (put_user(reg, uindices))
 935					return -EFAULT;
 936				uindices++;
 937			}
 938		}
 939	}
 940
 941	/* copy Smstateen csr regs */
 942	if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) {
 943		n3 = sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long);
 944
 945		for (int i = 0; i < n3; i++) {
 946			u64 size = IS_ENABLED(CONFIG_32BIT) ?
 947				   KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
 948			u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR |
 949					  KVM_REG_RISCV_CSR_SMSTATEEN | i;
 950
 951			if (uindices) {
 952				if (put_user(reg, uindices))
 953					return -EFAULT;
 954				uindices++;
 955			}
 956		}
 957	}
 958
 959	return n1 + n2 + n3;
 960}
 961
 962static inline unsigned long num_timer_regs(void)
 963{
 964	return sizeof(struct kvm_riscv_timer) / sizeof(u64);
 965}
 966
 967static int copy_timer_reg_indices(u64 __user *uindices)
 968{
 969	int n = num_timer_regs();
 970
 971	for (int i = 0; i < n; i++) {
 972		u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 |
 973			  KVM_REG_RISCV_TIMER | i;
 974
 975		if (uindices) {
 976			if (put_user(reg, uindices))
 977				return -EFAULT;
 978			uindices++;
 979		}
 980	}
 981
 982	return n;
 983}
 984
 985static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu)
 986{
 987	const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 988
 989	if (riscv_isa_extension_available(vcpu->arch.isa, f))
 990		return sizeof(cntx->fp.f) / sizeof(u32);
 991	else
 992		return 0;
 993}
 994
 995static int copy_fp_f_reg_indices(const struct kvm_vcpu *vcpu,
 996				u64 __user *uindices)
 997{
 998	int n = num_fp_f_regs(vcpu);
 999
1000	for (int i = 0; i < n; i++) {
1001		u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 |
1002			  KVM_REG_RISCV_FP_F | i;
1003
1004		if (uindices) {
1005			if (put_user(reg, uindices))
1006				return -EFAULT;
1007			uindices++;
1008		}
1009	}
1010
1011	return n;
1012}
1013
1014static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu)
1015{
1016	const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
1017
1018	if (riscv_isa_extension_available(vcpu->arch.isa, d))
1019		return sizeof(cntx->fp.d.f) / sizeof(u64) + 1;
1020	else
1021		return 0;
1022}
1023
1024static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu,
1025				u64 __user *uindices)
1026{
1027	int i;
1028	int n = num_fp_d_regs(vcpu);
1029	u64 reg;
1030
1031	/* copy fp.d.f indices */
1032	for (i = 0; i < n-1; i++) {
1033		reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 |
1034		      KVM_REG_RISCV_FP_D | i;
1035
1036		if (uindices) {
1037			if (put_user(reg, uindices))
1038				return -EFAULT;
1039			uindices++;
1040		}
1041	}
1042
1043	/* copy fp.d.fcsr indices */
1044	reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_D | i;
1045	if (uindices) {
1046		if (put_user(reg, uindices))
1047			return -EFAULT;
1048		uindices++;
1049	}
1050
1051	return n;
1052}
1053
1054static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu,
1055				u64 __user *uindices)
1056{
1057	unsigned long guest_ext;
1058	unsigned int n = 0;
1059
1060	for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
1061		u64 size = IS_ENABLED(CONFIG_32BIT) ?
1062			   KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
1063		u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i;
1064
1065		if (kvm_riscv_vcpu_isa_check_host(i, &guest_ext))
1066			continue;
1067
1068		if (uindices) {
1069			if (put_user(reg, uindices))
1070				return -EFAULT;
1071			uindices++;
1072		}
1073
1074		n++;
1075	}
1076
1077	return n;
1078}
1079
1080static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu)
1081{
1082	return copy_isa_ext_reg_indices(vcpu, NULL);
1083}
1084
1085static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu)
1086{
1087	return kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, NULL);
1088}
1089
1090static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu)
1091{
1092	return kvm_riscv_vcpu_reg_indices_sbi(vcpu, NULL);
1093}
1094
1095static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu)
1096{
1097	if (!riscv_isa_extension_available(vcpu->arch.isa, v))
1098		return 0;
1099
1100	/* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */
1101	return 37;
1102}
1103
1104static int copy_vector_reg_indices(const struct kvm_vcpu *vcpu,
1105				u64 __user *uindices)
1106{
1107	const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
1108	int n = num_vector_regs(vcpu);
1109	u64 reg, size;
1110	int i;
1111
1112	if (n == 0)
1113		return 0;
1114
1115	/* copy vstart, vl, vtype, vcsr and vlenb */
1116	size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
1117	for (i = 0; i < 5; i++) {
1118		reg = KVM_REG_RISCV | size | KVM_REG_RISCV_VECTOR | i;
1119
1120		if (uindices) {
1121			if (put_user(reg, uindices))
1122				return -EFAULT;
1123			uindices++;
1124		}
1125	}
1126
1127	/* vector_regs have a variable 'vlenb' size */
1128	size = __builtin_ctzl(cntx->vector.vlenb);
1129	size <<= KVM_REG_SIZE_SHIFT;
1130	for (i = 0; i < 32; i++) {
1131		reg = KVM_REG_RISCV | KVM_REG_RISCV_VECTOR | size |
1132			KVM_REG_RISCV_VECTOR_REG(i);
1133
1134		if (uindices) {
1135			if (put_user(reg, uindices))
1136				return -EFAULT;
1137			uindices++;
1138		}
1139	}
1140
1141	return n;
1142}
1143
1144/*
1145 * kvm_riscv_vcpu_num_regs - how many registers do we present via KVM_GET/SET_ONE_REG
1146 *
1147 * This is for all registers.
1148 */
1149unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu)
1150{
1151	unsigned long res = 0;
1152
1153	res += num_config_regs(vcpu);
1154	res += num_core_regs();
1155	res += num_csr_regs(vcpu);
1156	res += num_timer_regs();
1157	res += num_fp_f_regs(vcpu);
1158	res += num_fp_d_regs(vcpu);
1159	res += num_vector_regs(vcpu);
1160	res += num_isa_ext_regs(vcpu);
1161	res += num_sbi_ext_regs(vcpu);
1162	res += num_sbi_regs(vcpu);
1163
1164	return res;
1165}
1166
1167/*
1168 * kvm_riscv_vcpu_copy_reg_indices - get indices of all registers.
1169 */
1170int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu,
1171				    u64 __user *uindices)
1172{
1173	int ret;
1174
1175	ret = copy_config_reg_indices(vcpu, uindices);
1176	if (ret < 0)
1177		return ret;
1178	uindices += ret;
1179
1180	ret = copy_core_reg_indices(uindices);
1181	if (ret < 0)
1182		return ret;
1183	uindices += ret;
1184
1185	ret = copy_csr_reg_indices(vcpu, uindices);
1186	if (ret < 0)
1187		return ret;
1188	uindices += ret;
1189
1190	ret = copy_timer_reg_indices(uindices);
1191	if (ret < 0)
1192		return ret;
1193	uindices += ret;
1194
1195	ret = copy_fp_f_reg_indices(vcpu, uindices);
1196	if (ret < 0)
1197		return ret;
1198	uindices += ret;
1199
1200	ret = copy_fp_d_reg_indices(vcpu, uindices);
1201	if (ret < 0)
1202		return ret;
1203	uindices += ret;
1204
1205	ret = copy_vector_reg_indices(vcpu, uindices);
1206	if (ret < 0)
1207		return ret;
1208	uindices += ret;
1209
1210	ret = copy_isa_ext_reg_indices(vcpu, uindices);
1211	if (ret < 0)
1212		return ret;
1213	uindices += ret;
1214
1215	ret = kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, uindices);
1216	if (ret < 0)
1217		return ret;
1218	uindices += ret;
1219
1220	ret = kvm_riscv_vcpu_reg_indices_sbi(vcpu, uindices);
1221	if (ret < 0)
1222		return ret;
1223	uindices += ret;
1224
1225	return 0;
1226}
1227
1228int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu,
1229			   const struct kvm_one_reg *reg)
1230{
1231	switch (reg->id & KVM_REG_RISCV_TYPE_MASK) {
1232	case KVM_REG_RISCV_CONFIG:
1233		return kvm_riscv_vcpu_set_reg_config(vcpu, reg);
1234	case KVM_REG_RISCV_CORE:
1235		return kvm_riscv_vcpu_set_reg_core(vcpu, reg);
1236	case KVM_REG_RISCV_CSR:
1237		return kvm_riscv_vcpu_set_reg_csr(vcpu, reg);
1238	case KVM_REG_RISCV_TIMER:
1239		return kvm_riscv_vcpu_set_reg_timer(vcpu, reg);
1240	case KVM_REG_RISCV_FP_F:
1241		return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
1242						 KVM_REG_RISCV_FP_F);
1243	case KVM_REG_RISCV_FP_D:
1244		return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
1245						 KVM_REG_RISCV_FP_D);
1246	case KVM_REG_RISCV_VECTOR:
1247		return kvm_riscv_vcpu_set_reg_vector(vcpu, reg);
1248	case KVM_REG_RISCV_ISA_EXT:
1249		return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg);
1250	case KVM_REG_RISCV_SBI_EXT:
1251		return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg);
1252	case KVM_REG_RISCV_SBI_STATE:
1253		return kvm_riscv_vcpu_set_reg_sbi(vcpu, reg);
1254	default:
1255		break;
1256	}
1257
1258	return -ENOENT;
1259}
1260
1261int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu,
1262			   const struct kvm_one_reg *reg)
1263{
1264	switch (reg->id & KVM_REG_RISCV_TYPE_MASK) {
1265	case KVM_REG_RISCV_CONFIG:
1266		return kvm_riscv_vcpu_get_reg_config(vcpu, reg);
1267	case KVM_REG_RISCV_CORE:
1268		return kvm_riscv_vcpu_get_reg_core(vcpu, reg);
1269	case KVM_REG_RISCV_CSR:
1270		return kvm_riscv_vcpu_get_reg_csr(vcpu, reg);
1271	case KVM_REG_RISCV_TIMER:
1272		return kvm_riscv_vcpu_get_reg_timer(vcpu, reg);
1273	case KVM_REG_RISCV_FP_F:
1274		return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
1275						 KVM_REG_RISCV_FP_F);
1276	case KVM_REG_RISCV_FP_D:
1277		return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
1278						 KVM_REG_RISCV_FP_D);
1279	case KVM_REG_RISCV_VECTOR:
1280		return kvm_riscv_vcpu_get_reg_vector(vcpu, reg);
1281	case KVM_REG_RISCV_ISA_EXT:
1282		return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg);
1283	case KVM_REG_RISCV_SBI_EXT:
1284		return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg);
1285	case KVM_REG_RISCV_SBI_STATE:
1286		return kvm_riscv_vcpu_get_reg_sbi(vcpu, reg);
1287	default:
1288		break;
1289	}
1290
1291	return -ENOENT;
1292}