FILE
vcpu.c
File preview with syntax highlighting.
linux-6.19.14/arch/loongarch/kvm/vcpu.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6#include <linux/kvm_host.h> 7#include <asm/fpu.h> 8#include <asm/lbt.h> 9#include <asm/loongarch.h> 10#include <asm/setup.h> 11#include <asm/time.h> 12#include <asm/timex.h> 13 14#define CREATE_TRACE_POINTS 15#include "trace.h" 16 17const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { 18 KVM_GENERIC_VCPU_STATS(), 19 STATS_DESC_COUNTER(VCPU, int_exits), 20 STATS_DESC_COUNTER(VCPU, idle_exits), 21 STATS_DESC_COUNTER(VCPU, cpucfg_exits), 22 STATS_DESC_COUNTER(VCPU, signal_exits), 23 STATS_DESC_COUNTER(VCPU, hypercall_exits), 24 STATS_DESC_COUNTER(VCPU, ipi_read_exits), 25 STATS_DESC_COUNTER(VCPU, ipi_write_exits), 26 STATS_DESC_COUNTER(VCPU, eiointc_read_exits), 27 STATS_DESC_COUNTER(VCPU, eiointc_write_exits), 28 STATS_DESC_COUNTER(VCPU, pch_pic_read_exits), 29 STATS_DESC_COUNTER(VCPU, pch_pic_write_exits) 30}; 31 32const struct kvm_stats_header kvm_vcpu_stats_header = { 33 .name_size = KVM_STATS_NAME_SIZE, 34 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 35 .id_offset = sizeof(struct kvm_stats_header), 36 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 37 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 38 sizeof(kvm_vcpu_stats_desc), 39}; 40 41static inline void kvm_save_host_pmu(struct kvm_vcpu *vcpu) 42{ 43 struct kvm_context *context; 44 45 context = this_cpu_ptr(vcpu->kvm->arch.vmcs); 46 context->perf_cntr[0] = read_csr_perfcntr0(); 47 context->perf_cntr[1] = read_csr_perfcntr1(); 48 context->perf_cntr[2] = read_csr_perfcntr2(); 49 context->perf_cntr[3] = read_csr_perfcntr3(); 50 context->perf_ctrl[0] = write_csr_perfctrl0(0); 51 context->perf_ctrl[1] = write_csr_perfctrl1(0); 52 context->perf_ctrl[2] = write_csr_perfctrl2(0); 53 context->perf_ctrl[3] = write_csr_perfctrl3(0); 54} 55 56static inline void kvm_restore_host_pmu(struct kvm_vcpu *vcpu) 57{ 58 struct kvm_context *context; 59 60 context = this_cpu_ptr(vcpu->kvm->arch.vmcs); 61 write_csr_perfcntr0(context->perf_cntr[0]); 62 write_csr_perfcntr1(context->perf_cntr[1]); 63 write_csr_perfcntr2(context->perf_cntr[2]); 64 write_csr_perfcntr3(context->perf_cntr[3]); 65 write_csr_perfctrl0(context->perf_ctrl[0]); 66 write_csr_perfctrl1(context->perf_ctrl[1]); 67 write_csr_perfctrl2(context->perf_ctrl[2]); 68 write_csr_perfctrl3(context->perf_ctrl[3]); 69} 70 71 72static inline void kvm_save_guest_pmu(struct kvm_vcpu *vcpu) 73{ 74 struct loongarch_csrs *csr = vcpu->arch.csr; 75 76 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0); 77 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1); 78 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2); 79 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3); 80 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); 81 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); 82 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); 83 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 84} 85 86static inline void kvm_restore_guest_pmu(struct kvm_vcpu *vcpu) 87{ 88 struct loongarch_csrs *csr = vcpu->arch.csr; 89 90 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0); 91 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1); 92 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2); 93 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3); 94 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); 95 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); 96 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); 97 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 98} 99 100static int kvm_own_pmu(struct kvm_vcpu *vcpu) 101{ 102 unsigned long val; 103 104 if (!kvm_guest_has_pmu(&vcpu->arch)) 105 return -EINVAL; 106 107 kvm_save_host_pmu(vcpu); 108 109 /* Set PM0-PM(num) to guest */ 110 val = read_csr_gcfg() & ~CSR_GCFG_GPERF; 111 val |= (kvm_get_pmu_num(&vcpu->arch) + 1) << CSR_GCFG_GPERF_SHIFT; 112 write_csr_gcfg(val); 113 114 kvm_restore_guest_pmu(vcpu); 115 116 return 0; 117} 118 119static void kvm_lose_pmu(struct kvm_vcpu *vcpu) 120{ 121 unsigned long val; 122 struct loongarch_csrs *csr = vcpu->arch.csr; 123 124 if (!(vcpu->arch.aux_inuse & KVM_LARCH_PMU)) 125 return; 126 127 kvm_save_guest_pmu(vcpu); 128 129 /* Disable pmu access from guest */ 130 write_csr_gcfg(read_csr_gcfg() & ~CSR_GCFG_GPERF); 131 132 /* 133 * Clear KVM_LARCH_PMU if the guest is not using PMU CSRs when 134 * exiting the guest, so that the next time trap into the guest. 135 * We don't need to deal with PMU CSRs contexts. 136 * 137 * Otherwise set the request bit KVM_REQ_PMU to restore guest PMU 138 * before entering guest VM 139 */ 140 val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); 141 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); 142 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); 143 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 144 if (!(val & KVM_PMU_EVENT_ENABLED)) 145 vcpu->arch.aux_inuse &= ~KVM_LARCH_PMU; 146 else 147 kvm_make_request(KVM_REQ_PMU, vcpu); 148 149 kvm_restore_host_pmu(vcpu); 150} 151 152static void kvm_check_pmu(struct kvm_vcpu *vcpu) 153{ 154 if (kvm_check_request(KVM_REQ_PMU, vcpu)) { 155 kvm_own_pmu(vcpu); 156 vcpu->arch.aux_inuse |= KVM_LARCH_PMU; 157 } 158} 159 160static void kvm_update_stolen_time(struct kvm_vcpu *vcpu) 161{ 162 u32 version; 163 u64 steal; 164 gpa_t gpa; 165 struct kvm_memslots *slots; 166 struct kvm_steal_time __user *st; 167 struct gfn_to_hva_cache *ghc; 168 169 ghc = &vcpu->arch.st.cache; 170 gpa = vcpu->arch.st.guest_addr; 171 if (!(gpa & KVM_STEAL_PHYS_VALID)) 172 return; 173 174 gpa &= KVM_STEAL_PHYS_MASK; 175 slots = kvm_memslots(vcpu->kvm); 176 if (slots->generation != ghc->generation || gpa != ghc->gpa) { 177 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) { 178 ghc->gpa = INVALID_GPA; 179 return; 180 } 181 } 182 183 st = (struct kvm_steal_time __user *)ghc->hva; 184 unsafe_get_user(version, &st->version, out); 185 if (version & 1) 186 version += 1; /* first time write, random junk */ 187 188 version += 1; 189 unsafe_put_user(version, &st->version, out); 190 smp_wmb(); 191 192 unsafe_get_user(steal, &st->steal, out); 193 steal += current->sched_info.run_delay - vcpu->arch.st.last_steal; 194 vcpu->arch.st.last_steal = current->sched_info.run_delay; 195 unsafe_put_user(steal, &st->steal, out); 196 197 smp_wmb(); 198 version += 1; 199 unsafe_put_user(version, &st->version, out); 200out: 201 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 202} 203 204/* 205 * kvm_check_requests - check and handle pending vCPU requests 206 * 207 * Return: RESUME_GUEST if we should enter the guest 208 * RESUME_HOST if we should exit to userspace 209 */ 210static int kvm_check_requests(struct kvm_vcpu *vcpu) 211{ 212 if (!kvm_request_pending(vcpu)) 213 return RESUME_GUEST; 214 215 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 216 vcpu->arch.vpid = 0; /* Drop vpid for this vCPU */ 217 218 if (kvm_dirty_ring_check_request(vcpu)) 219 return RESUME_HOST; 220 221 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 222 kvm_update_stolen_time(vcpu); 223 224 return RESUME_GUEST; 225} 226 227static void kvm_late_check_requests(struct kvm_vcpu *vcpu) 228{ 229 lockdep_assert_irqs_disabled(); 230 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GPA, vcpu)) 231 if (vcpu->arch.flush_gpa != INVALID_GPA) { 232 kvm_flush_tlb_gpa(vcpu, vcpu->arch.flush_gpa); 233 vcpu->arch.flush_gpa = INVALID_GPA; 234 } 235} 236 237/* 238 * Check and handle pending signal and vCPU requests etc 239 * Run with irq enabled and preempt enabled 240 * 241 * Return: RESUME_GUEST if we should enter the guest 242 * RESUME_HOST if we should exit to userspace 243 * < 0 if we should exit to userspace, where the return value 244 * indicates an error 245 */ 246static int kvm_enter_guest_check(struct kvm_vcpu *vcpu) 247{ 248 int idx, ret; 249 250 /* 251 * Check conditions before entering the guest 252 */ 253 ret = kvm_xfer_to_guest_mode_handle_work(vcpu); 254 if (ret < 0) 255 return ret; 256 257 idx = srcu_read_lock(&vcpu->kvm->srcu); 258 ret = kvm_check_requests(vcpu); 259 srcu_read_unlock(&vcpu->kvm->srcu, idx); 260 261 return ret; 262} 263 264/* 265 * Called with irq enabled 266 * 267 * Return: RESUME_GUEST if we should enter the guest, and irq disabled 268 * Others if we should exit to userspace 269 */ 270static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu) 271{ 272 int ret; 273 274 do { 275 ret = kvm_enter_guest_check(vcpu); 276 if (ret != RESUME_GUEST) 277 break; 278 279 /* 280 * Handle vcpu timer, interrupts, check requests and 281 * check vmid before vcpu enter guest 282 */ 283 local_irq_disable(); 284 kvm_deliver_intr(vcpu); 285 kvm_deliver_exception(vcpu); 286 /* Make sure the vcpu mode has been written */ 287 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 288 kvm_check_vpid(vcpu); 289 kvm_check_pmu(vcpu); 290 291 /* 292 * Called after function kvm_check_vpid() 293 * Since it updates CSR.GSTAT used by kvm_flush_tlb_gpa(), 294 * and it may also clear KVM_REQ_TLB_FLUSH_GPA pending bit 295 */ 296 kvm_late_check_requests(vcpu); 297 vcpu->arch.host_eentry = csr_read64(LOONGARCH_CSR_EENTRY); 298 /* Clear KVM_LARCH_SWCSR_LATEST as CSR will change when enter guest */ 299 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 300 301 if (kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) { 302 if (vcpu->arch.aux_inuse & KVM_LARCH_PMU) { 303 kvm_lose_pmu(vcpu); 304 kvm_make_request(KVM_REQ_PMU, vcpu); 305 } 306 /* make sure the vcpu mode has been written */ 307 smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE); 308 local_irq_enable(); 309 ret = -EAGAIN; 310 } 311 } while (ret != RESUME_GUEST); 312 313 return ret; 314} 315 316/* 317 * Return 1 for resume guest and "<= 0" for resume host. 318 */ 319static int kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) 320{ 321 int ret = RESUME_GUEST; 322 unsigned long estat = vcpu->arch.host_estat; 323 u32 intr = estat & CSR_ESTAT_IS; 324 u32 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 325 326 vcpu->mode = OUTSIDE_GUEST_MODE; 327 328 /* Set a default exit reason */ 329 run->exit_reason = KVM_EXIT_UNKNOWN; 330 331 kvm_lose_pmu(vcpu); 332 333 guest_timing_exit_irqoff(); 334 guest_state_exit_irqoff(); 335 local_irq_enable(); 336 337 trace_kvm_exit(vcpu, ecode); 338 if (ecode) { 339 ret = kvm_handle_fault(vcpu, ecode); 340 } else { 341 WARN(!intr, "vm exiting with suspicious irq\n"); 342 ++vcpu->stat.int_exits; 343 } 344 345 if (ret == RESUME_GUEST) 346 ret = kvm_pre_enter_guest(vcpu); 347 348 if (ret != RESUME_GUEST) { 349 local_irq_disable(); 350 return ret; 351 } 352 353 guest_timing_enter_irqoff(); 354 guest_state_enter_irqoff(); 355 trace_kvm_reenter(vcpu); 356 357 return RESUME_GUEST; 358} 359 360int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 361{ 362 return !!(vcpu->arch.irq_pending) && 363 vcpu->arch.mp_state.mp_state == KVM_MP_STATE_RUNNABLE; 364} 365 366int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 367{ 368 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 369} 370 371bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 372{ 373 unsigned long val; 374 375 preempt_disable(); 376 val = gcsr_read(LOONGARCH_CSR_CRMD); 377 preempt_enable(); 378 379 return (val & CSR_PRMD_PPLV) == PLV_KERN; 380} 381 382#ifdef CONFIG_GUEST_PERF_EVENTS 383unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 384{ 385 return vcpu->arch.pc; 386} 387 388/* 389 * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, 390 * arrived in guest context. For LoongArch64, if PMU is not passthrough to VM, 391 * any event that arrives while a vCPU is loaded is considered to be "in guest". 392 */ 393bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) 394{ 395 return (vcpu && !(vcpu->arch.aux_inuse & KVM_LARCH_PMU)); 396} 397#endif 398 399bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) 400{ 401 return false; 402} 403 404vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 405{ 406 return VM_FAULT_SIGBUS; 407} 408 409int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 410 struct kvm_translation *tr) 411{ 412 return -EINVAL; 413} 414 415int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 416{ 417 int ret; 418 419 /* Protect from TOD sync and vcpu_load/put() */ 420 preempt_disable(); 421 ret = kvm_pending_timer(vcpu) || 422 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI); 423 preempt_enable(); 424 425 return ret; 426} 427 428int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) 429{ 430 int i; 431 432 kvm_debug("vCPU Register Dump:\n"); 433 kvm_debug("\tPC = 0x%08lx\n", vcpu->arch.pc); 434 kvm_debug("\tExceptions: %08lx\n", vcpu->arch.irq_pending); 435 436 for (i = 0; i < 32; i += 4) { 437 kvm_debug("\tGPR%02d: %08lx %08lx %08lx %08lx\n", i, 438 vcpu->arch.gprs[i], vcpu->arch.gprs[i + 1], 439 vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); 440 } 441 442 kvm_debug("\tCRMD: 0x%08lx, ESTAT: 0x%08lx\n", 443 kvm_read_hw_gcsr(LOONGARCH_CSR_CRMD), 444 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT)); 445 446 kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(LOONGARCH_CSR_ERA)); 447 448 return 0; 449} 450 451int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 452 struct kvm_mp_state *mp_state) 453{ 454 *mp_state = vcpu->arch.mp_state; 455 456 return 0; 457} 458 459int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 460 struct kvm_mp_state *mp_state) 461{ 462 int ret = 0; 463 464 switch (mp_state->mp_state) { 465 case KVM_MP_STATE_RUNNABLE: 466 vcpu->arch.mp_state = *mp_state; 467 break; 468 default: 469 ret = -EINVAL; 470 } 471 472 return ret; 473} 474 475int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 476 struct kvm_guest_debug *dbg) 477{ 478 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) 479 return -EINVAL; 480 481 if (dbg->control & KVM_GUESTDBG_ENABLE) 482 vcpu->guest_debug = dbg->control; 483 else 484 vcpu->guest_debug = 0; 485 486 return 0; 487} 488 489static inline int kvm_set_cpuid(struct kvm_vcpu *vcpu, u64 val) 490{ 491 int cpuid; 492 struct kvm_phyid_map *map; 493 struct loongarch_csrs *csr = vcpu->arch.csr; 494 495 if (val >= KVM_MAX_PHYID) 496 return -EINVAL; 497 498 map = vcpu->kvm->arch.phyid_map; 499 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID); 500 501 spin_lock(&vcpu->kvm->arch.phyid_map_lock); 502 if ((cpuid < KVM_MAX_PHYID) && map->phys_map[cpuid].enabled) { 503 /* Discard duplicated CPUID set operation */ 504 if (cpuid == val) { 505 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 506 return 0; 507 } 508 509 /* 510 * CPUID is already set before 511 * Forbid changing to a different CPUID at runtime 512 */ 513 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 514 return -EINVAL; 515 } 516 517 if (map->phys_map[val].enabled) { 518 /* Discard duplicated CPUID set operation */ 519 if (vcpu == map->phys_map[val].vcpu) { 520 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 521 return 0; 522 } 523 524 /* 525 * New CPUID is already set with other vcpu 526 * Forbid sharing the same CPUID between different vcpus 527 */ 528 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 529 return -EINVAL; 530 } 531 532 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, val); 533 map->phys_map[val].enabled = true; 534 map->phys_map[val].vcpu = vcpu; 535 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 536 537 return 0; 538} 539 540static inline void kvm_drop_cpuid(struct kvm_vcpu *vcpu) 541{ 542 int cpuid; 543 struct kvm_phyid_map *map; 544 struct loongarch_csrs *csr = vcpu->arch.csr; 545 546 map = vcpu->kvm->arch.phyid_map; 547 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID); 548 549 if (cpuid >= KVM_MAX_PHYID) 550 return; 551 552 spin_lock(&vcpu->kvm->arch.phyid_map_lock); 553 if (map->phys_map[cpuid].enabled) { 554 map->phys_map[cpuid].vcpu = NULL; 555 map->phys_map[cpuid].enabled = false; 556 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID); 557 } 558 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 559} 560 561struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) 562{ 563 struct kvm_phyid_map *map; 564 565 if (cpuid < 0) 566 return NULL; 567 568 if (cpuid >= KVM_MAX_PHYID) 569 return NULL; 570 571 map = kvm->arch.phyid_map; 572 if (!map->phys_map[cpuid].enabled) 573 return NULL; 574 575 return map->phys_map[cpuid].vcpu; 576} 577 578static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val) 579{ 580 unsigned long gintc; 581 struct loongarch_csrs *csr = vcpu->arch.csr; 582 583 if (get_gcsr_flag(id) & INVALID_GCSR) 584 return -EINVAL; 585 586 if (id == LOONGARCH_CSR_ESTAT) { 587 preempt_disable(); 588 vcpu_load(vcpu); 589 /* 590 * Sync pending interrupts into ESTAT so that interrupt 591 * remains during VM migration stage 592 */ 593 kvm_deliver_intr(vcpu); 594 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 595 vcpu_put(vcpu); 596 preempt_enable(); 597 598 /* ESTAT IP0~IP7 get from GINTC */ 599 gintc = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & 0xff; 600 *val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT) | (gintc << 2); 601 return 0; 602 } 603 604 /* 605 * Get software CSR state since software state is consistent 606 * with hardware for synchronous ioctl 607 */ 608 *val = kvm_read_sw_gcsr(csr, id); 609 610 return 0; 611} 612 613static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val) 614{ 615 int ret = 0, gintc; 616 struct loongarch_csrs *csr = vcpu->arch.csr; 617 618 if (get_gcsr_flag(id) & INVALID_GCSR) 619 return -EINVAL; 620 621 if (id == LOONGARCH_CSR_CPUID) 622 return kvm_set_cpuid(vcpu, val); 623 624 if (id == LOONGARCH_CSR_ESTAT) { 625 /* ESTAT IP0~IP7 inject through GINTC */ 626 gintc = (val >> 2) & 0xff; 627 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_GINTC, gintc); 628 629 gintc = val & ~(0xffUL << 2); 630 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_ESTAT, gintc); 631 632 return ret; 633 } 634 635 kvm_write_sw_gcsr(csr, id, val); 636 637 /* 638 * After modifying the PMU CSR register value of the vcpu. 639 * If the PMU CSRs are used, we need to set KVM_REQ_PMU. 640 */ 641 if (id >= LOONGARCH_CSR_PERFCTRL0 && id <= LOONGARCH_CSR_PERFCNTR3) { 642 unsigned long val; 643 644 val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0) | 645 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1) | 646 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2) | 647 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 648 649 if (val & KVM_PMU_EVENT_ENABLED) 650 kvm_make_request(KVM_REQ_PMU, vcpu); 651 } 652 653 return ret; 654} 655 656static int _kvm_get_cpucfg_mask(int id, u64 *v) 657{ 658 if (id < 0 || id >= KVM_MAX_CPUCFG_REGS) 659 return -EINVAL; 660 661 switch (id) { 662 case LOONGARCH_CPUCFG0: 663 *v = GENMASK(31, 0); 664 return 0; 665 case LOONGARCH_CPUCFG1: 666 *v = GENMASK(26, 0); 667 return 0; 668 case LOONGARCH_CPUCFG2: 669 /* CPUCFG2 features unconditionally supported by KVM */ 670 *v = CPUCFG2_FP | CPUCFG2_FPSP | CPUCFG2_FPDP | 671 CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV | 672 CPUCFG2_LSPW | CPUCFG2_LAM; 673 /* 674 * For the ISA extensions listed below, if one is supported 675 * by the host, then it is also supported by KVM. 676 */ 677 if (cpu_has_lsx) 678 *v |= CPUCFG2_LSX; 679 if (cpu_has_lasx) 680 *v |= CPUCFG2_LASX; 681 if (cpu_has_lbt_x86) 682 *v |= CPUCFG2_X86BT; 683 if (cpu_has_lbt_arm) 684 *v |= CPUCFG2_ARMBT; 685 if (cpu_has_lbt_mips) 686 *v |= CPUCFG2_MIPSBT; 687 if (cpu_has_ptw) 688 *v |= CPUCFG2_PTW; 689 690 return 0; 691 case LOONGARCH_CPUCFG3: 692 *v = GENMASK(16, 0); 693 return 0; 694 case LOONGARCH_CPUCFG4: 695 case LOONGARCH_CPUCFG5: 696 *v = GENMASK(31, 0); 697 return 0; 698 case LOONGARCH_CPUCFG6: 699 if (cpu_has_pmp) 700 *v = GENMASK(14, 0); 701 else 702 *v = 0; 703 return 0; 704 case LOONGARCH_CPUCFG16: 705 *v = GENMASK(16, 0); 706 return 0; 707 case LOONGARCH_CPUCFG17 ... LOONGARCH_CPUCFG20: 708 *v = GENMASK(30, 0); 709 return 0; 710 default: 711 /* 712 * CPUCFG bits should be zero if reserved by HW or not 713 * supported by KVM. 714 */ 715 *v = 0; 716 return 0; 717 } 718} 719 720static int kvm_check_cpucfg(int id, u64 val) 721{ 722 int ret; 723 u64 mask = 0; 724 725 ret = _kvm_get_cpucfg_mask(id, &mask); 726 if (ret) 727 return ret; 728 729 if (val & ~mask) 730 /* Unsupported features and/or the higher 32 bits should not be set */ 731 return -EINVAL; 732 733 switch (id) { 734 case LOONGARCH_CPUCFG1: 735 if ((val & CPUCFG1_MSGINT) && !cpu_has_msgint) 736 return -EINVAL; 737 return 0; 738 case LOONGARCH_CPUCFG2: 739 if (!(val & CPUCFG2_LLFTP)) 740 /* Guests must have a constant timer */ 741 return -EINVAL; 742 if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP))) 743 /* Single and double float point must both be set when FP is enabled */ 744 return -EINVAL; 745 if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP)) 746 /* LSX architecturally implies FP but val does not satisfy that */ 747 return -EINVAL; 748 if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX)) 749 /* LASX architecturally implies LSX and FP but val does not satisfy that */ 750 return -EINVAL; 751 return 0; 752 case LOONGARCH_CPUCFG6: 753 if (val & CPUCFG6_PMP) { 754 u32 host = read_cpucfg(LOONGARCH_CPUCFG6); 755 if ((val & CPUCFG6_PMBITS) != (host & CPUCFG6_PMBITS)) 756 return -EINVAL; 757 if ((val & CPUCFG6_PMNUM) > (host & CPUCFG6_PMNUM)) 758 return -EINVAL; 759 if ((val & CPUCFG6_UPM) && !(host & CPUCFG6_UPM)) 760 return -EINVAL; 761 } 762 return 0; 763 default: 764 /* 765 * Values for the other CPUCFG IDs are not being further validated 766 * besides the mask check above. 767 */ 768 return 0; 769 } 770} 771 772static int kvm_get_one_reg(struct kvm_vcpu *vcpu, 773 const struct kvm_one_reg *reg, u64 *v) 774{ 775 int id, ret = 0; 776 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 777 778 switch (type) { 779 case KVM_REG_LOONGARCH_CSR: 780 id = KVM_GET_IOC_CSR_IDX(reg->id); 781 ret = _kvm_getcsr(vcpu, id, v); 782 break; 783 case KVM_REG_LOONGARCH_CPUCFG: 784 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 785 if (id >= 0 && id < KVM_MAX_CPUCFG_REGS) 786 *v = vcpu->arch.cpucfg[id]; 787 else 788 ret = -EINVAL; 789 break; 790 case KVM_REG_LOONGARCH_LBT: 791 if (!kvm_guest_has_lbt(&vcpu->arch)) 792 return -ENXIO; 793 794 switch (reg->id) { 795 case KVM_REG_LOONGARCH_LBT_SCR0: 796 *v = vcpu->arch.lbt.scr0; 797 break; 798 case KVM_REG_LOONGARCH_LBT_SCR1: 799 *v = vcpu->arch.lbt.scr1; 800 break; 801 case KVM_REG_LOONGARCH_LBT_SCR2: 802 *v = vcpu->arch.lbt.scr2; 803 break; 804 case KVM_REG_LOONGARCH_LBT_SCR3: 805 *v = vcpu->arch.lbt.scr3; 806 break; 807 case KVM_REG_LOONGARCH_LBT_EFLAGS: 808 *v = vcpu->arch.lbt.eflags; 809 break; 810 case KVM_REG_LOONGARCH_LBT_FTOP: 811 *v = vcpu->arch.fpu.ftop; 812 break; 813 default: 814 ret = -EINVAL; 815 break; 816 } 817 break; 818 case KVM_REG_LOONGARCH_KVM: 819 switch (reg->id) { 820 case KVM_REG_LOONGARCH_COUNTER: 821 *v = get_cycles() + vcpu->kvm->arch.time_offset; 822 break; 823 case KVM_REG_LOONGARCH_DEBUG_INST: 824 *v = INSN_HVCL | KVM_HCALL_SWDBG; 825 break; 826 default: 827 ret = -EINVAL; 828 break; 829 } 830 break; 831 default: 832 ret = -EINVAL; 833 break; 834 } 835 836 return ret; 837} 838 839static int kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 840{ 841 int ret = 0; 842 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 843 844 switch (size) { 845 case KVM_REG_SIZE_U64: 846 ret = kvm_get_one_reg(vcpu, reg, &v); 847 if (ret) 848 return ret; 849 ret = put_user(v, (u64 __user *)(long)reg->addr); 850 break; 851 default: 852 ret = -EINVAL; 853 break; 854 } 855 856 return ret; 857} 858 859static int kvm_set_one_reg(struct kvm_vcpu *vcpu, 860 const struct kvm_one_reg *reg, u64 v) 861{ 862 int id, ret = 0; 863 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 864 865 switch (type) { 866 case KVM_REG_LOONGARCH_CSR: 867 id = KVM_GET_IOC_CSR_IDX(reg->id); 868 ret = _kvm_setcsr(vcpu, id, v); 869 break; 870 case KVM_REG_LOONGARCH_CPUCFG: 871 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 872 ret = kvm_check_cpucfg(id, v); 873 if (ret) 874 break; 875 vcpu->arch.cpucfg[id] = (u32)v; 876 if (id == LOONGARCH_CPUCFG6) 877 vcpu->arch.max_pmu_csrid = 878 LOONGARCH_CSR_PERFCTRL0 + 2 * kvm_get_pmu_num(&vcpu->arch) + 1; 879 break; 880 case KVM_REG_LOONGARCH_LBT: 881 if (!kvm_guest_has_lbt(&vcpu->arch)) 882 return -ENXIO; 883 884 switch (reg->id) { 885 case KVM_REG_LOONGARCH_LBT_SCR0: 886 vcpu->arch.lbt.scr0 = v; 887 break; 888 case KVM_REG_LOONGARCH_LBT_SCR1: 889 vcpu->arch.lbt.scr1 = v; 890 break; 891 case KVM_REG_LOONGARCH_LBT_SCR2: 892 vcpu->arch.lbt.scr2 = v; 893 break; 894 case KVM_REG_LOONGARCH_LBT_SCR3: 895 vcpu->arch.lbt.scr3 = v; 896 break; 897 case KVM_REG_LOONGARCH_LBT_EFLAGS: 898 vcpu->arch.lbt.eflags = v; 899 break; 900 case KVM_REG_LOONGARCH_LBT_FTOP: 901 vcpu->arch.fpu.ftop = v; 902 break; 903 default: 904 ret = -EINVAL; 905 break; 906 } 907 break; 908 case KVM_REG_LOONGARCH_KVM: 909 switch (reg->id) { 910 case KVM_REG_LOONGARCH_COUNTER: 911 /* 912 * gftoffset is relative with board, not vcpu 913 * only set for the first time for smp system 914 */ 915 if (vcpu->vcpu_id == 0) 916 vcpu->kvm->arch.time_offset = (signed long)(v - get_cycles()); 917 break; 918 case KVM_REG_LOONGARCH_VCPU_RESET: 919 vcpu->arch.st.guest_addr = 0; 920 memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); 921 memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); 922 923 /* 924 * When vCPU reset, clear the ESTAT and GINTC registers 925 * Other CSR registers are cleared with function _kvm_setcsr(). 926 */ 927 kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_GINTC, 0); 928 kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_ESTAT, 0); 929 break; 930 default: 931 ret = -EINVAL; 932 break; 933 } 934 break; 935 default: 936 ret = -EINVAL; 937 break; 938 } 939 940 return ret; 941} 942 943static int kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 944{ 945 int ret = 0; 946 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 947 948 switch (size) { 949 case KVM_REG_SIZE_U64: 950 ret = get_user(v, (u64 __user *)(long)reg->addr); 951 if (ret) 952 return ret; 953 break; 954 default: 955 return -EINVAL; 956 } 957 958 return kvm_set_one_reg(vcpu, reg, v); 959} 960 961int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 962{ 963 return -ENOIOCTLCMD; 964} 965 966int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 967{ 968 return -ENOIOCTLCMD; 969} 970 971int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 972{ 973 int i; 974 975 for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 976 regs->gpr[i] = vcpu->arch.gprs[i]; 977 978 regs->pc = vcpu->arch.pc; 979 980 return 0; 981} 982 983int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 984{ 985 int i; 986 987 for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 988 vcpu->arch.gprs[i] = regs->gpr[i]; 989 990 vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ 991 vcpu->arch.pc = regs->pc; 992 993 return 0; 994} 995 996static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 997 struct kvm_enable_cap *cap) 998{ 999 /* FPU is enabled by default, will support LSX/LASX later. */ 1000 return -EINVAL; 1001} 1002 1003static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu, 1004 struct kvm_device_attr *attr) 1005{ 1006 switch (attr->attr) { 1007 case LOONGARCH_CPUCFG2: 1008 case LOONGARCH_CPUCFG6: 1009 return 0; 1010 case CPUCFG_KVM_FEATURE: 1011 return 0; 1012 default: 1013 return -ENXIO; 1014 } 1015 1016 return -ENXIO; 1017} 1018 1019static int kvm_loongarch_pvtime_has_attr(struct kvm_vcpu *vcpu, 1020 struct kvm_device_attr *attr) 1021{ 1022 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) 1023 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 1024 return -ENXIO; 1025 1026 return 0; 1027} 1028 1029static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu, 1030 struct kvm_device_attr *attr) 1031{ 1032 int ret = -ENXIO; 1033 1034 switch (attr->group) { 1035 case KVM_LOONGARCH_VCPU_CPUCFG: 1036 ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr); 1037 break; 1038 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 1039 ret = kvm_loongarch_pvtime_has_attr(vcpu, attr); 1040 break; 1041 default: 1042 break; 1043 } 1044 1045 return ret; 1046} 1047 1048static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu, 1049 struct kvm_device_attr *attr) 1050{ 1051 int ret = 0; 1052 uint64_t val; 1053 uint64_t __user *uaddr = (uint64_t __user *)attr->addr; 1054 1055 switch (attr->attr) { 1056 case 0 ... (KVM_MAX_CPUCFG_REGS - 1): 1057 ret = _kvm_get_cpucfg_mask(attr->attr, &val); 1058 if (ret) 1059 return ret; 1060 break; 1061 case CPUCFG_KVM_FEATURE: 1062 val = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK; 1063 break; 1064 default: 1065 return -ENXIO; 1066 } 1067 1068 put_user(val, uaddr); 1069 1070 return ret; 1071} 1072 1073static int kvm_loongarch_pvtime_get_attr(struct kvm_vcpu *vcpu, 1074 struct kvm_device_attr *attr) 1075{ 1076 u64 gpa; 1077 u64 __user *user = (u64 __user *)attr->addr; 1078 1079 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) 1080 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 1081 return -ENXIO; 1082 1083 gpa = vcpu->arch.st.guest_addr; 1084 if (put_user(gpa, user)) 1085 return -EFAULT; 1086 1087 return 0; 1088} 1089 1090static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu, 1091 struct kvm_device_attr *attr) 1092{ 1093 int ret = -ENXIO; 1094 1095 switch (attr->group) { 1096 case KVM_LOONGARCH_VCPU_CPUCFG: 1097 ret = kvm_loongarch_cpucfg_get_attr(vcpu, attr); 1098 break; 1099 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 1100 ret = kvm_loongarch_pvtime_get_attr(vcpu, attr); 1101 break; 1102 default: 1103 break; 1104 } 1105 1106 return ret; 1107} 1108 1109static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, 1110 struct kvm_device_attr *attr) 1111{ 1112 u64 val, valid; 1113 u64 __user *user = (u64 __user *)attr->addr; 1114 struct kvm *kvm = vcpu->kvm; 1115 1116 switch (attr->attr) { 1117 case CPUCFG_KVM_FEATURE: 1118 if (get_user(val, user)) 1119 return -EFAULT; 1120 1121 valid = LOONGARCH_PV_FEAT_MASK; 1122 if (val & ~valid) 1123 return -EINVAL; 1124 1125 /* All vCPUs need set the same PV features */ 1126 if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) 1127 && ((kvm->arch.pv_features & valid) != val)) 1128 return -EINVAL; 1129 kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; 1130 return 0; 1131 default: 1132 return -ENXIO; 1133 } 1134} 1135 1136static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu, 1137 struct kvm_device_attr *attr) 1138{ 1139 int idx, ret = 0; 1140 u64 gpa, __user *user = (u64 __user *)attr->addr; 1141 struct kvm *kvm = vcpu->kvm; 1142 1143 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) 1144 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 1145 return -ENXIO; 1146 1147 if (get_user(gpa, user)) 1148 return -EFAULT; 1149 1150 if (gpa & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID)) 1151 return -EINVAL; 1152 1153 if (!(gpa & KVM_STEAL_PHYS_VALID)) { 1154 vcpu->arch.st.guest_addr = gpa; 1155 return 0; 1156 } 1157 1158 /* Check the address is in a valid memslot */ 1159 idx = srcu_read_lock(&kvm->srcu); 1160 if (kvm_is_error_hva(gfn_to_hva(kvm, gpa >> PAGE_SHIFT))) 1161 ret = -EINVAL; 1162 srcu_read_unlock(&kvm->srcu, idx); 1163 1164 if (!ret) { 1165 vcpu->arch.st.guest_addr = gpa; 1166 vcpu->arch.st.last_steal = current->sched_info.run_delay; 1167 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1168 } 1169 1170 return ret; 1171} 1172 1173static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu, 1174 struct kvm_device_attr *attr) 1175{ 1176 int ret = -ENXIO; 1177 1178 switch (attr->group) { 1179 case KVM_LOONGARCH_VCPU_CPUCFG: 1180 ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr); 1181 break; 1182 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 1183 ret = kvm_loongarch_pvtime_set_attr(vcpu, attr); 1184 break; 1185 default: 1186 break; 1187 } 1188 1189 return ret; 1190} 1191 1192long kvm_arch_vcpu_ioctl(struct file *filp, 1193 unsigned int ioctl, unsigned long arg) 1194{ 1195 long r; 1196 struct kvm_device_attr attr; 1197 void __user *argp = (void __user *)arg; 1198 struct kvm_vcpu *vcpu = filp->private_data; 1199 1200 /* 1201 * Only software CSR should be modified 1202 * 1203 * If any hardware CSR register is modified, vcpu_load/vcpu_put pair 1204 * should be used. Since CSR registers owns by this vcpu, if switch 1205 * to other vcpus, other vcpus need reload CSR registers. 1206 * 1207 * If software CSR is modified, bit KVM_LARCH_HWCSR_USABLE should 1208 * be clear in vcpu->arch.aux_inuse, and vcpu_load will check 1209 * aux_inuse flag and reload CSR registers form software. 1210 */ 1211 1212 switch (ioctl) { 1213 case KVM_SET_ONE_REG: 1214 case KVM_GET_ONE_REG: { 1215 struct kvm_one_reg reg; 1216 1217 r = -EFAULT; 1218 if (copy_from_user(®, argp, sizeof(reg))) 1219 break; 1220 if (ioctl == KVM_SET_ONE_REG) { 1221 r = kvm_set_reg(vcpu, ®); 1222 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 1223 } else 1224 r = kvm_get_reg(vcpu, ®); 1225 break; 1226 } 1227 case KVM_ENABLE_CAP: { 1228 struct kvm_enable_cap cap; 1229 1230 r = -EFAULT; 1231 if (copy_from_user(&cap, argp, sizeof(cap))) 1232 break; 1233 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1234 break; 1235 } 1236 case KVM_HAS_DEVICE_ATTR: { 1237 r = -EFAULT; 1238 if (copy_from_user(&attr, argp, sizeof(attr))) 1239 break; 1240 r = kvm_loongarch_vcpu_has_attr(vcpu, &attr); 1241 break; 1242 } 1243 case KVM_GET_DEVICE_ATTR: { 1244 r = -EFAULT; 1245 if (copy_from_user(&attr, argp, sizeof(attr))) 1246 break; 1247 r = kvm_loongarch_vcpu_get_attr(vcpu, &attr); 1248 break; 1249 } 1250 case KVM_SET_DEVICE_ATTR: { 1251 r = -EFAULT; 1252 if (copy_from_user(&attr, argp, sizeof(attr))) 1253 break; 1254 r = kvm_loongarch_vcpu_set_attr(vcpu, &attr); 1255 break; 1256 } 1257 default: 1258 r = -ENOIOCTLCMD; 1259 break; 1260 } 1261 1262 return r; 1263} 1264 1265int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1266{ 1267 int i = 0; 1268 1269 fpu->fcc = vcpu->arch.fpu.fcc; 1270 fpu->fcsr = vcpu->arch.fpu.fcsr; 1271 for (i = 0; i < NUM_FPU_REGS; i++) 1272 memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64); 1273 1274 return 0; 1275} 1276 1277int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1278{ 1279 int i = 0; 1280 1281 vcpu->arch.fpu.fcc = fpu->fcc; 1282 vcpu->arch.fpu.fcsr = fpu->fcsr; 1283 for (i = 0; i < NUM_FPU_REGS; i++) 1284 memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64); 1285 1286 return 0; 1287} 1288 1289#ifdef CONFIG_CPU_HAS_LBT 1290int kvm_own_lbt(struct kvm_vcpu *vcpu) 1291{ 1292 if (!kvm_guest_has_lbt(&vcpu->arch)) 1293 return -EINVAL; 1294 1295 preempt_disable(); 1296 if (!(vcpu->arch.aux_inuse & KVM_LARCH_LBT)) { 1297 set_csr_euen(CSR_EUEN_LBTEN); 1298 _restore_lbt(&vcpu->arch.lbt); 1299 vcpu->arch.aux_inuse |= KVM_LARCH_LBT; 1300 } 1301 preempt_enable(); 1302 1303 return 0; 1304} 1305 1306static void kvm_lose_lbt(struct kvm_vcpu *vcpu) 1307{ 1308 preempt_disable(); 1309 if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) { 1310 _save_lbt(&vcpu->arch.lbt); 1311 clear_csr_euen(CSR_EUEN_LBTEN); 1312 vcpu->arch.aux_inuse &= ~KVM_LARCH_LBT; 1313 } 1314 preempt_enable(); 1315} 1316 1317static void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) 1318{ 1319 /* 1320 * If TM is enabled, top register save/restore will 1321 * cause lbt exception, here enable lbt in advance 1322 */ 1323 if (fcsr & FPU_CSR_TM) 1324 kvm_own_lbt(vcpu); 1325} 1326 1327static void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) 1328{ 1329 if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1330 if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) 1331 return; 1332 kvm_check_fcsr(vcpu, read_fcsr(LOONGARCH_FCSR0)); 1333 } 1334} 1335#else 1336static inline void kvm_lose_lbt(struct kvm_vcpu *vcpu) { } 1337static inline void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) { } 1338static inline void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) { } 1339#endif 1340 1341/* Enable FPU and restore context */ 1342void kvm_own_fpu(struct kvm_vcpu *vcpu) 1343{ 1344 preempt_disable(); 1345 1346 /* 1347 * Enable FPU for guest 1348 * Set FR and FRE according to guest context 1349 */ 1350 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); 1351 set_csr_euen(CSR_EUEN_FPEN); 1352 1353 kvm_restore_fpu(&vcpu->arch.fpu); 1354 vcpu->arch.aux_inuse |= KVM_LARCH_FPU; 1355 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); 1356 1357 preempt_enable(); 1358} 1359 1360#ifdef CONFIG_CPU_HAS_LSX 1361/* Enable LSX and restore context */ 1362int kvm_own_lsx(struct kvm_vcpu *vcpu) 1363{ 1364 if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch)) 1365 return -EINVAL; 1366 1367 preempt_disable(); 1368 1369 /* Enable LSX for guest */ 1370 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); 1371 set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN); 1372 switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1373 case KVM_LARCH_FPU: 1374 /* 1375 * Guest FPU state already loaded, 1376 * only restore upper LSX state 1377 */ 1378 _restore_lsx_upper(&vcpu->arch.fpu); 1379 break; 1380 default: 1381 /* Neither FP or LSX already active, 1382 * restore full LSX state 1383 */ 1384 kvm_restore_lsx(&vcpu->arch.fpu); 1385 break; 1386 } 1387 1388 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX); 1389 vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU; 1390 preempt_enable(); 1391 1392 return 0; 1393} 1394#endif 1395 1396#ifdef CONFIG_CPU_HAS_LASX 1397/* Enable LASX and restore context */ 1398int kvm_own_lasx(struct kvm_vcpu *vcpu) 1399{ 1400 if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch) || !kvm_guest_has_lasx(&vcpu->arch)) 1401 return -EINVAL; 1402 1403 preempt_disable(); 1404 1405 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); 1406 set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 1407 switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) { 1408 case KVM_LARCH_LSX: 1409 case KVM_LARCH_LSX | KVM_LARCH_FPU: 1410 /* Guest LSX state already loaded, only restore upper LASX state */ 1411 _restore_lasx_upper(&vcpu->arch.fpu); 1412 break; 1413 case KVM_LARCH_FPU: 1414 /* Guest FP state already loaded, only restore upper LSX & LASX state */ 1415 _restore_lsx_upper(&vcpu->arch.fpu); 1416 _restore_lasx_upper(&vcpu->arch.fpu); 1417 break; 1418 default: 1419 /* Neither FP or LSX already active, restore full LASX state */ 1420 kvm_restore_lasx(&vcpu->arch.fpu); 1421 break; 1422 } 1423 1424 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); 1425 vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU; 1426 preempt_enable(); 1427 1428 return 0; 1429} 1430#endif 1431 1432/* Save context and disable FPU */ 1433void kvm_lose_fpu(struct kvm_vcpu *vcpu) 1434{ 1435 preempt_disable(); 1436 1437 kvm_check_fcsr_alive(vcpu); 1438 if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) { 1439 kvm_save_lasx(&vcpu->arch.fpu); 1440 vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX); 1441 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX); 1442 1443 /* Disable LASX & LSX & FPU */ 1444 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 1445 } else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) { 1446 kvm_save_lsx(&vcpu->arch.fpu); 1447 vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU); 1448 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX); 1449 1450 /* Disable LSX & FPU */ 1451 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN); 1452 } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1453 kvm_save_fpu(&vcpu->arch.fpu); 1454 vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU; 1455 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); 1456 1457 /* Disable FPU */ 1458 clear_csr_euen(CSR_EUEN_FPEN); 1459 } 1460 kvm_lose_lbt(vcpu); 1461 1462 preempt_enable(); 1463} 1464 1465int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) 1466{ 1467 int intr = (int)irq->irq; 1468 1469 if (intr > 0) 1470 kvm_queue_irq(vcpu, intr); 1471 else if (intr < 0) 1472 kvm_dequeue_irq(vcpu, -intr); 1473 else { 1474 kvm_err("%s: invalid interrupt ioctl %d\n", __func__, irq->irq); 1475 return -EINVAL; 1476 } 1477 1478 kvm_vcpu_kick(vcpu); 1479 1480 return 0; 1481} 1482 1483long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, 1484 unsigned long arg) 1485{ 1486 void __user *argp = (void __user *)arg; 1487 struct kvm_vcpu *vcpu = filp->private_data; 1488 1489 if (ioctl == KVM_INTERRUPT) { 1490 struct kvm_interrupt irq; 1491 1492 if (copy_from_user(&irq, argp, sizeof(irq))) 1493 return -EFAULT; 1494 1495 kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, irq.irq); 1496 1497 return kvm_vcpu_ioctl_interrupt(vcpu, &irq); 1498 } 1499 1500 return -ENOIOCTLCMD; 1501} 1502 1503int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 1504{ 1505 return 0; 1506} 1507 1508int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 1509{ 1510 unsigned long timer_hz; 1511 struct loongarch_csrs *csr; 1512 1513 vcpu->arch.vpid = 0; 1514 vcpu->arch.flush_gpa = INVALID_GPA; 1515 1516 hrtimer_setup(&vcpu->arch.swtimer, kvm_swtimer_wakeup, CLOCK_MONOTONIC, 1517 HRTIMER_MODE_ABS_PINNED_HARD); 1518 1519 /* Get GPA (=HVA) of PGD for kvm hypervisor */ 1520 vcpu->arch.kvm_pgd = __pa(vcpu->kvm->arch.pgd); 1521 1522 /* 1523 * Get PGD for primary mmu, virtual address is used since there is 1524 * memory access after loading from CSR_PGD in tlb exception fast path. 1525 */ 1526 vcpu->arch.host_pgd = (unsigned long)vcpu->kvm->mm->pgd; 1527 1528 vcpu->arch.handle_exit = kvm_handle_exit; 1529 vcpu->arch.guest_eentry = (unsigned long)kvm_loongarch_ops->exc_entry; 1530 vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); 1531 if (!vcpu->arch.csr) 1532 return -ENOMEM; 1533 1534 /* 1535 * All kvm exceptions share one exception entry, and host <-> guest 1536 * switch also switch ECFG.VS field, keep host ECFG.VS info here. 1537 */ 1538 vcpu->arch.host_ecfg = (read_csr_ecfg() & CSR_ECFG_VS); 1539 1540 /* Init */ 1541 vcpu->arch.last_sched_cpu = -1; 1542 1543 /* Init ipi_state lock */ 1544 spin_lock_init(&vcpu->arch.ipi_state.lock); 1545 1546 /* 1547 * Initialize guest register state to valid architectural reset state. 1548 */ 1549 timer_hz = calc_const_freq(); 1550 kvm_init_timer(vcpu, timer_hz); 1551 1552 /* Set Initialize mode for guest */ 1553 csr = vcpu->arch.csr; 1554 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CRMD, CSR_CRMD_DA); 1555 1556 /* Set cpuid */ 1557 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_TMID, vcpu->vcpu_id); 1558 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID); 1559 1560 /* Start with no pending virtual guest interrupts */ 1561 csr->csrs[LOONGARCH_CSR_GINTC] = 0; 1562 1563 return 0; 1564} 1565 1566void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 1567{ 1568} 1569 1570void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 1571{ 1572 int cpu; 1573 struct kvm_context *context; 1574 1575 hrtimer_cancel(&vcpu->arch.swtimer); 1576 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1577 kvm_drop_cpuid(vcpu); 1578 kfree(vcpu->arch.csr); 1579 1580 /* 1581 * If the vCPU is freed and reused as another vCPU, we don't want the 1582 * matching pointer wrongly hanging around in last_vcpu. 1583 */ 1584 for_each_possible_cpu(cpu) { 1585 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 1586 if (context->last_vcpu == vcpu) 1587 context->last_vcpu = NULL; 1588 } 1589} 1590 1591static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1592{ 1593 bool migrated; 1594 struct kvm_context *context; 1595 struct loongarch_csrs *csr = vcpu->arch.csr; 1596 1597 /* 1598 * Have we migrated to a different CPU? 1599 * If so, any old guest TLB state may be stale. 1600 */ 1601 migrated = (vcpu->arch.last_sched_cpu != cpu); 1602 1603 /* 1604 * Was this the last vCPU to run on this CPU? 1605 * If not, any old guest state from this vCPU will have been clobbered. 1606 */ 1607 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 1608 if (migrated || (context->last_vcpu != vcpu)) 1609 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 1610 context->last_vcpu = vcpu; 1611 1612 /* Restore timer state regardless */ 1613 kvm_restore_timer(vcpu); 1614 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1615 1616 /* Don't bother restoring registers multiple times unless necessary */ 1617 if (vcpu->arch.aux_inuse & KVM_LARCH_HWCSR_USABLE) 1618 return 0; 1619 1620 write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset); 1621 1622 /* Restore guest CSR registers */ 1623 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 1624 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 1625 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 1626 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_MISC); 1627 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 1628 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ERA); 1629 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADV); 1630 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADI); 1631 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 1632 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 1633 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 1634 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 1635 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 1636 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ASID); 1637 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 1638 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 1639 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 1640 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 1641 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 1642 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 1643 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 1644 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS0); 1645 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS1); 1646 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS2); 1647 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS3); 1648 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS4); 1649 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS5); 1650 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS6); 1651 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS7); 1652 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TMID); 1653 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 1654 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 1655 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 1656 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 1657 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 1658 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 1659 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 1660 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 1661 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 1662 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 1663 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 1664 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 1665 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 1666 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 1667 if (cpu_has_msgint) { 1668 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR0); 1669 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR1); 1670 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR2); 1671 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR3); 1672 } 1673 1674 /* Restore Root.GINTC from unused Guest.GINTC register */ 1675 write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]); 1676 1677 /* 1678 * We should clear linked load bit to break interrupted atomics. This 1679 * prevents a SC on the next vCPU from succeeding by matching a LL on 1680 * the previous vCPU. 1681 */ 1682 if (vcpu->kvm->created_vcpus > 1) 1683 set_gcsr_llbctl(CSR_LLBCTL_WCLLB); 1684 1685 vcpu->arch.aux_inuse |= KVM_LARCH_HWCSR_USABLE; 1686 1687 return 0; 1688} 1689 1690void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1691{ 1692 unsigned long flags; 1693 1694 local_irq_save(flags); 1695 /* Restore guest state to registers */ 1696 _kvm_vcpu_load(vcpu, cpu); 1697 local_irq_restore(flags); 1698} 1699 1700static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 1701{ 1702 struct loongarch_csrs *csr = vcpu->arch.csr; 1703 1704 kvm_lose_fpu(vcpu); 1705 1706 /* 1707 * Update CSR state from hardware if software CSR state is stale, 1708 * most CSR registers are kept unchanged during process context 1709 * switch except CSR registers like remaining timer tick value and 1710 * injected interrupt state. 1711 */ 1712 if (vcpu->arch.aux_inuse & KVM_LARCH_SWCSR_LATEST) 1713 goto out; 1714 1715 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 1716 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 1717 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 1718 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_MISC); 1719 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 1720 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ERA); 1721 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADV); 1722 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADI); 1723 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 1724 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 1725 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 1726 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 1727 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 1728 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ASID); 1729 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 1730 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 1731 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 1732 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 1733 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 1734 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 1735 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 1736 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG1); 1737 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG2); 1738 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG3); 1739 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS0); 1740 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS1); 1741 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS2); 1742 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS3); 1743 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS4); 1744 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS5); 1745 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS6); 1746 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS7); 1747 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TMID); 1748 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 1749 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 1750 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 1751 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 1752 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 1753 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 1754 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 1755 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 1756 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 1757 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 1758 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 1759 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 1760 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 1761 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 1762 if (cpu_has_msgint) { 1763 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR0); 1764 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR1); 1765 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR2); 1766 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR3); 1767 } 1768 1769 vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST; 1770 1771out: 1772 kvm_save_timer(vcpu); 1773 /* Save Root.GINTC into unused Guest.GINTC register */ 1774 csr->csrs[LOONGARCH_CSR_GINTC] = read_csr_gintc(); 1775 1776 return 0; 1777} 1778 1779void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1780{ 1781 int cpu; 1782 unsigned long flags; 1783 1784 local_irq_save(flags); 1785 cpu = smp_processor_id(); 1786 vcpu->arch.last_sched_cpu = cpu; 1787 1788 /* Save guest state in registers */ 1789 _kvm_vcpu_put(vcpu, cpu); 1790 local_irq_restore(flags); 1791} 1792 1793int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1794{ 1795 int r = -EINTR; 1796 struct kvm_run *run = vcpu->run; 1797 1798 if (vcpu->mmio_needed) { 1799 if (!vcpu->mmio_is_write) 1800 kvm_complete_mmio_read(vcpu, run); 1801 vcpu->mmio_needed = 0; 1802 } 1803 1804 switch (run->exit_reason) { 1805 case KVM_EXIT_HYPERCALL: 1806 kvm_complete_user_service(vcpu, run); 1807 break; 1808 case KVM_EXIT_LOONGARCH_IOCSR: 1809 if (!run->iocsr_io.is_write) 1810 kvm_complete_iocsr_read(vcpu, run); 1811 break; 1812 } 1813 1814 if (!vcpu->wants_to_run) 1815 return r; 1816 1817 /* Clear exit_reason */ 1818 run->exit_reason = KVM_EXIT_UNKNOWN; 1819 lose_fpu(1); 1820 vcpu_load(vcpu); 1821 kvm_sigset_activate(vcpu); 1822 r = kvm_pre_enter_guest(vcpu); 1823 if (r != RESUME_GUEST) 1824 goto out; 1825 1826 guest_timing_enter_irqoff(); 1827 guest_state_enter_irqoff(); 1828 trace_kvm_enter(vcpu); 1829 r = kvm_loongarch_ops->enter_guest(run, vcpu); 1830 1831 trace_kvm_out(vcpu); 1832 /* 1833 * Guest exit is already recorded at kvm_handle_exit() 1834 * return value must not be RESUME_GUEST 1835 */ 1836 local_irq_enable(); 1837out: 1838 kvm_sigset_deactivate(vcpu); 1839 vcpu_put(vcpu); 1840 1841 return r; 1842}
~/
Read-only file preview.