FILE
kvm_host.h
File preview with syntax highlighting.
linux-6.19.14/include/linux/kvm_host.h
1/* SPDX-License-Identifier: GPL-2.0-only */ 2#ifndef __KVM_HOST_H 3#define __KVM_HOST_H 4 5#include <linux/entry-virt.h> 6#include <linux/types.h> 7#include <linux/hardirq.h> 8#include <linux/list.h> 9#include <linux/mutex.h> 10#include <linux/spinlock.h> 11#include <linux/signal.h> 12#include <linux/sched.h> 13#include <linux/sched/stat.h> 14#include <linux/bug.h> 15#include <linux/minmax.h> 16#include <linux/mm.h> 17#include <linux/mmu_notifier.h> 18#include <linux/preempt.h> 19#include <linux/msi.h> 20#include <linux/slab.h> 21#include <linux/vmalloc.h> 22#include <linux/rcupdate.h> 23#include <linux/ratelimit.h> 24#include <linux/err.h> 25#include <linux/irqflags.h> 26#include <linux/context_tracking.h> 27#include <linux/irqbypass.h> 28#include <linux/rcuwait.h> 29#include <linux/refcount.h> 30#include <linux/nospec.h> 31#include <linux/notifier.h> 32#include <linux/ftrace.h> 33#include <linux/hashtable.h> 34#include <linux/instrumentation.h> 35#include <linux/interval_tree.h> 36#include <linux/rbtree.h> 37#include <linux/xarray.h> 38#include <asm/signal.h> 39 40#include <linux/kvm.h> 41#include <linux/kvm_para.h> 42 43#include <linux/kvm_types.h> 44 45#include <asm/kvm_host.h> 46#include <linux/kvm_dirty_ring.h> 47 48#ifndef KVM_MAX_VCPU_IDS 49#define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS 50#endif 51 52/* 53 * The bit 16 ~ bit 31 of kvm_userspace_memory_region::flags are internally 54 * used in kvm, other bits are visible for userspace which are defined in 55 * include/uapi/linux/kvm.h. 56 */ 57#define KVM_MEMSLOT_INVALID (1UL << 16) 58#define KVM_MEMSLOT_GMEM_ONLY (1UL << 17) 59 60/* 61 * Bit 63 of the memslot generation number is an "update in-progress flag", 62 * e.g. is temporarily set for the duration of kvm_swap_active_memslots(). 63 * This flag effectively creates a unique generation number that is used to 64 * mark cached memslot data, e.g. MMIO accesses, as potentially being stale, 65 * i.e. may (or may not) have come from the previous memslots generation. 66 * 67 * This is necessary because the actual memslots update is not atomic with 68 * respect to the generation number update. Updating the generation number 69 * first would allow a vCPU to cache a spte from the old memslots using the 70 * new generation number, and updating the generation number after switching 71 * to the new memslots would allow cache hits using the old generation number 72 * to reference the defunct memslots. 73 * 74 * This mechanism is used to prevent getting hits in KVM's caches while a 75 * memslot update is in-progress, and to prevent cache hits *after* updating 76 * the actual generation number against accesses that were inserted into the 77 * cache *before* the memslots were updated. 78 */ 79#define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS BIT_ULL(63) 80 81/* Two fragments for cross MMIO pages. */ 82#define KVM_MAX_MMIO_FRAGMENTS 2 83 84#ifndef KVM_MAX_NR_ADDRESS_SPACES 85#define KVM_MAX_NR_ADDRESS_SPACES 1 86#endif 87 88/* 89 * For the normal pfn, the highest 12 bits should be zero, 90 * so we can mask bit 62 ~ bit 52 to indicate the error pfn, 91 * mask bit 63 to indicate the noslot pfn. 92 */ 93#define KVM_PFN_ERR_MASK (0x7ffULL << 52) 94#define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52) 95#define KVM_PFN_NOSLOT (0x1ULL << 63) 96 97#define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK) 98#define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1) 99#define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2) 100#define KVM_PFN_ERR_SIGPENDING (KVM_PFN_ERR_MASK + 3) 101#define KVM_PFN_ERR_NEEDS_IO (KVM_PFN_ERR_MASK + 4) 102 103/* 104 * error pfns indicate that the gfn is in slot but faild to 105 * translate it to pfn on host. 106 */ 107static inline bool is_error_pfn(kvm_pfn_t pfn) 108{ 109 return !!(pfn & KVM_PFN_ERR_MASK); 110} 111 112/* 113 * KVM_PFN_ERR_SIGPENDING indicates that fetching the PFN was interrupted 114 * by a pending signal. Note, the signal may or may not be fatal. 115 */ 116static inline bool is_sigpending_pfn(kvm_pfn_t pfn) 117{ 118 return pfn == KVM_PFN_ERR_SIGPENDING; 119} 120 121/* 122 * error_noslot pfns indicate that the gfn can not be 123 * translated to pfn - it is not in slot or failed to 124 * translate it to pfn. 125 */ 126static inline bool is_error_noslot_pfn(kvm_pfn_t pfn) 127{ 128 return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK); 129} 130 131/* noslot pfn indicates that the gfn is not in slot. */ 132static inline bool is_noslot_pfn(kvm_pfn_t pfn) 133{ 134 return pfn == KVM_PFN_NOSLOT; 135} 136 137/* 138 * architectures with KVM_HVA_ERR_BAD other than PAGE_OFFSET (e.g. s390) 139 * provide own defines and kvm_is_error_hva 140 */ 141#ifndef KVM_HVA_ERR_BAD 142 143#define KVM_HVA_ERR_BAD (PAGE_OFFSET) 144#define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE) 145 146static inline bool kvm_is_error_hva(unsigned long addr) 147{ 148 return addr >= PAGE_OFFSET; 149} 150 151#endif 152 153static inline bool kvm_is_error_gpa(gpa_t gpa) 154{ 155 return gpa == INVALID_GPA; 156} 157 158#define KVM_REQUEST_MASK GENMASK(7,0) 159#define KVM_REQUEST_NO_WAKEUP BIT(8) 160#define KVM_REQUEST_WAIT BIT(9) 161#define KVM_REQUEST_NO_ACTION BIT(10) 162/* 163 * Architecture-independent vcpu->requests bit members 164 * Bits 3-7 are reserved for more arch-independent bits. 165 */ 166#define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 167#define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 168#define KVM_REQ_UNBLOCK 2 169#define KVM_REQ_DIRTY_RING_SOFT_FULL 3 170#define KVM_REQUEST_ARCH_BASE 8 171 172/* 173 * KVM_REQ_OUTSIDE_GUEST_MODE exists is purely as way to force the vCPU to 174 * OUTSIDE_GUEST_MODE. KVM_REQ_OUTSIDE_GUEST_MODE differs from a vCPU "kick" 175 * in that it ensures the vCPU has reached OUTSIDE_GUEST_MODE before continuing 176 * on. A kick only guarantees that the vCPU is on its way out, e.g. a previous 177 * kick may have set vcpu->mode to EXITING_GUEST_MODE, and so there's no 178 * guarantee the vCPU received an IPI and has actually exited guest mode. 179 */ 180#define KVM_REQ_OUTSIDE_GUEST_MODE (KVM_REQUEST_NO_ACTION | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) 181 182#define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \ 183 BUILD_BUG_ON((unsigned)(nr) >= (sizeof_field(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \ 184 (unsigned)(((nr) + KVM_REQUEST_ARCH_BASE) | (flags)); \ 185}) 186#define KVM_ARCH_REQ(nr) KVM_ARCH_REQ_FLAGS(nr, 0) 187 188bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req, 189 unsigned long *vcpu_bitmap); 190bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req); 191 192#define KVM_USERSPACE_IRQ_SOURCE_ID 0 193#define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1 194#define KVM_PIT_IRQ_SOURCE_ID 2 195 196extern struct mutex kvm_lock; 197extern struct list_head vm_list; 198 199struct kvm_io_range { 200 gpa_t addr; 201 int len; 202 struct kvm_io_device *dev; 203}; 204 205#define NR_IOBUS_DEVS 1000 206 207struct kvm_io_bus { 208 int dev_count; 209 int ioeventfd_count; 210 struct rcu_head rcu; 211 struct kvm_io_range range[]; 212}; 213 214enum kvm_bus { 215 KVM_MMIO_BUS, 216 KVM_PIO_BUS, 217 KVM_VIRTIO_CCW_NOTIFY_BUS, 218 KVM_FAST_MMIO_BUS, 219 KVM_IOCSR_BUS, 220 KVM_NR_BUSES 221}; 222 223int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 224 int len, const void *val); 225int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, 226 gpa_t addr, int len, const void *val, long cookie); 227int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, 228 int len, void *val); 229int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, 230 int len, struct kvm_io_device *dev); 231int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, 232 struct kvm_io_device *dev); 233struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, 234 gpa_t addr); 235 236#ifdef CONFIG_KVM_ASYNC_PF 237struct kvm_async_pf { 238 struct work_struct work; 239 struct list_head link; 240 struct list_head queue; 241 struct kvm_vcpu *vcpu; 242 gpa_t cr2_or_gpa; 243 unsigned long addr; 244 struct kvm_arch_async_pf arch; 245 bool wakeup_all; 246 bool notpresent_injected; 247}; 248 249void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu); 250void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu); 251bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 252 unsigned long hva, struct kvm_arch_async_pf *arch); 253int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu); 254#endif 255 256#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER 257union kvm_mmu_notifier_arg { 258 unsigned long attributes; 259}; 260 261enum kvm_gfn_range_filter { 262 KVM_FILTER_SHARED = BIT(0), 263 KVM_FILTER_PRIVATE = BIT(1), 264}; 265 266struct kvm_gfn_range { 267 struct kvm_memory_slot *slot; 268 gfn_t start; 269 gfn_t end; 270 union kvm_mmu_notifier_arg arg; 271 enum kvm_gfn_range_filter attr_filter; 272 bool may_block; 273 bool lockless; 274}; 275bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); 276bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range); 277bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range); 278#endif 279 280enum { 281 OUTSIDE_GUEST_MODE, 282 IN_GUEST_MODE, 283 EXITING_GUEST_MODE, 284 READING_SHADOW_PAGE_TABLES, 285}; 286 287struct kvm_host_map { 288 /* 289 * Only valid if the 'pfn' is managed by the host kernel (i.e. There is 290 * a 'struct page' for it. When using mem= kernel parameter some memory 291 * can be used as guest memory but they are not managed by host 292 * kernel). 293 */ 294 struct page *pinned_page; 295 struct page *page; 296 void *hva; 297 kvm_pfn_t pfn; 298 kvm_pfn_t gfn; 299 bool writable; 300}; 301 302/* 303 * Used to check if the mapping is valid or not. Never use 'kvm_host_map' 304 * directly to check for that. 305 */ 306static inline bool kvm_vcpu_mapped(struct kvm_host_map *map) 307{ 308 return !!map->hva; 309} 310 311static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop) 312{ 313 return single_task_running() && !need_resched() && ktime_before(cur, stop); 314} 315 316/* 317 * Sometimes a large or cross-page mmio needs to be broken up into separate 318 * exits for userspace servicing. 319 */ 320struct kvm_mmio_fragment { 321 gpa_t gpa; 322 void *data; 323 u64 val; 324 unsigned int len; 325}; 326 327struct kvm_vcpu { 328 struct kvm *kvm; 329#ifdef CONFIG_PREEMPT_NOTIFIERS 330 struct preempt_notifier preempt_notifier; 331#endif 332 int cpu; 333 int vcpu_id; /* id given by userspace at creation */ 334 int vcpu_idx; /* index into kvm->vcpu_array */ 335 int ____srcu_idx; /* Don't use this directly. You've been warned. */ 336#ifdef CONFIG_PROVE_RCU 337 int srcu_depth; 338#endif 339 int mode; 340 u64 requests; 341 unsigned long guest_debug; 342 343 struct mutex mutex; 344 struct kvm_run *run; 345 346#ifndef __KVM_HAVE_ARCH_WQP 347 struct rcuwait wait; 348#endif 349 struct pid *pid; 350 rwlock_t pid_lock; 351 int sigset_active; 352 sigset_t sigset; 353 unsigned int halt_poll_ns; 354 bool valid_wakeup; 355 356#ifdef CONFIG_HAS_IOMEM 357 int mmio_needed; 358 int mmio_read_completed; 359 int mmio_is_write; 360 int mmio_cur_fragment; 361 int mmio_nr_fragments; 362 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS]; 363#endif 364 365#ifdef CONFIG_KVM_ASYNC_PF 366 struct { 367 u32 queued; 368 struct list_head queue; 369 struct list_head done; 370 spinlock_t lock; 371 } async_pf; 372#endif 373 374#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 375 /* 376 * Cpu relax intercept or pause loop exit optimization 377 * in_spin_loop: set when a vcpu does a pause loop exit 378 * or cpu relax intercepted. 379 * dy_eligible: indicates whether vcpu is eligible for directed yield. 380 */ 381 struct { 382 bool in_spin_loop; 383 bool dy_eligible; 384 } spin_loop; 385#endif 386 bool wants_to_run; 387 bool preempted; 388 bool ready; 389 bool scheduled_out; 390 struct kvm_vcpu_arch arch; 391 struct kvm_vcpu_stat stat; 392 char stats_id[KVM_STATS_NAME_SIZE]; 393 struct kvm_dirty_ring dirty_ring; 394 395 /* 396 * The most recently used memslot by this vCPU and the slots generation 397 * for which it is valid. 398 * No wraparound protection is needed since generations won't overflow in 399 * thousands of years, even assuming 1M memslot operations per second. 400 */ 401 struct kvm_memory_slot *last_used_slot; 402 u64 last_used_slot_gen; 403}; 404 405/* 406 * Start accounting time towards a guest. 407 * Must be called before entering guest context. 408 */ 409static __always_inline void guest_timing_enter_irqoff(void) 410{ 411 /* 412 * This is running in ioctl context so its safe to assume that it's the 413 * stime pending cputime to flush. 414 */ 415 instrumentation_begin(); 416 vtime_account_guest_enter(); 417 instrumentation_end(); 418} 419 420/* 421 * Enter guest context and enter an RCU extended quiescent state. 422 * 423 * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is 424 * unsafe to use any code which may directly or indirectly use RCU, tracing 425 * (including IRQ flag tracing), or lockdep. All code in this period must be 426 * non-instrumentable. 427 */ 428static __always_inline void guest_context_enter_irqoff(void) 429{ 430 /* 431 * KVM does not hold any references to rcu protected data when it 432 * switches CPU into a guest mode. In fact switching to a guest mode 433 * is very similar to exiting to userspace from rcu point of view. In 434 * addition CPU may stay in a guest mode for quite a long time (up to 435 * one time slice). Lets treat guest mode as quiescent state, just like 436 * we do with user-mode execution. 437 */ 438 if (!context_tracking_guest_enter()) { 439 instrumentation_begin(); 440 rcu_virt_note_context_switch(); 441 instrumentation_end(); 442 } 443} 444 445/* 446 * Deprecated. Architectures should move to guest_timing_enter_irqoff() and 447 * guest_state_enter_irqoff(). 448 */ 449static __always_inline void guest_enter_irqoff(void) 450{ 451 guest_timing_enter_irqoff(); 452 guest_context_enter_irqoff(); 453} 454 455/** 456 * guest_state_enter_irqoff - Fixup state when entering a guest 457 * 458 * Entry to a guest will enable interrupts, but the kernel state is interrupts 459 * disabled when this is invoked. Also tell RCU about it. 460 * 461 * 1) Trace interrupts on state 462 * 2) Invoke context tracking if enabled to adjust RCU state 463 * 3) Tell lockdep that interrupts are enabled 464 * 465 * Invoked from architecture specific code before entering a guest. 466 * Must be called with interrupts disabled and the caller must be 467 * non-instrumentable. 468 * The caller has to invoke guest_timing_enter_irqoff() before this. 469 * 470 * Note: this is analogous to exit_to_user_mode(). 471 */ 472static __always_inline void guest_state_enter_irqoff(void) 473{ 474 instrumentation_begin(); 475 trace_hardirqs_on_prepare(); 476 lockdep_hardirqs_on_prepare(); 477 instrumentation_end(); 478 479 guest_context_enter_irqoff(); 480 lockdep_hardirqs_on(CALLER_ADDR0); 481} 482 483/* 484 * Exit guest context and exit an RCU extended quiescent state. 485 * 486 * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is 487 * unsafe to use any code which may directly or indirectly use RCU, tracing 488 * (including IRQ flag tracing), or lockdep. All code in this period must be 489 * non-instrumentable. 490 */ 491static __always_inline void guest_context_exit_irqoff(void) 492{ 493 /* 494 * Guest mode is treated as a quiescent state, see 495 * guest_context_enter_irqoff() for more details. 496 */ 497 if (!context_tracking_guest_exit()) { 498 instrumentation_begin(); 499 rcu_virt_note_context_switch(); 500 instrumentation_end(); 501 } 502} 503 504/* 505 * Stop accounting time towards a guest. 506 * Must be called after exiting guest context. 507 */ 508static __always_inline void guest_timing_exit_irqoff(void) 509{ 510 instrumentation_begin(); 511 /* Flush the guest cputime we spent on the guest */ 512 vtime_account_guest_exit(); 513 instrumentation_end(); 514} 515 516/* 517 * Deprecated. Architectures should move to guest_state_exit_irqoff() and 518 * guest_timing_exit_irqoff(). 519 */ 520static __always_inline void guest_exit_irqoff(void) 521{ 522 guest_context_exit_irqoff(); 523 guest_timing_exit_irqoff(); 524} 525 526static inline void guest_exit(void) 527{ 528 unsigned long flags; 529 530 local_irq_save(flags); 531 guest_exit_irqoff(); 532 local_irq_restore(flags); 533} 534 535/** 536 * guest_state_exit_irqoff - Establish state when returning from guest mode 537 * 538 * Entry from a guest disables interrupts, but guest mode is traced as 539 * interrupts enabled. Also with NO_HZ_FULL RCU might be idle. 540 * 541 * 1) Tell lockdep that interrupts are disabled 542 * 2) Invoke context tracking if enabled to reactivate RCU 543 * 3) Trace interrupts off state 544 * 545 * Invoked from architecture specific code after exiting a guest. 546 * Must be invoked with interrupts disabled and the caller must be 547 * non-instrumentable. 548 * The caller has to invoke guest_timing_exit_irqoff() after this. 549 * 550 * Note: this is analogous to enter_from_user_mode(). 551 */ 552static __always_inline void guest_state_exit_irqoff(void) 553{ 554 lockdep_hardirqs_off(CALLER_ADDR0); 555 guest_context_exit_irqoff(); 556 557 instrumentation_begin(); 558 trace_hardirqs_off_finish(); 559 instrumentation_end(); 560} 561 562static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) 563{ 564 /* 565 * The memory barrier ensures a previous write to vcpu->requests cannot 566 * be reordered with the read of vcpu->mode. It pairs with the general 567 * memory barrier following the write of vcpu->mode in VCPU RUN. 568 */ 569 smp_mb__before_atomic(); 570 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); 571} 572 573/* 574 * Some of the bitops functions do not support too long bitmaps. 575 * This number must be determined not to exceed such limits. 576 */ 577#define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1) 578 579/* 580 * Since at idle each memslot belongs to two memslot sets it has to contain 581 * two embedded nodes for each data structure that it forms a part of. 582 * 583 * Two memslot sets (one active and one inactive) are necessary so the VM 584 * continues to run on one memslot set while the other is being modified. 585 * 586 * These two memslot sets normally point to the same set of memslots. 587 * They can, however, be desynchronized when performing a memslot management 588 * operation by replacing the memslot to be modified by its copy. 589 * After the operation is complete, both memslot sets once again point to 590 * the same, common set of memslot data. 591 * 592 * The memslots themselves are independent of each other so they can be 593 * individually added or deleted. 594 */ 595struct kvm_memory_slot { 596 struct hlist_node id_node[2]; 597 struct interval_tree_node hva_node[2]; 598 struct rb_node gfn_node[2]; 599 gfn_t base_gfn; 600 unsigned long npages; 601 unsigned long *dirty_bitmap; 602 struct kvm_arch_memory_slot arch; 603 unsigned long userspace_addr; 604 u32 flags; 605 short id; 606 u16 as_id; 607 608#ifdef CONFIG_KVM_GUEST_MEMFD 609 struct { 610 /* 611 * Writes protected by kvm->slots_lock. Acquiring a 612 * reference via kvm_gmem_get_file() is protected by 613 * either kvm->slots_lock or kvm->srcu. 614 */ 615 struct file *file; 616 pgoff_t pgoff; 617 } gmem; 618#endif 619}; 620 621static inline bool kvm_slot_has_gmem(const struct kvm_memory_slot *slot) 622{ 623 return slot && (slot->flags & KVM_MEM_GUEST_MEMFD); 624} 625 626static inline bool kvm_slot_dirty_track_enabled(const struct kvm_memory_slot *slot) 627{ 628 return slot->flags & KVM_MEM_LOG_DIRTY_PAGES; 629} 630 631static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot) 632{ 633 return ALIGN(memslot->npages, BITS_PER_LONG) / 8; 634} 635 636static inline unsigned long *kvm_second_dirty_bitmap(struct kvm_memory_slot *memslot) 637{ 638 unsigned long len = kvm_dirty_bitmap_bytes(memslot); 639 640 return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap); 641} 642 643#ifndef KVM_DIRTY_LOG_MANUAL_CAPS 644#define KVM_DIRTY_LOG_MANUAL_CAPS KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE 645#endif 646 647struct kvm_s390_adapter_int { 648 u64 ind_addr; 649 u64 summary_addr; 650 u64 ind_offset; 651 u32 summary_offset; 652 u32 adapter_id; 653}; 654 655struct kvm_hv_sint { 656 u32 vcpu; 657 u32 sint; 658}; 659 660struct kvm_xen_evtchn { 661 u32 port; 662 u32 vcpu_id; 663 int vcpu_idx; 664 u32 priority; 665}; 666 667struct kvm_kernel_irq_routing_entry { 668 u32 gsi; 669 u32 type; 670 int (*set)(struct kvm_kernel_irq_routing_entry *e, 671 struct kvm *kvm, int irq_source_id, int level, 672 bool line_status); 673 union { 674 struct { 675 unsigned irqchip; 676 unsigned pin; 677 } irqchip; 678 struct { 679 u32 address_lo; 680 u32 address_hi; 681 u32 data; 682 u32 flags; 683 u32 devid; 684 } msi; 685 struct kvm_s390_adapter_int adapter; 686 struct kvm_hv_sint hv_sint; 687 struct kvm_xen_evtchn xen_evtchn; 688 }; 689 struct hlist_node link; 690}; 691 692#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 693struct kvm_irq_routing_table { 694 int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS]; 695 u32 nr_rt_entries; 696 /* 697 * Array indexed by gsi. Each entry contains list of irq chips 698 * the gsi is connected to. 699 */ 700 struct hlist_head map[] __counted_by(nr_rt_entries); 701}; 702#endif 703 704bool kvm_arch_irqchip_in_kernel(struct kvm *kvm); 705 706#ifndef KVM_INTERNAL_MEM_SLOTS 707#define KVM_INTERNAL_MEM_SLOTS 0 708#endif 709 710#define KVM_MEM_SLOTS_NUM SHRT_MAX 711#define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_INTERNAL_MEM_SLOTS) 712 713#if KVM_MAX_NR_ADDRESS_SPACES == 1 714static inline int kvm_arch_nr_memslot_as_ids(struct kvm *kvm) 715{ 716 return KVM_MAX_NR_ADDRESS_SPACES; 717} 718 719static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) 720{ 721 return 0; 722} 723#endif 724 725#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 726static inline bool kvm_arch_has_private_mem(struct kvm *kvm) 727{ 728 return false; 729} 730#endif 731 732#ifdef CONFIG_KVM_GUEST_MEMFD 733bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); 734 735static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) 736{ 737 u64 flags = GUEST_MEMFD_FLAG_MMAP; 738 739 if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) 740 flags |= GUEST_MEMFD_FLAG_INIT_SHARED; 741 742 return flags; 743} 744#endif 745 746#ifndef kvm_arch_has_readonly_mem 747static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) 748{ 749 return IS_ENABLED(CONFIG_HAVE_KVM_READONLY_MEM); 750} 751#endif 752 753struct kvm_memslots { 754 u64 generation; 755 atomic_long_t last_used_slot; 756 struct rb_root_cached hva_tree; 757 struct rb_root gfn_tree; 758 /* 759 * The mapping table from slot id to memslot. 760 * 761 * 7-bit bucket count matches the size of the old id to index array for 762 * 512 slots, while giving good performance with this slot count. 763 * Higher bucket counts bring only small performance improvements but 764 * always result in higher memory usage (even for lower memslot counts). 765 */ 766 DECLARE_HASHTABLE(id_hash, 7); 767 int node_idx; 768}; 769 770struct kvm { 771#ifdef KVM_HAVE_MMU_RWLOCK 772 rwlock_t mmu_lock; 773#else 774 spinlock_t mmu_lock; 775#endif /* KVM_HAVE_MMU_RWLOCK */ 776 777 struct mutex slots_lock; 778 779 /* 780 * Protects the arch-specific fields of struct kvm_memory_slots in 781 * use by the VM. To be used under the slots_lock (above) or in a 782 * kvm->srcu critical section where acquiring the slots_lock would 783 * lead to deadlock with the synchronize_srcu in 784 * kvm_swap_active_memslots(). 785 */ 786 struct mutex slots_arch_lock; 787 struct mm_struct *mm; /* userspace tied to this vm */ 788 unsigned long nr_memslot_pages; 789 /* The two memslot sets - active and inactive (per address space) */ 790 struct kvm_memslots __memslots[KVM_MAX_NR_ADDRESS_SPACES][2]; 791 /* The current active memslot set for each address space */ 792 struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; 793 struct xarray vcpu_array; 794 /* 795 * Protected by slots_lock, but can be read outside if an 796 * incorrect answer is acceptable. 797 */ 798 atomic_t nr_memslots_dirty_logging; 799 800 /* Used to wait for completion of MMU notifiers. */ 801 spinlock_t mn_invalidate_lock; 802 unsigned long mn_active_invalidate_count; 803 struct rcuwait mn_memslots_update_rcuwait; 804 805 /* For management / invalidation of gfn_to_pfn_caches */ 806 spinlock_t gpc_lock; 807 struct list_head gpc_list; 808 809 /* 810 * created_vcpus is protected by kvm->lock, and is incremented 811 * at the beginning of KVM_CREATE_VCPU. online_vcpus is only 812 * incremented after storing the kvm_vcpu pointer in vcpus, 813 * and is accessed atomically. 814 */ 815 atomic_t online_vcpus; 816 int max_vcpus; 817 int created_vcpus; 818 int last_boosted_vcpu; 819 struct list_head vm_list; 820 struct mutex lock; 821 struct kvm_io_bus __rcu *buses[KVM_NR_BUSES]; 822#ifdef CONFIG_HAVE_KVM_IRQCHIP 823 struct { 824 spinlock_t lock; 825 struct list_head items; 826 /* resampler_list update side is protected by resampler_lock. */ 827 struct list_head resampler_list; 828 struct mutex resampler_lock; 829 } irqfds; 830#endif 831 struct list_head ioeventfds; 832 struct kvm_vm_stat stat; 833 struct kvm_arch arch; 834 refcount_t users_count; 835#ifdef CONFIG_KVM_MMIO 836 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 837 spinlock_t ring_lock; 838 struct list_head coalesced_zones; 839#endif 840 841 struct mutex irq_lock; 842#ifdef CONFIG_HAVE_KVM_IRQCHIP 843 /* 844 * Update side is protected by irq_lock. 845 */ 846 struct kvm_irq_routing_table __rcu *irq_routing; 847 848 struct hlist_head irq_ack_notifier_list; 849#endif 850 851#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER 852 struct mmu_notifier mmu_notifier; 853 unsigned long mmu_invalidate_seq; 854 long mmu_invalidate_in_progress; 855 gfn_t mmu_invalidate_range_start; 856 gfn_t mmu_invalidate_range_end; 857#endif 858 struct list_head devices; 859 u64 manual_dirty_log_protect; 860 struct dentry *debugfs_dentry; 861 struct kvm_stat_data **debugfs_stat_data; 862 struct srcu_struct srcu; 863 struct srcu_struct irq_srcu; 864 pid_t userspace_pid; 865 bool override_halt_poll_ns; 866 unsigned int max_halt_poll_ns; 867 u32 dirty_ring_size; 868 bool dirty_ring_with_bitmap; 869 bool vm_bugged; 870 bool vm_dead; 871 872#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 873 struct notifier_block pm_notifier; 874#endif 875#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 876 /* Protected by slots_lock (for writes) and RCU (for reads) */ 877 struct xarray mem_attr_array; 878#endif 879 char stats_id[KVM_STATS_NAME_SIZE]; 880}; 881 882#define kvm_err(fmt, ...) \ 883 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 884#define kvm_info(fmt, ...) \ 885 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 886#define kvm_debug(fmt, ...) \ 887 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__) 888#define kvm_debug_ratelimited(fmt, ...) \ 889 pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \ 890 ## __VA_ARGS__) 891#define kvm_pr_unimpl(fmt, ...) \ 892 pr_err_ratelimited("kvm [%i]: " fmt, \ 893 task_tgid_nr(current), ## __VA_ARGS__) 894 895/* The guest did something we don't support. */ 896#define vcpu_unimpl(vcpu, fmt, ...) \ 897 kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt, \ 898 (vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__) 899 900#define vcpu_debug(vcpu, fmt, ...) \ 901 kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 902#define vcpu_debug_ratelimited(vcpu, fmt, ...) \ 903 kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id, \ 904 ## __VA_ARGS__) 905#define vcpu_err(vcpu, fmt, ...) \ 906 kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__) 907 908static inline void kvm_vm_dead(struct kvm *kvm) 909{ 910 kvm->vm_dead = true; 911 kvm_make_all_cpus_request(kvm, KVM_REQ_VM_DEAD); 912} 913 914static inline void kvm_vm_bugged(struct kvm *kvm) 915{ 916 kvm->vm_bugged = true; 917 kvm_vm_dead(kvm); 918} 919 920 921#define KVM_BUG(cond, kvm, fmt...) \ 922({ \ 923 bool __ret = !!(cond); \ 924 \ 925 if (WARN_ONCE(__ret && !(kvm)->vm_bugged, fmt)) \ 926 kvm_vm_bugged(kvm); \ 927 unlikely(__ret); \ 928}) 929 930#define KVM_BUG_ON(cond, kvm) \ 931({ \ 932 bool __ret = !!(cond); \ 933 \ 934 if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged)) \ 935 kvm_vm_bugged(kvm); \ 936 unlikely(__ret); \ 937}) 938 939/* 940 * Note, "data corruption" refers to corruption of host kernel data structures, 941 * not guest data. Guest data corruption, suspected or confirmed, that is tied 942 * and contained to a single VM should *never* BUG() and potentially panic the 943 * host, i.e. use this variant of KVM_BUG() if and only if a KVM data structure 944 * is corrupted and that corruption can have a cascading effect to other parts 945 * of the hosts and/or to other VMs. 946 */ 947#define KVM_BUG_ON_DATA_CORRUPTION(cond, kvm) \ 948({ \ 949 bool __ret = !!(cond); \ 950 \ 951 if (IS_ENABLED(CONFIG_BUG_ON_DATA_CORRUPTION)) \ 952 BUG_ON(__ret); \ 953 else if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged)) \ 954 kvm_vm_bugged(kvm); \ 955 unlikely(__ret); \ 956}) 957 958static inline void kvm_vcpu_srcu_read_lock(struct kvm_vcpu *vcpu) 959{ 960#ifdef CONFIG_PROVE_RCU 961 WARN_ONCE(vcpu->srcu_depth++, 962 "KVM: Illegal vCPU srcu_idx LOCK, depth=%d", vcpu->srcu_depth - 1); 963#endif 964 vcpu->____srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 965} 966 967static inline void kvm_vcpu_srcu_read_unlock(struct kvm_vcpu *vcpu) 968{ 969 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->____srcu_idx); 970 971#ifdef CONFIG_PROVE_RCU 972 WARN_ONCE(--vcpu->srcu_depth, 973 "KVM: Illegal vCPU srcu_idx UNLOCK, depth=%d", vcpu->srcu_depth); 974#endif 975} 976 977static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) 978{ 979 return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET); 980} 981 982/* 983 * Get a bus reference under the update-side lock. No long-term SRCU reader 984 * references are permitted, to avoid stale reads vs concurrent IO 985 * registrations. 986 */ 987static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) 988{ 989 return rcu_dereference_protected(kvm->buses[idx], 990 lockdep_is_held(&kvm->slots_lock)); 991} 992 993static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) 994{ 995 int num_vcpus = atomic_read(&kvm->online_vcpus); 996 997 /* 998 * Explicitly verify the target vCPU is online, as the anti-speculation 999 * logic only limits the CPU's ability to speculate, e.g. given a "bad" 1000 * index, clamping the index to 0 would return vCPU0, not NULL. 1001 */ 1002 if (i >= num_vcpus) 1003 return NULL; 1004 1005 i = array_index_nospec(i, num_vcpus); 1006 1007 /* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu. */ 1008 smp_rmb(); 1009 return xa_load(&kvm->vcpu_array, i); 1010} 1011 1012#define kvm_for_each_vcpu(idx, vcpup, kvm) \ 1013 if (atomic_read(&kvm->online_vcpus)) \ 1014 xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0, \ 1015 (atomic_read(&kvm->online_vcpus) - 1)) 1016 1017static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) 1018{ 1019 struct kvm_vcpu *vcpu = NULL; 1020 unsigned long i; 1021 1022 if (id < 0) 1023 return NULL; 1024 if (id < KVM_MAX_VCPUS) 1025 vcpu = kvm_get_vcpu(kvm, id); 1026 if (vcpu && vcpu->vcpu_id == id) 1027 return vcpu; 1028 kvm_for_each_vcpu(i, vcpu, kvm) 1029 if (vcpu->vcpu_id == id) 1030 return vcpu; 1031 return NULL; 1032} 1033 1034static inline bool kvm_is_vcpu_creation_in_progress(struct kvm *kvm) 1035{ 1036 lockdep_assert_held(&kvm->lock); 1037 1038 return kvm->created_vcpus != atomic_read(&kvm->online_vcpus); 1039} 1040 1041void kvm_destroy_vcpus(struct kvm *kvm); 1042 1043int kvm_trylock_all_vcpus(struct kvm *kvm); 1044int kvm_lock_all_vcpus(struct kvm *kvm); 1045void kvm_unlock_all_vcpus(struct kvm *kvm); 1046 1047void vcpu_load(struct kvm_vcpu *vcpu); 1048void vcpu_put(struct kvm_vcpu *vcpu); 1049 1050#ifdef CONFIG_KVM_IOAPIC 1051void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm); 1052#else 1053static inline void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm) 1054{ 1055} 1056#endif 1057 1058#ifdef CONFIG_HAVE_KVM_IRQCHIP 1059int kvm_irqfd_init(void); 1060void kvm_irqfd_exit(void); 1061#else 1062static inline int kvm_irqfd_init(void) 1063{ 1064 return 0; 1065} 1066 1067static inline void kvm_irqfd_exit(void) 1068{ 1069} 1070#endif 1071int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module); 1072void kvm_exit(void); 1073 1074void kvm_get_kvm(struct kvm *kvm); 1075bool kvm_get_kvm_safe(struct kvm *kvm); 1076void kvm_put_kvm(struct kvm *kvm); 1077bool file_is_kvm(struct file *file); 1078void kvm_put_kvm_no_destroy(struct kvm *kvm); 1079 1080static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) 1081{ 1082 as_id = array_index_nospec(as_id, KVM_MAX_NR_ADDRESS_SPACES); 1083 return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu, 1084 lockdep_is_held(&kvm->slots_lock) || 1085 !refcount_read(&kvm->users_count)); 1086} 1087 1088static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm) 1089{ 1090 return __kvm_memslots(kvm, 0); 1091} 1092 1093static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu) 1094{ 1095 int as_id = kvm_arch_vcpu_memslots_id(vcpu); 1096 1097 return __kvm_memslots(vcpu->kvm, as_id); 1098} 1099 1100static inline bool kvm_memslots_empty(struct kvm_memslots *slots) 1101{ 1102 return RB_EMPTY_ROOT(&slots->gfn_tree); 1103} 1104 1105bool kvm_are_all_memslots_empty(struct kvm *kvm); 1106 1107#define kvm_for_each_memslot(memslot, bkt, slots) \ 1108 hash_for_each(slots->id_hash, bkt, memslot, id_node[slots->node_idx]) \ 1109 if (WARN_ON_ONCE(!memslot->npages)) { \ 1110 } else 1111 1112static inline 1113struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id) 1114{ 1115 struct kvm_memory_slot *slot; 1116 int idx = slots->node_idx; 1117 1118 hash_for_each_possible(slots->id_hash, slot, id_node[idx], id) { 1119 if (slot->id == id) 1120 return slot; 1121 } 1122 1123 return NULL; 1124} 1125 1126/* Iterator used for walking memslots that overlap a gfn range. */ 1127struct kvm_memslot_iter { 1128 struct kvm_memslots *slots; 1129 struct rb_node *node; 1130 struct kvm_memory_slot *slot; 1131}; 1132 1133static inline void kvm_memslot_iter_next(struct kvm_memslot_iter *iter) 1134{ 1135 iter->node = rb_next(iter->node); 1136 if (!iter->node) 1137 return; 1138 1139 iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[iter->slots->node_idx]); 1140} 1141 1142static inline void kvm_memslot_iter_start(struct kvm_memslot_iter *iter, 1143 struct kvm_memslots *slots, 1144 gfn_t start) 1145{ 1146 int idx = slots->node_idx; 1147 struct rb_node *tmp; 1148 struct kvm_memory_slot *slot; 1149 1150 iter->slots = slots; 1151 1152 /* 1153 * Find the so called "upper bound" of a key - the first node that has 1154 * its key strictly greater than the searched one (the start gfn in our case). 1155 */ 1156 iter->node = NULL; 1157 for (tmp = slots->gfn_tree.rb_node; tmp; ) { 1158 slot = container_of(tmp, struct kvm_memory_slot, gfn_node[idx]); 1159 if (start < slot->base_gfn) { 1160 iter->node = tmp; 1161 tmp = tmp->rb_left; 1162 } else { 1163 tmp = tmp->rb_right; 1164 } 1165 } 1166 1167 /* 1168 * Find the slot with the lowest gfn that can possibly intersect with 1169 * the range, so we'll ideally have slot start <= range start 1170 */ 1171 if (iter->node) { 1172 /* 1173 * A NULL previous node means that the very first slot 1174 * already has a higher start gfn. 1175 * In this case slot start > range start. 1176 */ 1177 tmp = rb_prev(iter->node); 1178 if (tmp) 1179 iter->node = tmp; 1180 } else { 1181 /* a NULL node below means no slots */ 1182 iter->node = rb_last(&slots->gfn_tree); 1183 } 1184 1185 if (iter->node) { 1186 iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[idx]); 1187 1188 /* 1189 * It is possible in the slot start < range start case that the 1190 * found slot ends before or at range start (slot end <= range start) 1191 * and so it does not overlap the requested range. 1192 * 1193 * In such non-overlapping case the next slot (if it exists) will 1194 * already have slot start > range start, otherwise the logic above 1195 * would have found it instead of the current slot. 1196 */ 1197 if (iter->slot->base_gfn + iter->slot->npages <= start) 1198 kvm_memslot_iter_next(iter); 1199 } 1200} 1201 1202static inline bool kvm_memslot_iter_is_valid(struct kvm_memslot_iter *iter, gfn_t end) 1203{ 1204 if (!iter->node) 1205 return false; 1206 1207 /* 1208 * If this slot starts beyond or at the end of the range so does 1209 * every next one 1210 */ 1211 return iter->slot->base_gfn < end; 1212} 1213 1214/* Iterate over each memslot at least partially intersecting [start, end) range */ 1215#define kvm_for_each_memslot_in_gfn_range(iter, slots, start, end) \ 1216 for (kvm_memslot_iter_start(iter, slots, start); \ 1217 kvm_memslot_iter_is_valid(iter, end); \ 1218 kvm_memslot_iter_next(iter)) 1219 1220struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); 1221struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu); 1222struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn); 1223 1224/* 1225 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations: 1226 * - create a new memory slot 1227 * - delete an existing memory slot 1228 * - modify an existing memory slot 1229 * -- move it in the guest physical memory space 1230 * -- just change its flags 1231 * 1232 * Since flags can be changed by some of these operations, the following 1233 * differentiation is the best we can do for kvm_set_memory_region(): 1234 */ 1235enum kvm_mr_change { 1236 KVM_MR_CREATE, 1237 KVM_MR_DELETE, 1238 KVM_MR_MOVE, 1239 KVM_MR_FLAGS_ONLY, 1240}; 1241 1242int kvm_set_internal_memslot(struct kvm *kvm, 1243 const struct kvm_userspace_memory_region2 *mem); 1244void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot); 1245void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen); 1246int kvm_arch_prepare_memory_region(struct kvm *kvm, 1247 const struct kvm_memory_slot *old, 1248 struct kvm_memory_slot *new, 1249 enum kvm_mr_change change); 1250void kvm_arch_commit_memory_region(struct kvm *kvm, 1251 struct kvm_memory_slot *old, 1252 const struct kvm_memory_slot *new, 1253 enum kvm_mr_change change); 1254/* flush all memory translations */ 1255void kvm_arch_flush_shadow_all(struct kvm *kvm); 1256/* flush memory translations pointing to 'slot' */ 1257void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 1258 struct kvm_memory_slot *slot); 1259 1260int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn, 1261 struct page **pages, int nr_pages); 1262 1263struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn, bool write); 1264static inline struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) 1265{ 1266 return __gfn_to_page(kvm, gfn, true); 1267} 1268 1269unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn); 1270unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable); 1271unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn); 1272unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn, 1273 bool *writable); 1274 1275static inline void kvm_release_page_unused(struct page *page) 1276{ 1277 if (!page) 1278 return; 1279 1280 put_page(page); 1281} 1282 1283void kvm_release_page_clean(struct page *page); 1284void kvm_release_page_dirty(struct page *page); 1285 1286static inline void kvm_release_faultin_page(struct kvm *kvm, struct page *page, 1287 bool unused, bool dirty) 1288{ 1289 lockdep_assert_once(lockdep_is_held(&kvm->mmu_lock) || unused); 1290 1291 if (!page) 1292 return; 1293 1294 /* 1295 * If the page that KVM got from the *primary MMU* is writable, and KVM 1296 * installed or reused a SPTE, mark the page/folio dirty. Note, this 1297 * may mark a folio dirty even if KVM created a read-only SPTE, e.g. if 1298 * the GFN is write-protected. Folios can't be safely marked dirty 1299 * outside of mmu_lock as doing so could race with writeback on the 1300 * folio. As a result, KVM can't mark folios dirty in the fast page 1301 * fault handler, and so KVM must (somewhat) speculatively mark the 1302 * folio dirty if KVM could locklessly make the SPTE writable. 1303 */ 1304 if (unused) 1305 kvm_release_page_unused(page); 1306 else if (dirty) 1307 kvm_release_page_dirty(page); 1308 else 1309 kvm_release_page_clean(page); 1310} 1311 1312kvm_pfn_t __kvm_faultin_pfn(const struct kvm_memory_slot *slot, gfn_t gfn, 1313 unsigned int foll, bool *writable, 1314 struct page **refcounted_page); 1315 1316static inline kvm_pfn_t kvm_faultin_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, 1317 bool write, bool *writable, 1318 struct page **refcounted_page) 1319{ 1320 return __kvm_faultin_pfn(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, 1321 write ? FOLL_WRITE : 0, writable, refcounted_page); 1322} 1323 1324int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, 1325 int len); 1326int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); 1327int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1328 void *data, unsigned long len); 1329int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1330 void *data, unsigned int offset, 1331 unsigned long len); 1332int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, 1333 int offset, int len); 1334int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, 1335 unsigned long len); 1336int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1337 void *data, unsigned long len); 1338int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1339 void *data, unsigned int offset, 1340 unsigned long len); 1341int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, 1342 gpa_t gpa, unsigned long len); 1343 1344#define __kvm_get_guest(kvm, gfn, offset, v) \ 1345({ \ 1346 unsigned long __addr = gfn_to_hva(kvm, gfn); \ 1347 typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset); \ 1348 int __ret = -EFAULT; \ 1349 \ 1350 if (!kvm_is_error_hva(__addr)) \ 1351 __ret = get_user(v, __uaddr); \ 1352 __ret; \ 1353}) 1354 1355#define kvm_get_guest(kvm, gpa, v) \ 1356({ \ 1357 gpa_t __gpa = gpa; \ 1358 struct kvm *__kvm = kvm; \ 1359 \ 1360 __kvm_get_guest(__kvm, __gpa >> PAGE_SHIFT, \ 1361 offset_in_page(__gpa), v); \ 1362}) 1363 1364#define __kvm_put_guest(kvm, gfn, offset, v) \ 1365({ \ 1366 unsigned long __addr = gfn_to_hva(kvm, gfn); \ 1367 typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset); \ 1368 int __ret = -EFAULT; \ 1369 \ 1370 if (!kvm_is_error_hva(__addr)) \ 1371 __ret = put_user(v, __uaddr); \ 1372 if (!__ret) \ 1373 mark_page_dirty(kvm, gfn); \ 1374 __ret; \ 1375}) 1376 1377#define kvm_put_guest(kvm, gpa, v) \ 1378({ \ 1379 gpa_t __gpa = gpa; \ 1380 struct kvm *__kvm = kvm; \ 1381 \ 1382 __kvm_put_guest(__kvm, __gpa >> PAGE_SHIFT, \ 1383 offset_in_page(__gpa), v); \ 1384}) 1385 1386int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); 1387bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); 1388bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); 1389unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn); 1390void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn); 1391void mark_page_dirty(struct kvm *kvm, gfn_t gfn); 1392 1393int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map, 1394 bool writable); 1395void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map); 1396 1397static inline int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, 1398 struct kvm_host_map *map) 1399{ 1400 return __kvm_vcpu_map(vcpu, gpa, map, true); 1401} 1402 1403static inline int kvm_vcpu_map_readonly(struct kvm_vcpu *vcpu, gpa_t gpa, 1404 struct kvm_host_map *map) 1405{ 1406 return __kvm_vcpu_map(vcpu, gpa, map, false); 1407} 1408 1409unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); 1410unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); 1411int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, 1412 int len); 1413int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 1414 unsigned long len); 1415int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, 1416 unsigned long len); 1417int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data, 1418 int offset, int len); 1419int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, 1420 unsigned long len); 1421void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); 1422 1423/** 1424 * kvm_gpc_init - initialize gfn_to_pfn_cache. 1425 * 1426 * @gpc: struct gfn_to_pfn_cache object. 1427 * @kvm: pointer to kvm instance. 1428 * 1429 * This sets up a gfn_to_pfn_cache by initializing locks and assigning the 1430 * immutable attributes. Note, the cache must be zero-allocated (or zeroed by 1431 * the caller before init). 1432 */ 1433void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm); 1434 1435/** 1436 * kvm_gpc_activate - prepare a cached kernel mapping and HPA for a given guest 1437 * physical address. 1438 * 1439 * @gpc: struct gfn_to_pfn_cache object. 1440 * @gpa: guest physical address to map. 1441 * @len: sanity check; the range being access must fit a single page. 1442 * 1443 * @return: 0 for success. 1444 * -EINVAL for a mapping which would cross a page boundary. 1445 * -EFAULT for an untranslatable guest physical address. 1446 * 1447 * This primes a gfn_to_pfn_cache and links it into the @gpc->kvm's list for 1448 * invalidations to be processed. Callers are required to use kvm_gpc_check() 1449 * to ensure that the cache is valid before accessing the target page. 1450 */ 1451int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len); 1452 1453/** 1454 * kvm_gpc_activate_hva - prepare a cached kernel mapping and HPA for a given HVA. 1455 * 1456 * @gpc: struct gfn_to_pfn_cache object. 1457 * @hva: userspace virtual address to map. 1458 * @len: sanity check; the range being access must fit a single page. 1459 * 1460 * @return: 0 for success. 1461 * -EINVAL for a mapping which would cross a page boundary. 1462 * -EFAULT for an untranslatable guest physical address. 1463 * 1464 * The semantics of this function are the same as those of kvm_gpc_activate(). It 1465 * merely bypasses a layer of address translation. 1466 */ 1467int kvm_gpc_activate_hva(struct gfn_to_pfn_cache *gpc, unsigned long hva, unsigned long len); 1468 1469/** 1470 * kvm_gpc_check - check validity of a gfn_to_pfn_cache. 1471 * 1472 * @gpc: struct gfn_to_pfn_cache object. 1473 * @len: sanity check; the range being access must fit a single page. 1474 * 1475 * @return: %true if the cache is still valid and the address matches. 1476 * %false if the cache is not valid. 1477 * 1478 * Callers outside IN_GUEST_MODE context should hold a read lock on @gpc->lock 1479 * while calling this function, and then continue to hold the lock until the 1480 * access is complete. 1481 * 1482 * Callers in IN_GUEST_MODE may do so without locking, although they should 1483 * still hold a read lock on kvm->scru for the memslot checks. 1484 */ 1485bool kvm_gpc_check(struct gfn_to_pfn_cache *gpc, unsigned long len); 1486 1487/** 1488 * kvm_gpc_refresh - update a previously initialized cache. 1489 * 1490 * @gpc: struct gfn_to_pfn_cache object. 1491 * @len: sanity check; the range being access must fit a single page. 1492 * 1493 * @return: 0 for success. 1494 * -EINVAL for a mapping which would cross a page boundary. 1495 * -EFAULT for an untranslatable guest physical address. 1496 * 1497 * This will attempt to refresh a gfn_to_pfn_cache. Note that a successful 1498 * return from this function does not mean the page can be immediately 1499 * accessed because it may have raced with an invalidation. Callers must 1500 * still lock and check the cache status, as this function does not return 1501 * with the lock still held to permit access. 1502 */ 1503int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len); 1504 1505/** 1506 * kvm_gpc_deactivate - deactivate and unlink a gfn_to_pfn_cache. 1507 * 1508 * @gpc: struct gfn_to_pfn_cache object. 1509 * 1510 * This removes a cache from the VM's list to be processed on MMU notifier 1511 * invocation. 1512 */ 1513void kvm_gpc_deactivate(struct gfn_to_pfn_cache *gpc); 1514 1515static inline bool kvm_gpc_is_gpa_active(struct gfn_to_pfn_cache *gpc) 1516{ 1517 return gpc->active && !kvm_is_error_gpa(gpc->gpa); 1518} 1519 1520static inline bool kvm_gpc_is_hva_active(struct gfn_to_pfn_cache *gpc) 1521{ 1522 return gpc->active && kvm_is_error_gpa(gpc->gpa); 1523} 1524 1525void kvm_sigset_activate(struct kvm_vcpu *vcpu); 1526void kvm_sigset_deactivate(struct kvm_vcpu *vcpu); 1527 1528void kvm_vcpu_halt(struct kvm_vcpu *vcpu); 1529bool kvm_vcpu_block(struct kvm_vcpu *vcpu); 1530void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu); 1531void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu); 1532bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu); 1533 1534#ifndef CONFIG_S390 1535void __kvm_vcpu_kick(struct kvm_vcpu *vcpu, bool wait); 1536 1537static inline void kvm_vcpu_kick(struct kvm_vcpu *vcpu) 1538{ 1539 __kvm_vcpu_kick(vcpu, false); 1540} 1541#endif 1542 1543int kvm_vcpu_yield_to(struct kvm_vcpu *target); 1544void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu, bool yield_to_kernel_mode); 1545 1546void kvm_flush_remote_tlbs(struct kvm *kvm); 1547void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages); 1548void kvm_flush_remote_tlbs_memslot(struct kvm *kvm, 1549 const struct kvm_memory_slot *memslot); 1550 1551#ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE 1552int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min); 1553int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min); 1554int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc); 1555void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc); 1556void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); 1557#endif 1558 1559void kvm_mmu_invalidate_begin(struct kvm *kvm); 1560void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end); 1561void kvm_mmu_invalidate_end(struct kvm *kvm); 1562bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); 1563 1564long kvm_arch_dev_ioctl(struct file *filp, 1565 unsigned int ioctl, unsigned long arg); 1566long kvm_arch_vcpu_ioctl(struct file *filp, 1567 unsigned int ioctl, unsigned long arg); 1568long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, 1569 unsigned int ioctl, unsigned long arg); 1570vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf); 1571 1572int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext); 1573 1574void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, 1575 struct kvm_memory_slot *slot, 1576 gfn_t gfn_offset, 1577 unsigned long mask); 1578void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot); 1579 1580#ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT 1581int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); 1582int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, 1583 int *is_dirty, struct kvm_memory_slot **memslot); 1584#endif 1585 1586int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, 1587 bool line_status); 1588int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 1589 struct kvm_enable_cap *cap); 1590int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); 1591long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl, 1592 unsigned long arg); 1593 1594int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 1595int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); 1596 1597int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 1598 struct kvm_translation *tr); 1599 1600int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 1601int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); 1602int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 1603 struct kvm_sregs *sregs); 1604int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 1605 struct kvm_sregs *sregs); 1606int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 1607 struct kvm_mp_state *mp_state); 1608int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 1609 struct kvm_mp_state *mp_state); 1610int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 1611 struct kvm_guest_debug *dbg); 1612int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu); 1613 1614void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); 1615void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); 1616int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id); 1617int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu); 1618void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu); 1619void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); 1620 1621#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 1622int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state); 1623#endif 1624 1625#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS 1626void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); 1627#else 1628static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {} 1629#endif 1630 1631#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 1632/* 1633 * kvm_arch_{enable,disable}_virtualization() are called on one CPU, under 1634 * kvm_usage_lock, immediately after/before 0=>1 and 1=>0 transitions of 1635 * kvm_usage_count, i.e. at the beginning of the generic hardware enabling 1636 * sequence, and at the end of the generic hardware disabling sequence. 1637 */ 1638void kvm_arch_enable_virtualization(void); 1639void kvm_arch_disable_virtualization(void); 1640/* 1641 * kvm_arch_{enable,disable}_virtualization_cpu() are called on "every" CPU to 1642 * do the actual twiddling of hardware bits. The hooks are called on all 1643 * online CPUs when KVM enables/disabled virtualization, and on a single CPU 1644 * when that CPU is onlined/offlined (including for Resume/Suspend). 1645 */ 1646int kvm_arch_enable_virtualization_cpu(void); 1647void kvm_arch_disable_virtualization_cpu(void); 1648#endif 1649bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu); 1650int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); 1651bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu); 1652int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu); 1653bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu); 1654bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu); 1655bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu); 1656void kvm_arch_pre_destroy_vm(struct kvm *kvm); 1657void kvm_arch_create_vm_debugfs(struct kvm *kvm); 1658 1659#ifndef __KVM_HAVE_ARCH_VM_ALLOC 1660/* 1661 * All architectures that want to use vzalloc currently also 1662 * need their own kvm_arch_alloc_vm implementation. 1663 */ 1664static inline struct kvm *kvm_arch_alloc_vm(void) 1665{ 1666 return kzalloc(sizeof(struct kvm), GFP_KERNEL_ACCOUNT); 1667} 1668#endif 1669 1670static inline void __kvm_arch_free_vm(struct kvm *kvm) 1671{ 1672 kvfree(kvm); 1673} 1674 1675#ifndef __KVM_HAVE_ARCH_VM_FREE 1676static inline void kvm_arch_free_vm(struct kvm *kvm) 1677{ 1678 __kvm_arch_free_vm(kvm); 1679} 1680#endif 1681 1682#ifndef __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS 1683static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm) 1684{ 1685 return -ENOTSUPP; 1686} 1687#else 1688int kvm_arch_flush_remote_tlbs(struct kvm *kvm); 1689#endif 1690 1691#ifndef __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE 1692static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, 1693 gfn_t gfn, u64 nr_pages) 1694{ 1695 return -EOPNOTSUPP; 1696} 1697#else 1698int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages); 1699#endif 1700 1701#ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA 1702void kvm_arch_register_noncoherent_dma(struct kvm *kvm); 1703void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm); 1704bool kvm_arch_has_noncoherent_dma(struct kvm *kvm); 1705#else 1706static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 1707{ 1708} 1709 1710static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 1711{ 1712} 1713 1714static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 1715{ 1716 return false; 1717} 1718#endif 1719 1720static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu) 1721{ 1722#ifdef __KVM_HAVE_ARCH_WQP 1723 return vcpu->arch.waitp; 1724#else 1725 return &vcpu->wait; 1726#endif 1727} 1728 1729/* 1730 * Wake a vCPU if necessary, but don't do any stats/metadata updates. Returns 1731 * true if the vCPU was blocking and was awakened, false otherwise. 1732 */ 1733static inline bool __kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) 1734{ 1735 return !!rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu)); 1736} 1737 1738static inline bool kvm_vcpu_is_blocking(struct kvm_vcpu *vcpu) 1739{ 1740 return rcuwait_active(kvm_arch_vcpu_get_wait(vcpu)); 1741} 1742 1743#ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED 1744/* 1745 * returns true if the virtual interrupt controller is initialized and 1746 * ready to accept virtual IRQ. On some architectures the virtual interrupt 1747 * controller is dynamically instantiated and this is not always true. 1748 */ 1749bool kvm_arch_intc_initialized(struct kvm *kvm); 1750#else 1751static inline bool kvm_arch_intc_initialized(struct kvm *kvm) 1752{ 1753 return true; 1754} 1755#endif 1756 1757#ifdef CONFIG_GUEST_PERF_EVENTS 1758unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu); 1759 1760void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void)); 1761void kvm_unregister_perf_callbacks(void); 1762#else 1763static inline void kvm_register_perf_callbacks(void *ign) {} 1764static inline void kvm_unregister_perf_callbacks(void) {} 1765#endif /* CONFIG_GUEST_PERF_EVENTS */ 1766 1767int kvm_arch_init_vm(struct kvm *kvm, unsigned long type); 1768void kvm_arch_destroy_vm(struct kvm *kvm); 1769 1770int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); 1771 1772struct kvm_irq_ack_notifier { 1773 struct hlist_node link; 1774 unsigned gsi; 1775 void (*irq_acked)(struct kvm_irq_ack_notifier *kian); 1776}; 1777 1778int kvm_irq_map_gsi(struct kvm *kvm, 1779 struct kvm_kernel_irq_routing_entry *entries, int gsi); 1780int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin); 1781 1782int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, 1783 bool line_status); 1784int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm, 1785 int irq_source_id, int level, bool line_status); 1786int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 1787 struct kvm *kvm, int irq_source_id, 1788 int level, bool line_status); 1789bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin); 1790void kvm_notify_acked_gsi(struct kvm *kvm, int gsi); 1791void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin); 1792void kvm_register_irq_ack_notifier(struct kvm *kvm, 1793 struct kvm_irq_ack_notifier *kian); 1794void kvm_unregister_irq_ack_notifier(struct kvm *kvm, 1795 struct kvm_irq_ack_notifier *kian); 1796bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args); 1797 1798/* 1799 * Returns a pointer to the memslot if it contains gfn. 1800 * Otherwise returns NULL. 1801 */ 1802static inline struct kvm_memory_slot * 1803try_get_memslot(struct kvm_memory_slot *slot, gfn_t gfn) 1804{ 1805 if (!slot) 1806 return NULL; 1807 1808 if (gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages) 1809 return slot; 1810 else 1811 return NULL; 1812} 1813 1814/* 1815 * Returns a pointer to the memslot that contains gfn. Otherwise returns NULL. 1816 * 1817 * With "approx" set returns the memslot also when the address falls 1818 * in a hole. In that case one of the memslots bordering the hole is 1819 * returned. 1820 */ 1821static inline struct kvm_memory_slot * 1822search_memslots(struct kvm_memslots *slots, gfn_t gfn, bool approx) 1823{ 1824 struct kvm_memory_slot *slot; 1825 struct rb_node *node; 1826 int idx = slots->node_idx; 1827 1828 slot = NULL; 1829 for (node = slots->gfn_tree.rb_node; node; ) { 1830 slot = container_of(node, struct kvm_memory_slot, gfn_node[idx]); 1831 if (gfn >= slot->base_gfn) { 1832 if (gfn < slot->base_gfn + slot->npages) 1833 return slot; 1834 node = node->rb_right; 1835 } else 1836 node = node->rb_left; 1837 } 1838 1839 return approx ? slot : NULL; 1840} 1841 1842static inline struct kvm_memory_slot * 1843____gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn, bool approx) 1844{ 1845 struct kvm_memory_slot *slot; 1846 1847 slot = (struct kvm_memory_slot *)atomic_long_read(&slots->last_used_slot); 1848 slot = try_get_memslot(slot, gfn); 1849 if (slot) 1850 return slot; 1851 1852 slot = search_memslots(slots, gfn, approx); 1853 if (slot) { 1854 atomic_long_set(&slots->last_used_slot, (unsigned long)slot); 1855 return slot; 1856 } 1857 1858 return NULL; 1859} 1860 1861/* 1862 * __gfn_to_memslot() and its descendants are here to allow arch code to inline 1863 * the lookups in hot paths. gfn_to_memslot() itself isn't here as an inline 1864 * because that would bloat other code too much. 1865 */ 1866static inline struct kvm_memory_slot * 1867__gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn) 1868{ 1869 return ____gfn_to_memslot(slots, gfn, false); 1870} 1871 1872static inline unsigned long 1873__gfn_to_hva_memslot(const struct kvm_memory_slot *slot, gfn_t gfn) 1874{ 1875 /* 1876 * The index was checked originally in search_memslots. To avoid 1877 * that a malicious guest builds a Spectre gadget out of e.g. page 1878 * table walks, do not let the processor speculate loads outside 1879 * the guest's registered memslots. 1880 */ 1881 unsigned long offset = gfn - slot->base_gfn; 1882 offset = array_index_nospec(offset, slot->npages); 1883 return slot->userspace_addr + offset * PAGE_SIZE; 1884} 1885 1886static inline int memslot_id(struct kvm *kvm, gfn_t gfn) 1887{ 1888 return gfn_to_memslot(kvm, gfn)->id; 1889} 1890 1891static inline gfn_t 1892hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot) 1893{ 1894 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT; 1895 1896 return slot->base_gfn + gfn_offset; 1897} 1898 1899static inline gpa_t gfn_to_gpa(gfn_t gfn) 1900{ 1901 return (gpa_t)gfn << PAGE_SHIFT; 1902} 1903 1904static inline gfn_t gpa_to_gfn(gpa_t gpa) 1905{ 1906 return (gfn_t)(gpa >> PAGE_SHIFT); 1907} 1908 1909static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn) 1910{ 1911 return (hpa_t)pfn << PAGE_SHIFT; 1912} 1913 1914static inline bool kvm_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa) 1915{ 1916 unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); 1917 1918 return !kvm_is_error_hva(hva); 1919} 1920 1921static inline void kvm_gpc_mark_dirty_in_slot(struct gfn_to_pfn_cache *gpc) 1922{ 1923 lockdep_assert_held(&gpc->lock); 1924 1925 if (!gpc->memslot) 1926 return; 1927 1928 mark_page_dirty_in_slot(gpc->kvm, gpc->memslot, gpa_to_gfn(gpc->gpa)); 1929} 1930 1931enum kvm_stat_kind { 1932 KVM_STAT_VM, 1933 KVM_STAT_VCPU, 1934}; 1935 1936struct kvm_stat_data { 1937 struct kvm *kvm; 1938 const struct kvm_stats_desc *desc; 1939 enum kvm_stat_kind kind; 1940}; 1941 1942#define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz) \ 1943 .flags = type | unit | base | \ 1944 BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ 1945 BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ 1946 BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ 1947 .exponent = exp, \ 1948 .size = sz, \ 1949 .bucket_size = bsz 1950 1951#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1952{ \ 1953 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1954 .offset = offsetof(struct kvm_vm_stat, generic.stat), \ 1955 .name = #stat, \ 1956} 1957#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1958{ \ 1959 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1960 .offset = offsetof(struct kvm_vcpu_stat, generic.stat), \ 1961 .name = #stat, \ 1962} 1963#define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1964{ \ 1965 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1966 .offset = offsetof(struct kvm_vm_stat, stat), \ 1967 .name = #stat, \ 1968} 1969#define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz) \ 1970{ \ 1971 STATS_DESC_COMMON(type, unit, base, exp, sz, bsz), \ 1972 .offset = offsetof(struct kvm_vcpu_stat, stat), \ 1973 .name = #stat, \ 1974} 1975/* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */ 1976#define STATS_DESC(SCOPE, stat, type, unit, base, exp, sz, bsz) \ 1977 SCOPE##_STATS_DESC(stat, type, unit, base, exp, sz, bsz) 1978 1979#define STATS_DESC_CUMULATIVE(SCOPE, name, unit, base, exponent) \ 1980 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_CUMULATIVE, \ 1981 unit, base, exponent, 1, 0) 1982#define STATS_DESC_INSTANT(SCOPE, name, unit, base, exponent) \ 1983 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_INSTANT, \ 1984 unit, base, exponent, 1, 0) 1985#define STATS_DESC_PEAK(SCOPE, name, unit, base, exponent) \ 1986 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_PEAK, \ 1987 unit, base, exponent, 1, 0) 1988#define STATS_DESC_LINEAR_HIST(SCOPE, name, unit, base, exponent, sz, bsz) \ 1989 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_LINEAR_HIST, \ 1990 unit, base, exponent, sz, bsz) 1991#define STATS_DESC_LOG_HIST(SCOPE, name, unit, base, exponent, sz) \ 1992 STATS_DESC(SCOPE, name, KVM_STATS_TYPE_LOG_HIST, \ 1993 unit, base, exponent, sz, 0) 1994 1995/* Cumulative counter, read/write */ 1996#define STATS_DESC_COUNTER(SCOPE, name) \ 1997 STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_NONE, \ 1998 KVM_STATS_BASE_POW10, 0) 1999/* Instantaneous counter, read only */ 2000#define STATS_DESC_ICOUNTER(SCOPE, name) \ 2001 STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_NONE, \ 2002 KVM_STATS_BASE_POW10, 0) 2003/* Peak counter, read/write */ 2004#define STATS_DESC_PCOUNTER(SCOPE, name) \ 2005 STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_NONE, \ 2006 KVM_STATS_BASE_POW10, 0) 2007 2008/* Instantaneous boolean value, read only */ 2009#define STATS_DESC_IBOOLEAN(SCOPE, name) \ 2010 STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_BOOLEAN, \ 2011 KVM_STATS_BASE_POW10, 0) 2012/* Peak (sticky) boolean value, read/write */ 2013#define STATS_DESC_PBOOLEAN(SCOPE, name) \ 2014 STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_BOOLEAN, \ 2015 KVM_STATS_BASE_POW10, 0) 2016 2017/* Cumulative time in nanosecond */ 2018#define STATS_DESC_TIME_NSEC(SCOPE, name) \ 2019 STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ 2020 KVM_STATS_BASE_POW10, -9) 2021/* Linear histogram for time in nanosecond */ 2022#define STATS_DESC_LINHIST_TIME_NSEC(SCOPE, name, sz, bsz) \ 2023 STATS_DESC_LINEAR_HIST(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ 2024 KVM_STATS_BASE_POW10, -9, sz, bsz) 2025/* Logarithmic histogram for time in nanosecond */ 2026#define STATS_DESC_LOGHIST_TIME_NSEC(SCOPE, name, sz) \ 2027 STATS_DESC_LOG_HIST(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ 2028 KVM_STATS_BASE_POW10, -9, sz) 2029 2030#define KVM_GENERIC_VM_STATS() \ 2031 STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush), \ 2032 STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush_requests) 2033 2034#define KVM_GENERIC_VCPU_STATS() \ 2035 STATS_DESC_COUNTER(VCPU_GENERIC, halt_successful_poll), \ 2036 STATS_DESC_COUNTER(VCPU_GENERIC, halt_attempted_poll), \ 2037 STATS_DESC_COUNTER(VCPU_GENERIC, halt_poll_invalid), \ 2038 STATS_DESC_COUNTER(VCPU_GENERIC, halt_wakeup), \ 2039 STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_success_ns), \ 2040 STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_ns), \ 2041 STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_wait_ns), \ 2042 STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_poll_success_hist, \ 2043 HALT_POLL_HIST_COUNT), \ 2044 STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_hist, \ 2045 HALT_POLL_HIST_COUNT), \ 2046 STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_wait_hist, \ 2047 HALT_POLL_HIST_COUNT), \ 2048 STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking) 2049 2050ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, 2051 const struct kvm_stats_desc *desc, 2052 void *stats, size_t size_stats, 2053 char __user *user_buffer, size_t size, loff_t *offset); 2054 2055/** 2056 * kvm_stats_linear_hist_update() - Update bucket value for linear histogram 2057 * statistics data. 2058 * 2059 * @data: start address of the stats data 2060 * @size: the number of bucket of the stats data 2061 * @value: the new value used to update the linear histogram's bucket 2062 * @bucket_size: the size (width) of a bucket 2063 */ 2064static inline void kvm_stats_linear_hist_update(u64 *data, size_t size, 2065 u64 value, size_t bucket_size) 2066{ 2067 size_t index = div64_u64(value, bucket_size); 2068 2069 index = min(index, size - 1); 2070 ++data[index]; 2071} 2072 2073/** 2074 * kvm_stats_log_hist_update() - Update bucket value for logarithmic histogram 2075 * statistics data. 2076 * 2077 * @data: start address of the stats data 2078 * @size: the number of bucket of the stats data 2079 * @value: the new value used to update the logarithmic histogram's bucket 2080 */ 2081static inline void kvm_stats_log_hist_update(u64 *data, size_t size, u64 value) 2082{ 2083 size_t index = fls64(value); 2084 2085 index = min(index, size - 1); 2086 ++data[index]; 2087} 2088 2089#define KVM_STATS_LINEAR_HIST_UPDATE(array, value, bsize) \ 2090 kvm_stats_linear_hist_update(array, ARRAY_SIZE(array), value, bsize) 2091#define KVM_STATS_LOG_HIST_UPDATE(array, value) \ 2092 kvm_stats_log_hist_update(array, ARRAY_SIZE(array), value) 2093 2094 2095extern const struct kvm_stats_header kvm_vm_stats_header; 2096extern const struct kvm_stats_desc kvm_vm_stats_desc[]; 2097extern const struct kvm_stats_header kvm_vcpu_stats_header; 2098extern const struct kvm_stats_desc kvm_vcpu_stats_desc[]; 2099 2100#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER 2101static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq) 2102{ 2103 if (unlikely(kvm->mmu_invalidate_in_progress)) 2104 return 1; 2105 /* 2106 * Ensure the read of mmu_invalidate_in_progress happens before 2107 * the read of mmu_invalidate_seq. This interacts with the 2108 * smp_wmb() in mmu_notifier_invalidate_range_end to make sure 2109 * that the caller either sees the old (non-zero) value of 2110 * mmu_invalidate_in_progress or the new (incremented) value of 2111 * mmu_invalidate_seq. 2112 * 2113 * PowerPC Book3s HV KVM calls this under a per-page lock rather 2114 * than under kvm->mmu_lock, for scalability, so can't rely on 2115 * kvm->mmu_lock to keep things ordered. 2116 */ 2117 smp_rmb(); 2118 if (kvm->mmu_invalidate_seq != mmu_seq) 2119 return 1; 2120 return 0; 2121} 2122 2123static inline int mmu_invalidate_retry_gfn(struct kvm *kvm, 2124 unsigned long mmu_seq, 2125 gfn_t gfn) 2126{ 2127 lockdep_assert_held(&kvm->mmu_lock); 2128 /* 2129 * If mmu_invalidate_in_progress is non-zero, then the range maintained 2130 * by kvm_mmu_notifier_invalidate_range_start contains all addresses 2131 * that might be being invalidated. Note that it may include some false 2132 * positives, due to shortcuts when handing concurrent invalidations. 2133 */ 2134 if (unlikely(kvm->mmu_invalidate_in_progress)) { 2135 /* 2136 * Dropping mmu_lock after bumping mmu_invalidate_in_progress 2137 * but before updating the range is a KVM bug. 2138 */ 2139 if (WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA || 2140 kvm->mmu_invalidate_range_end == INVALID_GPA)) 2141 return 1; 2142 2143 if (gfn >= kvm->mmu_invalidate_range_start && 2144 gfn < kvm->mmu_invalidate_range_end) 2145 return 1; 2146 } 2147 2148 if (kvm->mmu_invalidate_seq != mmu_seq) 2149 return 1; 2150 return 0; 2151} 2152 2153/* 2154 * This lockless version of the range-based retry check *must* be paired with a 2155 * call to the locked version after acquiring mmu_lock, i.e. this is safe to 2156 * use only as a pre-check to avoid contending mmu_lock. This version *will* 2157 * get false negatives and false positives. 2158 */ 2159static inline bool mmu_invalidate_retry_gfn_unsafe(struct kvm *kvm, 2160 unsigned long mmu_seq, 2161 gfn_t gfn) 2162{ 2163 /* 2164 * Use READ_ONCE() to ensure the in-progress flag and sequence counter 2165 * are always read from memory, e.g. so that checking for retry in a 2166 * loop won't result in an infinite retry loop. Don't force loads for 2167 * start+end, as the key to avoiding infinite retry loops is observing 2168 * the 1=>0 transition of in-progress, i.e. getting false negatives 2169 * due to stale start+end values is acceptable. 2170 */ 2171 if (unlikely(READ_ONCE(kvm->mmu_invalidate_in_progress)) && 2172 gfn >= kvm->mmu_invalidate_range_start && 2173 gfn < kvm->mmu_invalidate_range_end) 2174 return true; 2175 2176 return READ_ONCE(kvm->mmu_invalidate_seq) != mmu_seq; 2177} 2178#endif 2179 2180#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING 2181 2182#define KVM_MAX_IRQ_ROUTES 4096 /* might need extension/rework in the future */ 2183 2184bool kvm_arch_can_set_irq_routing(struct kvm *kvm); 2185int kvm_set_irq_routing(struct kvm *kvm, 2186 const struct kvm_irq_routing_entry *entries, 2187 unsigned nr, 2188 unsigned flags); 2189int kvm_init_irq_routing(struct kvm *kvm); 2190int kvm_set_routing_entry(struct kvm *kvm, 2191 struct kvm_kernel_irq_routing_entry *e, 2192 const struct kvm_irq_routing_entry *ue); 2193void kvm_free_irq_routing(struct kvm *kvm); 2194 2195#else 2196 2197static inline void kvm_free_irq_routing(struct kvm *kvm) {} 2198 2199static inline int kvm_init_irq_routing(struct kvm *kvm) 2200{ 2201 return 0; 2202} 2203 2204#endif 2205 2206int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi); 2207 2208void kvm_eventfd_init(struct kvm *kvm); 2209int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args); 2210 2211#ifdef CONFIG_HAVE_KVM_IRQCHIP 2212int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args); 2213void kvm_irqfd_release(struct kvm *kvm); 2214bool kvm_notify_irqfd_resampler(struct kvm *kvm, 2215 unsigned int irqchip, 2216 unsigned int pin); 2217void kvm_irq_routing_update(struct kvm *); 2218#else 2219static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 2220{ 2221 return -EINVAL; 2222} 2223 2224static inline void kvm_irqfd_release(struct kvm *kvm) {} 2225 2226static inline bool kvm_notify_irqfd_resampler(struct kvm *kvm, 2227 unsigned int irqchip, 2228 unsigned int pin) 2229{ 2230 return false; 2231} 2232#endif /* CONFIG_HAVE_KVM_IRQCHIP */ 2233 2234void kvm_arch_irq_routing_update(struct kvm *kvm); 2235 2236static inline void __kvm_make_request(int req, struct kvm_vcpu *vcpu) 2237{ 2238 /* 2239 * Ensure the rest of the request is published to kvm_check_request's 2240 * caller. Paired with the smp_mb__after_atomic in kvm_check_request. 2241 */ 2242 smp_wmb(); 2243 set_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 2244} 2245 2246static __always_inline void kvm_make_request(int req, struct kvm_vcpu *vcpu) 2247{ 2248 /* 2249 * Request that don't require vCPU action should never be logged in 2250 * vcpu->requests. The vCPU won't clear the request, so it will stay 2251 * logged indefinitely and prevent the vCPU from entering the guest. 2252 */ 2253 BUILD_BUG_ON(!__builtin_constant_p(req) || 2254 (req & KVM_REQUEST_NO_ACTION)); 2255 2256 __kvm_make_request(req, vcpu); 2257} 2258 2259#ifndef CONFIG_S390 2260static inline void kvm_make_request_and_kick(int req, struct kvm_vcpu *vcpu) 2261{ 2262 kvm_make_request(req, vcpu); 2263 __kvm_vcpu_kick(vcpu, req & KVM_REQUEST_WAIT); 2264} 2265#endif 2266 2267static inline bool kvm_request_pending(struct kvm_vcpu *vcpu) 2268{ 2269 return READ_ONCE(vcpu->requests); 2270} 2271 2272static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu) 2273{ 2274 return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 2275} 2276 2277static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) 2278{ 2279 clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); 2280} 2281 2282static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) 2283{ 2284 if (kvm_test_request(req, vcpu)) { 2285 kvm_clear_request(req, vcpu); 2286 2287 /* 2288 * Ensure the rest of the request is visible to kvm_check_request's 2289 * caller. Paired with the smp_wmb in kvm_make_request. 2290 */ 2291 smp_mb__after_atomic(); 2292 return true; 2293 } else { 2294 return false; 2295 } 2296} 2297 2298#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 2299extern bool enable_virt_at_load; 2300extern bool kvm_rebooting; 2301#endif 2302 2303extern unsigned int halt_poll_ns; 2304extern unsigned int halt_poll_ns_grow; 2305extern unsigned int halt_poll_ns_grow_start; 2306extern unsigned int halt_poll_ns_shrink; 2307 2308struct kvm_device { 2309 const struct kvm_device_ops *ops; 2310 struct kvm *kvm; 2311 void *private; 2312 struct list_head vm_node; 2313}; 2314 2315/* create, destroy, and name are mandatory */ 2316struct kvm_device_ops { 2317 const char *name; 2318 2319 /* 2320 * create is called holding kvm->lock and any operations not suitable 2321 * to do while holding the lock should be deferred to init (see 2322 * below). 2323 */ 2324 int (*create)(struct kvm_device *dev, u32 type); 2325 2326 /* 2327 * init is called after create if create is successful and is called 2328 * outside of holding kvm->lock. 2329 */ 2330 void (*init)(struct kvm_device *dev); 2331 2332 /* 2333 * Destroy is responsible for freeing dev. 2334 * 2335 * Destroy may be called before or after destructors are called 2336 * on emulated I/O regions, depending on whether a reference is 2337 * held by a vcpu or other kvm component that gets destroyed 2338 * after the emulated I/O. 2339 */ 2340 void (*destroy)(struct kvm_device *dev); 2341 2342 /* 2343 * Release is an alternative method to free the device. It is 2344 * called when the device file descriptor is closed. Once 2345 * release is called, the destroy method will not be called 2346 * anymore as the device is removed from the device list of 2347 * the VM. kvm->lock is held. 2348 */ 2349 void (*release)(struct kvm_device *dev); 2350 2351 int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 2352 int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 2353 int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr); 2354 long (*ioctl)(struct kvm_device *dev, unsigned int ioctl, 2355 unsigned long arg); 2356 int (*mmap)(struct kvm_device *dev, struct vm_area_struct *vma); 2357}; 2358 2359struct kvm_device *kvm_device_from_filp(struct file *filp); 2360int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type); 2361void kvm_unregister_device_ops(u32 type); 2362 2363extern struct kvm_device_ops kvm_mpic_ops; 2364extern struct kvm_device_ops kvm_arm_vgic_v2_ops; 2365extern struct kvm_device_ops kvm_arm_vgic_v3_ops; 2366 2367#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT 2368 2369static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 2370{ 2371 vcpu->spin_loop.in_spin_loop = val; 2372} 2373static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 2374{ 2375 vcpu->spin_loop.dy_eligible = val; 2376} 2377 2378#else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 2379 2380static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val) 2381{ 2382} 2383 2384static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) 2385{ 2386} 2387#endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ 2388 2389static inline bool kvm_is_visible_memslot(struct kvm_memory_slot *memslot) 2390{ 2391 return (memslot && memslot->id < KVM_USER_MEM_SLOTS && 2392 !(memslot->flags & KVM_MEMSLOT_INVALID)); 2393} 2394 2395struct kvm_vcpu *kvm_get_running_vcpu(void); 2396struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); 2397 2398#if IS_ENABLED(CONFIG_HAVE_KVM_IRQ_BYPASS) 2399struct kvm_kernel_irqfd; 2400 2401bool kvm_arch_has_irq_bypass(void); 2402int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *, 2403 struct irq_bypass_producer *); 2404void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *, 2405 struct irq_bypass_producer *); 2406void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *); 2407void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *); 2408void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, 2409 struct kvm_kernel_irq_routing_entry *old, 2410 struct kvm_kernel_irq_routing_entry *new); 2411#endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ 2412 2413#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS 2414/* If we wakeup during the poll time, was it a sucessful poll? */ 2415static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 2416{ 2417 return vcpu->valid_wakeup; 2418} 2419 2420#else 2421static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) 2422{ 2423 return true; 2424} 2425#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ 2426 2427#ifdef CONFIG_HAVE_KVM_NO_POLL 2428/* Callback that tells if we must not poll */ 2429bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); 2430#else 2431static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 2432{ 2433 return false; 2434} 2435#endif /* CONFIG_HAVE_KVM_NO_POLL */ 2436 2437void kvm_arch_guest_memory_reclaimed(struct kvm *kvm); 2438 2439#ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE 2440int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); 2441#else 2442static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) 2443{ 2444 return 0; 2445} 2446#endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ 2447 2448#ifdef CONFIG_VIRT_XFER_TO_GUEST_WORK 2449static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) 2450{ 2451 vcpu->run->exit_reason = KVM_EXIT_INTR; 2452 vcpu->stat.signal_exits++; 2453} 2454 2455static inline int kvm_xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu) 2456{ 2457 int r = xfer_to_guest_mode_handle_work(); 2458 2459 if (r) { 2460 WARN_ON_ONCE(r != -EINTR); 2461 kvm_handle_signal_exit(vcpu); 2462 } 2463 return r; 2464} 2465#endif /* CONFIG_VIRT_XFER_TO_GUEST_WORK */ 2466 2467/* 2468 * If more than one page is being (un)accounted, @virt must be the address of 2469 * the first page of a block of pages what were allocated together (i.e 2470 * accounted together). 2471 * 2472 * kvm_account_pgtable_pages() is thread-safe because mod_lruvec_page_state() 2473 * is thread-safe. 2474 */ 2475static inline void kvm_account_pgtable_pages(void *virt, int nr) 2476{ 2477 mod_lruvec_page_state(virt_to_page(virt), NR_SECONDARY_PAGETABLE, nr); 2478} 2479 2480/* 2481 * This defines how many reserved entries we want to keep before we 2482 * kick the vcpu to the userspace to avoid dirty ring full. This 2483 * value can be tuned to higher if e.g. PML is enabled on the host. 2484 */ 2485#define KVM_DIRTY_RING_RSVD_ENTRIES 64 2486 2487/* Max number of entries allowed for each kvm dirty ring */ 2488#define KVM_DIRTY_RING_MAX_ENTRIES 65536 2489 2490static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu, 2491 gpa_t gpa, gpa_t size, 2492 bool is_write, bool is_exec, 2493 bool is_private) 2494{ 2495 vcpu->run->exit_reason = KVM_EXIT_MEMORY_FAULT; 2496 vcpu->run->memory_fault.gpa = gpa; 2497 vcpu->run->memory_fault.size = size; 2498 2499 /* RWX flags are not (yet) defined or communicated to userspace. */ 2500 vcpu->run->memory_fault.flags = 0; 2501 if (is_private) 2502 vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_PRIVATE; 2503} 2504 2505static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) 2506{ 2507 if (!IS_ENABLED(CONFIG_KVM_GUEST_MEMFD)) 2508 return false; 2509 2510 return slot->flags & KVM_MEMSLOT_GMEM_ONLY; 2511} 2512 2513#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES 2514static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) 2515{ 2516 return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); 2517} 2518 2519bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, 2520 unsigned long mask, unsigned long attrs); 2521bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, 2522 struct kvm_gfn_range *range); 2523bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, 2524 struct kvm_gfn_range *range); 2525 2526static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) 2527{ 2528 return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; 2529} 2530#else 2531static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) 2532{ 2533 return false; 2534} 2535#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ 2536 2537#ifdef CONFIG_KVM_GUEST_MEMFD 2538int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, 2539 gfn_t gfn, kvm_pfn_t *pfn, struct page **page, 2540 int *max_order); 2541#else 2542static inline int kvm_gmem_get_pfn(struct kvm *kvm, 2543 struct kvm_memory_slot *slot, gfn_t gfn, 2544 kvm_pfn_t *pfn, struct page **page, 2545 int *max_order) 2546{ 2547 KVM_BUG_ON(1, kvm); 2548 return -EIO; 2549} 2550#endif /* CONFIG_KVM_GUEST_MEMFD */ 2551 2552#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE 2553int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order); 2554#endif 2555 2556#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE 2557/** 2558 * kvm_gmem_populate() - Populate/prepare a GPA range with guest data 2559 * 2560 * @kvm: KVM instance 2561 * @gfn: starting GFN to be populated 2562 * @src: userspace-provided buffer containing data to copy into GFN range 2563 * (passed to @post_populate, and incremented on each iteration 2564 * if not NULL) 2565 * @npages: number of pages to copy from userspace-buffer 2566 * @post_populate: callback to issue for each gmem page that backs the GPA 2567 * range 2568 * @opaque: opaque data to pass to @post_populate callback 2569 * 2570 * This is primarily intended for cases where a gmem-backed GPA range needs 2571 * to be initialized with userspace-provided data prior to being mapped into 2572 * the guest as a private page. This should be called with the slots->lock 2573 * held so that caller-enforced invariants regarding the expected memory 2574 * attributes of the GPA range do not race with KVM_SET_MEMORY_ATTRIBUTES. 2575 * 2576 * Returns the number of pages that were populated. 2577 */ 2578typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 2579 void __user *src, int order, void *opaque); 2580 2581long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, 2582 kvm_gmem_populate_cb post_populate, void *opaque); 2583#endif 2584 2585#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE 2586void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); 2587#endif 2588 2589#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY 2590long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, 2591 struct kvm_pre_fault_memory *range); 2592#endif 2593 2594#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING 2595int kvm_enable_virtualization(void); 2596void kvm_disable_virtualization(void); 2597#else 2598static inline int kvm_enable_virtualization(void) { return 0; } 2599static inline void kvm_disable_virtualization(void) { } 2600#endif 2601 2602#endif
~/
Read-only file preview.