kvm_util.h

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (C) 2018, Google LLC.
   4 */
   5#ifndef SELFTEST_KVM_UTIL_H
   6#define SELFTEST_KVM_UTIL_H
   7
   8#include "test_util.h"
   9
  10#include <linux/compiler.h>
  11#include "linux/hashtable.h"
  12#include "linux/list.h"
  13#include <linux/kernel.h>
  14#include <linux/kvm.h>
  15#include "linux/rbtree.h"
  16#include <linux/types.h>
  17
  18#include <asm/atomic.h>
  19#include <asm/kvm.h>
  20
  21#include <sys/eventfd.h>
  22#include <sys/ioctl.h>
  23
  24#include <pthread.h>
  25
  26#include "kvm_syscalls.h"
  27#include "kvm_util_arch.h"
  28#include "kvm_util_types.h"
  29#include "sparsebit.h"
  30
  31#define KVM_DEV_PATH "/dev/kvm"
  32#define KVM_MAX_VCPUS 512
  33
  34#define NSEC_PER_SEC 1000000000L
  35
  36struct userspace_mem_region {
  37	struct kvm_userspace_memory_region2 region;
  38	struct sparsebit *unused_phy_pages;
  39	struct sparsebit *protected_phy_pages;
  40	int fd;
  41	off_t offset;
  42	enum vm_mem_backing_src_type backing_src_type;
  43	void *host_mem;
  44	void *host_alias;
  45	void *mmap_start;
  46	void *mmap_alias;
  47	size_t mmap_size;
  48	struct rb_node gpa_node;
  49	struct rb_node hva_node;
  50	struct hlist_node slot_node;
  51};
  52
  53struct kvm_binary_stats {
  54	int fd;
  55	struct kvm_stats_header header;
  56	struct kvm_stats_desc *desc;
  57};
  58
  59struct kvm_vcpu {
  60	struct list_head list;
  61	uint32_t id;
  62	int fd;
  63	struct kvm_vm *vm;
  64	struct kvm_run *run;
  65#ifdef __x86_64__
  66	struct kvm_cpuid2 *cpuid;
  67#endif
  68#ifdef __aarch64__
  69	struct kvm_vcpu_init init;
  70#endif
  71	struct kvm_binary_stats stats;
  72	struct kvm_dirty_gfn *dirty_gfns;
  73	uint32_t fetch_index;
  74	uint32_t dirty_gfns_count;
  75};
  76
  77struct userspace_mem_regions {
  78	struct rb_root gpa_tree;
  79	struct rb_root hva_tree;
  80	DECLARE_HASHTABLE(slot_hash, 9);
  81};
  82
  83enum kvm_mem_region_type {
  84	MEM_REGION_CODE,
  85	MEM_REGION_DATA,
  86	MEM_REGION_PT,
  87	MEM_REGION_TEST_DATA,
  88	NR_MEM_REGIONS,
  89};
  90
  91struct kvm_vm {
  92	int mode;
  93	unsigned long type;
  94	int kvm_fd;
  95	int fd;
  96	unsigned int pgtable_levels;
  97	unsigned int page_size;
  98	unsigned int page_shift;
  99	unsigned int pa_bits;
 100	unsigned int va_bits;
 101	uint64_t max_gfn;
 102	struct list_head vcpus;
 103	struct userspace_mem_regions regions;
 104	struct sparsebit *vpages_valid;
 105	struct sparsebit *vpages_mapped;
 106	bool has_irqchip;
 107	bool pgd_created;
 108	vm_paddr_t ucall_mmio_addr;
 109	vm_paddr_t pgd;
 110	vm_vaddr_t handlers;
 111	uint32_t dirty_ring_size;
 112	uint64_t gpa_tag_mask;
 113
 114	struct kvm_vm_arch arch;
 115
 116	struct kvm_binary_stats stats;
 117
 118	/*
 119	 * KVM region slots. These are the default memslots used by page
 120	 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
 121	 * memslot.
 122	 */
 123	uint32_t memslots[NR_MEM_REGIONS];
 124};
 125
 126struct vcpu_reg_sublist {
 127	const char *name;
 128	long capability;
 129	int feature;
 130	int feature_type;
 131	bool finalize;
 132	__u64 *regs;
 133	__u64 regs_n;
 134	__u64 *rejects_set;
 135	__u64 rejects_set_n;
 136	__u64 *skips_set;
 137	__u64 skips_set_n;
 138};
 139
 140struct vcpu_reg_list {
 141	char *name;
 142	struct vcpu_reg_sublist sublists[];
 143};
 144
 145#define for_each_sublist(c, s)		\
 146	for ((s) = &(c)->sublists[0]; (s)->regs; ++(s))
 147
 148#define kvm_for_each_vcpu(vm, i, vcpu)			\
 149	for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++)	\
 150		if (!((vcpu) = vm->vcpus[i]))		\
 151			continue;			\
 152		else
 153
 154struct userspace_mem_region *
 155memslot2region(struct kvm_vm *vm, uint32_t memslot);
 156
 157static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
 158							     enum kvm_mem_region_type type)
 159{
 160	assert(type < NR_MEM_REGIONS);
 161	return memslot2region(vm, vm->memslots[type]);
 162}
 163
 164/* Minimum allocated guest virtual and physical addresses */
 165#define KVM_UTIL_MIN_VADDR		0x2000
 166#define KVM_GUEST_PAGE_TABLE_MIN_PADDR	0x180000
 167
 168#define DEFAULT_GUEST_STACK_VADDR_MIN	0xab6000
 169#define DEFAULT_STACK_PGS		5
 170
 171enum vm_guest_mode {
 172	VM_MODE_P52V48_4K,
 173	VM_MODE_P52V48_16K,
 174	VM_MODE_P52V48_64K,
 175	VM_MODE_P48V48_4K,
 176	VM_MODE_P48V48_16K,
 177	VM_MODE_P48V48_64K,
 178	VM_MODE_P40V48_4K,
 179	VM_MODE_P40V48_16K,
 180	VM_MODE_P40V48_64K,
 181	VM_MODE_PXXVYY_4K,	/* For 48-bit or 57-bit VA, depending on host support */
 182	VM_MODE_P47V64_4K,
 183	VM_MODE_P44V64_4K,
 184	VM_MODE_P36V48_4K,
 185	VM_MODE_P36V48_16K,
 186	VM_MODE_P36V48_64K,
 187	VM_MODE_P47V47_16K,
 188	VM_MODE_P36V47_16K,
 189	NUM_VM_MODES,
 190};
 191
 192struct vm_shape {
 193	uint32_t type;
 194	uint8_t  mode;
 195	uint8_t  pad0;
 196	uint16_t pad1;
 197};
 198
 199kvm_static_assert(sizeof(struct vm_shape) == sizeof(uint64_t));
 200
 201#define VM_TYPE_DEFAULT			0
 202
 203#define VM_SHAPE(__mode)			\
 204({						\
 205	struct vm_shape shape = {		\
 206		.mode = (__mode),		\
 207		.type = VM_TYPE_DEFAULT		\
 208	};					\
 209						\
 210	shape;					\
 211})
 212
 213#if defined(__aarch64__)
 214
 215extern enum vm_guest_mode vm_mode_default;
 216
 217#define VM_MODE_DEFAULT			vm_mode_default
 218#define MIN_PAGE_SHIFT			12U
 219#define ptes_per_page(page_size)	((page_size) / 8)
 220
 221#elif defined(__x86_64__)
 222
 223#define VM_MODE_DEFAULT			VM_MODE_PXXVYY_4K
 224#define MIN_PAGE_SHIFT			12U
 225#define ptes_per_page(page_size)	((page_size) / 8)
 226
 227#elif defined(__s390x__)
 228
 229#define VM_MODE_DEFAULT			VM_MODE_P44V64_4K
 230#define MIN_PAGE_SHIFT			12U
 231#define ptes_per_page(page_size)	((page_size) / 16)
 232
 233#elif defined(__riscv)
 234
 235#if __riscv_xlen == 32
 236#error "RISC-V 32-bit kvm selftests not supported"
 237#endif
 238
 239#define VM_MODE_DEFAULT			VM_MODE_P40V48_4K
 240#define MIN_PAGE_SHIFT			12U
 241#define ptes_per_page(page_size)	((page_size) / 8)
 242
 243#elif defined(__loongarch__)
 244#define VM_MODE_DEFAULT			VM_MODE_P47V47_16K
 245#define MIN_PAGE_SHIFT			12U
 246#define ptes_per_page(page_size)	((page_size) / 8)
 247
 248#endif
 249
 250#define VM_SHAPE_DEFAULT	VM_SHAPE(VM_MODE_DEFAULT)
 251
 252#define MIN_PAGE_SIZE		(1U << MIN_PAGE_SHIFT)
 253#define PTES_PER_MIN_PAGE	ptes_per_page(MIN_PAGE_SIZE)
 254
 255struct vm_guest_mode_params {
 256	unsigned int pa_bits;
 257	unsigned int va_bits;
 258	unsigned int page_size;
 259	unsigned int page_shift;
 260};
 261extern const struct vm_guest_mode_params vm_guest_mode_params[];
 262
 263int __open_path_or_exit(const char *path, int flags, const char *enoent_help);
 264int open_path_or_exit(const char *path, int flags);
 265int open_kvm_dev_path_or_exit(void);
 266
 267int kvm_get_module_param_integer(const char *module_name, const char *param);
 268bool kvm_get_module_param_bool(const char *module_name, const char *param);
 269
 270static inline bool get_kvm_param_bool(const char *param)
 271{
 272	return kvm_get_module_param_bool("kvm", param);
 273}
 274
 275static inline int get_kvm_param_integer(const char *param)
 276{
 277	return kvm_get_module_param_integer("kvm", param);
 278}
 279
 280unsigned int kvm_check_cap(long cap);
 281
 282static inline bool kvm_has_cap(long cap)
 283{
 284	return kvm_check_cap(cap);
 285}
 286
 287/*
 288 * Use the "inner", double-underscore macro when reporting errors from within
 289 * other macros so that the name of ioctl() and not its literal numeric value
 290 * is printed on error.  The "outer" macro is strongly preferred when reporting
 291 * errors "directly", i.e. without an additional layer of macros, as it reduces
 292 * the probability of passing in the wrong string.
 293 */
 294#define __KVM_IOCTL_ERROR(_name, _ret)	__KVM_SYSCALL_ERROR(_name, _ret)
 295#define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret)
 296
 297#define kvm_do_ioctl(fd, cmd, arg)						\
 298({										\
 299	kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd));	\
 300	ioctl(fd, cmd, arg);							\
 301})
 302
 303#define __kvm_ioctl(kvm_fd, cmd, arg)				\
 304	kvm_do_ioctl(kvm_fd, cmd, arg)
 305
 306#define kvm_ioctl(kvm_fd, cmd, arg)				\
 307({								\
 308	int ret = __kvm_ioctl(kvm_fd, cmd, arg);		\
 309								\
 310	TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret));	\
 311})
 312
 313static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
 314
 315#define __vm_ioctl(vm, cmd, arg)				\
 316({								\
 317	static_assert_is_vm(vm);				\
 318	kvm_do_ioctl((vm)->fd, cmd, arg);			\
 319})
 320
 321/*
 322 * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if
 323 * the ioctl() failed because KVM killed/bugged the VM.  To detect a dead VM,
 324 * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before
 325 * selftests existed and (b) should never outright fail, i.e. is supposed to
 326 * return 0 or 1.  If KVM kills a VM, KVM returns -EIO for all ioctl()s for the
 327 * VM and its vCPUs, including KVM_CHECK_EXTENSION.
 328 */
 329#define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm)				\
 330do {											\
 331	int __errno = errno;								\
 332											\
 333	static_assert_is_vm(vm);							\
 334											\
 335	if (cond)									\
 336		break;									\
 337											\
 338	if (errno == EIO &&								\
 339	    __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) {	\
 340		TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO");	\
 341		TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues");	\
 342	}										\
 343	errno = __errno;								\
 344	TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret));				\
 345} while (0)
 346
 347#define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm)		\
 348	__TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm)
 349
 350#define vm_ioctl(vm, cmd, arg)					\
 351({								\
 352	int ret = __vm_ioctl(vm, cmd, arg);			\
 353								\
 354	__TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm);		\
 355})
 356
 357static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
 358
 359#define __vcpu_ioctl(vcpu, cmd, arg)				\
 360({								\
 361	static_assert_is_vcpu(vcpu);				\
 362	kvm_do_ioctl((vcpu)->fd, cmd, arg);			\
 363})
 364
 365#define vcpu_ioctl(vcpu, cmd, arg)				\
 366({								\
 367	int ret = __vcpu_ioctl(vcpu, cmd, arg);			\
 368								\
 369	__TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm);	\
 370})
 371
 372/*
 373 * Looks up and returns the value corresponding to the capability
 374 * (KVM_CAP_*) given by cap.
 375 */
 376static inline int vm_check_cap(struct kvm_vm *vm, long cap)
 377{
 378	int ret =  __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap);
 379
 380	TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm);
 381	return ret;
 382}
 383
 384static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 385{
 386	struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 387
 388	return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 389}
 390static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 391{
 392	struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 393
 394	vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 395}
 396
 397static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,
 398					    uint64_t size, uint64_t attributes)
 399{
 400	struct kvm_memory_attributes attr = {
 401		.attributes = attributes,
 402		.address = gpa,
 403		.size = size,
 404		.flags = 0,
 405	};
 406
 407	/*
 408	 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes.  These flows
 409	 * need significant enhancements to support multiple attributes.
 410	 */
 411	TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE,
 412		    "Update me to support multiple attributes!");
 413
 414	vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr);
 415}
 416
 417
 418static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa,
 419				      uint64_t size)
 420{
 421	vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
 422}
 423
 424static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa,
 425				     uint64_t size)
 426{
 427	vm_set_memory_attributes(vm, gpa, size, 0);
 428}
 429
 430void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size,
 431			    bool punch_hole);
 432
 433static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa,
 434					   uint64_t size)
 435{
 436	vm_guest_mem_fallocate(vm, gpa, size, true);
 437}
 438
 439static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa,
 440					 uint64_t size)
 441{
 442	vm_guest_mem_fallocate(vm, gpa, size, false);
 443}
 444
 445void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);
 446const char *vm_guest_mode_string(uint32_t i);
 447
 448void kvm_vm_free(struct kvm_vm *vmp);
 449void kvm_vm_restart(struct kvm_vm *vmp);
 450void kvm_vm_release(struct kvm_vm *vmp);
 451void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);
 452int kvm_memfd_alloc(size_t size, bool hugepages);
 453
 454void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
 455
 456static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
 457{
 458	struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
 459
 460	vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args);
 461}
 462
 463static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
 464					  uint64_t first_page, uint32_t num_pages)
 465{
 466	struct kvm_clear_dirty_log args = {
 467		.dirty_bitmap = log,
 468		.slot = slot,
 469		.first_page = first_page,
 470		.num_pages = num_pages
 471	};
 472
 473	vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args);
 474}
 475
 476static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
 477{
 478	return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL);
 479}
 480
 481static inline void kvm_vm_register_coalesced_io(struct kvm_vm *vm,
 482						uint64_t address,
 483						uint64_t size, bool pio)
 484{
 485	struct kvm_coalesced_mmio_zone zone = {
 486		.addr = address,
 487		.size = size,
 488		.pio  = pio,
 489	};
 490
 491	vm_ioctl(vm, KVM_REGISTER_COALESCED_MMIO, &zone);
 492}
 493
 494static inline void kvm_vm_unregister_coalesced_io(struct kvm_vm *vm,
 495						  uint64_t address,
 496						  uint64_t size, bool pio)
 497{
 498	struct kvm_coalesced_mmio_zone zone = {
 499		.addr = address,
 500		.size = size,
 501		.pio  = pio,
 502	};
 503
 504	vm_ioctl(vm, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 505}
 506
 507static inline int vm_get_stats_fd(struct kvm_vm *vm)
 508{
 509	int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL);
 510
 511	TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm);
 512	return fd;
 513}
 514
 515static inline int __kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
 516			      uint32_t flags)
 517{
 518	struct kvm_irqfd irqfd = {
 519		.fd = eventfd,
 520		.gsi = gsi,
 521		.flags = flags,
 522		.resamplefd = -1,
 523	};
 524
 525	return __vm_ioctl(vm, KVM_IRQFD, &irqfd);
 526}
 527
 528static inline void kvm_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd,
 529			      uint32_t flags)
 530{
 531	int ret = __kvm_irqfd(vm, gsi, eventfd, flags);
 532
 533	TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_IRQFD, ret, vm);
 534}
 535
 536static inline void kvm_assign_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd)
 537{
 538	kvm_irqfd(vm, gsi, eventfd, 0);
 539}
 540
 541static inline void kvm_deassign_irqfd(struct kvm_vm *vm, uint32_t gsi, int eventfd)
 542{
 543	kvm_irqfd(vm, gsi, eventfd, KVM_IRQFD_FLAG_DEASSIGN);
 544}
 545
 546static inline int kvm_new_eventfd(void)
 547{
 548	int fd = eventfd(0, 0);
 549
 550	TEST_ASSERT(fd >= 0, __KVM_SYSCALL_ERROR("eventfd()", fd));
 551	return fd;
 552}
 553
 554static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header)
 555{
 556	ssize_t ret;
 557
 558	ret = pread(stats_fd, header, sizeof(*header), 0);
 559	TEST_ASSERT(ret == sizeof(*header),
 560		    "Failed to read '%lu' header bytes, ret = '%ld'",
 561		    sizeof(*header), ret);
 562}
 563
 564struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
 565					      struct kvm_stats_header *header);
 566
 567static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header)
 568{
 569	 /*
 570	  * The base size of the descriptor is defined by KVM's ABI, but the
 571	  * size of the name field is variable, as far as KVM's ABI is
 572	  * concerned. For a given instance of KVM, the name field is the same
 573	  * size for all stats and is provided in the overall stats header.
 574	  */
 575	return sizeof(struct kvm_stats_desc) + header->name_size;
 576}
 577
 578static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats,
 579							  int index,
 580							  struct kvm_stats_header *header)
 581{
 582	/*
 583	 * Note, size_desc includes the size of the name field, which is
 584	 * variable. i.e. this is NOT equivalent to &stats_desc[i].
 585	 */
 586	return (void *)stats + index * get_stats_descriptor_size(header);
 587}
 588
 589void read_stat_data(int stats_fd, struct kvm_stats_header *header,
 590		    struct kvm_stats_desc *desc, uint64_t *data,
 591		    size_t max_elements);
 592
 593void kvm_get_stat(struct kvm_binary_stats *stats, const char *name,
 594		  uint64_t *data, size_t max_elements);
 595
 596#define __get_stat(stats, stat)							\
 597({										\
 598	uint64_t data;								\
 599										\
 600	kvm_get_stat(stats, #stat, &data, 1);					\
 601	data;									\
 602})
 603
 604#define vm_get_stat(vm, stat) __get_stat(&(vm)->stats, stat)
 605#define vcpu_get_stat(vcpu, stat) __get_stat(&(vcpu)->stats, stat)
 606
 607static inline bool read_smt_control(char *buf, size_t buf_size)
 608{
 609	FILE *f = fopen("/sys/devices/system/cpu/smt/control", "r");
 610	bool ret;
 611
 612	if (!f)
 613		return false;
 614
 615	ret = fread(buf, sizeof(*buf), buf_size, f) > 0;
 616	fclose(f);
 617
 618	return ret;
 619}
 620
 621static inline bool is_smt_possible(void)
 622{
 623	char buf[16];
 624
 625	if (read_smt_control(buf, sizeof(buf)) &&
 626	    (!strncmp(buf, "forceoff", 8) || !strncmp(buf, "notsupported", 12)))
 627		return false;
 628
 629	return true;
 630}
 631
 632static inline bool is_smt_on(void)
 633{
 634	char buf[16];
 635
 636	if (read_smt_control(buf, sizeof(buf)) && !strncmp(buf, "on", 2))
 637		return true;
 638
 639	return false;
 640}
 641
 642void vm_create_irqchip(struct kvm_vm *vm);
 643
 644static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
 645					uint64_t flags)
 646{
 647	struct kvm_create_guest_memfd guest_memfd = {
 648		.size = size,
 649		.flags = flags,
 650	};
 651
 652	return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
 653}
 654
 655static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
 656					uint64_t flags)
 657{
 658	int fd = __vm_create_guest_memfd(vm, size, flags);
 659
 660	TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd));
 661	return fd;
 662}
 663
 664void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 665			       uint64_t gpa, uint64_t size, void *hva);
 666int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 667				uint64_t gpa, uint64_t size, void *hva);
 668void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 669				uint64_t gpa, uint64_t size, void *hva,
 670				uint32_t guest_memfd, uint64_t guest_memfd_offset);
 671int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 672				 uint64_t gpa, uint64_t size, void *hva,
 673				 uint32_t guest_memfd, uint64_t guest_memfd_offset);
 674
 675void vm_userspace_mem_region_add(struct kvm_vm *vm,
 676				 enum vm_mem_backing_src_type src_type,
 677				 uint64_t gpa, uint32_t slot, uint64_t npages,
 678				 uint32_t flags);
 679void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
 680		uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags,
 681		int guest_memfd_fd, uint64_t guest_memfd_offset);
 682
 683#ifndef vm_arch_has_protected_memory
 684static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
 685{
 686	return false;
 687}
 688#endif
 689
 690void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
 691void vm_mem_region_reload(struct kvm_vm *vm, uint32_t slot);
 692void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
 693void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);
 694struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
 695void vm_populate_vaddr_bitmap(struct kvm_vm *vm);
 696vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
 697vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
 698vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
 699			    enum kvm_mem_region_type type);
 700vm_vaddr_t vm_vaddr_alloc_shared(struct kvm_vm *vm, size_t sz,
 701				 vm_vaddr_t vaddr_min,
 702				 enum kvm_mem_region_type type);
 703vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
 704vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
 705				 enum kvm_mem_region_type type);
 706vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
 707
 708void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 709	      unsigned int npages);
 710void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa);
 711void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva);
 712vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
 713void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa);
 714
 715#ifndef vcpu_arch_put_guest
 716#define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0)
 717#endif
 718
 719static inline vm_paddr_t vm_untag_gpa(struct kvm_vm *vm, vm_paddr_t gpa)
 720{
 721	return gpa & ~vm->gpa_tag_mask;
 722}
 723
 724void vcpu_run(struct kvm_vcpu *vcpu);
 725int _vcpu_run(struct kvm_vcpu *vcpu);
 726
 727static inline int __vcpu_run(struct kvm_vcpu *vcpu)
 728{
 729	return __vcpu_ioctl(vcpu, KVM_RUN, NULL);
 730}
 731
 732void vcpu_run_complete_io(struct kvm_vcpu *vcpu);
 733struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu);
 734
 735static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap,
 736				   uint64_t arg0)
 737{
 738	struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 739
 740	vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap);
 741}
 742
 743static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu,
 744					struct kvm_guest_debug *debug)
 745{
 746	vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug);
 747}
 748
 749static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu,
 750				     struct kvm_mp_state *mp_state)
 751{
 752	vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state);
 753}
 754static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu,
 755				     struct kvm_mp_state *mp_state)
 756{
 757	vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state);
 758}
 759
 760static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 761{
 762	vcpu_ioctl(vcpu, KVM_GET_REGS, regs);
 763}
 764
 765static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 766{
 767	vcpu_ioctl(vcpu, KVM_SET_REGS, regs);
 768}
 769static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 770{
 771	vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs);
 772
 773}
 774static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 775{
 776	vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
 777}
 778static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 779{
 780	return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
 781}
 782static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 783{
 784	vcpu_ioctl(vcpu, KVM_GET_FPU, fpu);
 785}
 786static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 787{
 788	vcpu_ioctl(vcpu, KVM_SET_FPU, fpu);
 789}
 790
 791static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
 792{
 793	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
 794
 795	return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
 796}
 797static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
 798{
 799	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 800
 801	return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
 802}
 803static inline uint64_t vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id)
 804{
 805	uint64_t val;
 806	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 807
 808	TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id);
 809
 810	vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
 811	return val;
 812}
 813static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
 814{
 815	struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 816
 817	TEST_ASSERT(KVM_REG_SIZE(id) <= sizeof(val), "Reg %lx too big", id);
 818
 819	vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
 820}
 821
 822#ifdef __KVM_HAVE_VCPU_EVENTS
 823static inline void vcpu_events_get(struct kvm_vcpu *vcpu,
 824				   struct kvm_vcpu_events *events)
 825{
 826	vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events);
 827}
 828static inline void vcpu_events_set(struct kvm_vcpu *vcpu,
 829				   struct kvm_vcpu_events *events)
 830{
 831	vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events);
 832}
 833#endif
 834#ifdef __x86_64__
 835static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu,
 836					 struct kvm_nested_state *state)
 837{
 838	vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state);
 839}
 840static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu,
 841					  struct kvm_nested_state *state)
 842{
 843	return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
 844}
 845
 846static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu,
 847					 struct kvm_nested_state *state)
 848{
 849	vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
 850}
 851#endif
 852static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu)
 853{
 854	int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL);
 855
 856	TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm);
 857	return fd;
 858}
 859
 860int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr);
 861
 862static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
 863{
 864	int ret = __kvm_has_device_attr(dev_fd, group, attr);
 865
 866	TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
 867}
 868
 869int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val);
 870
 871static inline void kvm_device_attr_get(int dev_fd, uint32_t group,
 872				       uint64_t attr, void *val)
 873{
 874	int ret = __kvm_device_attr_get(dev_fd, group, attr, val);
 875
 876	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret));
 877}
 878
 879int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val);
 880
 881static inline void kvm_device_attr_set(int dev_fd, uint32_t group,
 882				       uint64_t attr, void *val)
 883{
 884	int ret = __kvm_device_attr_set(dev_fd, group, attr, val);
 885
 886	TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret));
 887}
 888
 889static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
 890					 uint64_t attr)
 891{
 892	return __kvm_has_device_attr(vcpu->fd, group, attr);
 893}
 894
 895static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
 896					uint64_t attr)
 897{
 898	kvm_has_device_attr(vcpu->fd, group, attr);
 899}
 900
 901static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
 902					 uint64_t attr, void *val)
 903{
 904	return __kvm_device_attr_get(vcpu->fd, group, attr, val);
 905}
 906
 907static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
 908					uint64_t attr, void *val)
 909{
 910	kvm_device_attr_get(vcpu->fd, group, attr, val);
 911}
 912
 913static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
 914					 uint64_t attr, void *val)
 915{
 916	return __kvm_device_attr_set(vcpu->fd, group, attr, val);
 917}
 918
 919static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
 920					uint64_t attr, void *val)
 921{
 922	kvm_device_attr_set(vcpu->fd, group, attr, val);
 923}
 924
 925int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type);
 926int __kvm_create_device(struct kvm_vm *vm, uint64_t type);
 927
 928static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type)
 929{
 930	int fd = __kvm_create_device(vm, type);
 931
 932	TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd));
 933	return fd;
 934}
 935
 936void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);
 937
 938/*
 939 * VM VCPU Args Set
 940 *
 941 * Input Args:
 942 *   vm - Virtual Machine
 943 *   num - number of arguments
 944 *   ... - arguments, each of type uint64_t
 945 *
 946 * Output Args: None
 947 *
 948 * Return: None
 949 *
 950 * Sets the first @num input parameters for the function at @vcpu's entry point,
 951 * per the C calling convention of the architecture, to the values given as
 952 * variable args. Each of the variable args is expected to be of type uint64_t.
 953 * The maximum @num can be is specific to the architecture.
 954 */
 955void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);
 956
 957void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 958int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 959
 960#define KVM_MAX_IRQ_ROUTES		4096
 961
 962struct kvm_irq_routing *kvm_gsi_routing_create(void);
 963void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
 964		uint32_t gsi, uint32_t pin);
 965int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 966void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 967
 968const char *exit_reason_str(unsigned int exit_reason);
 969
 970vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
 971			     uint32_t memslot);
 972vm_paddr_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
 973				vm_paddr_t paddr_min, uint32_t memslot,
 974				bool protected);
 975vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);
 976
 977static inline vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
 978					    vm_paddr_t paddr_min, uint32_t memslot)
 979{
 980	/*
 981	 * By default, allocate memory as protected for VMs that support
 982	 * protected memory, as the majority of memory for such VMs is
 983	 * protected, i.e. using shared memory is effectively opt-in.
 984	 */
 985	return __vm_phy_pages_alloc(vm, num, paddr_min, memslot,
 986				    vm_arch_has_protected_memory(vm));
 987}
 988
 989/*
 990 * ____vm_create() does KVM_CREATE_VM and little else.  __vm_create() also
 991 * loads the test binary into guest memory and creates an IRQ chip (x86 only).
 992 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
 993 * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
 994 */
 995struct kvm_vm *____vm_create(struct vm_shape shape);
 996struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
 997			   uint64_t nr_extra_pages);
 998
 999static inline struct kvm_vm *vm_create_barebones(void)
1000{
1001	return ____vm_create(VM_SHAPE_DEFAULT);
1002}
1003
1004static inline struct kvm_vm *vm_create_barebones_type(unsigned long type)
1005{
1006	const struct vm_shape shape = {
1007		.mode = VM_MODE_DEFAULT,
1008		.type = type,
1009	};
1010
1011	return ____vm_create(shape);
1012}
1013
1014static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
1015{
1016	return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0);
1017}
1018
1019struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
1020				      uint64_t extra_mem_pages,
1021				      void *guest_code, struct kvm_vcpu *vcpus[]);
1022
1023static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
1024						  void *guest_code,
1025						  struct kvm_vcpu *vcpus[])
1026{
1027	return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0,
1028				      guest_code, vcpus);
1029}
1030
1031
1032struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
1033					       struct kvm_vcpu **vcpu,
1034					       uint64_t extra_mem_pages,
1035					       void *guest_code);
1036
1037/*
1038 * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages
1039 * additional pages of guest memory.  Returns the VM and vCPU (via out param).
1040 */
1041static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
1042						       uint64_t extra_mem_pages,
1043						       void *guest_code)
1044{
1045	return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu,
1046					       extra_mem_pages, guest_code);
1047}
1048
1049static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
1050						     void *guest_code)
1051{
1052	return __vm_create_with_one_vcpu(vcpu, 0, guest_code);
1053}
1054
1055static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape,
1056							   struct kvm_vcpu **vcpu,
1057							   void *guest_code)
1058{
1059	return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code);
1060}
1061
1062struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);
1063
1064void kvm_set_files_rlimit(uint32_t nr_vcpus);
1065
1066int __pin_task_to_cpu(pthread_t task, int cpu);
1067
1068static inline void pin_task_to_cpu(pthread_t task, int cpu)
1069{
1070	int r;
1071
1072	r = __pin_task_to_cpu(task, cpu);
1073	TEST_ASSERT(!r, "Failed to set thread affinity to pCPU '%u'", cpu);
1074}
1075
1076static inline int pin_task_to_any_cpu(pthread_t task)
1077{
1078	int cpu = sched_getcpu();
1079
1080	pin_task_to_cpu(task, cpu);
1081	return cpu;
1082}
1083
1084static inline void pin_self_to_cpu(int cpu)
1085{
1086	pin_task_to_cpu(pthread_self(), cpu);
1087}
1088
1089static inline int pin_self_to_any_cpu(void)
1090{
1091	return pin_task_to_any_cpu(pthread_self());
1092}
1093
1094void kvm_print_vcpu_pinning_help(void);
1095void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
1096			    int nr_vcpus);
1097
1098unsigned long vm_compute_max_gfn(struct kvm_vm *vm);
1099unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size);
1100unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages);
1101unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages);
1102static inline unsigned int
1103vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
1104{
1105	unsigned int n;
1106	n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages));
1107#ifdef __s390x__
1108	/* s390 requires 1M aligned guest sizes */
1109	n = (n + 255) & ~255;
1110#endif
1111	return n;
1112}
1113
1114#define sync_global_to_guest(vm, g) ({				\
1115	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
1116	memcpy(_p, &(g), sizeof(g));				\
1117})
1118
1119#define sync_global_from_guest(vm, g) ({			\
1120	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
1121	memcpy(&(g), _p, sizeof(g));				\
1122})
1123
1124/*
1125 * Write a global value, but only in the VM's (guest's) domain.  Primarily used
1126 * for "globals" that hold per-VM values (VMs always duplicate code and global
1127 * data into their own region of physical memory), but can be used anytime it's
1128 * undesirable to change the host's copy of the global.
1129 */
1130#define write_guest_global(vm, g, val) ({			\
1131	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
1132	typeof(g) _val = val;					\
1133								\
1134	memcpy(_p, &(_val), sizeof(g));				\
1135})
1136
1137void assert_on_unhandled_exception(struct kvm_vcpu *vcpu);
1138
1139void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu,
1140		    uint8_t indent);
1141
1142static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,
1143			     uint8_t indent)
1144{
1145	vcpu_arch_dump(stream, vcpu, indent);
1146}
1147
1148/*
1149 * Adds a vCPU with reasonable defaults (e.g. a stack)
1150 *
1151 * Input Args:
1152 *   vm - Virtual Machine
1153 *   vcpu_id - The id of the VCPU to add to the VM.
1154 */
1155struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
1156void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code);
1157
1158static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
1159					   void *guest_code)
1160{
1161	struct kvm_vcpu *vcpu = vm_arch_vcpu_add(vm, vcpu_id);
1162
1163	vcpu_arch_set_entry_point(vcpu, guest_code);
1164
1165	return vcpu;
1166}
1167
1168/* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */
1169struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id);
1170
1171static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm,
1172						uint32_t vcpu_id)
1173{
1174	return vm_arch_vcpu_recreate(vm, vcpu_id);
1175}
1176
1177void vcpu_arch_free(struct kvm_vcpu *vcpu);
1178
1179void virt_arch_pgd_alloc(struct kvm_vm *vm);
1180
1181static inline void virt_pgd_alloc(struct kvm_vm *vm)
1182{
1183	virt_arch_pgd_alloc(vm);
1184}
1185
1186/*
1187 * VM Virtual Page Map
1188 *
1189 * Input Args:
1190 *   vm - Virtual Machine
1191 *   vaddr - VM Virtual Address
1192 *   paddr - VM Physical Address
1193 *   memslot - Memory region slot for new virtual translation tables
1194 *
1195 * Output Args: None
1196 *
1197 * Return: None
1198 *
1199 * Within @vm, creates a virtual translation for the page starting
1200 * at @vaddr to the page starting at @paddr.
1201 */
1202void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr);
1203
1204static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
1205{
1206	virt_arch_pg_map(vm, vaddr, paddr);
1207	sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift);
1208}
1209
1210
1211/*
1212 * Address Guest Virtual to Guest Physical
1213 *
1214 * Input Args:
1215 *   vm - Virtual Machine
1216 *   gva - VM virtual address
1217 *
1218 * Output Args: None
1219 *
1220 * Return:
1221 *   Equivalent VM physical address
1222 *
1223 * Returns the VM physical address of the translated VM virtual
1224 * address given by @gva.
1225 */
1226vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva);
1227
1228static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
1229{
1230	return addr_arch_gva2gpa(vm, gva);
1231}
1232
1233/*
1234 * Virtual Translation Tables Dump
1235 *
1236 * Input Args:
1237 *   stream - Output FILE stream
1238 *   vm     - Virtual Machine
1239 *   indent - Left margin indent amount
1240 *
1241 * Output Args: None
1242 *
1243 * Return: None
1244 *
1245 * Dumps to the FILE stream given by @stream, the contents of all the
1246 * virtual translation tables for the VM given by @vm.
1247 */
1248void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
1249
1250static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1251{
1252	virt_arch_dump(stream, vm, indent);
1253}
1254
1255
1256static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm)
1257{
1258	return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0);
1259}
1260
1261/*
1262 * Arch hook that is invoked via a constructor, i.e. before exeucting main(),
1263 * to allow for arch-specific setup that is common to all tests, e.g. computing
1264 * the default guest "mode".
1265 */
1266void kvm_selftest_arch_init(void);
1267
1268void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus);
1269void kvm_arch_vm_finalize_vcpus(struct kvm_vm *vm);
1270void kvm_arch_vm_release(struct kvm_vm *vm);
1271
1272bool vm_is_gpa_protected(struct kvm_vm *vm, vm_paddr_t paddr);
1273
1274uint32_t guest_get_vcpuid(void);
1275
1276bool kvm_arch_has_default_irqchip(void);
1277
1278#endif /* SELFTEST_KVM_UTIL_H */