mshv_root_main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2024, Microsoft Corporation.
   4 *
   5 * The main part of the mshv_root module, providing APIs to create
   6 * and manage guest partitions.
   7 *
   8 * Authors: Microsoft Linux virtualization team
   9 */
  10
  11#include <linux/entry-virt.h>
  12#include <linux/kernel.h>
  13#include <linux/module.h>
  14#include <linux/fs.h>
  15#include <linux/miscdevice.h>
  16#include <linux/slab.h>
  17#include <linux/file.h>
  18#include <linux/anon_inodes.h>
  19#include <linux/mm.h>
  20#include <linux/io.h>
  21#include <linux/cpuhotplug.h>
  22#include <linux/random.h>
  23#include <asm/mshyperv.h>
  24#include <linux/hyperv.h>
  25#include <linux/notifier.h>
  26#include <linux/reboot.h>
  27#include <linux/kexec.h>
  28#include <linux/page-flags.h>
  29#include <linux/crash_dump.h>
  30#include <linux/panic_notifier.h>
  31#include <linux/vmalloc.h>
  32#include <linux/rseq.h>
  33
  34#include "mshv_eventfd.h"
  35#include "mshv.h"
  36#include "mshv_root.h"
  37
  38MODULE_AUTHOR("Microsoft");
  39MODULE_LICENSE("GPL");
  40MODULE_DESCRIPTION("Microsoft Hyper-V root partition VMM interface /dev/mshv");
  41
  42/* TODO move this to another file when debugfs code is added */
  43enum hv_stats_vp_counters {			/* HV_THREAD_COUNTER */
  44#if defined(CONFIG_X86)
  45	VpRootDispatchThreadBlocked			= 202,
  46#elif defined(CONFIG_ARM64)
  47	VpRootDispatchThreadBlocked			= 94,
  48#endif
  49	VpStatsMaxCounter
  50};
  51
  52struct hv_stats_page {
  53	union {
  54		u64 vp_cntrs[VpStatsMaxCounter];		/* VP counters */
  55		u8 data[HV_HYP_PAGE_SIZE];
  56	};
  57} __packed;
  58
  59struct mshv_root mshv_root;
  60
  61enum hv_scheduler_type hv_scheduler_type;
  62
  63/* Once we implement the fast extended hypercall ABI they can go away. */
  64static void * __percpu *root_scheduler_input;
  65static void * __percpu *root_scheduler_output;
  66
  67static long mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
  68static int mshv_dev_open(struct inode *inode, struct file *filp);
  69static int mshv_dev_release(struct inode *inode, struct file *filp);
  70static int mshv_vp_release(struct inode *inode, struct file *filp);
  71static long mshv_vp_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
  72static int mshv_partition_release(struct inode *inode, struct file *filp);
  73static long mshv_partition_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
  74static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma);
  75static vm_fault_t mshv_vp_fault(struct vm_fault *vmf);
  76static int mshv_init_async_handler(struct mshv_partition *partition);
  77static void mshv_async_hvcall_handler(void *data, u64 *status);
  78
  79static const union hv_input_vtl input_vtl_zero;
  80static const union hv_input_vtl input_vtl_normal = {
  81	.target_vtl = HV_NORMAL_VTL,
  82	.use_target_vtl = 1,
  83};
  84
  85static const struct vm_operations_struct mshv_vp_vm_ops = {
  86	.fault = mshv_vp_fault,
  87};
  88
  89static const struct file_operations mshv_vp_fops = {
  90	.owner = THIS_MODULE,
  91	.release = mshv_vp_release,
  92	.unlocked_ioctl = mshv_vp_ioctl,
  93	.llseek = noop_llseek,
  94	.mmap = mshv_vp_mmap,
  95};
  96
  97static const struct file_operations mshv_partition_fops = {
  98	.owner = THIS_MODULE,
  99	.release = mshv_partition_release,
 100	.unlocked_ioctl = mshv_partition_ioctl,
 101	.llseek = noop_llseek,
 102};
 103
 104static const struct file_operations mshv_dev_fops = {
 105	.owner = THIS_MODULE,
 106	.open = mshv_dev_open,
 107	.release = mshv_dev_release,
 108	.unlocked_ioctl = mshv_dev_ioctl,
 109	.llseek = noop_llseek,
 110};
 111
 112static struct miscdevice mshv_dev = {
 113	.minor = MISC_DYNAMIC_MINOR,
 114	.name = "mshv",
 115	.fops = &mshv_dev_fops,
 116	.mode = 0600,
 117};
 118
 119/*
 120 * Only allow hypercalls that have a u64 partition id as the first member of
 121 * the input structure.
 122 * These are sorted by value.
 123 */
 124static u16 mshv_passthru_hvcalls[] = {
 125	HVCALL_GET_PARTITION_PROPERTY,
 126	HVCALL_GET_PARTITION_PROPERTY_EX,
 127	HVCALL_SET_PARTITION_PROPERTY,
 128	HVCALL_INSTALL_INTERCEPT,
 129	HVCALL_GET_VP_REGISTERS,
 130	HVCALL_SET_VP_REGISTERS,
 131	HVCALL_TRANSLATE_VIRTUAL_ADDRESS,
 132	HVCALL_CLEAR_VIRTUAL_INTERRUPT,
 133	HVCALL_REGISTER_INTERCEPT_RESULT,
 134	HVCALL_ASSERT_VIRTUAL_INTERRUPT,
 135	HVCALL_GET_GPA_PAGES_ACCESS_STATES,
 136	HVCALL_SIGNAL_EVENT_DIRECT,
 137	HVCALL_POST_MESSAGE_DIRECT,
 138	HVCALL_GET_VP_CPUID_VALUES,
 139};
 140
 141/*
 142 * Only allow hypercalls that are safe to be called by the VMM with the host
 143 * partition as target (i.e. HV_PARTITION_ID_SELF). Carefully audit that a
 144 * hypercall cannot be misused by the VMM before adding it to this list.
 145 */
 146static u16 mshv_self_passthru_hvcalls[] = {
 147	HVCALL_GET_PARTITION_PROPERTY,
 148	HVCALL_GET_PARTITION_PROPERTY_EX,
 149};
 150
 151static bool mshv_hvcall_is_async(u16 code)
 152{
 153	switch (code) {
 154	case HVCALL_SET_PARTITION_PROPERTY:
 155		return true;
 156	default:
 157		break;
 158	}
 159	return false;
 160}
 161
 162static bool mshv_passthru_hvcall_allowed(u16 code, u64 pt_id)
 163{
 164	int i;
 165	int n = ARRAY_SIZE(mshv_passthru_hvcalls);
 166	u16 *allowed_hvcalls = mshv_passthru_hvcalls;
 167
 168	if (pt_id == HV_PARTITION_ID_SELF) {
 169		n = ARRAY_SIZE(mshv_self_passthru_hvcalls);
 170		allowed_hvcalls = mshv_self_passthru_hvcalls;
 171	}
 172
 173	for (i = 0; i < n; ++i)
 174		if (allowed_hvcalls[i] == code)
 175			return true;
 176
 177	return false;
 178}
 179
 180static int mshv_ioctl_passthru_hvcall(struct mshv_partition *partition,
 181				      bool partition_locked,
 182				      void __user *user_args)
 183{
 184	u64 status;
 185	int ret = 0;
 186	bool is_async;
 187	struct mshv_root_hvcall args;
 188	struct page *page;
 189	unsigned int pages_order;
 190	void *input_pg = NULL;
 191	void *output_pg = NULL;
 192	u16 reps_completed;
 193	u64 pt_id = partition ? partition->pt_id : HV_PARTITION_ID_SELF;
 194
 195	if (copy_from_user(&args, user_args, sizeof(args)))
 196		return -EFAULT;
 197
 198	if (args.status || !args.in_ptr || args.in_sz < sizeof(u64) ||
 199	    mshv_field_nonzero(args, rsvd) || args.in_sz > HV_HYP_PAGE_SIZE)
 200		return -EINVAL;
 201
 202	if (args.out_ptr && (!args.out_sz || args.out_sz > HV_HYP_PAGE_SIZE))
 203		return -EINVAL;
 204
 205	if (!mshv_passthru_hvcall_allowed(args.code, pt_id))
 206		return -EINVAL;
 207
 208	is_async = mshv_hvcall_is_async(args.code);
 209	if (is_async) {
 210		/* async hypercalls can only be called from partition fd */
 211		if (!partition || !partition_locked)
 212			return -EINVAL;
 213		ret = mshv_init_async_handler(partition);
 214		if (ret)
 215			return ret;
 216	}
 217
 218	pages_order = args.out_ptr ? 1 : 0;
 219	page = alloc_pages(GFP_KERNEL, pages_order);
 220	if (!page)
 221		return -ENOMEM;
 222	input_pg = page_address(page);
 223
 224	if (args.out_ptr)
 225		output_pg = (char *)input_pg + PAGE_SIZE;
 226	else
 227		output_pg = NULL;
 228
 229	if (copy_from_user(input_pg, (void __user *)args.in_ptr,
 230			   args.in_sz)) {
 231		ret = -EFAULT;
 232		goto free_pages_out;
 233	}
 234
 235	/*
 236	 * NOTE: This only works because all the allowed hypercalls' input
 237	 * structs begin with a u64 partition_id field.
 238	 */
 239	*(u64 *)input_pg = pt_id;
 240
 241	reps_completed = 0;
 242	do {
 243		if (args.reps) {
 244			status = hv_do_rep_hypercall_ex(args.code, args.reps,
 245							0, reps_completed,
 246							input_pg, output_pg);
 247			reps_completed = hv_repcomp(status);
 248		} else {
 249			status = hv_do_hypercall(args.code, input_pg, output_pg);
 250		}
 251
 252		if (hv_result(status) == HV_STATUS_CALL_PENDING) {
 253			if (is_async) {
 254				mshv_async_hvcall_handler(partition, &status);
 255			} else { /* Paranoia check. This shouldn't happen! */
 256				ret = -EBADFD;
 257				goto free_pages_out;
 258			}
 259		}
 260
 261		if (hv_result_success(status))
 262			break;
 263
 264		if (hv_result(status) != HV_STATUS_INSUFFICIENT_MEMORY)
 265			ret = hv_result_to_errno(status);
 266		else
 267			ret = hv_call_deposit_pages(NUMA_NO_NODE,
 268						    pt_id, 1);
 269	} while (!ret);
 270
 271	args.status = hv_result(status);
 272	args.reps = reps_completed;
 273	if (copy_to_user(user_args, &args, sizeof(args)))
 274		ret = -EFAULT;
 275
 276	if (!ret && output_pg &&
 277	    copy_to_user((void __user *)args.out_ptr, output_pg, args.out_sz))
 278		ret = -EFAULT;
 279
 280free_pages_out:
 281	free_pages((unsigned long)input_pg, pages_order);
 282
 283	return ret;
 284}
 285
 286static inline bool is_ghcb_mapping_available(void)
 287{
 288#if IS_ENABLED(CONFIG_X86_64)
 289	return ms_hyperv.ext_features & HV_VP_GHCB_ROOT_MAPPING_AVAILABLE;
 290#else
 291	return 0;
 292#endif
 293}
 294
 295static int mshv_get_vp_registers(u32 vp_index, u64 partition_id, u16 count,
 296				 struct hv_register_assoc *registers)
 297{
 298	return hv_call_get_vp_registers(vp_index, partition_id,
 299					count, input_vtl_zero, registers);
 300}
 301
 302static int mshv_set_vp_registers(u32 vp_index, u64 partition_id, u16 count,
 303				 struct hv_register_assoc *registers)
 304{
 305	return hv_call_set_vp_registers(vp_index, partition_id,
 306					count, input_vtl_zero, registers);
 307}
 308
 309/*
 310 * Explicit guest vCPU suspend is asynchronous by nature (as it is requested by
 311 * dom0 vCPU for guest vCPU) and thus it can race with "intercept" suspend,
 312 * done by the hypervisor.
 313 * "Intercept" suspend leads to asynchronous message delivery to dom0 which
 314 * should be awaited to keep the VP loop consistent (i.e. no message pending
 315 * upon VP resume).
 316 * VP intercept suspend can't be done when the VP is explicitly suspended
 317 * already, and thus can be only two possible race scenarios:
 318 *   1. implicit suspend bit set -> explicit suspend bit set -> message sent
 319 *   2. implicit suspend bit set -> message sent -> explicit suspend bit set
 320 * Checking for implicit suspend bit set after explicit suspend request has
 321 * succeeded in either case allows us to reliably identify, if there is a
 322 * message to receive and deliver to VMM.
 323 */
 324static int
 325mshv_suspend_vp(const struct mshv_vp *vp, bool *message_in_flight)
 326{
 327	struct hv_register_assoc explicit_suspend = {
 328		.name = HV_REGISTER_EXPLICIT_SUSPEND
 329	};
 330	struct hv_register_assoc intercept_suspend = {
 331		.name = HV_REGISTER_INTERCEPT_SUSPEND
 332	};
 333	union hv_explicit_suspend_register *es =
 334		&explicit_suspend.value.explicit_suspend;
 335	union hv_intercept_suspend_register *is =
 336		&intercept_suspend.value.intercept_suspend;
 337	int ret;
 338
 339	es->suspended = 1;
 340
 341	ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
 342				    1, &explicit_suspend);
 343	if (ret) {
 344		vp_err(vp, "Failed to explicitly suspend vCPU\n");
 345		return ret;
 346	}
 347
 348	ret = mshv_get_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
 349				    1, &intercept_suspend);
 350	if (ret) {
 351		vp_err(vp, "Failed to get intercept suspend state\n");
 352		return ret;
 353	}
 354
 355	*message_in_flight = is->suspended;
 356
 357	return 0;
 358}
 359
 360/*
 361 * This function is used when VPs are scheduled by the hypervisor's
 362 * scheduler.
 363 *
 364 * Caller has to make sure the registers contain cleared
 365 * HV_REGISTER_INTERCEPT_SUSPEND and HV_REGISTER_EXPLICIT_SUSPEND registers
 366 * exactly in this order (the hypervisor clears them sequentially) to avoid
 367 * potential invalid clearing a newly arrived HV_REGISTER_INTERCEPT_SUSPEND
 368 * after VP is released from HV_REGISTER_EXPLICIT_SUSPEND in case of the
 369 * opposite order.
 370 */
 371static long mshv_run_vp_with_hyp_scheduler(struct mshv_vp *vp)
 372{
 373	long ret;
 374	struct hv_register_assoc suspend_regs[2] = {
 375			{ .name = HV_REGISTER_INTERCEPT_SUSPEND },
 376			{ .name = HV_REGISTER_EXPLICIT_SUSPEND }
 377	};
 378	size_t count = ARRAY_SIZE(suspend_regs);
 379
 380	/* Resume VP execution */
 381	ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
 382				    count, suspend_regs);
 383	if (ret) {
 384		vp_err(vp, "Failed to resume vp execution. %lx\n", ret);
 385		return ret;
 386	}
 387
 388	ret = wait_event_interruptible(vp->run.vp_suspend_queue,
 389				       vp->run.kicked_by_hv == 1);
 390	if (ret) {
 391		bool message_in_flight;
 392
 393		/*
 394		 * Otherwise the waiting was interrupted by a signal: suspend
 395		 * the vCPU explicitly and copy message in flight (if any).
 396		 */
 397		ret = mshv_suspend_vp(vp, &message_in_flight);
 398		if (ret)
 399			return ret;
 400
 401		/* Return if no message in flight */
 402		if (!message_in_flight)
 403			return -EINTR;
 404
 405		/* Wait for the message in flight. */
 406		wait_event(vp->run.vp_suspend_queue, vp->run.kicked_by_hv == 1);
 407	}
 408
 409	/*
 410	 * Reset the flag to make the wait_event call above work
 411	 * next time.
 412	 */
 413	vp->run.kicked_by_hv = 0;
 414
 415	return 0;
 416}
 417
 418static int
 419mshv_vp_dispatch(struct mshv_vp *vp, u32 flags,
 420		 struct hv_output_dispatch_vp *res)
 421{
 422	struct hv_input_dispatch_vp *input;
 423	struct hv_output_dispatch_vp *output;
 424	u64 status;
 425
 426	preempt_disable();
 427	input = *this_cpu_ptr(root_scheduler_input);
 428	output = *this_cpu_ptr(root_scheduler_output);
 429
 430	memset(input, 0, sizeof(*input));
 431	memset(output, 0, sizeof(*output));
 432
 433	input->partition_id = vp->vp_partition->pt_id;
 434	input->vp_index = vp->vp_index;
 435	input->time_slice = 0; /* Run forever until something happens */
 436	input->spec_ctrl = 0; /* TODO: set sensible flags */
 437	input->flags = flags;
 438
 439	vp->run.flags.root_sched_dispatched = 1;
 440	status = hv_do_hypercall(HVCALL_DISPATCH_VP, input, output);
 441	vp->run.flags.root_sched_dispatched = 0;
 442
 443	*res = *output;
 444	preempt_enable();
 445
 446	if (!hv_result_success(status))
 447		vp_err(vp, "%s: status %s\n", __func__,
 448		       hv_result_to_string(status));
 449
 450	return hv_result_to_errno(status);
 451}
 452
 453static int
 454mshv_vp_clear_explicit_suspend(struct mshv_vp *vp)
 455{
 456	struct hv_register_assoc explicit_suspend = {
 457		.name = HV_REGISTER_EXPLICIT_SUSPEND,
 458		.value.explicit_suspend.suspended = 0,
 459	};
 460	int ret;
 461
 462	ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
 463				    1, &explicit_suspend);
 464
 465	if (ret)
 466		vp_err(vp, "Failed to unsuspend\n");
 467
 468	return ret;
 469}
 470
 471#if IS_ENABLED(CONFIG_X86_64)
 472static u64 mshv_vp_interrupt_pending(struct mshv_vp *vp)
 473{
 474	if (!vp->vp_register_page)
 475		return 0;
 476	return vp->vp_register_page->interrupt_vectors.as_uint64;
 477}
 478#else
 479static u64 mshv_vp_interrupt_pending(struct mshv_vp *vp)
 480{
 481	return 0;
 482}
 483#endif
 484
 485static bool mshv_vp_dispatch_thread_blocked(struct mshv_vp *vp)
 486{
 487	struct hv_stats_page **stats = vp->vp_stats_pages;
 488	u64 *self_vp_cntrs = stats[HV_STATS_AREA_SELF]->vp_cntrs;
 489	u64 *parent_vp_cntrs = stats[HV_STATS_AREA_PARENT]->vp_cntrs;
 490
 491	if (self_vp_cntrs[VpRootDispatchThreadBlocked])
 492		return self_vp_cntrs[VpRootDispatchThreadBlocked];
 493	return parent_vp_cntrs[VpRootDispatchThreadBlocked];
 494}
 495
 496static int
 497mshv_vp_wait_for_hv_kick(struct mshv_vp *vp)
 498{
 499	int ret;
 500
 501	ret = wait_event_interruptible(vp->run.vp_suspend_queue,
 502				       (vp->run.kicked_by_hv == 1 &&
 503					!mshv_vp_dispatch_thread_blocked(vp)) ||
 504				       mshv_vp_interrupt_pending(vp));
 505	if (ret)
 506		return -EINTR;
 507
 508	vp->run.flags.root_sched_blocked = 0;
 509	vp->run.kicked_by_hv = 0;
 510
 511	return 0;
 512}
 513
 514/* Must be called with interrupts enabled */
 515static long mshv_run_vp_with_root_scheduler(struct mshv_vp *vp)
 516{
 517	long ret;
 518
 519	if (vp->run.flags.root_sched_blocked) {
 520		/*
 521		 * Dispatch state of this VP is blocked. Need to wait
 522		 * for the hypervisor to clear the blocked state before
 523		 * dispatching it.
 524		 */
 525		ret = mshv_vp_wait_for_hv_kick(vp);
 526		if (ret)
 527			return ret;
 528	}
 529
 530	do {
 531		u32 flags = 0;
 532		struct hv_output_dispatch_vp output;
 533
 534		if (__xfer_to_guest_mode_work_pending()) {
 535			ret = xfer_to_guest_mode_handle_work();
 536			if (ret)
 537				break;
 538		}
 539
 540		if (vp->run.flags.intercept_suspend)
 541			flags |= HV_DISPATCH_VP_FLAG_CLEAR_INTERCEPT_SUSPEND;
 542
 543		if (mshv_vp_interrupt_pending(vp))
 544			flags |= HV_DISPATCH_VP_FLAG_SCAN_INTERRUPT_INJECTION;
 545
 546		ret = mshv_vp_dispatch(vp, flags, &output);
 547		if (ret)
 548			break;
 549
 550		vp->run.flags.intercept_suspend = 0;
 551
 552		if (output.dispatch_state == HV_VP_DISPATCH_STATE_BLOCKED) {
 553			if (output.dispatch_event ==
 554						HV_VP_DISPATCH_EVENT_SUSPEND) {
 555				/*
 556				 * TODO: remove the warning once VP canceling
 557				 *	 is supported
 558				 */
 559				WARN_ONCE(atomic64_read(&vp->run.vp_signaled_count),
 560					  "%s: vp#%d: unexpected explicit suspend\n",
 561					  __func__, vp->vp_index);
 562				/*
 563				 * Need to clear explicit suspend before
 564				 * dispatching.
 565				 * Explicit suspend is either:
 566				 * - set right after the first VP dispatch or
 567				 * - set explicitly via hypercall
 568				 * Since the latter case is not yet supported,
 569				 * simply clear it here.
 570				 */
 571				ret = mshv_vp_clear_explicit_suspend(vp);
 572				if (ret)
 573					break;
 574
 575				ret = mshv_vp_wait_for_hv_kick(vp);
 576				if (ret)
 577					break;
 578			} else {
 579				vp->run.flags.root_sched_blocked = 1;
 580				ret = mshv_vp_wait_for_hv_kick(vp);
 581				if (ret)
 582					break;
 583			}
 584		} else {
 585			/* HV_VP_DISPATCH_STATE_READY */
 586			if (output.dispatch_event ==
 587						HV_VP_DISPATCH_EVENT_INTERCEPT)
 588				vp->run.flags.intercept_suspend = 1;
 589		}
 590	} while (!vp->run.flags.intercept_suspend);
 591
 592	rseq_virt_userspace_exit();
 593
 594	return ret;
 595}
 596
 597static_assert(sizeof(struct hv_message) <= MSHV_RUN_VP_BUF_SZ,
 598	      "sizeof(struct hv_message) must not exceed MSHV_RUN_VP_BUF_SZ");
 599
 600static struct mshv_mem_region *
 601mshv_partition_region_by_gfn(struct mshv_partition *partition, u64 gfn)
 602{
 603	struct mshv_mem_region *region;
 604
 605	hlist_for_each_entry(region, &partition->pt_mem_regions, hnode) {
 606		if (gfn >= region->start_gfn &&
 607		    gfn < region->start_gfn + region->nr_pages)
 608			return region;
 609	}
 610
 611	return NULL;
 612}
 613
 614static struct mshv_mem_region *
 615mshv_partition_region_by_gfn_get(struct mshv_partition *p, u64 gfn)
 616{
 617	struct mshv_mem_region *region;
 618
 619	spin_lock(&p->pt_mem_regions_lock);
 620	region = mshv_partition_region_by_gfn(p, gfn);
 621	if (!region || !mshv_region_get(region)) {
 622		spin_unlock(&p->pt_mem_regions_lock);
 623		return NULL;
 624	}
 625	spin_unlock(&p->pt_mem_regions_lock);
 626
 627	return region;
 628}
 629
 630/**
 631 * mshv_handle_gpa_intercept - Handle GPA (Guest Physical Address) intercepts.
 632 * @vp: Pointer to the virtual processor structure.
 633 *
 634 * This function processes GPA intercepts by identifying the memory region
 635 * corresponding to the intercepted GPA, aligning the page offset, and
 636 * mapping the required pages. It ensures that the region is valid and
 637 * handles faults efficiently by mapping multiple pages at once.
 638 *
 639 * Return: true if the intercept was handled successfully, false otherwise.
 640 */
 641static bool mshv_handle_gpa_intercept(struct mshv_vp *vp)
 642{
 643	struct mshv_partition *p = vp->vp_partition;
 644	struct mshv_mem_region *region;
 645	bool ret = false;
 646	u64 gfn;
 647#if defined(CONFIG_X86_64)
 648	struct hv_x64_memory_intercept_message *msg =
 649		(struct hv_x64_memory_intercept_message *)
 650		vp->vp_intercept_msg_page->u.payload;
 651#elif defined(CONFIG_ARM64)
 652	struct hv_arm64_memory_intercept_message *msg =
 653		(struct hv_arm64_memory_intercept_message *)
 654		vp->vp_intercept_msg_page->u.payload;
 655#endif
 656	enum hv_intercept_access_type access_type =
 657		msg->header.intercept_access_type;
 658
 659	gfn = HVPFN_DOWN(msg->guest_physical_address);
 660
 661	region = mshv_partition_region_by_gfn_get(p, gfn);
 662	if (!region)
 663		return false;
 664
 665	if (access_type == HV_INTERCEPT_ACCESS_WRITE &&
 666	    !(region->hv_map_flags & HV_MAP_GPA_WRITABLE))
 667		goto put_region;
 668
 669	if (access_type == HV_INTERCEPT_ACCESS_EXECUTE &&
 670	    !(region->hv_map_flags & HV_MAP_GPA_EXECUTABLE))
 671		goto put_region;
 672
 673	/* Only movable memory ranges are supported for GPA intercepts */
 674	if (region->type == MSHV_REGION_TYPE_MEM_MOVABLE)
 675		ret = mshv_region_handle_gfn_fault(region, gfn);
 676
 677put_region:
 678	mshv_region_put(region);
 679
 680	return ret;
 681}
 682
 683static bool mshv_vp_handle_intercept(struct mshv_vp *vp)
 684{
 685	switch (vp->vp_intercept_msg_page->header.message_type) {
 686	case HVMSG_GPA_INTERCEPT:
 687		return mshv_handle_gpa_intercept(vp);
 688	}
 689	return false;
 690}
 691
 692static long mshv_vp_ioctl_run_vp(struct mshv_vp *vp, void __user *ret_msg)
 693{
 694	long rc;
 695
 696	do {
 697		if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
 698			rc = mshv_run_vp_with_root_scheduler(vp);
 699		else
 700			rc = mshv_run_vp_with_hyp_scheduler(vp);
 701	} while (rc == 0 && mshv_vp_handle_intercept(vp));
 702
 703	if (rc)
 704		return rc;
 705
 706	if (copy_to_user(ret_msg, vp->vp_intercept_msg_page,
 707			 sizeof(struct hv_message)))
 708		rc = -EFAULT;
 709
 710	return rc;
 711}
 712
 713static int
 714mshv_vp_ioctl_get_set_state_pfn(struct mshv_vp *vp,
 715				struct hv_vp_state_data state_data,
 716				unsigned long user_pfn, size_t page_count,
 717				bool is_set)
 718{
 719	int completed, ret = 0;
 720	unsigned long check;
 721	struct page **pages;
 722
 723	if (page_count > INT_MAX)
 724		return -EINVAL;
 725	/*
 726	 * Check the arithmetic for wraparound/overflow.
 727	 * The last page address in the buffer is:
 728	 * (user_pfn + (page_count - 1)) * PAGE_SIZE
 729	 */
 730	if (check_add_overflow(user_pfn, (page_count - 1), &check))
 731		return -EOVERFLOW;
 732	if (check_mul_overflow(check, PAGE_SIZE, &check))
 733		return -EOVERFLOW;
 734
 735	/* Pin user pages so hypervisor can copy directly to them */
 736	pages = kcalloc(page_count, sizeof(struct page *), GFP_KERNEL);
 737	if (!pages)
 738		return -ENOMEM;
 739
 740	for (completed = 0; completed < page_count; completed += ret) {
 741		unsigned long user_addr = (user_pfn + completed) * PAGE_SIZE;
 742		int remaining = page_count - completed;
 743
 744		ret = pin_user_pages_fast(user_addr, remaining, FOLL_WRITE,
 745					  &pages[completed]);
 746		if (ret < 0) {
 747			vp_err(vp, "%s: Failed to pin user pages error %i\n",
 748			       __func__, ret);
 749			goto unpin_pages;
 750		}
 751	}
 752
 753	if (is_set)
 754		ret = hv_call_set_vp_state(vp->vp_index,
 755					   vp->vp_partition->pt_id,
 756					   state_data, page_count, pages,
 757					   0, NULL);
 758	else
 759		ret = hv_call_get_vp_state(vp->vp_index,
 760					   vp->vp_partition->pt_id,
 761					   state_data, page_count, pages,
 762					   NULL);
 763
 764unpin_pages:
 765	unpin_user_pages(pages, completed);
 766	kfree(pages);
 767	return ret;
 768}
 769
 770static long
 771mshv_vp_ioctl_get_set_state(struct mshv_vp *vp,
 772			    struct mshv_get_set_vp_state __user *user_args,
 773			    bool is_set)
 774{
 775	struct mshv_get_set_vp_state args;
 776	long ret = 0;
 777	union hv_output_get_vp_state vp_state;
 778	u32 data_sz;
 779	struct hv_vp_state_data state_data = {};
 780
 781	if (copy_from_user(&args, user_args, sizeof(args)))
 782		return -EFAULT;
 783
 784	if (args.type >= MSHV_VP_STATE_COUNT || mshv_field_nonzero(args, rsvd) ||
 785	    !args.buf_sz || !PAGE_ALIGNED(args.buf_sz) ||
 786	    !PAGE_ALIGNED(args.buf_ptr))
 787		return -EINVAL;
 788
 789	if (!access_ok((void __user *)args.buf_ptr, args.buf_sz))
 790		return -EFAULT;
 791
 792	switch (args.type) {
 793	case MSHV_VP_STATE_LAPIC:
 794		state_data.type = HV_GET_SET_VP_STATE_LAPIC_STATE;
 795		data_sz = HV_HYP_PAGE_SIZE;
 796		break;
 797	case MSHV_VP_STATE_XSAVE:
 798	{
 799		u64 data_sz_64;
 800
 801		ret = hv_call_get_partition_property(vp->vp_partition->pt_id,
 802						     HV_PARTITION_PROPERTY_XSAVE_STATES,
 803						     &state_data.xsave.states.as_uint64);
 804		if (ret)
 805			return ret;
 806
 807		ret = hv_call_get_partition_property(vp->vp_partition->pt_id,
 808						     HV_PARTITION_PROPERTY_MAX_XSAVE_DATA_SIZE,
 809						     &data_sz_64);
 810		if (ret)
 811			return ret;
 812
 813		data_sz = (u32)data_sz_64;
 814		state_data.xsave.flags = 0;
 815		/* Always request legacy states */
 816		state_data.xsave.states.legacy_x87 = 1;
 817		state_data.xsave.states.legacy_sse = 1;
 818		state_data.type = HV_GET_SET_VP_STATE_XSAVE;
 819		break;
 820	}
 821	case MSHV_VP_STATE_SIMP:
 822		state_data.type = HV_GET_SET_VP_STATE_SIM_PAGE;
 823		data_sz = HV_HYP_PAGE_SIZE;
 824		break;
 825	case MSHV_VP_STATE_SIEFP:
 826		state_data.type = HV_GET_SET_VP_STATE_SIEF_PAGE;
 827		data_sz = HV_HYP_PAGE_SIZE;
 828		break;
 829	case MSHV_VP_STATE_SYNTHETIC_TIMERS:
 830		state_data.type = HV_GET_SET_VP_STATE_SYNTHETIC_TIMERS;
 831		data_sz = sizeof(vp_state.synthetic_timers_state);
 832		break;
 833	default:
 834		return -EINVAL;
 835	}
 836
 837	if (copy_to_user(&user_args->buf_sz, &data_sz, sizeof(user_args->buf_sz)))
 838		return -EFAULT;
 839
 840	if (data_sz > args.buf_sz)
 841		return -EINVAL;
 842
 843	/* If the data is transmitted via pfns, delegate to helper */
 844	if (state_data.type & HV_GET_SET_VP_STATE_TYPE_PFN) {
 845		unsigned long user_pfn = PFN_DOWN(args.buf_ptr);
 846		size_t page_count = PFN_DOWN(args.buf_sz);
 847
 848		return mshv_vp_ioctl_get_set_state_pfn(vp, state_data, user_pfn,
 849						       page_count, is_set);
 850	}
 851
 852	/* Paranoia check - this shouldn't happen! */
 853	if (data_sz > sizeof(vp_state)) {
 854		vp_err(vp, "Invalid vp state data size!\n");
 855		return -EINVAL;
 856	}
 857
 858	if (is_set) {
 859		if (copy_from_user(&vp_state, (__user void *)args.buf_ptr, data_sz))
 860			return -EFAULT;
 861
 862		return hv_call_set_vp_state(vp->vp_index,
 863					    vp->vp_partition->pt_id,
 864					    state_data, 0, NULL,
 865					    sizeof(vp_state), (u8 *)&vp_state);
 866	}
 867
 868	ret = hv_call_get_vp_state(vp->vp_index, vp->vp_partition->pt_id,
 869				   state_data, 0, NULL, &vp_state);
 870	if (ret)
 871		return ret;
 872
 873	if (copy_to_user((void __user *)args.buf_ptr, &vp_state, data_sz))
 874		return -EFAULT;
 875
 876	return 0;
 877}
 878
 879static long
 880mshv_vp_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 881{
 882	struct mshv_vp *vp = filp->private_data;
 883	long r = -ENOTTY;
 884
 885	if (mutex_lock_killable(&vp->vp_mutex))
 886		return -EINTR;
 887
 888	switch (ioctl) {
 889	case MSHV_RUN_VP:
 890		r = mshv_vp_ioctl_run_vp(vp, (void __user *)arg);
 891		break;
 892	case MSHV_GET_VP_STATE:
 893		r = mshv_vp_ioctl_get_set_state(vp, (void __user *)arg, false);
 894		break;
 895	case MSHV_SET_VP_STATE:
 896		r = mshv_vp_ioctl_get_set_state(vp, (void __user *)arg, true);
 897		break;
 898	case MSHV_ROOT_HVCALL:
 899		r = mshv_ioctl_passthru_hvcall(vp->vp_partition, false,
 900					       (void __user *)arg);
 901		break;
 902	default:
 903		vp_warn(vp, "Invalid ioctl: %#x\n", ioctl);
 904		break;
 905	}
 906	mutex_unlock(&vp->vp_mutex);
 907
 908	return r;
 909}
 910
 911static vm_fault_t mshv_vp_fault(struct vm_fault *vmf)
 912{
 913	struct mshv_vp *vp = vmf->vma->vm_file->private_data;
 914
 915	switch (vmf->vma->vm_pgoff) {
 916	case MSHV_VP_MMAP_OFFSET_REGISTERS:
 917		vmf->page = virt_to_page(vp->vp_register_page);
 918		break;
 919	case MSHV_VP_MMAP_OFFSET_INTERCEPT_MESSAGE:
 920		vmf->page = virt_to_page(vp->vp_intercept_msg_page);
 921		break;
 922	case MSHV_VP_MMAP_OFFSET_GHCB:
 923		vmf->page = virt_to_page(vp->vp_ghcb_page);
 924		break;
 925	default:
 926		return VM_FAULT_SIGBUS;
 927	}
 928
 929	get_page(vmf->page);
 930
 931	return 0;
 932}
 933
 934static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma)
 935{
 936	struct mshv_vp *vp = file->private_data;
 937
 938	switch (vma->vm_pgoff) {
 939	case MSHV_VP_MMAP_OFFSET_REGISTERS:
 940		if (!vp->vp_register_page)
 941			return -ENODEV;
 942		break;
 943	case MSHV_VP_MMAP_OFFSET_INTERCEPT_MESSAGE:
 944		if (!vp->vp_intercept_msg_page)
 945			return -ENODEV;
 946		break;
 947	case MSHV_VP_MMAP_OFFSET_GHCB:
 948		if (!vp->vp_ghcb_page)
 949			return -ENODEV;
 950		break;
 951	default:
 952		return -EINVAL;
 953	}
 954
 955	vma->vm_ops = &mshv_vp_vm_ops;
 956	return 0;
 957}
 958
 959static int
 960mshv_vp_release(struct inode *inode, struct file *filp)
 961{
 962	struct mshv_vp *vp = filp->private_data;
 963
 964	/* Rest of VP cleanup happens in destroy_partition() */
 965	mshv_partition_put(vp->vp_partition);
 966	return 0;
 967}
 968
 969static void mshv_vp_stats_unmap(u64 partition_id, u32 vp_index,
 970				void *stats_pages[])
 971{
 972	union hv_stats_object_identity identity = {
 973		.vp.partition_id = partition_id,
 974		.vp.vp_index = vp_index,
 975	};
 976
 977	identity.vp.stats_area_type = HV_STATS_AREA_SELF;
 978	hv_unmap_stats_page(HV_STATS_OBJECT_VP, NULL, &identity);
 979
 980	identity.vp.stats_area_type = HV_STATS_AREA_PARENT;
 981	hv_unmap_stats_page(HV_STATS_OBJECT_VP, NULL, &identity);
 982}
 983
 984static int mshv_vp_stats_map(u64 partition_id, u32 vp_index,
 985			     void *stats_pages[])
 986{
 987	union hv_stats_object_identity identity = {
 988		.vp.partition_id = partition_id,
 989		.vp.vp_index = vp_index,
 990	};
 991	int err;
 992
 993	identity.vp.stats_area_type = HV_STATS_AREA_SELF;
 994	err = hv_map_stats_page(HV_STATS_OBJECT_VP, &identity,
 995				&stats_pages[HV_STATS_AREA_SELF]);
 996	if (err)
 997		return err;
 998
 999	identity.vp.stats_area_type = HV_STATS_AREA_PARENT;
1000	err = hv_map_stats_page(HV_STATS_OBJECT_VP, &identity,
1001				&stats_pages[HV_STATS_AREA_PARENT]);
1002	if (err)
1003		goto unmap_self;
1004
1005	if (!stats_pages[HV_STATS_AREA_PARENT])
1006		stats_pages[HV_STATS_AREA_PARENT] = stats_pages[HV_STATS_AREA_SELF];
1007
1008	return 0;
1009
1010unmap_self:
1011	identity.vp.stats_area_type = HV_STATS_AREA_SELF;
1012	hv_unmap_stats_page(HV_STATS_OBJECT_VP, NULL, &identity);
1013	return err;
1014}
1015
1016static long
1017mshv_partition_ioctl_create_vp(struct mshv_partition *partition,
1018			       void __user *arg)
1019{
1020	struct mshv_create_vp args;
1021	struct mshv_vp *vp;
1022	struct page *intercept_msg_page, *register_page, *ghcb_page;
1023	void *stats_pages[2];
1024	long ret;
1025
1026	if (copy_from_user(&args, arg, sizeof(args)))
1027		return -EFAULT;
1028
1029	if (args.vp_index >= MSHV_MAX_VPS)
1030		return -EINVAL;
1031
1032	if (partition->pt_vp_array[args.vp_index])
1033		return -EEXIST;
1034
1035	ret = hv_call_create_vp(NUMA_NO_NODE, partition->pt_id, args.vp_index,
1036				0 /* Only valid for root partition VPs */);
1037	if (ret)
1038		return ret;
1039
1040	ret = hv_map_vp_state_page(partition->pt_id, args.vp_index,
1041				   HV_VP_STATE_PAGE_INTERCEPT_MESSAGE,
1042				   input_vtl_zero, &intercept_msg_page);
1043	if (ret)
1044		goto destroy_vp;
1045
1046	if (!mshv_partition_encrypted(partition)) {
1047		ret = hv_map_vp_state_page(partition->pt_id, args.vp_index,
1048					   HV_VP_STATE_PAGE_REGISTERS,
1049					   input_vtl_zero, &register_page);
1050		if (ret)
1051			goto unmap_intercept_message_page;
1052	}
1053
1054	if (mshv_partition_encrypted(partition) &&
1055	    is_ghcb_mapping_available()) {
1056		ret = hv_map_vp_state_page(partition->pt_id, args.vp_index,
1057					   HV_VP_STATE_PAGE_GHCB,
1058					   input_vtl_normal, &ghcb_page);
1059		if (ret)
1060			goto unmap_register_page;
1061	}
1062
1063	/*
1064	 * This mapping of the stats page is for detecting if dispatch thread
1065	 * is blocked - only relevant for root scheduler
1066	 */
1067	if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT) {
1068		ret = mshv_vp_stats_map(partition->pt_id, args.vp_index,
1069					stats_pages);
1070		if (ret)
1071			goto unmap_ghcb_page;
1072	}
1073
1074	vp = kzalloc(sizeof(*vp), GFP_KERNEL);
1075	if (!vp)
1076		goto unmap_stats_pages;
1077
1078	vp->vp_partition = mshv_partition_get(partition);
1079	if (!vp->vp_partition) {
1080		ret = -EBADF;
1081		goto free_vp;
1082	}
1083
1084	mutex_init(&vp->vp_mutex);
1085	init_waitqueue_head(&vp->run.vp_suspend_queue);
1086	atomic64_set(&vp->run.vp_signaled_count, 0);
1087
1088	vp->vp_index = args.vp_index;
1089	vp->vp_intercept_msg_page = page_to_virt(intercept_msg_page);
1090	if (!mshv_partition_encrypted(partition))
1091		vp->vp_register_page = page_to_virt(register_page);
1092
1093	if (mshv_partition_encrypted(partition) && is_ghcb_mapping_available())
1094		vp->vp_ghcb_page = page_to_virt(ghcb_page);
1095
1096	if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
1097		memcpy(vp->vp_stats_pages, stats_pages, sizeof(stats_pages));
1098
1099	/*
1100	 * Keep anon_inode_getfd last: it installs fd in the file struct and
1101	 * thus makes the state accessible in user space.
1102	 */
1103	ret = anon_inode_getfd("mshv_vp", &mshv_vp_fops, vp,
1104			       O_RDWR | O_CLOEXEC);
1105	if (ret < 0)
1106		goto put_partition;
1107
1108	/* already exclusive with the partition mutex for all ioctls */
1109	partition->pt_vp_count++;
1110	partition->pt_vp_array[args.vp_index] = vp;
1111
1112	return ret;
1113
1114put_partition:
1115	mshv_partition_put(partition);
1116free_vp:
1117	kfree(vp);
1118unmap_stats_pages:
1119	if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
1120		mshv_vp_stats_unmap(partition->pt_id, args.vp_index, stats_pages);
1121unmap_ghcb_page:
1122	if (mshv_partition_encrypted(partition) && is_ghcb_mapping_available())
1123		hv_unmap_vp_state_page(partition->pt_id, args.vp_index,
1124				       HV_VP_STATE_PAGE_GHCB, ghcb_page,
1125				       input_vtl_normal);
1126unmap_register_page:
1127	if (!mshv_partition_encrypted(partition))
1128		hv_unmap_vp_state_page(partition->pt_id, args.vp_index,
1129				       HV_VP_STATE_PAGE_REGISTERS,
1130				       register_page, input_vtl_zero);
1131unmap_intercept_message_page:
1132	hv_unmap_vp_state_page(partition->pt_id, args.vp_index,
1133			       HV_VP_STATE_PAGE_INTERCEPT_MESSAGE,
1134			       intercept_msg_page, input_vtl_zero);
1135destroy_vp:
1136	hv_call_delete_vp(partition->pt_id, args.vp_index);
1137	return ret;
1138}
1139
1140static int mshv_init_async_handler(struct mshv_partition *partition)
1141{
1142	if (completion_done(&partition->async_hypercall)) {
1143		pt_err(partition,
1144		       "Cannot issue async hypercall while another one in progress!\n");
1145		return -EPERM;
1146	}
1147
1148	reinit_completion(&partition->async_hypercall);
1149	return 0;
1150}
1151
1152static void mshv_async_hvcall_handler(void *data, u64 *status)
1153{
1154	struct mshv_partition *partition = data;
1155
1156	wait_for_completion(&partition->async_hypercall);
1157	pt_dbg(partition, "Async hypercall completed!\n");
1158
1159	*status = partition->async_hypercall_status;
1160}
1161
1162/*
1163 * NB: caller checks and makes sure mem->size is page aligned
1164 * Returns: 0 with regionpp updated on success, or -errno
1165 */
1166static int mshv_partition_create_region(struct mshv_partition *partition,
1167					struct mshv_user_mem_region *mem,
1168					struct mshv_mem_region **regionpp,
1169					bool is_mmio)
1170{
1171	struct mshv_mem_region *rg;
1172	u64 nr_pages = HVPFN_DOWN(mem->size);
1173
1174	/* Reject overlapping regions */
1175	spin_lock(&partition->pt_mem_regions_lock);
1176	hlist_for_each_entry(rg, &partition->pt_mem_regions, hnode) {
1177		if (mem->guest_pfn + nr_pages <= rg->start_gfn ||
1178		    rg->start_gfn + rg->nr_pages <= mem->guest_pfn)
1179			continue;
1180		spin_unlock(&partition->pt_mem_regions_lock);
1181		return -EEXIST;
1182	}
1183	spin_unlock(&partition->pt_mem_regions_lock);
1184
1185	rg = mshv_region_create(mem->guest_pfn, nr_pages,
1186				mem->userspace_addr, mem->flags);
1187	if (IS_ERR(rg))
1188		return PTR_ERR(rg);
1189
1190	if (is_mmio)
1191		rg->type = MSHV_REGION_TYPE_MMIO;
1192	else if (mshv_partition_encrypted(partition) ||
1193		 !mshv_region_movable_init(rg))
1194		rg->type = MSHV_REGION_TYPE_MEM_PINNED;
1195	else
1196		rg->type = MSHV_REGION_TYPE_MEM_MOVABLE;
1197
1198	rg->partition = partition;
1199
1200	*regionpp = rg;
1201
1202	return 0;
1203}
1204
1205/**
1206 * mshv_prepare_pinned_region - Pin and map memory regions
1207 * @region: Pointer to the memory region structure
1208 *
1209 * This function processes memory regions that are explicitly marked as pinned.
1210 * Pinned regions are preallocated, mapped upfront, and do not rely on fault-based
1211 * population. The function ensures the region is properly populated, handles
1212 * encryption requirements for SNP partitions if applicable, maps the region,
1213 * and performs necessary sharing or eviction operations based on the mapping
1214 * result.
1215 *
1216 * Return: 0 on success, negative error code on failure.
1217 */
1218static int mshv_prepare_pinned_region(struct mshv_mem_region *region)
1219{
1220	struct mshv_partition *partition = region->partition;
1221	int ret;
1222
1223	ret = mshv_region_pin(region);
1224	if (ret) {
1225		pt_err(partition, "Failed to pin memory region: %d\n",
1226		       ret);
1227		goto err_out;
1228	}
1229
1230	/*
1231	 * For an SNP partition it is a requirement that for every memory region
1232	 * that we are going to map for this partition we should make sure that
1233	 * host access to that region is released. This is ensured by doing an
1234	 * additional hypercall which will update the SLAT to release host
1235	 * access to guest memory regions.
1236	 */
1237	if (mshv_partition_encrypted(partition)) {
1238		ret = mshv_region_unshare(region);
1239		if (ret) {
1240			pt_err(partition,
1241			       "Failed to unshare memory region (guest_pfn: %llu): %d\n",
1242			       region->start_gfn, ret);
1243			goto invalidate_region;
1244		}
1245	}
1246
1247	ret = mshv_region_map(region);
1248	if (ret && mshv_partition_encrypted(partition)) {
1249		int shrc;
1250
1251		shrc = mshv_region_share(region);
1252		if (!shrc)
1253			goto invalidate_region;
1254
1255		pt_err(partition,
1256		       "Failed to share memory region (guest_pfn: %llu): %d\n",
1257		       region->start_gfn, shrc);
1258		/*
1259		 * Don't unpin if marking shared failed because pages are no
1260		 * longer mapped in the host, ie root, anymore.
1261		 */
1262		goto err_out;
1263	}
1264
1265	return 0;
1266
1267invalidate_region:
1268	mshv_region_invalidate(region);
1269err_out:
1270	return ret;
1271}
1272
1273/*
1274 * This maps two things: guest RAM and for pci passthru mmio space.
1275 *
1276 * mmio:
1277 *  - vfio overloads vm_pgoff to store the mmio start pfn/spa.
1278 *  - Two things need to happen for mapping mmio range:
1279 *	1. mapped in the uaddr so VMM can access it.
1280 *	2. mapped in the hwpt (gfn <-> mmio phys addr) so guest can access it.
1281 *
1282 *   This function takes care of the second. The first one is managed by vfio,
1283 *   and hence is taken care of via vfio_pci_mmap_fault().
1284 */
1285static long
1286mshv_map_user_memory(struct mshv_partition *partition,
1287		     struct mshv_user_mem_region mem)
1288{
1289	struct mshv_mem_region *region;
1290	struct vm_area_struct *vma;
1291	bool is_mmio;
1292	ulong mmio_pfn;
1293	long ret;
1294
1295	if (mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP) ||
1296	    !access_ok((const void __user *)mem.userspace_addr, mem.size))
1297		return -EINVAL;
1298
1299	mmap_read_lock(current->mm);
1300	vma = vma_lookup(current->mm, mem.userspace_addr);
1301	is_mmio = vma ? !!(vma->vm_flags & (VM_IO | VM_PFNMAP)) : 0;
1302	mmio_pfn = is_mmio ? vma->vm_pgoff : 0;
1303	mmap_read_unlock(current->mm);
1304
1305	if (!vma)
1306		return -EINVAL;
1307
1308	ret = mshv_partition_create_region(partition, &mem, &region,
1309					   is_mmio);
1310	if (ret)
1311		return ret;
1312
1313	switch (region->type) {
1314	case MSHV_REGION_TYPE_MEM_PINNED:
1315		ret = mshv_prepare_pinned_region(region);
1316		break;
1317	case MSHV_REGION_TYPE_MEM_MOVABLE:
1318		/*
1319		 * For movable memory regions, remap with no access to let
1320		 * the hypervisor track dirty pages, enabling pre-copy live
1321		 * migration.
1322		 */
1323		ret = hv_call_map_gpa_pages(partition->pt_id,
1324					    region->start_gfn,
1325					    region->nr_pages,
1326					    HV_MAP_GPA_NO_ACCESS, NULL);
1327		break;
1328	case MSHV_REGION_TYPE_MMIO:
1329		ret = hv_call_map_mmio_pages(partition->pt_id,
1330					     region->start_gfn,
1331					     mmio_pfn,
1332					     region->nr_pages);
1333		break;
1334	}
1335
1336	if (ret)
1337		goto errout;
1338
1339	spin_lock(&partition->pt_mem_regions_lock);
1340	hlist_add_head(&region->hnode, &partition->pt_mem_regions);
1341	spin_unlock(&partition->pt_mem_regions_lock);
1342
1343	return 0;
1344
1345errout:
1346	mshv_region_put(region);
1347	return ret;
1348}
1349
1350/* Called for unmapping both the guest ram and the mmio space */
1351static long
1352mshv_unmap_user_memory(struct mshv_partition *partition,
1353		       struct mshv_user_mem_region mem)
1354{
1355	struct mshv_mem_region *region;
1356
1357	if (!(mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP)))
1358		return -EINVAL;
1359
1360	spin_lock(&partition->pt_mem_regions_lock);
1361
1362	region = mshv_partition_region_by_gfn(partition, mem.guest_pfn);
1363	if (!region) {
1364		spin_unlock(&partition->pt_mem_regions_lock);
1365		return -ENOENT;
1366	}
1367
1368	/* Paranoia check */
1369	if (region->start_uaddr != mem.userspace_addr ||
1370	    region->start_gfn != mem.guest_pfn ||
1371	    region->nr_pages != HVPFN_DOWN(mem.size)) {
1372		spin_unlock(&partition->pt_mem_regions_lock);
1373		return -EINVAL;
1374	}
1375
1376	hlist_del(&region->hnode);
1377
1378	spin_unlock(&partition->pt_mem_regions_lock);
1379
1380	mshv_region_put(region);
1381
1382	return 0;
1383}
1384
1385static long
1386mshv_partition_ioctl_set_memory(struct mshv_partition *partition,
1387				struct mshv_user_mem_region __user *user_mem)
1388{
1389	struct mshv_user_mem_region mem;
1390
1391	if (copy_from_user(&mem, user_mem, sizeof(mem)))
1392		return -EFAULT;
1393
1394	if (!mem.size ||
1395	    !PAGE_ALIGNED(mem.size) ||
1396	    !PAGE_ALIGNED(mem.userspace_addr) ||
1397	    (mem.flags & ~MSHV_SET_MEM_FLAGS_MASK) ||
1398	    mshv_field_nonzero(mem, rsvd))
1399		return -EINVAL;
1400
1401	if (mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP))
1402		return mshv_unmap_user_memory(partition, mem);
1403
1404	return mshv_map_user_memory(partition, mem);
1405}
1406
1407static long
1408mshv_partition_ioctl_ioeventfd(struct mshv_partition *partition,
1409			       void __user *user_args)
1410{
1411	struct mshv_user_ioeventfd args;
1412
1413	if (copy_from_user(&args, user_args, sizeof(args)))
1414		return -EFAULT;
1415
1416	return mshv_set_unset_ioeventfd(partition, &args);
1417}
1418
1419static long
1420mshv_partition_ioctl_irqfd(struct mshv_partition *partition,
1421			   void __user *user_args)
1422{
1423	struct mshv_user_irqfd args;
1424
1425	if (copy_from_user(&args, user_args, sizeof(args)))
1426		return -EFAULT;
1427
1428	return mshv_set_unset_irqfd(partition, &args);
1429}
1430
1431static long
1432mshv_partition_ioctl_get_gpap_access_bitmap(struct mshv_partition *partition,
1433					    void __user *user_args)
1434{
1435	struct mshv_gpap_access_bitmap args;
1436	union hv_gpa_page_access_state *states;
1437	long ret, i;
1438	union hv_gpa_page_access_state_flags hv_flags = {};
1439	u8 hv_type_mask;
1440	ulong bitmap_buf_sz, states_buf_sz;
1441	int written = 0;
1442
1443	if (copy_from_user(&args, user_args, sizeof(args)))
1444		return -EFAULT;
1445
1446	if (args.access_type >= MSHV_GPAP_ACCESS_TYPE_COUNT ||
1447	    args.access_op >= MSHV_GPAP_ACCESS_OP_COUNT ||
1448	    mshv_field_nonzero(args, rsvd) || !args.page_count ||
1449	    !args.bitmap_ptr)
1450		return -EINVAL;
1451
1452	if (check_mul_overflow(args.page_count, sizeof(*states), &states_buf_sz))
1453		return -E2BIG;
1454
1455	/* Num bytes needed to store bitmap; one bit per page rounded up */
1456	bitmap_buf_sz = DIV_ROUND_UP(args.page_count, 8);
1457
1458	/* Sanity check */
1459	if (bitmap_buf_sz > states_buf_sz)
1460		return -EBADFD;
1461
1462	switch (args.access_type) {
1463	case MSHV_GPAP_ACCESS_TYPE_ACCESSED:
1464		hv_type_mask = 1;
1465		if (args.access_op == MSHV_GPAP_ACCESS_OP_CLEAR) {
1466			hv_flags.clear_accessed = 1;
1467			/* not accessed implies not dirty */
1468			hv_flags.clear_dirty = 1;
1469		} else { /* MSHV_GPAP_ACCESS_OP_SET */
1470			hv_flags.set_accessed = 1;
1471		}
1472		break;
1473	case MSHV_GPAP_ACCESS_TYPE_DIRTY:
1474		hv_type_mask = 2;
1475		if (args.access_op == MSHV_GPAP_ACCESS_OP_CLEAR) {
1476			hv_flags.clear_dirty = 1;
1477		} else { /* MSHV_GPAP_ACCESS_OP_SET */
1478			hv_flags.set_dirty = 1;
1479			/* dirty implies accessed */
1480			hv_flags.set_accessed = 1;
1481		}
1482		break;
1483	}
1484
1485	states = vzalloc(states_buf_sz);
1486	if (!states)
1487		return -ENOMEM;
1488
1489	ret = hv_call_get_gpa_access_states(partition->pt_id, args.page_count,
1490					    args.gpap_base, hv_flags, &written,
1491					    states);
1492	if (ret)
1493		goto free_return;
1494
1495	/*
1496	 * Overwrite states buffer with bitmap - the bits in hv_type_mask
1497	 * correspond to bitfields in hv_gpa_page_access_state
1498	 */
1499	for (i = 0; i < written; ++i)
1500		__assign_bit(i, (ulong *)states,
1501			     states[i].as_uint8 & hv_type_mask);
1502
1503	/* zero the unused bits in the last byte(s) of the returned bitmap */
1504	for (i = written; i < bitmap_buf_sz * 8; ++i)
1505		__clear_bit(i, (ulong *)states);
1506
1507	if (copy_to_user((void __user *)args.bitmap_ptr, states, bitmap_buf_sz))
1508		ret = -EFAULT;
1509
1510free_return:
1511	vfree(states);
1512	return ret;
1513}
1514
1515static long
1516mshv_partition_ioctl_set_msi_routing(struct mshv_partition *partition,
1517				     void __user *user_args)
1518{
1519	struct mshv_user_irq_entry *entries = NULL;
1520	struct mshv_user_irq_table args;
1521	long ret;
1522
1523	if (copy_from_user(&args, user_args, sizeof(args)))
1524		return -EFAULT;
1525
1526	if (args.nr > MSHV_MAX_GUEST_IRQS ||
1527	    mshv_field_nonzero(args, rsvd))
1528		return -EINVAL;
1529
1530	if (args.nr) {
1531		struct mshv_user_irq_table __user *urouting = user_args;
1532
1533		entries = vmemdup_user(urouting->entries,
1534				       array_size(sizeof(*entries),
1535						  args.nr));
1536		if (IS_ERR(entries))
1537			return PTR_ERR(entries);
1538	}
1539	ret = mshv_update_routing_table(partition, entries, args.nr);
1540	kvfree(entries);
1541
1542	return ret;
1543}
1544
1545static long
1546mshv_partition_ioctl_initialize(struct mshv_partition *partition)
1547{
1548	long ret;
1549
1550	if (partition->pt_initialized)
1551		return 0;
1552
1553	ret = hv_call_initialize_partition(partition->pt_id);
1554	if (ret)
1555		goto withdraw_mem;
1556
1557	partition->pt_initialized = true;
1558
1559	return 0;
1560
1561withdraw_mem:
1562	hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id);
1563
1564	return ret;
1565}
1566
1567static long
1568mshv_partition_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1569{
1570	struct mshv_partition *partition = filp->private_data;
1571	long ret;
1572	void __user *uarg = (void __user *)arg;
1573
1574	if (mutex_lock_killable(&partition->pt_mutex))
1575		return -EINTR;
1576
1577	switch (ioctl) {
1578	case MSHV_INITIALIZE_PARTITION:
1579		ret = mshv_partition_ioctl_initialize(partition);
1580		break;
1581	case MSHV_SET_GUEST_MEMORY:
1582		ret = mshv_partition_ioctl_set_memory(partition, uarg);
1583		break;
1584	case MSHV_CREATE_VP:
1585		ret = mshv_partition_ioctl_create_vp(partition, uarg);
1586		break;
1587	case MSHV_IRQFD:
1588		ret = mshv_partition_ioctl_irqfd(partition, uarg);
1589		break;
1590	case MSHV_IOEVENTFD:
1591		ret = mshv_partition_ioctl_ioeventfd(partition, uarg);
1592		break;
1593	case MSHV_SET_MSI_ROUTING:
1594		ret = mshv_partition_ioctl_set_msi_routing(partition, uarg);
1595		break;
1596	case MSHV_GET_GPAP_ACCESS_BITMAP:
1597		ret = mshv_partition_ioctl_get_gpap_access_bitmap(partition,
1598								  uarg);
1599		break;
1600	case MSHV_ROOT_HVCALL:
1601		ret = mshv_ioctl_passthru_hvcall(partition, true, uarg);
1602		break;
1603	default:
1604		ret = -ENOTTY;
1605	}
1606
1607	mutex_unlock(&partition->pt_mutex);
1608	return ret;
1609}
1610
1611static int
1612disable_vp_dispatch(struct mshv_vp *vp)
1613{
1614	int ret;
1615	struct hv_register_assoc dispatch_suspend = {
1616		.name = HV_REGISTER_DISPATCH_SUSPEND,
1617		.value.dispatch_suspend.suspended = 1,
1618	};
1619
1620	ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
1621				    1, &dispatch_suspend);
1622	if (ret)
1623		vp_err(vp, "failed to suspend\n");
1624
1625	return ret;
1626}
1627
1628static int
1629get_vp_signaled_count(struct mshv_vp *vp, u64 *count)
1630{
1631	int ret;
1632	struct hv_register_assoc root_signal_count = {
1633		.name = HV_REGISTER_VP_ROOT_SIGNAL_COUNT,
1634	};
1635
1636	ret = mshv_get_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
1637				    1, &root_signal_count);
1638
1639	if (ret) {
1640		vp_err(vp, "Failed to get root signal count");
1641		*count = 0;
1642		return ret;
1643	}
1644
1645	*count = root_signal_count.value.reg64;
1646
1647	return ret;
1648}
1649
1650static void
1651drain_vp_signals(struct mshv_vp *vp)
1652{
1653	u64 hv_signal_count;
1654	u64 vp_signal_count;
1655
1656	get_vp_signaled_count(vp, &hv_signal_count);
1657
1658	vp_signal_count = atomic64_read(&vp->run.vp_signaled_count);
1659
1660	/*
1661	 * There should be at most 1 outstanding notification, but be extra
1662	 * careful anyway.
1663	 */
1664	while (hv_signal_count != vp_signal_count) {
1665		WARN_ON(hv_signal_count - vp_signal_count != 1);
1666
1667		if (wait_event_interruptible(vp->run.vp_suspend_queue,
1668					     vp->run.kicked_by_hv == 1))
1669			break;
1670		vp->run.kicked_by_hv = 0;
1671		vp_signal_count = atomic64_read(&vp->run.vp_signaled_count);
1672	}
1673}
1674
1675static void drain_all_vps(const struct mshv_partition *partition)
1676{
1677	int i;
1678	struct mshv_vp *vp;
1679
1680	/*
1681	 * VPs are reachable from ISR. It is safe to not take the partition
1682	 * lock because nobody else can enter this function and drop the
1683	 * partition from the list.
1684	 */
1685	for (i = 0; i < MSHV_MAX_VPS; i++) {
1686		vp = partition->pt_vp_array[i];
1687		if (!vp)
1688			continue;
1689		/*
1690		 * Disable dispatching of the VP in the hypervisor. After this
1691		 * the hypervisor guarantees it won't generate any signals for
1692		 * the VP and the hypervisor's VP signal count won't change.
1693		 */
1694		disable_vp_dispatch(vp);
1695		drain_vp_signals(vp);
1696	}
1697}
1698
1699static void
1700remove_partition(struct mshv_partition *partition)
1701{
1702	spin_lock(&mshv_root.pt_ht_lock);
1703	hlist_del_rcu(&partition->pt_hnode);
1704	spin_unlock(&mshv_root.pt_ht_lock);
1705
1706	synchronize_rcu();
1707}
1708
1709/*
1710 * Tear down a partition and remove it from the list.
1711 * Partition's refcount must be 0
1712 */
1713static void destroy_partition(struct mshv_partition *partition)
1714{
1715	struct mshv_vp *vp;
1716	struct mshv_mem_region *region;
1717	struct hlist_node *n;
1718	int i;
1719
1720	if (refcount_read(&partition->pt_ref_count)) {
1721		pt_err(partition,
1722		       "Attempt to destroy partition but refcount > 0\n");
1723		return;
1724	}
1725
1726	if (partition->pt_initialized) {
1727		/*
1728		 * We only need to drain signals for root scheduler. This should be
1729		 * done before removing the partition from the partition list.
1730		 */
1731		if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
1732			drain_all_vps(partition);
1733
1734		/* Remove vps */
1735		for (i = 0; i < MSHV_MAX_VPS; ++i) {
1736			vp = partition->pt_vp_array[i];
1737			if (!vp)
1738				continue;
1739
1740			if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
1741				mshv_vp_stats_unmap(partition->pt_id, vp->vp_index,
1742						    (void **)vp->vp_stats_pages);
1743
1744			if (vp->vp_register_page) {
1745				(void)hv_unmap_vp_state_page(partition->pt_id,
1746							     vp->vp_index,
1747							     HV_VP_STATE_PAGE_REGISTERS,
1748							     virt_to_page(vp->vp_register_page),
1749							     input_vtl_zero);
1750				vp->vp_register_page = NULL;
1751			}
1752
1753			(void)hv_unmap_vp_state_page(partition->pt_id,
1754						     vp->vp_index,
1755						     HV_VP_STATE_PAGE_INTERCEPT_MESSAGE,
1756						     virt_to_page(vp->vp_intercept_msg_page),
1757						     input_vtl_zero);
1758			vp->vp_intercept_msg_page = NULL;
1759
1760			if (vp->vp_ghcb_page) {
1761				(void)hv_unmap_vp_state_page(partition->pt_id,
1762							     vp->vp_index,
1763							     HV_VP_STATE_PAGE_GHCB,
1764							     virt_to_page(vp->vp_ghcb_page),
1765							     input_vtl_normal);
1766				vp->vp_ghcb_page = NULL;
1767			}
1768
1769			kfree(vp);
1770
1771			partition->pt_vp_array[i] = NULL;
1772		}
1773
1774		/* Deallocates and unmaps everything including vcpus, GPA mappings etc */
1775		hv_call_finalize_partition(partition->pt_id);
1776
1777		partition->pt_initialized = false;
1778	}
1779
1780	remove_partition(partition);
1781
1782	hlist_for_each_entry_safe(region, n, &partition->pt_mem_regions,
1783				  hnode) {
1784		hlist_del(&region->hnode);
1785		mshv_region_put(region);
1786	}
1787
1788	/* Withdraw and free all pages we deposited */
1789	hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id);
1790	hv_call_delete_partition(partition->pt_id);
1791
1792	mshv_free_routing_table(partition);
1793	kfree(partition);
1794}
1795
1796struct
1797mshv_partition *mshv_partition_get(struct mshv_partition *partition)
1798{
1799	if (refcount_inc_not_zero(&partition->pt_ref_count))
1800		return partition;
1801	return NULL;
1802}
1803
1804struct
1805mshv_partition *mshv_partition_find(u64 partition_id)
1806	__must_hold(RCU)
1807{
1808	struct mshv_partition *p;
1809
1810	hash_for_each_possible_rcu(mshv_root.pt_htable, p, pt_hnode,
1811				   partition_id)
1812		if (p->pt_id == partition_id)
1813			return p;
1814
1815	return NULL;
1816}
1817
1818void
1819mshv_partition_put(struct mshv_partition *partition)
1820{
1821	if (refcount_dec_and_test(&partition->pt_ref_count))
1822		destroy_partition(partition);
1823}
1824
1825static int
1826mshv_partition_release(struct inode *inode, struct file *filp)
1827{
1828	struct mshv_partition *partition = filp->private_data;
1829
1830	mshv_eventfd_release(partition);
1831
1832	cleanup_srcu_struct(&partition->pt_irq_srcu);
1833
1834	mshv_partition_put(partition);
1835
1836	return 0;
1837}
1838
1839static int
1840add_partition(struct mshv_partition *partition)
1841{
1842	spin_lock(&mshv_root.pt_ht_lock);
1843
1844	hash_add_rcu(mshv_root.pt_htable, &partition->pt_hnode,
1845		     partition->pt_id);
1846
1847	spin_unlock(&mshv_root.pt_ht_lock);
1848
1849	return 0;
1850}
1851
1852static_assert(MSHV_NUM_CPU_FEATURES_BANKS ==
1853	      HV_PARTITION_PROCESSOR_FEATURES_BANKS);
1854
1855static long mshv_ioctl_process_pt_flags(void __user *user_arg, u64 *pt_flags,
1856					struct hv_partition_creation_properties *cr_props,
1857					union hv_partition_isolation_properties *isol_props)
1858{
1859	int i;
1860	struct mshv_create_partition_v2 args;
1861	union hv_partition_processor_features *disabled_procs;
1862	union hv_partition_processor_xsave_features *disabled_xsave;
1863
1864	/* First, copy v1 struct in case user is on previous versions */
1865	if (copy_from_user(&args, user_arg,
1866			   sizeof(struct mshv_create_partition)))
1867		return -EFAULT;
1868
1869	if ((args.pt_flags & ~MSHV_PT_FLAGS_MASK) ||
1870	    args.pt_isolation >= MSHV_PT_ISOLATION_COUNT)
1871		return -EINVAL;
1872
1873	disabled_procs = &cr_props->disabled_processor_features;
1874	disabled_xsave = &cr_props->disabled_processor_xsave_features;
1875
1876	/* Check if user provided newer struct with feature fields */
1877	if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_CPU_AND_XSAVE_FEATURES)) {
1878		if (copy_from_user(&args, user_arg, sizeof(args)))
1879			return -EFAULT;
1880
1881		/* Re-validate v1 fields after second copy_from_user() */
1882		if ((args.pt_flags & ~MSHV_PT_FLAGS_MASK) ||
1883		    args.pt_isolation >= MSHV_PT_ISOLATION_COUNT)
1884			return -EINVAL;
1885
1886		if (args.pt_num_cpu_fbanks != MSHV_NUM_CPU_FEATURES_BANKS ||
1887		    mshv_field_nonzero(args, pt_rsvd) ||
1888		    mshv_field_nonzero(args, pt_rsvd1))
1889			return -EINVAL;
1890
1891		/*
1892		 * Note this assumes MSHV_NUM_CPU_FEATURES_BANKS will never
1893		 * change and equals HV_PARTITION_PROCESSOR_FEATURES_BANKS
1894		 * (i.e. 2).
1895		 *
1896		 * Further banks (index >= 2) will be modifiable as 'early'
1897		 * properties via the set partition property hypercall.
1898		 */
1899		for (i = 0; i < HV_PARTITION_PROCESSOR_FEATURES_BANKS; i++)
1900			disabled_procs->as_uint64[i] = args.pt_cpu_fbanks[i];
1901
1902#if IS_ENABLED(CONFIG_X86_64)
1903		disabled_xsave->as_uint64 = args.pt_disabled_xsave;
1904#else
1905		/*
1906		 * In practice this field is ignored on arm64, but safer to
1907		 * zero it in case it is ever used.
1908		 */
1909		disabled_xsave->as_uint64 = 0;
1910
1911		if (mshv_field_nonzero(args, pt_rsvd2))
1912			return -EINVAL;
1913#endif
1914	} else {
1915		/*
1916		 * v1 behavior: try to enable everything. The hypervisor will
1917		 * disable features that are not supported. The banks can be
1918		 * queried via the get partition property hypercall.
1919		 */
1920		for (i = 0; i < HV_PARTITION_PROCESSOR_FEATURES_BANKS; i++)
1921			disabled_procs->as_uint64[i] = 0;
1922
1923		disabled_xsave->as_uint64 = 0;
1924	}
1925
1926	/* Only support EXO partitions */
1927	*pt_flags = HV_PARTITION_CREATION_FLAG_EXO_PARTITION |
1928		    HV_PARTITION_CREATION_FLAG_INTERCEPT_MESSAGE_PAGE_ENABLED;
1929
1930	if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_LAPIC))
1931		*pt_flags |= HV_PARTITION_CREATION_FLAG_LAPIC_ENABLED;
1932	if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_X2APIC))
1933		*pt_flags |= HV_PARTITION_CREATION_FLAG_X2APIC_CAPABLE;
1934	if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_GPA_SUPER_PAGES))
1935		*pt_flags |= HV_PARTITION_CREATION_FLAG_GPA_SUPER_PAGES_ENABLED;
1936
1937	isol_props->as_uint64 = 0;
1938
1939	switch (args.pt_isolation) {
1940	case MSHV_PT_ISOLATION_NONE:
1941		isol_props->isolation_type = HV_PARTITION_ISOLATION_TYPE_NONE;
1942		break;
1943	}
1944
1945	return 0;
1946}
1947
1948static long
1949mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev)
1950{
1951	u64 creation_flags;
1952	struct hv_partition_creation_properties creation_properties;
1953	union hv_partition_isolation_properties isolation_properties;
1954	struct mshv_partition *partition;
1955	long ret;
1956
1957	ret = mshv_ioctl_process_pt_flags(user_arg, &creation_flags,
1958					  &creation_properties,
1959					  &isolation_properties);
1960	if (ret)
1961		return ret;
1962
1963	partition = kzalloc(sizeof(*partition), GFP_KERNEL);
1964	if (!partition)
1965		return -ENOMEM;
1966
1967	partition->pt_module_dev = module_dev;
1968	partition->isolation_type = isolation_properties.isolation_type;
1969
1970	refcount_set(&partition->pt_ref_count, 1);
1971
1972	mutex_init(&partition->pt_mutex);
1973
1974	mutex_init(&partition->pt_irq_lock);
1975
1976	init_completion(&partition->async_hypercall);
1977
1978	INIT_HLIST_HEAD(&partition->irq_ack_notifier_list);
1979
1980	INIT_HLIST_HEAD(&partition->pt_devices);
1981
1982	spin_lock_init(&partition->pt_mem_regions_lock);
1983	INIT_HLIST_HEAD(&partition->pt_mem_regions);
1984
1985	mshv_eventfd_init(partition);
1986
1987	ret = init_srcu_struct(&partition->pt_irq_srcu);
1988	if (ret)
1989		goto free_partition;
1990
1991	ret = hv_call_create_partition(creation_flags,
1992				       creation_properties,
1993				       isolation_properties,
1994				       &partition->pt_id);
1995	if (ret)
1996		goto cleanup_irq_srcu;
1997
1998	ret = add_partition(partition);
1999	if (ret)
2000		goto delete_partition;
2001
2002	ret = mshv_init_async_handler(partition);
2003	if (!ret) {
2004		ret = FD_ADD(O_CLOEXEC, anon_inode_getfile("mshv_partition",
2005							   &mshv_partition_fops,
2006							   partition, O_RDWR));
2007		if (ret >= 0)
2008			return ret;
2009	}
2010	remove_partition(partition);
2011delete_partition:
2012	hv_call_delete_partition(partition->pt_id);
2013cleanup_irq_srcu:
2014	cleanup_srcu_struct(&partition->pt_irq_srcu);
2015free_partition:
2016	kfree(partition);
2017
2018	return ret;
2019}
2020
2021static long mshv_dev_ioctl(struct file *filp, unsigned int ioctl,
2022			   unsigned long arg)
2023{
2024	struct miscdevice *misc = filp->private_data;
2025
2026	switch (ioctl) {
2027	case MSHV_CREATE_PARTITION:
2028		return mshv_ioctl_create_partition((void __user *)arg,
2029						misc->this_device);
2030	case MSHV_ROOT_HVCALL:
2031		return mshv_ioctl_passthru_hvcall(NULL, false,
2032					(void __user *)arg);
2033	}
2034
2035	return -ENOTTY;
2036}
2037
2038static int
2039mshv_dev_open(struct inode *inode, struct file *filp)
2040{
2041	return 0;
2042}
2043
2044static int
2045mshv_dev_release(struct inode *inode, struct file *filp)
2046{
2047	return 0;
2048}
2049
2050static int mshv_cpuhp_online;
2051static int mshv_root_sched_online;
2052
2053static const char *scheduler_type_to_string(enum hv_scheduler_type type)
2054{
2055	switch (type) {
2056	case HV_SCHEDULER_TYPE_LP:
2057		return "classic scheduler without SMT";
2058	case HV_SCHEDULER_TYPE_LP_SMT:
2059		return "classic scheduler with SMT";
2060	case HV_SCHEDULER_TYPE_CORE_SMT:
2061		return "core scheduler";
2062	case HV_SCHEDULER_TYPE_ROOT:
2063		return "root scheduler";
2064	default:
2065		return "unknown scheduler";
2066	};
2067}
2068
2069/* TODO move this to hv_common.c when needed outside */
2070static int __init hv_retrieve_scheduler_type(enum hv_scheduler_type *out)
2071{
2072	struct hv_input_get_system_property *input;
2073	struct hv_output_get_system_property *output;
2074	unsigned long flags;
2075	u64 status;
2076
2077	local_irq_save(flags);
2078	input = *this_cpu_ptr(hyperv_pcpu_input_arg);
2079	output = *this_cpu_ptr(hyperv_pcpu_output_arg);
2080
2081	memset(input, 0, sizeof(*input));
2082	memset(output, 0, sizeof(*output));
2083	input->property_id = HV_SYSTEM_PROPERTY_SCHEDULER_TYPE;
2084
2085	status = hv_do_hypercall(HVCALL_GET_SYSTEM_PROPERTY, input, output);
2086	if (!hv_result_success(status)) {
2087		local_irq_restore(flags);
2088		pr_err("%s: %s\n", __func__, hv_result_to_string(status));
2089		return hv_result_to_errno(status);
2090	}
2091
2092	*out = output->scheduler_type;
2093	local_irq_restore(flags);
2094
2095	return 0;
2096}
2097
2098/* Retrieve and stash the supported scheduler type */
2099static int __init mshv_retrieve_scheduler_type(struct device *dev)
2100{
2101	int ret = 0;
2102
2103	if (hv_l1vh_partition())
2104		hv_scheduler_type = HV_SCHEDULER_TYPE_CORE_SMT;
2105	else
2106		ret = hv_retrieve_scheduler_type(&hv_scheduler_type);
2107
2108	if (ret)
2109		return ret;
2110
2111	dev_info(dev, "Hypervisor using %s\n",
2112		 scheduler_type_to_string(hv_scheduler_type));
2113
2114	switch (hv_scheduler_type) {
2115	case HV_SCHEDULER_TYPE_CORE_SMT:
2116	case HV_SCHEDULER_TYPE_LP_SMT:
2117	case HV_SCHEDULER_TYPE_ROOT:
2118	case HV_SCHEDULER_TYPE_LP:
2119		/* Supported scheduler, nothing to do */
2120		break;
2121	default:
2122		dev_err(dev, "unsupported scheduler 0x%x, bailing.\n",
2123			hv_scheduler_type);
2124		return -EOPNOTSUPP;
2125	}
2126
2127	return 0;
2128}
2129
2130static int mshv_root_scheduler_init(unsigned int cpu)
2131{
2132	void **inputarg, **outputarg, *p;
2133
2134	inputarg = (void **)this_cpu_ptr(root_scheduler_input);
2135	outputarg = (void **)this_cpu_ptr(root_scheduler_output);
2136
2137	/* Allocate two consecutive pages. One for input, one for output. */
2138	p = kmalloc(2 * HV_HYP_PAGE_SIZE, GFP_KERNEL);
2139	if (!p)
2140		return -ENOMEM;
2141
2142	*inputarg = p;
2143	*outputarg = (char *)p + HV_HYP_PAGE_SIZE;
2144
2145	return 0;
2146}
2147
2148static int mshv_root_scheduler_cleanup(unsigned int cpu)
2149{
2150	void *p, **inputarg, **outputarg;
2151
2152	inputarg = (void **)this_cpu_ptr(root_scheduler_input);
2153	outputarg = (void **)this_cpu_ptr(root_scheduler_output);
2154
2155	p = *inputarg;
2156
2157	*inputarg = NULL;
2158	*outputarg = NULL;
2159
2160	kfree(p);
2161
2162	return 0;
2163}
2164
2165/* Must be called after retrieving the scheduler type */
2166static int
2167root_scheduler_init(struct device *dev)
2168{
2169	int ret;
2170
2171	if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT)
2172		return 0;
2173
2174	root_scheduler_input = alloc_percpu(void *);
2175	root_scheduler_output = alloc_percpu(void *);
2176
2177	if (!root_scheduler_input || !root_scheduler_output) {
2178		dev_err(dev, "Failed to allocate root scheduler buffers\n");
2179		ret = -ENOMEM;
2180		goto out;
2181	}
2182
2183	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_root_sched",
2184				mshv_root_scheduler_init,
2185				mshv_root_scheduler_cleanup);
2186
2187	if (ret < 0) {
2188		dev_err(dev, "Failed to setup root scheduler state: %i\n", ret);
2189		goto out;
2190	}
2191
2192	mshv_root_sched_online = ret;
2193
2194	return 0;
2195
2196out:
2197	free_percpu(root_scheduler_input);
2198	free_percpu(root_scheduler_output);
2199	return ret;
2200}
2201
2202static void
2203root_scheduler_deinit(void)
2204{
2205	if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT)
2206		return;
2207
2208	cpuhp_remove_state(mshv_root_sched_online);
2209	free_percpu(root_scheduler_input);
2210	free_percpu(root_scheduler_output);
2211}
2212
2213static int mshv_reboot_notify(struct notifier_block *nb,
2214			      unsigned long code, void *unused)
2215{
2216	cpuhp_remove_state(mshv_cpuhp_online);
2217	return 0;
2218}
2219
2220struct notifier_block mshv_reboot_nb = {
2221	.notifier_call = mshv_reboot_notify,
2222};
2223
2224static void mshv_root_partition_exit(void)
2225{
2226	unregister_reboot_notifier(&mshv_reboot_nb);
2227	root_scheduler_deinit();
2228}
2229
2230static int __init mshv_root_partition_init(struct device *dev)
2231{
2232	int err;
2233
2234	err = root_scheduler_init(dev);
2235	if (err)
2236		return err;
2237
2238	err = register_reboot_notifier(&mshv_reboot_nb);
2239	if (err)
2240		goto root_sched_deinit;
2241
2242	return 0;
2243
2244root_sched_deinit:
2245	root_scheduler_deinit();
2246	return err;
2247}
2248
2249static void mshv_init_vmm_caps(struct device *dev)
2250{
2251	/*
2252	 * This can only fail here if HVCALL_GET_PARTITION_PROPERTY_EX or
2253	 * HV_PARTITION_PROPERTY_VMM_CAPABILITIES are not supported. In that
2254	 * case it's valid to proceed as if all vmm_caps are disabled (zero).
2255	 */
2256	if (hv_call_get_partition_property_ex(HV_PARTITION_ID_SELF,
2257					      HV_PARTITION_PROPERTY_VMM_CAPABILITIES,
2258					      0, &mshv_root.vmm_caps,
2259					      sizeof(mshv_root.vmm_caps)))
2260		dev_warn(dev, "Unable to get VMM capabilities\n");
2261
2262	dev_dbg(dev, "vmm_caps = %#llx\n", mshv_root.vmm_caps.as_uint64[0]);
2263}
2264
2265static int __init mshv_parent_partition_init(void)
2266{
2267	int ret;
2268	struct device *dev;
2269	union hv_hypervisor_version_info version_info;
2270
2271	if (!hv_parent_partition() || is_kdump_kernel())
2272		return -ENODEV;
2273
2274	if (hv_get_hypervisor_version(&version_info))
2275		return -ENODEV;
2276
2277	ret = misc_register(&mshv_dev);
2278	if (ret)
2279		return ret;
2280
2281	dev = mshv_dev.this_device;
2282
2283	if (version_info.build_number < MSHV_HV_MIN_VERSION ||
2284	    version_info.build_number > MSHV_HV_MAX_VERSION) {
2285		dev_err(dev, "Running on unvalidated Hyper-V version\n");
2286		dev_err(dev, "Versions: current: %u  min: %u  max: %u\n",
2287			version_info.build_number, MSHV_HV_MIN_VERSION,
2288			MSHV_HV_MAX_VERSION);
2289	}
2290
2291	mshv_root.synic_pages = alloc_percpu(struct hv_synic_pages);
2292	if (!mshv_root.synic_pages) {
2293		dev_err(dev, "Failed to allocate percpu synic page\n");
2294		ret = -ENOMEM;
2295		goto device_deregister;
2296	}
2297
2298	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_synic",
2299				mshv_synic_init,
2300				mshv_synic_cleanup);
2301	if (ret < 0) {
2302		dev_err(dev, "Failed to setup cpu hotplug state: %i\n", ret);
2303		goto free_synic_pages;
2304	}
2305
2306	mshv_cpuhp_online = ret;
2307
2308	ret = mshv_retrieve_scheduler_type(dev);
2309	if (ret)
2310		goto remove_cpu_state;
2311
2312	if (hv_root_partition())
2313		ret = mshv_root_partition_init(dev);
2314	if (ret)
2315		goto remove_cpu_state;
2316
2317	mshv_init_vmm_caps(dev);
2318
2319	ret = mshv_irqfd_wq_init();
2320	if (ret)
2321		goto exit_partition;
2322
2323	spin_lock_init(&mshv_root.pt_ht_lock);
2324	hash_init(mshv_root.pt_htable);
2325
2326	hv_setup_mshv_handler(mshv_isr);
2327
2328	return 0;
2329
2330exit_partition:
2331	if (hv_root_partition())
2332		mshv_root_partition_exit();
2333remove_cpu_state:
2334	cpuhp_remove_state(mshv_cpuhp_online);
2335free_synic_pages:
2336	free_percpu(mshv_root.synic_pages);
2337device_deregister:
2338	misc_deregister(&mshv_dev);
2339	return ret;
2340}
2341
2342static void __exit mshv_parent_partition_exit(void)
2343{
2344	hv_setup_mshv_handler(NULL);
2345	mshv_port_table_fini();
2346	misc_deregister(&mshv_dev);
2347	mshv_irqfd_wq_cleanup();
2348	if (hv_root_partition())
2349		mshv_root_partition_exit();
2350	cpuhp_remove_state(mshv_cpuhp_online);
2351	free_percpu(mshv_root.synic_pages);
2352}
2353
2354module_init(mshv_parent_partition_init);
2355module_exit(mshv_parent_partition_exit);