mshv_vtl_main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2023, Microsoft Corporation.
   4 *
   5 * Author:
   6 *   Roman Kisel <romank@linux.microsoft.com>
   7 *   Saurabh Sengar <ssengar@linux.microsoft.com>
   8 *   Naman Jain <namjain@linux.microsoft.com>
   9 */
  10
  11#include <linux/kernel.h>
  12#include <linux/module.h>
  13#include <linux/miscdevice.h>
  14#include <linux/anon_inodes.h>
  15#include <linux/cpuhotplug.h>
  16#include <linux/count_zeros.h>
  17#include <linux/entry-virt.h>
  18#include <linux/eventfd.h>
  19#include <linux/poll.h>
  20#include <linux/file.h>
  21#include <linux/vmalloc.h>
  22#include <asm/debugreg.h>
  23#include <asm/mshyperv.h>
  24#include <trace/events/ipi.h>
  25#include <uapi/asm/mtrr.h>
  26#include <uapi/linux/mshv.h>
  27#include <hyperv/hvhdk.h>
  28
  29#include "../../kernel/fpu/legacy.h"
  30#include "mshv.h"
  31#include "mshv_vtl.h"
  32#include "hyperv_vmbus.h"
  33
  34MODULE_AUTHOR("Microsoft");
  35MODULE_LICENSE("GPL");
  36MODULE_DESCRIPTION("Microsoft Hyper-V VTL Driver");
  37
  38#define MSHV_ENTRY_REASON_LOWER_VTL_CALL     0x1
  39#define MSHV_ENTRY_REASON_INTERRUPT          0x2
  40#define MSHV_ENTRY_REASON_INTERCEPT          0x3
  41
  42#define MSHV_REAL_OFF_SHIFT	16
  43#define MSHV_PG_OFF_CPU_MASK	(BIT_ULL(MSHV_REAL_OFF_SHIFT) - 1)
  44#define MSHV_RUN_PAGE_OFFSET	0
  45#define MSHV_REG_PAGE_OFFSET	1
  46#define VTL2_VMBUS_SINT_INDEX	7
  47
  48static struct device *mem_dev;
  49
  50static struct tasklet_struct msg_dpc;
  51static wait_queue_head_t fd_wait_queue;
  52static bool has_message;
  53static struct eventfd_ctx *flag_eventfds[HV_EVENT_FLAGS_COUNT];
  54static DEFINE_MUTEX(flag_lock);
  55static bool __read_mostly mshv_has_reg_page;
  56
  57/* hvcall code is of type u16, allocate a bitmap of size (1 << 16) to accommodate it */
  58#define MAX_BITMAP_SIZE ((U16_MAX + 1) / 8)
  59
  60struct mshv_vtl_hvcall_fd {
  61	u8 allow_bitmap[MAX_BITMAP_SIZE];
  62	bool allow_map_initialized;
  63	/*
  64	 * Used to protect hvcall setup in IOCTLs
  65	 */
  66	struct mutex init_mutex;
  67	struct miscdevice *dev;
  68};
  69
  70struct mshv_vtl_poll_file {
  71	struct file *file;
  72	wait_queue_entry_t wait;
  73	wait_queue_head_t *wqh;
  74	poll_table pt;
  75	int cpu;
  76};
  77
  78struct mshv_vtl {
  79	struct device *module_dev;
  80	u64 id;
  81};
  82
  83struct mshv_vtl_per_cpu {
  84	struct mshv_vtl_run *run;
  85	struct page *reg_page;
  86};
  87
  88/* SYNIC_OVERLAY_PAGE_MSR - internal, identical to hv_synic_simp */
  89union hv_synic_overlay_page_msr {
  90	u64 as_uint64;
  91	struct {
  92		u64 enabled: 1;
  93		u64 reserved: 11;
  94		u64 pfn: 52;
  95	} __packed;
  96};
  97
  98static struct mutex mshv_vtl_poll_file_lock;
  99static union hv_register_vsm_page_offsets mshv_vsm_page_offsets;
 100static union hv_register_vsm_capabilities mshv_vsm_capabilities;
 101
 102static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file);
 103static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions);
 104static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu);
 105
 106static const union hv_input_vtl input_vtl_zero;
 107static const union hv_input_vtl input_vtl_normal = {
 108	.use_target_vtl = 1,
 109};
 110
 111static const struct file_operations mshv_vtl_fops;
 112
 113static long
 114mshv_ioctl_create_vtl(void __user *user_arg, struct device *module_dev)
 115{
 116	struct mshv_vtl *vtl;
 117	struct file *file;
 118	int fd;
 119
 120	vtl = kzalloc(sizeof(*vtl), GFP_KERNEL);
 121	if (!vtl)
 122		return -ENOMEM;
 123
 124	fd = get_unused_fd_flags(O_CLOEXEC);
 125	if (fd < 0) {
 126		kfree(vtl);
 127		return fd;
 128	}
 129	file = anon_inode_getfile("mshv_vtl", &mshv_vtl_fops,
 130				  vtl, O_RDWR);
 131	if (IS_ERR(file)) {
 132		kfree(vtl);
 133		return PTR_ERR(file);
 134	}
 135	vtl->module_dev = module_dev;
 136	fd_install(fd, file);
 137
 138	return fd;
 139}
 140
 141static long
 142mshv_ioctl_check_extension(void __user *user_arg)
 143{
 144	u32 arg;
 145
 146	if (copy_from_user(&arg, user_arg, sizeof(arg)))
 147		return -EFAULT;
 148
 149	switch (arg) {
 150	case MSHV_CAP_CORE_API_STABLE:
 151		return 0;
 152	case MSHV_CAP_REGISTER_PAGE:
 153		return mshv_has_reg_page;
 154	case MSHV_CAP_VTL_RETURN_ACTION:
 155		return mshv_vsm_capabilities.return_action_available;
 156	case MSHV_CAP_DR6_SHARED:
 157		return mshv_vsm_capabilities.dr6_shared;
 158	}
 159
 160	return -EOPNOTSUPP;
 161}
 162
 163static long
 164mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 165{
 166	struct miscdevice *misc = filp->private_data;
 167
 168	switch (ioctl) {
 169	case MSHV_CHECK_EXTENSION:
 170		return mshv_ioctl_check_extension((void __user *)arg);
 171	case MSHV_CREATE_VTL:
 172		return mshv_ioctl_create_vtl((void __user *)arg, misc->this_device);
 173	}
 174
 175	return -ENOTTY;
 176}
 177
 178static const struct file_operations mshv_dev_fops = {
 179	.owner		= THIS_MODULE,
 180	.unlocked_ioctl	= mshv_dev_ioctl,
 181	.llseek		= noop_llseek,
 182};
 183
 184static struct miscdevice mshv_dev = {
 185	.minor = MISC_DYNAMIC_MINOR,
 186	.name = "mshv",
 187	.fops = &mshv_dev_fops,
 188	.mode = 0600,
 189};
 190
 191static struct mshv_vtl_run *mshv_vtl_this_run(void)
 192{
 193	return *this_cpu_ptr(&mshv_vtl_per_cpu.run);
 194}
 195
 196static struct mshv_vtl_run *mshv_vtl_cpu_run(int cpu)
 197{
 198	return *per_cpu_ptr(&mshv_vtl_per_cpu.run, cpu);
 199}
 200
 201static struct page *mshv_vtl_cpu_reg_page(int cpu)
 202{
 203	return *per_cpu_ptr(&mshv_vtl_per_cpu.reg_page, cpu);
 204}
 205
 206static void mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu *per_cpu)
 207{
 208	struct hv_register_assoc reg_assoc = {};
 209	union hv_synic_overlay_page_msr overlay = {};
 210	struct page *reg_page;
 211
 212	reg_page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL);
 213	if (!reg_page) {
 214		WARN(1, "failed to allocate register page\n");
 215		return;
 216	}
 217
 218	overlay.enabled = 1;
 219	overlay.pfn = page_to_hvpfn(reg_page);
 220	reg_assoc.name = HV_X64_REGISTER_REG_PAGE;
 221	reg_assoc.value.reg64 = overlay.as_uint64;
 222
 223	if (hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
 224				     1, input_vtl_zero, &reg_assoc)) {
 225		WARN(1, "failed to setup register page\n");
 226		__free_page(reg_page);
 227		return;
 228	}
 229
 230	per_cpu->reg_page = reg_page;
 231	mshv_has_reg_page = true;
 232}
 233
 234static void mshv_vtl_synic_enable_regs(unsigned int cpu)
 235{
 236	union hv_synic_sint sint;
 237
 238	sint.as_uint64 = 0;
 239	sint.vector = HYPERVISOR_CALLBACK_VECTOR;
 240	sint.masked = false;
 241	sint.auto_eoi = hv_recommend_using_aeoi();
 242
 243	/* Enable intercepts */
 244	if (!mshv_vsm_capabilities.intercept_page_available)
 245		hv_set_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX,
 246			   sint.as_uint64);
 247
 248	/* VTL2 Host VSP SINT is (un)masked when the user mode requests that */
 249}
 250
 251static int mshv_vtl_get_vsm_regs(void)
 252{
 253	struct hv_register_assoc registers[2];
 254	int ret, count = 2;
 255
 256	registers[0].name = HV_REGISTER_VSM_CODE_PAGE_OFFSETS;
 257	registers[1].name = HV_REGISTER_VSM_CAPABILITIES;
 258
 259	ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
 260				       count, input_vtl_zero, registers);
 261	if (ret)
 262		return ret;
 263
 264	mshv_vsm_page_offsets.as_uint64 = registers[0].value.reg64;
 265	mshv_vsm_capabilities.as_uint64 = registers[1].value.reg64;
 266
 267	return ret;
 268}
 269
 270static int mshv_vtl_configure_vsm_partition(struct device *dev)
 271{
 272	union hv_register_vsm_partition_config config;
 273	struct hv_register_assoc reg_assoc;
 274
 275	config.as_uint64 = 0;
 276	config.default_vtl_protection_mask = HV_MAP_GPA_PERMISSIONS_MASK;
 277	config.enable_vtl_protection = 1;
 278	config.zero_memory_on_reset = 1;
 279	config.intercept_vp_startup = 1;
 280	config.intercept_cpuid_unimplemented = 1;
 281
 282	if (mshv_vsm_capabilities.intercept_page_available) {
 283		dev_dbg(dev, "using intercept page\n");
 284		config.intercept_page = 1;
 285	}
 286
 287	reg_assoc.name = HV_REGISTER_VSM_PARTITION_CONFIG;
 288	reg_assoc.value.reg64 = config.as_uint64;
 289
 290	return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
 291				       1, input_vtl_zero, &reg_assoc);
 292}
 293
 294static void mshv_vtl_vmbus_isr(void)
 295{
 296	struct hv_per_cpu_context *per_cpu;
 297	struct hv_message *msg;
 298	u32 message_type;
 299	union hv_synic_event_flags *event_flags;
 300	struct eventfd_ctx *eventfd;
 301	u16 i;
 302
 303	per_cpu = this_cpu_ptr(hv_context.cpu_context);
 304	if (smp_processor_id() == 0) {
 305		msg = (struct hv_message *)per_cpu->hyp_synic_message_page + VTL2_VMBUS_SINT_INDEX;
 306		message_type = READ_ONCE(msg->header.message_type);
 307		if (message_type != HVMSG_NONE)
 308			tasklet_schedule(&msg_dpc);
 309	}
 310
 311	event_flags = (union hv_synic_event_flags *)per_cpu->hyp_synic_event_page +
 312			VTL2_VMBUS_SINT_INDEX;
 313	for_each_set_bit(i, event_flags->flags, HV_EVENT_FLAGS_COUNT) {
 314		if (!sync_test_and_clear_bit(i, event_flags->flags))
 315			continue;
 316		rcu_read_lock();
 317		eventfd = READ_ONCE(flag_eventfds[i]);
 318		if (eventfd)
 319			eventfd_signal(eventfd);
 320		rcu_read_unlock();
 321	}
 322
 323	vmbus_isr();
 324}
 325
 326static int mshv_vtl_alloc_context(unsigned int cpu)
 327{
 328	struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu);
 329
 330	per_cpu->run = (struct mshv_vtl_run *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
 331	if (!per_cpu->run)
 332		return -ENOMEM;
 333
 334	if (mshv_vsm_capabilities.intercept_page_available)
 335		mshv_vtl_configure_reg_page(per_cpu);
 336
 337	mshv_vtl_synic_enable_regs(cpu);
 338
 339	return 0;
 340}
 341
 342static int mshv_vtl_cpuhp_online;
 343
 344static int hv_vtl_setup_synic(void)
 345{
 346	int ret;
 347
 348	/* Use our isr to first filter out packets destined for userspace */
 349	hv_setup_vmbus_handler(mshv_vtl_vmbus_isr);
 350
 351	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vtl:online",
 352				mshv_vtl_alloc_context, NULL);
 353	if (ret < 0) {
 354		hv_setup_vmbus_handler(vmbus_isr);
 355		return ret;
 356	}
 357
 358	mshv_vtl_cpuhp_online = ret;
 359
 360	return 0;
 361}
 362
 363static void hv_vtl_remove_synic(void)
 364{
 365	cpuhp_remove_state(mshv_vtl_cpuhp_online);
 366	hv_setup_vmbus_handler(vmbus_isr);
 367}
 368
 369static int vtl_get_vp_register(struct hv_register_assoc *reg)
 370{
 371	return hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
 372					1, input_vtl_normal, reg);
 373}
 374
 375static int vtl_set_vp_register(struct hv_register_assoc *reg)
 376{
 377	return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF,
 378					1, input_vtl_normal, reg);
 379}
 380
 381static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg)
 382{
 383	struct mshv_vtl_ram_disposition vtl0_mem;
 384	struct dev_pagemap *pgmap;
 385	void *addr;
 386
 387	if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem)))
 388		return -EFAULT;
 389	/* vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design */
 390	if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) {
 391		dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n",
 392			vtl0_mem.start_pfn, vtl0_mem.last_pfn);
 393		return -EFAULT;
 394	}
 395
 396	pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL);
 397	if (!pgmap)
 398		return -ENOMEM;
 399
 400	pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn);
 401	pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1;
 402	pgmap->nr_range = 1;
 403	pgmap->type = MEMORY_DEVICE_GENERIC;
 404
 405	/*
 406	 * Determine the highest page order that can be used for the given memory range.
 407	 * This works best when the range is aligned; i.e. both the start and the length.
 408	 */
 409	pgmap->vmemmap_shift = count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn);
 410	dev_dbg(vtl->module_dev,
 411		"Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n",
 412		vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift);
 413
 414	addr = devm_memremap_pages(mem_dev, pgmap);
 415	if (IS_ERR(addr)) {
 416		dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr));
 417		kfree(pgmap);
 418		return -EFAULT;
 419	}
 420
 421	/* Don't free pgmap, since it has to stick around until the memory
 422	 * is unmapped, which will never happen as there is no scenario
 423	 * where VTL0 can be released/shutdown without bringing down VTL2.
 424	 */
 425	return 0;
 426}
 427
 428static void mshv_vtl_cancel(int cpu)
 429{
 430	int here = get_cpu();
 431
 432	if (here != cpu) {
 433		if (!xchg_relaxed(&mshv_vtl_cpu_run(cpu)->cancel, 1))
 434			smp_send_reschedule(cpu);
 435	} else {
 436		WRITE_ONCE(mshv_vtl_this_run()->cancel, 1);
 437	}
 438	put_cpu();
 439}
 440
 441static int mshv_vtl_poll_file_wake(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key)
 442{
 443	struct mshv_vtl_poll_file *poll_file = container_of(wait, struct mshv_vtl_poll_file, wait);
 444
 445	mshv_vtl_cancel(poll_file->cpu);
 446
 447	return 0;
 448}
 449
 450static void mshv_vtl_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
 451{
 452	struct mshv_vtl_poll_file *poll_file = container_of(pt, struct mshv_vtl_poll_file, pt);
 453
 454	WARN_ON(poll_file->wqh);
 455	poll_file->wqh = wqh;
 456	add_wait_queue(wqh, &poll_file->wait);
 457}
 458
 459static int mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user *user_input)
 460{
 461	struct file *file, *old_file;
 462	struct mshv_vtl_poll_file *poll_file;
 463	struct mshv_vtl_set_poll_file input;
 464
 465	if (copy_from_user(&input, user_input, sizeof(input)))
 466		return -EFAULT;
 467
 468	if (input.cpu >= num_possible_cpus() || !cpu_online(input.cpu))
 469		return -EINVAL;
 470	/*
 471	 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists.
 472	 * CPU is expected to remain online after above cpu_online() check.
 473	 */
 474
 475	file = NULL;
 476	file = fget(input.fd);
 477	if (!file)
 478		return -EBADFD;
 479
 480	poll_file = per_cpu_ptr(&mshv_vtl_poll_file, READ_ONCE(input.cpu));
 481	if (!poll_file)
 482		return -EINVAL;
 483
 484	mutex_lock(&mshv_vtl_poll_file_lock);
 485
 486	if (poll_file->wqh)
 487		remove_wait_queue(poll_file->wqh, &poll_file->wait);
 488	poll_file->wqh = NULL;
 489
 490	old_file = poll_file->file;
 491	poll_file->file = file;
 492	poll_file->cpu = input.cpu;
 493
 494	if (file) {
 495		init_waitqueue_func_entry(&poll_file->wait, mshv_vtl_poll_file_wake);
 496		init_poll_funcptr(&poll_file->pt, mshv_vtl_ptable_queue_proc);
 497		vfs_poll(file, &poll_file->pt);
 498	}
 499
 500	mutex_unlock(&mshv_vtl_poll_file_lock);
 501
 502	if (old_file)
 503		fput(old_file);
 504
 505	return 0;
 506}
 507
 508/* Static table mapping register names to their corresponding actions */
 509static const struct {
 510	enum hv_register_name reg_name;
 511	int debug_reg_num;  /* -1 if not a debug register */
 512	u32 msr_addr;       /* 0 if not an MSR */
 513} reg_table[] = {
 514	/* Debug registers */
 515	{HV_X64_REGISTER_DR0, 0, 0},
 516	{HV_X64_REGISTER_DR1, 1, 0},
 517	{HV_X64_REGISTER_DR2, 2, 0},
 518	{HV_X64_REGISTER_DR3, 3, 0},
 519	{HV_X64_REGISTER_DR6, 6, 0},
 520	/* MTRR MSRs */
 521	{HV_X64_REGISTER_MSR_MTRR_CAP, -1, MSR_MTRRcap},
 522	{HV_X64_REGISTER_MSR_MTRR_DEF_TYPE, -1, MSR_MTRRdefType},
 523	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, -1, MTRRphysBase_MSR(0)},
 524	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1, -1, MTRRphysBase_MSR(1)},
 525	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2, -1, MTRRphysBase_MSR(2)},
 526	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3, -1, MTRRphysBase_MSR(3)},
 527	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4, -1, MTRRphysBase_MSR(4)},
 528	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5, -1, MTRRphysBase_MSR(5)},
 529	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6, -1, MTRRphysBase_MSR(6)},
 530	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7, -1, MTRRphysBase_MSR(7)},
 531	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8, -1, MTRRphysBase_MSR(8)},
 532	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9, -1, MTRRphysBase_MSR(9)},
 533	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA, -1, MTRRphysBase_MSR(0xa)},
 534	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB, -1, MTRRphysBase_MSR(0xb)},
 535	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC, -1, MTRRphysBase_MSR(0xc)},
 536	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASED, -1, MTRRphysBase_MSR(0xd)},
 537	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE, -1, MTRRphysBase_MSR(0xe)},
 538	{HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF, -1, MTRRphysBase_MSR(0xf)},
 539	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, -1, MTRRphysMask_MSR(0)},
 540	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1, -1, MTRRphysMask_MSR(1)},
 541	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2, -1, MTRRphysMask_MSR(2)},
 542	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3, -1, MTRRphysMask_MSR(3)},
 543	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4, -1, MTRRphysMask_MSR(4)},
 544	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5, -1, MTRRphysMask_MSR(5)},
 545	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6, -1, MTRRphysMask_MSR(6)},
 546	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7, -1, MTRRphysMask_MSR(7)},
 547	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8, -1, MTRRphysMask_MSR(8)},
 548	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9, -1, MTRRphysMask_MSR(9)},
 549	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA, -1, MTRRphysMask_MSR(0xa)},
 550	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB, -1, MTRRphysMask_MSR(0xb)},
 551	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC, -1, MTRRphysMask_MSR(0xc)},
 552	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD, -1, MTRRphysMask_MSR(0xd)},
 553	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE, -1, MTRRphysMask_MSR(0xe)},
 554	{HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF, -1, MTRRphysMask_MSR(0xf)},
 555	{HV_X64_REGISTER_MSR_MTRR_FIX64K00000, -1, MSR_MTRRfix64K_00000},
 556	{HV_X64_REGISTER_MSR_MTRR_FIX16K80000, -1, MSR_MTRRfix16K_80000},
 557	{HV_X64_REGISTER_MSR_MTRR_FIX16KA0000, -1, MSR_MTRRfix16K_A0000},
 558	{HV_X64_REGISTER_MSR_MTRR_FIX4KC0000, -1, MSR_MTRRfix4K_C0000},
 559	{HV_X64_REGISTER_MSR_MTRR_FIX4KC8000, -1, MSR_MTRRfix4K_C8000},
 560	{HV_X64_REGISTER_MSR_MTRR_FIX4KD0000, -1, MSR_MTRRfix4K_D0000},
 561	{HV_X64_REGISTER_MSR_MTRR_FIX4KD8000, -1, MSR_MTRRfix4K_D8000},
 562	{HV_X64_REGISTER_MSR_MTRR_FIX4KE0000, -1, MSR_MTRRfix4K_E0000},
 563	{HV_X64_REGISTER_MSR_MTRR_FIX4KE8000, -1, MSR_MTRRfix4K_E8000},
 564	{HV_X64_REGISTER_MSR_MTRR_FIX4KF0000, -1, MSR_MTRRfix4K_F0000},
 565	{HV_X64_REGISTER_MSR_MTRR_FIX4KF8000, -1, MSR_MTRRfix4K_F8000},
 566};
 567
 568static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set)
 569{
 570	u64 *reg64;
 571	enum hv_register_name gpr_name;
 572	int i;
 573
 574	gpr_name = regs->name;
 575	reg64 = &regs->value.reg64;
 576
 577	/* Search for the register in the table */
 578	for (i = 0; i < ARRAY_SIZE(reg_table); i++) {
 579		if (reg_table[i].reg_name != gpr_name)
 580			continue;
 581		if (reg_table[i].debug_reg_num != -1) {
 582			/* Handle debug registers */
 583			if (gpr_name == HV_X64_REGISTER_DR6 &&
 584			    !mshv_vsm_capabilities.dr6_shared)
 585				goto hypercall;
 586			if (set)
 587				native_set_debugreg(reg_table[i].debug_reg_num, *reg64);
 588			else
 589				*reg64 = native_get_debugreg(reg_table[i].debug_reg_num);
 590		} else {
 591			/* Handle MSRs */
 592			if (set)
 593				wrmsrl(reg_table[i].msr_addr, *reg64);
 594			else
 595				rdmsrl(reg_table[i].msr_addr, *reg64);
 596		}
 597		return 0;
 598	}
 599
 600hypercall:
 601	return 1;
 602}
 603
 604static void mshv_vtl_return(struct mshv_vtl_cpu_context *vtl0)
 605{
 606	struct hv_vp_assist_page *hvp;
 607
 608	hvp = hv_vp_assist_page[smp_processor_id()];
 609
 610	/*
 611	 * Process signal event direct set in the run page, if any.
 612	 */
 613	if (mshv_vsm_capabilities.return_action_available) {
 614		u32 offset = READ_ONCE(mshv_vtl_this_run()->vtl_ret_action_size);
 615
 616		WRITE_ONCE(mshv_vtl_this_run()->vtl_ret_action_size, 0);
 617
 618		/*
 619		 * Hypervisor will take care of clearing out the actions
 620		 * set in the assist page.
 621		 */
 622		memcpy(hvp->vtl_ret_actions,
 623		       mshv_vtl_this_run()->vtl_ret_actions,
 624		       min_t(u32, offset, sizeof(hvp->vtl_ret_actions)));
 625	}
 626
 627	mshv_vtl_return_call(vtl0);
 628}
 629
 630static bool mshv_vtl_process_intercept(void)
 631{
 632	struct hv_per_cpu_context *mshv_cpu;
 633	void *synic_message_page;
 634	struct hv_message *msg;
 635	u32 message_type;
 636
 637	mshv_cpu = this_cpu_ptr(hv_context.cpu_context);
 638	synic_message_page = mshv_cpu->hyp_synic_message_page;
 639	if (unlikely(!synic_message_page))
 640		return true;
 641
 642	msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX;
 643	message_type = READ_ONCE(msg->header.message_type);
 644	if (message_type == HVMSG_NONE)
 645		return true;
 646
 647	memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg));
 648	vmbus_signal_eom(msg, message_type);
 649
 650	return false;
 651}
 652
 653static int mshv_vtl_ioctl_return_to_lower_vtl(void)
 654{
 655	preempt_disable();
 656	for (;;) {
 657		unsigned long irq_flags;
 658		struct hv_vp_assist_page *hvp;
 659		int ret;
 660
 661		if (__xfer_to_guest_mode_work_pending()) {
 662			preempt_enable();
 663			ret = xfer_to_guest_mode_handle_work();
 664			if (ret)
 665				return ret;
 666			preempt_disable();
 667		}
 668
 669		local_irq_save(irq_flags);
 670		if (READ_ONCE(mshv_vtl_this_run()->cancel)) {
 671			local_irq_restore(irq_flags);
 672			preempt_enable();
 673			return -EINTR;
 674		}
 675
 676		mshv_vtl_return(&mshv_vtl_this_run()->cpu_context);
 677		local_irq_restore(irq_flags);
 678
 679		hvp = hv_vp_assist_page[smp_processor_id()];
 680		this_cpu_inc(num_vtl0_transitions);
 681		switch (hvp->vtl_entry_reason) {
 682		case MSHV_ENTRY_REASON_INTERRUPT:
 683			if (!mshv_vsm_capabilities.intercept_page_available &&
 684			    likely(!mshv_vtl_process_intercept()))
 685				goto done;
 686			break;
 687
 688		case MSHV_ENTRY_REASON_INTERCEPT:
 689			WARN_ON(!mshv_vsm_capabilities.intercept_page_available);
 690			memcpy(mshv_vtl_this_run()->exit_message, hvp->intercept_message,
 691			       sizeof(hvp->intercept_message));
 692			goto done;
 693
 694		default:
 695			panic("unknown entry reason: %d", hvp->vtl_entry_reason);
 696		}
 697	}
 698
 699done:
 700	preempt_enable();
 701
 702	return 0;
 703}
 704
 705static long
 706mshv_vtl_ioctl_get_regs(void __user *user_args)
 707{
 708	struct mshv_vp_registers args;
 709	struct hv_register_assoc reg;
 710	long ret;
 711
 712	if (copy_from_user(&args, user_args, sizeof(args)))
 713		return -EFAULT;
 714
 715	/*  This IOCTL supports processing only one register at a time. */
 716	if (args.count != 1)
 717		return -EINVAL;
 718
 719	if (copy_from_user(&reg, (void __user *)args.regs_ptr,
 720			   sizeof(reg)))
 721		return -EFAULT;
 722
 723	ret = mshv_vtl_get_set_reg(&reg, false);
 724	if (!ret)
 725		goto copy_args; /* No need of hypercall */
 726	ret = vtl_get_vp_register(&reg);
 727	if (ret)
 728		return ret;
 729
 730copy_args:
 731	if (copy_to_user((void __user *)args.regs_ptr, &reg, sizeof(reg)))
 732		ret = -EFAULT;
 733
 734	return ret;
 735}
 736
 737static long
 738mshv_vtl_ioctl_set_regs(void __user *user_args)
 739{
 740	struct mshv_vp_registers args;
 741	struct hv_register_assoc reg;
 742	long ret;
 743
 744	if (copy_from_user(&args, user_args, sizeof(args)))
 745		return -EFAULT;
 746
 747	/*  This IOCTL supports processing only one register at a time. */
 748	if (args.count != 1)
 749		return -EINVAL;
 750
 751	if (copy_from_user(&reg, (void __user *)args.regs_ptr, sizeof(reg)))
 752		return -EFAULT;
 753
 754	ret = mshv_vtl_get_set_reg(&reg, true);
 755	if (!ret)
 756		return ret; /* No need of hypercall */
 757	ret = vtl_set_vp_register(&reg);
 758
 759	return ret;
 760}
 761
 762static long
 763mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 764{
 765	long ret;
 766	struct mshv_vtl *vtl = filp->private_data;
 767
 768	switch (ioctl) {
 769	case MSHV_SET_POLL_FILE:
 770		ret = mshv_vtl_ioctl_set_poll_file((struct mshv_vtl_set_poll_file __user *)arg);
 771		break;
 772	case MSHV_GET_VP_REGISTERS:
 773		ret = mshv_vtl_ioctl_get_regs((void __user *)arg);
 774		break;
 775	case MSHV_SET_VP_REGISTERS:
 776		ret = mshv_vtl_ioctl_set_regs((void __user *)arg);
 777		break;
 778	case MSHV_RETURN_TO_LOWER_VTL:
 779		ret = mshv_vtl_ioctl_return_to_lower_vtl();
 780		break;
 781	case MSHV_ADD_VTL0_MEMORY:
 782		ret = mshv_vtl_ioctl_add_vtl0_mem(vtl, (void __user *)arg);
 783		break;
 784	default:
 785		dev_err(vtl->module_dev, "invalid vtl ioctl: %#x\n", ioctl);
 786		ret = -ENOTTY;
 787	}
 788
 789	return ret;
 790}
 791
 792static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf)
 793{
 794	struct page *page;
 795	int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK;
 796	int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT;
 797
 798	if (!cpu_online(cpu))
 799		return VM_FAULT_SIGBUS;
 800	/*
 801	 * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists.
 802	 * CPU is expected to remain online after above cpu_online() check.
 803	 */
 804
 805	if (real_off == MSHV_RUN_PAGE_OFFSET) {
 806		page = virt_to_page(mshv_vtl_cpu_run(cpu));
 807	} else if (real_off == MSHV_REG_PAGE_OFFSET) {
 808		if (!mshv_has_reg_page)
 809			return VM_FAULT_SIGBUS;
 810		page = mshv_vtl_cpu_reg_page(cpu);
 811	} else {
 812		return VM_FAULT_NOPAGE;
 813	}
 814
 815	get_page(page);
 816	vmf->page = page;
 817
 818	return 0;
 819}
 820
 821static const struct vm_operations_struct mshv_vtl_vm_ops = {
 822	.fault = mshv_vtl_fault,
 823};
 824
 825static int mshv_vtl_mmap(struct file *filp, struct vm_area_struct *vma)
 826{
 827	vma->vm_ops = &mshv_vtl_vm_ops;
 828
 829	return 0;
 830}
 831
 832static int mshv_vtl_release(struct inode *inode, struct file *filp)
 833{
 834	struct mshv_vtl *vtl = filp->private_data;
 835
 836	kfree(vtl);
 837
 838	return 0;
 839}
 840
 841static const struct file_operations mshv_vtl_fops = {
 842	.owner = THIS_MODULE,
 843	.unlocked_ioctl = mshv_vtl_ioctl,
 844	.release = mshv_vtl_release,
 845	.mmap = mshv_vtl_mmap,
 846};
 847
 848static void mshv_vtl_synic_mask_vmbus_sint(const u8 *mask)
 849{
 850	union hv_synic_sint sint;
 851
 852	sint.as_uint64 = 0;
 853	sint.vector = HYPERVISOR_CALLBACK_VECTOR;
 854	sint.masked = (*mask != 0);
 855	sint.auto_eoi = hv_recommend_using_aeoi();
 856
 857	hv_set_msr(HV_MSR_SINT0 + VTL2_VMBUS_SINT_INDEX,
 858		   sint.as_uint64);
 859
 860	if (!sint.masked)
 861		pr_debug("%s: Unmasking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id());
 862	else
 863		pr_debug("%s: Masking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id());
 864}
 865
 866static void mshv_vtl_read_remote(void *buffer)
 867{
 868	struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context);
 869	struct hv_message *msg = (struct hv_message *)mshv_cpu->hyp_synic_message_page +
 870					VTL2_VMBUS_SINT_INDEX;
 871	u32 message_type = READ_ONCE(msg->header.message_type);
 872
 873	WRITE_ONCE(has_message, false);
 874	if (message_type == HVMSG_NONE)
 875		return;
 876
 877	memcpy(buffer, msg, sizeof(*msg));
 878	vmbus_signal_eom(msg, message_type);
 879}
 880
 881static bool vtl_synic_mask_vmbus_sint_masked = true;
 882
 883static ssize_t mshv_vtl_sint_read(struct file *filp, char __user *arg, size_t size, loff_t *offset)
 884{
 885	struct hv_message msg = {};
 886	int ret;
 887
 888	if (size < sizeof(msg))
 889		return -EINVAL;
 890
 891	for (;;) {
 892		smp_call_function_single(VMBUS_CONNECT_CPU, mshv_vtl_read_remote, &msg, true);
 893		if (msg.header.message_type != HVMSG_NONE)
 894			break;
 895
 896		if (READ_ONCE(vtl_synic_mask_vmbus_sint_masked))
 897			return 0; /* EOF */
 898
 899		if (filp->f_flags & O_NONBLOCK)
 900			return -EAGAIN;
 901
 902		ret = wait_event_interruptible(fd_wait_queue,
 903					       READ_ONCE(has_message) ||
 904						READ_ONCE(vtl_synic_mask_vmbus_sint_masked));
 905		if (ret)
 906			return ret;
 907	}
 908
 909	if (copy_to_user(arg, &msg, sizeof(msg)))
 910		return -EFAULT;
 911
 912	return sizeof(msg);
 913}
 914
 915static __poll_t mshv_vtl_sint_poll(struct file *filp, poll_table *wait)
 916{
 917	__poll_t mask = 0;
 918
 919	poll_wait(filp, &fd_wait_queue, wait);
 920	if (READ_ONCE(has_message) || READ_ONCE(vtl_synic_mask_vmbus_sint_masked))
 921		mask |= EPOLLIN | EPOLLRDNORM;
 922
 923	return mask;
 924}
 925
 926static void mshv_vtl_sint_on_msg_dpc(unsigned long data)
 927{
 928	WRITE_ONCE(has_message, true);
 929	wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN);
 930}
 931
 932static int mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user *arg)
 933{
 934	struct mshv_vtl_sint_post_msg message;
 935	u8 payload[HV_MESSAGE_PAYLOAD_BYTE_COUNT];
 936
 937	if (copy_from_user(&message, arg, sizeof(message)))
 938		return -EFAULT;
 939	if (message.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT)
 940		return -EINVAL;
 941	if (copy_from_user(payload, (void __user *)message.payload_ptr,
 942			   message.payload_size))
 943		return -EFAULT;
 944
 945	return hv_post_message((union hv_connection_id)message.connection_id,
 946			       message.message_type, (void *)payload,
 947			       message.payload_size);
 948}
 949
 950static int mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user *arg)
 951{
 952	u64 input, status;
 953	struct mshv_vtl_signal_event signal_event;
 954
 955	if (copy_from_user(&signal_event, arg, sizeof(signal_event)))
 956		return -EFAULT;
 957
 958	input = signal_event.connection_id | ((u64)signal_event.flag << 32);
 959
 960	status = hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, input);
 961
 962	return hv_result_to_errno(status);
 963}
 964
 965static int mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user *arg)
 966{
 967	struct mshv_vtl_set_eventfd set_eventfd;
 968	struct eventfd_ctx *eventfd, *old_eventfd;
 969
 970	if (copy_from_user(&set_eventfd, arg, sizeof(set_eventfd)))
 971		return -EFAULT;
 972	if (set_eventfd.flag >= HV_EVENT_FLAGS_COUNT)
 973		return -EINVAL;
 974
 975	eventfd = NULL;
 976	if (set_eventfd.fd >= 0) {
 977		eventfd = eventfd_ctx_fdget(set_eventfd.fd);
 978		if (IS_ERR(eventfd))
 979			return PTR_ERR(eventfd);
 980	}
 981
 982	guard(mutex)(&flag_lock);
 983	old_eventfd = READ_ONCE(flag_eventfds[set_eventfd.flag]);
 984	WRITE_ONCE(flag_eventfds[set_eventfd.flag], eventfd);
 985
 986	if (old_eventfd) {
 987		synchronize_rcu();
 988		eventfd_ctx_put(old_eventfd);
 989	}
 990
 991	return 0;
 992}
 993
 994static int mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user *arg)
 995{
 996	static DEFINE_MUTEX(vtl2_vmbus_sint_mask_mutex);
 997	struct mshv_sint_mask mask;
 998
 999	if (copy_from_user(&mask, arg, sizeof(mask)))
1000		return -EFAULT;
1001	guard(mutex)(&vtl2_vmbus_sint_mask_mutex);
1002	on_each_cpu((smp_call_func_t)mshv_vtl_synic_mask_vmbus_sint, &mask.mask, 1);
1003	WRITE_ONCE(vtl_synic_mask_vmbus_sint_masked, mask.mask != 0);
1004	if (mask.mask)
1005		wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN);
1006
1007	return 0;
1008}
1009
1010static long mshv_vtl_sint_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1011{
1012	switch (cmd) {
1013	case MSHV_SINT_POST_MESSAGE:
1014		return mshv_vtl_sint_ioctl_post_msg((struct mshv_vtl_sint_post_msg __user *)arg);
1015	case MSHV_SINT_SIGNAL_EVENT:
1016		return mshv_vtl_sint_ioctl_signal_event((struct mshv_vtl_signal_event __user *)arg);
1017	case MSHV_SINT_SET_EVENTFD:
1018		return mshv_vtl_sint_ioctl_set_eventfd((struct mshv_vtl_set_eventfd __user *)arg);
1019	case MSHV_SINT_PAUSE_MESSAGE_STREAM:
1020		return mshv_vtl_sint_ioctl_pause_msg_stream((struct mshv_sint_mask __user *)arg);
1021	default:
1022		return -ENOIOCTLCMD;
1023	}
1024}
1025
1026static const struct file_operations mshv_vtl_sint_ops = {
1027	.owner = THIS_MODULE,
1028	.read = mshv_vtl_sint_read,
1029	.poll = mshv_vtl_sint_poll,
1030	.unlocked_ioctl = mshv_vtl_sint_ioctl,
1031};
1032
1033static struct miscdevice mshv_vtl_sint_dev = {
1034	.name = "mshv_sint",
1035	.fops = &mshv_vtl_sint_ops,
1036	.mode = 0600,
1037	.minor = MISC_DYNAMIC_MINOR,
1038};
1039
1040static int mshv_vtl_hvcall_dev_open(struct inode *node, struct file *f)
1041{
1042	struct miscdevice *dev = f->private_data;
1043	struct mshv_vtl_hvcall_fd *fd;
1044
1045	if (!capable(CAP_SYS_ADMIN))
1046		return -EPERM;
1047
1048	fd = vzalloc(sizeof(*fd));
1049	if (!fd)
1050		return -ENOMEM;
1051	fd->dev = dev;
1052	f->private_data = fd;
1053	mutex_init(&fd->init_mutex);
1054
1055	return 0;
1056}
1057
1058static int mshv_vtl_hvcall_dev_release(struct inode *node, struct file *f)
1059{
1060	struct mshv_vtl_hvcall_fd *fd;
1061
1062	fd = f->private_data;
1063	if (fd) {
1064		vfree(fd);
1065		f->private_data = NULL;
1066	}
1067
1068	return 0;
1069}
1070
1071static int mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd *fd,
1072				    struct mshv_vtl_hvcall_setup __user *hvcall_setup_user)
1073{
1074	struct mshv_vtl_hvcall_setup hvcall_setup;
1075
1076	guard(mutex)(&fd->init_mutex);
1077
1078	if (fd->allow_map_initialized) {
1079		dev_err(fd->dev->this_device,
1080			"Hypercall allow map has already been set, pid %d\n",
1081			current->pid);
1082		return -EINVAL;
1083	}
1084
1085	if (copy_from_user(&hvcall_setup, hvcall_setup_user,
1086			   sizeof(struct mshv_vtl_hvcall_setup))) {
1087		return -EFAULT;
1088	}
1089	if (hvcall_setup.bitmap_array_size > ARRAY_SIZE(fd->allow_bitmap))
1090		return -EINVAL;
1091
1092	if (copy_from_user(&fd->allow_bitmap,
1093			   (void __user *)hvcall_setup.allow_bitmap_ptr,
1094			   hvcall_setup.bitmap_array_size)) {
1095		return -EFAULT;
1096	}
1097
1098	dev_info(fd->dev->this_device, "Hypercall allow map has been set, pid %d\n",
1099		 current->pid);
1100	fd->allow_map_initialized = true;
1101	return 0;
1102}
1103
1104static bool mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd *fd, u16 call_code)
1105{
1106	return test_bit(call_code, (unsigned long *)fd->allow_bitmap);
1107}
1108
1109static int mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd *fd,
1110				struct mshv_vtl_hvcall __user *hvcall_user)
1111{
1112	struct mshv_vtl_hvcall hvcall;
1113	void *in, *out;
1114	int ret;
1115
1116	if (copy_from_user(&hvcall, hvcall_user, sizeof(struct mshv_vtl_hvcall)))
1117		return -EFAULT;
1118	if (hvcall.input_size > HV_HYP_PAGE_SIZE)
1119		return -EINVAL;
1120	if (hvcall.output_size > HV_HYP_PAGE_SIZE)
1121		return -EINVAL;
1122
1123	/*
1124	 * By default, all hypercalls are not allowed.
1125	 * The user mode code has to set up the allow bitmap once.
1126	 */
1127
1128	if (!mshv_vtl_hvcall_is_allowed(fd, hvcall.control & 0xFFFF)) {
1129		dev_err(fd->dev->this_device,
1130			"Hypercall with control data %#llx isn't allowed\n",
1131			hvcall.control);
1132		return -EPERM;
1133	}
1134
1135	/*
1136	 * This may create a problem for Confidential VM (CVM) usecase where we need to use
1137	 * Hyper-V driver allocated per-cpu input and output pages (hyperv_pcpu_input_arg and
1138	 * hyperv_pcpu_output_arg) for making a hypervisor call.
1139	 *
1140	 * TODO: Take care of this when CVM support is added.
1141	 */
1142	in = (void *)__get_free_page(GFP_KERNEL);
1143	out = (void *)__get_free_page(GFP_KERNEL);
1144
1145	if (copy_from_user(in, (void __user *)hvcall.input_ptr, hvcall.input_size)) {
1146		ret = -EFAULT;
1147		goto free_pages;
1148	}
1149
1150	hvcall.status = hv_do_hypercall(hvcall.control, in, out);
1151
1152	if (copy_to_user((void __user *)hvcall.output_ptr, out, hvcall.output_size)) {
1153		ret = -EFAULT;
1154		goto free_pages;
1155	}
1156	ret = put_user(hvcall.status, &hvcall_user->status);
1157free_pages:
1158	free_page((unsigned long)in);
1159	free_page((unsigned long)out);
1160
1161	return ret;
1162}
1163
1164static long mshv_vtl_hvcall_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1165{
1166	struct mshv_vtl_hvcall_fd *fd = f->private_data;
1167
1168	switch (cmd) {
1169	case MSHV_HVCALL_SETUP:
1170		return mshv_vtl_hvcall_do_setup(fd, (struct mshv_vtl_hvcall_setup __user *)arg);
1171	case MSHV_HVCALL:
1172		return mshv_vtl_hvcall_call(fd, (struct mshv_vtl_hvcall __user *)arg);
1173	default:
1174		break;
1175	}
1176
1177	return -ENOIOCTLCMD;
1178}
1179
1180static const struct file_operations mshv_vtl_hvcall_dev_file_ops = {
1181	.owner = THIS_MODULE,
1182	.open = mshv_vtl_hvcall_dev_open,
1183	.release = mshv_vtl_hvcall_dev_release,
1184	.unlocked_ioctl = mshv_vtl_hvcall_dev_ioctl,
1185};
1186
1187static struct miscdevice mshv_vtl_hvcall_dev = {
1188	.name = "mshv_hvcall",
1189	.nodename = "mshv_hvcall",
1190	.fops = &mshv_vtl_hvcall_dev_file_ops,
1191	.mode = 0600,
1192	.minor = MISC_DYNAMIC_MINOR,
1193};
1194
1195static int mshv_vtl_low_open(struct inode *inodep, struct file *filp)
1196{
1197	pid_t pid = task_pid_vnr(current);
1198	uid_t uid = current_uid().val;
1199	int ret = 0;
1200
1201	pr_debug("%s: Opening VTL low, task group %d, uid %d\n", __func__, pid, uid);
1202
1203	if (capable(CAP_SYS_ADMIN)) {
1204		filp->private_data = inodep;
1205	} else {
1206		pr_err("%s: VTL low open failed: CAP_SYS_ADMIN required. task group %d, uid %d",
1207		       __func__, pid, uid);
1208		ret = -EPERM;
1209	}
1210
1211	return ret;
1212}
1213
1214static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn)
1215{
1216	unsigned long mask = size - 1;
1217	unsigned long start = vmf->address & ~mask;
1218	unsigned long end = start + size;
1219	bool is_valid;
1220
1221	is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) &&
1222		start >= vmf->vma->vm_start &&
1223		end <= vmf->vma->vm_end;
1224
1225	if (is_valid)
1226		*pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT);
1227
1228	return is_valid;
1229}
1230
1231static vm_fault_t mshv_vtl_low_huge_fault(struct vm_fault *vmf, unsigned int order)
1232{
1233	unsigned long pfn = vmf->pgoff;
1234	vm_fault_t ret = VM_FAULT_FALLBACK;
1235
1236	switch (order) {
1237	case 0:
1238		return vmf_insert_mixed(vmf->vma, vmf->address, pfn);
1239
1240	case PMD_ORDER:
1241		if (can_fault(vmf, PMD_SIZE, &pfn))
1242			ret = vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
1243		return ret;
1244
1245	case PUD_ORDER:
1246		if (can_fault(vmf, PUD_SIZE, &pfn))
1247			ret = vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
1248		return ret;
1249
1250	default:
1251		return VM_FAULT_SIGBUS;
1252	}
1253}
1254
1255static vm_fault_t mshv_vtl_low_fault(struct vm_fault *vmf)
1256{
1257	return mshv_vtl_low_huge_fault(vmf, 0);
1258}
1259
1260static const struct vm_operations_struct mshv_vtl_low_vm_ops = {
1261	.fault = mshv_vtl_low_fault,
1262	.huge_fault = mshv_vtl_low_huge_fault,
1263};
1264
1265static int mshv_vtl_low_mmap(struct file *filp, struct vm_area_struct *vma)
1266{
1267	vma->vm_ops = &mshv_vtl_low_vm_ops;
1268	vm_flags_set(vma, VM_HUGEPAGE | VM_MIXEDMAP);
1269
1270	return 0;
1271}
1272
1273static const struct file_operations mshv_vtl_low_file_ops = {
1274	.owner		= THIS_MODULE,
1275	.open		= mshv_vtl_low_open,
1276	.mmap		= mshv_vtl_low_mmap,
1277};
1278
1279static struct miscdevice mshv_vtl_low = {
1280	.name = "mshv_vtl_low",
1281	.nodename = "mshv_vtl_low",
1282	.fops = &mshv_vtl_low_file_ops,
1283	.mode = 0600,
1284	.minor = MISC_DYNAMIC_MINOR,
1285};
1286
1287static int __init mshv_vtl_init(void)
1288{
1289	int ret;
1290	struct device *dev = mshv_dev.this_device;
1291
1292	/*
1293	 * This creates /dev/mshv which provides functionality to create VTLs and partitions.
1294	 */
1295	ret = misc_register(&mshv_dev);
1296	if (ret) {
1297		dev_err(dev, "mshv device register failed: %d\n", ret);
1298		goto free_dev;
1299	}
1300
1301	tasklet_init(&msg_dpc, mshv_vtl_sint_on_msg_dpc, 0);
1302	init_waitqueue_head(&fd_wait_queue);
1303
1304	if (mshv_vtl_get_vsm_regs()) {
1305		dev_emerg(dev, "Unable to get VSM capabilities !!\n");
1306		ret = -ENODEV;
1307		goto free_dev;
1308	}
1309	if (mshv_vtl_configure_vsm_partition(dev)) {
1310		dev_emerg(dev, "VSM configuration failed !!\n");
1311		ret = -ENODEV;
1312		goto free_dev;
1313	}
1314
1315	mshv_vtl_return_call_init(mshv_vsm_page_offsets.vtl_return_offset);
1316	ret = hv_vtl_setup_synic();
1317	if (ret)
1318		goto free_dev;
1319
1320	/*
1321	 * mshv_sint device adds VMBus relay ioctl support.
1322	 * This provides a channel for VTL0 to communicate with VTL2.
1323	 */
1324	ret = misc_register(&mshv_vtl_sint_dev);
1325	if (ret)
1326		goto free_synic;
1327
1328	/*
1329	 * mshv_hvcall device adds interface to enable userspace for direct hypercalls support.
1330	 */
1331	ret = misc_register(&mshv_vtl_hvcall_dev);
1332	if (ret)
1333		goto free_sint;
1334
1335	/*
1336	 * mshv_vtl_low device is used to map VTL0 address space to a user-mode process in VTL2.
1337	 * It implements mmap() to allow a user-mode process in VTL2 to map to the address of VTL0.
1338	 */
1339	ret = misc_register(&mshv_vtl_low);
1340	if (ret)
1341		goto free_hvcall;
1342
1343	/*
1344	 * "mshv vtl mem dev" device is later used to setup VTL0 memory.
1345	 */
1346	mem_dev = kzalloc(sizeof(*mem_dev), GFP_KERNEL);
1347	if (!mem_dev) {
1348		ret = -ENOMEM;
1349		goto free_low;
1350	}
1351
1352	mutex_init(&mshv_vtl_poll_file_lock);
1353
1354	device_initialize(mem_dev);
1355	dev_set_name(mem_dev, "mshv vtl mem dev");
1356	ret = device_add(mem_dev);
1357	if (ret) {
1358		dev_err(dev, "mshv vtl mem dev add: %d\n", ret);
1359		goto free_mem;
1360	}
1361
1362	return 0;
1363
1364free_mem:
1365	kfree(mem_dev);
1366free_low:
1367	misc_deregister(&mshv_vtl_low);
1368free_hvcall:
1369	misc_deregister(&mshv_vtl_hvcall_dev);
1370free_sint:
1371	misc_deregister(&mshv_vtl_sint_dev);
1372free_synic:
1373	hv_vtl_remove_synic();
1374free_dev:
1375	misc_deregister(&mshv_dev);
1376
1377	return ret;
1378}
1379
1380static void __exit mshv_vtl_exit(void)
1381{
1382	device_del(mem_dev);
1383	kfree(mem_dev);
1384	misc_deregister(&mshv_vtl_low);
1385	misc_deregister(&mshv_vtl_hvcall_dev);
1386	misc_deregister(&mshv_vtl_sint_dev);
1387	hv_vtl_remove_synic();
1388	misc_deregister(&mshv_dev);
1389}
1390
1391module_init(mshv_vtl_init);
1392module_exit(mshv_vtl_exit);