ioctl.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*  Copyright(c) 2016-20 Intel Corporation. */
   3
   4#include <asm/mman.h>
   5#include <asm/sgx.h>
   6#include <crypto/sha2.h>
   7#include <linux/mman.h>
   8#include <linux/delay.h>
   9#include <linux/file.h>
  10#include <linux/hashtable.h>
  11#include <linux/highmem.h>
  12#include <linux/ratelimit.h>
  13#include <linux/sched/signal.h>
  14#include <linux/shmem_fs.h>
  15#include <linux/slab.h>
  16#include <linux/suspend.h>
  17#include "driver.h"
  18#include "encl.h"
  19#include "encls.h"
  20
  21struct sgx_va_page *sgx_encl_grow(struct sgx_encl *encl, bool reclaim)
  22{
  23	struct sgx_va_page *va_page = NULL;
  24	void *err;
  25
  26	BUILD_BUG_ON(SGX_VA_SLOT_COUNT !=
  27		(SGX_ENCL_PAGE_VA_OFFSET_MASK >> 3) + 1);
  28
  29	if (!(encl->page_cnt % SGX_VA_SLOT_COUNT)) {
  30		va_page = kzalloc(sizeof(*va_page), GFP_KERNEL);
  31		if (!va_page)
  32			return ERR_PTR(-ENOMEM);
  33
  34		va_page->epc_page = sgx_alloc_va_page(reclaim);
  35		if (IS_ERR(va_page->epc_page)) {
  36			err = ERR_CAST(va_page->epc_page);
  37			kfree(va_page);
  38			return err;
  39		}
  40
  41		WARN_ON_ONCE(encl->page_cnt % SGX_VA_SLOT_COUNT);
  42	}
  43	encl->page_cnt++;
  44	return va_page;
  45}
  46
  47void sgx_encl_shrink(struct sgx_encl *encl, struct sgx_va_page *va_page)
  48{
  49	encl->page_cnt--;
  50
  51	if (va_page) {
  52		sgx_encl_free_epc_page(va_page->epc_page);
  53		list_del(&va_page->list);
  54		kfree(va_page);
  55	}
  56}
  57
  58static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs)
  59{
  60	struct sgx_epc_page *secs_epc;
  61	struct sgx_va_page *va_page;
  62	struct sgx_pageinfo pginfo;
  63	struct sgx_secinfo secinfo;
  64	unsigned long encl_size;
  65	struct file *backing;
  66	long ret;
  67
  68	/*
  69	 * ECREATE would detect this too, but checking here also ensures
  70	 * that the 'encl_size' calculations below can never overflow.
  71	 */
  72	if (!is_power_of_2(secs->size))
  73		return -EINVAL;
  74
  75	va_page = sgx_encl_grow(encl, true);
  76	if (IS_ERR(va_page))
  77		return PTR_ERR(va_page);
  78	else if (va_page)
  79		list_add(&va_page->list, &encl->va_pages);
  80	/* else the tail page of the VA page list had free slots. */
  81
  82	/* The extra page goes to SECS. */
  83	encl_size = secs->size + PAGE_SIZE;
  84
  85	backing = shmem_file_setup("SGX backing", encl_size + (encl_size >> 5),
  86				   VM_NORESERVE);
  87	if (IS_ERR(backing)) {
  88		ret = PTR_ERR(backing);
  89		goto err_out_shrink;
  90	}
  91
  92	encl->backing = backing;
  93
  94	secs_epc = sgx_alloc_epc_page(&encl->secs, true);
  95	if (IS_ERR(secs_epc)) {
  96		ret = PTR_ERR(secs_epc);
  97		goto err_out_backing;
  98	}
  99
 100	encl->secs.epc_page = secs_epc;
 101
 102	pginfo.addr = 0;
 103	pginfo.contents = (unsigned long)secs;
 104	pginfo.metadata = (unsigned long)&secinfo;
 105	pginfo.secs = 0;
 106	memset(&secinfo, 0, sizeof(secinfo));
 107
 108	ret = __ecreate((void *)&pginfo, sgx_get_epc_virt_addr(secs_epc));
 109	if (ret) {
 110		ret = -EIO;
 111		goto err_out;
 112	}
 113
 114	if (secs->attributes & SGX_ATTR_DEBUG)
 115		set_bit(SGX_ENCL_DEBUG, &encl->flags);
 116
 117	encl->secs.encl = encl;
 118	encl->secs.type = SGX_PAGE_TYPE_SECS;
 119	encl->base = secs->base;
 120	encl->size = secs->size;
 121	encl->attributes = secs->attributes;
 122	encl->attributes_mask = SGX_ATTR_UNPRIV_MASK;
 123
 124	/* Set only after completion, as encl->lock has not been taken. */
 125	set_bit(SGX_ENCL_CREATED, &encl->flags);
 126
 127	return 0;
 128
 129err_out:
 130	sgx_encl_free_epc_page(encl->secs.epc_page);
 131	encl->secs.epc_page = NULL;
 132
 133err_out_backing:
 134	fput(encl->backing);
 135	encl->backing = NULL;
 136
 137err_out_shrink:
 138	sgx_encl_shrink(encl, va_page);
 139
 140	return ret;
 141}
 142
 143/**
 144 * sgx_ioc_enclave_create() - handler for %SGX_IOC_ENCLAVE_CREATE
 145 * @encl:	An enclave pointer.
 146 * @arg:	The ioctl argument.
 147 *
 148 * Allocate kernel data structures for the enclave and invoke ECREATE.
 149 *
 150 * Return:
 151 * - 0:		Success.
 152 * - -EIO:	ECREATE failed.
 153 * - -errno:	POSIX error.
 154 */
 155static long sgx_ioc_enclave_create(struct sgx_encl *encl, void __user *arg)
 156{
 157	struct sgx_enclave_create create_arg;
 158	void *secs;
 159	int ret;
 160
 161	if (test_bit(SGX_ENCL_CREATED, &encl->flags))
 162		return -EINVAL;
 163
 164	if (copy_from_user(&create_arg, arg, sizeof(create_arg)))
 165		return -EFAULT;
 166
 167	secs = kmalloc(PAGE_SIZE, GFP_KERNEL);
 168	if (!secs)
 169		return -ENOMEM;
 170
 171	if (copy_from_user(secs, (void __user *)create_arg.src, PAGE_SIZE))
 172		ret = -EFAULT;
 173	else
 174		ret = sgx_encl_create(encl, secs);
 175
 176	kfree(secs);
 177	return ret;
 178}
 179
 180static int sgx_validate_secinfo(struct sgx_secinfo *secinfo)
 181{
 182	u64 perm = secinfo->flags & SGX_SECINFO_PERMISSION_MASK;
 183	u64 pt   = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK;
 184
 185	if (pt != SGX_SECINFO_REG && pt != SGX_SECINFO_TCS)
 186		return -EINVAL;
 187
 188	if ((perm & SGX_SECINFO_W) && !(perm & SGX_SECINFO_R))
 189		return -EINVAL;
 190
 191	/*
 192	 * CPU will silently overwrite the permissions as zero, which means
 193	 * that we need to validate it ourselves.
 194	 */
 195	if (pt == SGX_SECINFO_TCS && perm)
 196		return -EINVAL;
 197
 198	if (secinfo->flags & SGX_SECINFO_RESERVED_MASK)
 199		return -EINVAL;
 200
 201	if (memchr_inv(secinfo->reserved, 0, sizeof(secinfo->reserved)))
 202		return -EINVAL;
 203
 204	return 0;
 205}
 206
 207static int __sgx_encl_add_page(struct sgx_encl *encl,
 208			       struct sgx_encl_page *encl_page,
 209			       struct sgx_epc_page *epc_page,
 210			       struct sgx_secinfo *secinfo, unsigned long src)
 211{
 212	struct sgx_pageinfo pginfo;
 213	struct vm_area_struct *vma;
 214	struct page *src_page;
 215	int ret;
 216
 217	/* Deny noexec. */
 218	vma = find_vma(current->mm, src);
 219	if (!vma)
 220		return -EFAULT;
 221
 222	if (!(vma->vm_flags & VM_MAYEXEC))
 223		return -EACCES;
 224
 225	ret = get_user_pages(src, 1, 0, &src_page);
 226	if (ret < 1)
 227		return -EFAULT;
 228
 229	pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page);
 230	pginfo.addr = encl_page->desc & PAGE_MASK;
 231	pginfo.metadata = (unsigned long)secinfo;
 232	pginfo.contents = (unsigned long)kmap_local_page(src_page);
 233
 234	ret = __eadd(&pginfo, sgx_get_epc_virt_addr(epc_page));
 235
 236	kunmap_local((void *)pginfo.contents);
 237	put_page(src_page);
 238
 239	return ret ? -EIO : 0;
 240}
 241
 242/*
 243 * If the caller requires measurement of the page as a proof for the content,
 244 * use EEXTEND to add a measurement for 256 bytes of the page. Repeat this
 245 * operation until the entire page is measured.
 246 */
 247static int __sgx_encl_extend(struct sgx_encl *encl,
 248			     struct sgx_epc_page *epc_page)
 249{
 250	unsigned long offset;
 251	int ret;
 252
 253	for (offset = 0; offset < PAGE_SIZE; offset += SGX_EEXTEND_BLOCK_SIZE) {
 254		ret = __eextend(sgx_get_epc_virt_addr(encl->secs.epc_page),
 255				sgx_get_epc_virt_addr(epc_page) + offset);
 256		if (ret) {
 257			if (encls_failed(ret))
 258				ENCLS_WARN(ret, "EEXTEND");
 259
 260			return -EIO;
 261		}
 262	}
 263
 264	return 0;
 265}
 266
 267static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src,
 268			     unsigned long offset, struct sgx_secinfo *secinfo,
 269			     unsigned long flags)
 270{
 271	struct sgx_encl_page *encl_page;
 272	struct sgx_epc_page *epc_page;
 273	struct sgx_va_page *va_page;
 274	int ret;
 275
 276	encl_page = sgx_encl_page_alloc(encl, offset, secinfo->flags);
 277	if (IS_ERR(encl_page))
 278		return PTR_ERR(encl_page);
 279
 280	epc_page = sgx_alloc_epc_page(encl_page, true);
 281	if (IS_ERR(epc_page)) {
 282		kfree(encl_page);
 283		return PTR_ERR(epc_page);
 284	}
 285
 286	va_page = sgx_encl_grow(encl, true);
 287	if (IS_ERR(va_page)) {
 288		ret = PTR_ERR(va_page);
 289		goto err_out_free;
 290	}
 291
 292	mmap_read_lock(current->mm);
 293	mutex_lock(&encl->lock);
 294
 295	/*
 296	 * Adding to encl->va_pages must be done under encl->lock.  Ditto for
 297	 * deleting (via sgx_encl_shrink()) in the error path.
 298	 */
 299	if (va_page)
 300		list_add(&va_page->list, &encl->va_pages);
 301
 302	/*
 303	 * Insert prior to EADD in case of OOM.  EADD modifies MRENCLAVE, i.e.
 304	 * can't be gracefully unwound, while failure on EADD/EXTEND is limited
 305	 * to userspace errors (or kernel/hardware bugs).
 306	 */
 307	ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc),
 308			encl_page, GFP_KERNEL);
 309	if (ret)
 310		goto err_out_unlock;
 311
 312	ret = __sgx_encl_add_page(encl, encl_page, epc_page, secinfo,
 313				  src);
 314	if (ret)
 315		goto err_out;
 316
 317	/*
 318	 * Complete the "add" before doing the "extend" so that the "add"
 319	 * isn't in a half-baked state in the extremely unlikely scenario
 320	 * the enclave will be destroyed in response to EEXTEND failure.
 321	 */
 322	encl_page->encl = encl;
 323	encl_page->epc_page = epc_page;
 324	encl_page->type = (secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK) >> 8;
 325	encl->secs_child_cnt++;
 326
 327	if (flags & SGX_PAGE_MEASURE) {
 328		ret = __sgx_encl_extend(encl, epc_page);
 329		if (ret)
 330			goto err_out;
 331	}
 332
 333	sgx_mark_page_reclaimable(encl_page->epc_page);
 334	mutex_unlock(&encl->lock);
 335	mmap_read_unlock(current->mm);
 336	return ret;
 337
 338err_out:
 339	xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc));
 340
 341err_out_unlock:
 342	sgx_encl_shrink(encl, va_page);
 343	mutex_unlock(&encl->lock);
 344	mmap_read_unlock(current->mm);
 345
 346err_out_free:
 347	sgx_encl_free_epc_page(epc_page);
 348	kfree(encl_page);
 349
 350	return ret;
 351}
 352
 353/*
 354 * Ensure user provided offset and length values are valid for
 355 * an enclave.
 356 */
 357static int sgx_validate_offset_length(struct sgx_encl *encl,
 358				      unsigned long offset,
 359				      unsigned long length)
 360{
 361	if (!IS_ALIGNED(offset, PAGE_SIZE))
 362		return -EINVAL;
 363
 364	if (!length || !IS_ALIGNED(length, PAGE_SIZE))
 365		return -EINVAL;
 366
 367	if (offset + length < offset)
 368		return -EINVAL;
 369
 370	if (offset + length - PAGE_SIZE >= encl->size)
 371		return -EINVAL;
 372
 373	return 0;
 374}
 375
 376/**
 377 * sgx_ioc_enclave_add_pages() - The handler for %SGX_IOC_ENCLAVE_ADD_PAGES
 378 * @encl:       an enclave pointer
 379 * @arg:	a user pointer to a struct sgx_enclave_add_pages instance
 380 *
 381 * Add one or more pages to an uninitialized enclave, and optionally extend the
 382 * measurement with the contents of the page. The SECINFO and measurement mask
 383 * are applied to all pages.
 384 *
 385 * A SECINFO for a TCS is required to always contain zero permissions because
 386 * CPU silently zeros them. Allowing anything else would cause a mismatch in
 387 * the measurement.
 388 *
 389 * mmap()'s protection bits are capped by the page permissions. For each page
 390 * address, the maximum protection bits are computed with the following
 391 * heuristics:
 392 *
 393 * 1. A regular page: PROT_R, PROT_W and PROT_X match the SECINFO permissions.
 394 * 2. A TCS page: PROT_R | PROT_W.
 395 *
 396 * mmap() is not allowed to surpass the minimum of the maximum protection bits
 397 * within the given address range.
 398 *
 399 * The function deinitializes kernel data structures for enclave and returns
 400 * -EIO in any of the following conditions:
 401 *
 402 * - Enclave Page Cache (EPC), the physical memory holding enclaves, has
 403 *   been invalidated. This will cause EADD and EEXTEND to fail.
 404 * - If the source address is corrupted somehow when executing EADD.
 405 *
 406 * Return:
 407 * - 0:		Success.
 408 * - -EACCES:	The source page is located in a noexec partition.
 409 * - -ENOMEM:	Out of EPC pages.
 410 * - -EINTR:	The call was interrupted before data was processed.
 411 * - -EIO:	Either EADD or EEXTEND failed because invalid source address
 412 *		or power cycle.
 413 * - -errno:	POSIX error.
 414 */
 415static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg)
 416{
 417	struct sgx_enclave_add_pages add_arg;
 418	struct sgx_secinfo secinfo;
 419	unsigned long c;
 420	int ret;
 421
 422	if (!test_bit(SGX_ENCL_CREATED, &encl->flags) ||
 423	    test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
 424		return -EINVAL;
 425
 426	if (copy_from_user(&add_arg, arg, sizeof(add_arg)))
 427		return -EFAULT;
 428
 429	if (!IS_ALIGNED(add_arg.src, PAGE_SIZE))
 430		return -EINVAL;
 431
 432	if (sgx_validate_offset_length(encl, add_arg.offset, add_arg.length))
 433		return -EINVAL;
 434
 435	if (copy_from_user(&secinfo, (void __user *)add_arg.secinfo,
 436			   sizeof(secinfo)))
 437		return -EFAULT;
 438
 439	if (sgx_validate_secinfo(&secinfo))
 440		return -EINVAL;
 441
 442	for (c = 0 ; c < add_arg.length; c += PAGE_SIZE) {
 443		if (signal_pending(current)) {
 444			if (!c)
 445				ret = -ERESTARTSYS;
 446
 447			break;
 448		}
 449
 450		if (need_resched())
 451			cond_resched();
 452
 453		ret = sgx_encl_add_page(encl, add_arg.src + c, add_arg.offset + c,
 454					&secinfo, add_arg.flags);
 455		if (ret)
 456			break;
 457	}
 458
 459	add_arg.count = c;
 460
 461	if (copy_to_user(arg, &add_arg, sizeof(add_arg)))
 462		return -EFAULT;
 463
 464	return ret;
 465}
 466
 467static int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct,
 468			 void *token)
 469{
 470	u64 mrsigner[4];
 471	int i, j;
 472	void *addr;
 473	int ret;
 474
 475	/*
 476	 * Deny initializing enclaves with attributes (namely provisioning)
 477	 * that have not been explicitly allowed.
 478	 */
 479	if (encl->attributes & ~encl->attributes_mask)
 480		return -EACCES;
 481
 482	/*
 483	 * Attributes should not be enforced *only* against what's available on
 484	 * platform (done in sgx_encl_create) but checked and enforced against
 485	 * the mask for enforcement in sigstruct. For example an enclave could
 486	 * opt to sign with AVX bit in xfrm, but still be loadable on a platform
 487	 * without it if the sigstruct->body.attributes_mask does not turn that
 488	 * bit on.
 489	 */
 490	if (sigstruct->body.attributes & sigstruct->body.attributes_mask &
 491	    sgx_attributes_reserved_mask)
 492		return -EINVAL;
 493
 494	if (sigstruct->body.miscselect & sigstruct->body.misc_mask &
 495	    sgx_misc_reserved_mask)
 496		return -EINVAL;
 497
 498	if (sigstruct->body.xfrm & sigstruct->body.xfrm_mask &
 499	    sgx_xfrm_reserved_mask)
 500		return -EINVAL;
 501
 502	sha256(sigstruct->modulus, SGX_MODULUS_SIZE, (u8 *)mrsigner);
 503
 504	mutex_lock(&encl->lock);
 505
 506	/*
 507	 * ENCLS[EINIT] is interruptible because it has such a high latency,
 508	 * e.g. 50k+ cycles on success. If an IRQ/NMI/SMI becomes pending,
 509	 * EINIT may fail with SGX_UNMASKED_EVENT so that the event can be
 510	 * serviced.
 511	 */
 512	for (i = 0; i < SGX_EINIT_SLEEP_COUNT; i++) {
 513		for (j = 0; j < SGX_EINIT_SPIN_COUNT; j++) {
 514			addr = sgx_get_epc_virt_addr(encl->secs.epc_page);
 515
 516			preempt_disable();
 517
 518			sgx_update_lepubkeyhash(mrsigner);
 519
 520			ret = __einit(sigstruct, token, addr);
 521
 522			preempt_enable();
 523
 524			if (ret == SGX_UNMASKED_EVENT)
 525				continue;
 526			else
 527				break;
 528		}
 529
 530		if (ret != SGX_UNMASKED_EVENT)
 531			break;
 532
 533		msleep_interruptible(SGX_EINIT_SLEEP_TIME);
 534
 535		if (signal_pending(current)) {
 536			ret = -ERESTARTSYS;
 537			goto err_out;
 538		}
 539	}
 540
 541	if (encls_faulted(ret)) {
 542		if (encls_failed(ret))
 543			ENCLS_WARN(ret, "EINIT");
 544
 545		ret = -EIO;
 546	} else if (ret) {
 547		pr_debug("EINIT returned %d\n", ret);
 548		ret = -EPERM;
 549	} else {
 550		set_bit(SGX_ENCL_INITIALIZED, &encl->flags);
 551	}
 552
 553err_out:
 554	mutex_unlock(&encl->lock);
 555	return ret;
 556}
 557
 558/**
 559 * sgx_ioc_enclave_init() - handler for %SGX_IOC_ENCLAVE_INIT
 560 * @encl:	an enclave pointer
 561 * @arg:	userspace pointer to a struct sgx_enclave_init instance
 562 *
 563 * Flush any outstanding enqueued EADD operations and perform EINIT.  The
 564 * Launch Enclave Public Key Hash MSRs are rewritten as necessary to match
 565 * the enclave's MRSIGNER, which is calculated from the provided sigstruct.
 566 *
 567 * Return:
 568 * - 0:		Success.
 569 * - -EPERM:	Invalid SIGSTRUCT.
 570 * - -EIO:	EINIT failed because of a power cycle.
 571 * - -errno:	POSIX error.
 572 */
 573static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg)
 574{
 575	struct sgx_sigstruct *sigstruct;
 576	struct sgx_enclave_init init_arg;
 577	void *token;
 578	int ret;
 579
 580	if (!test_bit(SGX_ENCL_CREATED, &encl->flags) ||
 581	    test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
 582		return -EINVAL;
 583
 584	if (copy_from_user(&init_arg, arg, sizeof(init_arg)))
 585		return -EFAULT;
 586
 587	/*
 588	 * 'sigstruct' must be on a page boundary and 'token' on a 512 byte
 589	 * boundary.  kmalloc() will give this alignment when allocating
 590	 * PAGE_SIZE bytes.
 591	 */
 592	sigstruct = kmalloc(PAGE_SIZE, GFP_KERNEL);
 593	if (!sigstruct)
 594		return -ENOMEM;
 595
 596	token = (void *)((unsigned long)sigstruct + PAGE_SIZE / 2);
 597	memset(token, 0, SGX_LAUNCH_TOKEN_SIZE);
 598
 599	if (copy_from_user(sigstruct, (void __user *)init_arg.sigstruct,
 600			   sizeof(*sigstruct))) {
 601		ret = -EFAULT;
 602		goto out;
 603	}
 604
 605	/*
 606	 * A legacy field used with Intel signed enclaves. These used to mean
 607	 * regular and architectural enclaves. The CPU only accepts these values
 608	 * but they do not have any other meaning.
 609	 *
 610	 * Thus, reject any other values.
 611	 */
 612	if (sigstruct->header.vendor != 0x0000 &&
 613	    sigstruct->header.vendor != 0x8086) {
 614		ret = -EINVAL;
 615		goto out;
 616	}
 617
 618	ret = sgx_encl_init(encl, sigstruct, token);
 619
 620out:
 621	kfree(sigstruct);
 622	return ret;
 623}
 624
 625/**
 626 * sgx_ioc_enclave_provision() - handler for %SGX_IOC_ENCLAVE_PROVISION
 627 * @encl:	an enclave pointer
 628 * @arg:	userspace pointer to a struct sgx_enclave_provision instance
 629 *
 630 * Allow ATTRIBUTE.PROVISION_KEY for an enclave by providing a file handle to
 631 * /dev/sgx_provision.
 632 *
 633 * Return:
 634 * - 0:		Success.
 635 * - -errno:	Otherwise.
 636 */
 637static long sgx_ioc_enclave_provision(struct sgx_encl *encl, void __user *arg)
 638{
 639	struct sgx_enclave_provision params;
 640
 641	if (copy_from_user(&params, arg, sizeof(params)))
 642		return -EFAULT;
 643
 644	return sgx_set_attribute(&encl->attributes_mask, params.fd);
 645}
 646
 647/*
 648 * Ensure enclave is ready for SGX2 functions. Readiness is checked
 649 * by ensuring the hardware supports SGX2 and the enclave is initialized
 650 * and thus able to handle requests to modify pages within it.
 651 */
 652static int sgx_ioc_sgx2_ready(struct sgx_encl *encl)
 653{
 654	if (!(cpu_feature_enabled(X86_FEATURE_SGX2)))
 655		return -ENODEV;
 656
 657	if (!test_bit(SGX_ENCL_INITIALIZED, &encl->flags))
 658		return -EINVAL;
 659
 660	return 0;
 661}
 662
 663/*
 664 * Some SGX functions require that no cached linear-to-physical address
 665 * mappings are present before they can succeed. Collaborate with
 666 * hardware via ENCLS[ETRACK] to ensure that all cached
 667 * linear-to-physical address mappings belonging to all threads of
 668 * the enclave are cleared. See sgx_encl_cpumask() for details.
 669 *
 670 * Must be called with enclave's mutex held from the time the
 671 * SGX function requiring that no cached linear-to-physical mappings
 672 * are present is executed until this ETRACK flow is complete.
 673 */
 674static int sgx_enclave_etrack(struct sgx_encl *encl)
 675{
 676	void *epc_virt;
 677	int ret;
 678
 679	epc_virt = sgx_get_epc_virt_addr(encl->secs.epc_page);
 680	ret = __etrack(epc_virt);
 681	if (ret) {
 682		/*
 683		 * ETRACK only fails when there is an OS issue. For
 684		 * example, two consecutive ETRACK was sent without
 685		 * completed IPI between.
 686		 */
 687		pr_err_once("ETRACK returned %d (0x%x)", ret, ret);
 688		/*
 689		 * Send IPIs to kick CPUs out of the enclave and
 690		 * try ETRACK again.
 691		 */
 692		on_each_cpu_mask(sgx_encl_cpumask(encl), sgx_ipi_cb, NULL, 1);
 693		ret = __etrack(epc_virt);
 694		if (ret) {
 695			pr_err_once("ETRACK repeat returned %d (0x%x)",
 696				    ret, ret);
 697			return -EFAULT;
 698		}
 699	}
 700	on_each_cpu_mask(sgx_encl_cpumask(encl), sgx_ipi_cb, NULL, 1);
 701
 702	return 0;
 703}
 704
 705/**
 706 * sgx_enclave_restrict_permissions() - Restrict EPCM permissions
 707 * @encl:	Enclave to which the pages belong.
 708 * @modp:	Checked parameters from user on which pages need modifying and
 709 *              their new permissions.
 710 *
 711 * Return:
 712 * - 0:		Success.
 713 * - -errno:	Otherwise.
 714 */
 715static long
 716sgx_enclave_restrict_permissions(struct sgx_encl *encl,
 717				 struct sgx_enclave_restrict_permissions *modp)
 718{
 719	struct sgx_encl_page *entry;
 720	struct sgx_secinfo secinfo;
 721	unsigned long addr;
 722	unsigned long c;
 723	void *epc_virt;
 724	int ret;
 725
 726	memset(&secinfo, 0, sizeof(secinfo));
 727	secinfo.flags = modp->permissions & SGX_SECINFO_PERMISSION_MASK;
 728
 729	for (c = 0 ; c < modp->length; c += PAGE_SIZE) {
 730		addr = encl->base + modp->offset + c;
 731
 732		sgx_reclaim_direct();
 733
 734		mutex_lock(&encl->lock);
 735
 736		entry = sgx_encl_load_page(encl, addr);
 737		if (IS_ERR(entry)) {
 738			ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
 739			goto out_unlock;
 740		}
 741
 742		/*
 743		 * Changing EPCM permissions is only supported on regular
 744		 * SGX pages. Attempting this change on other pages will
 745		 * result in #PF.
 746		 */
 747		if (entry->type != SGX_PAGE_TYPE_REG) {
 748			ret = -EINVAL;
 749			goto out_unlock;
 750		}
 751
 752		/*
 753		 * Apart from ensuring that read-access remains, do not verify
 754		 * the permission bits requested. Kernel has no control over
 755		 * how EPCM permissions can be relaxed from within the enclave.
 756		 * ENCLS[EMODPR] can only remove existing EPCM permissions,
 757		 * attempting to set new permissions will be ignored by the
 758		 * hardware.
 759		 */
 760
 761		/* Change EPCM permissions. */
 762		epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
 763		ret = __emodpr(&secinfo, epc_virt);
 764		if (encls_faulted(ret)) {
 765			/*
 766			 * All possible faults should be avoidable:
 767			 * parameters have been checked, will only change
 768			 * permissions of a regular page, and no concurrent
 769			 * SGX1/SGX2 ENCLS instructions since these
 770			 * are protected with mutex.
 771			 */
 772			pr_err_once("EMODPR encountered exception %d\n",
 773				    ENCLS_TRAPNR(ret));
 774			ret = -EFAULT;
 775			goto out_unlock;
 776		}
 777		if (encls_failed(ret)) {
 778			modp->result = ret;
 779			ret = -EFAULT;
 780			goto out_unlock;
 781		}
 782
 783		ret = sgx_enclave_etrack(encl);
 784		if (ret) {
 785			ret = -EFAULT;
 786			goto out_unlock;
 787		}
 788
 789		mutex_unlock(&encl->lock);
 790	}
 791
 792	ret = 0;
 793	goto out;
 794
 795out_unlock:
 796	mutex_unlock(&encl->lock);
 797out:
 798	modp->count = c;
 799
 800	return ret;
 801}
 802
 803/**
 804 * sgx_ioc_enclave_restrict_permissions() - handler for
 805 *                                        %SGX_IOC_ENCLAVE_RESTRICT_PERMISSIONS
 806 * @encl:	an enclave pointer
 807 * @arg:	userspace pointer to a &struct sgx_enclave_restrict_permissions
 808 *		instance
 809 *
 810 * SGX2 distinguishes between relaxing and restricting the enclave page
 811 * permissions maintained by the hardware (EPCM permissions) of pages
 812 * belonging to an initialized enclave (after SGX_IOC_ENCLAVE_INIT).
 813 *
 814 * EPCM permissions cannot be restricted from within the enclave, the enclave
 815 * requires the kernel to run the privileged level 0 instructions ENCLS[EMODPR]
 816 * and ENCLS[ETRACK]. An attempt to relax EPCM permissions with this call
 817 * will be ignored by the hardware.
 818 *
 819 * Return:
 820 * - 0:		Success
 821 * - -errno:	Otherwise
 822 */
 823static long sgx_ioc_enclave_restrict_permissions(struct sgx_encl *encl,
 824						 void __user *arg)
 825{
 826	struct sgx_enclave_restrict_permissions params;
 827	long ret;
 828
 829	ret = sgx_ioc_sgx2_ready(encl);
 830	if (ret)
 831		return ret;
 832
 833	if (copy_from_user(&params, arg, sizeof(params)))
 834		return -EFAULT;
 835
 836	if (sgx_validate_offset_length(encl, params.offset, params.length))
 837		return -EINVAL;
 838
 839	if (params.permissions & ~SGX_SECINFO_PERMISSION_MASK)
 840		return -EINVAL;
 841
 842	/*
 843	 * Fail early if invalid permissions requested to prevent ENCLS[EMODPR]
 844	 * from faulting later when the CPU does the same check.
 845	 */
 846	if ((params.permissions & SGX_SECINFO_W) &&
 847	    !(params.permissions & SGX_SECINFO_R))
 848		return -EINVAL;
 849
 850	if (params.result || params.count)
 851		return -EINVAL;
 852
 853	ret = sgx_enclave_restrict_permissions(encl, &params);
 854
 855	if (copy_to_user(arg, &params, sizeof(params)))
 856		return -EFAULT;
 857
 858	return ret;
 859}
 860
 861/**
 862 * sgx_enclave_modify_types() - Modify type of SGX enclave pages
 863 * @encl:	Enclave to which the pages belong.
 864 * @modt:	Checked parameters from user about which pages need modifying
 865 *              and their new page type.
 866 *
 867 * Return:
 868 * - 0:		Success
 869 * - -errno:	Otherwise
 870 */
 871static long sgx_enclave_modify_types(struct sgx_encl *encl,
 872				     struct sgx_enclave_modify_types *modt)
 873{
 874	unsigned long max_prot_restore;
 875	enum sgx_page_type page_type;
 876	struct sgx_encl_page *entry;
 877	struct sgx_secinfo secinfo;
 878	unsigned long prot;
 879	unsigned long addr;
 880	unsigned long c;
 881	void *epc_virt;
 882	int ret;
 883
 884	page_type = modt->page_type & SGX_PAGE_TYPE_MASK;
 885
 886	/*
 887	 * The only new page types allowed by hardware are PT_TCS and PT_TRIM.
 888	 */
 889	if (page_type != SGX_PAGE_TYPE_TCS && page_type != SGX_PAGE_TYPE_TRIM)
 890		return -EINVAL;
 891
 892	memset(&secinfo, 0, sizeof(secinfo));
 893
 894	secinfo.flags = page_type << 8;
 895
 896	for (c = 0 ; c < modt->length; c += PAGE_SIZE) {
 897		addr = encl->base + modt->offset + c;
 898
 899		sgx_reclaim_direct();
 900
 901		mutex_lock(&encl->lock);
 902
 903		entry = sgx_encl_load_page(encl, addr);
 904		if (IS_ERR(entry)) {
 905			ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
 906			goto out_unlock;
 907		}
 908
 909		/*
 910		 * Borrow the logic from the Intel SDM. Regular pages
 911		 * (SGX_PAGE_TYPE_REG) can change type to SGX_PAGE_TYPE_TCS
 912		 * or SGX_PAGE_TYPE_TRIM but TCS pages can only be trimmed.
 913		 * CET pages not supported yet.
 914		 */
 915		if (!(entry->type == SGX_PAGE_TYPE_REG ||
 916		      (entry->type == SGX_PAGE_TYPE_TCS &&
 917		       page_type == SGX_PAGE_TYPE_TRIM))) {
 918			ret = -EINVAL;
 919			goto out_unlock;
 920		}
 921
 922		max_prot_restore = entry->vm_max_prot_bits;
 923
 924		/*
 925		 * Once a regular page becomes a TCS page it cannot be
 926		 * changed back. So the maximum allowed protection reflects
 927		 * the TCS page that is always RW from kernel perspective but
 928		 * will be inaccessible from within enclave. Before doing
 929		 * so, do make sure that the new page type continues to
 930		 * respect the originally vetted page permissions.
 931		 */
 932		if (entry->type == SGX_PAGE_TYPE_REG &&
 933		    page_type == SGX_PAGE_TYPE_TCS) {
 934			if (~entry->vm_max_prot_bits & (VM_READ | VM_WRITE)) {
 935				ret = -EPERM;
 936				goto out_unlock;
 937			}
 938			prot = PROT_READ | PROT_WRITE;
 939			entry->vm_max_prot_bits = calc_vm_prot_bits(prot, 0);
 940
 941			/*
 942			 * Prevent page from being reclaimed while mutex
 943			 * is released.
 944			 */
 945			if (sgx_unmark_page_reclaimable(entry->epc_page)) {
 946				ret = -EAGAIN;
 947				goto out_entry_changed;
 948			}
 949
 950			/*
 951			 * Do not keep encl->lock because of dependency on
 952			 * mmap_lock acquired in sgx_zap_enclave_ptes().
 953			 */
 954			mutex_unlock(&encl->lock);
 955
 956			sgx_zap_enclave_ptes(encl, addr);
 957
 958			mutex_lock(&encl->lock);
 959
 960			sgx_mark_page_reclaimable(entry->epc_page);
 961		}
 962
 963		/* Change EPC type */
 964		epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
 965		ret = __emodt(&secinfo, epc_virt);
 966		if (encls_faulted(ret)) {
 967			/*
 968			 * All possible faults should be avoidable:
 969			 * parameters have been checked, will only change
 970			 * valid page types, and no concurrent
 971			 * SGX1/SGX2 ENCLS instructions since these are
 972			 * protected with mutex.
 973			 */
 974			pr_err_once("EMODT encountered exception %d\n",
 975				    ENCLS_TRAPNR(ret));
 976			ret = -EFAULT;
 977			goto out_entry_changed;
 978		}
 979		if (encls_failed(ret)) {
 980			modt->result = ret;
 981			ret = -EFAULT;
 982			goto out_entry_changed;
 983		}
 984
 985		ret = sgx_enclave_etrack(encl);
 986		if (ret) {
 987			ret = -EFAULT;
 988			goto out_unlock;
 989		}
 990
 991		entry->type = page_type;
 992
 993		mutex_unlock(&encl->lock);
 994	}
 995
 996	ret = 0;
 997	goto out;
 998
 999out_entry_changed:
1000	entry->vm_max_prot_bits = max_prot_restore;
1001out_unlock:
1002	mutex_unlock(&encl->lock);
1003out:
1004	modt->count = c;
1005
1006	return ret;
1007}
1008
1009/**
1010 * sgx_ioc_enclave_modify_types() - handler for %SGX_IOC_ENCLAVE_MODIFY_TYPES
1011 * @encl:	an enclave pointer
1012 * @arg:	userspace pointer to a &struct sgx_enclave_modify_types instance
1013 *
1014 * Ability to change the enclave page type supports the following use cases:
1015 *
1016 * * It is possible to add TCS pages to an enclave by changing the type of
1017 *   regular pages (%SGX_PAGE_TYPE_REG) to TCS (%SGX_PAGE_TYPE_TCS) pages.
1018 *   With this support the number of threads supported by an initialized
1019 *   enclave can be increased dynamically.
1020 *
1021 * * Regular or TCS pages can dynamically be removed from an initialized
1022 *   enclave by changing the page type to %SGX_PAGE_TYPE_TRIM. Changing the
1023 *   page type to %SGX_PAGE_TYPE_TRIM marks the page for removal with actual
1024 *   removal done by handler of %SGX_IOC_ENCLAVE_REMOVE_PAGES ioctl() called
1025 *   after ENCLU[EACCEPT] is run on %SGX_PAGE_TYPE_TRIM page from within the
1026 *   enclave.
1027 *
1028 * Return:
1029 * - 0:		Success
1030 * - -errno:	Otherwise
1031 */
1032static long sgx_ioc_enclave_modify_types(struct sgx_encl *encl,
1033					 void __user *arg)
1034{
1035	struct sgx_enclave_modify_types params;
1036	long ret;
1037
1038	ret = sgx_ioc_sgx2_ready(encl);
1039	if (ret)
1040		return ret;
1041
1042	if (copy_from_user(&params, arg, sizeof(params)))
1043		return -EFAULT;
1044
1045	if (sgx_validate_offset_length(encl, params.offset, params.length))
1046		return -EINVAL;
1047
1048	if (params.page_type & ~SGX_PAGE_TYPE_MASK)
1049		return -EINVAL;
1050
1051	if (params.result || params.count)
1052		return -EINVAL;
1053
1054	ret = sgx_enclave_modify_types(encl, &params);
1055
1056	if (copy_to_user(arg, &params, sizeof(params)))
1057		return -EFAULT;
1058
1059	return ret;
1060}
1061
1062/**
1063 * sgx_encl_remove_pages() - Remove trimmed pages from SGX enclave
1064 * @encl:	Enclave to which the pages belong
1065 * @params:	Checked parameters from user on which pages need to be removed
1066 *
1067 * Return:
1068 * - 0:		Success.
1069 * - -errno:	Otherwise.
1070 */
1071static long sgx_encl_remove_pages(struct sgx_encl *encl,
1072				  struct sgx_enclave_remove_pages *params)
1073{
1074	struct sgx_encl_page *entry;
1075	struct sgx_secinfo secinfo;
1076	unsigned long addr;
1077	unsigned long c;
1078	void *epc_virt;
1079	int ret;
1080
1081	memset(&secinfo, 0, sizeof(secinfo));
1082	secinfo.flags = SGX_SECINFO_R | SGX_SECINFO_W | SGX_SECINFO_X;
1083
1084	for (c = 0 ; c < params->length; c += PAGE_SIZE) {
1085		addr = encl->base + params->offset + c;
1086
1087		sgx_reclaim_direct();
1088
1089		mutex_lock(&encl->lock);
1090
1091		entry = sgx_encl_load_page(encl, addr);
1092		if (IS_ERR(entry)) {
1093			ret = PTR_ERR(entry) == -EBUSY ? -EAGAIN : -EFAULT;
1094			goto out_unlock;
1095		}
1096
1097		if (entry->type != SGX_PAGE_TYPE_TRIM) {
1098			ret = -EPERM;
1099			goto out_unlock;
1100		}
1101
1102		/*
1103		 * ENCLS[EMODPR] is a no-op instruction used to inform if
1104		 * ENCLU[EACCEPT] was run from within the enclave. If
1105		 * ENCLS[EMODPR] is run with RWX on a trimmed page that is
1106		 * not yet accepted then it will return
1107		 * %SGX_PAGE_NOT_MODIFIABLE, after the trimmed page is
1108		 * accepted the instruction will encounter a page fault.
1109		 */
1110		epc_virt = sgx_get_epc_virt_addr(entry->epc_page);
1111		ret = __emodpr(&secinfo, epc_virt);
1112		if (!encls_faulted(ret) || ENCLS_TRAPNR(ret) != X86_TRAP_PF) {
1113			ret = -EPERM;
1114			goto out_unlock;
1115		}
1116
1117		if (sgx_unmark_page_reclaimable(entry->epc_page)) {
1118			ret = -EBUSY;
1119			goto out_unlock;
1120		}
1121
1122		/*
1123		 * Do not keep encl->lock because of dependency on
1124		 * mmap_lock acquired in sgx_zap_enclave_ptes().
1125		 */
1126		mutex_unlock(&encl->lock);
1127
1128		sgx_zap_enclave_ptes(encl, addr);
1129
1130		mutex_lock(&encl->lock);
1131
1132		sgx_encl_free_epc_page(entry->epc_page);
1133		encl->secs_child_cnt--;
1134		entry->epc_page = NULL;
1135		xa_erase(&encl->page_array, PFN_DOWN(entry->desc));
1136		sgx_encl_shrink(encl, NULL);
1137		kfree(entry);
1138
1139		mutex_unlock(&encl->lock);
1140	}
1141
1142	ret = 0;
1143	goto out;
1144
1145out_unlock:
1146	mutex_unlock(&encl->lock);
1147out:
1148	params->count = c;
1149
1150	return ret;
1151}
1152
1153/**
1154 * sgx_ioc_enclave_remove_pages() - handler for %SGX_IOC_ENCLAVE_REMOVE_PAGES
1155 * @encl:	an enclave pointer
1156 * @arg:	userspace pointer to &struct sgx_enclave_remove_pages instance
1157 *
1158 * Final step of the flow removing pages from an initialized enclave. The
1159 * complete flow is:
1160 *
1161 * 1) User changes the type of the pages to be removed to %SGX_PAGE_TYPE_TRIM
1162 *    using the %SGX_IOC_ENCLAVE_MODIFY_TYPES ioctl().
1163 * 2) User approves the page removal by running ENCLU[EACCEPT] from within
1164 *    the enclave.
1165 * 3) User initiates actual page removal using the
1166 *    %SGX_IOC_ENCLAVE_REMOVE_PAGES ioctl() that is handled here.
1167 *
1168 * First remove any page table entries pointing to the page and then proceed
1169 * with the actual removal of the enclave page and data in support of it.
1170 *
1171 * VA pages are not affected by this removal. It is thus possible that the
1172 * enclave may end up with more VA pages than needed to support all its
1173 * pages.
1174 *
1175 * Return:
1176 * - 0:		Success
1177 * - -errno:	Otherwise
1178 */
1179static long sgx_ioc_enclave_remove_pages(struct sgx_encl *encl,
1180					 void __user *arg)
1181{
1182	struct sgx_enclave_remove_pages params;
1183	long ret;
1184
1185	ret = sgx_ioc_sgx2_ready(encl);
1186	if (ret)
1187		return ret;
1188
1189	if (copy_from_user(&params, arg, sizeof(params)))
1190		return -EFAULT;
1191
1192	if (sgx_validate_offset_length(encl, params.offset, params.length))
1193		return -EINVAL;
1194
1195	if (params.count)
1196		return -EINVAL;
1197
1198	ret = sgx_encl_remove_pages(encl, &params);
1199
1200	if (copy_to_user(arg, &params, sizeof(params)))
1201		return -EFAULT;
1202
1203	return ret;
1204}
1205
1206long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1207{
1208	struct sgx_encl *encl = filep->private_data;
1209	int ret;
1210
1211	if (test_and_set_bit(SGX_ENCL_IOCTL, &encl->flags))
1212		return -EBUSY;
1213
1214	switch (cmd) {
1215	case SGX_IOC_ENCLAVE_CREATE:
1216		ret = sgx_ioc_enclave_create(encl, (void __user *)arg);
1217		break;
1218	case SGX_IOC_ENCLAVE_ADD_PAGES:
1219		ret = sgx_ioc_enclave_add_pages(encl, (void __user *)arg);
1220		break;
1221	case SGX_IOC_ENCLAVE_INIT:
1222		ret = sgx_ioc_enclave_init(encl, (void __user *)arg);
1223		break;
1224	case SGX_IOC_ENCLAVE_PROVISION:
1225		ret = sgx_ioc_enclave_provision(encl, (void __user *)arg);
1226		break;
1227	case SGX_IOC_ENCLAVE_RESTRICT_PERMISSIONS:
1228		ret = sgx_ioc_enclave_restrict_permissions(encl,
1229							   (void __user *)arg);
1230		break;
1231	case SGX_IOC_ENCLAVE_MODIFY_TYPES:
1232		ret = sgx_ioc_enclave_modify_types(encl, (void __user *)arg);
1233		break;
1234	case SGX_IOC_ENCLAVE_REMOVE_PAGES:
1235		ret = sgx_ioc_enclave_remove_pages(encl, (void __user *)arg);
1236		break;
1237	default:
1238		ret = -ENOIOCTLCMD;
1239		break;
1240	}
1241
1242	clear_bit(SGX_ENCL_IOCTL, &encl->flags);
1243	return ret;
1244}