main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*  Copyright(c) 2016-20 Intel Corporation. */
   3
   4#include <linux/file.h>
   5#include <linux/freezer.h>
   6#include <linux/highmem.h>
   7#include <linux/kthread.h>
   8#include <linux/kvm_types.h>
   9#include <linux/miscdevice.h>
  10#include <linux/node.h>
  11#include <linux/pagemap.h>
  12#include <linux/ratelimit.h>
  13#include <linux/sched/mm.h>
  14#include <linux/sched/signal.h>
  15#include <linux/slab.h>
  16#include <linux/sysfs.h>
  17#include <linux/vmalloc.h>
  18#include <asm/msr.h>
  19#include <asm/sgx.h>
  20#include <asm/archrandom.h>
  21#include "driver.h"
  22#include "encl.h"
  23#include "encls.h"
  24
  25struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS];
  26static int sgx_nr_epc_sections;
  27static struct task_struct *ksgxd_tsk;
  28static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq);
  29static DEFINE_XARRAY(sgx_epc_address_space);
  30
  31/*
  32 * These variables are part of the state of the reclaimer, and must be accessed
  33 * with sgx_reclaimer_lock acquired.
  34 */
  35static LIST_HEAD(sgx_active_page_list);
  36static DEFINE_SPINLOCK(sgx_reclaimer_lock);
  37
  38static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0);
  39
  40/* Nodes with one or more EPC sections. */
  41static nodemask_t sgx_numa_mask;
  42
  43/*
  44 * Array with one list_head for each possible NUMA node.  Each
  45 * list contains all the sgx_epc_section's which are on that
  46 * node.
  47 */
  48static struct sgx_numa_node *sgx_numa_nodes;
  49
  50static LIST_HEAD(sgx_dirty_page_list);
  51
  52/*
  53 * Reset post-kexec EPC pages to the uninitialized state. The pages are removed
  54 * from the input list, and made available for the page allocator. SECS pages
  55 * prepending their children in the input list are left intact.
  56 *
  57 * Return 0 when sanitization was successful or kthread was stopped, and the
  58 * number of unsanitized pages otherwise.
  59 */
  60static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list)
  61{
  62	unsigned long left_dirty = 0;
  63	struct sgx_epc_page *page;
  64	LIST_HEAD(dirty);
  65	int ret;
  66
  67	/* dirty_page_list is thread-local, no need for a lock: */
  68	while (!list_empty(dirty_page_list)) {
  69		if (kthread_should_stop())
  70			return 0;
  71
  72		page = list_first_entry(dirty_page_list, struct sgx_epc_page, list);
  73
  74		/*
  75		 * Checking page->poison without holding the node->lock
  76		 * is racy, but losing the race (i.e. poison is set just
  77		 * after the check) just means __eremove() will be uselessly
  78		 * called for a page that sgx_free_epc_page() will put onto
  79		 * the node->sgx_poison_page_list later.
  80		 */
  81		if (page->poison) {
  82			struct sgx_epc_section *section = &sgx_epc_sections[page->section];
  83			struct sgx_numa_node *node = section->node;
  84
  85			spin_lock(&node->lock);
  86			list_move(&page->list, &node->sgx_poison_page_list);
  87			spin_unlock(&node->lock);
  88
  89			continue;
  90		}
  91
  92		ret = __eremove(sgx_get_epc_virt_addr(page));
  93		if (!ret) {
  94			/*
  95			 * page is now sanitized.  Make it available via the SGX
  96			 * page allocator:
  97			 */
  98			list_del(&page->list);
  99			sgx_free_epc_page(page);
 100		} else {
 101			/* The page is not yet clean - move to the dirty list. */
 102			list_move_tail(&page->list, &dirty);
 103			left_dirty++;
 104		}
 105
 106		cond_resched();
 107	}
 108
 109	list_splice(&dirty, dirty_page_list);
 110	return left_dirty;
 111}
 112
 113static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page)
 114{
 115	struct sgx_encl_page *page = epc_page->owner;
 116	struct sgx_encl *encl = page->encl;
 117	struct sgx_encl_mm *encl_mm;
 118	bool ret = true;
 119	int idx;
 120
 121	idx = srcu_read_lock(&encl->srcu);
 122
 123	list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) {
 124		if (!mmget_not_zero(encl_mm->mm))
 125			continue;
 126
 127		mmap_read_lock(encl_mm->mm);
 128		ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page);
 129		mmap_read_unlock(encl_mm->mm);
 130
 131		mmput_async(encl_mm->mm);
 132
 133		if (!ret)
 134			break;
 135	}
 136
 137	srcu_read_unlock(&encl->srcu, idx);
 138
 139	if (!ret)
 140		return false;
 141
 142	return true;
 143}
 144
 145static void sgx_reclaimer_block(struct sgx_epc_page *epc_page)
 146{
 147	struct sgx_encl_page *page = epc_page->owner;
 148	unsigned long addr = page->desc & PAGE_MASK;
 149	struct sgx_encl *encl = page->encl;
 150	int ret;
 151
 152	sgx_zap_enclave_ptes(encl, addr);
 153
 154	mutex_lock(&encl->lock);
 155
 156	ret = __eblock(sgx_get_epc_virt_addr(epc_page));
 157	if (encls_failed(ret))
 158		ENCLS_WARN(ret, "EBLOCK");
 159
 160	mutex_unlock(&encl->lock);
 161}
 162
 163static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot,
 164			  struct sgx_backing *backing)
 165{
 166	struct sgx_pageinfo pginfo;
 167	int ret;
 168
 169	pginfo.addr = 0;
 170	pginfo.secs = 0;
 171
 172	pginfo.contents = (unsigned long)kmap_local_page(backing->contents);
 173	pginfo.metadata = (unsigned long)kmap_local_page(backing->pcmd) +
 174			  backing->pcmd_offset;
 175
 176	ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot);
 177	set_page_dirty(backing->pcmd);
 178	set_page_dirty(backing->contents);
 179
 180	kunmap_local((void *)(unsigned long)(pginfo.metadata -
 181					      backing->pcmd_offset));
 182	kunmap_local((void *)(unsigned long)pginfo.contents);
 183
 184	return ret;
 185}
 186
 187void sgx_ipi_cb(void *info)
 188{
 189}
 190
 191/*
 192 * Swap page to the regular memory transformed to the blocked state by using
 193 * EBLOCK, which means that it can no longer be referenced (no new TLB entries).
 194 *
 195 * The first trial just tries to write the page assuming that some other thread
 196 * has reset the count for threads inside the enclave by using ETRACK, and
 197 * previous thread count has been zeroed out. The second trial calls ETRACK
 198 * before EWB. If that fails we kick all the HW threads out, and then do EWB,
 199 * which should be guaranteed the succeed.
 200 */
 201static void sgx_encl_ewb(struct sgx_epc_page *epc_page,
 202			 struct sgx_backing *backing)
 203{
 204	struct sgx_encl_page *encl_page = epc_page->owner;
 205	struct sgx_encl *encl = encl_page->encl;
 206	struct sgx_va_page *va_page;
 207	unsigned int va_offset;
 208	void *va_slot;
 209	int ret;
 210
 211	encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED;
 212
 213	va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
 214				   list);
 215	va_offset = sgx_alloc_va_slot(va_page);
 216	va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset;
 217	if (sgx_va_page_full(va_page))
 218		list_move_tail(&va_page->list, &encl->va_pages);
 219
 220	ret = __sgx_encl_ewb(epc_page, va_slot, backing);
 221	if (ret == SGX_NOT_TRACKED) {
 222		ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page));
 223		if (ret) {
 224			if (encls_failed(ret))
 225				ENCLS_WARN(ret, "ETRACK");
 226		}
 227
 228		ret = __sgx_encl_ewb(epc_page, va_slot, backing);
 229		if (ret == SGX_NOT_TRACKED) {
 230			/*
 231			 * Slow path, send IPIs to kick cpus out of the
 232			 * enclave.  Note, it's imperative that the cpu
 233			 * mask is generated *after* ETRACK, else we'll
 234			 * miss cpus that entered the enclave between
 235			 * generating the mask and incrementing epoch.
 236			 */
 237			on_each_cpu_mask(sgx_encl_cpumask(encl),
 238					 sgx_ipi_cb, NULL, 1);
 239			ret = __sgx_encl_ewb(epc_page, va_slot, backing);
 240		}
 241	}
 242
 243	if (ret) {
 244		if (encls_failed(ret))
 245			ENCLS_WARN(ret, "EWB");
 246
 247		sgx_free_va_slot(va_page, va_offset);
 248	} else {
 249		encl_page->desc |= va_offset;
 250		encl_page->va_page = va_page;
 251	}
 252}
 253
 254static void sgx_reclaimer_write(struct sgx_epc_page *epc_page,
 255				struct sgx_backing *backing)
 256{
 257	struct sgx_encl_page *encl_page = epc_page->owner;
 258	struct sgx_encl *encl = encl_page->encl;
 259	struct sgx_backing secs_backing;
 260	int ret;
 261
 262	mutex_lock(&encl->lock);
 263
 264	sgx_encl_ewb(epc_page, backing);
 265	encl_page->epc_page = NULL;
 266	encl->secs_child_cnt--;
 267	sgx_encl_put_backing(backing);
 268
 269	if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) {
 270		ret = sgx_encl_alloc_backing(encl, PFN_DOWN(encl->size),
 271					   &secs_backing);
 272		if (ret)
 273			goto out;
 274
 275		sgx_encl_ewb(encl->secs.epc_page, &secs_backing);
 276
 277		sgx_encl_free_epc_page(encl->secs.epc_page);
 278		encl->secs.epc_page = NULL;
 279
 280		sgx_encl_put_backing(&secs_backing);
 281	}
 282
 283out:
 284	mutex_unlock(&encl->lock);
 285}
 286
 287/*
 288 * Take a fixed number of pages from the head of the active page pool and
 289 * reclaim them to the enclave's private shmem files. Skip the pages, which have
 290 * been accessed since the last scan. Move those pages to the tail of active
 291 * page pool so that the pages get scanned in LRU like fashion.
 292 *
 293 * Batch process a chunk of pages (at the moment 16) in order to degrade amount
 294 * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit
 295 * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI
 296 * + EWB) but not sufficiently. Reclaiming one page at a time would also be
 297 * problematic as it would increase the lock contention too much, which would
 298 * halt forward progress.
 299 */
 300static void sgx_reclaim_pages(void)
 301{
 302	struct sgx_epc_page *chunk[SGX_NR_TO_SCAN];
 303	struct sgx_backing backing[SGX_NR_TO_SCAN];
 304	struct sgx_encl_page *encl_page;
 305	struct sgx_epc_page *epc_page;
 306	pgoff_t page_index;
 307	int cnt = 0;
 308	int ret;
 309	int i;
 310
 311	spin_lock(&sgx_reclaimer_lock);
 312	for (i = 0; i < SGX_NR_TO_SCAN; i++) {
 313		if (list_empty(&sgx_active_page_list))
 314			break;
 315
 316		epc_page = list_first_entry(&sgx_active_page_list,
 317					    struct sgx_epc_page, list);
 318		list_del_init(&epc_page->list);
 319		encl_page = epc_page->owner;
 320
 321		if (kref_get_unless_zero(&encl_page->encl->refcount) != 0)
 322			chunk[cnt++] = epc_page;
 323		else
 324			/* The owner is freeing the page. No need to add the
 325			 * page back to the list of reclaimable pages.
 326			 */
 327			epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
 328	}
 329	spin_unlock(&sgx_reclaimer_lock);
 330
 331	for (i = 0; i < cnt; i++) {
 332		epc_page = chunk[i];
 333		encl_page = epc_page->owner;
 334
 335		if (!sgx_reclaimer_age(epc_page))
 336			goto skip;
 337
 338		page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base);
 339
 340		mutex_lock(&encl_page->encl->lock);
 341		ret = sgx_encl_alloc_backing(encl_page->encl, page_index, &backing[i]);
 342		if (ret) {
 343			mutex_unlock(&encl_page->encl->lock);
 344			goto skip;
 345		}
 346
 347		encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED;
 348		mutex_unlock(&encl_page->encl->lock);
 349		continue;
 350
 351skip:
 352		spin_lock(&sgx_reclaimer_lock);
 353		list_add_tail(&epc_page->list, &sgx_active_page_list);
 354		spin_unlock(&sgx_reclaimer_lock);
 355
 356		kref_put(&encl_page->encl->refcount, sgx_encl_release);
 357
 358		chunk[i] = NULL;
 359	}
 360
 361	for (i = 0; i < cnt; i++) {
 362		epc_page = chunk[i];
 363		if (epc_page)
 364			sgx_reclaimer_block(epc_page);
 365	}
 366
 367	for (i = 0; i < cnt; i++) {
 368		epc_page = chunk[i];
 369		if (!epc_page)
 370			continue;
 371
 372		encl_page = epc_page->owner;
 373		sgx_reclaimer_write(epc_page, &backing[i]);
 374
 375		kref_put(&encl_page->encl->refcount, sgx_encl_release);
 376		epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
 377
 378		sgx_free_epc_page(epc_page);
 379	}
 380}
 381
 382static bool sgx_should_reclaim(unsigned long watermark)
 383{
 384	return atomic_long_read(&sgx_nr_free_pages) < watermark &&
 385	       !list_empty(&sgx_active_page_list);
 386}
 387
 388/*
 389 * sgx_reclaim_direct() should be called (without enclave's mutex held)
 390 * in locations where SGX memory resources might be low and might be
 391 * needed in order to make forward progress.
 392 */
 393void sgx_reclaim_direct(void)
 394{
 395	if (sgx_should_reclaim(SGX_NR_LOW_PAGES))
 396		sgx_reclaim_pages();
 397}
 398
 399static int ksgxd(void *p)
 400{
 401	set_freezable();
 402
 403	/*
 404	 * Sanitize pages in order to recover from kexec(). The 2nd pass is
 405	 * required for SECS pages, whose child pages blocked EREMOVE.
 406	 */
 407	__sgx_sanitize_pages(&sgx_dirty_page_list);
 408	WARN_ON(__sgx_sanitize_pages(&sgx_dirty_page_list));
 409
 410	while (!kthread_should_stop()) {
 411		if (try_to_freeze())
 412			continue;
 413
 414		wait_event_freezable(ksgxd_waitq,
 415				     kthread_should_stop() ||
 416				     sgx_should_reclaim(SGX_NR_HIGH_PAGES));
 417
 418		if (sgx_should_reclaim(SGX_NR_HIGH_PAGES))
 419			sgx_reclaim_pages();
 420
 421		cond_resched();
 422	}
 423
 424	return 0;
 425}
 426
 427static bool __init sgx_page_reclaimer_init(void)
 428{
 429	struct task_struct *tsk;
 430
 431	tsk = kthread_run(ksgxd, NULL, "ksgxd");
 432	if (IS_ERR(tsk))
 433		return false;
 434
 435	ksgxd_tsk = tsk;
 436
 437	return true;
 438}
 439
 440bool current_is_ksgxd(void)
 441{
 442	return current == ksgxd_tsk;
 443}
 444
 445static struct sgx_epc_page *__sgx_alloc_epc_page_from_node(int nid)
 446{
 447	struct sgx_numa_node *node = &sgx_numa_nodes[nid];
 448	struct sgx_epc_page *page = NULL;
 449
 450	spin_lock(&node->lock);
 451
 452	if (list_empty(&node->free_page_list)) {
 453		spin_unlock(&node->lock);
 454		return NULL;
 455	}
 456
 457	page = list_first_entry(&node->free_page_list, struct sgx_epc_page, list);
 458	list_del_init(&page->list);
 459	page->flags = 0;
 460
 461	spin_unlock(&node->lock);
 462	atomic_long_dec(&sgx_nr_free_pages);
 463
 464	return page;
 465}
 466
 467/**
 468 * __sgx_alloc_epc_page() - Allocate an EPC page
 469 *
 470 * Iterate through NUMA nodes and reserve ia free EPC page to the caller. Start
 471 * from the NUMA node, where the caller is executing.
 472 *
 473 * Return:
 474 * - an EPC page:	A borrowed EPC pages were available.
 475 * - NULL:		Out of EPC pages.
 476 */
 477struct sgx_epc_page *__sgx_alloc_epc_page(void)
 478{
 479	struct sgx_epc_page *page;
 480	int nid_of_current = numa_node_id();
 481	int nid_start, nid;
 482
 483	/*
 484	 * Try local node first. If it doesn't have an EPC section,
 485	 * fall back to the non-local NUMA nodes.
 486	 */
 487	if (node_isset(nid_of_current, sgx_numa_mask))
 488		nid_start = nid_of_current;
 489	else
 490		nid_start = next_node_in(nid_of_current, sgx_numa_mask);
 491
 492	nid = nid_start;
 493	do {
 494		page = __sgx_alloc_epc_page_from_node(nid);
 495		if (page)
 496			return page;
 497
 498		nid = next_node_in(nid, sgx_numa_mask);
 499	} while (nid != nid_start);
 500
 501	return ERR_PTR(-ENOMEM);
 502}
 503
 504/**
 505 * sgx_mark_page_reclaimable() - Mark a page as reclaimable
 506 * @page:	EPC page
 507 *
 508 * Mark a page as reclaimable and add it to the active page list. Pages
 509 * are automatically removed from the active list when freed.
 510 */
 511void sgx_mark_page_reclaimable(struct sgx_epc_page *page)
 512{
 513	spin_lock(&sgx_reclaimer_lock);
 514	page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED;
 515	list_add_tail(&page->list, &sgx_active_page_list);
 516	spin_unlock(&sgx_reclaimer_lock);
 517}
 518
 519/**
 520 * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list
 521 * @page:	EPC page
 522 *
 523 * Clear the reclaimable flag and remove the page from the active page list.
 524 *
 525 * Return:
 526 *   0 on success,
 527 *   -EBUSY if the page is in the process of being reclaimed
 528 */
 529int sgx_unmark_page_reclaimable(struct sgx_epc_page *page)
 530{
 531	spin_lock(&sgx_reclaimer_lock);
 532	if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) {
 533		/* The page is being reclaimed. */
 534		if (list_empty(&page->list)) {
 535			spin_unlock(&sgx_reclaimer_lock);
 536			return -EBUSY;
 537		}
 538
 539		list_del(&page->list);
 540		page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED;
 541	}
 542	spin_unlock(&sgx_reclaimer_lock);
 543
 544	return 0;
 545}
 546
 547/**
 548 * sgx_alloc_epc_page() - Allocate an EPC page
 549 * @owner:	the owner of the EPC page
 550 * @reclaim:	reclaim pages if necessary
 551 *
 552 * Iterate through EPC sections and borrow a free EPC page to the caller. When a
 553 * page is no longer needed it must be released with sgx_free_epc_page(). If
 554 * @reclaim is set to true, directly reclaim pages when we are out of pages. No
 555 * mm's can be locked when @reclaim is set to true.
 556 *
 557 * Finally, wake up ksgxd when the number of pages goes below the watermark
 558 * before returning back to the caller.
 559 *
 560 * Return:
 561 *   an EPC page,
 562 *   -errno on error
 563 */
 564struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim)
 565{
 566	struct sgx_epc_page *page;
 567
 568	for ( ; ; ) {
 569		page = __sgx_alloc_epc_page();
 570		if (!IS_ERR(page)) {
 571			page->owner = owner;
 572			break;
 573		}
 574
 575		if (list_empty(&sgx_active_page_list))
 576			return ERR_PTR(-ENOMEM);
 577
 578		if (!reclaim) {
 579			page = ERR_PTR(-EBUSY);
 580			break;
 581		}
 582
 583		if (signal_pending(current)) {
 584			page = ERR_PTR(-ERESTARTSYS);
 585			break;
 586		}
 587
 588		sgx_reclaim_pages();
 589		cond_resched();
 590	}
 591
 592	if (sgx_should_reclaim(SGX_NR_LOW_PAGES))
 593		wake_up(&ksgxd_waitq);
 594
 595	return page;
 596}
 597
 598/**
 599 * sgx_free_epc_page() - Free an EPC page
 600 * @page:	an EPC page
 601 *
 602 * Put the EPC page back to the list of free pages. It's the caller's
 603 * responsibility to make sure that the page is in uninitialized state. In other
 604 * words, do EREMOVE, EWB or whatever operation is necessary before calling
 605 * this function.
 606 */
 607void sgx_free_epc_page(struct sgx_epc_page *page)
 608{
 609	struct sgx_epc_section *section = &sgx_epc_sections[page->section];
 610	struct sgx_numa_node *node = section->node;
 611
 612	spin_lock(&node->lock);
 613
 614	page->owner = NULL;
 615	if (page->poison)
 616		list_add(&page->list, &node->sgx_poison_page_list);
 617	else
 618		list_add_tail(&page->list, &node->free_page_list);
 619	page->flags = SGX_EPC_PAGE_IS_FREE;
 620
 621	spin_unlock(&node->lock);
 622	atomic_long_inc(&sgx_nr_free_pages);
 623}
 624
 625static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size,
 626					 unsigned long index,
 627					 struct sgx_epc_section *section)
 628{
 629	unsigned long nr_pages = size >> PAGE_SHIFT;
 630	unsigned long i;
 631
 632	section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB);
 633	if (!section->virt_addr)
 634		return false;
 635
 636	section->pages = vmalloc_array(nr_pages, sizeof(struct sgx_epc_page));
 637	if (!section->pages) {
 638		memunmap(section->virt_addr);
 639		return false;
 640	}
 641
 642	section->phys_addr = phys_addr;
 643	xa_store_range(&sgx_epc_address_space, section->phys_addr,
 644		       phys_addr + size - 1, section, GFP_KERNEL);
 645
 646	for (i = 0; i < nr_pages; i++) {
 647		section->pages[i].section = index;
 648		section->pages[i].flags = 0;
 649		section->pages[i].owner = NULL;
 650		section->pages[i].poison = 0;
 651		list_add_tail(&section->pages[i].list, &sgx_dirty_page_list);
 652	}
 653
 654	return true;
 655}
 656
 657bool arch_is_platform_page(u64 paddr)
 658{
 659	return !!xa_load(&sgx_epc_address_space, paddr);
 660}
 661EXPORT_SYMBOL_GPL(arch_is_platform_page);
 662
 663static struct sgx_epc_page *sgx_paddr_to_page(u64 paddr)
 664{
 665	struct sgx_epc_section *section;
 666
 667	section = xa_load(&sgx_epc_address_space, paddr);
 668	if (!section)
 669		return NULL;
 670
 671	return &section->pages[PFN_DOWN(paddr - section->phys_addr)];
 672}
 673
 674/*
 675 * Called in process context to handle a hardware reported
 676 * error in an SGX EPC page.
 677 * If the MF_ACTION_REQUIRED bit is set in flags, then the
 678 * context is the task that consumed the poison data. Otherwise
 679 * this is called from a kernel thread unrelated to the page.
 680 */
 681int arch_memory_failure(unsigned long pfn, int flags)
 682{
 683	struct sgx_epc_page *page = sgx_paddr_to_page(pfn << PAGE_SHIFT);
 684	struct sgx_epc_section *section;
 685	struct sgx_numa_node *node;
 686
 687	/*
 688	 * mm/memory-failure.c calls this routine for all errors
 689	 * where there isn't a "struct page" for the address. But that
 690	 * includes other address ranges besides SGX.
 691	 */
 692	if (!page)
 693		return -ENXIO;
 694
 695	/*
 696	 * If poison was consumed synchronously. Send a SIGBUS to
 697	 * the task. Hardware has already exited the SGX enclave and
 698	 * will not allow re-entry to an enclave that has a memory
 699	 * error. The signal may help the task understand why the
 700	 * enclave is broken.
 701	 */
 702	if (flags & MF_ACTION_REQUIRED)
 703		force_sig(SIGBUS);
 704
 705	section = &sgx_epc_sections[page->section];
 706	node = section->node;
 707
 708	spin_lock(&node->lock);
 709
 710	/* Already poisoned? Nothing more to do */
 711	if (page->poison)
 712		goto out;
 713
 714	page->poison = 1;
 715
 716	/*
 717	 * If the page is on a free list, move it to the per-node
 718	 * poison page list.
 719	 */
 720	if (page->flags & SGX_EPC_PAGE_IS_FREE) {
 721		list_move(&page->list, &node->sgx_poison_page_list);
 722		goto out;
 723	}
 724
 725	sgx_unmark_page_reclaimable(page);
 726
 727	/*
 728	 * TBD: Add additional plumbing to enable pre-emptive
 729	 * action for asynchronous poison notification. Until
 730	 * then just hope that the poison:
 731	 * a) is not accessed - sgx_free_epc_page() will deal with it
 732	 *    when the user gives it back
 733	 * b) results in a recoverable machine check rather than
 734	 *    a fatal one
 735	 */
 736out:
 737	spin_unlock(&node->lock);
 738	return 0;
 739}
 740
 741/*
 742 * A section metric is concatenated in a way that @low bits 12-31 define the
 743 * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the
 744 * metric.
 745 */
 746static inline u64 __init sgx_calc_section_metric(u64 low, u64 high)
 747{
 748	return (low & GENMASK_ULL(31, 12)) +
 749	       ((high & GENMASK_ULL(19, 0)) << 32);
 750}
 751
 752#ifdef CONFIG_NUMA
 753static ssize_t sgx_total_bytes_show(struct device *dev, struct device_attribute *attr, char *buf)
 754{
 755	return sysfs_emit(buf, "%lu\n", sgx_numa_nodes[dev->id].size);
 756}
 757static DEVICE_ATTR_RO(sgx_total_bytes);
 758
 759static umode_t arch_node_attr_is_visible(struct kobject *kobj,
 760		struct attribute *attr, int idx)
 761{
 762	/* Make all x86/ attributes invisible when SGX is not initialized: */
 763	if (nodes_empty(sgx_numa_mask))
 764		return 0;
 765
 766	return attr->mode;
 767}
 768
 769static struct attribute *arch_node_dev_attrs[] = {
 770	&dev_attr_sgx_total_bytes.attr,
 771	NULL,
 772};
 773
 774const struct attribute_group arch_node_dev_group = {
 775	.name = "x86",
 776	.attrs = arch_node_dev_attrs,
 777	.is_visible = arch_node_attr_is_visible,
 778};
 779
 780static void __init arch_update_sysfs_visibility(int nid)
 781{
 782	struct node *node = node_devices[nid];
 783	int ret;
 784
 785	ret = sysfs_update_group(&node->dev.kobj, &arch_node_dev_group);
 786
 787	if (ret)
 788		pr_err("sysfs update failed (%d), files may be invisible", ret);
 789}
 790#else /* !CONFIG_NUMA */
 791static void __init arch_update_sysfs_visibility(int nid) {}
 792#endif
 793
 794static bool __init sgx_page_cache_init(void)
 795{
 796	u32 eax, ebx, ecx, edx, type;
 797	u64 pa, size;
 798	int nid;
 799	int i;
 800
 801	sgx_numa_nodes = kmalloc_array(num_possible_nodes(), sizeof(*sgx_numa_nodes), GFP_KERNEL);
 802	if (!sgx_numa_nodes)
 803		return false;
 804
 805	for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) {
 806		cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx);
 807
 808		type = eax & SGX_CPUID_EPC_MASK;
 809		if (type == SGX_CPUID_EPC_INVALID)
 810			break;
 811
 812		if (type != SGX_CPUID_EPC_SECTION) {
 813			pr_err_once("Unknown EPC section type: %u\n", type);
 814			break;
 815		}
 816
 817		pa   = sgx_calc_section_metric(eax, ebx);
 818		size = sgx_calc_section_metric(ecx, edx);
 819
 820		pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1);
 821
 822		if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) {
 823			pr_err("No free memory for an EPC section\n");
 824			break;
 825		}
 826
 827		nid = numa_map_to_online_node(phys_to_target_node(pa));
 828		if (nid == NUMA_NO_NODE) {
 829			/* The physical address is already printed above. */
 830			pr_warn(FW_BUG "Unable to map EPC section to online node. Fallback to the NUMA node 0.\n");
 831			nid = 0;
 832		}
 833
 834		if (!node_isset(nid, sgx_numa_mask)) {
 835			spin_lock_init(&sgx_numa_nodes[nid].lock);
 836			INIT_LIST_HEAD(&sgx_numa_nodes[nid].free_page_list);
 837			INIT_LIST_HEAD(&sgx_numa_nodes[nid].sgx_poison_page_list);
 838			node_set(nid, sgx_numa_mask);
 839			sgx_numa_nodes[nid].size = 0;
 840
 841			/* Make SGX-specific node sysfs files visible: */
 842			arch_update_sysfs_visibility(nid);
 843		}
 844
 845		sgx_epc_sections[i].node =  &sgx_numa_nodes[nid];
 846		sgx_numa_nodes[nid].size += size;
 847
 848		sgx_nr_epc_sections++;
 849	}
 850
 851	if (!sgx_nr_epc_sections) {
 852		pr_err("There are zero EPC sections.\n");
 853		return false;
 854	}
 855
 856	for_each_online_node(nid) {
 857		if (!node_isset(nid, sgx_numa_mask) &&
 858		    node_state(nid, N_MEMORY) && node_state(nid, N_CPU))
 859			pr_info("node%d has both CPUs and memory but doesn't have an EPC section\n",
 860				nid);
 861	}
 862
 863	return true;
 864}
 865
 866/*
 867 * Update the SGX_LEPUBKEYHASH MSRs to the values specified by caller.
 868 * Bare-metal driver requires to update them to hash of enclave's signer
 869 * before EINIT. KVM needs to update them to guest's virtual MSR values
 870 * before doing EINIT from guest.
 871 */
 872void sgx_update_lepubkeyhash(u64 *lepubkeyhash)
 873{
 874	int i;
 875
 876	WARN_ON_ONCE(preemptible());
 877
 878	for (i = 0; i < 4; i++)
 879		wrmsrq(MSR_IA32_SGXLEPUBKEYHASH0 + i, lepubkeyhash[i]);
 880}
 881
 882const struct file_operations sgx_provision_fops = {
 883	.owner			= THIS_MODULE,
 884};
 885
 886static struct miscdevice sgx_dev_provision = {
 887	.minor = MISC_DYNAMIC_MINOR,
 888	.name = "sgx_provision",
 889	.nodename = "sgx_provision",
 890	.fops = &sgx_provision_fops,
 891};
 892
 893/**
 894 * sgx_set_attribute() - Update allowed attributes given file descriptor
 895 * @allowed_attributes:		Pointer to allowed enclave attributes
 896 * @attribute_fd:		File descriptor for specific attribute
 897 *
 898 * Append enclave attribute indicated by file descriptor to allowed
 899 * attributes. Currently only SGX_ATTR_PROVISIONKEY indicated by
 900 * /dev/sgx_provision is supported.
 901 *
 902 * Return:
 903 * -0:		SGX_ATTR_PROVISIONKEY is appended to allowed_attributes
 904 * -EINVAL:	Invalid, or not supported file descriptor
 905 */
 906int sgx_set_attribute(unsigned long *allowed_attributes,
 907		      unsigned int attribute_fd)
 908{
 909	CLASS(fd, f)(attribute_fd);
 910
 911	if (fd_empty(f))
 912		return -EINVAL;
 913
 914	if (fd_file(f)->f_op != &sgx_provision_fops)
 915		return -EINVAL;
 916
 917	*allowed_attributes |= SGX_ATTR_PROVISIONKEY;
 918	return 0;
 919}
 920EXPORT_SYMBOL_FOR_KVM(sgx_set_attribute);
 921
 922/* Counter to count the active SGX users */
 923static int sgx_usage_count;
 924
 925/**
 926 * sgx_update_svn() - Attempt to call ENCLS[EUPDATESVN].
 927 *
 928 * This instruction attempts to update CPUSVN to the
 929 * currently loaded microcode update SVN and generate new
 930 * cryptographic assets.
 931 *
 932 * Return:
 933 * * %0:       - Success or not supported
 934 * * %-EAGAIN: - Can be safely retried, failure is due to lack of
 935 * *             entropy in RNG
 936 * * %-EIO:    - Unexpected error, retries are not advisable
 937 */
 938static int sgx_update_svn(void)
 939{
 940	int ret;
 941
 942	/*
 943	 * If EUPDATESVN is not available, it is ok to
 944	 * silently skip it to comply with legacy behavior.
 945	 */
 946	if (!cpu_feature_enabled(X86_FEATURE_SGX_EUPDATESVN))
 947		return 0;
 948
 949	/*
 950	 * EPC is guaranteed to be empty when there are no users.
 951	 * Ensure we are on our first user before proceeding further.
 952	 */
 953	WARN(sgx_usage_count, "Elevated usage count when calling EUPDATESVN\n");
 954
 955	for (int i = 0; i < RDRAND_RETRY_LOOPS; i++) {
 956		ret = __eupdatesvn();
 957
 958		/* Stop on success or unexpected errors: */
 959		if (ret != SGX_INSUFFICIENT_ENTROPY)
 960			break;
 961	}
 962
 963	switch (ret) {
 964	case 0:
 965		/*
 966		 * SVN successfully updated.
 967		 * Let users know when the update was successful.
 968		 */
 969		pr_info("SVN updated successfully\n");
 970		return 0;
 971	case SGX_NO_UPDATE:
 972		/*
 973		 * SVN update failed since the current SVN is
 974		 * not newer than CPUSVN. This is the most
 975		 * common case and indicates no harm.
 976		 */
 977		return 0;
 978	case SGX_INSUFFICIENT_ENTROPY:
 979		/*
 980		 * SVN update failed due to lack of entropy in DRNG.
 981		 * Indicate to userspace that it should retry.
 982		 */
 983		return -EAGAIN;
 984	default:
 985		break;
 986	}
 987
 988	/*
 989	 * EUPDATESVN was called when EPC is empty, all other error
 990	 * codes are unexpected.
 991	 */
 992	ENCLS_WARN(ret, "EUPDATESVN");
 993	return -EIO;
 994}
 995
 996/* Mutex to ensure no concurrent EPC accesses during EUPDATESVN */
 997static DEFINE_MUTEX(sgx_svn_lock);
 998
 999int sgx_inc_usage_count(void)
1000{
1001	int ret;
1002
1003	guard(mutex)(&sgx_svn_lock);
1004
1005	if (!sgx_usage_count) {
1006		ret = sgx_update_svn();
1007		if (ret)
1008			return ret;
1009	}
1010
1011	sgx_usage_count++;
1012
1013	return 0;
1014}
1015
1016void sgx_dec_usage_count(void)
1017{
1018	guard(mutex)(&sgx_svn_lock);
1019	sgx_usage_count--;
1020}
1021
1022static int __init sgx_init(void)
1023{
1024	int ret;
1025	int i;
1026
1027	if (!cpu_feature_enabled(X86_FEATURE_SGX))
1028		return -ENODEV;
1029
1030	if (!sgx_page_cache_init())
1031		return -ENOMEM;
1032
1033	if (!sgx_page_reclaimer_init()) {
1034		ret = -ENOMEM;
1035		goto err_page_cache;
1036	}
1037
1038	ret = misc_register(&sgx_dev_provision);
1039	if (ret)
1040		goto err_kthread;
1041
1042	/*
1043	 * Always try to initialize the native *and* KVM drivers.
1044	 * The KVM driver is less picky than the native one and
1045	 * can function if the native one is not supported on the
1046	 * current system or fails to initialize.
1047	 *
1048	 * Error out only if both fail to initialize.
1049	 */
1050	ret = sgx_drv_init();
1051
1052	if (sgx_vepc_init() && ret)
1053		goto err_provision;
1054
1055	return 0;
1056
1057err_provision:
1058	misc_deregister(&sgx_dev_provision);
1059
1060err_kthread:
1061	kthread_stop(ksgxd_tsk);
1062
1063err_page_cache:
1064	for (i = 0; i < sgx_nr_epc_sections; i++) {
1065		vfree(sgx_epc_sections[i].pages);
1066		memunmap(sgx_epc_sections[i].virt_addr);
1067	}
1068
1069	return ret;
1070}
1071
1072device_initcall(sgx_init);