efi.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Common EFI (Extensible Firmware Interface) support functions
  4 * Based on Extensible Firmware Interface Specification version 1.0
  5 *
  6 * Copyright (C) 1999 VA Linux Systems
  7 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
  8 * Copyright (C) 1999-2002 Hewlett-Packard Co.
  9 *	David Mosberger-Tang <davidm@hpl.hp.com>
 10 *	Stephane Eranian <eranian@hpl.hp.com>
 11 * Copyright (C) 2005-2008 Intel Co.
 12 *	Fenghua Yu <fenghua.yu@intel.com>
 13 *	Bibo Mao <bibo.mao@intel.com>
 14 *	Chandramouli Narayanan <mouli@linux.intel.com>
 15 *	Huang Ying <ying.huang@intel.com>
 16 * Copyright (C) 2013 SuSE Labs
 17 *	Borislav Petkov <bp@suse.de> - runtime services VA mapping
 18 *
 19 * Copied from efi_32.c to eliminate the duplicated code between EFI
 20 * 32/64 support code. --ying 2007-10-26
 21 *
 22 * All EFI Runtime Services are not implemented yet as EFI only
 23 * supports physical mode addressing on SoftSDV. This is to be fixed
 24 * in a future version.  --drummond 1999-07-20
 25 *
 26 * Implemented EFI runtime services and virtual mode calls.  --davidm
 27 *
 28 * Goutham Rao: <goutham.rao@intel.com>
 29 *	Skip non-WB memory and ignore empty memory ranges.
 30 */
 31
 32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 33
 34#include <linux/kernel.h>
 35#include <linux/init.h>
 36#include <linux/efi.h>
 37#include <linux/efi-bgrt.h>
 38#include <linux/export.h>
 39#include <linux/memblock.h>
 40#include <linux/slab.h>
 41#include <linux/spinlock.h>
 42#include <linux/uaccess.h>
 43#include <linux/time.h>
 44#include <linux/io.h>
 45#include <linux/reboot.h>
 46#include <linux/bcd.h>
 47
 48#include <asm/setup.h>
 49#include <asm/efi.h>
 50#include <asm/e820/api.h>
 51#include <asm/time.h>
 52#include <asm/tlbflush.h>
 53#include <asm/x86_init.h>
 54#include <asm/uv/uv.h>
 55
 56static unsigned long efi_systab_phys __initdata;
 57static unsigned long efi_runtime, efi_nr_tables;
 58
 59unsigned long efi_fw_vendor, efi_config_table;
 60
 61static const efi_config_table_type_t arch_tables[] __initconst = {
 62#ifdef CONFIG_X86_UV
 63	{UV_SYSTEM_TABLE_GUID,		&uv_systab_phys,	"UVsystab"	},
 64#endif
 65	{},
 66};
 67
 68static const unsigned long * const efi_tables[] = {
 69	&efi.acpi,
 70	&efi.acpi20,
 71	&efi.smbios,
 72	&efi.smbios3,
 73#ifdef CONFIG_X86_UV
 74	&uv_systab_phys,
 75#endif
 76	&efi_fw_vendor,
 77	&efi_runtime,
 78	&efi_config_table,
 79	&efi.esrt,
 80	&efi_mem_attr_table,
 81#ifdef CONFIG_EFI_RCI2_TABLE
 82	&rci2_table_phys,
 83#endif
 84	&efi.tpm_log,
 85	&efi.tpm_final_log,
 86	&efi_rng_seed,
 87#ifdef CONFIG_LOAD_UEFI_KEYS
 88	&efi.mokvar_table,
 89#endif
 90#ifdef CONFIG_EFI_COCO_SECRET
 91	&efi.coco_secret,
 92#endif
 93#ifdef CONFIG_UNACCEPTED_MEMORY
 94	&efi.unaccepted,
 95#endif
 96};
 97
 98u64 efi_setup;		/* efi setup_data physical address */
 99
100static int add_efi_memmap __initdata;
101static int __init setup_add_efi_memmap(char *arg)
102{
103	add_efi_memmap = 1;
104	return 0;
105}
106early_param("add_efi_memmap", setup_add_efi_memmap);
107
108/*
109 * Tell the kernel about the EFI memory map.  This might include
110 * more than the max 128 entries that can fit in the passed in e820
111 * legacy (zeropage) memory map, but the kernel's e820 table can hold
112 * E820_MAX_ENTRIES.
113 */
114
115static void __init do_add_efi_memmap(void)
116{
117	efi_memory_desc_t *md;
118
119	if (!efi_enabled(EFI_MEMMAP))
120		return;
121
122	for_each_efi_memory_desc(md) {
123		unsigned long long start = md->phys_addr;
124		unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
125		int e820_type;
126
127		switch (md->type) {
128		case EFI_LOADER_CODE:
129		case EFI_LOADER_DATA:
130		case EFI_BOOT_SERVICES_CODE:
131		case EFI_BOOT_SERVICES_DATA:
132		case EFI_CONVENTIONAL_MEMORY:
133			if (efi_soft_reserve_enabled()
134			    && (md->attribute & EFI_MEMORY_SP))
135				e820_type = E820_TYPE_SOFT_RESERVED;
136			else if (md->attribute & EFI_MEMORY_WB)
137				e820_type = E820_TYPE_RAM;
138			else
139				e820_type = E820_TYPE_RESERVED;
140			break;
141		case EFI_ACPI_RECLAIM_MEMORY:
142			e820_type = E820_TYPE_ACPI;
143			break;
144		case EFI_ACPI_MEMORY_NVS:
145			e820_type = E820_TYPE_NVS;
146			break;
147		case EFI_UNUSABLE_MEMORY:
148			e820_type = E820_TYPE_UNUSABLE;
149			break;
150		case EFI_PERSISTENT_MEMORY:
151			e820_type = E820_TYPE_PMEM;
152			break;
153		default:
154			/*
155			 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
156			 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
157			 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
158			 */
159			e820_type = E820_TYPE_RESERVED;
160			break;
161		}
162
163		e820__range_add(start, size, e820_type);
164	}
165	e820__update_table(e820_table);
166}
167
168/*
169 * Given add_efi_memmap defaults to 0 and there is no alternative
170 * e820 mechanism for soft-reserved memory, import the full EFI memory
171 * map if soft reservations are present and enabled. Otherwise, the
172 * mechanism to disable the kernel's consideration of EFI_MEMORY_SP is
173 * the efi=nosoftreserve option.
174 */
175static bool do_efi_soft_reserve(void)
176{
177	efi_memory_desc_t *md;
178
179	if (!efi_enabled(EFI_MEMMAP))
180		return false;
181
182	if (!efi_soft_reserve_enabled())
183		return false;
184
185	for_each_efi_memory_desc(md)
186		if (md->type == EFI_CONVENTIONAL_MEMORY &&
187		    (md->attribute & EFI_MEMORY_SP))
188			return true;
189	return false;
190}
191
192int __init efi_memblock_x86_reserve_range(void)
193{
194	struct efi_info *e = &boot_params.efi_info;
195	struct efi_memory_map_data data;
196	phys_addr_t pmap;
197	int rv;
198
199	if (efi_enabled(EFI_PARAVIRT))
200		return 0;
201
202	/* Can't handle firmware tables above 4GB on i386 */
203	if (IS_ENABLED(CONFIG_X86_32) && e->efi_memmap_hi > 0) {
204		pr_err("Memory map is above 4GB, disabling EFI.\n");
205		return -EINVAL;
206	}
207	pmap = (phys_addr_t)(e->efi_memmap | ((u64)e->efi_memmap_hi << 32));
208
209	data.phys_map		= pmap;
210	data.size 		= e->efi_memmap_size;
211	data.desc_size		= e->efi_memdesc_size;
212	data.desc_version	= e->efi_memdesc_version;
213
214	if (!efi_enabled(EFI_PARAVIRT)) {
215		rv = efi_memmap_init_early(&data);
216		if (rv)
217			return rv;
218	}
219
220	if (add_efi_memmap || do_efi_soft_reserve())
221		do_add_efi_memmap();
222
223	WARN(efi.memmap.desc_version != 1,
224	     "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
225	     efi.memmap.desc_version);
226
227	memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
228	set_bit(EFI_PRESERVE_BS_REGIONS, &efi.flags);
229
230	return 0;
231}
232
233#define OVERFLOW_ADDR_SHIFT	(64 - EFI_PAGE_SHIFT)
234#define OVERFLOW_ADDR_MASK	(U64_MAX << OVERFLOW_ADDR_SHIFT)
235#define U64_HIGH_BIT		(~(U64_MAX >> 1))
236
237static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
238{
239	u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
240	u64 end_hi = 0;
241	char buf[64];
242
243	if (md->num_pages == 0) {
244		end = 0;
245	} else if (md->num_pages > EFI_PAGES_MAX ||
246		   EFI_PAGES_MAX - md->num_pages <
247		   (md->phys_addr >> EFI_PAGE_SHIFT)) {
248		end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
249			>> OVERFLOW_ADDR_SHIFT;
250
251		if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
252			end_hi += 1;
253	} else {
254		return true;
255	}
256
257	pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
258
259	if (end_hi) {
260		pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
261			i, efi_md_typeattr_format(buf, sizeof(buf), md),
262			md->phys_addr, end_hi, end);
263	} else {
264		pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
265			i, efi_md_typeattr_format(buf, sizeof(buf), md),
266			md->phys_addr, end);
267	}
268	return false;
269}
270
271static void __init efi_clean_memmap(void)
272{
273	efi_memory_desc_t *out = efi.memmap.map;
274	const efi_memory_desc_t *in = out;
275	const efi_memory_desc_t *end = efi.memmap.map_end;
276	int i, n_removal;
277
278	for (i = n_removal = 0; in < end; i++) {
279		if (efi_memmap_entry_valid(in, i)) {
280			if (out != in)
281				memcpy(out, in, efi.memmap.desc_size);
282			out = (void *)out + efi.memmap.desc_size;
283		} else {
284			n_removal++;
285		}
286		in = (void *)in + efi.memmap.desc_size;
287	}
288
289	if (n_removal > 0) {
290		struct efi_memory_map_data data = {
291			.phys_map	= efi.memmap.phys_map,
292			.desc_version	= efi.memmap.desc_version,
293			.desc_size	= efi.memmap.desc_size,
294			.size		= efi.memmap.desc_size * (efi.memmap.nr_map - n_removal),
295			.flags		= 0,
296		};
297
298		pr_warn("Removing %d invalid memory map entries.\n", n_removal);
299		efi_memmap_install(&data);
300	}
301}
302
303/*
304 * Firmware can use EfiMemoryMappedIO to request that MMIO regions be
305 * mapped by the OS so they can be accessed by EFI runtime services, but
306 * should have no other significance to the OS (UEFI r2.10, sec 7.2).
307 * However, most bootloaders and EFI stubs convert EfiMemoryMappedIO
308 * regions to E820_TYPE_RESERVED entries, which prevent Linux from
309 * allocating space from them (see remove_e820_regions()).
310 *
311 * Some platforms use EfiMemoryMappedIO entries for PCI MMCONFIG space and
312 * PCI host bridge windows, which means Linux can't allocate BAR space for
313 * hot-added devices.
314 *
315 * Remove large EfiMemoryMappedIO regions from the E820 map to avoid this
316 * problem.
317 *
318 * Retain small EfiMemoryMappedIO regions because on some platforms, these
319 * describe non-window space that's included in host bridge _CRS.  If we
320 * assign that space to PCI devices, they don't work.
321 */
322static void __init efi_remove_e820_mmio(void)
323{
324	efi_memory_desc_t *md;
325	u64 size, start, end;
326	int i = 0;
327
328	for_each_efi_memory_desc(md) {
329		if (md->type == EFI_MEMORY_MAPPED_IO) {
330			size = md->num_pages << EFI_PAGE_SHIFT;
331			start = md->phys_addr;
332			end = start + size - 1;
333			if (size >= 256*1024) {
334				pr_info("Remove mem%02u: MMIO range=[0x%08llx-0x%08llx] (%lluMB) from e820 map\n",
335					i, start, end, size >> 20);
336				e820__range_remove(start, size,
337						   E820_TYPE_RESERVED, 1);
338			} else {
339				pr_info("Not removing mem%02u: MMIO range=[0x%08llx-0x%08llx] (%lluKB) from e820 map\n",
340					i, start, end, size >> 10);
341			}
342		}
343		i++;
344	}
345}
346
347void __init efi_print_memmap(void)
348{
349	efi_memory_desc_t *md;
350	int i = 0;
351
352	for_each_efi_memory_desc(md) {
353		char buf[64];
354
355		pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
356			i++, efi_md_typeattr_format(buf, sizeof(buf), md),
357			md->phys_addr,
358			md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
359			(md->num_pages >> (20 - EFI_PAGE_SHIFT)));
360	}
361}
362
363static int __init efi_systab_init(unsigned long phys)
364{
365	int size = efi_enabled(EFI_64BIT) ? sizeof(efi_system_table_64_t)
366					  : sizeof(efi_system_table_32_t);
367	const efi_table_hdr_t *hdr;
368	bool over4g = false;
369	void *p;
370	int ret;
371
372	hdr = p = early_memremap_ro(phys, size);
373	if (p == NULL) {
374		pr_err("Couldn't map the system table!\n");
375		return -ENOMEM;
376	}
377
378	ret = efi_systab_check_header(hdr);
379	if (ret) {
380		early_memunmap(p, size);
381		return ret;
382	}
383
384	if (efi_enabled(EFI_64BIT)) {
385		const efi_system_table_64_t *systab64 = p;
386
387		efi_runtime	= systab64->runtime;
388		over4g		= systab64->runtime > U32_MAX;
389
390		if (efi_setup) {
391			struct efi_setup_data *data;
392
393			data = early_memremap_ro(efi_setup, sizeof(*data));
394			if (!data) {
395				early_memunmap(p, size);
396				return -ENOMEM;
397			}
398
399			efi_fw_vendor		= (unsigned long)data->fw_vendor;
400			efi_config_table	= (unsigned long)data->tables;
401
402			over4g |= data->fw_vendor	> U32_MAX ||
403				  data->tables		> U32_MAX;
404
405			early_memunmap(data, sizeof(*data));
406		} else {
407			efi_fw_vendor		= systab64->fw_vendor;
408			efi_config_table	= systab64->tables;
409
410			over4g |= systab64->fw_vendor	> U32_MAX ||
411				  systab64->tables	> U32_MAX;
412		}
413		efi_nr_tables = systab64->nr_tables;
414	} else {
415		const efi_system_table_32_t *systab32 = p;
416
417		efi_fw_vendor		= systab32->fw_vendor;
418		efi_runtime		= systab32->runtime;
419		efi_config_table	= systab32->tables;
420		efi_nr_tables		= systab32->nr_tables;
421	}
422
423	efi.runtime_version = hdr->revision;
424
425	efi_systab_report_header(hdr, efi_fw_vendor);
426	early_memunmap(p, size);
427
428	if (IS_ENABLED(CONFIG_X86_32) && over4g) {
429		pr_err("EFI data located above 4GB, disabling EFI.\n");
430		return -EINVAL;
431	}
432
433	return 0;
434}
435
436static int __init efi_config_init(const efi_config_table_type_t *arch_tables)
437{
438	void *config_tables;
439	int sz, ret;
440
441	if (efi_nr_tables == 0)
442		return 0;
443
444	if (efi_enabled(EFI_64BIT))
445		sz = sizeof(efi_config_table_64_t);
446	else
447		sz = sizeof(efi_config_table_32_t);
448
449	/*
450	 * Let's see what config tables the firmware passed to us.
451	 */
452	config_tables = early_memremap(efi_config_table, efi_nr_tables * sz);
453	if (config_tables == NULL) {
454		pr_err("Could not map Configuration table!\n");
455		return -ENOMEM;
456	}
457
458	ret = efi_config_parse_tables(config_tables, efi_nr_tables,
459				      arch_tables);
460
461	early_memunmap(config_tables, efi_nr_tables * sz);
462	return ret;
463}
464
465void __init efi_init(void)
466{
467	if (IS_ENABLED(CONFIG_X86_32) &&
468	    (boot_params.efi_info.efi_systab_hi ||
469	     boot_params.efi_info.efi_memmap_hi)) {
470		pr_info("Table located above 4GB, disabling EFI.\n");
471		return;
472	}
473
474	efi_systab_phys = boot_params.efi_info.efi_systab |
475			  ((__u64)boot_params.efi_info.efi_systab_hi << 32);
476
477	if (efi_systab_init(efi_systab_phys))
478		return;
479
480	if (efi_reuse_config(efi_config_table, efi_nr_tables))
481		return;
482
483	if (efi_config_init(arch_tables))
484		return;
485
486	/*
487	 * Note: We currently don't support runtime services on an EFI
488	 * that doesn't match the kernel 32/64-bit mode.
489	 */
490
491	if (!efi_runtime_supported())
492		pr_err("No EFI runtime due to 32/64-bit mismatch with kernel\n");
493
494	if (!efi_runtime_supported() || efi_runtime_disabled()) {
495		efi_memmap_unmap();
496		return;
497	}
498
499	set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
500	efi_clean_memmap();
501
502	efi_remove_e820_mmio();
503
504	if (efi_enabled(EFI_DBG))
505		efi_print_memmap();
506}
507
508/* Merge contiguous regions of the same type and attribute */
509static void __init efi_merge_regions(void)
510{
511	efi_memory_desc_t *md, *prev_md = NULL;
512
513	for_each_efi_memory_desc(md) {
514		u64 prev_size;
515
516		if (!prev_md) {
517			prev_md = md;
518			continue;
519		}
520
521		if (prev_md->type != md->type ||
522		    prev_md->attribute != md->attribute) {
523			prev_md = md;
524			continue;
525		}
526
527		prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
528
529		if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
530			prev_md->num_pages += md->num_pages;
531			md->type = EFI_RESERVED_TYPE;
532			md->attribute = 0;
533			continue;
534		}
535		prev_md = md;
536	}
537}
538
539static void *realloc_pages(void *old_memmap, int old_shift)
540{
541	void *ret;
542
543	ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
544	if (!ret)
545		goto out;
546
547	/*
548	 * A first-time allocation doesn't have anything to copy.
549	 */
550	if (!old_memmap)
551		return ret;
552
553	memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
554
555out:
556	free_pages((unsigned long)old_memmap, old_shift);
557	return ret;
558}
559
560/*
561 * Iterate the EFI memory map in reverse order because the regions
562 * will be mapped top-down. The end result is the same as if we had
563 * mapped things forward, but doesn't require us to change the
564 * existing implementation of efi_map_region().
565 */
566static inline void *efi_map_next_entry_reverse(void *entry)
567{
568	/* Initial call */
569	if (!entry)
570		return efi.memmap.map_end - efi.memmap.desc_size;
571
572	entry -= efi.memmap.desc_size;
573	if (entry < efi.memmap.map)
574		return NULL;
575
576	return entry;
577}
578
579/*
580 * efi_map_next_entry - Return the next EFI memory map descriptor
581 * @entry: Previous EFI memory map descriptor
582 *
583 * This is a helper function to iterate over the EFI memory map, which
584 * we do in different orders depending on the current configuration.
585 *
586 * To begin traversing the memory map @entry must be %NULL.
587 *
588 * Returns %NULL when we reach the end of the memory map.
589 */
590static void *efi_map_next_entry(void *entry)
591{
592	if (efi_enabled(EFI_64BIT)) {
593		/*
594		 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
595		 * config table feature requires us to map all entries
596		 * in the same order as they appear in the EFI memory
597		 * map. That is to say, entry N must have a lower
598		 * virtual address than entry N+1. This is because the
599		 * firmware toolchain leaves relative references in
600		 * the code/data sections, which are split and become
601		 * separate EFI memory regions. Mapping things
602		 * out-of-order leads to the firmware accessing
603		 * unmapped addresses.
604		 *
605		 * Since we need to map things this way whether or not
606		 * the kernel actually makes use of
607		 * EFI_PROPERTIES_TABLE, let's just switch to this
608		 * scheme by default for 64-bit.
609		 */
610		return efi_map_next_entry_reverse(entry);
611	}
612
613	/* Initial call */
614	if (!entry)
615		return efi.memmap.map;
616
617	entry += efi.memmap.desc_size;
618	if (entry >= efi.memmap.map_end)
619		return NULL;
620
621	return entry;
622}
623
624static bool should_map_region(efi_memory_desc_t *md)
625{
626	/*
627	 * Runtime regions always require runtime mappings (obviously).
628	 */
629	if (md->attribute & EFI_MEMORY_RUNTIME)
630		return true;
631
632	/*
633	 * 32-bit EFI doesn't suffer from the bug that requires us to
634	 * reserve boot services regions, and mixed mode support
635	 * doesn't exist for 32-bit kernels.
636	 */
637	if (IS_ENABLED(CONFIG_X86_32))
638		return false;
639
640	/*
641	 * EFI specific purpose memory may be reserved by default
642	 * depending on kernel config and boot options.
643	 */
644	if (md->type == EFI_CONVENTIONAL_MEMORY &&
645	    efi_soft_reserve_enabled() &&
646	    (md->attribute & EFI_MEMORY_SP))
647		return false;
648
649	/*
650	 * Map all of RAM so that we can access arguments in the 1:1
651	 * mapping when making EFI runtime calls.
652	 */
653	if (efi_is_mixed()) {
654		if (md->type == EFI_CONVENTIONAL_MEMORY ||
655		    md->type == EFI_LOADER_DATA ||
656		    md->type == EFI_LOADER_CODE)
657			return true;
658	}
659
660	/*
661	 * Map boot services regions as a workaround for buggy
662	 * firmware that accesses them even when they shouldn't.
663	 *
664	 * See efi_{reserve,free}_boot_services().
665	 */
666	if (md->type == EFI_BOOT_SERVICES_CODE ||
667	    md->type == EFI_BOOT_SERVICES_DATA)
668		return true;
669
670	return false;
671}
672
673/*
674 * Map the efi memory ranges of the runtime services and update new_mmap with
675 * virtual addresses.
676 */
677static void * __init efi_map_regions(int *count, int *pg_shift)
678{
679	void *p, *new_memmap = NULL;
680	unsigned long left = 0;
681	unsigned long desc_size;
682	efi_memory_desc_t *md;
683
684	desc_size = efi.memmap.desc_size;
685
686	p = NULL;
687	while ((p = efi_map_next_entry(p))) {
688		md = p;
689
690		if (!should_map_region(md))
691			continue;
692
693		efi_map_region(md);
694
695		if (left < desc_size) {
696			new_memmap = realloc_pages(new_memmap, *pg_shift);
697			if (!new_memmap)
698				return NULL;
699
700			left += PAGE_SIZE << *pg_shift;
701			(*pg_shift)++;
702		}
703
704		memcpy(new_memmap + (*count * desc_size), md, desc_size);
705
706		left -= desc_size;
707		(*count)++;
708	}
709
710	return new_memmap;
711}
712
713static void __init kexec_enter_virtual_mode(void)
714{
715#ifdef CONFIG_KEXEC_CORE
716	efi_memory_desc_t *md;
717	unsigned int num_pages;
718
719	/*
720	 * We don't do virtual mode, since we don't do runtime services, on
721	 * non-native EFI.
722	 */
723	if (efi_is_mixed()) {
724		efi_memmap_unmap();
725		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
726		return;
727	}
728
729	if (efi_alloc_page_tables()) {
730		pr_err("Failed to allocate EFI page tables\n");
731		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
732		return;
733	}
734
735	/*
736	* Map efi regions which were passed via setup_data. The virt_addr is a
737	* fixed addr which was used in first kernel of a kexec boot.
738	*/
739	for_each_efi_memory_desc(md)
740		efi_map_region_fixed(md); /* FIXME: add error handling */
741
742	/*
743	 * Unregister the early EFI memmap from efi_init() and install
744	 * the new EFI memory map.
745	 */
746	efi_memmap_unmap();
747
748	if (efi_memmap_init_late(efi.memmap.phys_map,
749				 efi.memmap.desc_size * efi.memmap.nr_map)) {
750		pr_err("Failed to remap late EFI memory map\n");
751		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
752		return;
753	}
754
755	num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
756	num_pages >>= PAGE_SHIFT;
757
758	if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
759		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
760		return;
761	}
762
763	efi_sync_low_kernel_mappings();
764	efi_native_runtime_setup();
765	efi_runtime_update_mappings();
766#endif
767}
768
769/*
770 * This function will switch the EFI runtime services to virtual mode.
771 * Essentially, we look through the EFI memmap and map every region that
772 * has the runtime attribute bit set in its memory descriptor into the
773 * efi_pgd page table.
774 *
775 * The new method does a pagetable switch in a preemption-safe manner
776 * so that we're in a different address space when calling a runtime
777 * function. For function arguments passing we do copy the PUDs of the
778 * kernel page table into efi_pgd prior to each call.
779 *
780 * Specially for kexec boot, efi runtime maps in previous kernel should
781 * be passed in via setup_data. In that case runtime ranges will be mapped
782 * to the same virtual addresses as the first kernel, see
783 * kexec_enter_virtual_mode().
784 */
785static void __init __efi_enter_virtual_mode(void)
786{
787	int count = 0, pg_shift = 0;
788	void *new_memmap = NULL;
789	efi_status_t status;
790	unsigned long pa;
791
792	if (efi_alloc_page_tables()) {
793		pr_err("Failed to allocate EFI page tables\n");
794		goto err;
795	}
796
797	efi_merge_regions();
798	new_memmap = efi_map_regions(&count, &pg_shift);
799	if (!new_memmap) {
800		pr_err("Error reallocating memory, EFI runtime non-functional!\n");
801		goto err;
802	}
803
804	pa = __pa(new_memmap);
805
806	/*
807	 * Unregister the early EFI memmap from efi_init() and install
808	 * the new EFI memory map that we are about to pass to the
809	 * firmware via SetVirtualAddressMap().
810	 */
811	efi_memmap_unmap();
812
813	if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
814		pr_err("Failed to remap late EFI memory map\n");
815		goto err;
816	}
817
818	if (efi_enabled(EFI_DBG)) {
819		pr_info("EFI runtime memory map:\n");
820		efi_print_memmap();
821	}
822
823	if (efi_setup_page_tables(pa, 1 << pg_shift))
824		goto err;
825
826	efi_sync_low_kernel_mappings();
827
828	status = efi_set_virtual_address_map(efi.memmap.desc_size * count,
829					     efi.memmap.desc_size,
830					     efi.memmap.desc_version,
831					     (efi_memory_desc_t *)pa,
832					     efi_systab_phys);
833	if (status != EFI_SUCCESS) {
834		pr_err("Unable to switch EFI into virtual mode (status=%lx)!\n",
835		       status);
836		goto err;
837	}
838
839	efi_check_for_embedded_firmwares();
840	efi_unmap_boot_services();
841
842	if (!efi_is_mixed())
843		efi_native_runtime_setup();
844	else
845		efi_thunk_runtime_setup();
846
847	/*
848	 * Apply more restrictive page table mapping attributes now that
849	 * SVAM() has been called and the firmware has performed all
850	 * necessary relocation fixups for the new virtual addresses.
851	 */
852	efi_runtime_update_mappings();
853
854	/* clean DUMMY object */
855	efi_delete_dummy_variable();
856	return;
857
858err:
859	clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
860}
861
862void __init efi_enter_virtual_mode(void)
863{
864	if (efi_enabled(EFI_PARAVIRT))
865		return;
866
867	efi.runtime = (efi_runtime_services_t *)efi_runtime;
868
869	if (efi_setup)
870		kexec_enter_virtual_mode();
871	else
872		__efi_enter_virtual_mode();
873
874	efi_dump_pagetable();
875}
876
877bool efi_is_table_address(unsigned long phys_addr)
878{
879	unsigned int i;
880
881	if (phys_addr == EFI_INVALID_TABLE_ADDR)
882		return false;
883
884	for (i = 0; i < ARRAY_SIZE(efi_tables); i++)
885		if (*(efi_tables[i]) == phys_addr)
886			return true;
887
888	return false;
889}
890
891#define EFI_FIELD(var) efi_ ## var
892
893#define EFI_ATTR_SHOW(name) \
894static ssize_t name##_show(struct kobject *kobj, \
895				struct kobj_attribute *attr, char *buf) \
896{ \
897	return sprintf(buf, "0x%lx\n", EFI_FIELD(name)); \
898}
899
900EFI_ATTR_SHOW(fw_vendor);
901EFI_ATTR_SHOW(runtime);
902EFI_ATTR_SHOW(config_table);
903
904struct kobj_attribute efi_attr_fw_vendor = __ATTR_RO(fw_vendor);
905struct kobj_attribute efi_attr_runtime = __ATTR_RO(runtime);
906struct kobj_attribute efi_attr_config_table = __ATTR_RO(config_table);
907
908umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n)
909{
910	if (attr == &efi_attr_fw_vendor.attr) {
911		if (efi_enabled(EFI_PARAVIRT) ||
912				efi_fw_vendor == EFI_INVALID_TABLE_ADDR)
913			return 0;
914	} else if (attr == &efi_attr_runtime.attr) {
915		if (efi_runtime == EFI_INVALID_TABLE_ADDR)
916			return 0;
917	} else if (attr == &efi_attr_config_table.attr) {
918		if (efi_config_table == EFI_INVALID_TABLE_ADDR)
919			return 0;
920	}
921	return attr->mode;
922}
923
924enum efi_secureboot_mode __x86_ima_efi_boot_mode(void)
925{
926	return boot_params.secure_boot;
927}