x86-stub.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2
   3/* -----------------------------------------------------------------------
   4 *
   5 *   Copyright 2011 Intel Corporation; author Matt Fleming
   6 *
   7 * ----------------------------------------------------------------------- */
   8
   9#include <linux/efi.h>
  10#include <linux/pci.h>
  11#include <linux/stddef.h>
  12
  13#include <asm/efi.h>
  14#include <asm/e820/types.h>
  15#include <asm/setup.h>
  16#include <asm/desc.h>
  17#include <asm/boot.h>
  18#include <asm/kaslr.h>
  19#include <asm/sev.h>
  20
  21#include "efistub.h"
  22#include "x86-stub.h"
  23
  24extern char _bss[], _ebss[];
  25
  26const efi_system_table_t *efi_system_table;
  27const efi_dxe_services_table_t *efi_dxe_table;
  28static efi_loaded_image_t *image = NULL;
  29static efi_memory_attribute_protocol_t *memattr;
  30
  31typedef union sev_memory_acceptance_protocol sev_memory_acceptance_protocol_t;
  32union sev_memory_acceptance_protocol {
  33	struct {
  34		efi_status_t (__efiapi * allow_unaccepted_memory)(
  35			sev_memory_acceptance_protocol_t *);
  36	};
  37	struct {
  38		u32 allow_unaccepted_memory;
  39	} mixed_mode;
  40};
  41
  42static efi_status_t
  43preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom)
  44{
  45	struct pci_setup_rom *rom __free(efi_pool) = NULL;
  46	efi_status_t status;
  47	unsigned long size;
  48	uint64_t romsize;
  49	void *romimage;
  50
  51	/*
  52	 * Some firmware images contain EFI function pointers at the place where
  53	 * the romimage and romsize fields are supposed to be. Typically the EFI
  54	 * code is mapped at high addresses, translating to an unrealistically
  55	 * large romsize. The UEFI spec limits the size of option ROMs to 16
  56	 * MiB so we reject any ROMs over 16 MiB in size to catch this.
  57	 */
  58	romimage = efi_table_attr(pci, romimage);
  59	romsize = efi_table_attr(pci, romsize);
  60	if (!romimage || !romsize || romsize > SZ_16M)
  61		return EFI_INVALID_PARAMETER;
  62
  63	size = romsize + sizeof(*rom);
  64
  65	status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size,
  66			     (void **)&rom);
  67	if (status != EFI_SUCCESS) {
  68		efi_err("Failed to allocate memory for 'rom'\n");
  69		return status;
  70	}
  71
  72	memset(rom, 0, sizeof(*rom));
  73
  74	rom->data.type	= SETUP_PCI;
  75	rom->data.len	= size - sizeof(struct setup_data);
  76	rom->data.next	= 0;
  77	rom->pcilen	= romsize;
  78
  79	status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16,
  80				PCI_VENDOR_ID, 1, &rom->vendor);
  81
  82	if (status != EFI_SUCCESS) {
  83		efi_err("Failed to read rom->vendor\n");
  84		return status;
  85	}
  86
  87	status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16,
  88				PCI_DEVICE_ID, 1, &rom->devid);
  89
  90	if (status != EFI_SUCCESS) {
  91		efi_err("Failed to read rom->devid\n");
  92		return status;
  93	}
  94
  95	status = efi_call_proto(pci, get_location, &rom->segment, &rom->bus,
  96				&rom->device, &rom->function);
  97
  98	if (status != EFI_SUCCESS)
  99		return status;
 100
 101	memcpy(rom->romdata, romimage, romsize);
 102	*__rom = no_free_ptr(rom);
 103	return EFI_SUCCESS;
 104}
 105
 106/*
 107 * There's no way to return an informative status from this function,
 108 * because any analysis (and printing of error messages) needs to be
 109 * done directly at the EFI function call-site.
 110 *
 111 * For example, EFI_INVALID_PARAMETER could indicate a bug or maybe we
 112 * just didn't find any PCI devices, but there's no way to tell outside
 113 * the context of the call.
 114 */
 115static void setup_efi_pci(struct boot_params *params)
 116{
 117	efi_status_t status;
 118	efi_handle_t *pci_handle __free(efi_pool) = NULL;
 119	efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
 120	struct setup_data *data;
 121	unsigned long num;
 122	efi_handle_t h;
 123
 124	status = efi_bs_call(locate_handle_buffer, EFI_LOCATE_BY_PROTOCOL,
 125			     &pci_proto, NULL, &num, &pci_handle);
 126	if (status != EFI_SUCCESS)
 127		return;
 128
 129	data = (struct setup_data *)(unsigned long)params->hdr.setup_data;
 130
 131	while (data && data->next)
 132		data = (struct setup_data *)(unsigned long)data->next;
 133
 134	for_each_efi_handle(h, pci_handle, num) {
 135		efi_pci_io_protocol_t *pci = NULL;
 136		struct pci_setup_rom *rom;
 137
 138		status = efi_bs_call(handle_protocol, h, &pci_proto,
 139				     (void **)&pci);
 140		if (status != EFI_SUCCESS || !pci)
 141			continue;
 142
 143		status = preserve_pci_rom_image(pci, &rom);
 144		if (status != EFI_SUCCESS)
 145			continue;
 146
 147		if (data)
 148			data->next = (unsigned long)rom;
 149		else
 150			params->hdr.setup_data = (unsigned long)rom;
 151
 152		data = (struct setup_data *)rom;
 153	}
 154}
 155
 156static void retrieve_apple_device_properties(struct boot_params *boot_params)
 157{
 158	efi_guid_t guid = APPLE_PROPERTIES_PROTOCOL_GUID;
 159	struct setup_data *data, *new;
 160	efi_status_t status;
 161	u32 size = 0;
 162	apple_properties_protocol_t *p;
 163
 164	status = efi_bs_call(locate_protocol, &guid, NULL, (void **)&p);
 165	if (status != EFI_SUCCESS)
 166		return;
 167
 168	if (efi_table_attr(p, version) != 0x10000) {
 169		efi_err("Unsupported properties proto version\n");
 170		return;
 171	}
 172
 173	efi_call_proto(p, get_all, NULL, &size);
 174	if (!size)
 175		return;
 176
 177	do {
 178		status = efi_bs_call(allocate_pool, EFI_LOADER_DATA,
 179				     size + sizeof(struct setup_data),
 180				     (void **)&new);
 181		if (status != EFI_SUCCESS) {
 182			efi_err("Failed to allocate memory for 'properties'\n");
 183			return;
 184		}
 185
 186		status = efi_call_proto(p, get_all, new->data, &size);
 187
 188		if (status == EFI_BUFFER_TOO_SMALL)
 189			efi_bs_call(free_pool, new);
 190	} while (status == EFI_BUFFER_TOO_SMALL);
 191
 192	new->type = SETUP_APPLE_PROPERTIES;
 193	new->len  = size;
 194	new->next = 0;
 195
 196	data = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data;
 197	if (!data) {
 198		boot_params->hdr.setup_data = (unsigned long)new;
 199	} else {
 200		while (data->next)
 201			data = (struct setup_data *)(unsigned long)data->next;
 202		data->next = (unsigned long)new;
 203	}
 204}
 205
 206struct smbios_entry_point {
 207	u8	anchor[4];
 208	u8	ep_checksum;
 209	u8	ep_length;
 210	u8	major_version;
 211	u8	minor_version;
 212	u16	max_size_entry;
 213	u8	ep_rev;
 214	u8	reserved[5];
 215
 216	struct __packed {
 217		u8	anchor[5];
 218		u8	checksum;
 219		u16	st_length;
 220		u32	st_address;
 221		u16	number_of_entries;
 222		u8	bcd_rev;
 223	} intm;
 224};
 225
 226static bool verify_ep_checksum(const void *ptr, int length)
 227{
 228	u8 sum = 0;
 229
 230	for (int i = 0; i < length; i++)
 231		sum += ((u8 *)ptr)[i];
 232
 233	return sum == 0;
 234}
 235
 236static bool verify_ep_integrity(const struct smbios_entry_point *ep)
 237{
 238	if (memcmp(ep->anchor, "_SM_", sizeof(ep->anchor)) != 0)
 239		return false;
 240
 241	if (memcmp(ep->intm.anchor, "_DMI_", sizeof(ep->intm.anchor)) != 0)
 242		return false;
 243
 244	if (!verify_ep_checksum(ep, ep->ep_length) ||
 245	    !verify_ep_checksum(&ep->intm, sizeof(ep->intm)))
 246		return false;
 247
 248	return true;
 249}
 250
 251static const struct efi_smbios_record *search_record(void *table, u32 length,
 252						     u8 type)
 253{
 254	const u8 *p, *end;
 255
 256	p = (u8 *)table;
 257	end = p + length;
 258
 259	while (p + sizeof(struct efi_smbios_record) < end) {
 260		const struct efi_smbios_record *hdr =
 261			(struct efi_smbios_record *)p;
 262		const u8 *next;
 263
 264		if (hdr->type == type)
 265			return hdr;
 266
 267		/* Type 127 = End-of-Table */
 268		if (hdr->type == 0x7F)
 269			return NULL;
 270
 271		/* Jumping to the unformed section */
 272		next = p + hdr->length;
 273
 274		/* Unformed section ends with 0000h */
 275		while ((next[0] != 0 || next[1] != 0) && next + 1 < end)
 276			next++;
 277
 278		next += 2;
 279		p = next;
 280	}
 281
 282	return NULL;
 283}
 284
 285static const struct efi_smbios_record *get_table_record(u8 type)
 286{
 287	const struct smbios_entry_point *ep;
 288
 289	/*
 290	 * Locate the legacy 32-bit SMBIOS entrypoint in memory, and parse it
 291	 * directly. Needed by some Macs that do not implement the EFI protocol.
 292	 */
 293	ep = get_efi_config_table(SMBIOS_TABLE_GUID);
 294	if (!ep)
 295		return NULL;
 296
 297	if (!verify_ep_integrity(ep))
 298		return NULL;
 299
 300	return search_record((void *)(unsigned long)ep->intm.st_address,
 301			     ep->intm.st_length, type);
 302}
 303
 304static bool apple_match_product_name(void)
 305{
 306	static const char type1_product_matches[][15] = {
 307		"MacBookPro11,3",
 308		"MacBookPro11,5",
 309		"MacBookPro13,3",
 310		"MacBookPro14,3",
 311		"MacBookPro15,1",
 312		"MacBookPro15,3",
 313		"MacBookPro16,1",
 314		"MacBookPro16,4",
 315	};
 316	const struct efi_smbios_type1_record *record;
 317	const u8 *product;
 318
 319	record = (struct efi_smbios_type1_record *)
 320			(efi_get_smbios_record(1) ?: get_table_record(1));
 321	if (!record)
 322		return false;
 323
 324	product = efi_get_smbios_string(record, product_name);
 325	if (!product)
 326		return false;
 327
 328	for (int i = 0; i < ARRAY_SIZE(type1_product_matches); i++) {
 329		if (!strcmp(product, type1_product_matches[i]))
 330			return true;
 331	}
 332
 333	return false;
 334}
 335
 336static void apple_set_os(void)
 337{
 338	struct {
 339		unsigned long version;
 340		efi_status_t (__efiapi *set_os_version)(const char *);
 341		efi_status_t (__efiapi *set_os_vendor)(const char *);
 342	} *set_os;
 343	efi_status_t status;
 344
 345	if (!efi_is_64bit() || !apple_match_product_name())
 346		return;
 347
 348	status = efi_bs_call(locate_protocol, &APPLE_SET_OS_PROTOCOL_GUID, NULL,
 349			     (void **)&set_os);
 350	if (status != EFI_SUCCESS)
 351		return;
 352
 353	if (set_os->version >= 2) {
 354		status = set_os->set_os_vendor("Apple Inc.");
 355		if (status != EFI_SUCCESS)
 356			efi_err("Failed to set OS vendor via apple_set_os\n");
 357	}
 358
 359	if (set_os->version > 0) {
 360		/* The version being set doesn't seem to matter */
 361		status = set_os->set_os_version("Mac OS X 10.9");
 362		if (status != EFI_SUCCESS)
 363			efi_err("Failed to set OS version via apple_set_os\n");
 364	}
 365}
 366
 367efi_status_t efi_adjust_memory_range_protection(unsigned long start,
 368						unsigned long size)
 369{
 370	efi_status_t status;
 371	efi_gcd_memory_space_desc_t desc;
 372	unsigned long end, next;
 373	unsigned long rounded_start, rounded_end;
 374	unsigned long unprotect_start, unprotect_size;
 375
 376	rounded_start = rounddown(start, EFI_PAGE_SIZE);
 377	rounded_end = roundup(start + size, EFI_PAGE_SIZE);
 378
 379	if (memattr != NULL) {
 380		status = efi_call_proto(memattr, set_memory_attributes,
 381					rounded_start,
 382					rounded_end - rounded_start,
 383					EFI_MEMORY_RO);
 384		if (status != EFI_SUCCESS) {
 385			efi_warn("Failed to set EFI_MEMORY_RO attribute\n");
 386			return status;
 387		}
 388
 389		status = efi_call_proto(memattr, clear_memory_attributes,
 390					rounded_start,
 391					rounded_end - rounded_start,
 392					EFI_MEMORY_XP);
 393		if (status != EFI_SUCCESS)
 394			efi_warn("Failed to clear EFI_MEMORY_XP attribute\n");
 395		return status;
 396	}
 397
 398	if (efi_dxe_table == NULL)
 399		return EFI_SUCCESS;
 400
 401	/*
 402	 * Don't modify memory region attributes, if they are
 403	 * already suitable, to lower the possibility to
 404	 * encounter firmware bugs.
 405	 */
 406
 407	for (end = start + size; start < end; start = next) {
 408
 409		status = efi_dxe_call(get_memory_space_descriptor, start, &desc);
 410
 411		if (status != EFI_SUCCESS)
 412			break;
 413
 414		next = desc.base_address + desc.length;
 415
 416		/*
 417		 * Only system memory and more reliable memory are suitable for
 418		 * trampoline/kernel image placement. So only those memory types
 419		 * may need to have attributes modified.
 420		 */
 421
 422		if ((desc.gcd_memory_type != EfiGcdMemoryTypeSystemMemory &&
 423		     desc.gcd_memory_type != EfiGcdMemoryTypeMoreReliable) ||
 424		    (desc.attributes & (EFI_MEMORY_RO | EFI_MEMORY_XP)) == 0)
 425			continue;
 426
 427		unprotect_start = max(rounded_start, (unsigned long)desc.base_address);
 428		unprotect_size = min(rounded_end, next) - unprotect_start;
 429
 430		status = efi_dxe_call(set_memory_space_attributes,
 431				      unprotect_start, unprotect_size,
 432				      EFI_MEMORY_WB);
 433
 434		if (status != EFI_SUCCESS) {
 435			efi_warn("Unable to unprotect memory range [%08lx,%08lx]: %lx\n",
 436				 unprotect_start,
 437				 unprotect_start + unprotect_size,
 438				 status);
 439			break;
 440		}
 441	}
 442	return EFI_SUCCESS;
 443}
 444
 445static void setup_unaccepted_memory(void)
 446{
 447	efi_guid_t mem_acceptance_proto = OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID;
 448	sev_memory_acceptance_protocol_t *proto;
 449	efi_status_t status;
 450
 451	if (!IS_ENABLED(CONFIG_UNACCEPTED_MEMORY))
 452		return;
 453
 454	/*
 455	 * Enable unaccepted memory before calling exit boot services in order
 456	 * for the UEFI to not accept all memory on EBS.
 457	 */
 458	status = efi_bs_call(locate_protocol, &mem_acceptance_proto, NULL,
 459			     (void **)&proto);
 460	if (status != EFI_SUCCESS)
 461		return;
 462
 463	status = efi_call_proto(proto, allow_unaccepted_memory);
 464	if (status != EFI_SUCCESS)
 465		efi_err("Memory acceptance protocol failed\n");
 466}
 467
 468static efi_char16_t *efistub_fw_vendor(void)
 469{
 470	unsigned long vendor = efi_table_attr(efi_system_table, fw_vendor);
 471
 472	return (efi_char16_t *)vendor;
 473}
 474
 475static const efi_char16_t apple[] = L"Apple";
 476
 477static void setup_quirks(struct boot_params *boot_params)
 478{
 479	if (!memcmp(efistub_fw_vendor(), apple, sizeof(apple))) {
 480		if (IS_ENABLED(CONFIG_APPLE_PROPERTIES))
 481			retrieve_apple_device_properties(boot_params);
 482
 483		apple_set_os();
 484	}
 485}
 486
 487static void setup_graphics(struct boot_params *boot_params)
 488{
 489	struct screen_info *si = memset(&boot_params->screen_info, 0, sizeof(*si));
 490	struct edid_info *edid = memset(&boot_params->edid_info, 0, sizeof(*edid));
 491
 492	efi_setup_graphics(si, edid);
 493}
 494
 495static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status)
 496{
 497	efi_bs_call(exit, handle, status, 0, NULL);
 498	for(;;)
 499		asm("hlt");
 500}
 501
 502/*
 503 * Because the x86 boot code expects to be passed a boot_params we
 504 * need to create one ourselves (usually the bootloader would create
 505 * one for us).
 506 */
 507static efi_status_t efi_allocate_bootparams(efi_handle_t handle,
 508					    struct boot_params **bp)
 509{
 510	efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
 511	struct boot_params *boot_params;
 512	struct setup_header *hdr;
 513	efi_status_t status;
 514	unsigned long alloc;
 515	char *cmdline_ptr;
 516
 517	status = efi_bs_call(handle_protocol, handle, &proto, (void **)&image);
 518	if (status != EFI_SUCCESS) {
 519		efi_err("Failed to get handle for LOADED_IMAGE_PROTOCOL\n");
 520		return status;
 521	}
 522
 523	status = efi_allocate_pages(PARAM_SIZE, &alloc, ULONG_MAX);
 524	if (status != EFI_SUCCESS)
 525		return status;
 526
 527	boot_params = memset((void *)alloc, 0x0, PARAM_SIZE);
 528	hdr	    = &boot_params->hdr;
 529
 530	/* Assign the setup_header fields that the kernel actually cares about */
 531	hdr->root_flags	= 1;
 532	hdr->vid_mode	= 0xffff;
 533
 534	hdr->type_of_loader = 0x21;
 535	hdr->initrd_addr_max = INT_MAX;
 536
 537	/* Convert unicode cmdline to ascii */
 538	cmdline_ptr = efi_convert_cmdline(image);
 539	if (!cmdline_ptr) {
 540		efi_free(PARAM_SIZE, alloc);
 541		return EFI_OUT_OF_RESOURCES;
 542	}
 543
 544	efi_set_u64_split((unsigned long)cmdline_ptr, &hdr->cmd_line_ptr,
 545			  &boot_params->ext_cmd_line_ptr);
 546
 547	*bp = boot_params;
 548	return EFI_SUCCESS;
 549}
 550
 551static void add_e820ext(struct boot_params *params,
 552			struct setup_data *e820ext, u32 nr_entries)
 553{
 554	struct setup_data *data;
 555
 556	e820ext->type = SETUP_E820_EXT;
 557	e820ext->len  = nr_entries * sizeof(struct boot_e820_entry);
 558	e820ext->next = 0;
 559
 560	data = (struct setup_data *)(unsigned long)params->hdr.setup_data;
 561
 562	while (data && data->next)
 563		data = (struct setup_data *)(unsigned long)data->next;
 564
 565	if (data)
 566		data->next = (unsigned long)e820ext;
 567	else
 568		params->hdr.setup_data = (unsigned long)e820ext;
 569}
 570
 571static efi_status_t
 572setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_size)
 573{
 574	struct boot_e820_entry *entry = params->e820_table;
 575	struct efi_info *efi = &params->efi_info;
 576	struct boot_e820_entry *prev = NULL;
 577	u32 nr_entries;
 578	u32 nr_desc;
 579	int i;
 580
 581	nr_entries = 0;
 582	nr_desc = efi->efi_memmap_size / efi->efi_memdesc_size;
 583
 584	for (i = 0; i < nr_desc; i++) {
 585		efi_memory_desc_t *d;
 586		unsigned int e820_type = 0;
 587		unsigned long m = efi->efi_memmap;
 588
 589#ifdef CONFIG_X86_64
 590		m |= (u64)efi->efi_memmap_hi << 32;
 591#endif
 592
 593		d = efi_memdesc_ptr(m, efi->efi_memdesc_size, i);
 594		switch (d->type) {
 595		case EFI_RESERVED_TYPE:
 596		case EFI_RUNTIME_SERVICES_CODE:
 597		case EFI_RUNTIME_SERVICES_DATA:
 598		case EFI_MEMORY_MAPPED_IO:
 599		case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
 600		case EFI_PAL_CODE:
 601			e820_type = E820_TYPE_RESERVED;
 602			break;
 603
 604		case EFI_UNUSABLE_MEMORY:
 605			e820_type = E820_TYPE_UNUSABLE;
 606			break;
 607
 608		case EFI_ACPI_RECLAIM_MEMORY:
 609			e820_type = E820_TYPE_ACPI;
 610			break;
 611
 612		case EFI_LOADER_CODE:
 613		case EFI_LOADER_DATA:
 614		case EFI_BOOT_SERVICES_CODE:
 615		case EFI_BOOT_SERVICES_DATA:
 616		case EFI_CONVENTIONAL_MEMORY:
 617			if (efi_soft_reserve_enabled() &&
 618			    (d->attribute & EFI_MEMORY_SP))
 619				e820_type = E820_TYPE_SOFT_RESERVED;
 620			else
 621				e820_type = E820_TYPE_RAM;
 622			break;
 623
 624		case EFI_ACPI_MEMORY_NVS:
 625			e820_type = E820_TYPE_NVS;
 626			break;
 627
 628		case EFI_PERSISTENT_MEMORY:
 629			e820_type = E820_TYPE_PMEM;
 630			break;
 631
 632		case EFI_UNACCEPTED_MEMORY:
 633			if (!IS_ENABLED(CONFIG_UNACCEPTED_MEMORY))
 634				continue;
 635			e820_type = E820_TYPE_RAM;
 636			process_unaccepted_memory(d->phys_addr,
 637						  d->phys_addr + PAGE_SIZE * d->num_pages);
 638			break;
 639		default:
 640			continue;
 641		}
 642
 643		/* Merge adjacent mappings */
 644		if (prev && prev->type == e820_type &&
 645		    (prev->addr + prev->size) == d->phys_addr) {
 646			prev->size += d->num_pages << 12;
 647			continue;
 648		}
 649
 650		if (nr_entries == ARRAY_SIZE(params->e820_table)) {
 651			u32 need = (nr_desc - i) * sizeof(struct e820_entry) +
 652				   sizeof(struct setup_data);
 653
 654			if (!e820ext || e820ext_size < need)
 655				return EFI_BUFFER_TOO_SMALL;
 656
 657			/* boot_params map full, switch to e820 extended */
 658			entry = (struct boot_e820_entry *)e820ext->data;
 659		}
 660
 661		entry->addr = d->phys_addr;
 662		entry->size = d->num_pages << PAGE_SHIFT;
 663		entry->type = e820_type;
 664		prev = entry++;
 665		nr_entries++;
 666	}
 667
 668	if (nr_entries > ARRAY_SIZE(params->e820_table)) {
 669		u32 nr_e820ext = nr_entries - ARRAY_SIZE(params->e820_table);
 670
 671		add_e820ext(params, e820ext, nr_e820ext);
 672		nr_entries -= nr_e820ext;
 673	}
 674
 675	params->e820_entries = (u8)nr_entries;
 676
 677	return EFI_SUCCESS;
 678}
 679
 680static efi_status_t alloc_e820ext(u32 nr_desc, struct setup_data **e820ext,
 681				  u32 *e820ext_size)
 682{
 683	efi_status_t status;
 684	unsigned long size;
 685
 686	size = sizeof(struct setup_data) +
 687		sizeof(struct e820_entry) * nr_desc;
 688
 689	if (*e820ext) {
 690		efi_bs_call(free_pool, *e820ext);
 691		*e820ext = NULL;
 692		*e820ext_size = 0;
 693	}
 694
 695	status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size,
 696			     (void **)e820ext);
 697	if (status == EFI_SUCCESS)
 698		*e820ext_size = size;
 699
 700	return status;
 701}
 702
 703static efi_status_t allocate_e820(struct boot_params *params,
 704				  struct setup_data **e820ext,
 705				  u32 *e820ext_size)
 706{
 707	struct efi_boot_memmap *map __free(efi_pool) = NULL;
 708	efi_status_t status;
 709	__u32 nr_desc;
 710
 711	status = efi_get_memory_map(&map, false);
 712	if (status != EFI_SUCCESS)
 713		return status;
 714
 715	nr_desc = map->map_size / map->desc_size;
 716	if (nr_desc > ARRAY_SIZE(params->e820_table) - EFI_MMAP_NR_SLACK_SLOTS) {
 717		u32 nr_e820ext = nr_desc - ARRAY_SIZE(params->e820_table) +
 718				 EFI_MMAP_NR_SLACK_SLOTS;
 719
 720		status = alloc_e820ext(nr_e820ext, e820ext, e820ext_size);
 721		if (status != EFI_SUCCESS)
 722			return status;
 723	}
 724
 725	if (IS_ENABLED(CONFIG_UNACCEPTED_MEMORY))
 726		return allocate_unaccepted_bitmap(nr_desc, map);
 727
 728	return EFI_SUCCESS;
 729}
 730
 731struct exit_boot_struct {
 732	struct boot_params	*boot_params;
 733	struct efi_info		*efi;
 734};
 735
 736static efi_status_t exit_boot_func(struct efi_boot_memmap *map,
 737				   void *priv)
 738{
 739	const char *signature;
 740	struct exit_boot_struct *p = priv;
 741
 742	signature = efi_is_64bit() ? EFI64_LOADER_SIGNATURE
 743				   : EFI32_LOADER_SIGNATURE;
 744	memcpy(&p->efi->efi_loader_signature, signature, sizeof(__u32));
 745
 746	efi_set_u64_split((unsigned long)efi_system_table,
 747			  &p->efi->efi_systab, &p->efi->efi_systab_hi);
 748	p->efi->efi_memdesc_size	= map->desc_size;
 749	p->efi->efi_memdesc_version	= map->desc_ver;
 750	efi_set_u64_split((unsigned long)map->map,
 751			  &p->efi->efi_memmap, &p->efi->efi_memmap_hi);
 752	p->efi->efi_memmap_size		= map->map_size;
 753
 754	return EFI_SUCCESS;
 755}
 756
 757static efi_status_t exit_boot(struct boot_params *boot_params, void *handle)
 758{
 759	struct setup_data *e820ext = NULL;
 760	__u32 e820ext_size = 0;
 761	efi_status_t status;
 762	struct exit_boot_struct priv;
 763
 764	priv.boot_params	= boot_params;
 765	priv.efi		= &boot_params->efi_info;
 766
 767	status = allocate_e820(boot_params, &e820ext, &e820ext_size);
 768	if (status != EFI_SUCCESS)
 769		return status;
 770
 771	/* Might as well exit boot services now */
 772	status = efi_exit_boot_services(handle, &priv, exit_boot_func);
 773	if (status != EFI_SUCCESS)
 774		return status;
 775
 776	/* Historic? */
 777	boot_params->alt_mem_k	= 32 * 1024;
 778
 779	status = setup_e820(boot_params, e820ext, e820ext_size);
 780	if (status != EFI_SUCCESS)
 781		return status;
 782
 783	return EFI_SUCCESS;
 784}
 785
 786static bool have_unsupported_snp_features(void)
 787{
 788	u64 unsupported;
 789
 790	unsupported = snp_get_unsupported_features(sev_get_status());
 791	if (unsupported) {
 792		efi_err("Unsupported SEV-SNP features detected: 0x%llx\n",
 793			unsupported);
 794		return true;
 795	}
 796	return false;
 797}
 798
 799static void efi_get_seed(void *seed, int size)
 800{
 801	efi_get_random_bytes(size, seed);
 802
 803	/*
 804	 * This only updates seed[0] when running on 32-bit, but in that case,
 805	 * seed[1] is not used anyway, as there is no virtual KASLR on 32-bit.
 806	 */
 807	*(unsigned long *)seed ^= kaslr_get_random_long("EFI");
 808}
 809
 810static void error(char *str)
 811{
 812	efi_warn("Decompression failed: %s\n", str);
 813}
 814
 815static const char *cmdline_memmap_override;
 816
 817static efi_status_t parse_options(const char *cmdline)
 818{
 819	static const char opts[][14] = {
 820		"mem=", "memmap=", "hugepages="
 821	};
 822
 823	for (int i = 0; i < ARRAY_SIZE(opts); i++) {
 824		const char *p = strstr(cmdline, opts[i]);
 825
 826		if (p == cmdline || (p > cmdline && isspace(p[-1]))) {
 827			cmdline_memmap_override = opts[i];
 828			break;
 829		}
 830	}
 831
 832	return efi_parse_options(cmdline);
 833}
 834
 835static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry,
 836					  struct boot_params *boot_params)
 837{
 838	unsigned long virt_addr = LOAD_PHYSICAL_ADDR;
 839	unsigned long addr, alloc_size, entry;
 840	efi_status_t status;
 841	u32 seed[2] = {};
 842
 843	boot_params_ptr	= boot_params;
 844
 845	/* determine the required size of the allocation */
 846	alloc_size = ALIGN(max_t(unsigned long, output_len, kernel_total_size),
 847			   MIN_KERNEL_ALIGN);
 848
 849	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && !efi_nokaslr) {
 850		u64 range = KERNEL_IMAGE_SIZE - LOAD_PHYSICAL_ADDR - kernel_total_size;
 851		static const efi_char16_t ami[] = L"American Megatrends";
 852
 853		efi_get_seed(seed, sizeof(seed));
 854
 855		virt_addr += (range * seed[1]) >> 32;
 856		virt_addr &= ~(CONFIG_PHYSICAL_ALIGN - 1);
 857
 858		/*
 859		 * Older Dell systems with AMI UEFI firmware v2.0 may hang
 860		 * while decompressing the kernel if physical address
 861		 * randomization is enabled.
 862		 *
 863		 * https://bugzilla.kernel.org/show_bug.cgi?id=218173
 864		 */
 865		if (efi_system_table->hdr.revision <= EFI_2_00_SYSTEM_TABLE_REVISION &&
 866		    !memcmp(efistub_fw_vendor(), ami, sizeof(ami))) {
 867			efi_debug("AMI firmware v2.0 or older detected - disabling physical KASLR\n");
 868			seed[0] = 0;
 869		} else if (cmdline_memmap_override) {
 870			efi_info("%s detected on the kernel command line - disabling physical KASLR\n",
 871				 cmdline_memmap_override);
 872			seed[0] = 0;
 873		}
 874
 875		boot_params->hdr.loadflags |= KASLR_FLAG;
 876	}
 877
 878	status = efi_random_alloc(alloc_size, CONFIG_PHYSICAL_ALIGN, &addr,
 879				  seed[0], EFI_LOADER_CODE,
 880				  LOAD_PHYSICAL_ADDR,
 881				  EFI_X86_KERNEL_ALLOC_LIMIT);
 882	if (status != EFI_SUCCESS)
 883		return status;
 884
 885	entry = decompress_kernel((void *)addr, virt_addr, error);
 886	if (entry == ULONG_MAX) {
 887		efi_free(alloc_size, addr);
 888		return EFI_LOAD_ERROR;
 889	}
 890
 891	*kernel_entry = addr + entry;
 892
 893	return efi_adjust_memory_range_protection(addr, kernel_text_size) ?:
 894	       efi_adjust_memory_range_protection(addr + kernel_inittext_offset,
 895						  kernel_inittext_size);
 896}
 897
 898static void __noreturn enter_kernel(unsigned long kernel_addr,
 899				    struct boot_params *boot_params)
 900{
 901	/* enter decompressed kernel with boot_params pointer in RSI/ESI */
 902	asm("jmp *%0"::"r"(kernel_addr), "S"(boot_params));
 903
 904	unreachable();
 905}
 906
 907/*
 908 * On success, this routine will jump to the relocated image directly and never
 909 * return.  On failure, it will exit to the firmware via efi_exit() instead of
 910 * returning.
 911 */
 912void __noreturn efi_stub_entry(efi_handle_t handle,
 913			       efi_system_table_t *sys_table_arg,
 914			       struct boot_params *boot_params)
 915
 916{
 917	efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID;
 918	const struct linux_efi_initrd *initrd = NULL;
 919	unsigned long kernel_entry;
 920	struct setup_header *hdr;
 921	efi_status_t status;
 922
 923	efi_system_table = sys_table_arg;
 924	/* Check if we were booted by the EFI firmware */
 925	if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
 926		efi_exit(handle, EFI_INVALID_PARAMETER);
 927
 928	if (!IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL) || !boot_params) {
 929		status = efi_allocate_bootparams(handle, &boot_params);
 930		if (status != EFI_SUCCESS)
 931			efi_exit(handle, status);
 932	}
 933
 934	hdr = &boot_params->hdr;
 935
 936	if (have_unsupported_snp_features())
 937		efi_exit(handle, EFI_UNSUPPORTED);
 938
 939	if (IS_ENABLED(CONFIG_EFI_DXE_MEM_ATTRIBUTES)) {
 940		efi_dxe_table = get_efi_config_table(EFI_DXE_SERVICES_TABLE_GUID);
 941		if (efi_dxe_table &&
 942		    efi_dxe_table->hdr.signature != EFI_DXE_SERVICES_TABLE_SIGNATURE) {
 943			efi_warn("Ignoring DXE services table: invalid signature\n");
 944			efi_dxe_table = NULL;
 945		}
 946	}
 947
 948	/* grab the memory attributes protocol if it exists */
 949	efi_bs_call(locate_protocol, &guid, NULL, (void **)&memattr);
 950
 951	status = efi_setup_5level_paging();
 952	if (status != EFI_SUCCESS) {
 953		efi_err("efi_setup_5level_paging() failed!\n");
 954		goto fail;
 955	}
 956
 957#ifdef CONFIG_CMDLINE_BOOL
 958	status = parse_options(CONFIG_CMDLINE);
 959	if (status != EFI_SUCCESS) {
 960		efi_err("Failed to parse options\n");
 961		goto fail;
 962	}
 963#endif
 964	if (!IS_ENABLED(CONFIG_CMDLINE_OVERRIDE)) {
 965		unsigned long cmdline_paddr = ((u64)hdr->cmd_line_ptr |
 966					       ((u64)boot_params->ext_cmd_line_ptr << 32));
 967		status = parse_options((char *)cmdline_paddr);
 968		if (status != EFI_SUCCESS) {
 969			efi_err("Failed to parse options\n");
 970			goto fail;
 971		}
 972	}
 973
 974	if (efi_mem_encrypt > 0)
 975		hdr->xloadflags |= XLF_MEM_ENCRYPTION;
 976
 977	status = efi_decompress_kernel(&kernel_entry, boot_params);
 978	if (status != EFI_SUCCESS) {
 979		efi_err("Failed to decompress kernel\n");
 980		goto fail;
 981	}
 982
 983	/*
 984	 * At this point, an initrd may already have been loaded by the
 985	 * bootloader and passed via bootparams. We permit an initrd loaded
 986	 * from the LINUX_EFI_INITRD_MEDIA_GUID device path to supersede it.
 987	 *
 988	 * If the device path is not present, any command-line initrd=
 989	 * arguments will be processed only if image is not NULL, which will be
 990	 * the case only if we were loaded via the PE entry point.
 991	 */
 992	status = efi_load_initrd(image, hdr->initrd_addr_max, ULONG_MAX,
 993				 &initrd);
 994	if (status != EFI_SUCCESS)
 995		goto fail;
 996	if (initrd && initrd->size > 0) {
 997		efi_set_u64_split(initrd->base, &hdr->ramdisk_image,
 998				  &boot_params->ext_ramdisk_image);
 999		efi_set_u64_split(initrd->size, &hdr->ramdisk_size,
1000				  &boot_params->ext_ramdisk_size);
1001	}
1002
1003
1004	/*
1005	 * If the boot loader gave us a value for secure_boot then we use that,
1006	 * otherwise we ask the BIOS.
1007	 */
1008	if (boot_params->secure_boot == efi_secureboot_mode_unset)
1009		boot_params->secure_boot = efi_get_secureboot();
1010
1011	/* Ask the firmware to clear memory on unclean shutdown */
1012	efi_enable_reset_attack_mitigation();
1013
1014	efi_random_get_seed();
1015
1016	efi_retrieve_eventlog();
1017
1018	setup_graphics(boot_params);
1019
1020	setup_efi_pci(boot_params);
1021
1022	setup_quirks(boot_params);
1023
1024	setup_unaccepted_memory();
1025
1026	status = exit_boot(boot_params, handle);
1027	if (status != EFI_SUCCESS) {
1028		efi_err("exit_boot() failed!\n");
1029		goto fail;
1030	}
1031
1032	/*
1033	 * Call the SEV init code while still running with the firmware's
1034	 * GDT/IDT, so #VC exceptions will be handled by EFI.
1035	 */
1036	sev_enable(boot_params);
1037
1038	efi_5level_switch();
1039
1040	enter_kernel(kernel_entry, boot_params);
1041fail:
1042	efi_err("efi_stub_entry() failed!\n");
1043
1044	efi_exit(handle, status);
1045}
1046
1047efi_status_t __efiapi efi_pe_entry(efi_handle_t handle,
1048				   efi_system_table_t *sys_table_arg)
1049{
1050	efi_stub_entry(handle, sys_table_arg, NULL);
1051}
1052
1053#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
1054void efi_handover_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
1055			struct boot_params *boot_params)
1056{
1057	memset(_bss, 0, _ebss - _bss);
1058	efi_stub_entry(handle, sys_table_arg, boot_params);
1059}
1060
1061#ifndef CONFIG_EFI_MIXED
1062extern __alias(efi_handover_entry)
1063void efi32_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
1064		      struct boot_params *boot_params);
1065
1066extern __alias(efi_handover_entry)
1067void efi64_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
1068		      struct boot_params *boot_params);
1069#endif
1070#endif