misc.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * misc.c
  4 *
  5 * This is a collection of several routines used to extract the kernel
  6 * which includes KASLR relocation, decompression, ELF parsing, and
  7 * relocation processing. Additionally included are the screen and serial
  8 * output functions and related debugging support functions.
  9 *
 10 * malloc by Hannu Savolainen 1993 and Matthias Urlichs 1994
 11 * puts by Nick Holloway 1993, better puts by Martin Mares 1995
 12 * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
 13 */
 14
 15#include "misc.h"
 16#include "error.h"
 17#include "../string.h"
 18#include "../voffset.h"
 19#include <asm/bootparam_utils.h>
 20
 21/*
 22 * WARNING!!
 23 * This code is compiled with -fPIC and it is relocated dynamically at
 24 * run time, but no relocation processing is performed. This means that
 25 * it is not safe to place pointers in static structures.
 26 */
 27
 28/* Macros used by the included decompressor code below. */
 29#define STATIC		static
 30/* Define an externally visible malloc()/free(). */
 31#define MALLOC_VISIBLE
 32#include <linux/decompress/mm.h>
 33
 34/*
 35 * Provide definitions of memzero and memmove as some of the decompressors will
 36 * try to define their own functions if these are not defined as macros.
 37 */
 38#define memzero(s, n)	memset((s), 0, (n))
 39#ifndef memmove
 40#define memmove		memmove
 41/* Functions used by the included decompressor code below. */
 42void *memmove(void *dest, const void *src, size_t n);
 43#endif
 44
 45/*
 46 * This is set up by the setup-routine at boot-time
 47 */
 48struct boot_params *boot_params_ptr;
 49
 50struct port_io_ops pio_ops;
 51
 52memptr free_mem_ptr;
 53memptr free_mem_end_ptr;
 54int spurious_nmi_count;
 55
 56static char *vidmem;
 57static int vidport;
 58
 59/* These might be accessed before .bss is cleared, so use .data instead. */
 60static int lines __section(".data");
 61static int cols __section(".data");
 62
 63#ifdef CONFIG_KERNEL_GZIP
 64#include "../../../../lib/decompress_inflate.c"
 65#endif
 66
 67#ifdef CONFIG_KERNEL_BZIP2
 68#include "../../../../lib/decompress_bunzip2.c"
 69#endif
 70
 71#ifdef CONFIG_KERNEL_LZMA
 72#include "../../../../lib/decompress_unlzma.c"
 73#endif
 74
 75#ifdef CONFIG_KERNEL_XZ
 76#include "../../../../lib/decompress_unxz.c"
 77#endif
 78
 79#ifdef CONFIG_KERNEL_LZO
 80#include "../../../../lib/decompress_unlzo.c"
 81#endif
 82
 83#ifdef CONFIG_KERNEL_LZ4
 84#include "../../../../lib/decompress_unlz4.c"
 85#endif
 86
 87#ifdef CONFIG_KERNEL_ZSTD
 88#include "../../../../lib/decompress_unzstd.c"
 89#endif
 90/*
 91 * NOTE: When adding a new decompressor, please update the analysis in
 92 * ../header.S.
 93 */
 94
 95static void scroll(void)
 96{
 97	int i;
 98
 99	memmove(vidmem, vidmem + cols * 2, (lines - 1) * cols * 2);
100	for (i = (lines - 1) * cols * 2; i < lines * cols * 2; i += 2)
101		vidmem[i] = ' ';
102}
103
104#define XMTRDY          0x20
105
106#define TXR             0       /*  Transmit register (WRITE) */
107#define LSR             5       /*  Line Status               */
108static void serial_putchar(int ch)
109{
110	unsigned timeout = 0xffff;
111
112	while ((inb(early_serial_base + LSR) & XMTRDY) == 0 && --timeout)
113		cpu_relax();
114
115	outb(ch, early_serial_base + TXR);
116}
117
118void __putstr(const char *s)
119{
120	int x, y, pos;
121	char c;
122
123	if (early_serial_base) {
124		const char *str = s;
125		while (*str) {
126			if (*str == '\n')
127				serial_putchar('\r');
128			serial_putchar(*str++);
129		}
130	}
131
132	if (lines == 0 || cols == 0)
133		return;
134
135	x = boot_params_ptr->screen_info.orig_x;
136	y = boot_params_ptr->screen_info.orig_y;
137
138	while ((c = *s++) != '\0') {
139		if (c == '\n') {
140			x = 0;
141			if (++y >= lines) {
142				scroll();
143				y--;
144			}
145		} else {
146			vidmem[(x + cols * y) * 2] = c;
147			if (++x >= cols) {
148				x = 0;
149				if (++y >= lines) {
150					scroll();
151					y--;
152				}
153			}
154		}
155	}
156
157	boot_params_ptr->screen_info.orig_x = x;
158	boot_params_ptr->screen_info.orig_y = y;
159
160	pos = (x + cols * y) * 2;	/* Update cursor position */
161	outb(14, vidport);
162	outb(0xff & (pos >> 9), vidport+1);
163	outb(15, vidport);
164	outb(0xff & (pos >> 1), vidport+1);
165}
166
167static noinline void __putnum(unsigned long value, unsigned int base,
168			      int mindig)
169{
170	char buf[8*sizeof(value)+1];
171	char *p;
172
173	p = buf + sizeof(buf);
174	*--p = '\0';
175
176	while (mindig-- > 0 || value) {
177		unsigned char digit = value % base;
178		digit += (digit >= 10) ? ('a'-10) : '0';
179		*--p = digit;
180
181		value /= base;
182	}
183
184	__putstr(p);
185}
186
187void __puthex(unsigned long value)
188{
189	__putnum(value, 16, sizeof(value)*2);
190}
191
192void __putdec(unsigned long value)
193{
194	__putnum(value, 10, 1);
195}
196
197#ifdef CONFIG_X86_NEED_RELOCS
198static void handle_relocations(void *output, unsigned long output_len,
199			       unsigned long virt_addr)
200{
201	int *reloc;
202	unsigned long delta, map, ptr;
203	unsigned long min_addr = (unsigned long)output;
204	unsigned long max_addr = min_addr + (VO___bss_start - VO__text);
205
206	/*
207	 * Calculate the delta between where vmlinux was linked to load
208	 * and where it was actually loaded.
209	 */
210	delta = min_addr - LOAD_PHYSICAL_ADDR;
211
212	/*
213	 * The kernel contains a table of relocation addresses. Those
214	 * addresses have the final load address of the kernel in virtual
215	 * memory. We are currently working in the self map. So we need to
216	 * create an adjustment for kernel memory addresses to the self map.
217	 * This will involve subtracting out the base address of the kernel.
218	 */
219	map = delta - __START_KERNEL_map;
220
221	/*
222	 * 32-bit always performs relocations. 64-bit relocations are only
223	 * needed if KASLR has chosen a different starting address offset
224	 * from __START_KERNEL_map.
225	 */
226	if (IS_ENABLED(CONFIG_X86_64))
227		delta = virt_addr - LOAD_PHYSICAL_ADDR;
228
229	if (!delta) {
230		debug_putstr("No relocation needed... ");
231		return;
232	}
233	debug_putstr("Performing relocations... ");
234
235	/*
236	 * Process relocations: 32 bit relocations first then 64 bit after.
237	 * Two sets of binary relocations are added to the end of the kernel
238	 * before compression. Each relocation table entry is the kernel
239	 * address of the location which needs to be updated stored as a
240	 * 32-bit value which is sign extended to 64 bits.
241	 *
242	 * Format is:
243	 *
244	 * kernel bits...
245	 * 0 - zero terminator for 64 bit relocations
246	 * 64 bit relocation repeated
247	 * 0 - zero terminator for 32 bit relocations
248	 * 32 bit relocation repeated
249	 *
250	 * So we work backwards from the end of the decompressed image.
251	 */
252	for (reloc = output + output_len - sizeof(*reloc); *reloc; reloc--) {
253		long extended = *reloc;
254		extended += map;
255
256		ptr = (unsigned long)extended;
257		if (ptr < min_addr || ptr > max_addr)
258			error("32-bit relocation outside of kernel!\n");
259
260		*(uint32_t *)ptr += delta;
261	}
262#ifdef CONFIG_X86_64
263	for (reloc--; *reloc; reloc--) {
264		long extended = *reloc;
265		extended += map;
266
267		ptr = (unsigned long)extended;
268		if (ptr < min_addr || ptr > max_addr)
269			error("64-bit relocation outside of kernel!\n");
270
271		*(uint64_t *)ptr += delta;
272	}
273#endif
274}
275#else
276static inline void handle_relocations(void *output, unsigned long output_len,
277				      unsigned long virt_addr)
278{ }
279#endif
280
281static size_t parse_elf(void *output)
282{
283#ifdef CONFIG_X86_64
284	Elf64_Ehdr ehdr;
285	Elf64_Phdr *phdrs, *phdr;
286#else
287	Elf32_Ehdr ehdr;
288	Elf32_Phdr *phdrs, *phdr;
289#endif
290	void *dest;
291	int i;
292
293	memcpy(&ehdr, output, sizeof(ehdr));
294	if (ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
295	   ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
296	   ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
297	   ehdr.e_ident[EI_MAG3] != ELFMAG3)
298		error("Kernel is not a valid ELF file");
299
300	debug_putstr("Parsing ELF... ");
301
302	phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
303	if (!phdrs)
304		error("Failed to allocate space for phdrs");
305
306	memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum);
307
308	for (i = 0; i < ehdr.e_phnum; i++) {
309		phdr = &phdrs[i];
310
311		switch (phdr->p_type) {
312		case PT_LOAD:
313#ifdef CONFIG_X86_64
314			if ((phdr->p_align % 0x200000) != 0)
315				error("Alignment of LOAD segment isn't multiple of 2MB");
316#endif
317#ifdef CONFIG_RELOCATABLE
318			dest = output;
319			dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
320#else
321			dest = (void *)(phdr->p_paddr);
322#endif
323			memmove(dest, output + phdr->p_offset, phdr->p_filesz);
324			break;
325		default: /* Ignore other PT_* */ break;
326		}
327	}
328
329	free(phdrs);
330
331	return ehdr.e_entry - LOAD_PHYSICAL_ADDR;
332}
333
334const unsigned long kernel_text_size = VO___start_rodata - VO__text;
335const unsigned long kernel_inittext_offset = VO__sinittext - VO__text;
336const unsigned long kernel_inittext_size = VO___inittext_end - VO__sinittext;
337const unsigned long kernel_total_size = VO__end - VO__text;
338
339static u8 boot_heap[BOOT_HEAP_SIZE] __aligned(4);
340
341extern unsigned char input_data[];
342extern unsigned int input_len, output_len;
343
344unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr,
345				void (*error)(char *x))
346{
347	unsigned long entry;
348
349	if (!free_mem_ptr) {
350		free_mem_ptr     = (unsigned long)boot_heap;
351		free_mem_end_ptr = (unsigned long)boot_heap + sizeof(boot_heap);
352	}
353
354	if (__decompress(input_data, input_len, NULL, NULL, outbuf, output_len,
355			 NULL, error) < 0)
356		return ULONG_MAX;
357
358	entry = parse_elf(outbuf);
359	handle_relocations(outbuf, output_len, virt_addr);
360
361	return entry;
362}
363
364/*
365 * Set the memory encryption xloadflag based on the mem_encrypt= command line
366 * parameter, if provided.
367 */
368static void parse_mem_encrypt(struct setup_header *hdr)
369{
370	int on = cmdline_find_option_bool("mem_encrypt=on");
371	int off = cmdline_find_option_bool("mem_encrypt=off");
372
373	if (on > off)
374		hdr->xloadflags |= XLF_MEM_ENCRYPTION;
375}
376
377static void early_sev_detect(void)
378{
379	/*
380	 * Accessing video memory causes guest termination because
381	 * the boot stage2 #VC handler of SEV-ES/SNP guests does not
382	 * support MMIO handling and kexec -c adds screen_info to the
383	 * boot parameters passed to the kexec kernel, which causes
384	 * console output to be dumped to both video and serial.
385	 */
386	if (sev_status & MSR_AMD64_SEV_ES_ENABLED)
387		lines = cols = 0;
388}
389
390/*
391 * The compressed kernel image (ZO), has been moved so that its position
392 * is against the end of the buffer used to hold the uncompressed kernel
393 * image (VO) and the execution environment (.bss, .brk), which makes sure
394 * there is room to do the in-place decompression. (See header.S for the
395 * calculations.)
396 *
397 *                             |-----compressed kernel image------|
398 *                             V                                  V
399 * 0                       extract_offset                      +INIT_SIZE
400 * |-----------|---------------|-------------------------|--------|
401 *             |               |                         |        |
402 *           VO__text      startup_32 of ZO          VO__end    ZO__end
403 *             ^                                         ^
404 *             |-------uncompressed kernel image---------|
405 *
406 */
407asmlinkage __visible void *extract_kernel(void *rmode, unsigned char *output)
408{
409	unsigned long virt_addr = LOAD_PHYSICAL_ADDR;
410	memptr heap = (memptr)boot_heap;
411	unsigned long needed_size;
412	size_t entry_offset;
413
414	/* Retain x86 boot parameters pointer passed from startup_32/64. */
415	boot_params_ptr = rmode;
416
417	/* Clear flags intended for solely in-kernel use. */
418	boot_params_ptr->hdr.loadflags &= ~KASLR_FLAG;
419
420	parse_mem_encrypt(&boot_params_ptr->hdr);
421
422	sanitize_boot_params(boot_params_ptr);
423
424	if (boot_params_ptr->screen_info.orig_video_mode == 7) {
425		vidmem = (char *) 0xb0000;
426		vidport = 0x3b4;
427	} else {
428		vidmem = (char *) 0xb8000;
429		vidport = 0x3d4;
430	}
431
432	lines = boot_params_ptr->screen_info.orig_video_lines;
433	cols = boot_params_ptr->screen_info.orig_video_cols;
434
435	init_default_io_ops();
436
437	/*
438	 * Detect TDX guest environment.
439	 *
440	 * It has to be done before console_init() in order to use
441	 * paravirtualized port I/O operations if needed.
442	 */
443	early_tdx_detect();
444
445	early_sev_detect();
446
447	console_init();
448
449	/*
450	 * Save RSDP address for later use. Have this after console_init()
451	 * so that early debugging output from the RSDP parsing code can be
452	 * collected.
453	 */
454	boot_params_ptr->acpi_rsdp_addr = get_rsdp_addr();
455
456	debug_putstr("early console in extract_kernel\n");
457
458	free_mem_ptr     = heap;	/* Heap */
459	free_mem_end_ptr = heap + BOOT_HEAP_SIZE;
460
461	/*
462	 * The memory hole needed for the kernel is the larger of either
463	 * the entire decompressed kernel plus relocation table, or the
464	 * entire decompressed kernel plus .bss and .brk sections.
465	 *
466	 * On X86_64, the memory is mapped with PMD pages. Round the
467	 * size up so that the full extent of PMD pages mapped is
468	 * included in the check against the valid memory table
469	 * entries. This ensures the full mapped area is usable RAM
470	 * and doesn't include any reserved areas.
471	 */
472	needed_size = max_t(unsigned long, output_len, kernel_total_size);
473#ifdef CONFIG_X86_64
474	needed_size = ALIGN(needed_size, MIN_KERNEL_ALIGN);
475#endif
476
477	/* Report initial kernel position details. */
478	debug_putaddr(input_data);
479	debug_putaddr(input_len);
480	debug_putaddr(output);
481	debug_putaddr(output_len);
482	debug_putaddr(kernel_total_size);
483	debug_putaddr(needed_size);
484
485#ifdef CONFIG_X86_64
486	/* Report address of 32-bit trampoline */
487	debug_putaddr(trampoline_32bit);
488#endif
489
490	choose_random_location((unsigned long)input_data, input_len,
491				(unsigned long *)&output,
492				needed_size,
493				&virt_addr);
494
495	/* Validate memory location choices. */
496	if ((unsigned long)output & (MIN_KERNEL_ALIGN - 1))
497		error("Destination physical address inappropriately aligned");
498	if (virt_addr & (MIN_KERNEL_ALIGN - 1))
499		error("Destination virtual address inappropriately aligned");
500#ifdef CONFIG_X86_64
501	if (heap > 0x3fffffffffffUL)
502		error("Destination address too large");
503	if (virt_addr + needed_size > KERNEL_IMAGE_SIZE)
504		error("Destination virtual address is beyond the kernel mapping area");
505#else
506	if (heap > ((-__PAGE_OFFSET-(128<<20)-1) & 0x7fffffff))
507		error("Destination address too large");
508#endif
509#ifndef CONFIG_RELOCATABLE
510	if (virt_addr != LOAD_PHYSICAL_ADDR)
511		error("Destination virtual address changed when not relocatable");
512#endif
513
514	debug_putstr("\nDecompressing Linux... ");
515
516	if (init_unaccepted_memory()) {
517		debug_putstr("Accepting memory... ");
518		accept_memory(__pa(output), needed_size);
519	}
520
521	entry_offset = decompress_kernel(output, virt_addr, error);
522
523	debug_putstr("done.\nBooting the kernel (entry_offset: 0x");
524	debug_puthex(entry_offset);
525	debug_putstr(").\n");
526
527	/* Disable exception handling before booting the kernel */
528	cleanup_exception_handling();
529
530	if (spurious_nmi_count) {
531		error_putstr("Spurious early NMIs ignored: ");
532		error_putdec(spurious_nmi_count);
533		error_putstr("\n");
534	}
535
536	return output + entry_offset;
537}