machine_kexec_file.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * kexec_file for riscv, use vmlinux as the dump-capture kernel image.
  4 *
  5 * Copyright (C) 2021 Huawei Technologies Co, Ltd.
  6 *
  7 * Author: Liao Chang (liaochang1@huawei.com)
  8 */
  9#include <linux/kexec.h>
 10#include <linux/elf.h>
 11#include <linux/slab.h>
 12#include <linux/of.h>
 13#include <linux/libfdt.h>
 14#include <linux/types.h>
 15#include <linux/memblock.h>
 16#include <linux/vmalloc.h>
 17#include <asm/setup.h>
 18#include <asm/insn.h>
 19
 20const struct kexec_file_ops * const kexec_file_loaders[] = {
 21	&elf_kexec_ops,
 22	&image_kexec_ops,
 23	NULL
 24};
 25
 26int arch_kimage_file_post_load_cleanup(struct kimage *image)
 27{
 28	kvfree(image->arch.fdt);
 29	image->arch.fdt = NULL;
 30
 31	vfree(image->elf_headers);
 32	image->elf_headers = NULL;
 33	image->elf_headers_sz = 0;
 34
 35	return kexec_image_post_load_cleanup_default(image);
 36}
 37
 38#ifdef CONFIG_CRASH_DUMP
 39static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
 40{
 41	unsigned int *nr_ranges = arg;
 42
 43	(*nr_ranges)++;
 44	return 0;
 45}
 46
 47static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
 48{
 49	struct crash_mem *cmem = arg;
 50
 51	cmem->ranges[cmem->nr_ranges].start = res->start;
 52	cmem->ranges[cmem->nr_ranges].end = res->end;
 53	cmem->nr_ranges++;
 54
 55	return 0;
 56}
 57
 58static int prepare_elf_headers(void **addr, unsigned long *sz)
 59{
 60	struct crash_mem *cmem;
 61	unsigned int nr_ranges;
 62	int ret;
 63
 64	nr_ranges = 1; /* For exclusion of crashkernel region */
 65	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
 66
 67	cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
 68	if (!cmem)
 69		return -ENOMEM;
 70
 71	cmem->max_nr_ranges = nr_ranges;
 72	cmem->nr_ranges = 0;
 73	ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
 74	if (ret)
 75		goto out;
 76
 77	/* Exclude crashkernel region */
 78	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
 79	if (!ret)
 80		ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
 81
 82out:
 83	kfree(cmem);
 84	return ret;
 85}
 86
 87static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
 88				 unsigned long cmdline_len)
 89{
 90	int elfcorehdr_strlen;
 91	char *cmdline_ptr;
 92
 93	cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
 94	if (!cmdline_ptr)
 95		return NULL;
 96
 97	elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
 98		image->elf_load_addr);
 99
100	if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
101		pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
102		kfree(cmdline_ptr);
103		return NULL;
104	}
105
106	memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
107	/* Ensure it's nul terminated */
108	cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
109	return cmdline_ptr;
110}
111#endif
112
113#define RISCV_IMM_BITS 12
114#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
115#define RISCV_CONST_HIGH_PART(x) \
116	(((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
117#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
118
119#define ENCODE_ITYPE_IMM(x) \
120	(RV_X(x, 0, 12) << 20)
121#define ENCODE_BTYPE_IMM(x) \
122	((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
123	(RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
124#define ENCODE_UTYPE_IMM(x) \
125	(RV_X(x, 12, 20) << 12)
126#define ENCODE_JTYPE_IMM(x) \
127	((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
128	(RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
129#define ENCODE_CBTYPE_IMM(x) \
130	((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
131	(RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
132#define ENCODE_CJTYPE_IMM(x) \
133	((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
134	(RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
135	(RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
136#define ENCODE_UJTYPE_IMM(x) \
137	(ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
138	(ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
139#define ENCODE_UITYPE_IMM(x) \
140	(ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))
141
142#define CLEAN_IMM(type, x) \
143	((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
144
145int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
146				     Elf_Shdr *section,
147				     const Elf_Shdr *relsec,
148				     const Elf_Shdr *symtab)
149{
150	const char *strtab, *name, *shstrtab;
151	const Elf_Shdr *sechdrs;
152	Elf64_Rela *relas;
153	int i, r_type;
154
155	/* String & section header string table */
156	sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
157	strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
158	shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
159
160	relas = (void *)pi->ehdr + relsec->sh_offset;
161
162	for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
163		const Elf_Sym *sym;	/* symbol to relocate */
164		unsigned long addr;	/* final location after relocation */
165		unsigned long val;	/* relocated symbol value */
166		unsigned long sec_base;	/* relocated symbol value */
167		void *loc;		/* tmp location to modify */
168
169		sym = (void *)pi->ehdr + symtab->sh_offset;
170		sym += ELF64_R_SYM(relas[i].r_info);
171
172		if (sym->st_name)
173			name = strtab + sym->st_name;
174		else
175			name = shstrtab + sechdrs[sym->st_shndx].sh_name;
176
177		loc = pi->purgatory_buf;
178		loc += section->sh_offset;
179		loc += relas[i].r_offset;
180
181		if (sym->st_shndx == SHN_ABS)
182			sec_base = 0;
183		else if (sym->st_shndx >= pi->ehdr->e_shnum) {
184			pr_err("Invalid section %d for symbol %s\n",
185			       sym->st_shndx, name);
186			return -ENOEXEC;
187		} else
188			sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
189
190		val = sym->st_value;
191		val += sec_base;
192		val += relas[i].r_addend;
193
194		addr = section->sh_addr + relas[i].r_offset;
195
196		r_type = ELF64_R_TYPE(relas[i].r_info);
197
198		switch (r_type) {
199		case R_RISCV_BRANCH:
200			*(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
201				 ENCODE_BTYPE_IMM(val - addr);
202			break;
203		case R_RISCV_JAL:
204			*(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
205				 ENCODE_JTYPE_IMM(val - addr);
206			break;
207		/*
208		 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
209		 * sym is expected to be next to R_RISCV_PCREL_HI20
210		 * in purgatory relsec. Handle it like R_RISCV_CALL
211		 * sym, instead of searching the whole relsec.
212		 */
213		case R_RISCV_PCREL_HI20:
214		case R_RISCV_CALL_PLT:
215		case R_RISCV_CALL:
216			*(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
217				 ENCODE_UJTYPE_IMM(val - addr);
218			break;
219		case R_RISCV_RVC_BRANCH:
220			*(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
221				 ENCODE_CBTYPE_IMM(val - addr);
222			break;
223		case R_RISCV_RVC_JUMP:
224			*(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
225				 ENCODE_CJTYPE_IMM(val - addr);
226			break;
227		case R_RISCV_ADD16:
228			*(u16 *)loc += val;
229			break;
230		case R_RISCV_SUB16:
231			*(u16 *)loc -= val;
232			break;
233		case R_RISCV_ADD32:
234			*(u32 *)loc += val;
235			break;
236		case R_RISCV_SUB32:
237			*(u32 *)loc -= val;
238			break;
239		/* It has been applied by R_RISCV_PCREL_HI20 sym */
240		case R_RISCV_PCREL_LO12_I:
241		case R_RISCV_ALIGN:
242		case R_RISCV_RELAX:
243			break;
244		case R_RISCV_64:
245			*(u64 *)loc = val;
246			break;
247		default:
248			pr_err("Unknown rela relocation: %d\n", r_type);
249			return -ENOEXEC;
250		}
251	}
252	return 0;
253}
254
255
256int load_extra_segments(struct kimage *image, unsigned long kernel_start,
257			    unsigned long kernel_len, char *initrd,
258			    unsigned long initrd_len, char *cmdline,
259			    unsigned long cmdline_len)
260{
261	int ret;
262	void *fdt;
263	unsigned long initrd_pbase = 0UL;
264	struct kexec_buf kbuf = {};
265	char *modified_cmdline = NULL;
266
267	kbuf.image = image;
268	kbuf.buf_min = kernel_start + kernel_len;
269	kbuf.buf_max = ULONG_MAX;
270
271#ifdef CONFIG_CRASH_DUMP
272	/* Add elfcorehdr */
273	if (image->type == KEXEC_TYPE_CRASH) {
274		void *headers;
275		unsigned long headers_sz;
276		ret = prepare_elf_headers(&headers, &headers_sz);
277		if (ret) {
278			pr_err("Preparing elf core header failed\n");
279			goto out;
280		}
281
282		kbuf.buffer = headers;
283		kbuf.bufsz = headers_sz;
284		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
285		kbuf.memsz = headers_sz;
286		kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
287		kbuf.top_down = true;
288
289		ret = kexec_add_buffer(&kbuf);
290		if (ret) {
291			vfree(headers);
292			goto out;
293		}
294		image->elf_headers = headers;
295		image->elf_load_addr = kbuf.mem;
296		image->elf_headers_sz = headers_sz;
297
298		kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
299			      image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
300
301		/* Setup cmdline for kdump kernel case */
302		modified_cmdline = setup_kdump_cmdline(image, cmdline,
303						       cmdline_len);
304		if (!modified_cmdline) {
305			pr_err("Setting up cmdline for kdump kernel failed\n");
306			ret = -EINVAL;
307			goto out;
308		}
309		cmdline = modified_cmdline;
310	}
311#endif
312
313#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
314	/* Add purgatory to the image */
315	kbuf.top_down = true;
316	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
317	ret = kexec_load_purgatory(image, &kbuf);
318	if (ret) {
319		pr_err("Error loading purgatory ret=%d\n", ret);
320		goto out;
321	}
322	kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem);
323
324	ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
325					     &kernel_start,
326					     sizeof(kernel_start), 0);
327	if (ret)
328		pr_err("Error update purgatory ret=%d\n", ret);
329#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */
330
331	/* Add the initrd to the image */
332	if (initrd != NULL) {
333		kbuf.buffer = initrd;
334		kbuf.bufsz = kbuf.memsz = initrd_len;
335		kbuf.buf_align = PAGE_SIZE;
336		kbuf.top_down = true;
337		kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
338		ret = kexec_add_buffer(&kbuf);
339		if (ret)
340			goto out;
341		initrd_pbase = kbuf.mem;
342		kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase);
343	}
344
345	/* Add the DTB to the image */
346	fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
347					   initrd_len, cmdline, 0);
348	if (!fdt) {
349		pr_err("Error setting up the new device tree.\n");
350		ret = -EINVAL;
351		goto out;
352	}
353
354	fdt_pack(fdt);
355	kbuf.buffer = fdt;
356	kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
357	kbuf.buf_align = PAGE_SIZE;
358	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
359	kbuf.top_down = true;
360	ret = kexec_add_buffer(&kbuf);
361	if (ret) {
362		pr_err("Error add DTB kbuf ret=%d\n", ret);
363		goto out_free_fdt;
364	}
365	/* Cache the fdt buffer address for memory cleanup */
366	image->arch.fdt = fdt;
367	kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem);
368	goto out;
369
370out_free_fdt:
371	kvfree(fdt);
372out:
373	kfree(modified_cmdline);
374	return ret;
375}