kexec_file.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * kexec: kexec_file_load system call
   4 *
   5 * Copyright (C) 2014 Red Hat Inc.
   6 * Authors:
   7 *      Vivek Goyal <vgoyal@redhat.com>
   8 */
   9
  10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  11
  12#include <linux/capability.h>
  13#include <linux/mm.h>
  14#include <linux/file.h>
  15#include <linux/slab.h>
  16#include <linux/kexec.h>
  17#include <linux/memblock.h>
  18#include <linux/mutex.h>
  19#include <linux/list.h>
  20#include <linux/fs.h>
  21#include <linux/ima.h>
  22#include <crypto/sha2.h>
  23#include <linux/elf.h>
  24#include <linux/elfcore.h>
  25#include <linux/kernel.h>
  26#include <linux/kernel_read_file.h>
  27#include <linux/syscalls.h>
  28#include <linux/vmalloc.h>
  29#include <linux/dma-map-ops.h>
  30#include "kexec_internal.h"
  31
  32#ifdef CONFIG_KEXEC_SIG
  33static bool sig_enforce = IS_ENABLED(CONFIG_KEXEC_SIG_FORCE);
  34
  35void set_kexec_sig_enforced(void)
  36{
  37	sig_enforce = true;
  38}
  39#endif
  40
  41#ifdef CONFIG_IMA_KEXEC
  42static bool check_ima_segment_index(struct kimage *image, int i)
  43{
  44	if (image->is_ima_segment_index_set && i == image->ima_segment_index)
  45		return true;
  46	else
  47		return false;
  48}
  49#else
  50static bool check_ima_segment_index(struct kimage *image, int i)
  51{
  52	return false;
  53}
  54#endif
  55
  56static int kexec_calculate_store_digests(struct kimage *image);
  57
  58/* Maximum size in bytes for kernel/initrd files. */
  59#define KEXEC_FILE_SIZE_MAX	min_t(s64, 4LL << 30, SSIZE_MAX)
  60
  61/*
  62 * Currently this is the only default function that is exported as some
  63 * architectures need it to do additional handlings.
  64 * In the future, other default functions may be exported too if required.
  65 */
  66int kexec_image_probe_default(struct kimage *image, void *buf,
  67			      unsigned long buf_len)
  68{
  69	const struct kexec_file_ops * const *fops;
  70	int ret = -ENOEXEC;
  71
  72	for (fops = &kexec_file_loaders[0]; *fops && (*fops)->probe; ++fops) {
  73		ret = (*fops)->probe(buf, buf_len);
  74		if (!ret) {
  75			image->fops = *fops;
  76			return ret;
  77		}
  78	}
  79
  80	return ret;
  81}
  82
  83static void *kexec_image_load_default(struct kimage *image)
  84{
  85	if (!image->fops || !image->fops->load)
  86		return ERR_PTR(-ENOEXEC);
  87
  88	return image->fops->load(image, image->kernel_buf,
  89				 image->kernel_buf_len, image->initrd_buf,
  90				 image->initrd_buf_len, image->cmdline_buf,
  91				 image->cmdline_buf_len);
  92}
  93
  94int kexec_image_post_load_cleanup_default(struct kimage *image)
  95{
  96	if (!image->fops || !image->fops->cleanup)
  97		return 0;
  98
  99	return image->fops->cleanup(image->image_loader_data);
 100}
 101
 102/*
 103 * Free up memory used by kernel, initrd, and command line. This is temporary
 104 * memory allocation which is not needed any more after these buffers have
 105 * been loaded into separate segments and have been copied elsewhere.
 106 */
 107void kimage_file_post_load_cleanup(struct kimage *image)
 108{
 109	struct purgatory_info *pi = &image->purgatory_info;
 110
 111	vfree(image->kernel_buf);
 112	image->kernel_buf = NULL;
 113
 114	vfree(image->initrd_buf);
 115	image->initrd_buf = NULL;
 116
 117	kfree(image->cmdline_buf);
 118	image->cmdline_buf = NULL;
 119
 120	vfree(pi->purgatory_buf);
 121	pi->purgatory_buf = NULL;
 122
 123	vfree(pi->sechdrs);
 124	pi->sechdrs = NULL;
 125
 126#ifdef CONFIG_IMA_KEXEC
 127	vfree(image->ima_buffer);
 128	image->ima_buffer = NULL;
 129#endif /* CONFIG_IMA_KEXEC */
 130
 131	/* See if architecture has anything to cleanup post load */
 132	arch_kimage_file_post_load_cleanup(image);
 133
 134	/*
 135	 * Above call should have called into bootloader to free up
 136	 * any data stored in kimage->image_loader_data. It should
 137	 * be ok now to free it up.
 138	 */
 139	kfree(image->image_loader_data);
 140	image->image_loader_data = NULL;
 141
 142	kexec_file_dbg_print = false;
 143}
 144
 145#ifdef CONFIG_KEXEC_SIG
 146#ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION
 147int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len)
 148{
 149	int ret;
 150
 151	ret = verify_pefile_signature(kernel, kernel_len,
 152				      VERIFY_USE_SECONDARY_KEYRING,
 153				      VERIFYING_KEXEC_PE_SIGNATURE);
 154	if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) {
 155		ret = verify_pefile_signature(kernel, kernel_len,
 156					      VERIFY_USE_PLATFORM_KEYRING,
 157					      VERIFYING_KEXEC_PE_SIGNATURE);
 158	}
 159	return ret;
 160}
 161#endif
 162
 163static int kexec_image_verify_sig(struct kimage *image, void *buf,
 164				  unsigned long buf_len)
 165{
 166	if (!image->fops || !image->fops->verify_sig) {
 167		pr_debug("kernel loader does not support signature verification.\n");
 168		return -EKEYREJECTED;
 169	}
 170
 171	return image->fops->verify_sig(buf, buf_len);
 172}
 173
 174static int
 175kimage_validate_signature(struct kimage *image)
 176{
 177	int ret;
 178
 179	ret = kexec_image_verify_sig(image, image->kernel_buf,
 180				     image->kernel_buf_len);
 181	if (ret) {
 182
 183		if (sig_enforce) {
 184			pr_notice("Enforced kernel signature verification failed (%d).\n", ret);
 185			return ret;
 186		}
 187
 188		/*
 189		 * If IMA is guaranteed to appraise a signature on the kexec
 190		 * image, permit it even if the kernel is otherwise locked
 191		 * down.
 192		 */
 193		if (!ima_appraise_signature(READING_KEXEC_IMAGE) &&
 194		    security_locked_down(LOCKDOWN_KEXEC))
 195			return -EPERM;
 196
 197		pr_debug("kernel signature verification failed (%d).\n", ret);
 198	}
 199
 200	return 0;
 201}
 202#endif
 203
 204static int kexec_post_load(struct kimage *image, unsigned long flags)
 205{
 206#ifdef CONFIG_IMA_KEXEC
 207	if (!(flags & KEXEC_FILE_ON_CRASH))
 208		ima_kexec_post_load(image);
 209#endif
 210	return machine_kexec_post_load(image);
 211}
 212
 213/*
 214 * In file mode list of segments is prepared by kernel. Copy relevant
 215 * data from user space, do error checking, prepare segment list
 216 */
 217static int
 218kimage_file_prepare_segments(struct kimage *image, int kernel_fd, int initrd_fd,
 219			     const char __user *cmdline_ptr,
 220			     unsigned long cmdline_len, unsigned flags)
 221{
 222	ssize_t ret;
 223	void *ldata;
 224
 225	ret = kernel_read_file_from_fd(kernel_fd, 0, &image->kernel_buf,
 226				       KEXEC_FILE_SIZE_MAX, NULL,
 227				       READING_KEXEC_IMAGE);
 228	if (ret < 0)
 229		return ret;
 230	image->kernel_buf_len = ret;
 231	kexec_dprintk("kernel: %p kernel_size: %#lx\n",
 232		      image->kernel_buf, image->kernel_buf_len);
 233
 234	/* Call arch image probe handlers */
 235	ret = arch_kexec_kernel_image_probe(image, image->kernel_buf,
 236					    image->kernel_buf_len);
 237	if (ret)
 238		goto out;
 239
 240#ifdef CONFIG_KEXEC_SIG
 241	ret = kimage_validate_signature(image);
 242
 243	if (ret)
 244		goto out;
 245#endif
 246	/* It is possible that there no initramfs is being loaded */
 247	if (!(flags & KEXEC_FILE_NO_INITRAMFS)) {
 248		ret = kernel_read_file_from_fd(initrd_fd, 0, &image->initrd_buf,
 249					       KEXEC_FILE_SIZE_MAX, NULL,
 250					       READING_KEXEC_INITRAMFS);
 251		if (ret < 0)
 252			goto out;
 253		image->initrd_buf_len = ret;
 254		ret = 0;
 255	}
 256
 257	image->no_cma = !!(flags & KEXEC_FILE_NO_CMA);
 258	image->force_dtb = flags & KEXEC_FILE_FORCE_DTB;
 259
 260	if (cmdline_len) {
 261		image->cmdline_buf = memdup_user(cmdline_ptr, cmdline_len);
 262		if (IS_ERR(image->cmdline_buf)) {
 263			ret = PTR_ERR(image->cmdline_buf);
 264			image->cmdline_buf = NULL;
 265			goto out;
 266		}
 267
 268		image->cmdline_buf_len = cmdline_len;
 269
 270		/* command line should be a string with last byte null */
 271		if (image->cmdline_buf[cmdline_len - 1] != '\0') {
 272			ret = -EINVAL;
 273			goto out;
 274		}
 275
 276		ima_kexec_cmdline(kernel_fd, image->cmdline_buf,
 277				  image->cmdline_buf_len - 1);
 278	}
 279
 280	/* IMA needs to pass the measurement list to the next kernel. */
 281	ima_add_kexec_buffer(image);
 282
 283	/* If KHO is active, add its images to the list */
 284	ret = kho_fill_kimage(image);
 285	if (ret)
 286		goto out;
 287
 288	/* Call image load handler */
 289	ldata = kexec_image_load_default(image);
 290
 291	if (IS_ERR(ldata)) {
 292		ret = PTR_ERR(ldata);
 293		goto out;
 294	}
 295
 296	image->image_loader_data = ldata;
 297out:
 298	/* In case of error, free up all allocated memory in this function */
 299	if (ret)
 300		kimage_file_post_load_cleanup(image);
 301	return ret;
 302}
 303
 304static int
 305kimage_file_alloc_init(struct kimage **rimage, int kernel_fd,
 306		       int initrd_fd, const char __user *cmdline_ptr,
 307		       unsigned long cmdline_len, unsigned long flags)
 308{
 309	int ret;
 310	struct kimage *image;
 311	bool kexec_on_panic = flags & KEXEC_FILE_ON_CRASH;
 312
 313	image = do_kimage_alloc_init();
 314	if (!image)
 315		return -ENOMEM;
 316
 317	kexec_file_dbg_print = !!(flags & KEXEC_FILE_DEBUG);
 318	image->file_mode = 1;
 319
 320#ifdef CONFIG_CRASH_DUMP
 321	if (kexec_on_panic) {
 322		/* Enable special crash kernel control page alloc policy. */
 323		image->control_page = crashk_res.start;
 324		image->type = KEXEC_TYPE_CRASH;
 325	}
 326#endif
 327
 328	ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd,
 329					   cmdline_ptr, cmdline_len, flags);
 330	if (ret)
 331		goto out_free_image;
 332
 333	ret = sanity_check_segment_list(image);
 334	if (ret)
 335		goto out_free_post_load_bufs;
 336
 337	ret = -ENOMEM;
 338	image->control_code_page = kimage_alloc_control_pages(image,
 339					   get_order(KEXEC_CONTROL_PAGE_SIZE));
 340	if (!image->control_code_page) {
 341		pr_err("Could not allocate control_code_buffer\n");
 342		goto out_free_post_load_bufs;
 343	}
 344
 345	if (!kexec_on_panic) {
 346		image->swap_page = kimage_alloc_control_pages(image, 0);
 347		if (!image->swap_page) {
 348			pr_err("Could not allocate swap buffer\n");
 349			goto out_free_control_pages;
 350		}
 351	}
 352
 353	*rimage = image;
 354	return 0;
 355out_free_control_pages:
 356	kimage_free_page_list(&image->control_pages);
 357out_free_post_load_bufs:
 358	kimage_file_post_load_cleanup(image);
 359out_free_image:
 360	kfree(image);
 361	return ret;
 362}
 363
 364SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd,
 365		unsigned long, cmdline_len, const char __user *, cmdline_ptr,
 366		unsigned long, flags)
 367{
 368	int image_type = (flags & KEXEC_FILE_ON_CRASH) ?
 369			 KEXEC_TYPE_CRASH : KEXEC_TYPE_DEFAULT;
 370	struct kimage **dest_image, *image;
 371	int ret = 0, i;
 372
 373	/* We only trust the superuser with rebooting the system. */
 374	if (!kexec_load_permitted(image_type))
 375		return -EPERM;
 376
 377	/* Make sure we have a legal set of flags */
 378	if (flags != (flags & KEXEC_FILE_FLAGS))
 379		return -EINVAL;
 380
 381	image = NULL;
 382
 383	if (!kexec_trylock())
 384		return -EBUSY;
 385
 386#ifdef CONFIG_CRASH_DUMP
 387	if (image_type == KEXEC_TYPE_CRASH) {
 388		dest_image = &kexec_crash_image;
 389		if (kexec_crash_image)
 390			arch_kexec_unprotect_crashkres();
 391	} else
 392#endif
 393		dest_image = &kexec_image;
 394
 395	if (flags & KEXEC_FILE_UNLOAD)
 396		goto exchange;
 397
 398	/*
 399	 * In case of crash, new kernel gets loaded in reserved region. It is
 400	 * same memory where old crash kernel might be loaded. Free any
 401	 * current crash dump kernel before we corrupt it.
 402	 */
 403	if (flags & KEXEC_FILE_ON_CRASH)
 404		kimage_free(xchg(&kexec_crash_image, NULL));
 405
 406	ret = kimage_file_alloc_init(&image, kernel_fd, initrd_fd, cmdline_ptr,
 407				     cmdline_len, flags);
 408	if (ret)
 409		goto out;
 410
 411#ifdef CONFIG_CRASH_HOTPLUG
 412	if ((flags & KEXEC_FILE_ON_CRASH) && arch_crash_hotplug_support(image, flags))
 413		image->hotplug_support = 1;
 414#endif
 415
 416	ret = machine_kexec_prepare(image);
 417	if (ret)
 418		goto out;
 419
 420	/*
 421	 * Some architecture(like S390) may touch the crash memory before
 422	 * machine_kexec_prepare(), we must copy vmcoreinfo data after it.
 423	 */
 424	ret = kimage_crash_copy_vmcoreinfo(image);
 425	if (ret)
 426		goto out;
 427
 428	ret = kexec_calculate_store_digests(image);
 429	if (ret)
 430		goto out;
 431
 432	kexec_dprintk("nr_segments = %lu\n", image->nr_segments);
 433	for (i = 0; i < image->nr_segments; i++) {
 434		struct kexec_segment *ksegment;
 435
 436		ksegment = &image->segment[i];
 437		kexec_dprintk("segment[%d]: buf=0x%p bufsz=0x%zx mem=0x%lx memsz=0x%zx\n",
 438			      i, ksegment->buf, ksegment->bufsz, ksegment->mem,
 439			      ksegment->memsz);
 440
 441		ret = kimage_load_segment(image, i);
 442		if (ret)
 443			goto out;
 444	}
 445
 446	kimage_terminate(image);
 447
 448	ret = kexec_post_load(image, flags);
 449	if (ret)
 450		goto out;
 451
 452	kexec_dprintk("kexec_file_load: type:%u, start:0x%lx head:0x%lx flags:0x%lx\n",
 453		      image->type, image->start, image->head, flags);
 454	/*
 455	 * Free up any temporary buffers allocated which are not needed
 456	 * after image has been loaded
 457	 */
 458	kimage_file_post_load_cleanup(image);
 459exchange:
 460	image = xchg(dest_image, image);
 461out:
 462#ifdef CONFIG_CRASH_DUMP
 463	if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image)
 464		arch_kexec_protect_crashkres();
 465#endif
 466
 467	kexec_unlock();
 468	kimage_free(image);
 469	return ret;
 470}
 471
 472static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
 473				    struct kexec_buf *kbuf)
 474{
 475	struct kimage *image = kbuf->image;
 476	unsigned long temp_start, temp_end;
 477
 478	temp_end = min(end, kbuf->buf_max);
 479	temp_start = temp_end - kbuf->memsz + 1;
 480	kexec_random_range_start(temp_start, temp_end, kbuf, &temp_start);
 481
 482	do {
 483		/* align down start */
 484		temp_start = ALIGN_DOWN(temp_start, kbuf->buf_align);
 485
 486		if (temp_start < start || temp_start < kbuf->buf_min)
 487			return 0;
 488
 489		temp_end = temp_start + kbuf->memsz - 1;
 490
 491		/*
 492		 * Make sure this does not conflict with any of existing
 493		 * segments
 494		 */
 495		if (kimage_is_destination_range(image, temp_start, temp_end)) {
 496			temp_start = temp_start - PAGE_SIZE;
 497			continue;
 498		}
 499
 500		/* Make sure this does not conflict with exclude range */
 501		if (arch_check_excluded_range(image, temp_start, temp_end)) {
 502			temp_start = temp_start - PAGE_SIZE;
 503			continue;
 504		}
 505
 506		/* We found a suitable memory range */
 507		break;
 508	} while (1);
 509
 510	/* If we are here, we found a suitable memory range */
 511	kbuf->mem = temp_start;
 512
 513	/* Success, stop navigating through remaining System RAM ranges */
 514	return 1;
 515}
 516
 517static int locate_mem_hole_bottom_up(unsigned long start, unsigned long end,
 518				     struct kexec_buf *kbuf)
 519{
 520	struct kimage *image = kbuf->image;
 521	unsigned long temp_start, temp_end;
 522
 523	temp_start = max(start, kbuf->buf_min);
 524
 525	kexec_random_range_start(temp_start, end, kbuf, &temp_start);
 526
 527	do {
 528		temp_start = ALIGN(temp_start, kbuf->buf_align);
 529		temp_end = temp_start + kbuf->memsz - 1;
 530
 531		if (temp_end > end || temp_end > kbuf->buf_max)
 532			return 0;
 533		/*
 534		 * Make sure this does not conflict with any of existing
 535		 * segments
 536		 */
 537		if (kimage_is_destination_range(image, temp_start, temp_end)) {
 538			temp_start = temp_start + PAGE_SIZE;
 539			continue;
 540		}
 541
 542		/* Make sure this does not conflict with exclude range */
 543		if (arch_check_excluded_range(image, temp_start, temp_end)) {
 544			temp_start = temp_start + PAGE_SIZE;
 545			continue;
 546		}
 547
 548		/* We found a suitable memory range */
 549		break;
 550	} while (1);
 551
 552	/* If we are here, we found a suitable memory range */
 553	kbuf->mem = temp_start;
 554
 555	/* Success, stop navigating through remaining System RAM ranges */
 556	return 1;
 557}
 558
 559static int locate_mem_hole_callback(struct resource *res, void *arg)
 560{
 561	struct kexec_buf *kbuf = (struct kexec_buf *)arg;
 562	u64 start = res->start, end = res->end;
 563	unsigned long sz = end - start + 1;
 564
 565	/* Returning 0 will take to next memory range */
 566
 567	/* Don't use memory that will be detected and handled by a driver. */
 568	if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED)
 569		return 0;
 570
 571	if (sz < kbuf->memsz)
 572		return 0;
 573
 574	if (end < kbuf->buf_min || start > kbuf->buf_max)
 575		return 0;
 576
 577	/*
 578	 * Allocate memory top down with-in ram range. Otherwise bottom up
 579	 * allocation.
 580	 */
 581	if (kbuf->top_down)
 582		return locate_mem_hole_top_down(start, end, kbuf);
 583	return locate_mem_hole_bottom_up(start, end, kbuf);
 584}
 585
 586#ifdef CONFIG_ARCH_KEEP_MEMBLOCK
 587static int kexec_walk_memblock(struct kexec_buf *kbuf,
 588			       int (*func)(struct resource *, void *))
 589{
 590	int ret = 0;
 591	u64 i;
 592	phys_addr_t mstart, mend;
 593	struct resource res = { };
 594
 595#ifdef CONFIG_CRASH_DUMP
 596	if (kbuf->image->type == KEXEC_TYPE_CRASH)
 597		return func(&crashk_res, kbuf);
 598#endif
 599
 600	/*
 601	 * Using MEMBLOCK_NONE will properly skip MEMBLOCK_DRIVER_MANAGED. See
 602	 * IORESOURCE_SYSRAM_DRIVER_MANAGED handling in
 603	 * locate_mem_hole_callback().
 604	 */
 605	if (kbuf->top_down) {
 606		for_each_free_mem_range_reverse(i, NUMA_NO_NODE, MEMBLOCK_NONE,
 607						&mstart, &mend, NULL) {
 608			/*
 609			 * In memblock, end points to the first byte after the
 610			 * range while in kexec, end points to the last byte
 611			 * in the range.
 612			 */
 613			res.start = mstart;
 614			res.end = mend - 1;
 615			ret = func(&res, kbuf);
 616			if (ret)
 617				break;
 618		}
 619	} else {
 620		for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
 621					&mstart, &mend, NULL) {
 622			/*
 623			 * In memblock, end points to the first byte after the
 624			 * range while in kexec, end points to the last byte
 625			 * in the range.
 626			 */
 627			res.start = mstart;
 628			res.end = mend - 1;
 629			ret = func(&res, kbuf);
 630			if (ret)
 631				break;
 632		}
 633	}
 634
 635	return ret;
 636}
 637#else
 638static int kexec_walk_memblock(struct kexec_buf *kbuf,
 639			       int (*func)(struct resource *, void *))
 640{
 641	return 0;
 642}
 643#endif
 644
 645/**
 646 * kexec_walk_resources - call func(data) on free memory regions
 647 * @kbuf:	Context info for the search. Also passed to @func.
 648 * @func:	Function to call for each memory region.
 649 *
 650 * Return: The memory walk will stop when func returns a non-zero value
 651 * and that value will be returned. If all free regions are visited without
 652 * func returning non-zero, then zero will be returned.
 653 */
 654static int kexec_walk_resources(struct kexec_buf *kbuf,
 655				int (*func)(struct resource *, void *))
 656{
 657#ifdef CONFIG_CRASH_DUMP
 658	if (kbuf->image->type == KEXEC_TYPE_CRASH)
 659		return walk_iomem_res_desc(crashk_res.desc,
 660					   IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
 661					   crashk_res.start, crashk_res.end,
 662					   kbuf, func);
 663#endif
 664	if (kbuf->top_down)
 665		return walk_system_ram_res_rev(0, ULONG_MAX, kbuf, func);
 666	else
 667		return walk_system_ram_res(0, ULONG_MAX, kbuf, func);
 668}
 669
 670static int kexec_alloc_contig(struct kexec_buf *kbuf)
 671{
 672	size_t nr_pages = kbuf->memsz >> PAGE_SHIFT;
 673	unsigned long mem;
 674	struct page *p;
 675
 676	/* User space disabled CMA allocations, bail out. */
 677	if (kbuf->image->no_cma)
 678		return -EPERM;
 679
 680	/* Skip CMA logic for crash kernel */
 681	if (kbuf->image->type == KEXEC_TYPE_CRASH)
 682		return -EPERM;
 683
 684	p = dma_alloc_from_contiguous(NULL, nr_pages, get_order(kbuf->buf_align), true);
 685	if (!p)
 686		return -ENOMEM;
 687
 688	pr_debug("allocated %zu DMA pages at 0x%lx", nr_pages, page_to_boot_pfn(p));
 689
 690	mem = page_to_boot_pfn(p) << PAGE_SHIFT;
 691
 692	if (kimage_is_destination_range(kbuf->image, mem, mem + kbuf->memsz)) {
 693		/* Our region is already in use by a statically defined one. Bail out. */
 694		pr_debug("CMA overlaps existing mem: 0x%lx+0x%lx\n", mem, kbuf->memsz);
 695		dma_release_from_contiguous(NULL, p, nr_pages);
 696		return -EBUSY;
 697	}
 698
 699	kbuf->mem = page_to_boot_pfn(p) << PAGE_SHIFT;
 700	kbuf->cma = p;
 701
 702	arch_kexec_post_alloc_pages(page_address(p), (int)nr_pages, 0);
 703
 704	return 0;
 705}
 706
 707/**
 708 * kexec_locate_mem_hole - find free memory for the purgatory or the next kernel
 709 * @kbuf:	Parameters for the memory search.
 710 *
 711 * On success, kbuf->mem will have the start address of the memory region found.
 712 *
 713 * Return: 0 on success, negative errno on error.
 714 */
 715int kexec_locate_mem_hole(struct kexec_buf *kbuf)
 716{
 717	int ret;
 718
 719	/* Arch knows where to place */
 720	if (kbuf->mem != KEXEC_BUF_MEM_UNKNOWN)
 721		return 0;
 722
 723	/*
 724	 * If KHO is active, only use KHO scratch memory. All other memory
 725	 * could potentially be handed over.
 726	 */
 727	ret = kho_locate_mem_hole(kbuf, locate_mem_hole_callback);
 728	if (ret <= 0)
 729		return ret;
 730
 731	/*
 732	 * Try to find a free physically contiguous block of memory first. With that, we
 733	 * can avoid any copying at kexec time.
 734	 */
 735	if (!kexec_alloc_contig(kbuf))
 736		return 0;
 737
 738	if (!IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
 739		ret = kexec_walk_resources(kbuf, locate_mem_hole_callback);
 740	else
 741		ret = kexec_walk_memblock(kbuf, locate_mem_hole_callback);
 742
 743	return ret == 1 ? 0 : -EADDRNOTAVAIL;
 744}
 745
 746/**
 747 * kexec_add_buffer - place a buffer in a kexec segment
 748 * @kbuf:	Buffer contents and memory parameters.
 749 *
 750 * This function assumes that kexec_lock is held.
 751 * On successful return, @kbuf->mem will have the physical address of
 752 * the buffer in memory.
 753 *
 754 * Return: 0 on success, negative errno on error.
 755 */
 756int kexec_add_buffer(struct kexec_buf *kbuf)
 757{
 758	struct kexec_segment *ksegment;
 759	int ret;
 760
 761	/* Currently adding segment this way is allowed only in file mode */
 762	if (!kbuf->image->file_mode)
 763		return -EINVAL;
 764
 765	if (kbuf->image->nr_segments >= KEXEC_SEGMENT_MAX)
 766		return -EINVAL;
 767
 768	/*
 769	 * Make sure we are not trying to add buffer after allocating
 770	 * control pages. All segments need to be placed first before
 771	 * any control pages are allocated. As control page allocation
 772	 * logic goes through list of segments to make sure there are
 773	 * no destination overlaps.
 774	 */
 775	if (!list_empty(&kbuf->image->control_pages)) {
 776		WARN_ON(1);
 777		return -EINVAL;
 778	}
 779
 780	/* Ensure minimum alignment needed for segments. */
 781	kbuf->memsz = ALIGN(kbuf->memsz, PAGE_SIZE);
 782	kbuf->buf_align = max(kbuf->buf_align, PAGE_SIZE);
 783	kbuf->cma = NULL;
 784
 785	/* Walk the RAM ranges and allocate a suitable range for the buffer */
 786	ret = arch_kexec_locate_mem_hole(kbuf);
 787	if (ret)
 788		return ret;
 789
 790	/* Found a suitable memory range */
 791	ksegment = &kbuf->image->segment[kbuf->image->nr_segments];
 792	ksegment->kbuf = kbuf->buffer;
 793	ksegment->bufsz = kbuf->bufsz;
 794	ksegment->mem = kbuf->mem;
 795	ksegment->memsz = kbuf->memsz;
 796	kbuf->image->segment_cma[kbuf->image->nr_segments] = kbuf->cma;
 797	kbuf->image->nr_segments++;
 798	return 0;
 799}
 800
 801/* Calculate and store the digest of segments */
 802static int kexec_calculate_store_digests(struct kimage *image)
 803{
 804	struct sha256_ctx sctx;
 805	int ret = 0, i, j, zero_buf_sz, sha_region_sz;
 806	size_t nullsz;
 807	u8 digest[SHA256_DIGEST_SIZE];
 808	void *zero_buf;
 809	struct kexec_sha_region *sha_regions;
 810	struct purgatory_info *pi = &image->purgatory_info;
 811
 812	if (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY))
 813		return 0;
 814
 815	zero_buf = __va(page_to_pfn(ZERO_PAGE(0)) << PAGE_SHIFT);
 816	zero_buf_sz = PAGE_SIZE;
 817
 818	sha_region_sz = KEXEC_SEGMENT_MAX * sizeof(struct kexec_sha_region);
 819	sha_regions = vzalloc(sha_region_sz);
 820	if (!sha_regions)
 821		return -ENOMEM;
 822
 823	sha256_init(&sctx);
 824
 825	for (j = i = 0; i < image->nr_segments; i++) {
 826		struct kexec_segment *ksegment;
 827
 828#ifdef CONFIG_CRASH_HOTPLUG
 829		/* Exclude elfcorehdr segment to allow future changes via hotplug */
 830		if (i == image->elfcorehdr_index)
 831			continue;
 832#endif
 833
 834		ksegment = &image->segment[i];
 835		/*
 836		 * Skip purgatory as it will be modified once we put digest
 837		 * info in purgatory.
 838		 */
 839		if (ksegment->kbuf == pi->purgatory_buf)
 840			continue;
 841
 842		/*
 843		 * Skip the segment if ima_segment_index is set and matches
 844		 * the current index
 845		 */
 846		if (check_ima_segment_index(image, i))
 847			continue;
 848
 849		sha256_update(&sctx, ksegment->kbuf, ksegment->bufsz);
 850
 851		/*
 852		 * Assume rest of the buffer is filled with zero and
 853		 * update digest accordingly.
 854		 */
 855		nullsz = ksegment->memsz - ksegment->bufsz;
 856		while (nullsz) {
 857			unsigned long bytes = nullsz;
 858
 859			if (bytes > zero_buf_sz)
 860				bytes = zero_buf_sz;
 861			sha256_update(&sctx, zero_buf, bytes);
 862			nullsz -= bytes;
 863		}
 864
 865		sha_regions[j].start = ksegment->mem;
 866		sha_regions[j].len = ksegment->memsz;
 867		j++;
 868	}
 869
 870	sha256_final(&sctx, digest);
 871
 872	ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions",
 873					     sha_regions, sha_region_sz, 0);
 874	if (ret)
 875		goto out_free_sha_regions;
 876
 877	ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha256_digest",
 878					     digest, SHA256_DIGEST_SIZE, 0);
 879out_free_sha_regions:
 880	vfree(sha_regions);
 881	return ret;
 882}
 883
 884#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
 885/*
 886 * kexec_purgatory_find_symbol - find a symbol in the purgatory
 887 * @pi:		Purgatory to search in.
 888 * @name:	Name of the symbol.
 889 *
 890 * Return: pointer to symbol in read-only symtab on success, NULL on error.
 891 */
 892static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi,
 893						  const char *name)
 894{
 895	const Elf_Shdr *sechdrs;
 896	const Elf_Ehdr *ehdr;
 897	const Elf_Sym *syms;
 898	const char *strtab;
 899	int i, k;
 900
 901	if (!pi->ehdr)
 902		return NULL;
 903
 904	ehdr = pi->ehdr;
 905	sechdrs = (void *)ehdr + ehdr->e_shoff;
 906
 907	for (i = 0; i < ehdr->e_shnum; i++) {
 908		if (sechdrs[i].sh_type != SHT_SYMTAB)
 909			continue;
 910
 911		if (sechdrs[i].sh_link >= ehdr->e_shnum)
 912			/* Invalid strtab section number */
 913			continue;
 914		strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset;
 915		syms = (void *)ehdr + sechdrs[i].sh_offset;
 916
 917		/* Go through symbols for a match */
 918		for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) {
 919			if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL)
 920				continue;
 921
 922			if (strcmp(strtab + syms[k].st_name, name) != 0)
 923				continue;
 924
 925			if (syms[k].st_shndx == SHN_UNDEF ||
 926			    syms[k].st_shndx >= ehdr->e_shnum) {
 927				pr_debug("Symbol: %s has bad section index %d.\n",
 928					name, syms[k].st_shndx);
 929				return NULL;
 930			}
 931
 932			/* Found the symbol we are looking for */
 933			return &syms[k];
 934		}
 935	}
 936
 937	return NULL;
 938}
 939/*
 940 * kexec_purgatory_setup_kbuf - prepare buffer to load purgatory.
 941 * @pi:		Purgatory to be loaded.
 942 * @kbuf:	Buffer to setup.
 943 *
 944 * Allocates the memory needed for the buffer. Caller is responsible to free
 945 * the memory after use.
 946 *
 947 * Return: 0 on success, negative errno on error.
 948 */
 949static int kexec_purgatory_setup_kbuf(struct purgatory_info *pi,
 950				      struct kexec_buf *kbuf)
 951{
 952	const Elf_Shdr *sechdrs;
 953	unsigned long bss_align;
 954	unsigned long bss_sz;
 955	unsigned long align;
 956	int i, ret;
 957
 958	sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
 959	kbuf->buf_align = bss_align = 1;
 960	kbuf->bufsz = bss_sz = 0;
 961
 962	for (i = 0; i < pi->ehdr->e_shnum; i++) {
 963		if (!(sechdrs[i].sh_flags & SHF_ALLOC))
 964			continue;
 965
 966		align = sechdrs[i].sh_addralign;
 967		if (sechdrs[i].sh_type != SHT_NOBITS) {
 968			if (kbuf->buf_align < align)
 969				kbuf->buf_align = align;
 970			kbuf->bufsz = ALIGN(kbuf->bufsz, align);
 971			kbuf->bufsz += sechdrs[i].sh_size;
 972		} else {
 973			if (bss_align < align)
 974				bss_align = align;
 975			bss_sz = ALIGN(bss_sz, align);
 976			bss_sz += sechdrs[i].sh_size;
 977		}
 978	}
 979	kbuf->bufsz = ALIGN(kbuf->bufsz, bss_align);
 980	kbuf->memsz = kbuf->bufsz + bss_sz;
 981	if (kbuf->buf_align < bss_align)
 982		kbuf->buf_align = bss_align;
 983
 984	kbuf->buffer = vzalloc(kbuf->bufsz);
 985	if (!kbuf->buffer)
 986		return -ENOMEM;
 987	pi->purgatory_buf = kbuf->buffer;
 988
 989	ret = kexec_add_buffer(kbuf);
 990	if (ret)
 991		goto out;
 992
 993	return 0;
 994out:
 995	vfree(pi->purgatory_buf);
 996	pi->purgatory_buf = NULL;
 997	return ret;
 998}
 999
1000/*
1001 * kexec_purgatory_setup_sechdrs - prepares the pi->sechdrs buffer.
1002 * @pi:		Purgatory to be loaded.
1003 * @kbuf:	Buffer prepared to store purgatory.
1004 *
1005 * Allocates the memory needed for the buffer. Caller is responsible to free
1006 * the memory after use.
1007 *
1008 * Return: 0 on success, negative errno on error.
1009 */
1010static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi,
1011					 struct kexec_buf *kbuf)
1012{
1013	unsigned long bss_addr;
1014	unsigned long offset;
1015	size_t sechdrs_size;
1016	Elf_Shdr *sechdrs;
1017	const Elf_Sym *entry_sym;
1018	u16 entry_shndx = 0;
1019	unsigned long entry_off = 0;
1020	bool start_fixed = false;
1021	int i;
1022
1023	/*
1024	 * The section headers in kexec_purgatory are read-only. In order to
1025	 * have them modifiable make a temporary copy.
1026	 */
1027	sechdrs_size = array_size(sizeof(Elf_Shdr), pi->ehdr->e_shnum);
1028	sechdrs = vzalloc(sechdrs_size);
1029	if (!sechdrs)
1030		return -ENOMEM;
1031	memcpy(sechdrs, (void *)pi->ehdr + pi->ehdr->e_shoff, sechdrs_size);
1032	pi->sechdrs = sechdrs;
1033
1034	offset = 0;
1035	bss_addr = kbuf->mem + kbuf->bufsz;
1036	kbuf->image->start = pi->ehdr->e_entry;
1037
1038	entry_sym = kexec_purgatory_find_symbol(pi, "purgatory_start");
1039	if (entry_sym) {
1040		entry_shndx = entry_sym->st_shndx;
1041		entry_off = entry_sym->st_value;
1042	}
1043
1044	for (i = 0; i < pi->ehdr->e_shnum; i++) {
1045		unsigned long align;
1046		void *src, *dst;
1047
1048		if (!(sechdrs[i].sh_flags & SHF_ALLOC))
1049			continue;
1050
1051		align = sechdrs[i].sh_addralign;
1052		if (sechdrs[i].sh_type == SHT_NOBITS) {
1053			bss_addr = ALIGN(bss_addr, align);
1054			sechdrs[i].sh_addr = bss_addr;
1055			bss_addr += sechdrs[i].sh_size;
1056			continue;
1057		}
1058
1059		offset = ALIGN(offset, align);
1060
1061		if (!start_fixed && entry_sym && i == entry_shndx &&
1062		    (sechdrs[i].sh_flags & SHF_EXECINSTR) &&
1063		    entry_off < sechdrs[i].sh_size) {
1064			kbuf->image->start = kbuf->mem + offset + entry_off;
1065			start_fixed = true;
1066		}
1067
1068		/*
1069		 * Check if the segment contains the entry point, if so,
1070		 * calculate the value of image->start based on it.
1071		 * If the compiler has produced more than one .text section
1072		 * (Eg: .text.hot), they are generally after the main .text
1073		 * section, and they shall not be used to calculate
1074		 * image->start. So do not re-calculate image->start if it
1075		 * is not set to the initial value, and warn the user so they
1076		 * have a chance to fix their purgatory's linker script.
1077		 */
1078		if (!start_fixed && sechdrs[i].sh_flags & SHF_EXECINSTR &&
1079		    pi->ehdr->e_entry >= sechdrs[i].sh_addr &&
1080		    pi->ehdr->e_entry < (sechdrs[i].sh_addr
1081					 + sechdrs[i].sh_size) &&
1082		    kbuf->image->start == pi->ehdr->e_entry) {
1083			kbuf->image->start -= sechdrs[i].sh_addr;
1084			kbuf->image->start += kbuf->mem + offset;
1085			start_fixed = true;
1086		}
1087
1088		src = (void *)pi->ehdr + sechdrs[i].sh_offset;
1089		dst = pi->purgatory_buf + offset;
1090		memcpy(dst, src, sechdrs[i].sh_size);
1091
1092		sechdrs[i].sh_addr = kbuf->mem + offset;
1093		sechdrs[i].sh_offset = offset;
1094		offset += sechdrs[i].sh_size;
1095	}
1096
1097	return 0;
1098}
1099
1100static int kexec_apply_relocations(struct kimage *image)
1101{
1102	int i, ret;
1103	struct purgatory_info *pi = &image->purgatory_info;
1104	const Elf_Shdr *sechdrs;
1105
1106	sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
1107
1108	for (i = 0; i < pi->ehdr->e_shnum; i++) {
1109		const Elf_Shdr *relsec;
1110		const Elf_Shdr *symtab;
1111		Elf_Shdr *section;
1112
1113		relsec = sechdrs + i;
1114
1115		if (relsec->sh_type != SHT_RELA &&
1116		    relsec->sh_type != SHT_REL)
1117			continue;
1118
1119		/*
1120		 * For section of type SHT_RELA/SHT_REL,
1121		 * ->sh_link contains section header index of associated
1122		 * symbol table. And ->sh_info contains section header
1123		 * index of section to which relocations apply.
1124		 */
1125		if (relsec->sh_info >= pi->ehdr->e_shnum ||
1126		    relsec->sh_link >= pi->ehdr->e_shnum)
1127			return -ENOEXEC;
1128
1129		section = pi->sechdrs + relsec->sh_info;
1130		symtab = sechdrs + relsec->sh_link;
1131
1132		if (!(section->sh_flags & SHF_ALLOC))
1133			continue;
1134
1135		/*
1136		 * symtab->sh_link contain section header index of associated
1137		 * string table.
1138		 */
1139		if (symtab->sh_link >= pi->ehdr->e_shnum)
1140			/* Invalid section number? */
1141			continue;
1142
1143		/*
1144		 * Respective architecture needs to provide support for applying
1145		 * relocations of type SHT_RELA/SHT_REL.
1146		 */
1147		if (relsec->sh_type == SHT_RELA)
1148			ret = arch_kexec_apply_relocations_add(pi, section,
1149							       relsec, symtab);
1150		else if (relsec->sh_type == SHT_REL)
1151			ret = arch_kexec_apply_relocations(pi, section,
1152							   relsec, symtab);
1153		if (ret)
1154			return ret;
1155	}
1156
1157	return 0;
1158}
1159
1160/*
1161 * kexec_load_purgatory - Load and relocate the purgatory object.
1162 * @image:	Image to add the purgatory to.
1163 * @kbuf:	Memory parameters to use.
1164 *
1165 * Allocates the memory needed for image->purgatory_info.sechdrs and
1166 * image->purgatory_info.purgatory_buf/kbuf->buffer. Caller is responsible
1167 * to free the memory after use.
1168 *
1169 * Return: 0 on success, negative errno on error.
1170 */
1171int kexec_load_purgatory(struct kimage *image, struct kexec_buf *kbuf)
1172{
1173	struct purgatory_info *pi = &image->purgatory_info;
1174	int ret;
1175
1176	if (kexec_purgatory_size <= 0)
1177		return -EINVAL;
1178
1179	pi->ehdr = (const Elf_Ehdr *)kexec_purgatory;
1180
1181	ret = kexec_purgatory_setup_kbuf(pi, kbuf);
1182	if (ret)
1183		return ret;
1184
1185	ret = kexec_purgatory_setup_sechdrs(pi, kbuf);
1186	if (ret)
1187		goto out_free_kbuf;
1188
1189	ret = kexec_apply_relocations(image);
1190	if (ret)
1191		goto out;
1192
1193	return 0;
1194out:
1195	vfree(pi->sechdrs);
1196	pi->sechdrs = NULL;
1197out_free_kbuf:
1198	vfree(pi->purgatory_buf);
1199	pi->purgatory_buf = NULL;
1200	return ret;
1201}
1202
1203void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name)
1204{
1205	struct purgatory_info *pi = &image->purgatory_info;
1206	const Elf_Sym *sym;
1207	Elf_Shdr *sechdr;
1208
1209	sym = kexec_purgatory_find_symbol(pi, name);
1210	if (!sym)
1211		return ERR_PTR(-EINVAL);
1212
1213	sechdr = &pi->sechdrs[sym->st_shndx];
1214
1215	/*
1216	 * Returns the address where symbol will finally be loaded after
1217	 * kexec_load_segment()
1218	 */
1219	return (void *)(sechdr->sh_addr + sym->st_value);
1220}
1221
1222/*
1223 * Get or set value of a symbol. If "get_value" is true, symbol value is
1224 * returned in buf otherwise symbol value is set based on value in buf.
1225 */
1226int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name,
1227				   void *buf, unsigned int size, bool get_value)
1228{
1229	struct purgatory_info *pi = &image->purgatory_info;
1230	const Elf_Sym *sym;
1231	Elf_Shdr *sec;
1232	char *sym_buf;
1233
1234	sym = kexec_purgatory_find_symbol(pi, name);
1235	if (!sym)
1236		return -EINVAL;
1237
1238	if (sym->st_size != size) {
1239		pr_err("symbol %s size mismatch: expected %lu actual %u\n",
1240		       name, (unsigned long)sym->st_size, size);
1241		return -EINVAL;
1242	}
1243
1244	sec = pi->sechdrs + sym->st_shndx;
1245
1246	if (sec->sh_type == SHT_NOBITS) {
1247		pr_err("symbol %s is in a bss section. Cannot %s\n", name,
1248		       get_value ? "get" : "set");
1249		return -EINVAL;
1250	}
1251
1252	sym_buf = (char *)pi->purgatory_buf + sec->sh_offset + sym->st_value;
1253
1254	if (get_value)
1255		memcpy((void *)buf, sym_buf, size);
1256	else
1257		memcpy((void *)sym_buf, buf, size);
1258
1259	return 0;
1260}
1261#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */