kexec_handover.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * kexec_handover.c - kexec handover metadata processing
   4 * Copyright (C) 2023 Alexander Graf <graf@amazon.com>
   5 * Copyright (C) 2025 Microsoft Corporation, Mike Rapoport <rppt@kernel.org>
   6 * Copyright (C) 2025 Google LLC, Changyuan Lyu <changyuanl@google.com>
   7 * Copyright (C) 2025 Pasha Tatashin <pasha.tatashin@soleen.com>
   8 */
   9
  10#define pr_fmt(fmt) "KHO: " fmt
  11
  12#include <linux/cleanup.h>
  13#include <linux/cma.h>
  14#include <linux/kmemleak.h>
  15#include <linux/count_zeros.h>
  16#include <linux/kexec.h>
  17#include <linux/kexec_handover.h>
  18#include <linux/libfdt.h>
  19#include <linux/list.h>
  20#include <linux/memblock.h>
  21#include <linux/page-isolation.h>
  22#include <linux/unaligned.h>
  23#include <linux/vmalloc.h>
  24
  25#include <asm/early_ioremap.h>
  26
  27#include "kexec_handover_internal.h"
  28/*
  29 * KHO is tightly coupled with mm init and needs access to some of mm
  30 * internal APIs.
  31 */
  32#include "../../mm/internal.h"
  33#include "../kexec_internal.h"
  34#include "kexec_handover_internal.h"
  35
  36#define KHO_FDT_COMPATIBLE "kho-v1"
  37#define PROP_PRESERVED_MEMORY_MAP "preserved-memory-map"
  38#define PROP_SUB_FDT "fdt"
  39
  40#define KHO_PAGE_MAGIC 0x4b484f50U /* ASCII for 'KHOP' */
  41
  42/*
  43 * KHO uses page->private, which is an unsigned long, to store page metadata.
  44 * Use it to store both the magic and the order.
  45 */
  46union kho_page_info {
  47	unsigned long page_private;
  48	struct {
  49		unsigned int order;
  50		unsigned int magic;
  51	};
  52};
  53
  54static_assert(sizeof(union kho_page_info) == sizeof(((struct page *)0)->private));
  55
  56static bool kho_enable __ro_after_init = IS_ENABLED(CONFIG_KEXEC_HANDOVER_ENABLE_DEFAULT);
  57
  58bool kho_is_enabled(void)
  59{
  60	return kho_enable;
  61}
  62EXPORT_SYMBOL_GPL(kho_is_enabled);
  63
  64static int __init kho_parse_enable(char *p)
  65{
  66	return kstrtobool(p, &kho_enable);
  67}
  68early_param("kho", kho_parse_enable);
  69
  70/*
  71 * Keep track of memory that is to be preserved across KHO.
  72 *
  73 * The serializing side uses two levels of xarrays to manage chunks of per-order
  74 * PAGE_SIZE byte bitmaps. For instance if PAGE_SIZE = 4096, the entire 1G order
  75 * of a 8TB system would fit inside a single 4096 byte bitmap. For order 0
  76 * allocations each bitmap will cover 128M of address space. Thus, for 16G of
  77 * memory at most 512K of bitmap memory will be needed for order 0.
  78 *
  79 * This approach is fully incremental, as the serialization progresses folios
  80 * can continue be aggregated to the tracker. The final step, immediately prior
  81 * to kexec would serialize the xarray information into a linked list for the
  82 * successor kernel to parse.
  83 */
  84
  85#define PRESERVE_BITS (PAGE_SIZE * 8)
  86
  87struct kho_mem_phys_bits {
  88	DECLARE_BITMAP(preserve, PRESERVE_BITS);
  89};
  90
  91static_assert(sizeof(struct kho_mem_phys_bits) == PAGE_SIZE);
  92
  93struct kho_mem_phys {
  94	/*
  95	 * Points to kho_mem_phys_bits, a sparse bitmap array. Each bit is sized
  96	 * to order.
  97	 */
  98	struct xarray phys_bits;
  99};
 100
 101struct kho_mem_track {
 102	/* Points to kho_mem_phys, each order gets its own bitmap tree */
 103	struct xarray orders;
 104};
 105
 106struct khoser_mem_chunk;
 107
 108struct kho_out {
 109	void *fdt;
 110	bool finalized;
 111	struct mutex lock; /* protects KHO FDT finalization */
 112
 113	struct kho_mem_track track;
 114	struct kho_debugfs dbg;
 115};
 116
 117static struct kho_out kho_out = {
 118	.lock = __MUTEX_INITIALIZER(kho_out.lock),
 119	.track = {
 120		.orders = XARRAY_INIT(kho_out.track.orders, 0),
 121	},
 122	.finalized = false,
 123};
 124
 125static void *xa_load_or_alloc(struct xarray *xa, unsigned long index)
 126{
 127	void *res = xa_load(xa, index);
 128
 129	if (res)
 130		return res;
 131
 132	void *elm __free(free_page) = (void *)get_zeroed_page(GFP_KERNEL);
 133
 134	if (!elm)
 135		return ERR_PTR(-ENOMEM);
 136
 137	if (WARN_ON(kho_scratch_overlap(virt_to_phys(elm), PAGE_SIZE)))
 138		return ERR_PTR(-EINVAL);
 139
 140	res = xa_cmpxchg(xa, index, NULL, elm, GFP_KERNEL);
 141	if (xa_is_err(res))
 142		return ERR_PTR(xa_err(res));
 143	else if (res)
 144		return res;
 145
 146	return no_free_ptr(elm);
 147}
 148
 149static void __kho_unpreserve_order(struct kho_mem_track *track, unsigned long pfn,
 150				   unsigned int order)
 151{
 152	struct kho_mem_phys_bits *bits;
 153	struct kho_mem_phys *physxa;
 154	const unsigned long pfn_high = pfn >> order;
 155
 156	physxa = xa_load(&track->orders, order);
 157	if (WARN_ON_ONCE(!physxa))
 158		return;
 159
 160	bits = xa_load(&physxa->phys_bits, pfn_high / PRESERVE_BITS);
 161	if (WARN_ON_ONCE(!bits))
 162		return;
 163
 164	clear_bit(pfn_high % PRESERVE_BITS, bits->preserve);
 165}
 166
 167static void __kho_unpreserve(struct kho_mem_track *track, unsigned long pfn,
 168			     unsigned long end_pfn)
 169{
 170	unsigned int order;
 171
 172	while (pfn < end_pfn) {
 173		order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
 174
 175		__kho_unpreserve_order(track, pfn, order);
 176
 177		pfn += 1 << order;
 178	}
 179}
 180
 181static int __kho_preserve_order(struct kho_mem_track *track, unsigned long pfn,
 182				unsigned int order)
 183{
 184	struct kho_mem_phys_bits *bits;
 185	struct kho_mem_phys *physxa, *new_physxa;
 186	const unsigned long pfn_high = pfn >> order;
 187
 188	might_sleep();
 189	physxa = xa_load(&track->orders, order);
 190	if (!physxa) {
 191		int err;
 192
 193		new_physxa = kzalloc(sizeof(*physxa), GFP_KERNEL);
 194		if (!new_physxa)
 195			return -ENOMEM;
 196
 197		xa_init(&new_physxa->phys_bits);
 198		physxa = xa_cmpxchg(&track->orders, order, NULL, new_physxa,
 199				    GFP_KERNEL);
 200
 201		err = xa_err(physxa);
 202		if (err || physxa) {
 203			xa_destroy(&new_physxa->phys_bits);
 204			kfree(new_physxa);
 205
 206			if (err)
 207				return err;
 208		} else {
 209			physxa = new_physxa;
 210		}
 211	}
 212
 213	bits = xa_load_or_alloc(&physxa->phys_bits, pfn_high / PRESERVE_BITS);
 214	if (IS_ERR(bits))
 215		return PTR_ERR(bits);
 216
 217	set_bit(pfn_high % PRESERVE_BITS, bits->preserve);
 218
 219	return 0;
 220}
 221
 222static struct page *kho_restore_page(phys_addr_t phys, bool is_folio)
 223{
 224	struct page *page = pfn_to_online_page(PHYS_PFN(phys));
 225	unsigned int nr_pages, ref_cnt;
 226	union kho_page_info info;
 227
 228	if (!page)
 229		return NULL;
 230
 231	info.page_private = page->private;
 232	/*
 233	 * deserialize_bitmap() only sets the magic on the head page. This magic
 234	 * check also implicitly makes sure phys is order-aligned since for
 235	 * non-order-aligned phys addresses, magic will never be set.
 236	 */
 237	if (WARN_ON_ONCE(info.magic != KHO_PAGE_MAGIC || info.order > MAX_PAGE_ORDER))
 238		return NULL;
 239	nr_pages = (1 << info.order);
 240
 241	/* Clear private to make sure later restores on this page error out. */
 242	page->private = 0;
 243	/* Head page gets refcount of 1. */
 244	set_page_count(page, 1);
 245
 246	/*
 247	 * For higher order folios, tail pages get a page count of zero.
 248	 * For physically contiguous order-0 pages every pages gets a page
 249	 * count of 1
 250	 */
 251	ref_cnt = is_folio ? 0 : 1;
 252	for (unsigned int i = 1; i < nr_pages; i++)
 253		set_page_count(page + i, ref_cnt);
 254
 255	if (is_folio && info.order)
 256		prep_compound_page(page, info.order);
 257
 258	/* Always mark headpage's codetag as empty to avoid accounting mismatch */
 259	clear_page_tag_ref(page);
 260	if (!is_folio) {
 261		/* Also do that for the non-compound tail pages */
 262		for (unsigned int i = 1; i < nr_pages; i++)
 263			clear_page_tag_ref(page + i);
 264	}
 265
 266	adjust_managed_page_count(page, nr_pages);
 267	return page;
 268}
 269
 270/**
 271 * kho_restore_folio - recreates the folio from the preserved memory.
 272 * @phys: physical address of the folio.
 273 *
 274 * Return: pointer to the struct folio on success, NULL on failure.
 275 */
 276struct folio *kho_restore_folio(phys_addr_t phys)
 277{
 278	struct page *page = kho_restore_page(phys, true);
 279
 280	return page ? page_folio(page) : NULL;
 281}
 282EXPORT_SYMBOL_GPL(kho_restore_folio);
 283
 284/**
 285 * kho_restore_pages - restore list of contiguous order 0 pages.
 286 * @phys: physical address of the first page.
 287 * @nr_pages: number of pages.
 288 *
 289 * Restore a contiguous list of order 0 pages that was preserved with
 290 * kho_preserve_pages().
 291 *
 292 * Return: 0 on success, error code on failure
 293 */
 294struct page *kho_restore_pages(phys_addr_t phys, unsigned int nr_pages)
 295{
 296	const unsigned long start_pfn = PHYS_PFN(phys);
 297	const unsigned long end_pfn = start_pfn + nr_pages;
 298	unsigned long pfn = start_pfn;
 299
 300	while (pfn < end_pfn) {
 301		const unsigned int order =
 302			min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
 303		struct page *page = kho_restore_page(PFN_PHYS(pfn), false);
 304
 305		if (!page)
 306			return NULL;
 307		pfn += 1 << order;
 308	}
 309
 310	return pfn_to_page(start_pfn);
 311}
 312EXPORT_SYMBOL_GPL(kho_restore_pages);
 313
 314/* Serialize and deserialize struct kho_mem_phys across kexec
 315 *
 316 * Record all the bitmaps in a linked list of pages for the next kernel to
 317 * process. Each chunk holds bitmaps of the same order and each block of bitmaps
 318 * starts at a given physical address. This allows the bitmaps to be sparse. The
 319 * xarray is used to store them in a tree while building up the data structure,
 320 * but the KHO successor kernel only needs to process them once in order.
 321 *
 322 * All of this memory is normal kmalloc() memory and is not marked for
 323 * preservation. The successor kernel will remain isolated to the scratch space
 324 * until it completes processing this list. Once processed all the memory
 325 * storing these ranges will be marked as free.
 326 */
 327
 328struct khoser_mem_bitmap_ptr {
 329	phys_addr_t phys_start;
 330	DECLARE_KHOSER_PTR(bitmap, struct kho_mem_phys_bits *);
 331};
 332
 333struct khoser_mem_chunk_hdr {
 334	DECLARE_KHOSER_PTR(next, struct khoser_mem_chunk *);
 335	unsigned int order;
 336	unsigned int num_elms;
 337};
 338
 339#define KHOSER_BITMAP_SIZE                                   \
 340	((PAGE_SIZE - sizeof(struct khoser_mem_chunk_hdr)) / \
 341	 sizeof(struct khoser_mem_bitmap_ptr))
 342
 343struct khoser_mem_chunk {
 344	struct khoser_mem_chunk_hdr hdr;
 345	struct khoser_mem_bitmap_ptr bitmaps[KHOSER_BITMAP_SIZE];
 346};
 347
 348static_assert(sizeof(struct khoser_mem_chunk) == PAGE_SIZE);
 349
 350static struct khoser_mem_chunk *new_chunk(struct khoser_mem_chunk *cur_chunk,
 351					  unsigned long order)
 352{
 353	struct khoser_mem_chunk *chunk __free(free_page) = NULL;
 354
 355	chunk = (void *)get_zeroed_page(GFP_KERNEL);
 356	if (!chunk)
 357		return ERR_PTR(-ENOMEM);
 358
 359	if (WARN_ON(kho_scratch_overlap(virt_to_phys(chunk), PAGE_SIZE)))
 360		return ERR_PTR(-EINVAL);
 361
 362	chunk->hdr.order = order;
 363	if (cur_chunk)
 364		KHOSER_STORE_PTR(cur_chunk->hdr.next, chunk);
 365	return no_free_ptr(chunk);
 366}
 367
 368static void kho_mem_ser_free(struct khoser_mem_chunk *first_chunk)
 369{
 370	struct khoser_mem_chunk *chunk = first_chunk;
 371
 372	while (chunk) {
 373		struct khoser_mem_chunk *tmp = chunk;
 374
 375		chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
 376		free_page((unsigned long)tmp);
 377	}
 378}
 379
 380/*
 381 *  Update memory map property, if old one is found discard it via
 382 *  kho_mem_ser_free().
 383 */
 384static void kho_update_memory_map(struct khoser_mem_chunk *first_chunk)
 385{
 386	void *ptr;
 387	u64 phys;
 388
 389	ptr = fdt_getprop_w(kho_out.fdt, 0, PROP_PRESERVED_MEMORY_MAP, NULL);
 390
 391	/* Check and discard previous memory map */
 392	phys = get_unaligned((u64 *)ptr);
 393	if (phys)
 394		kho_mem_ser_free((struct khoser_mem_chunk *)phys_to_virt(phys));
 395
 396	/* Update with the new value */
 397	phys = first_chunk ? (u64)virt_to_phys(first_chunk) : 0;
 398	put_unaligned(phys, (u64 *)ptr);
 399}
 400
 401static int kho_mem_serialize(struct kho_out *kho_out)
 402{
 403	struct khoser_mem_chunk *first_chunk = NULL;
 404	struct khoser_mem_chunk *chunk = NULL;
 405	struct kho_mem_phys *physxa;
 406	unsigned long order;
 407	int err = -ENOMEM;
 408
 409	xa_for_each(&kho_out->track.orders, order, physxa) {
 410		struct kho_mem_phys_bits *bits;
 411		unsigned long phys;
 412
 413		chunk = new_chunk(chunk, order);
 414		if (IS_ERR(chunk)) {
 415			err = PTR_ERR(chunk);
 416			goto err_free;
 417		}
 418
 419		if (!first_chunk)
 420			first_chunk = chunk;
 421
 422		xa_for_each(&physxa->phys_bits, phys, bits) {
 423			struct khoser_mem_bitmap_ptr *elm;
 424
 425			if (chunk->hdr.num_elms == ARRAY_SIZE(chunk->bitmaps)) {
 426				chunk = new_chunk(chunk, order);
 427				if (IS_ERR(chunk)) {
 428					err = PTR_ERR(chunk);
 429					goto err_free;
 430				}
 431			}
 432
 433			elm = &chunk->bitmaps[chunk->hdr.num_elms];
 434			chunk->hdr.num_elms++;
 435			elm->phys_start = (phys * PRESERVE_BITS)
 436					  << (order + PAGE_SHIFT);
 437			KHOSER_STORE_PTR(elm->bitmap, bits);
 438		}
 439	}
 440
 441	kho_update_memory_map(first_chunk);
 442
 443	return 0;
 444
 445err_free:
 446	kho_mem_ser_free(first_chunk);
 447	return err;
 448}
 449
 450static void __init deserialize_bitmap(unsigned int order,
 451				      struct khoser_mem_bitmap_ptr *elm)
 452{
 453	struct kho_mem_phys_bits *bitmap = KHOSER_LOAD_PTR(elm->bitmap);
 454	unsigned long bit;
 455
 456	for_each_set_bit(bit, bitmap->preserve, PRESERVE_BITS) {
 457		int sz = 1 << (order + PAGE_SHIFT);
 458		phys_addr_t phys =
 459			elm->phys_start + (bit << (order + PAGE_SHIFT));
 460		struct page *page = phys_to_page(phys);
 461		union kho_page_info info;
 462
 463		memblock_reserve(phys, sz);
 464		memblock_reserved_mark_noinit(phys, sz);
 465		info.magic = KHO_PAGE_MAGIC;
 466		info.order = order;
 467		page->private = info.page_private;
 468	}
 469}
 470
 471/* Returns physical address of the preserved memory map from FDT */
 472static phys_addr_t __init kho_get_mem_map_phys(const void *fdt)
 473{
 474	const void *mem_ptr;
 475	int len;
 476
 477	mem_ptr = fdt_getprop(fdt, 0, PROP_PRESERVED_MEMORY_MAP, &len);
 478	if (!mem_ptr || len != sizeof(u64)) {
 479		pr_err("failed to get preserved memory bitmaps\n");
 480		return 0;
 481	}
 482
 483	return get_unaligned((const u64 *)mem_ptr);
 484}
 485
 486static void __init kho_mem_deserialize(struct khoser_mem_chunk *chunk)
 487{
 488	while (chunk) {
 489		unsigned int i;
 490
 491		for (i = 0; i != chunk->hdr.num_elms; i++)
 492			deserialize_bitmap(chunk->hdr.order,
 493					   &chunk->bitmaps[i]);
 494		chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
 495	}
 496}
 497
 498/*
 499 * With KHO enabled, memory can become fragmented because KHO regions may
 500 * be anywhere in physical address space. The scratch regions give us a
 501 * safe zones that we will never see KHO allocations from. This is where we
 502 * can later safely load our new kexec images into and then use the scratch
 503 * area for early allocations that happen before page allocator is
 504 * initialized.
 505 */
 506struct kho_scratch *kho_scratch;
 507unsigned int kho_scratch_cnt;
 508
 509/*
 510 * The scratch areas are scaled by default as percent of memory allocated from
 511 * memblock. A user can override the scale with command line parameter:
 512 *
 513 * kho_scratch=N%
 514 *
 515 * It is also possible to explicitly define size for a lowmem, a global and
 516 * per-node scratch areas:
 517 *
 518 * kho_scratch=l[KMG],n[KMG],m[KMG]
 519 *
 520 * The explicit size definition takes precedence over scale definition.
 521 */
 522static unsigned int scratch_scale __initdata = 200;
 523static phys_addr_t scratch_size_global __initdata;
 524static phys_addr_t scratch_size_pernode __initdata;
 525static phys_addr_t scratch_size_lowmem __initdata;
 526
 527static int __init kho_parse_scratch_size(char *p)
 528{
 529	size_t len;
 530	unsigned long sizes[3];
 531	size_t total_size = 0;
 532	int i;
 533
 534	if (!p)
 535		return -EINVAL;
 536
 537	len = strlen(p);
 538	if (!len)
 539		return -EINVAL;
 540
 541	/* parse nn% */
 542	if (p[len - 1] == '%') {
 543		/* unsigned int max is 4,294,967,295, 10 chars */
 544		char s_scale[11] = {};
 545		int ret = 0;
 546
 547		if (len > ARRAY_SIZE(s_scale))
 548			return -EINVAL;
 549
 550		memcpy(s_scale, p, len - 1);
 551		ret = kstrtouint(s_scale, 10, &scratch_scale);
 552		if (!ret)
 553			pr_notice("scratch scale is %d%%\n", scratch_scale);
 554		return ret;
 555	}
 556
 557	/* parse ll[KMG],mm[KMG],nn[KMG] */
 558	for (i = 0; i < ARRAY_SIZE(sizes); i++) {
 559		char *endp = p;
 560
 561		if (i > 0) {
 562			if (*p != ',')
 563				return -EINVAL;
 564			p += 1;
 565		}
 566
 567		sizes[i] = memparse(p, &endp);
 568		if (endp == p)
 569			return -EINVAL;
 570		p = endp;
 571		total_size += sizes[i];
 572	}
 573
 574	if (!total_size)
 575		return -EINVAL;
 576
 577	/* The string should be fully consumed by now. */
 578	if (*p)
 579		return -EINVAL;
 580
 581	scratch_size_lowmem = sizes[0];
 582	scratch_size_global = sizes[1];
 583	scratch_size_pernode = sizes[2];
 584	scratch_scale = 0;
 585
 586	pr_notice("scratch areas: lowmem: %lluMiB global: %lluMiB pernode: %lldMiB\n",
 587		  (u64)(scratch_size_lowmem >> 20),
 588		  (u64)(scratch_size_global >> 20),
 589		  (u64)(scratch_size_pernode >> 20));
 590
 591	return 0;
 592}
 593early_param("kho_scratch", kho_parse_scratch_size);
 594
 595static void __init scratch_size_update(void)
 596{
 597	phys_addr_t size;
 598
 599	if (!scratch_scale)
 600		return;
 601
 602	size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT,
 603					   NUMA_NO_NODE);
 604	size = size * scratch_scale / 100;
 605	scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES);
 606
 607	size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
 608					   NUMA_NO_NODE);
 609	size = size * scratch_scale / 100 - scratch_size_lowmem;
 610	scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES);
 611}
 612
 613static phys_addr_t __init scratch_size_node(int nid)
 614{
 615	phys_addr_t size;
 616
 617	if (scratch_scale) {
 618		size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
 619						   nid);
 620		size = size * scratch_scale / 100;
 621	} else {
 622		size = scratch_size_pernode;
 623	}
 624
 625	return round_up(size, CMA_MIN_ALIGNMENT_BYTES);
 626}
 627
 628/**
 629 * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
 630 *
 631 * With KHO we can preserve arbitrary pages in the system. To ensure we still
 632 * have a large contiguous region of memory when we search the physical address
 633 * space for target memory, let's make sure we always have a large CMA region
 634 * active. This CMA region will only be used for movable pages which are not a
 635 * problem for us during KHO because we can just move them somewhere else.
 636 */
 637static void __init kho_reserve_scratch(void)
 638{
 639	phys_addr_t addr, size;
 640	int nid, i = 0;
 641
 642	if (!kho_enable)
 643		return;
 644
 645	scratch_size_update();
 646
 647	/* FIXME: deal with node hot-plug/remove */
 648	kho_scratch_cnt = nodes_weight(node_states[N_MEMORY]) + 2;
 649	size = kho_scratch_cnt * sizeof(*kho_scratch);
 650	kho_scratch = memblock_alloc(size, PAGE_SIZE);
 651	if (!kho_scratch)
 652		goto err_disable_kho;
 653
 654	/*
 655	 * reserve scratch area in low memory for lowmem allocations in the
 656	 * next kernel
 657	 */
 658	size = scratch_size_lowmem;
 659	addr = memblock_phys_alloc_range(size, CMA_MIN_ALIGNMENT_BYTES, 0,
 660					 ARCH_LOW_ADDRESS_LIMIT);
 661	if (!addr)
 662		goto err_free_scratch_desc;
 663
 664	kho_scratch[i].addr = addr;
 665	kho_scratch[i].size = size;
 666	i++;
 667
 668	/* reserve large contiguous area for allocations without nid */
 669	size = scratch_size_global;
 670	addr = memblock_phys_alloc(size, CMA_MIN_ALIGNMENT_BYTES);
 671	if (!addr)
 672		goto err_free_scratch_areas;
 673
 674	kho_scratch[i].addr = addr;
 675	kho_scratch[i].size = size;
 676	i++;
 677
 678	/*
 679	 * Loop over nodes that have both memory and are online. Skip
 680	 * memoryless nodes, as we can not allocate scratch areas there.
 681	 */
 682	for_each_node_state(nid, N_MEMORY) {
 683		size = scratch_size_node(nid);
 684		addr = memblock_alloc_range_nid(size, CMA_MIN_ALIGNMENT_BYTES,
 685						0, MEMBLOCK_ALLOC_ACCESSIBLE,
 686						nid, true);
 687		if (!addr)
 688			goto err_free_scratch_areas;
 689
 690		kho_scratch[i].addr = addr;
 691		kho_scratch[i].size = size;
 692		i++;
 693	}
 694
 695	return;
 696
 697err_free_scratch_areas:
 698	for (i--; i >= 0; i--)
 699		memblock_phys_free(kho_scratch[i].addr, kho_scratch[i].size);
 700err_free_scratch_desc:
 701	memblock_free(kho_scratch, kho_scratch_cnt * sizeof(*kho_scratch));
 702err_disable_kho:
 703	pr_warn("Failed to reserve scratch area, disabling kexec handover\n");
 704	kho_enable = false;
 705}
 706
 707/**
 708 * kho_add_subtree - record the physical address of a sub FDT in KHO root tree.
 709 * @name: name of the sub tree.
 710 * @fdt: the sub tree blob.
 711 *
 712 * Creates a new child node named @name in KHO root FDT and records
 713 * the physical address of @fdt. The pages of @fdt must also be preserved
 714 * by KHO for the new kernel to retrieve it after kexec.
 715 *
 716 * A debugfs blob entry is also created at
 717 * ``/sys/kernel/debug/kho/out/sub_fdts/@name`` when kernel is configured with
 718 * CONFIG_KEXEC_HANDOVER_DEBUGFS
 719 *
 720 * Return: 0 on success, error code on failure
 721 */
 722int kho_add_subtree(const char *name, void *fdt)
 723{
 724	phys_addr_t phys = virt_to_phys(fdt);
 725	void *root_fdt = kho_out.fdt;
 726	int err = -ENOMEM;
 727	int off, fdt_err;
 728
 729	guard(mutex)(&kho_out.lock);
 730
 731	fdt_err = fdt_open_into(root_fdt, root_fdt, PAGE_SIZE);
 732	if (fdt_err < 0)
 733		return err;
 734
 735	off = fdt_add_subnode(root_fdt, 0, name);
 736	if (off < 0) {
 737		if (off == -FDT_ERR_EXISTS)
 738			err = -EEXIST;
 739		goto out_pack;
 740	}
 741
 742	err = fdt_setprop(root_fdt, off, PROP_SUB_FDT, &phys, sizeof(phys));
 743	if (err < 0)
 744		goto out_pack;
 745
 746	WARN_ON_ONCE(kho_debugfs_fdt_add(&kho_out.dbg, name, fdt, false));
 747
 748out_pack:
 749	fdt_pack(root_fdt);
 750
 751	return err;
 752}
 753EXPORT_SYMBOL_GPL(kho_add_subtree);
 754
 755void kho_remove_subtree(void *fdt)
 756{
 757	phys_addr_t target_phys = virt_to_phys(fdt);
 758	void *root_fdt = kho_out.fdt;
 759	int off;
 760	int err;
 761
 762	guard(mutex)(&kho_out.lock);
 763
 764	err = fdt_open_into(root_fdt, root_fdt, PAGE_SIZE);
 765	if (err < 0)
 766		return;
 767
 768	for (off = fdt_first_subnode(root_fdt, 0); off >= 0;
 769	     off = fdt_next_subnode(root_fdt, off)) {
 770		const u64 *val;
 771		int len;
 772
 773		val = fdt_getprop(root_fdt, off, PROP_SUB_FDT, &len);
 774		if (!val || len != sizeof(phys_addr_t))
 775			continue;
 776
 777		if ((phys_addr_t)*val == target_phys) {
 778			fdt_del_node(root_fdt, off);
 779			kho_debugfs_fdt_remove(&kho_out.dbg, fdt);
 780			break;
 781		}
 782	}
 783
 784	fdt_pack(root_fdt);
 785}
 786EXPORT_SYMBOL_GPL(kho_remove_subtree);
 787
 788/**
 789 * kho_preserve_folio - preserve a folio across kexec.
 790 * @folio: folio to preserve.
 791 *
 792 * Instructs KHO to preserve the whole folio across kexec. The order
 793 * will be preserved as well.
 794 *
 795 * Return: 0 on success, error code on failure
 796 */
 797int kho_preserve_folio(struct folio *folio)
 798{
 799	const unsigned long pfn = folio_pfn(folio);
 800	const unsigned int order = folio_order(folio);
 801	struct kho_mem_track *track = &kho_out.track;
 802
 803	if (WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
 804		return -EINVAL;
 805
 806	return __kho_preserve_order(track, pfn, order);
 807}
 808EXPORT_SYMBOL_GPL(kho_preserve_folio);
 809
 810/**
 811 * kho_unpreserve_folio - unpreserve a folio.
 812 * @folio: folio to unpreserve.
 813 *
 814 * Instructs KHO to unpreserve a folio that was preserved by
 815 * kho_preserve_folio() before. The provided @folio (pfn and order)
 816 * must exactly match a previously preserved folio.
 817 */
 818void kho_unpreserve_folio(struct folio *folio)
 819{
 820	const unsigned long pfn = folio_pfn(folio);
 821	const unsigned int order = folio_order(folio);
 822	struct kho_mem_track *track = &kho_out.track;
 823
 824	__kho_unpreserve_order(track, pfn, order);
 825}
 826EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
 827
 828/**
 829 * kho_preserve_pages - preserve contiguous pages across kexec
 830 * @page: first page in the list.
 831 * @nr_pages: number of pages.
 832 *
 833 * Preserve a contiguous list of order 0 pages. Must be restored using
 834 * kho_restore_pages() to ensure the pages are restored properly as order 0.
 835 *
 836 * Return: 0 on success, error code on failure
 837 */
 838int kho_preserve_pages(struct page *page, unsigned int nr_pages)
 839{
 840	struct kho_mem_track *track = &kho_out.track;
 841	const unsigned long start_pfn = page_to_pfn(page);
 842	const unsigned long end_pfn = start_pfn + nr_pages;
 843	unsigned long pfn = start_pfn;
 844	unsigned long failed_pfn = 0;
 845	int err = 0;
 846
 847	if (WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT,
 848					nr_pages << PAGE_SHIFT))) {
 849		return -EINVAL;
 850	}
 851
 852	while (pfn < end_pfn) {
 853		const unsigned int order =
 854			min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
 855
 856		err = __kho_preserve_order(track, pfn, order);
 857		if (err) {
 858			failed_pfn = pfn;
 859			break;
 860		}
 861
 862		pfn += 1 << order;
 863	}
 864
 865	if (err)
 866		__kho_unpreserve(track, start_pfn, failed_pfn);
 867
 868	return err;
 869}
 870EXPORT_SYMBOL_GPL(kho_preserve_pages);
 871
 872/**
 873 * kho_unpreserve_pages - unpreserve contiguous pages.
 874 * @page: first page in the list.
 875 * @nr_pages: number of pages.
 876 *
 877 * Instructs KHO to unpreserve @nr_pages contiguous pages starting from @page.
 878 * This must be called with the same @page and @nr_pages as the corresponding
 879 * kho_preserve_pages() call. Unpreserving arbitrary sub-ranges of larger
 880 * preserved blocks is not supported.
 881 */
 882void kho_unpreserve_pages(struct page *page, unsigned int nr_pages)
 883{
 884	struct kho_mem_track *track = &kho_out.track;
 885	const unsigned long start_pfn = page_to_pfn(page);
 886	const unsigned long end_pfn = start_pfn + nr_pages;
 887
 888	__kho_unpreserve(track, start_pfn, end_pfn);
 889}
 890EXPORT_SYMBOL_GPL(kho_unpreserve_pages);
 891
 892struct kho_vmalloc_hdr {
 893	DECLARE_KHOSER_PTR(next, struct kho_vmalloc_chunk *);
 894};
 895
 896#define KHO_VMALLOC_SIZE				\
 897	((PAGE_SIZE - sizeof(struct kho_vmalloc_hdr)) / \
 898	 sizeof(phys_addr_t))
 899
 900struct kho_vmalloc_chunk {
 901	struct kho_vmalloc_hdr hdr;
 902	phys_addr_t phys[KHO_VMALLOC_SIZE];
 903};
 904
 905static_assert(sizeof(struct kho_vmalloc_chunk) == PAGE_SIZE);
 906
 907/* vmalloc flags KHO supports */
 908#define KHO_VMALLOC_SUPPORTED_FLAGS	(VM_ALLOC | VM_ALLOW_HUGE_VMAP)
 909
 910/* KHO internal flags for vmalloc preservations */
 911#define KHO_VMALLOC_ALLOC	0x0001
 912#define KHO_VMALLOC_HUGE_VMAP	0x0002
 913
 914static unsigned short vmalloc_flags_to_kho(unsigned int vm_flags)
 915{
 916	unsigned short kho_flags = 0;
 917
 918	if (vm_flags & VM_ALLOC)
 919		kho_flags |= KHO_VMALLOC_ALLOC;
 920	if (vm_flags & VM_ALLOW_HUGE_VMAP)
 921		kho_flags |= KHO_VMALLOC_HUGE_VMAP;
 922
 923	return kho_flags;
 924}
 925
 926static unsigned int kho_flags_to_vmalloc(unsigned short kho_flags)
 927{
 928	unsigned int vm_flags = 0;
 929
 930	if (kho_flags & KHO_VMALLOC_ALLOC)
 931		vm_flags |= VM_ALLOC;
 932	if (kho_flags & KHO_VMALLOC_HUGE_VMAP)
 933		vm_flags |= VM_ALLOW_HUGE_VMAP;
 934
 935	return vm_flags;
 936}
 937
 938static struct kho_vmalloc_chunk *new_vmalloc_chunk(struct kho_vmalloc_chunk *cur)
 939{
 940	struct kho_vmalloc_chunk *chunk;
 941	int err;
 942
 943	chunk = (struct kho_vmalloc_chunk *)get_zeroed_page(GFP_KERNEL);
 944	if (!chunk)
 945		return NULL;
 946
 947	err = kho_preserve_pages(virt_to_page(chunk), 1);
 948	if (err)
 949		goto err_free;
 950	if (cur)
 951		KHOSER_STORE_PTR(cur->hdr.next, chunk);
 952	return chunk;
 953
 954err_free:
 955	free_page((unsigned long)chunk);
 956	return NULL;
 957}
 958
 959static void kho_vmalloc_unpreserve_chunk(struct kho_vmalloc_chunk *chunk,
 960					 unsigned short order)
 961{
 962	struct kho_mem_track *track = &kho_out.track;
 963	unsigned long pfn = PHYS_PFN(virt_to_phys(chunk));
 964
 965	__kho_unpreserve(track, pfn, pfn + 1);
 966
 967	for (int i = 0; i < ARRAY_SIZE(chunk->phys) && chunk->phys[i]; i++) {
 968		pfn = PHYS_PFN(chunk->phys[i]);
 969		__kho_unpreserve(track, pfn, pfn + (1 << order));
 970	}
 971}
 972
 973/**
 974 * kho_preserve_vmalloc - preserve memory allocated with vmalloc() across kexec
 975 * @ptr: pointer to the area in vmalloc address space
 976 * @preservation: placeholder for preservation metadata
 977 *
 978 * Instructs KHO to preserve the area in vmalloc address space at @ptr. The
 979 * physical pages mapped at @ptr will be preserved and on successful return
 980 * @preservation will hold the physical address of a structure that describes
 981 * the preservation.
 982 *
 983 * NOTE: The memory allocated with vmalloc_node() variants cannot be reliably
 984 * restored on the same node
 985 *
 986 * Return: 0 on success, error code on failure
 987 */
 988int kho_preserve_vmalloc(void *ptr, struct kho_vmalloc *preservation)
 989{
 990	struct kho_vmalloc_chunk *chunk;
 991	struct vm_struct *vm = find_vm_area(ptr);
 992	unsigned int order, flags, nr_contig_pages;
 993	unsigned int idx = 0;
 994	int err;
 995
 996	if (!vm)
 997		return -EINVAL;
 998
 999	if (vm->flags & ~KHO_VMALLOC_SUPPORTED_FLAGS)
1000		return -EOPNOTSUPP;
1001
1002	flags = vmalloc_flags_to_kho(vm->flags);
1003	order = get_vm_area_page_order(vm);
1004
1005	chunk = new_vmalloc_chunk(NULL);
1006	if (!chunk)
1007		return -ENOMEM;
1008	KHOSER_STORE_PTR(preservation->first, chunk);
1009
1010	nr_contig_pages = (1 << order);
1011	for (int i = 0; i < vm->nr_pages; i += nr_contig_pages) {
1012		phys_addr_t phys = page_to_phys(vm->pages[i]);
1013
1014		err = kho_preserve_pages(vm->pages[i], nr_contig_pages);
1015		if (err)
1016			goto err_free;
1017
1018		chunk->phys[idx++] = phys;
1019		if (idx == ARRAY_SIZE(chunk->phys)) {
1020			chunk = new_vmalloc_chunk(chunk);
1021			if (!chunk) {
1022				err = -ENOMEM;
1023				goto err_free;
1024			}
1025			idx = 0;
1026		}
1027	}
1028
1029	preservation->total_pages = vm->nr_pages;
1030	preservation->flags = flags;
1031	preservation->order = order;
1032
1033	return 0;
1034
1035err_free:
1036	kho_unpreserve_vmalloc(preservation);
1037	return err;
1038}
1039EXPORT_SYMBOL_GPL(kho_preserve_vmalloc);
1040
1041/**
1042 * kho_unpreserve_vmalloc - unpreserve memory allocated with vmalloc()
1043 * @preservation: preservation metadata returned by kho_preserve_vmalloc()
1044 *
1045 * Instructs KHO to unpreserve the area in vmalloc address space that was
1046 * previously preserved with kho_preserve_vmalloc().
1047 */
1048void kho_unpreserve_vmalloc(struct kho_vmalloc *preservation)
1049{
1050	struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(preservation->first);
1051
1052	while (chunk) {
1053		struct kho_vmalloc_chunk *tmp = chunk;
1054
1055		kho_vmalloc_unpreserve_chunk(chunk, preservation->order);
1056
1057		chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
1058		free_page((unsigned long)tmp);
1059	}
1060}
1061EXPORT_SYMBOL_GPL(kho_unpreserve_vmalloc);
1062
1063/**
1064 * kho_restore_vmalloc - recreates and populates an area in vmalloc address
1065 * space from the preserved memory.
1066 * @preservation: preservation metadata.
1067 *
1068 * Recreates an area in vmalloc address space and populates it with memory that
1069 * was preserved using kho_preserve_vmalloc().
1070 *
1071 * Return: pointer to the area in the vmalloc address space, NULL on failure.
1072 */
1073void *kho_restore_vmalloc(const struct kho_vmalloc *preservation)
1074{
1075	struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(preservation->first);
1076	unsigned int align, order, shift, vm_flags;
1077	unsigned long total_pages, contig_pages;
1078	unsigned long addr, size;
1079	struct vm_struct *area;
1080	struct page **pages;
1081	unsigned int idx = 0;
1082	int err;
1083
1084	vm_flags = kho_flags_to_vmalloc(preservation->flags);
1085	if (vm_flags & ~KHO_VMALLOC_SUPPORTED_FLAGS)
1086		return NULL;
1087
1088	total_pages = preservation->total_pages;
1089	pages = kvmalloc_array(total_pages, sizeof(*pages), GFP_KERNEL);
1090	if (!pages)
1091		return NULL;
1092	order = preservation->order;
1093	contig_pages = (1 << order);
1094	shift = PAGE_SHIFT + order;
1095	align = 1 << shift;
1096
1097	while (chunk) {
1098		struct page *page;
1099
1100		for (int i = 0; i < ARRAY_SIZE(chunk->phys) && chunk->phys[i]; i++) {
1101			phys_addr_t phys = chunk->phys[i];
1102
1103			if (idx + contig_pages > total_pages)
1104				goto err_free_pages_array;
1105
1106			page = kho_restore_pages(phys, contig_pages);
1107			if (!page)
1108				goto err_free_pages_array;
1109
1110			for (int j = 0; j < contig_pages; j++)
1111				pages[idx++] = page + j;
1112
1113			phys += contig_pages * PAGE_SIZE;
1114		}
1115
1116		page = kho_restore_pages(virt_to_phys(chunk), 1);
1117		if (!page)
1118			goto err_free_pages_array;
1119		chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
1120		__free_page(page);
1121	}
1122
1123	if (idx != total_pages)
1124		goto err_free_pages_array;
1125
1126	area = __get_vm_area_node(total_pages * PAGE_SIZE, align, shift,
1127				  vm_flags, VMALLOC_START, VMALLOC_END,
1128				  NUMA_NO_NODE, GFP_KERNEL,
1129				  __builtin_return_address(0));
1130	if (!area)
1131		goto err_free_pages_array;
1132
1133	addr = (unsigned long)area->addr;
1134	size = get_vm_area_size(area);
1135	err = vmap_pages_range(addr, addr + size, PAGE_KERNEL, pages, shift);
1136	if (err)
1137		goto err_free_vm_area;
1138
1139	area->nr_pages = total_pages;
1140	area->pages = pages;
1141
1142	return area->addr;
1143
1144err_free_vm_area:
1145	free_vm_area(area);
1146err_free_pages_array:
1147	kvfree(pages);
1148	return NULL;
1149}
1150EXPORT_SYMBOL_GPL(kho_restore_vmalloc);
1151
1152/**
1153 * kho_alloc_preserve - Allocate, zero, and preserve memory.
1154 * @size: The number of bytes to allocate.
1155 *
1156 * Allocates a physically contiguous block of zeroed pages that is large
1157 * enough to hold @size bytes. The allocated memory is then registered with
1158 * KHO for preservation across a kexec.
1159 *
1160 * Note: The actual allocated size will be rounded up to the nearest
1161 * power-of-two page boundary.
1162 *
1163 * @return A virtual pointer to the allocated and preserved memory on success,
1164 * or an ERR_PTR() encoded error on failure.
1165 */
1166void *kho_alloc_preserve(size_t size)
1167{
1168	struct folio *folio;
1169	int order, ret;
1170
1171	if (!size)
1172		return ERR_PTR(-EINVAL);
1173
1174	order = get_order(size);
1175	if (order > MAX_PAGE_ORDER)
1176		return ERR_PTR(-E2BIG);
1177
1178	folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, order);
1179	if (!folio)
1180		return ERR_PTR(-ENOMEM);
1181
1182	ret = kho_preserve_folio(folio);
1183	if (ret) {
1184		folio_put(folio);
1185		return ERR_PTR(ret);
1186	}
1187
1188	return folio_address(folio);
1189}
1190EXPORT_SYMBOL_GPL(kho_alloc_preserve);
1191
1192/**
1193 * kho_unpreserve_free - Unpreserve and free memory.
1194 * @mem:  Pointer to the memory allocated by kho_alloc_preserve().
1195 *
1196 * Unregisters the memory from KHO preservation and frees the underlying
1197 * pages back to the system. This function should be called to clean up
1198 * memory allocated with kho_alloc_preserve().
1199 */
1200void kho_unpreserve_free(void *mem)
1201{
1202	struct folio *folio;
1203
1204	if (!mem)
1205		return;
1206
1207	folio = virt_to_folio(mem);
1208	kho_unpreserve_folio(folio);
1209	folio_put(folio);
1210}
1211EXPORT_SYMBOL_GPL(kho_unpreserve_free);
1212
1213/**
1214 * kho_restore_free - Restore and free memory after kexec.
1215 * @mem:  Pointer to the memory (in the new kernel's address space)
1216 * that was allocated by the old kernel.
1217 *
1218 * This function is intended to be called in the new kernel (post-kexec)
1219 * to take ownership of and free a memory region that was preserved by the
1220 * old kernel using kho_alloc_preserve().
1221 *
1222 * It first restores the pages from KHO (using their physical address)
1223 * and then frees the pages back to the new kernel's page allocator.
1224 */
1225void kho_restore_free(void *mem)
1226{
1227	struct folio *folio;
1228
1229	if (!mem)
1230		return;
1231
1232	folio = kho_restore_folio(__pa(mem));
1233	if (!WARN_ON(!folio))
1234		folio_put(folio);
1235}
1236EXPORT_SYMBOL_GPL(kho_restore_free);
1237
1238int kho_finalize(void)
1239{
1240	int ret;
1241
1242	if (!kho_enable)
1243		return -EOPNOTSUPP;
1244
1245	guard(mutex)(&kho_out.lock);
1246	ret = kho_mem_serialize(&kho_out);
1247	if (ret)
1248		return ret;
1249
1250	kho_out.finalized = true;
1251
1252	return 0;
1253}
1254
1255bool kho_finalized(void)
1256{
1257	guard(mutex)(&kho_out.lock);
1258	return kho_out.finalized;
1259}
1260
1261struct kho_in {
1262	phys_addr_t fdt_phys;
1263	phys_addr_t scratch_phys;
1264	phys_addr_t mem_map_phys;
1265	struct kho_debugfs dbg;
1266};
1267
1268static struct kho_in kho_in = {
1269};
1270
1271static const void *kho_get_fdt(void)
1272{
1273	return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL;
1274}
1275
1276/**
1277 * is_kho_boot - check if current kernel was booted via KHO-enabled
1278 * kexec
1279 *
1280 * This function checks if the current kernel was loaded through a kexec
1281 * operation with KHO enabled, by verifying that a valid KHO FDT
1282 * was passed.
1283 *
1284 * Note: This function returns reliable results only after
1285 * kho_populate() has been called during early boot. Before that,
1286 * it may return false even if KHO data is present.
1287 *
1288 * Return: true if booted via KHO-enabled kexec, false otherwise
1289 */
1290bool is_kho_boot(void)
1291{
1292	return !!kho_get_fdt();
1293}
1294EXPORT_SYMBOL_GPL(is_kho_boot);
1295
1296/**
1297 * kho_retrieve_subtree - retrieve a preserved sub FDT by its name.
1298 * @name: the name of the sub FDT passed to kho_add_subtree().
1299 * @phys: if found, the physical address of the sub FDT is stored in @phys.
1300 *
1301 * Retrieve a preserved sub FDT named @name and store its physical
1302 * address in @phys.
1303 *
1304 * Return: 0 on success, error code on failure
1305 */
1306int kho_retrieve_subtree(const char *name, phys_addr_t *phys)
1307{
1308	const void *fdt = kho_get_fdt();
1309	const u64 *val;
1310	int offset, len;
1311
1312	if (!fdt)
1313		return -ENOENT;
1314
1315	if (!phys)
1316		return -EINVAL;
1317
1318	offset = fdt_subnode_offset(fdt, 0, name);
1319	if (offset < 0)
1320		return -ENOENT;
1321
1322	val = fdt_getprop(fdt, offset, PROP_SUB_FDT, &len);
1323	if (!val || len != sizeof(*val))
1324		return -EINVAL;
1325
1326	*phys = (phys_addr_t)*val;
1327
1328	return 0;
1329}
1330EXPORT_SYMBOL_GPL(kho_retrieve_subtree);
1331
1332static __init int kho_out_fdt_setup(void)
1333{
1334	void *root = kho_out.fdt;
1335	u64 empty_mem_map = 0;
1336	int err;
1337
1338	err = fdt_create(root, PAGE_SIZE);
1339	err |= fdt_finish_reservemap(root);
1340	err |= fdt_begin_node(root, "");
1341	err |= fdt_property_string(root, "compatible", KHO_FDT_COMPATIBLE);
1342	err |= fdt_property(root, PROP_PRESERVED_MEMORY_MAP, &empty_mem_map,
1343			    sizeof(empty_mem_map));
1344	err |= fdt_end_node(root);
1345	err |= fdt_finish(root);
1346
1347	return err;
1348}
1349
1350static __init int kho_init(void)
1351{
1352	const void *fdt = kho_get_fdt();
1353	int err = 0;
1354
1355	if (!kho_enable)
1356		return 0;
1357
1358	kho_out.fdt = kho_alloc_preserve(PAGE_SIZE);
1359	if (IS_ERR(kho_out.fdt)) {
1360		err = PTR_ERR(kho_out.fdt);
1361		goto err_free_scratch;
1362	}
1363
1364	err = kho_debugfs_init();
1365	if (err)
1366		goto err_free_fdt;
1367
1368	err = kho_out_debugfs_init(&kho_out.dbg);
1369	if (err)
1370		goto err_free_fdt;
1371
1372	err = kho_out_fdt_setup();
1373	if (err)
1374		goto err_free_fdt;
1375
1376	if (fdt) {
1377		kho_in_debugfs_init(&kho_in.dbg, fdt);
1378		return 0;
1379	}
1380
1381	for (int i = 0; i < kho_scratch_cnt; i++) {
1382		unsigned long base_pfn = PHYS_PFN(kho_scratch[i].addr);
1383		unsigned long count = kho_scratch[i].size >> PAGE_SHIFT;
1384		unsigned long pfn;
1385
1386		/*
1387		 * When debug_pagealloc is enabled, __free_pages() clears the
1388		 * corresponding PRESENT bit in the kernel page table.
1389		 * Subsequent kmemleak scans of these pages cause the
1390		 * non-PRESENT page faults.
1391		 * Mark scratch areas with kmemleak_ignore_phys() to exclude
1392		 * them from kmemleak scanning.
1393		 */
1394		kmemleak_ignore_phys(kho_scratch[i].addr);
1395		for (pfn = base_pfn; pfn < base_pfn + count;
1396		     pfn += pageblock_nr_pages)
1397			init_cma_reserved_pageblock(pfn_to_page(pfn));
1398	}
1399
1400	WARN_ON_ONCE(kho_debugfs_fdt_add(&kho_out.dbg, "fdt",
1401					 kho_out.fdt, true));
1402
1403	return 0;
1404
1405err_free_fdt:
1406	kho_unpreserve_free(kho_out.fdt);
1407err_free_scratch:
1408	kho_out.fdt = NULL;
1409	for (int i = 0; i < kho_scratch_cnt; i++) {
1410		void *start = __va(kho_scratch[i].addr);
1411		void *end = start + kho_scratch[i].size;
1412
1413		free_reserved_area(start, end, -1, "");
1414	}
1415	kho_enable = false;
1416	return err;
1417}
1418fs_initcall(kho_init);
1419
1420static void __init kho_release_scratch(void)
1421{
1422	phys_addr_t start, end;
1423	u64 i;
1424
1425	memmap_init_kho_scratch_pages();
1426
1427	/*
1428	 * Mark scratch mem as CMA before we return it. That way we
1429	 * ensure that no kernel allocations happen on it. That means
1430	 * we can reuse it as scratch memory again later.
1431	 */
1432	__for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE,
1433			     MEMBLOCK_KHO_SCRATCH, &start, &end, NULL) {
1434		ulong start_pfn = pageblock_start_pfn(PFN_DOWN(start));
1435		ulong end_pfn = pageblock_align(PFN_UP(end));
1436		ulong pfn;
1437
1438		for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages)
1439			init_pageblock_migratetype(pfn_to_page(pfn),
1440						   MIGRATE_CMA, false);
1441	}
1442}
1443
1444void __init kho_memory_init(void)
1445{
1446	if (kho_in.mem_map_phys) {
1447		kho_scratch = phys_to_virt(kho_in.scratch_phys);
1448		kho_release_scratch();
1449		kho_mem_deserialize(phys_to_virt(kho_in.mem_map_phys));
1450	} else {
1451		kho_reserve_scratch();
1452	}
1453}
1454
1455void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
1456			 phys_addr_t scratch_phys, u64 scratch_len)
1457{
1458	struct kho_scratch *scratch = NULL;
1459	phys_addr_t mem_map_phys;
1460	void *fdt = NULL;
1461	int err = 0;
1462	unsigned int scratch_cnt = scratch_len / sizeof(*kho_scratch);
1463
1464	/* Validate the input FDT */
1465	fdt = early_memremap(fdt_phys, fdt_len);
1466	if (!fdt) {
1467		pr_warn("setup: failed to memremap FDT (0x%llx)\n", fdt_phys);
1468		err = -EFAULT;
1469		goto out;
1470	}
1471	err = fdt_check_header(fdt);
1472	if (err) {
1473		pr_warn("setup: handover FDT (0x%llx) is invalid: %d\n",
1474			fdt_phys, err);
1475		err = -EINVAL;
1476		goto out;
1477	}
1478	err = fdt_node_check_compatible(fdt, 0, KHO_FDT_COMPATIBLE);
1479	if (err) {
1480		pr_warn("setup: handover FDT (0x%llx) is incompatible with '%s': %d\n",
1481			fdt_phys, KHO_FDT_COMPATIBLE, err);
1482		err = -EINVAL;
1483		goto out;
1484	}
1485
1486	mem_map_phys = kho_get_mem_map_phys(fdt);
1487	if (!mem_map_phys) {
1488		err = -ENOENT;
1489		goto out;
1490	}
1491
1492	scratch = early_memremap(scratch_phys, scratch_len);
1493	if (!scratch) {
1494		pr_warn("setup: failed to memremap scratch (phys=0x%llx, len=%lld)\n",
1495			scratch_phys, scratch_len);
1496		err = -EFAULT;
1497		goto out;
1498	}
1499
1500	/*
1501	 * We pass a safe contiguous blocks of memory to use for early boot
1502	 * purporses from the previous kernel so that we can resize the
1503	 * memblock array as needed.
1504	 */
1505	for (int i = 0; i < scratch_cnt; i++) {
1506		struct kho_scratch *area = &scratch[i];
1507		u64 size = area->size;
1508
1509		memblock_add(area->addr, size);
1510		err = memblock_mark_kho_scratch(area->addr, size);
1511		if (WARN_ON(err)) {
1512			pr_warn("failed to mark the scratch region 0x%pa+0x%pa: %pe",
1513				&area->addr, &size, ERR_PTR(err));
1514			goto out;
1515		}
1516		pr_debug("Marked 0x%pa+0x%pa as scratch", &area->addr, &size);
1517	}
1518
1519	memblock_reserve(scratch_phys, scratch_len);
1520
1521	/*
1522	 * Now that we have a viable region of scratch memory, let's tell
1523	 * the memblocks allocator to only use that for any allocations.
1524	 * That way we ensure that nothing scribbles over in use data while
1525	 * we initialize the page tables which we will need to ingest all
1526	 * memory reservations from the previous kernel.
1527	 */
1528	memblock_set_kho_scratch_only();
1529
1530	kho_in.fdt_phys = fdt_phys;
1531	kho_in.scratch_phys = scratch_phys;
1532	kho_in.mem_map_phys = mem_map_phys;
1533	kho_scratch_cnt = scratch_cnt;
1534	pr_info("found kexec handover data.\n");
1535
1536out:
1537	if (fdt)
1538		early_memunmap(fdt, fdt_len);
1539	if (scratch)
1540		early_memunmap(scratch, scratch_len);
1541	if (err)
1542		pr_warn("disabling KHO revival: %d\n", err);
1543}
1544
1545/* Helper functions for kexec_file_load */
1546
1547int kho_fill_kimage(struct kimage *image)
1548{
1549	ssize_t scratch_size;
1550	int err = 0;
1551	struct kexec_buf scratch;
1552
1553	if (!kho_enable)
1554		return 0;
1555
1556	image->kho.fdt = virt_to_phys(kho_out.fdt);
1557
1558	scratch_size = sizeof(*kho_scratch) * kho_scratch_cnt;
1559	scratch = (struct kexec_buf){
1560		.image = image,
1561		.buffer = kho_scratch,
1562		.bufsz = scratch_size,
1563		.mem = KEXEC_BUF_MEM_UNKNOWN,
1564		.memsz = scratch_size,
1565		.buf_align = SZ_64K, /* Makes it easier to map */
1566		.buf_max = ULONG_MAX,
1567		.top_down = true,
1568	};
1569	err = kexec_add_buffer(&scratch);
1570	if (err)
1571		return err;
1572	image->kho.scratch = &image->segment[image->nr_segments - 1];
1573
1574	return 0;
1575}
1576
1577static int kho_walk_scratch(struct kexec_buf *kbuf,
1578			    int (*func)(struct resource *, void *))
1579{
1580	int ret = 0;
1581	int i;
1582
1583	for (i = 0; i < kho_scratch_cnt; i++) {
1584		struct resource res = {
1585			.start = kho_scratch[i].addr,
1586			.end = kho_scratch[i].addr + kho_scratch[i].size - 1,
1587		};
1588
1589		/* Try to fit the kimage into our KHO scratch region */
1590		ret = func(&res, kbuf);
1591		if (ret)
1592			break;
1593	}
1594
1595	return ret;
1596}
1597
1598int kho_locate_mem_hole(struct kexec_buf *kbuf,
1599			int (*func)(struct resource *, void *))
1600{
1601	int ret;
1602
1603	if (!kho_enable || kbuf->image->type == KEXEC_TYPE_CRASH)
1604		return 1;
1605
1606	ret = kho_walk_scratch(kbuf, func);
1607
1608	return ret == 1 ? 0 : -EADDRNOTAVAIL;
1609}