fadump.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Firmware Assisted dump: A robust mechanism to get reliable kernel crash
   4 * dump with assistance from firmware. This approach does not use kexec,
   5 * instead firmware assists in booting the kdump kernel while preserving
   6 * memory contents. The most of the code implementation has been adapted
   7 * from phyp assisted dump implementation written by Linas Vepstas and
   8 * Manish Ahuja
   9 *
  10 * Copyright 2011 IBM Corporation
  11 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
  12 */
  13
  14#undef DEBUG
  15#define pr_fmt(fmt) "fadump: " fmt
  16
  17#include <linux/string.h>
  18#include <linux/memblock.h>
  19#include <linux/delay.h>
  20#include <linux/seq_file.h>
  21#include <linux/crash_dump.h>
  22#include <linux/kobject.h>
  23#include <linux/sysfs.h>
  24#include <linux/slab.h>
  25#include <linux/cma.h>
  26#include <linux/hugetlb.h>
  27#include <linux/debugfs.h>
  28#include <linux/of.h>
  29#include <linux/of_fdt.h>
  30
  31#include <asm/page.h>
  32#include <asm/fadump.h>
  33#include <asm/fadump-internal.h>
  34#include <asm/setup.h>
  35#include <asm/interrupt.h>
  36#include <asm/prom.h>
  37
  38/*
  39 * The CPU who acquired the lock to trigger the fadump crash should
  40 * wait for other CPUs to enter.
  41 *
  42 * The timeout is in milliseconds.
  43 */
  44#define CRASH_TIMEOUT		500
  45
  46static struct fw_dump fw_dump;
  47
  48static void __init fadump_reserve_crash_area(u64 base);
  49
  50#ifndef CONFIG_PRESERVE_FA_DUMP
  51
  52static struct kobject *fadump_kobj;
  53
  54static atomic_t cpus_in_fadump;
  55static DEFINE_MUTEX(fadump_mutex);
  56
  57#define RESERVED_RNGS_SZ	16384 /* 16K - 128 entries */
  58#define RESERVED_RNGS_CNT	(RESERVED_RNGS_SZ / \
  59				 sizeof(struct fadump_memory_range))
  60static struct fadump_memory_range rngs[RESERVED_RNGS_CNT];
  61static struct fadump_mrange_info
  62reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true };
  63
  64static void __init early_init_dt_scan_reserved_ranges(unsigned long node);
  65
  66#ifdef CONFIG_CMA
  67static struct cma *fadump_cma;
  68
  69/*
  70 * fadump_cma_init() - Initialize CMA area from a fadump reserved memory
  71 *
  72 * This function initializes CMA area from fadump reserved memory.
  73 * The total size of fadump reserved memory covers for boot memory size
  74 * + cpu data size + hpte size and metadata.
  75 * Initialize only the area equivalent to boot memory size for CMA use.
  76 * The remaining portion of fadump reserved memory will be not given
  77 * to CMA and pages for those will stay reserved. boot memory size is
  78 * aligned per CMA requirement to satisy cma_init_reserved_mem() call.
  79 * But for some reason even if it fails we still have the memory reservation
  80 * with us and we can still continue doing fadump.
  81 */
  82void __init fadump_cma_init(void)
  83{
  84	unsigned long long base, size, end;
  85	int rc;
  86
  87	if (!fw_dump.fadump_supported || !fw_dump.fadump_enabled ||
  88			fw_dump.dump_active)
  89		return;
  90	/*
  91	 * Do not use CMA if user has provided fadump=nocma kernel parameter.
  92	 */
  93	if (fw_dump.nocma || !fw_dump.boot_memory_size)
  94		return;
  95
  96	/*
  97	 * [base, end) should be reserved during early init in
  98	 * fadump_reserve_mem(). No need to check this here as
  99	 * cma_init_reserved_mem() already checks for overlap.
 100	 * Here we give the aligned chunk of this reserved memory to CMA.
 101	 */
 102	base = fw_dump.reserve_dump_area_start;
 103	size = fw_dump.boot_memory_size;
 104	end = base + size;
 105
 106	base = ALIGN(base, CMA_MIN_ALIGNMENT_BYTES);
 107	end = ALIGN_DOWN(end, CMA_MIN_ALIGNMENT_BYTES);
 108	size = end - base;
 109
 110	if (end <= base) {
 111		pr_warn("%s: Too less memory to give to CMA\n", __func__);
 112		return;
 113	}
 114
 115	rc = cma_init_reserved_mem(base, size, 0, "fadump_cma", &fadump_cma);
 116	if (rc) {
 117		pr_err("Failed to init cma area for firmware-assisted dump,%d\n", rc);
 118		/*
 119		 * Though the CMA init has failed we still have memory
 120		 * reservation with us. The reserved memory will be
 121		 * blocked from production system usage.  Hence return 1,
 122		 * so that we can continue with fadump.
 123		 */
 124		return;
 125	}
 126
 127	/*
 128	 *  If CMA activation fails, keep the pages reserved, instead of
 129	 *  exposing them to buddy allocator. Same as 'fadump=nocma' case.
 130	 */
 131	cma_reserve_pages_on_error(fadump_cma);
 132
 133	/*
 134	 * So we now have successfully initialized cma area for fadump.
 135	 */
 136	pr_info("Initialized [0x%llx, %luMB] cma area from [0x%lx, %luMB] "
 137		"bytes of memory reserved for firmware-assisted dump\n",
 138		cma_get_base(fadump_cma), cma_get_size(fadump_cma) >> 20,
 139		fw_dump.reserve_dump_area_start,
 140		fw_dump.boot_memory_size >> 20);
 141	return;
 142}
 143#endif /* CONFIG_CMA */
 144
 145/*
 146 * Additional parameters meant for capture kernel are placed in a dedicated area.
 147 * If this is capture kernel boot, append these parameters to bootargs.
 148 */
 149void __init fadump_append_bootargs(void)
 150{
 151	char *append_args;
 152	size_t len;
 153
 154	if (!fw_dump.dump_active || !fw_dump.param_area_supported || !fw_dump.param_area)
 155		return;
 156
 157	if (fw_dump.param_area < fw_dump.boot_mem_top) {
 158		if (memblock_reserve(fw_dump.param_area, COMMAND_LINE_SIZE)) {
 159			pr_warn("WARNING: Can't use additional parameters area!\n");
 160			fw_dump.param_area = 0;
 161			return;
 162		}
 163	}
 164
 165	append_args = (char *)fw_dump.param_area;
 166	len = strlen(boot_command_line);
 167
 168	/*
 169	 * Too late to fail even if cmdline size exceeds. Truncate additional parameters
 170	 * to cmdline size and proceed anyway.
 171	 */
 172	if (len + strlen(append_args) >= COMMAND_LINE_SIZE - 1)
 173		pr_warn("WARNING: Appending parameters exceeds cmdline size. Truncating!\n");
 174
 175	pr_debug("Cmdline: %s\n", boot_command_line);
 176	snprintf(boot_command_line + len, COMMAND_LINE_SIZE - len, " %s", append_args);
 177	pr_info("Updated cmdline: %s\n", boot_command_line);
 178}
 179
 180/* Scan the Firmware Assisted dump configuration details. */
 181int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname,
 182				      int depth, void *data)
 183{
 184	if (depth == 0) {
 185		early_init_dt_scan_reserved_ranges(node);
 186		return 0;
 187	}
 188
 189	if (depth != 1)
 190		return 0;
 191
 192	if (strcmp(uname, "rtas") == 0) {
 193		rtas_fadump_dt_scan(&fw_dump, node);
 194		return 1;
 195	}
 196
 197	if (strcmp(uname, "ibm,opal") == 0) {
 198		opal_fadump_dt_scan(&fw_dump, node);
 199		return 1;
 200	}
 201
 202	return 0;
 203}
 204
 205/*
 206 * If fadump is registered, check if the memory provided
 207 * falls within boot memory area and reserved memory area.
 208 */
 209int is_fadump_memory_area(u64 addr, unsigned long size)
 210{
 211	u64 d_start, d_end;
 212
 213	if (!fw_dump.dump_registered)
 214		return 0;
 215
 216	if (!size)
 217		return 0;
 218
 219	d_start = fw_dump.reserve_dump_area_start;
 220	d_end = d_start + fw_dump.reserve_dump_area_size;
 221	if (((addr + size) > d_start) && (addr <= d_end))
 222		return 1;
 223
 224	return (addr <= fw_dump.boot_mem_top);
 225}
 226
 227int should_fadump_crash(void)
 228{
 229	if (!fw_dump.dump_registered || !fw_dump.fadumphdr_addr)
 230		return 0;
 231	return 1;
 232}
 233
 234int is_fadump_active(void)
 235{
 236	return fw_dump.dump_active;
 237}
 238
 239/*
 240 * Returns true, if there are no holes in memory area between d_start to d_end,
 241 * false otherwise.
 242 */
 243static bool is_fadump_mem_area_contiguous(u64 d_start, u64 d_end)
 244{
 245	phys_addr_t reg_start, reg_end;
 246	bool ret = false;
 247	u64 i, start, end;
 248
 249	for_each_mem_range(i, &reg_start, &reg_end) {
 250		start = max_t(u64, d_start, reg_start);
 251		end = min_t(u64, d_end, reg_end);
 252		if (d_start < end) {
 253			/* Memory hole from d_start to start */
 254			if (start > d_start)
 255				break;
 256
 257			if (end == d_end) {
 258				ret = true;
 259				break;
 260			}
 261
 262			d_start = end + 1;
 263		}
 264	}
 265
 266	return ret;
 267}
 268
 269/*
 270 * Returns true, if there are no holes in reserved memory area,
 271 * false otherwise.
 272 */
 273bool is_fadump_reserved_mem_contiguous(void)
 274{
 275	u64 d_start, d_end;
 276
 277	d_start	= fw_dump.reserve_dump_area_start;
 278	d_end	= d_start + fw_dump.reserve_dump_area_size;
 279	return is_fadump_mem_area_contiguous(d_start, d_end);
 280}
 281
 282/* Print firmware assisted dump configurations for debugging purpose. */
 283static void __init fadump_show_config(void)
 284{
 285	int i;
 286
 287	pr_debug("Support for firmware-assisted dump (fadump): %s\n",
 288			(fw_dump.fadump_supported ? "present" : "no support"));
 289
 290	if (!fw_dump.fadump_supported)
 291		return;
 292
 293	pr_debug("Fadump enabled    : %s\n", str_yes_no(fw_dump.fadump_enabled));
 294	pr_debug("Dump Active       : %s\n", str_yes_no(fw_dump.dump_active));
 295	pr_debug("Dump section sizes:\n");
 296	pr_debug("    CPU state data size: %lx\n", fw_dump.cpu_state_data_size);
 297	pr_debug("    HPTE region size   : %lx\n", fw_dump.hpte_region_size);
 298	pr_debug("    Boot memory size   : %lx\n", fw_dump.boot_memory_size);
 299	pr_debug("    Boot memory top    : %llx\n", fw_dump.boot_mem_top);
 300	pr_debug("Boot memory regions cnt: %llx\n", fw_dump.boot_mem_regs_cnt);
 301	for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) {
 302		pr_debug("[%03d] base = %llx, size = %llx\n", i,
 303			 fw_dump.boot_mem_addr[i], fw_dump.boot_mem_sz[i]);
 304	}
 305}
 306
 307/**
 308 * fadump_calculate_reserve_size(): reserve variable boot area 5% of System RAM
 309 *
 310 * Function to find the largest memory size we need to reserve during early
 311 * boot process. This will be the size of the memory that is required for a
 312 * kernel to boot successfully.
 313 *
 314 * This function has been taken from phyp-assisted dump feature implementation.
 315 *
 316 * returns larger of 256MB or 5% rounded down to multiples of 256MB.
 317 *
 318 * TODO: Come up with better approach to find out more accurate memory size
 319 * that is required for a kernel to boot successfully.
 320 *
 321 */
 322static __init u64 fadump_calculate_reserve_size(void)
 323{
 324	u64 base, size, bootmem_min;
 325	int ret;
 326
 327	if (fw_dump.reserve_bootvar)
 328		pr_warn("'fadump_reserve_mem=' parameter is deprecated in favor of 'crashkernel=' parameter.\n");
 329
 330	/*
 331	 * Check if the size is specified through crashkernel= cmdline
 332	 * option. If yes, then use that but ignore base as fadump reserves
 333	 * memory at a predefined offset.
 334	 */
 335	ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
 336				&size, &base, NULL, NULL, NULL);
 337	if (ret == 0 && size > 0) {
 338		unsigned long max_size;
 339
 340		if (fw_dump.reserve_bootvar)
 341			pr_info("Using 'crashkernel=' parameter for memory reservation.\n");
 342
 343		fw_dump.reserve_bootvar = (unsigned long)size;
 344
 345		/*
 346		 * Adjust if the boot memory size specified is above
 347		 * the upper limit.
 348		 */
 349		max_size = memblock_phys_mem_size() / MAX_BOOT_MEM_RATIO;
 350		if (fw_dump.reserve_bootvar > max_size) {
 351			fw_dump.reserve_bootvar = max_size;
 352			pr_info("Adjusted boot memory size to %luMB\n",
 353				(fw_dump.reserve_bootvar >> 20));
 354		}
 355
 356		return fw_dump.reserve_bootvar;
 357	} else if (fw_dump.reserve_bootvar) {
 358		/*
 359		 * 'fadump_reserve_mem=' is being used to reserve memory
 360		 * for firmware-assisted dump.
 361		 */
 362		return fw_dump.reserve_bootvar;
 363	}
 364
 365	/* divide by 20 to get 5% of value */
 366	size = memblock_phys_mem_size() / 20;
 367
 368	/* round it down in multiples of 256 */
 369	size = size & ~0x0FFFFFFFUL;
 370
 371	/* Truncate to memory_limit. We don't want to over reserve the memory.*/
 372	if (memory_limit && size > memory_limit)
 373		size = memory_limit;
 374
 375	bootmem_min = fw_dump.ops->fadump_get_bootmem_min();
 376	return (size > bootmem_min ? size : bootmem_min);
 377}
 378
 379/*
 380 * Calculate the total memory size required to be reserved for
 381 * firmware-assisted dump registration.
 382 */
 383static unsigned long __init get_fadump_area_size(void)
 384{
 385	unsigned long size = 0;
 386
 387	size += fw_dump.cpu_state_data_size;
 388	size += fw_dump.hpte_region_size;
 389	/*
 390	 * Account for pagesize alignment of boot memory area destination address.
 391	 * This faciliates in mmap reading of first kernel's memory.
 392	 */
 393	size = PAGE_ALIGN(size);
 394	size += fw_dump.boot_memory_size;
 395	size += sizeof(struct fadump_crash_info_header);
 396
 397	/* This is to hold kernel metadata on platforms that support it */
 398	size += (fw_dump.ops->fadump_get_metadata_size ?
 399		 fw_dump.ops->fadump_get_metadata_size() : 0);
 400	return size;
 401}
 402
 403static int __init add_boot_mem_region(unsigned long rstart,
 404				      unsigned long rsize)
 405{
 406	int max_boot_mem_rgns = fw_dump.ops->fadump_max_boot_mem_rgns();
 407	int i = fw_dump.boot_mem_regs_cnt++;
 408
 409	if (fw_dump.boot_mem_regs_cnt > max_boot_mem_rgns) {
 410		fw_dump.boot_mem_regs_cnt = max_boot_mem_rgns;
 411		return 0;
 412	}
 413
 414	pr_debug("Added boot memory range[%d] [%#016lx-%#016lx)\n",
 415		 i, rstart, (rstart + rsize));
 416	fw_dump.boot_mem_addr[i] = rstart;
 417	fw_dump.boot_mem_sz[i] = rsize;
 418	return 1;
 419}
 420
 421/*
 422 * Firmware usually has a hard limit on the data it can copy per region.
 423 * Honour that by splitting a memory range into multiple regions.
 424 */
 425static int __init add_boot_mem_regions(unsigned long mstart,
 426				       unsigned long msize)
 427{
 428	unsigned long rstart, rsize, max_size;
 429	int ret = 1;
 430
 431	rstart = mstart;
 432	max_size = fw_dump.max_copy_size ? fw_dump.max_copy_size : msize;
 433	while (msize) {
 434		if (msize > max_size)
 435			rsize = max_size;
 436		else
 437			rsize = msize;
 438
 439		ret = add_boot_mem_region(rstart, rsize);
 440		if (!ret)
 441			break;
 442
 443		msize -= rsize;
 444		rstart += rsize;
 445	}
 446
 447	return ret;
 448}
 449
 450static int __init fadump_get_boot_mem_regions(void)
 451{
 452	unsigned long size, cur_size, hole_size, last_end;
 453	unsigned long mem_size = fw_dump.boot_memory_size;
 454	phys_addr_t reg_start, reg_end;
 455	int ret = 1;
 456	u64 i;
 457
 458	fw_dump.boot_mem_regs_cnt = 0;
 459
 460	last_end = 0;
 461	hole_size = 0;
 462	cur_size = 0;
 463	for_each_mem_range(i, &reg_start, &reg_end) {
 464		size = reg_end - reg_start;
 465		hole_size += (reg_start - last_end);
 466
 467		if ((cur_size + size) >= mem_size) {
 468			size = (mem_size - cur_size);
 469			ret = add_boot_mem_regions(reg_start, size);
 470			break;
 471		}
 472
 473		mem_size -= size;
 474		cur_size += size;
 475		ret = add_boot_mem_regions(reg_start, size);
 476		if (!ret)
 477			break;
 478
 479		last_end = reg_end;
 480	}
 481	fw_dump.boot_mem_top = PAGE_ALIGN(fw_dump.boot_memory_size + hole_size);
 482
 483	return ret;
 484}
 485
 486/*
 487 * Returns true, if the given range overlaps with reserved memory ranges
 488 * starting at idx. Also, updates idx to index of overlapping memory range
 489 * with the given memory range.
 490 * False, otherwise.
 491 */
 492static bool __init overlaps_reserved_ranges(u64 base, u64 end, int *idx)
 493{
 494	bool ret = false;
 495	int i;
 496
 497	for (i = *idx; i < reserved_mrange_info.mem_range_cnt; i++) {
 498		u64 rbase = reserved_mrange_info.mem_ranges[i].base;
 499		u64 rend = rbase + reserved_mrange_info.mem_ranges[i].size;
 500
 501		if (end <= rbase)
 502			break;
 503
 504		if ((end > rbase) &&  (base < rend)) {
 505			*idx = i;
 506			ret = true;
 507			break;
 508		}
 509	}
 510
 511	return ret;
 512}
 513
 514/*
 515 * Locate a suitable memory area to reserve memory for FADump. While at it,
 516 * lookup reserved-ranges & avoid overlap with them, as they are used by F/W.
 517 */
 518static u64 __init fadump_locate_reserve_mem(u64 base, u64 size)
 519{
 520	struct fadump_memory_range *mrngs;
 521	phys_addr_t mstart, mend;
 522	int idx = 0;
 523	u64 i, ret = 0;
 524
 525	mrngs = reserved_mrange_info.mem_ranges;
 526	for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
 527				&mstart, &mend, NULL) {
 528		pr_debug("%llu) mstart: %llx, mend: %llx, base: %llx\n",
 529			 i, mstart, mend, base);
 530
 531		if (mstart > base)
 532			base = PAGE_ALIGN(mstart);
 533
 534		while ((mend > base) && ((mend - base) >= size)) {
 535			if (!overlaps_reserved_ranges(base, base+size, &idx)) {
 536				ret = base;
 537				goto out;
 538			}
 539
 540			base = mrngs[idx].base + mrngs[idx].size;
 541			base = PAGE_ALIGN(base);
 542		}
 543	}
 544
 545out:
 546	return ret;
 547}
 548
 549int __init fadump_reserve_mem(void)
 550{
 551	u64 base, size, mem_boundary, bootmem_min;
 552	int ret = 1;
 553
 554	if (!fw_dump.fadump_enabled)
 555		return 0;
 556
 557	if (!fw_dump.fadump_supported) {
 558		pr_info("Firmware-Assisted Dump is not supported on this hardware\n");
 559		goto error_out;
 560	}
 561
 562	/*
 563	 * Initialize boot memory size
 564	 * If dump is active then we have already calculated the size during
 565	 * first kernel.
 566	 */
 567	if (!fw_dump.dump_active) {
 568		fw_dump.boot_memory_size =
 569			PAGE_ALIGN(fadump_calculate_reserve_size());
 570
 571		bootmem_min = fw_dump.ops->fadump_get_bootmem_min();
 572		if (fw_dump.boot_memory_size < bootmem_min) {
 573			pr_err("Can't enable fadump with boot memory size (0x%lx) less than 0x%llx\n",
 574			       fw_dump.boot_memory_size, bootmem_min);
 575			goto error_out;
 576		}
 577
 578		if (!fadump_get_boot_mem_regions()) {
 579			pr_err("Too many holes in boot memory area to enable fadump\n");
 580			goto error_out;
 581		}
 582	}
 583
 584	if (memory_limit)
 585		mem_boundary = memory_limit;
 586	else
 587		mem_boundary = memblock_end_of_DRAM();
 588
 589	base = fw_dump.boot_mem_top;
 590	size = get_fadump_area_size();
 591	fw_dump.reserve_dump_area_size = size;
 592	if (fw_dump.dump_active) {
 593		pr_info("Firmware-assisted dump is active.\n");
 594
 595#ifdef CONFIG_HUGETLB_PAGE
 596		/*
 597		 * FADump capture kernel doesn't care much about hugepages.
 598		 * In fact, handling hugepages in capture kernel is asking for
 599		 * trouble. So, disable HugeTLB support when fadump is active.
 600		 */
 601		hugetlb_disabled = true;
 602#endif
 603		/*
 604		 * If last boot has crashed then reserve all the memory
 605		 * above boot memory size so that we don't touch it until
 606		 * dump is written to disk by userspace tool. This memory
 607		 * can be released for general use by invalidating fadump.
 608		 */
 609		fadump_reserve_crash_area(base);
 610
 611		pr_debug("fadumphdr_addr = %#016lx\n", fw_dump.fadumphdr_addr);
 612		pr_debug("Reserve dump area start address: 0x%lx\n",
 613			 fw_dump.reserve_dump_area_start);
 614	} else {
 615		/*
 616		 * Reserve memory at an offset closer to bottom of the RAM to
 617		 * minimize the impact of memory hot-remove operation.
 618		 */
 619		base = fadump_locate_reserve_mem(base, size);
 620
 621		if (!base || (base + size > mem_boundary)) {
 622			pr_err("Failed to find memory chunk for reservation!\n");
 623			goto error_out;
 624		}
 625		fw_dump.reserve_dump_area_start = base;
 626
 627		/*
 628		 * Calculate the kernel metadata address and register it with
 629		 * f/w if the platform supports.
 630		 */
 631		if (fw_dump.ops->fadump_setup_metadata &&
 632		    (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))
 633			goto error_out;
 634
 635		if (memblock_reserve(base, size)) {
 636			pr_err("Failed to reserve memory!\n");
 637			goto error_out;
 638		}
 639
 640		pr_info("Reserved %lldMB of memory at %#016llx (System RAM: %lldMB)\n",
 641			(size >> 20), base, (memblock_phys_mem_size() >> 20));
 642	}
 643
 644	return ret;
 645error_out:
 646	fw_dump.fadump_enabled = 0;
 647	fw_dump.reserve_dump_area_size = 0;
 648	return 0;
 649}
 650
 651/* Look for fadump= cmdline option. */
 652static int __init early_fadump_param(char *p)
 653{
 654	if (!p)
 655		return 1;
 656
 657	if (strncmp(p, "on", 2) == 0)
 658		fw_dump.fadump_enabled = 1;
 659	else if (strncmp(p, "off", 3) == 0)
 660		fw_dump.fadump_enabled = 0;
 661	else if (strncmp(p, "nocma", 5) == 0) {
 662		fw_dump.fadump_enabled = 1;
 663		fw_dump.nocma = 1;
 664	}
 665
 666	return 0;
 667}
 668early_param("fadump", early_fadump_param);
 669
 670/*
 671 * Look for fadump_reserve_mem= cmdline option
 672 * TODO: Remove references to 'fadump_reserve_mem=' parameter,
 673 *       the sooner 'crashkernel=' parameter is accustomed to.
 674 */
 675static int __init early_fadump_reserve_mem(char *p)
 676{
 677	if (p)
 678		fw_dump.reserve_bootvar = memparse(p, &p);
 679	return 0;
 680}
 681early_param("fadump_reserve_mem", early_fadump_reserve_mem);
 682
 683void crash_fadump(struct pt_regs *regs, const char *str)
 684{
 685	unsigned int msecs;
 686	struct fadump_crash_info_header *fdh = NULL;
 687	int old_cpu, this_cpu;
 688	/* Do not include first CPU */
 689	unsigned int ncpus = num_online_cpus() - 1;
 690
 691	if (!should_fadump_crash())
 692		return;
 693
 694	/*
 695	 * old_cpu == -1 means this is the first CPU which has come here,
 696	 * go ahead and trigger fadump.
 697	 *
 698	 * old_cpu != -1 means some other CPU has already on its way
 699	 * to trigger fadump, just keep looping here.
 700	 */
 701	this_cpu = smp_processor_id();
 702	old_cpu = cmpxchg(&crashing_cpu, -1, this_cpu);
 703
 704	if (old_cpu != -1) {
 705		atomic_inc(&cpus_in_fadump);
 706
 707		/*
 708		 * We can't loop here indefinitely. Wait as long as fadump
 709		 * is in force. If we race with fadump un-registration this
 710		 * loop will break and then we go down to normal panic path
 711		 * and reboot. If fadump is in force the first crashing
 712		 * cpu will definitely trigger fadump.
 713		 */
 714		while (fw_dump.dump_registered)
 715			cpu_relax();
 716		return;
 717	}
 718
 719	fdh = __va(fw_dump.fadumphdr_addr);
 720	fdh->crashing_cpu = crashing_cpu;
 721	crash_save_vmcoreinfo();
 722
 723	if (regs)
 724		fdh->regs = *regs;
 725	else
 726		ppc_save_regs(&fdh->regs);
 727
 728	fdh->cpu_mask = *cpu_online_mask;
 729
 730	/*
 731	 * If we came in via system reset, wait a while for the secondary
 732	 * CPUs to enter.
 733	 */
 734	if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) {
 735		msecs = CRASH_TIMEOUT;
 736		while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))
 737			mdelay(1);
 738	}
 739
 740	fw_dump.ops->fadump_trigger(fdh, str);
 741}
 742
 743u32 *__init fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs)
 744{
 745	struct elf_prstatus prstatus;
 746
 747	memset(&prstatus, 0, sizeof(prstatus));
 748	/*
 749	 * FIXME: How do i get PID? Do I really need it?
 750	 * prstatus.pr_pid = ????
 751	 */
 752	elf_core_copy_regs(&prstatus.pr_reg, regs);
 753	buf = append_elf_note(buf, NN_PRSTATUS, NT_PRSTATUS,
 754			      &prstatus, sizeof(prstatus));
 755	return buf;
 756}
 757
 758void __init fadump_update_elfcore_header(char *bufp)
 759{
 760	struct elf_phdr *phdr;
 761
 762	bufp += sizeof(struct elfhdr);
 763
 764	/* First note is a place holder for cpu notes info. */
 765	phdr = (struct elf_phdr *)bufp;
 766
 767	if (phdr->p_type == PT_NOTE) {
 768		phdr->p_paddr	= __pa(fw_dump.cpu_notes_buf_vaddr);
 769		phdr->p_offset	= phdr->p_paddr;
 770		phdr->p_filesz	= fw_dump.cpu_notes_buf_size;
 771		phdr->p_memsz = fw_dump.cpu_notes_buf_size;
 772	}
 773	return;
 774}
 775
 776static void *__init fadump_alloc_buffer(unsigned long size)
 777{
 778	unsigned long count, i;
 779	struct page *page;
 780	void *vaddr;
 781
 782	vaddr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
 783	if (!vaddr)
 784		return NULL;
 785
 786	count = PAGE_ALIGN(size) / PAGE_SIZE;
 787	page = virt_to_page(vaddr);
 788	for (i = 0; i < count; i++)
 789		mark_page_reserved(page + i);
 790	return vaddr;
 791}
 792
 793static void fadump_free_buffer(unsigned long vaddr, unsigned long size)
 794{
 795	free_reserved_area((void *)vaddr, (void *)(vaddr + size), -1, NULL);
 796}
 797
 798s32 __init fadump_setup_cpu_notes_buf(u32 num_cpus)
 799{
 800	/* Allocate buffer to hold cpu crash notes. */
 801	fw_dump.cpu_notes_buf_size = num_cpus * sizeof(note_buf_t);
 802	fw_dump.cpu_notes_buf_size = PAGE_ALIGN(fw_dump.cpu_notes_buf_size);
 803	fw_dump.cpu_notes_buf_vaddr =
 804		(unsigned long)fadump_alloc_buffer(fw_dump.cpu_notes_buf_size);
 805	if (!fw_dump.cpu_notes_buf_vaddr) {
 806		pr_err("Failed to allocate %ld bytes for CPU notes buffer\n",
 807		       fw_dump.cpu_notes_buf_size);
 808		return -ENOMEM;
 809	}
 810
 811	pr_debug("Allocated buffer for cpu notes of size %ld at 0x%lx\n",
 812		 fw_dump.cpu_notes_buf_size,
 813		 fw_dump.cpu_notes_buf_vaddr);
 814	return 0;
 815}
 816
 817void fadump_free_cpu_notes_buf(void)
 818{
 819	if (!fw_dump.cpu_notes_buf_vaddr)
 820		return;
 821
 822	fadump_free_buffer(fw_dump.cpu_notes_buf_vaddr,
 823			   fw_dump.cpu_notes_buf_size);
 824	fw_dump.cpu_notes_buf_vaddr = 0;
 825	fw_dump.cpu_notes_buf_size = 0;
 826}
 827
 828static void fadump_free_mem_ranges(struct fadump_mrange_info *mrange_info)
 829{
 830	if (mrange_info->is_static) {
 831		mrange_info->mem_range_cnt = 0;
 832		return;
 833	}
 834
 835	kfree(mrange_info->mem_ranges);
 836	memset((void *)((u64)mrange_info + RNG_NAME_SZ), 0,
 837	       (sizeof(struct fadump_mrange_info) - RNG_NAME_SZ));
 838}
 839
 840/*
 841 * Allocate or reallocate mem_ranges array in incremental units
 842 * of PAGE_SIZE.
 843 */
 844static int fadump_alloc_mem_ranges(struct fadump_mrange_info *mrange_info)
 845{
 846	struct fadump_memory_range *new_array;
 847	u64 new_size;
 848
 849	new_size = mrange_info->mem_ranges_sz + PAGE_SIZE;
 850	pr_debug("Allocating %llu bytes of memory for %s memory ranges\n",
 851		 new_size, mrange_info->name);
 852
 853	new_array = krealloc(mrange_info->mem_ranges, new_size, GFP_KERNEL);
 854	if (new_array == NULL) {
 855		pr_err("Insufficient memory for setting up %s memory ranges\n",
 856		       mrange_info->name);
 857		fadump_free_mem_ranges(mrange_info);
 858		return -ENOMEM;
 859	}
 860
 861	mrange_info->mem_ranges = new_array;
 862	mrange_info->mem_ranges_sz = new_size;
 863	mrange_info->max_mem_ranges = (new_size /
 864				       sizeof(struct fadump_memory_range));
 865	return 0;
 866}
 867static inline int fadump_add_mem_range(struct fadump_mrange_info *mrange_info,
 868				       u64 base, u64 end)
 869{
 870	struct fadump_memory_range *mem_ranges = mrange_info->mem_ranges;
 871	bool is_adjacent = false;
 872	u64 start, size;
 873
 874	if (base == end)
 875		return 0;
 876
 877	/*
 878	 * Fold adjacent memory ranges to bring down the memory ranges/
 879	 * PT_LOAD segments count.
 880	 */
 881	if (mrange_info->mem_range_cnt) {
 882		start = mem_ranges[mrange_info->mem_range_cnt - 1].base;
 883		size  = mem_ranges[mrange_info->mem_range_cnt - 1].size;
 884
 885		/*
 886		 * Boot memory area needs separate PT_LOAD segment(s) as it
 887		 * is moved to a different location at the time of crash.
 888		 * So, fold only if the region is not boot memory area.
 889		 */
 890		if ((start + size) == base && start >= fw_dump.boot_mem_top)
 891			is_adjacent = true;
 892	}
 893	if (!is_adjacent) {
 894		/* resize the array on reaching the limit */
 895		if (mrange_info->mem_range_cnt == mrange_info->max_mem_ranges) {
 896			int ret;
 897
 898			if (mrange_info->is_static) {
 899				pr_err("Reached array size limit for %s memory ranges\n",
 900				       mrange_info->name);
 901				return -ENOSPC;
 902			}
 903
 904			ret = fadump_alloc_mem_ranges(mrange_info);
 905			if (ret)
 906				return ret;
 907
 908			/* Update to the new resized array */
 909			mem_ranges = mrange_info->mem_ranges;
 910		}
 911
 912		start = base;
 913		mem_ranges[mrange_info->mem_range_cnt].base = start;
 914		mrange_info->mem_range_cnt++;
 915	}
 916
 917	mem_ranges[mrange_info->mem_range_cnt - 1].size = (end - start);
 918	pr_debug("%s_memory_range[%d] [%#016llx-%#016llx], %#llx bytes\n",
 919		 mrange_info->name, (mrange_info->mem_range_cnt - 1),
 920		 start, end - 1, (end - start));
 921	return 0;
 922}
 923
 924static int fadump_init_elfcore_header(char *bufp)
 925{
 926	struct elfhdr *elf;
 927
 928	elf = (struct elfhdr *) bufp;
 929	bufp += sizeof(struct elfhdr);
 930	memcpy(elf->e_ident, ELFMAG, SELFMAG);
 931	elf->e_ident[EI_CLASS] = ELF_CLASS;
 932	elf->e_ident[EI_DATA] = ELF_DATA;
 933	elf->e_ident[EI_VERSION] = EV_CURRENT;
 934	elf->e_ident[EI_OSABI] = ELF_OSABI;
 935	memset(elf->e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD);
 936	elf->e_type = ET_CORE;
 937	elf->e_machine = ELF_ARCH;
 938	elf->e_version = EV_CURRENT;
 939	elf->e_entry = 0;
 940	elf->e_phoff = sizeof(struct elfhdr);
 941	elf->e_shoff = 0;
 942
 943	if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2))
 944		elf->e_flags = 2;
 945	else if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1))
 946		elf->e_flags = 1;
 947	else
 948		elf->e_flags = 0;
 949
 950	elf->e_ehsize = sizeof(struct elfhdr);
 951	elf->e_phentsize = sizeof(struct elf_phdr);
 952	elf->e_phnum = 0;
 953	elf->e_shentsize = 0;
 954	elf->e_shnum = 0;
 955	elf->e_shstrndx = 0;
 956
 957	return 0;
 958}
 959
 960/*
 961 * If the given physical address falls within the boot memory region then
 962 * return the relocated address that points to the dump region reserved
 963 * for saving initial boot memory contents.
 964 */
 965static inline unsigned long fadump_relocate(unsigned long paddr)
 966{
 967	unsigned long raddr, rstart, rend, rlast, hole_size;
 968	int i;
 969
 970	hole_size = 0;
 971	rlast = 0;
 972	raddr = paddr;
 973	for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) {
 974		rstart = fw_dump.boot_mem_addr[i];
 975		rend = rstart + fw_dump.boot_mem_sz[i];
 976		hole_size += (rstart - rlast);
 977
 978		if (paddr >= rstart && paddr < rend) {
 979			raddr += fw_dump.boot_mem_dest_addr - hole_size;
 980			break;
 981		}
 982
 983		rlast = rend;
 984	}
 985
 986	pr_debug("vmcoreinfo: paddr = 0x%lx, raddr = 0x%lx\n", paddr, raddr);
 987	return raddr;
 988}
 989
 990static void __init populate_elf_pt_load(struct elf_phdr *phdr, u64 start,
 991			     u64 size, unsigned long long offset)
 992{
 993	phdr->p_align	= 0;
 994	phdr->p_memsz	= size;
 995	phdr->p_filesz	= size;
 996	phdr->p_paddr	= start;
 997	phdr->p_offset	= offset;
 998	phdr->p_type	= PT_LOAD;
 999	phdr->p_flags	= PF_R|PF_W|PF_X;
1000	phdr->p_vaddr	= (unsigned long)__va(start);
1001}
1002
1003static void __init fadump_populate_elfcorehdr(struct fadump_crash_info_header *fdh)
1004{
1005	char *bufp;
1006	struct elfhdr *elf;
1007	struct elf_phdr *phdr;
1008	u64 boot_mem_dest_offset;
1009	unsigned long long i, ra_start, ra_end, ra_size, mstart, mend;
1010
1011	bufp = (char *) fw_dump.elfcorehdr_addr;
1012	fadump_init_elfcore_header(bufp);
1013	elf = (struct elfhdr *)bufp;
1014	bufp += sizeof(struct elfhdr);
1015
1016	/*
1017	 * Set up ELF PT_NOTE, a placeholder for CPU notes information.
1018	 * The notes info will be populated later by platform-specific code.
1019	 * Hence, this PT_NOTE will always be the first ELF note.
1020	 *
1021	 * NOTE: Any new ELF note addition should be placed after this note.
1022	 */
1023	phdr = (struct elf_phdr *)bufp;
1024	bufp += sizeof(struct elf_phdr);
1025	phdr->p_type = PT_NOTE;
1026	phdr->p_flags	= 0;
1027	phdr->p_vaddr	= 0;
1028	phdr->p_align	= 0;
1029	phdr->p_offset	= 0;
1030	phdr->p_paddr	= 0;
1031	phdr->p_filesz	= 0;
1032	phdr->p_memsz	= 0;
1033	/* Increment number of program headers. */
1034	(elf->e_phnum)++;
1035
1036	/* setup ELF PT_NOTE for vmcoreinfo */
1037	phdr = (struct elf_phdr *)bufp;
1038	bufp += sizeof(struct elf_phdr);
1039	phdr->p_type	= PT_NOTE;
1040	phdr->p_flags	= 0;
1041	phdr->p_vaddr	= 0;
1042	phdr->p_align	= 0;
1043	phdr->p_paddr	= phdr->p_offset = fdh->vmcoreinfo_raddr;
1044	phdr->p_memsz	= phdr->p_filesz = fdh->vmcoreinfo_size;
1045	/* Increment number of program headers. */
1046	(elf->e_phnum)++;
1047
1048	/*
1049	 * Setup PT_LOAD sections. first include boot memory regions
1050	 * and then add rest of the memory regions.
1051	 */
1052	boot_mem_dest_offset = fw_dump.boot_mem_dest_addr;
1053	for (i = 0; i < fw_dump.boot_mem_regs_cnt; i++) {
1054		phdr = (struct elf_phdr *)bufp;
1055		bufp += sizeof(struct elf_phdr);
1056		populate_elf_pt_load(phdr, fw_dump.boot_mem_addr[i],
1057				     fw_dump.boot_mem_sz[i],
1058				     boot_mem_dest_offset);
1059		/* Increment number of program headers. */
1060		(elf->e_phnum)++;
1061		boot_mem_dest_offset += fw_dump.boot_mem_sz[i];
1062	}
1063
1064	/* Memory reserved for fadump in first kernel */
1065	ra_start = fw_dump.reserve_dump_area_start;
1066	ra_size = get_fadump_area_size();
1067	ra_end = ra_start + ra_size;
1068
1069	phdr = (struct elf_phdr *)bufp;
1070	for_each_mem_range(i, &mstart, &mend) {
1071		/* Boot memory regions already added, skip them now */
1072		if (mstart < fw_dump.boot_mem_top) {
1073			if (mend > fw_dump.boot_mem_top)
1074				mstart = fw_dump.boot_mem_top;
1075			else
1076				continue;
1077		}
1078
1079		/* Handle memblock regions overlaps with fadump reserved area */
1080		if ((ra_start < mend) && (ra_end > mstart)) {
1081			if ((mstart < ra_start) && (mend > ra_end)) {
1082				populate_elf_pt_load(phdr, mstart, ra_start - mstart, mstart);
1083				/* Increment number of program headers. */
1084				(elf->e_phnum)++;
1085				bufp += sizeof(struct elf_phdr);
1086				phdr = (struct elf_phdr *)bufp;
1087				populate_elf_pt_load(phdr, ra_end, mend - ra_end, ra_end);
1088			} else if (mstart < ra_start) {
1089				populate_elf_pt_load(phdr, mstart, ra_start - mstart, mstart);
1090			} else if (ra_end < mend) {
1091				populate_elf_pt_load(phdr, ra_end, mend - ra_end, ra_end);
1092			}
1093		} else {
1094		/* No overlap with fadump reserved memory region */
1095			populate_elf_pt_load(phdr, mstart, mend - mstart, mstart);
1096		}
1097
1098		/* Increment number of program headers. */
1099		(elf->e_phnum)++;
1100		bufp += sizeof(struct elf_phdr);
1101		phdr = (struct elf_phdr *) bufp;
1102	}
1103}
1104
1105static unsigned long init_fadump_header(unsigned long addr)
1106{
1107	struct fadump_crash_info_header *fdh;
1108
1109	if (!addr)
1110		return 0;
1111
1112	fdh = __va(addr);
1113	addr += sizeof(struct fadump_crash_info_header);
1114
1115	memset(fdh, 0, sizeof(struct fadump_crash_info_header));
1116	fdh->magic_number = FADUMP_CRASH_INFO_MAGIC;
1117	fdh->version = FADUMP_HEADER_VERSION;
1118	/* We will set the crashing cpu id in crash_fadump() during crash. */
1119	fdh->crashing_cpu = FADUMP_CPU_UNKNOWN;
1120
1121	/*
1122	 * The physical address and size of vmcoreinfo are required in the
1123	 * second kernel to prepare elfcorehdr.
1124	 */
1125	fdh->vmcoreinfo_raddr = fadump_relocate(paddr_vmcoreinfo_note());
1126	fdh->vmcoreinfo_size = VMCOREINFO_NOTE_SIZE;
1127
1128
1129	fdh->pt_regs_sz = sizeof(struct pt_regs);
1130	/*
1131	 * When LPAR is terminated by PYHP, ensure all possible CPUs'
1132	 * register data is processed while exporting the vmcore.
1133	 */
1134	fdh->cpu_mask = *cpu_possible_mask;
1135	fdh->cpu_mask_sz = sizeof(struct cpumask);
1136
1137	return addr;
1138}
1139
1140static int register_fadump(void)
1141{
1142	unsigned long addr;
1143
1144	/*
1145	 * If no memory is reserved then we can not register for firmware-
1146	 * assisted dump.
1147	 */
1148	if (!fw_dump.reserve_dump_area_size)
1149		return -ENODEV;
1150
1151	addr = fw_dump.fadumphdr_addr;
1152
1153	/* Initialize fadump crash info header. */
1154	addr = init_fadump_header(addr);
1155
1156	/* register the future kernel dump with firmware. */
1157	pr_debug("Registering for firmware-assisted kernel dump...\n");
1158	return fw_dump.ops->fadump_register(&fw_dump);
1159}
1160
1161void fadump_cleanup(void)
1162{
1163	if (!fw_dump.fadump_supported)
1164		return;
1165
1166	/* Invalidate the registration only if dump is active. */
1167	if (fw_dump.dump_active) {
1168		pr_debug("Invalidating firmware-assisted dump registration\n");
1169		fw_dump.ops->fadump_invalidate(&fw_dump);
1170	} else if (fw_dump.dump_registered) {
1171		/* Un-register Firmware-assisted dump if it was registered. */
1172		fw_dump.ops->fadump_unregister(&fw_dump);
1173	}
1174
1175	if (fw_dump.ops->fadump_cleanup)
1176		fw_dump.ops->fadump_cleanup(&fw_dump);
1177}
1178
1179static void fadump_free_reserved_memory(unsigned long start_pfn,
1180					unsigned long end_pfn)
1181{
1182	unsigned long pfn;
1183	unsigned long time_limit = jiffies + HZ;
1184
1185	pr_info("freeing reserved memory (0x%llx - 0x%llx)\n",
1186		PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
1187
1188	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1189		free_reserved_page(pfn_to_page(pfn));
1190
1191		if (time_after(jiffies, time_limit)) {
1192			cond_resched();
1193			time_limit = jiffies + HZ;
1194		}
1195	}
1196}
1197
1198/*
1199 * Skip memory holes and free memory that was actually reserved.
1200 */
1201static void fadump_release_reserved_area(u64 start, u64 end)
1202{
1203	unsigned long reg_spfn, reg_epfn;
1204	u64 tstart, tend, spfn, epfn;
1205	int i;
1206
1207	spfn = PHYS_PFN(start);
1208	epfn = PHYS_PFN(end);
1209
1210	for_each_mem_pfn_range(i, MAX_NUMNODES, &reg_spfn, &reg_epfn, NULL) {
1211		tstart = max_t(u64, spfn, reg_spfn);
1212		tend   = min_t(u64, epfn, reg_epfn);
1213
1214		if (tstart < tend) {
1215			fadump_free_reserved_memory(tstart, tend);
1216
1217			if (tend == epfn)
1218				break;
1219
1220			spfn = tend;
1221		}
1222	}
1223}
1224
1225/*
1226 * Sort the mem ranges in-place and merge adjacent ranges
1227 * to minimize the memory ranges count.
1228 */
1229static void sort_and_merge_mem_ranges(struct fadump_mrange_info *mrange_info)
1230{
1231	struct fadump_memory_range *mem_ranges;
1232	u64 base, size;
1233	int i, j, idx;
1234
1235	if (!reserved_mrange_info.mem_range_cnt)
1236		return;
1237
1238	/* Sort the memory ranges */
1239	mem_ranges = mrange_info->mem_ranges;
1240	for (i = 0; i < mrange_info->mem_range_cnt; i++) {
1241		idx = i;
1242		for (j = (i + 1); j < mrange_info->mem_range_cnt; j++) {
1243			if (mem_ranges[idx].base > mem_ranges[j].base)
1244				idx = j;
1245		}
1246		if (idx != i)
1247			swap(mem_ranges[idx], mem_ranges[i]);
1248	}
1249
1250	/* Merge adjacent reserved ranges */
1251	idx = 0;
1252	for (i = 1; i < mrange_info->mem_range_cnt; i++) {
1253		base = mem_ranges[i-1].base;
1254		size = mem_ranges[i-1].size;
1255		if (mem_ranges[i].base == (base + size))
1256			mem_ranges[idx].size += mem_ranges[i].size;
1257		else {
1258			idx++;
1259			if (i == idx)
1260				continue;
1261
1262			mem_ranges[idx] = mem_ranges[i];
1263		}
1264	}
1265	mrange_info->mem_range_cnt = idx + 1;
1266}
1267
1268/*
1269 * Scan reserved-ranges to consider them while reserving/releasing
1270 * memory for FADump.
1271 */
1272static void __init early_init_dt_scan_reserved_ranges(unsigned long node)
1273{
1274	const __be32 *prop;
1275	int len, ret = -1;
1276	unsigned long i;
1277
1278	/* reserved-ranges already scanned */
1279	if (reserved_mrange_info.mem_range_cnt != 0)
1280		return;
1281
1282	prop = of_get_flat_dt_prop(node, "reserved-ranges", &len);
1283	if (!prop)
1284		return;
1285
1286	/*
1287	 * Each reserved range is an (address,size) pair, 2 cells each,
1288	 * totalling 4 cells per range.
1289	 */
1290	for (i = 0; i < len / (sizeof(*prop) * 4); i++) {
1291		u64 base, size;
1292
1293		base = of_read_number(prop + (i * 4) + 0, 2);
1294		size = of_read_number(prop + (i * 4) + 2, 2);
1295
1296		if (size) {
1297			ret = fadump_add_mem_range(&reserved_mrange_info,
1298						   base, base + size);
1299			if (ret < 0) {
1300				pr_warn("some reserved ranges are ignored!\n");
1301				break;
1302			}
1303		}
1304	}
1305
1306	/* Compact reserved ranges */
1307	sort_and_merge_mem_ranges(&reserved_mrange_info);
1308}
1309
1310/*
1311 * Release the memory that was reserved during early boot to preserve the
1312 * crash'ed kernel's memory contents except reserved dump area (permanent
1313 * reservation) and reserved ranges used by F/W. The released memory will
1314 * be available for general use.
1315 */
1316static void fadump_release_memory(u64 begin, u64 end)
1317{
1318	u64 ra_start, ra_end, tstart;
1319	int i, ret;
1320
1321	ra_start = fw_dump.reserve_dump_area_start;
1322	ra_end = ra_start + fw_dump.reserve_dump_area_size;
1323
1324	/*
1325	 * If reserved ranges array limit is hit, overwrite the last reserved
1326	 * memory range with reserved dump area to ensure it is excluded from
1327	 * the memory being released (reused for next FADump registration).
1328	 */
1329	if (reserved_mrange_info.mem_range_cnt ==
1330	    reserved_mrange_info.max_mem_ranges)
1331		reserved_mrange_info.mem_range_cnt--;
1332
1333	ret = fadump_add_mem_range(&reserved_mrange_info, ra_start, ra_end);
1334	if (ret != 0)
1335		return;
1336
1337	/* Get the reserved ranges list in order first. */
1338	sort_and_merge_mem_ranges(&reserved_mrange_info);
1339
1340	/* Exclude reserved ranges and release remaining memory */
1341	tstart = begin;
1342	for (i = 0; i < reserved_mrange_info.mem_range_cnt; i++) {
1343		ra_start = reserved_mrange_info.mem_ranges[i].base;
1344		ra_end = ra_start + reserved_mrange_info.mem_ranges[i].size;
1345
1346		if (tstart >= ra_end)
1347			continue;
1348
1349		if (tstart < ra_start)
1350			fadump_release_reserved_area(tstart, ra_start);
1351		tstart = ra_end;
1352	}
1353
1354	if (tstart < end)
1355		fadump_release_reserved_area(tstart, end);
1356}
1357
1358static void fadump_free_elfcorehdr_buf(void)
1359{
1360	if (fw_dump.elfcorehdr_addr == 0 || fw_dump.elfcorehdr_size == 0)
1361		return;
1362
1363	/*
1364	 * Before freeing the memory of `elfcorehdr`, reset the global
1365	 * `elfcorehdr_addr` to prevent modules like `vmcore` from accessing
1366	 * invalid memory.
1367	 */
1368	elfcorehdr_addr = ELFCORE_ADDR_ERR;
1369	fadump_free_buffer(fw_dump.elfcorehdr_addr, fw_dump.elfcorehdr_size);
1370	fw_dump.elfcorehdr_addr = 0;
1371	fw_dump.elfcorehdr_size = 0;
1372}
1373
1374static void fadump_invalidate_release_mem(void)
1375{
1376	scoped_guard(mutex, &fadump_mutex) {
1377		if (!fw_dump.dump_active)
1378			return;
1379		fadump_cleanup();
1380	}
1381
1382	fadump_free_elfcorehdr_buf();
1383	fadump_release_memory(fw_dump.boot_mem_top, memblock_end_of_DRAM());
1384	fadump_free_cpu_notes_buf();
1385
1386	/*
1387	 * Setup kernel metadata and initialize the kernel dump
1388	 * memory structure for FADump re-registration.
1389	 */
1390	if (fw_dump.ops->fadump_setup_metadata &&
1391	    (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))
1392		pr_warn("Failed to setup kernel metadata!\n");
1393	fw_dump.ops->fadump_init_mem_struct(&fw_dump);
1394}
1395
1396static ssize_t release_mem_store(struct kobject *kobj,
1397				 struct kobj_attribute *attr,
1398				 const char *buf, size_t count)
1399{
1400	int input = -1;
1401
1402	if (!fw_dump.dump_active)
1403		return -EPERM;
1404
1405	if (kstrtoint(buf, 0, &input))
1406		return -EINVAL;
1407
1408	if (input == 1) {
1409		/*
1410		 * Take away the '/proc/vmcore'. We are releasing the dump
1411		 * memory, hence it will not be valid anymore.
1412		 */
1413#ifdef CONFIG_PROC_VMCORE
1414		vmcore_cleanup();
1415#endif
1416		fadump_invalidate_release_mem();
1417
1418	} else
1419		return -EINVAL;
1420	return count;
1421}
1422
1423/* Release the reserved memory and disable the FADump */
1424static void __init unregister_fadump(void)
1425{
1426	fadump_cleanup();
1427	fadump_release_memory(fw_dump.reserve_dump_area_start,
1428			      fw_dump.reserve_dump_area_size);
1429	fw_dump.fadump_enabled = 0;
1430	kobject_put(fadump_kobj);
1431}
1432
1433static ssize_t enabled_show(struct kobject *kobj,
1434			    struct kobj_attribute *attr,
1435			    char *buf)
1436{
1437	return sprintf(buf, "%d\n", fw_dump.fadump_enabled);
1438}
1439
1440/*
1441 * /sys/kernel/fadump/hotplug_ready sysfs node returns 1, which inidcates
1442 * to usersapce that fadump re-registration is not required on memory
1443 * hotplug events.
1444 */
1445static ssize_t hotplug_ready_show(struct kobject *kobj,
1446				      struct kobj_attribute *attr,
1447				      char *buf)
1448{
1449	return sprintf(buf, "%d\n", 1);
1450}
1451
1452static ssize_t mem_reserved_show(struct kobject *kobj,
1453				 struct kobj_attribute *attr,
1454				 char *buf)
1455{
1456	return sprintf(buf, "%ld\n", fw_dump.reserve_dump_area_size);
1457}
1458
1459static ssize_t registered_show(struct kobject *kobj,
1460			       struct kobj_attribute *attr,
1461			       char *buf)
1462{
1463	return sprintf(buf, "%d\n", fw_dump.dump_registered);
1464}
1465
1466static ssize_t bootargs_append_show(struct kobject *kobj,
1467				   struct kobj_attribute *attr,
1468				   char *buf)
1469{
1470	return sprintf(buf, "%s\n", (char *)__va(fw_dump.param_area));
1471}
1472
1473static ssize_t bootargs_append_store(struct kobject *kobj,
1474				   struct kobj_attribute *attr,
1475				   const char *buf, size_t count)
1476{
1477	char *params;
1478
1479	if (!fw_dump.fadump_enabled || fw_dump.dump_active)
1480		return -EPERM;
1481
1482	if (count >= COMMAND_LINE_SIZE)
1483		return -EINVAL;
1484
1485	/*
1486	 * Fail here instead of handling this scenario with
1487	 * some silly workaround in capture kernel.
1488	 */
1489	if (saved_command_line_len + count >= COMMAND_LINE_SIZE) {
1490		pr_err("Appending parameters exceeds cmdline size!\n");
1491		return -ENOSPC;
1492	}
1493
1494	params = __va(fw_dump.param_area);
1495	strscpy_pad(params, buf, COMMAND_LINE_SIZE);
1496	/* Remove newline character at the end. */
1497	if (params[count-1] == '\n')
1498		params[count-1] = '\0';
1499
1500	return count;
1501}
1502
1503static ssize_t registered_store(struct kobject *kobj,
1504				struct kobj_attribute *attr,
1505				const char *buf, size_t count)
1506{
1507	int ret = 0;
1508	int input = -1;
1509
1510	if (!fw_dump.fadump_enabled || fw_dump.dump_active)
1511		return -EPERM;
1512
1513	if (kstrtoint(buf, 0, &input))
1514		return -EINVAL;
1515
1516	mutex_lock(&fadump_mutex);
1517
1518	switch (input) {
1519	case 0:
1520		if (fw_dump.dump_registered == 0) {
1521			goto unlock_out;
1522		}
1523
1524		/* Un-register Firmware-assisted dump */
1525		pr_debug("Un-register firmware-assisted dump\n");
1526		fw_dump.ops->fadump_unregister(&fw_dump);
1527		break;
1528	case 1:
1529		if (fw_dump.dump_registered == 1) {
1530			/* Un-register Firmware-assisted dump */
1531			fw_dump.ops->fadump_unregister(&fw_dump);
1532		}
1533		/* Register Firmware-assisted dump */
1534		ret = register_fadump();
1535		break;
1536	default:
1537		ret = -EINVAL;
1538		break;
1539	}
1540
1541unlock_out:
1542	mutex_unlock(&fadump_mutex);
1543	return ret < 0 ? ret : count;
1544}
1545
1546static int fadump_region_show(struct seq_file *m, void *private)
1547{
1548	if (!fw_dump.fadump_enabled)
1549		return 0;
1550
1551	mutex_lock(&fadump_mutex);
1552	fw_dump.ops->fadump_region_show(&fw_dump, m);
1553	mutex_unlock(&fadump_mutex);
1554	return 0;
1555}
1556
1557static struct kobj_attribute release_attr = __ATTR_WO(release_mem);
1558static struct kobj_attribute enable_attr = __ATTR_RO(enabled);
1559static struct kobj_attribute register_attr = __ATTR_RW(registered);
1560static struct kobj_attribute mem_reserved_attr = __ATTR_RO(mem_reserved);
1561static struct kobj_attribute hotplug_ready_attr = __ATTR_RO(hotplug_ready);
1562static struct kobj_attribute bootargs_append_attr = __ATTR_RW(bootargs_append);
1563
1564static struct attribute *fadump_attrs[] = {
1565	&enable_attr.attr,
1566	&register_attr.attr,
1567	&mem_reserved_attr.attr,
1568	&hotplug_ready_attr.attr,
1569	NULL,
1570};
1571
1572ATTRIBUTE_GROUPS(fadump);
1573
1574DEFINE_SHOW_ATTRIBUTE(fadump_region);
1575
1576static void __init fadump_init_files(void)
1577{
1578	int rc = 0;
1579
1580	fadump_kobj = kobject_create_and_add("fadump", kernel_kobj);
1581	if (!fadump_kobj) {
1582		pr_err("failed to create fadump kobject\n");
1583		return;
1584	}
1585
1586	if (fw_dump.param_area) {
1587		rc = sysfs_create_file(fadump_kobj, &bootargs_append_attr.attr);
1588		if (rc)
1589			pr_err("unable to create bootargs_append sysfs file (%d)\n", rc);
1590	}
1591
1592	debugfs_create_file("fadump_region", 0444, arch_debugfs_dir, NULL,
1593			    &fadump_region_fops);
1594
1595	if (fw_dump.dump_active) {
1596		rc = sysfs_create_file(fadump_kobj, &release_attr.attr);
1597		if (rc)
1598			pr_err("unable to create release_mem sysfs file (%d)\n",
1599			       rc);
1600	}
1601
1602	rc = sysfs_create_groups(fadump_kobj, fadump_groups);
1603	if (rc) {
1604		pr_err("sysfs group creation failed (%d), unregistering FADump",
1605		       rc);
1606		unregister_fadump();
1607		return;
1608	}
1609
1610	/*
1611	 * The FADump sysfs are moved from kernel_kobj to fadump_kobj need to
1612	 * create symlink at old location to maintain backward compatibility.
1613	 *
1614	 *      - fadump_enabled -> fadump/enabled
1615	 *      - fadump_registered -> fadump/registered
1616	 *      - fadump_release_mem -> fadump/release_mem
1617	 */
1618	rc = compat_only_sysfs_link_entry_to_kobj(kernel_kobj, fadump_kobj,
1619						  "enabled", "fadump_enabled");
1620	if (rc) {
1621		pr_err("unable to create fadump_enabled symlink (%d)", rc);
1622		return;
1623	}
1624
1625	rc = compat_only_sysfs_link_entry_to_kobj(kernel_kobj, fadump_kobj,
1626						  "registered",
1627						  "fadump_registered");
1628	if (rc) {
1629		pr_err("unable to create fadump_registered symlink (%d)", rc);
1630		sysfs_remove_link(kernel_kobj, "fadump_enabled");
1631		return;
1632	}
1633
1634	if (fw_dump.dump_active) {
1635		rc = compat_only_sysfs_link_entry_to_kobj(kernel_kobj,
1636							  fadump_kobj,
1637							  "release_mem",
1638							  "fadump_release_mem");
1639		if (rc)
1640			pr_err("unable to create fadump_release_mem symlink (%d)",
1641			       rc);
1642	}
1643	return;
1644}
1645
1646static int __init fadump_setup_elfcorehdr_buf(void)
1647{
1648	int elf_phdr_cnt;
1649	unsigned long elfcorehdr_size;
1650
1651	/*
1652	 * Program header for CPU notes comes first, followed by one for
1653	 * vmcoreinfo, and the remaining program headers correspond to
1654	 * memory regions.
1655	 */
1656	elf_phdr_cnt = 2 + fw_dump.boot_mem_regs_cnt + memblock_num_regions(memory);
1657	elfcorehdr_size = sizeof(struct elfhdr) + (elf_phdr_cnt * sizeof(struct elf_phdr));
1658	elfcorehdr_size = PAGE_ALIGN(elfcorehdr_size);
1659
1660	fw_dump.elfcorehdr_addr = (u64)fadump_alloc_buffer(elfcorehdr_size);
1661	if (!fw_dump.elfcorehdr_addr) {
1662		pr_err("Failed to allocate %lu bytes for elfcorehdr\n",
1663		       elfcorehdr_size);
1664		return -ENOMEM;
1665	}
1666	fw_dump.elfcorehdr_size = elfcorehdr_size;
1667	return 0;
1668}
1669
1670/*
1671 * Check if the fadump header of crashed kernel is compatible with fadump kernel.
1672 *
1673 * It checks the magic number, endianness, and size of non-primitive type
1674 * members of fadump header to ensure safe dump collection.
1675 */
1676static bool __init is_fadump_header_compatible(struct fadump_crash_info_header *fdh)
1677{
1678	if (fdh->magic_number == FADUMP_CRASH_INFO_MAGIC_OLD) {
1679		pr_err("Old magic number, can't process the dump.\n");
1680		return false;
1681	}
1682
1683	if (fdh->magic_number != FADUMP_CRASH_INFO_MAGIC) {
1684		if (fdh->magic_number == swab64(FADUMP_CRASH_INFO_MAGIC))
1685			pr_err("Endianness mismatch between the crashed and fadump kernels.\n");
1686		else
1687			pr_err("Fadump header is corrupted.\n");
1688
1689		return false;
1690	}
1691
1692	/*
1693	 * Dump collection is not safe if the size of non-primitive type members
1694	 * of the fadump header do not match between crashed and fadump kernel.
1695	 */
1696	if (fdh->pt_regs_sz != sizeof(struct pt_regs) ||
1697	    fdh->cpu_mask_sz != sizeof(struct cpumask)) {
1698		pr_err("Fadump header size mismatch.\n");
1699		return false;
1700	}
1701
1702	return true;
1703}
1704
1705static void __init fadump_process(void)
1706{
1707	struct fadump_crash_info_header *fdh;
1708
1709	fdh = (struct fadump_crash_info_header *) __va(fw_dump.fadumphdr_addr);
1710	if (!fdh) {
1711		pr_err("Crash info header is empty.\n");
1712		goto err_out;
1713	}
1714
1715	/* Avoid processing the dump if fadump header isn't compatible */
1716	if (!is_fadump_header_compatible(fdh))
1717		goto err_out;
1718
1719	/* Allocate buffer for elfcorehdr */
1720	if (fadump_setup_elfcorehdr_buf())
1721		goto err_out;
1722
1723	fadump_populate_elfcorehdr(fdh);
1724
1725	/* Let platform update the CPU notes in elfcorehdr */
1726	if (fw_dump.ops->fadump_process(&fw_dump) < 0)
1727		goto err_out;
1728
1729	/*
1730	 * elfcorehdr is now ready to be exported.
1731	 *
1732	 * set elfcorehdr_addr so that vmcore module will export the
1733	 * elfcorehdr through '/proc/vmcore'.
1734	 */
1735	elfcorehdr_addr = virt_to_phys((void *)fw_dump.elfcorehdr_addr);
1736	return;
1737
1738err_out:
1739	fadump_invalidate_release_mem();
1740}
1741
1742/*
1743 * Reserve memory to store additional parameters to be passed
1744 * for fadump/capture kernel.
1745 */
1746void __init fadump_setup_param_area(void)
1747{
1748	phys_addr_t range_start, range_end;
1749
1750	if (!fw_dump.fadump_enabled)
1751		return;
1752
1753	if (!fw_dump.param_area_supported || fw_dump.dump_active)
1754		return;
1755
1756	/* This memory can't be used by PFW or bootloader as it is shared across kernels */
1757	if (early_radix_enabled()) {
1758		/*
1759		 * Anywhere in the upper half should be good enough as all memory
1760		 * is accessible in real mode.
1761		 */
1762		range_start = memblock_end_of_DRAM() / 2;
1763		range_end = memblock_end_of_DRAM();
1764	} else {
1765		/*
1766		 * Memory range for passing additional parameters for HASH MMU
1767		 * must meet the following conditions:
1768		 * 1. The first memory block size must be higher than the
1769		 *    minimum RMA (MIN_RMA) size. Bootloader can use memory
1770		 *    upto RMA size. So it should be avoided.
1771		 * 2. The range should be between MIN_RMA and RMA size (ppc64_rma_size)
1772		 * 3. It must not overlap with the fadump reserved area.
1773		 */
1774		if (ppc64_rma_size < MIN_RMA*1024*1024)
1775			return;
1776
1777		range_start = MIN_RMA * 1024 * 1024;
1778		range_end = min(ppc64_rma_size, fw_dump.boot_mem_top);
1779	}
1780
1781	fw_dump.param_area = memblock_phys_alloc_range(COMMAND_LINE_SIZE,
1782						       COMMAND_LINE_SIZE,
1783						       range_start,
1784						       range_end);
1785	if (!fw_dump.param_area) {
1786		pr_warn("WARNING: Could not setup area to pass additional parameters!\n");
1787		return;
1788	}
1789
1790	memset((void *)fw_dump.param_area, 0, COMMAND_LINE_SIZE);
1791}
1792
1793/*
1794 * Prepare for firmware-assisted dump.
1795 */
1796int __init setup_fadump(void)
1797{
1798	if (!fw_dump.fadump_supported)
1799		return 0;
1800
1801	fadump_init_files();
1802	fadump_show_config();
1803
1804	if (!fw_dump.fadump_enabled)
1805		return 1;
1806
1807	/*
1808	 * If dump data is available then see if it is valid and prepare for
1809	 * saving it to the disk.
1810	 */
1811	if (fw_dump.dump_active) {
1812		fadump_process();
1813	}
1814	/* Initialize the kernel dump memory structure and register with f/w */
1815	else if (fw_dump.reserve_dump_area_size) {
1816		fw_dump.ops->fadump_init_mem_struct(&fw_dump);
1817		register_fadump();
1818	}
1819
1820	/*
1821	 * In case of panic, fadump is triggered via ppc_panic_event()
1822	 * panic notifier. Setting crash_kexec_post_notifiers to 'true'
1823	 * lets panic() function take crash friendly path before panic
1824	 * notifiers are invoked.
1825	 */
1826	crash_kexec_post_notifiers = true;
1827
1828	return 1;
1829}
1830/*
1831 * Use subsys_initcall_sync() here because there is dependency with
1832 * crash_save_vmcoreinfo_init(), which must run first to ensure vmcoreinfo initialization
1833 * is done before registering with f/w.
1834 */
1835subsys_initcall_sync(setup_fadump);
1836#else /* !CONFIG_PRESERVE_FA_DUMP */
1837
1838/* Scan the Firmware Assisted dump configuration details. */
1839int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname,
1840				      int depth, void *data)
1841{
1842	if ((depth != 1) || (strcmp(uname, "ibm,opal") != 0))
1843		return 0;
1844
1845	opal_fadump_dt_scan(&fw_dump, node);
1846	return 1;
1847}
1848
1849/*
1850 * When dump is active but PRESERVE_FA_DUMP is enabled on the kernel,
1851 * preserve crash data. The subsequent memory preserving kernel boot
1852 * is likely to process this crash data.
1853 */
1854int __init fadump_reserve_mem(void)
1855{
1856	if (fw_dump.dump_active) {
1857		/*
1858		 * If last boot has crashed then reserve all the memory
1859		 * above boot memory to preserve crash data.
1860		 */
1861		pr_info("Preserving crash data for processing in next boot.\n");
1862		fadump_reserve_crash_area(fw_dump.boot_mem_top);
1863	} else
1864		pr_debug("FADump-aware kernel..\n");
1865
1866	return 1;
1867}
1868#endif /* CONFIG_PRESERVE_FA_DUMP */
1869
1870/* Preserve everything above the base address */
1871static void __init fadump_reserve_crash_area(u64 base)
1872{
1873	u64 i, mstart, mend, msize;
1874
1875	for_each_mem_range(i, &mstart, &mend) {
1876		msize  = mend - mstart;
1877
1878		if ((mstart + msize) < base)
1879			continue;
1880
1881		if (mstart < base) {
1882			msize -= (base - mstart);
1883			mstart = base;
1884		}
1885
1886		pr_info("Reserving %lluMB of memory at %#016llx for preserving crash data",
1887			(msize >> 20), mstart);
1888		memblock_reserve(mstart, msize);
1889	}
1890}