vmcore.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *	fs/proc/vmcore.c Interface for accessing the crash
   4 * 				 dump from the system's previous life.
   5 * 	Heavily borrowed from fs/proc/kcore.c
   6 *	Created by: Hariprasad Nellitheertha (hari@in.ibm.com)
   7 *	Copyright (C) IBM Corporation, 2004. All rights reserved
   8 *
   9 */
  10
  11#define pr_fmt(fmt) "vmcore: " fmt
  12
  13#include <linux/mm.h>
  14#include <linux/kcore.h>
  15#include <linux/user.h>
  16#include <linux/elf.h>
  17#include <linux/elfcore.h>
  18#include <linux/export.h>
  19#include <linux/slab.h>
  20#include <linux/highmem.h>
  21#include <linux/printk.h>
  22#include <linux/memblock.h>
  23#include <linux/init.h>
  24#include <linux/crash_dump.h>
  25#include <linux/list.h>
  26#include <linux/moduleparam.h>
  27#include <linux/mutex.h>
  28#include <linux/vmalloc.h>
  29#include <linux/pagemap.h>
  30#include <linux/uio.h>
  31#include <linux/cc_platform.h>
  32#include <asm/io.h>
  33#include "internal.h"
  34
  35/* List representing chunks of contiguous memory areas and their offsets in
  36 * vmcore file.
  37 */
  38static LIST_HEAD(vmcore_list);
  39
  40/* Stores the pointer to the buffer containing kernel elf core headers. */
  41static char *elfcorebuf;
  42static size_t elfcorebuf_sz;
  43static size_t elfcorebuf_sz_orig;
  44
  45static char *elfnotes_buf;
  46static size_t elfnotes_sz;
  47/* Size of all notes minus the device dump notes */
  48static size_t elfnotes_orig_sz;
  49
  50/* Total size of vmcore file. */
  51static u64 vmcore_size;
  52
  53static struct proc_dir_entry *proc_vmcore;
  54
  55#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP
  56struct vmcoredd_node {
  57	struct list_head list;	/* List of dumps */
  58	void *buf;		/* Buffer containing device's dump */
  59	unsigned int size;	/* Size of the buffer */
  60};
  61
  62/* Device Dump list and mutex to synchronize access to list */
  63static LIST_HEAD(vmcoredd_list);
  64
  65static bool vmcoredd_disabled;
  66core_param(novmcoredd, vmcoredd_disabled, bool, 0);
  67#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
  68
  69/* Device Dump Size */
  70static size_t vmcoredd_orig_sz;
  71
  72static DEFINE_MUTEX(vmcore_mutex);
  73
  74DEFINE_STATIC_SRCU(vmcore_cb_srcu);
  75/* List of registered vmcore callbacks. */
  76static LIST_HEAD(vmcore_cb_list);
  77/* Whether the vmcore has been opened once. */
  78static bool vmcore_opened;
  79/* Whether the vmcore is currently open. */
  80static unsigned int vmcore_open;
  81
  82static void vmcore_process_device_ram(struct vmcore_cb *cb);
  83
  84void register_vmcore_cb(struct vmcore_cb *cb)
  85{
  86	INIT_LIST_HEAD(&cb->next);
  87	mutex_lock(&vmcore_mutex);
  88	list_add_tail(&cb->next, &vmcore_cb_list);
  89	/*
  90	 * Registering a vmcore callback after the vmcore was opened is
  91	 * very unusual (e.g., manual driver loading).
  92	 */
  93	if (vmcore_opened)
  94		pr_warn_once("Unexpected vmcore callback registration\n");
  95	if (!vmcore_open && cb->get_device_ram)
  96		vmcore_process_device_ram(cb);
  97	mutex_unlock(&vmcore_mutex);
  98}
  99EXPORT_SYMBOL_GPL(register_vmcore_cb);
 100
 101void unregister_vmcore_cb(struct vmcore_cb *cb)
 102{
 103	mutex_lock(&vmcore_mutex);
 104	list_del_rcu(&cb->next);
 105	/*
 106	 * Unregistering a vmcore callback after the vmcore was opened is
 107	 * very unusual (e.g., forced driver removal), but we cannot stop
 108	 * unregistering.
 109	 */
 110	if (vmcore_opened)
 111		pr_warn_once("Unexpected vmcore callback unregistration\n");
 112	mutex_unlock(&vmcore_mutex);
 113
 114	synchronize_srcu(&vmcore_cb_srcu);
 115}
 116EXPORT_SYMBOL_GPL(unregister_vmcore_cb);
 117
 118static bool pfn_is_ram(unsigned long pfn)
 119{
 120	struct vmcore_cb *cb;
 121	bool ret = true;
 122
 123	list_for_each_entry_srcu(cb, &vmcore_cb_list, next,
 124				 srcu_read_lock_held(&vmcore_cb_srcu)) {
 125		if (unlikely(!cb->pfn_is_ram))
 126			continue;
 127		ret = cb->pfn_is_ram(cb, pfn);
 128		if (!ret)
 129			break;
 130	}
 131
 132	return ret;
 133}
 134
 135static int open_vmcore(struct inode *inode, struct file *file)
 136{
 137	mutex_lock(&vmcore_mutex);
 138	vmcore_opened = true;
 139	if (vmcore_open + 1 == 0) {
 140		mutex_unlock(&vmcore_mutex);
 141		return -EBUSY;
 142	}
 143	vmcore_open++;
 144	mutex_unlock(&vmcore_mutex);
 145
 146	return 0;
 147}
 148
 149static int release_vmcore(struct inode *inode, struct file *file)
 150{
 151	mutex_lock(&vmcore_mutex);
 152	vmcore_open--;
 153	mutex_unlock(&vmcore_mutex);
 154
 155	return 0;
 156}
 157
 158/* Reads a page from the oldmem device from given offset. */
 159ssize_t read_from_oldmem(struct iov_iter *iter, size_t count,
 160			 u64 *ppos, bool encrypted)
 161{
 162	unsigned long pfn, offset;
 163	ssize_t nr_bytes;
 164	ssize_t read = 0, tmp;
 165	int idx;
 166
 167	if (!count)
 168		return 0;
 169
 170	offset = (unsigned long)(*ppos % PAGE_SIZE);
 171	pfn = (unsigned long)(*ppos / PAGE_SIZE);
 172
 173	idx = srcu_read_lock(&vmcore_cb_srcu);
 174	do {
 175		if (count > (PAGE_SIZE - offset))
 176			nr_bytes = PAGE_SIZE - offset;
 177		else
 178			nr_bytes = count;
 179
 180		/* If pfn is not ram, return zeros for sparse dump files */
 181		if (!pfn_is_ram(pfn)) {
 182			tmp = iov_iter_zero(nr_bytes, iter);
 183		} else {
 184			if (encrypted)
 185				tmp = copy_oldmem_page_encrypted(iter, pfn,
 186								 nr_bytes,
 187								 offset);
 188			else
 189				tmp = copy_oldmem_page(iter, pfn, nr_bytes,
 190						       offset);
 191		}
 192		if (tmp < nr_bytes) {
 193			srcu_read_unlock(&vmcore_cb_srcu, idx);
 194			return -EFAULT;
 195		}
 196
 197		*ppos += nr_bytes;
 198		count -= nr_bytes;
 199		read += nr_bytes;
 200		++pfn;
 201		offset = 0;
 202	} while (count);
 203	srcu_read_unlock(&vmcore_cb_srcu, idx);
 204
 205	return read;
 206}
 207
 208/*
 209 * Architectures may override this function to allocate ELF header in 2nd kernel
 210 */
 211int __weak elfcorehdr_alloc(unsigned long long *addr, unsigned long long *size)
 212{
 213	return 0;
 214}
 215
 216/*
 217 * Architectures may override this function to free header
 218 */
 219void __weak elfcorehdr_free(unsigned long long addr)
 220{}
 221
 222/*
 223 * Architectures may override this function to read from ELF header
 224 */
 225ssize_t __weak elfcorehdr_read(char *buf, size_t count, u64 *ppos)
 226{
 227	struct kvec kvec = { .iov_base = buf, .iov_len = count };
 228	struct iov_iter iter;
 229
 230	iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count);
 231
 232	return read_from_oldmem(&iter, count, ppos, false);
 233}
 234
 235/*
 236 * Architectures may override this function to read from notes sections
 237 */
 238ssize_t __weak elfcorehdr_read_notes(char *buf, size_t count, u64 *ppos)
 239{
 240	struct kvec kvec = { .iov_base = buf, .iov_len = count };
 241	struct iov_iter iter;
 242
 243	iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count);
 244
 245	return read_from_oldmem(&iter, count, ppos,
 246			cc_platform_has(CC_ATTR_MEM_ENCRYPT));
 247}
 248
 249/*
 250 * Architectures may override this function to map oldmem
 251 */
 252int __weak remap_oldmem_pfn_range(struct vm_area_struct *vma,
 253				  unsigned long from, unsigned long pfn,
 254				  unsigned long size, pgprot_t prot)
 255{
 256	prot = pgprot_encrypted(prot);
 257	return remap_pfn_range(vma, from, pfn, size, prot);
 258}
 259
 260/*
 261 * Architectures which support memory encryption override this.
 262 */
 263ssize_t __weak copy_oldmem_page_encrypted(struct iov_iter *iter,
 264		unsigned long pfn, size_t csize, unsigned long offset)
 265{
 266	return copy_oldmem_page(iter, pfn, csize, offset);
 267}
 268
 269#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP
 270static int vmcoredd_copy_dumps(struct iov_iter *iter, u64 start, size_t size)
 271{
 272	struct vmcoredd_node *dump;
 273	u64 offset = 0;
 274	size_t tsz;
 275	char *buf;
 276
 277	list_for_each_entry(dump, &vmcoredd_list, list) {
 278		if (start < offset + dump->size) {
 279			tsz = min(offset + (u64)dump->size - start, (u64)size);
 280			buf = dump->buf + start - offset;
 281			if (copy_to_iter(buf, tsz, iter) < tsz)
 282				return -EFAULT;
 283
 284			size -= tsz;
 285			start += tsz;
 286
 287			/* Leave now if buffer filled already */
 288			if (!size)
 289				return 0;
 290		}
 291		offset += dump->size;
 292	}
 293
 294	return 0;
 295}
 296
 297#ifdef CONFIG_MMU
 298static int vmcoredd_mmap_dumps(struct vm_area_struct *vma, unsigned long dst,
 299			       u64 start, size_t size)
 300{
 301	struct vmcoredd_node *dump;
 302	u64 offset = 0;
 303	size_t tsz;
 304	char *buf;
 305
 306	list_for_each_entry(dump, &vmcoredd_list, list) {
 307		if (start < offset + dump->size) {
 308			tsz = min(offset + (u64)dump->size - start, (u64)size);
 309			buf = dump->buf + start - offset;
 310			if (remap_vmalloc_range_partial(vma, dst, buf, 0,
 311							tsz))
 312				return -EFAULT;
 313
 314			size -= tsz;
 315			start += tsz;
 316			dst += tsz;
 317
 318			/* Leave now if buffer filled already */
 319			if (!size)
 320				return 0;
 321		}
 322		offset += dump->size;
 323	}
 324
 325	return 0;
 326}
 327#endif /* CONFIG_MMU */
 328#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
 329
 330/* Read from the ELF header and then the crash dump. On error, negative value is
 331 * returned otherwise number of bytes read are returned.
 332 */
 333static ssize_t __read_vmcore(struct iov_iter *iter, loff_t *fpos)
 334{
 335	struct vmcore_range *m = NULL;
 336	ssize_t acc = 0, tmp;
 337	size_t tsz;
 338	u64 start;
 339
 340	if (!iov_iter_count(iter) || *fpos >= vmcore_size)
 341		return 0;
 342
 343	iov_iter_truncate(iter, vmcore_size - *fpos);
 344
 345	/* Read ELF core header */
 346	if (*fpos < elfcorebuf_sz) {
 347		tsz = min(elfcorebuf_sz - (size_t)*fpos, iov_iter_count(iter));
 348		if (copy_to_iter(elfcorebuf + *fpos, tsz, iter) < tsz)
 349			return -EFAULT;
 350		*fpos += tsz;
 351		acc += tsz;
 352
 353		/* leave now if filled buffer already */
 354		if (!iov_iter_count(iter))
 355			return acc;
 356	}
 357
 358	/* Read ELF note segment */
 359	if (*fpos < elfcorebuf_sz + elfnotes_sz) {
 360		void *kaddr;
 361
 362		/* We add device dumps before other elf notes because the
 363		 * other elf notes may not fill the elf notes buffer
 364		 * completely and we will end up with zero-filled data
 365		 * between the elf notes and the device dumps. Tools will
 366		 * then try to decode this zero-filled data as valid notes
 367		 * and we don't want that. Hence, adding device dumps before
 368		 * the other elf notes ensure that zero-filled data can be
 369		 * avoided.
 370		 */
 371#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP
 372		/* Read device dumps */
 373		if (*fpos < elfcorebuf_sz + vmcoredd_orig_sz) {
 374			tsz = min(elfcorebuf_sz + vmcoredd_orig_sz -
 375				  (size_t)*fpos, iov_iter_count(iter));
 376			start = *fpos - elfcorebuf_sz;
 377			if (vmcoredd_copy_dumps(iter, start, tsz))
 378				return -EFAULT;
 379
 380			*fpos += tsz;
 381			acc += tsz;
 382
 383			/* leave now if filled buffer already */
 384			if (!iov_iter_count(iter))
 385				return acc;
 386		}
 387#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
 388
 389		/* Read remaining elf notes */
 390		tsz = min(elfcorebuf_sz + elfnotes_sz - (size_t)*fpos,
 391			  iov_iter_count(iter));
 392		kaddr = elfnotes_buf + *fpos - elfcorebuf_sz - vmcoredd_orig_sz;
 393		if (copy_to_iter(kaddr, tsz, iter) < tsz)
 394			return -EFAULT;
 395
 396		*fpos += tsz;
 397		acc += tsz;
 398
 399		/* leave now if filled buffer already */
 400		if (!iov_iter_count(iter))
 401			return acc;
 402
 403		cond_resched();
 404	}
 405
 406	list_for_each_entry(m, &vmcore_list, list) {
 407		if (*fpos < m->offset + m->size) {
 408			tsz = (size_t)min_t(unsigned long long,
 409					    m->offset + m->size - *fpos,
 410					    iov_iter_count(iter));
 411			start = m->paddr + *fpos - m->offset;
 412			tmp = read_from_oldmem(iter, tsz, &start,
 413					cc_platform_has(CC_ATTR_MEM_ENCRYPT));
 414			if (tmp < 0)
 415				return tmp;
 416			*fpos += tsz;
 417			acc += tsz;
 418
 419			/* leave now if filled buffer already */
 420			if (!iov_iter_count(iter))
 421				return acc;
 422		}
 423
 424		cond_resched();
 425	}
 426
 427	return acc;
 428}
 429
 430static ssize_t read_vmcore(struct kiocb *iocb, struct iov_iter *iter)
 431{
 432	return __read_vmcore(iter, &iocb->ki_pos);
 433}
 434
 435/**
 436 * vmcore_alloc_buf - allocate buffer in vmalloc memory
 437 * @size: size of buffer
 438 *
 439 * If CONFIG_MMU is defined, use vmalloc_user() to allow users to mmap
 440 * the buffer to user-space by means of remap_vmalloc_range().
 441 *
 442 * If CONFIG_MMU is not defined, use vzalloc() since mmap_vmcore() is
 443 * disabled and there's no need to allow users to mmap the buffer.
 444 */
 445static inline char *vmcore_alloc_buf(size_t size)
 446{
 447#ifdef CONFIG_MMU
 448	return vmalloc_user(size);
 449#else
 450	return vzalloc(size);
 451#endif
 452}
 453
 454/*
 455 * Disable mmap_vmcore() if CONFIG_MMU is not defined. MMU is
 456 * essential for mmap_vmcore() in order to map physically
 457 * non-contiguous objects (ELF header, ELF note segment and memory
 458 * regions in the 1st kernel pointed to by PT_LOAD entries) into
 459 * virtually contiguous user-space in ELF layout.
 460 */
 461#ifdef CONFIG_MMU
 462
 463/*
 464 * The vmcore fault handler uses the page cache and fills data using the
 465 * standard __read_vmcore() function.
 466 *
 467 * On s390 the fault handler is used for memory regions that can't be mapped
 468 * directly with remap_pfn_range().
 469 */
 470static vm_fault_t mmap_vmcore_fault(struct vm_fault *vmf)
 471{
 472#ifdef CONFIG_S390
 473	struct address_space *mapping = vmf->vma->vm_file->f_mapping;
 474	pgoff_t index = vmf->pgoff;
 475	struct iov_iter iter;
 476	struct kvec kvec;
 477	struct page *page;
 478	loff_t offset;
 479	int rc;
 480
 481	page = find_or_create_page(mapping, index, GFP_KERNEL);
 482	if (!page)
 483		return VM_FAULT_OOM;
 484	if (!PageUptodate(page)) {
 485		offset = (loff_t) index << PAGE_SHIFT;
 486		kvec.iov_base = page_address(page);
 487		kvec.iov_len = PAGE_SIZE;
 488		iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, PAGE_SIZE);
 489
 490		rc = __read_vmcore(&iter, &offset);
 491		if (rc < 0) {
 492			unlock_page(page);
 493			put_page(page);
 494			return vmf_error(rc);
 495		}
 496		SetPageUptodate(page);
 497	}
 498	unlock_page(page);
 499	vmf->page = page;
 500	return 0;
 501#else
 502	return VM_FAULT_SIGBUS;
 503#endif
 504}
 505
 506static const struct vm_operations_struct vmcore_mmap_ops = {
 507	.fault = mmap_vmcore_fault,
 508};
 509
 510/*
 511 * remap_oldmem_pfn_checked - do remap_oldmem_pfn_range replacing all pages
 512 * reported as not being ram with the zero page.
 513 *
 514 * @vma: vm_area_struct describing requested mapping
 515 * @from: start remapping from
 516 * @pfn: page frame number to start remapping to
 517 * @size: remapping size
 518 * @prot: protection bits
 519 *
 520 * Returns zero on success, -EAGAIN on failure.
 521 */
 522static int remap_oldmem_pfn_checked(struct vm_area_struct *vma,
 523				    unsigned long from, unsigned long pfn,
 524				    unsigned long size, pgprot_t prot)
 525{
 526	unsigned long map_size;
 527	unsigned long pos_start, pos_end, pos;
 528	unsigned long zeropage_pfn = my_zero_pfn(0);
 529	size_t len = 0;
 530
 531	pos_start = pfn;
 532	pos_end = pfn + (size >> PAGE_SHIFT);
 533
 534	for (pos = pos_start; pos < pos_end; ++pos) {
 535		if (!pfn_is_ram(pos)) {
 536			/*
 537			 * We hit a page which is not ram. Remap the continuous
 538			 * region between pos_start and pos-1 and replace
 539			 * the non-ram page at pos with the zero page.
 540			 */
 541			if (pos > pos_start) {
 542				/* Remap continuous region */
 543				map_size = (pos - pos_start) << PAGE_SHIFT;
 544				if (remap_oldmem_pfn_range(vma, from + len,
 545							   pos_start, map_size,
 546							   prot))
 547					goto fail;
 548				len += map_size;
 549			}
 550			/* Remap the zero page */
 551			if (remap_oldmem_pfn_range(vma, from + len,
 552						   zeropage_pfn,
 553						   PAGE_SIZE, prot))
 554				goto fail;
 555			len += PAGE_SIZE;
 556			pos_start = pos + 1;
 557		}
 558	}
 559	if (pos > pos_start) {
 560		/* Remap the rest */
 561		map_size = (pos - pos_start) << PAGE_SHIFT;
 562		if (remap_oldmem_pfn_range(vma, from + len, pos_start,
 563					   map_size, prot))
 564			goto fail;
 565	}
 566	return 0;
 567fail:
 568	do_munmap(vma->vm_mm, from, len, NULL);
 569	return -EAGAIN;
 570}
 571
 572static int vmcore_remap_oldmem_pfn(struct vm_area_struct *vma,
 573			    unsigned long from, unsigned long pfn,
 574			    unsigned long size, pgprot_t prot)
 575{
 576	int ret, idx;
 577
 578	/*
 579	 * Check if a callback was registered to avoid looping over all
 580	 * pages without a reason.
 581	 */
 582	idx = srcu_read_lock(&vmcore_cb_srcu);
 583	if (!list_empty(&vmcore_cb_list))
 584		ret = remap_oldmem_pfn_checked(vma, from, pfn, size, prot);
 585	else
 586		ret = remap_oldmem_pfn_range(vma, from, pfn, size, prot);
 587	srcu_read_unlock(&vmcore_cb_srcu, idx);
 588	return ret;
 589}
 590
 591static int mmap_vmcore(struct file *file, struct vm_area_struct *vma)
 592{
 593	size_t size = vma->vm_end - vma->vm_start;
 594	u64 start, end, len, tsz;
 595	struct vmcore_range *m;
 596
 597	start = (u64)vma->vm_pgoff << PAGE_SHIFT;
 598	end = start + size;
 599
 600	if (size > vmcore_size || end > vmcore_size)
 601		return -EINVAL;
 602
 603	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
 604		return -EPERM;
 605
 606	vm_flags_mod(vma, VM_MIXEDMAP, VM_MAYWRITE | VM_MAYEXEC);
 607	vma->vm_ops = &vmcore_mmap_ops;
 608
 609	len = 0;
 610
 611	if (start < elfcorebuf_sz) {
 612		u64 pfn;
 613
 614		tsz = min(elfcorebuf_sz - (size_t)start, size);
 615		pfn = __pa(elfcorebuf + start) >> PAGE_SHIFT;
 616		if (remap_pfn_range(vma, vma->vm_start, pfn, tsz,
 617				    vma->vm_page_prot))
 618			return -EAGAIN;
 619		size -= tsz;
 620		start += tsz;
 621		len += tsz;
 622
 623		if (size == 0)
 624			return 0;
 625	}
 626
 627	if (start < elfcorebuf_sz + elfnotes_sz) {
 628		void *kaddr;
 629
 630		/* We add device dumps before other elf notes because the
 631		 * other elf notes may not fill the elf notes buffer
 632		 * completely and we will end up with zero-filled data
 633		 * between the elf notes and the device dumps. Tools will
 634		 * then try to decode this zero-filled data as valid notes
 635		 * and we don't want that. Hence, adding device dumps before
 636		 * the other elf notes ensure that zero-filled data can be
 637		 * avoided. This also ensures that the device dumps and
 638		 * other elf notes can be properly mmaped at page aligned
 639		 * address.
 640		 */
 641#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP
 642		/* Read device dumps */
 643		if (start < elfcorebuf_sz + vmcoredd_orig_sz) {
 644			u64 start_off;
 645
 646			tsz = min(elfcorebuf_sz + vmcoredd_orig_sz -
 647				  (size_t)start, size);
 648			start_off = start - elfcorebuf_sz;
 649			if (vmcoredd_mmap_dumps(vma, vma->vm_start + len,
 650						start_off, tsz))
 651				goto fail;
 652
 653			size -= tsz;
 654			start += tsz;
 655			len += tsz;
 656
 657			/* leave now if filled buffer already */
 658			if (!size)
 659				return 0;
 660		}
 661#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
 662
 663		/* Read remaining elf notes */
 664		tsz = min(elfcorebuf_sz + elfnotes_sz - (size_t)start, size);
 665		kaddr = elfnotes_buf + start - elfcorebuf_sz - vmcoredd_orig_sz;
 666		if (remap_vmalloc_range_partial(vma, vma->vm_start + len,
 667						kaddr, 0, tsz))
 668			goto fail;
 669
 670		size -= tsz;
 671		start += tsz;
 672		len += tsz;
 673
 674		if (size == 0)
 675			return 0;
 676	}
 677
 678	list_for_each_entry(m, &vmcore_list, list) {
 679		if (start < m->offset + m->size) {
 680			u64 paddr = 0;
 681
 682			tsz = (size_t)min_t(unsigned long long,
 683					    m->offset + m->size - start, size);
 684			paddr = m->paddr + start - m->offset;
 685			if (vmcore_remap_oldmem_pfn(vma, vma->vm_start + len,
 686						    paddr >> PAGE_SHIFT, tsz,
 687						    vma->vm_page_prot))
 688				goto fail;
 689			size -= tsz;
 690			start += tsz;
 691			len += tsz;
 692
 693			if (size == 0)
 694				return 0;
 695		}
 696	}
 697
 698	return 0;
 699fail:
 700	do_munmap(vma->vm_mm, vma->vm_start, len, NULL);
 701	return -EAGAIN;
 702}
 703#else
 704static int mmap_vmcore(struct file *file, struct vm_area_struct *vma)
 705{
 706	return -ENOSYS;
 707}
 708#endif
 709
 710static const struct proc_ops vmcore_proc_ops = {
 711	.proc_open	= open_vmcore,
 712	.proc_release	= release_vmcore,
 713	.proc_read_iter	= read_vmcore,
 714	.proc_lseek	= default_llseek,
 715	.proc_mmap	= mmap_vmcore,
 716};
 717
 718static u64 get_vmcore_size(size_t elfsz, size_t elfnotesegsz,
 719			   struct list_head *vc_list)
 720{
 721	struct vmcore_range *m;
 722	u64 size;
 723
 724	size = elfsz + elfnotesegsz;
 725	list_for_each_entry(m, vc_list, list) {
 726		size += m->size;
 727	}
 728	return size;
 729}
 730
 731/**
 732 * update_note_header_size_elf64 - update p_memsz member of each PT_NOTE entry
 733 *
 734 * @ehdr_ptr: ELF header
 735 *
 736 * This function updates p_memsz member of each PT_NOTE entry in the
 737 * program header table pointed to by @ehdr_ptr to real size of ELF
 738 * note segment.
 739 */
 740static int __init update_note_header_size_elf64(const Elf64_Ehdr *ehdr_ptr)
 741{
 742	int i, rc=0;
 743	Elf64_Phdr *phdr_ptr;
 744	Elf64_Nhdr *nhdr_ptr;
 745
 746	phdr_ptr = (Elf64_Phdr *)(ehdr_ptr + 1);
 747	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
 748		void *notes_section;
 749		u64 offset, max_sz, sz, real_sz = 0;
 750		if (phdr_ptr->p_type != PT_NOTE)
 751			continue;
 752		max_sz = phdr_ptr->p_memsz;
 753		offset = phdr_ptr->p_offset;
 754		notes_section = kmalloc(max_sz, GFP_KERNEL);
 755		if (!notes_section)
 756			return -ENOMEM;
 757		rc = elfcorehdr_read_notes(notes_section, max_sz, &offset);
 758		if (rc < 0) {
 759			kfree(notes_section);
 760			return rc;
 761		}
 762		nhdr_ptr = notes_section;
 763		while (nhdr_ptr->n_namesz != 0) {
 764			sz = sizeof(Elf64_Nhdr) +
 765				(((u64)nhdr_ptr->n_namesz + 3) & ~3) +
 766				(((u64)nhdr_ptr->n_descsz + 3) & ~3);
 767			if ((real_sz + sz) > max_sz) {
 768				pr_warn("Warning: Exceeded p_memsz, dropping PT_NOTE entry n_namesz=0x%x, n_descsz=0x%x\n",
 769					nhdr_ptr->n_namesz, nhdr_ptr->n_descsz);
 770				break;
 771			}
 772			real_sz += sz;
 773			nhdr_ptr = (Elf64_Nhdr*)((char*)nhdr_ptr + sz);
 774		}
 775		kfree(notes_section);
 776		phdr_ptr->p_memsz = real_sz;
 777		if (real_sz == 0) {
 778			pr_warn("Warning: Zero PT_NOTE entries found\n");
 779		}
 780	}
 781
 782	return 0;
 783}
 784
 785/**
 786 * get_note_number_and_size_elf64 - get the number of PT_NOTE program
 787 * headers and sum of real size of their ELF note segment headers and
 788 * data.
 789 *
 790 * @ehdr_ptr: ELF header
 791 * @nr_ptnote: buffer for the number of PT_NOTE program headers
 792 * @sz_ptnote: buffer for size of unique PT_NOTE program header
 793 *
 794 * This function is used to merge multiple PT_NOTE program headers
 795 * into a unique single one. The resulting unique entry will have
 796 * @sz_ptnote in its phdr->p_mem.
 797 *
 798 * It is assumed that program headers with PT_NOTE type pointed to by
 799 * @ehdr_ptr has already been updated by update_note_header_size_elf64
 800 * and each of PT_NOTE program headers has actual ELF note segment
 801 * size in its p_memsz member.
 802 */
 803static int __init get_note_number_and_size_elf64(const Elf64_Ehdr *ehdr_ptr,
 804						 int *nr_ptnote, u64 *sz_ptnote)
 805{
 806	int i;
 807	Elf64_Phdr *phdr_ptr;
 808
 809	*nr_ptnote = *sz_ptnote = 0;
 810
 811	phdr_ptr = (Elf64_Phdr *)(ehdr_ptr + 1);
 812	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
 813		if (phdr_ptr->p_type != PT_NOTE)
 814			continue;
 815		*nr_ptnote += 1;
 816		*sz_ptnote += phdr_ptr->p_memsz;
 817	}
 818
 819	return 0;
 820}
 821
 822/**
 823 * copy_notes_elf64 - copy ELF note segments in a given buffer
 824 *
 825 * @ehdr_ptr: ELF header
 826 * @notes_buf: buffer into which ELF note segments are copied
 827 *
 828 * This function is used to copy ELF note segment in the 1st kernel
 829 * into the buffer @notes_buf in the 2nd kernel. It is assumed that
 830 * size of the buffer @notes_buf is equal to or larger than sum of the
 831 * real ELF note segment headers and data.
 832 *
 833 * It is assumed that program headers with PT_NOTE type pointed to by
 834 * @ehdr_ptr has already been updated by update_note_header_size_elf64
 835 * and each of PT_NOTE program headers has actual ELF note segment
 836 * size in its p_memsz member.
 837 */
 838static int __init copy_notes_elf64(const Elf64_Ehdr *ehdr_ptr, char *notes_buf)
 839{
 840	int i, rc=0;
 841	Elf64_Phdr *phdr_ptr;
 842
 843	phdr_ptr = (Elf64_Phdr*)(ehdr_ptr + 1);
 844
 845	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
 846		u64 offset;
 847		if (phdr_ptr->p_type != PT_NOTE)
 848			continue;
 849		offset = phdr_ptr->p_offset;
 850		rc = elfcorehdr_read_notes(notes_buf, phdr_ptr->p_memsz,
 851					   &offset);
 852		if (rc < 0)
 853			return rc;
 854		notes_buf += phdr_ptr->p_memsz;
 855	}
 856
 857	return 0;
 858}
 859
 860/* Merges all the PT_NOTE headers into one. */
 861static int __init merge_note_headers_elf64(char *elfptr, size_t *elfsz,
 862					   char **notes_buf, size_t *notes_sz)
 863{
 864	int i, nr_ptnote=0, rc=0;
 865	char *tmp;
 866	Elf64_Ehdr *ehdr_ptr;
 867	Elf64_Phdr phdr;
 868	u64 phdr_sz = 0, note_off;
 869
 870	ehdr_ptr = (Elf64_Ehdr *)elfptr;
 871
 872	rc = update_note_header_size_elf64(ehdr_ptr);
 873	if (rc < 0)
 874		return rc;
 875
 876	rc = get_note_number_and_size_elf64(ehdr_ptr, &nr_ptnote, &phdr_sz);
 877	if (rc < 0)
 878		return rc;
 879
 880	*notes_sz = roundup(phdr_sz, PAGE_SIZE);
 881	*notes_buf = vmcore_alloc_buf(*notes_sz);
 882	if (!*notes_buf)
 883		return -ENOMEM;
 884
 885	rc = copy_notes_elf64(ehdr_ptr, *notes_buf);
 886	if (rc < 0)
 887		return rc;
 888
 889	/* Prepare merged PT_NOTE program header. */
 890	phdr.p_type    = PT_NOTE;
 891	phdr.p_flags   = 0;
 892	note_off = sizeof(Elf64_Ehdr) +
 893			(ehdr_ptr->e_phnum - nr_ptnote +1) * sizeof(Elf64_Phdr);
 894	phdr.p_offset  = roundup(note_off, PAGE_SIZE);
 895	phdr.p_vaddr   = phdr.p_paddr = 0;
 896	phdr.p_filesz  = phdr.p_memsz = phdr_sz;
 897	phdr.p_align   = 4;
 898
 899	/* Add merged PT_NOTE program header*/
 900	tmp = elfptr + sizeof(Elf64_Ehdr);
 901	memcpy(tmp, &phdr, sizeof(phdr));
 902	tmp += sizeof(phdr);
 903
 904	/* Remove unwanted PT_NOTE program headers. */
 905	i = (nr_ptnote - 1) * sizeof(Elf64_Phdr);
 906	*elfsz = *elfsz - i;
 907	memmove(tmp, tmp+i, ((*elfsz)-sizeof(Elf64_Ehdr)-sizeof(Elf64_Phdr)));
 908	memset(elfptr + *elfsz, 0, i);
 909	*elfsz = roundup(*elfsz, PAGE_SIZE);
 910
 911	/* Modify e_phnum to reflect merged headers. */
 912	ehdr_ptr->e_phnum = ehdr_ptr->e_phnum - nr_ptnote + 1;
 913
 914	/* Store the size of all notes.  We need this to update the note
 915	 * header when the device dumps will be added.
 916	 */
 917	elfnotes_orig_sz = phdr.p_memsz;
 918
 919	return 0;
 920}
 921
 922/**
 923 * update_note_header_size_elf32 - update p_memsz member of each PT_NOTE entry
 924 *
 925 * @ehdr_ptr: ELF header
 926 *
 927 * This function updates p_memsz member of each PT_NOTE entry in the
 928 * program header table pointed to by @ehdr_ptr to real size of ELF
 929 * note segment.
 930 */
 931static int __init update_note_header_size_elf32(const Elf32_Ehdr *ehdr_ptr)
 932{
 933	int i, rc=0;
 934	Elf32_Phdr *phdr_ptr;
 935	Elf32_Nhdr *nhdr_ptr;
 936
 937	phdr_ptr = (Elf32_Phdr *)(ehdr_ptr + 1);
 938	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
 939		void *notes_section;
 940		u64 offset, max_sz, sz, real_sz = 0;
 941		if (phdr_ptr->p_type != PT_NOTE)
 942			continue;
 943		max_sz = phdr_ptr->p_memsz;
 944		offset = phdr_ptr->p_offset;
 945		notes_section = kmalloc(max_sz, GFP_KERNEL);
 946		if (!notes_section)
 947			return -ENOMEM;
 948		rc = elfcorehdr_read_notes(notes_section, max_sz, &offset);
 949		if (rc < 0) {
 950			kfree(notes_section);
 951			return rc;
 952		}
 953		nhdr_ptr = notes_section;
 954		while (nhdr_ptr->n_namesz != 0) {
 955			sz = sizeof(Elf32_Nhdr) +
 956				(((u64)nhdr_ptr->n_namesz + 3) & ~3) +
 957				(((u64)nhdr_ptr->n_descsz + 3) & ~3);
 958			if ((real_sz + sz) > max_sz) {
 959				pr_warn("Warning: Exceeded p_memsz, dropping PT_NOTE entry n_namesz=0x%x, n_descsz=0x%x\n",
 960					nhdr_ptr->n_namesz, nhdr_ptr->n_descsz);
 961				break;
 962			}
 963			real_sz += sz;
 964			nhdr_ptr = (Elf32_Nhdr*)((char*)nhdr_ptr + sz);
 965		}
 966		kfree(notes_section);
 967		phdr_ptr->p_memsz = real_sz;
 968		if (real_sz == 0) {
 969			pr_warn("Warning: Zero PT_NOTE entries found\n");
 970		}
 971	}
 972
 973	return 0;
 974}
 975
 976/**
 977 * get_note_number_and_size_elf32 - get the number of PT_NOTE program
 978 * headers and sum of real size of their ELF note segment headers and
 979 * data.
 980 *
 981 * @ehdr_ptr: ELF header
 982 * @nr_ptnote: buffer for the number of PT_NOTE program headers
 983 * @sz_ptnote: buffer for size of unique PT_NOTE program header
 984 *
 985 * This function is used to merge multiple PT_NOTE program headers
 986 * into a unique single one. The resulting unique entry will have
 987 * @sz_ptnote in its phdr->p_mem.
 988 *
 989 * It is assumed that program headers with PT_NOTE type pointed to by
 990 * @ehdr_ptr has already been updated by update_note_header_size_elf32
 991 * and each of PT_NOTE program headers has actual ELF note segment
 992 * size in its p_memsz member.
 993 */
 994static int __init get_note_number_and_size_elf32(const Elf32_Ehdr *ehdr_ptr,
 995						 int *nr_ptnote, u64 *sz_ptnote)
 996{
 997	int i;
 998	Elf32_Phdr *phdr_ptr;
 999
1000	*nr_ptnote = *sz_ptnote = 0;
1001
1002	phdr_ptr = (Elf32_Phdr *)(ehdr_ptr + 1);
1003	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
1004		if (phdr_ptr->p_type != PT_NOTE)
1005			continue;
1006		*nr_ptnote += 1;
1007		*sz_ptnote += phdr_ptr->p_memsz;
1008	}
1009
1010	return 0;
1011}
1012
1013/**
1014 * copy_notes_elf32 - copy ELF note segments in a given buffer
1015 *
1016 * @ehdr_ptr: ELF header
1017 * @notes_buf: buffer into which ELF note segments are copied
1018 *
1019 * This function is used to copy ELF note segment in the 1st kernel
1020 * into the buffer @notes_buf in the 2nd kernel. It is assumed that
1021 * size of the buffer @notes_buf is equal to or larger than sum of the
1022 * real ELF note segment headers and data.
1023 *
1024 * It is assumed that program headers with PT_NOTE type pointed to by
1025 * @ehdr_ptr has already been updated by update_note_header_size_elf32
1026 * and each of PT_NOTE program headers has actual ELF note segment
1027 * size in its p_memsz member.
1028 */
1029static int __init copy_notes_elf32(const Elf32_Ehdr *ehdr_ptr, char *notes_buf)
1030{
1031	int i, rc=0;
1032	Elf32_Phdr *phdr_ptr;
1033
1034	phdr_ptr = (Elf32_Phdr*)(ehdr_ptr + 1);
1035
1036	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
1037		u64 offset;
1038		if (phdr_ptr->p_type != PT_NOTE)
1039			continue;
1040		offset = phdr_ptr->p_offset;
1041		rc = elfcorehdr_read_notes(notes_buf, phdr_ptr->p_memsz,
1042					   &offset);
1043		if (rc < 0)
1044			return rc;
1045		notes_buf += phdr_ptr->p_memsz;
1046	}
1047
1048	return 0;
1049}
1050
1051/* Merges all the PT_NOTE headers into one. */
1052static int __init merge_note_headers_elf32(char *elfptr, size_t *elfsz,
1053					   char **notes_buf, size_t *notes_sz)
1054{
1055	int i, nr_ptnote=0, rc=0;
1056	char *tmp;
1057	Elf32_Ehdr *ehdr_ptr;
1058	Elf32_Phdr phdr;
1059	u64 phdr_sz = 0, note_off;
1060
1061	ehdr_ptr = (Elf32_Ehdr *)elfptr;
1062
1063	rc = update_note_header_size_elf32(ehdr_ptr);
1064	if (rc < 0)
1065		return rc;
1066
1067	rc = get_note_number_and_size_elf32(ehdr_ptr, &nr_ptnote, &phdr_sz);
1068	if (rc < 0)
1069		return rc;
1070
1071	*notes_sz = roundup(phdr_sz, PAGE_SIZE);
1072	*notes_buf = vmcore_alloc_buf(*notes_sz);
1073	if (!*notes_buf)
1074		return -ENOMEM;
1075
1076	rc = copy_notes_elf32(ehdr_ptr, *notes_buf);
1077	if (rc < 0)
1078		return rc;
1079
1080	/* Prepare merged PT_NOTE program header. */
1081	phdr.p_type    = PT_NOTE;
1082	phdr.p_flags   = 0;
1083	note_off = sizeof(Elf32_Ehdr) +
1084			(ehdr_ptr->e_phnum - nr_ptnote +1) * sizeof(Elf32_Phdr);
1085	phdr.p_offset  = roundup(note_off, PAGE_SIZE);
1086	phdr.p_vaddr   = phdr.p_paddr = 0;
1087	phdr.p_filesz  = phdr.p_memsz = phdr_sz;
1088	phdr.p_align   = 4;
1089
1090	/* Add merged PT_NOTE program header*/
1091	tmp = elfptr + sizeof(Elf32_Ehdr);
1092	memcpy(tmp, &phdr, sizeof(phdr));
1093	tmp += sizeof(phdr);
1094
1095	/* Remove unwanted PT_NOTE program headers. */
1096	i = (nr_ptnote - 1) * sizeof(Elf32_Phdr);
1097	*elfsz = *elfsz - i;
1098	memmove(tmp, tmp+i, ((*elfsz)-sizeof(Elf32_Ehdr)-sizeof(Elf32_Phdr)));
1099	memset(elfptr + *elfsz, 0, i);
1100	*elfsz = roundup(*elfsz, PAGE_SIZE);
1101
1102	/* Modify e_phnum to reflect merged headers. */
1103	ehdr_ptr->e_phnum = ehdr_ptr->e_phnum - nr_ptnote + 1;
1104
1105	/* Store the size of all notes.  We need this to update the note
1106	 * header when the device dumps will be added.
1107	 */
1108	elfnotes_orig_sz = phdr.p_memsz;
1109
1110	return 0;
1111}
1112
1113/* Add memory chunks represented by program headers to vmcore list. Also update
1114 * the new offset fields of exported program headers. */
1115static int __init process_ptload_program_headers_elf64(char *elfptr,
1116						size_t elfsz,
1117						size_t elfnotes_sz,
1118						struct list_head *vc_list)
1119{
1120	int i;
1121	Elf64_Ehdr *ehdr_ptr;
1122	Elf64_Phdr *phdr_ptr;
1123	loff_t vmcore_off;
1124
1125	ehdr_ptr = (Elf64_Ehdr *)elfptr;
1126	phdr_ptr = (Elf64_Phdr*)(elfptr + sizeof(Elf64_Ehdr)); /* PT_NOTE hdr */
1127
1128	/* Skip ELF header, program headers and ELF note segment. */
1129	vmcore_off = elfsz + elfnotes_sz;
1130
1131	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
1132		u64 paddr, start, end, size;
1133
1134		if (phdr_ptr->p_type != PT_LOAD)
1135			continue;
1136
1137		paddr = phdr_ptr->p_offset;
1138		start = rounddown(paddr, PAGE_SIZE);
1139		end = roundup(paddr + phdr_ptr->p_memsz, PAGE_SIZE);
1140		size = end - start;
1141
1142		if (vmcore_alloc_add_range(vc_list, start, size))
1143			return -ENOMEM;
1144
1145		/* Update the program header offset. */
1146		phdr_ptr->p_offset = vmcore_off + (paddr - start);
1147		vmcore_off = vmcore_off + size;
1148	}
1149	return 0;
1150}
1151
1152static int __init process_ptload_program_headers_elf32(char *elfptr,
1153						size_t elfsz,
1154						size_t elfnotes_sz,
1155						struct list_head *vc_list)
1156{
1157	int i;
1158	Elf32_Ehdr *ehdr_ptr;
1159	Elf32_Phdr *phdr_ptr;
1160	loff_t vmcore_off;
1161
1162	ehdr_ptr = (Elf32_Ehdr *)elfptr;
1163	phdr_ptr = (Elf32_Phdr*)(elfptr + sizeof(Elf32_Ehdr)); /* PT_NOTE hdr */
1164
1165	/* Skip ELF header, program headers and ELF note segment. */
1166	vmcore_off = elfsz + elfnotes_sz;
1167
1168	for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) {
1169		u64 paddr, start, end, size;
1170
1171		if (phdr_ptr->p_type != PT_LOAD)
1172			continue;
1173
1174		paddr = phdr_ptr->p_offset;
1175		start = rounddown(paddr, PAGE_SIZE);
1176		end = roundup(paddr + phdr_ptr->p_memsz, PAGE_SIZE);
1177		size = end - start;
1178
1179		if (vmcore_alloc_add_range(vc_list, start, size))
1180			return -ENOMEM;
1181
1182		/* Update the program header offset */
1183		phdr_ptr->p_offset = vmcore_off + (paddr - start);
1184		vmcore_off = vmcore_off + size;
1185	}
1186	return 0;
1187}
1188
1189/* Sets offset fields of vmcore elements. */
1190static void set_vmcore_list_offsets(size_t elfsz, size_t elfnotes_sz,
1191				    struct list_head *vc_list)
1192{
1193	struct vmcore_range *m;
1194	loff_t vmcore_off;
1195
1196	/* Skip ELF header, program headers and ELF note segment. */
1197	vmcore_off = elfsz + elfnotes_sz;
1198
1199	list_for_each_entry(m, vc_list, list) {
1200		m->offset = vmcore_off;
1201		vmcore_off += m->size;
1202	}
1203}
1204
1205static void free_elfcorebuf(void)
1206{
1207	free_pages((unsigned long)elfcorebuf, get_order(elfcorebuf_sz_orig));
1208	elfcorebuf = NULL;
1209	vfree(elfnotes_buf);
1210	elfnotes_buf = NULL;
1211}
1212
1213static int __init parse_crash_elf64_headers(void)
1214{
1215	int rc=0;
1216	Elf64_Ehdr ehdr;
1217	u64 addr;
1218
1219	addr = elfcorehdr_addr;
1220
1221	/* Read ELF header */
1222	rc = elfcorehdr_read((char *)&ehdr, sizeof(Elf64_Ehdr), &addr);
1223	if (rc < 0)
1224		return rc;
1225
1226	/* Do some basic Verification. */
1227	if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0 ||
1228		(ehdr.e_type != ET_CORE) ||
1229		!vmcore_elf64_check_arch(&ehdr) ||
1230		ehdr.e_ident[EI_CLASS] != ELFCLASS64 ||
1231		ehdr.e_ident[EI_VERSION] != EV_CURRENT ||
1232		ehdr.e_version != EV_CURRENT ||
1233		ehdr.e_ehsize != sizeof(Elf64_Ehdr) ||
1234		ehdr.e_phentsize != sizeof(Elf64_Phdr) ||
1235		ehdr.e_phnum == 0) {
1236		pr_warn("Warning: Core image elf header is not sane\n");
1237		return -EINVAL;
1238	}
1239
1240	/* Read in all elf headers. */
1241	elfcorebuf_sz_orig = sizeof(Elf64_Ehdr) +
1242				ehdr.e_phnum * sizeof(Elf64_Phdr);
1243	elfcorebuf_sz = elfcorebuf_sz_orig;
1244	elfcorebuf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1245					      get_order(elfcorebuf_sz_orig));
1246	if (!elfcorebuf)
1247		return -ENOMEM;
1248	addr = elfcorehdr_addr;
1249	rc = elfcorehdr_read(elfcorebuf, elfcorebuf_sz_orig, &addr);
1250	if (rc < 0)
1251		goto fail;
1252
1253	/* Merge all PT_NOTE headers into one. */
1254	rc = merge_note_headers_elf64(elfcorebuf, &elfcorebuf_sz,
1255				      &elfnotes_buf, &elfnotes_sz);
1256	if (rc)
1257		goto fail;
1258	rc = process_ptload_program_headers_elf64(elfcorebuf, elfcorebuf_sz,
1259						  elfnotes_sz, &vmcore_list);
1260	if (rc)
1261		goto fail;
1262	set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list);
1263	return 0;
1264fail:
1265	free_elfcorebuf();
1266	return rc;
1267}
1268
1269static int __init parse_crash_elf32_headers(void)
1270{
1271	int rc=0;
1272	Elf32_Ehdr ehdr;
1273	u64 addr;
1274
1275	addr = elfcorehdr_addr;
1276
1277	/* Read ELF header */
1278	rc = elfcorehdr_read((char *)&ehdr, sizeof(Elf32_Ehdr), &addr);
1279	if (rc < 0)
1280		return rc;
1281
1282	/* Do some basic Verification. */
1283	if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0 ||
1284		(ehdr.e_type != ET_CORE) ||
1285		!vmcore_elf32_check_arch(&ehdr) ||
1286		ehdr.e_ident[EI_CLASS] != ELFCLASS32||
1287		ehdr.e_ident[EI_VERSION] != EV_CURRENT ||
1288		ehdr.e_version != EV_CURRENT ||
1289		ehdr.e_ehsize != sizeof(Elf32_Ehdr) ||
1290		ehdr.e_phentsize != sizeof(Elf32_Phdr) ||
1291		ehdr.e_phnum == 0) {
1292		pr_warn("Warning: Core image elf header is not sane\n");
1293		return -EINVAL;
1294	}
1295
1296	/* Read in all elf headers. */
1297	elfcorebuf_sz_orig = sizeof(Elf32_Ehdr) + ehdr.e_phnum * sizeof(Elf32_Phdr);
1298	elfcorebuf_sz = elfcorebuf_sz_orig;
1299	elfcorebuf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1300					      get_order(elfcorebuf_sz_orig));
1301	if (!elfcorebuf)
1302		return -ENOMEM;
1303	addr = elfcorehdr_addr;
1304	rc = elfcorehdr_read(elfcorebuf, elfcorebuf_sz_orig, &addr);
1305	if (rc < 0)
1306		goto fail;
1307
1308	/* Merge all PT_NOTE headers into one. */
1309	rc = merge_note_headers_elf32(elfcorebuf, &elfcorebuf_sz,
1310				      &elfnotes_buf, &elfnotes_sz);
1311	if (rc)
1312		goto fail;
1313	rc = process_ptload_program_headers_elf32(elfcorebuf, elfcorebuf_sz,
1314						  elfnotes_sz, &vmcore_list);
1315	if (rc)
1316		goto fail;
1317	set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list);
1318	return 0;
1319fail:
1320	free_elfcorebuf();
1321	return rc;
1322}
1323
1324static int __init parse_crash_elf_headers(void)
1325{
1326	unsigned char e_ident[EI_NIDENT];
1327	u64 addr;
1328	int rc=0;
1329
1330	addr = elfcorehdr_addr;
1331	rc = elfcorehdr_read(e_ident, EI_NIDENT, &addr);
1332	if (rc < 0)
1333		return rc;
1334	if (memcmp(e_ident, ELFMAG, SELFMAG) != 0) {
1335		pr_warn("Warning: Core image elf header not found\n");
1336		return -EINVAL;
1337	}
1338
1339	if (e_ident[EI_CLASS] == ELFCLASS64) {
1340		rc = parse_crash_elf64_headers();
1341		if (rc)
1342			return rc;
1343	} else if (e_ident[EI_CLASS] == ELFCLASS32) {
1344		rc = parse_crash_elf32_headers();
1345		if (rc)
1346			return rc;
1347	} else {
1348		pr_warn("Warning: Core image elf header is not sane\n");
1349		return -EINVAL;
1350	}
1351
1352	/* Determine vmcore size. */
1353	vmcore_size = get_vmcore_size(elfcorebuf_sz, elfnotes_sz,
1354				      &vmcore_list);
1355
1356	return 0;
1357}
1358
1359#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP
1360/**
1361 * vmcoredd_write_header - Write vmcore device dump header at the
1362 * beginning of the dump's buffer.
1363 * @buf: Output buffer where the note is written
1364 * @data: Dump info
1365 * @size: Size of the dump
1366 *
1367 * Fills beginning of the dump's buffer with vmcore device dump header.
1368 */
1369static void vmcoredd_write_header(void *buf, struct vmcoredd_data *data,
1370				  u32 size)
1371{
1372	struct vmcoredd_header *vdd_hdr = (struct vmcoredd_header *)buf;
1373
1374	vdd_hdr->n_namesz = sizeof(vdd_hdr->name);
1375	vdd_hdr->n_descsz = size + sizeof(vdd_hdr->dump_name);
1376	vdd_hdr->n_type = NT_VMCOREDD;
1377
1378	strscpy_pad(vdd_hdr->name, VMCOREDD_NOTE_NAME);
1379	strscpy_pad(vdd_hdr->dump_name, data->dump_name);
1380}
1381
1382/**
1383 * vmcoredd_update_program_headers - Update all ELF program headers
1384 * @elfptr: Pointer to elf header
1385 * @elfnotesz: Size of elf notes aligned to page size
1386 * @vmcoreddsz: Size of device dumps to be added to elf note header
1387 *
1388 * Determine type of ELF header (Elf64 or Elf32) and update the elf note size.
1389 * Also update the offsets of all the program headers after the elf note header.
1390 */
1391static void vmcoredd_update_program_headers(char *elfptr, size_t elfnotesz,
1392					    size_t vmcoreddsz)
1393{
1394	unsigned char *e_ident = (unsigned char *)elfptr;
1395	u64 start, end, size;
1396	loff_t vmcore_off;
1397	u32 i;
1398
1399	vmcore_off = elfcorebuf_sz + elfnotesz;
1400
1401	if (e_ident[EI_CLASS] == ELFCLASS64) {
1402		Elf64_Ehdr *ehdr = (Elf64_Ehdr *)elfptr;
1403		Elf64_Phdr *phdr = (Elf64_Phdr *)(elfptr + sizeof(Elf64_Ehdr));
1404
1405		/* Update all program headers */
1406		for (i = 0; i < ehdr->e_phnum; i++, phdr++) {
1407			if (phdr->p_type == PT_NOTE) {
1408				/* Update note size */
1409				phdr->p_memsz = elfnotes_orig_sz + vmcoreddsz;
1410				phdr->p_filesz = phdr->p_memsz;
1411				continue;
1412			}
1413
1414			start = rounddown(phdr->p_offset, PAGE_SIZE);
1415			end = roundup(phdr->p_offset + phdr->p_memsz,
1416				      PAGE_SIZE);
1417			size = end - start;
1418			phdr->p_offset = vmcore_off + (phdr->p_offset - start);
1419			vmcore_off += size;
1420		}
1421	} else {
1422		Elf32_Ehdr *ehdr = (Elf32_Ehdr *)elfptr;
1423		Elf32_Phdr *phdr = (Elf32_Phdr *)(elfptr + sizeof(Elf32_Ehdr));
1424
1425		/* Update all program headers */
1426		for (i = 0; i < ehdr->e_phnum; i++, phdr++) {
1427			if (phdr->p_type == PT_NOTE) {
1428				/* Update note size */
1429				phdr->p_memsz = elfnotes_orig_sz + vmcoreddsz;
1430				phdr->p_filesz = phdr->p_memsz;
1431				continue;
1432			}
1433
1434			start = rounddown(phdr->p_offset, PAGE_SIZE);
1435			end = roundup(phdr->p_offset + phdr->p_memsz,
1436				      PAGE_SIZE);
1437			size = end - start;
1438			phdr->p_offset = vmcore_off + (phdr->p_offset - start);
1439			vmcore_off += size;
1440		}
1441	}
1442}
1443
1444/**
1445 * vmcoredd_update_size - Update the total size of the device dumps and update
1446 * ELF header
1447 * @dump_size: Size of the current device dump to be added to total size
1448 *
1449 * Update the total size of all the device dumps and update the ELF program
1450 * headers. Calculate the new offsets for the vmcore list and update the
1451 * total vmcore size.
1452 */
1453static void vmcoredd_update_size(size_t dump_size)
1454{
1455	vmcoredd_orig_sz += dump_size;
1456	elfnotes_sz = roundup(elfnotes_orig_sz, PAGE_SIZE) + vmcoredd_orig_sz;
1457	vmcoredd_update_program_headers(elfcorebuf, elfnotes_sz,
1458					vmcoredd_orig_sz);
1459
1460	/* Update vmcore list offsets */
1461	set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list);
1462
1463	vmcore_size = get_vmcore_size(elfcorebuf_sz, elfnotes_sz,
1464				      &vmcore_list);
1465	proc_vmcore->size = vmcore_size;
1466}
1467
1468/**
1469 * vmcore_add_device_dump - Add a buffer containing device dump to vmcore
1470 * @data: dump info.
1471 *
1472 * Allocate a buffer and invoke the calling driver's dump collect routine.
1473 * Write ELF note at the beginning of the buffer to indicate vmcore device
1474 * dump and add the dump to global list.
1475 */
1476int vmcore_add_device_dump(struct vmcoredd_data *data)
1477{
1478	struct vmcoredd_node *dump;
1479	void *buf = NULL;
1480	size_t data_size;
1481	int ret;
1482
1483	if (vmcoredd_disabled) {
1484		pr_err_once("Device dump is disabled\n");
1485		return -EINVAL;
1486	}
1487
1488	if (!data || !strlen(data->dump_name) ||
1489	    !data->vmcoredd_callback || !data->size)
1490		return -EINVAL;
1491
1492	dump = vzalloc(sizeof(*dump));
1493	if (!dump)
1494		return -ENOMEM;
1495
1496	/* Keep size of the buffer page aligned so that it can be mmaped */
1497	data_size = roundup(sizeof(struct vmcoredd_header) + data->size,
1498			    PAGE_SIZE);
1499
1500	/* Allocate buffer for driver's to write their dumps */
1501	buf = vmcore_alloc_buf(data_size);
1502	if (!buf) {
1503		ret = -ENOMEM;
1504		goto out_err;
1505	}
1506
1507	vmcoredd_write_header(buf, data, data_size -
1508			      sizeof(struct vmcoredd_header));
1509
1510	/* Invoke the driver's dump collection routing */
1511	ret = data->vmcoredd_callback(data, buf +
1512				      sizeof(struct vmcoredd_header));
1513	if (ret)
1514		goto out_err;
1515
1516	dump->buf = buf;
1517	dump->size = data_size;
1518
1519	/* Add the dump to driver sysfs list and update the elfcore hdr */
1520	scoped_guard(mutex, &vmcore_mutex) {
1521		if (vmcore_opened)
1522			pr_warn_once("Unexpected adding of device dump\n");
1523		if (vmcore_open) {
1524			ret = -EBUSY;
1525			goto out_err;
1526		}
1527
1528		list_add_tail(&dump->list, &vmcoredd_list);
1529		vmcoredd_update_size(data_size);
1530	}
1531	return 0;
1532
1533out_err:
1534	vfree(buf);
1535	vfree(dump);
1536
1537	return ret;
1538}
1539EXPORT_SYMBOL(vmcore_add_device_dump);
1540#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
1541
1542#ifdef CONFIG_PROC_VMCORE_DEVICE_RAM
1543static int vmcore_realloc_elfcore_buffer_elf64(size_t new_size)
1544{
1545	char *elfcorebuf_new;
1546
1547	if (WARN_ON_ONCE(new_size < elfcorebuf_sz))
1548		return -EINVAL;
1549	if (get_order(elfcorebuf_sz_orig) == get_order(new_size)) {
1550		elfcorebuf_sz_orig = new_size;
1551		return 0;
1552	}
1553
1554	elfcorebuf_new = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1555						  get_order(new_size));
1556	if (!elfcorebuf_new)
1557		return -ENOMEM;
1558	memcpy(elfcorebuf_new, elfcorebuf, elfcorebuf_sz);
1559	free_pages((unsigned long)elfcorebuf, get_order(elfcorebuf_sz_orig));
1560	elfcorebuf = elfcorebuf_new;
1561	elfcorebuf_sz_orig = new_size;
1562	return 0;
1563}
1564
1565static void vmcore_reset_offsets_elf64(void)
1566{
1567	Elf64_Phdr *phdr_start = (Elf64_Phdr *)(elfcorebuf + sizeof(Elf64_Ehdr));
1568	loff_t vmcore_off = elfcorebuf_sz + elfnotes_sz;
1569	Elf64_Ehdr *ehdr = (Elf64_Ehdr *)elfcorebuf;
1570	Elf64_Phdr *phdr;
1571	int i;
1572
1573	for (i = 0, phdr = phdr_start; i < ehdr->e_phnum; i++, phdr++) {
1574		u64 start, end;
1575
1576		/*
1577		 * After merge_note_headers_elf64() we should only have a single
1578		 * PT_NOTE entry that starts immediately after elfcorebuf_sz.
1579		 */
1580		if (phdr->p_type == PT_NOTE) {
1581			phdr->p_offset = elfcorebuf_sz;
1582			continue;
1583		}
1584
1585		start = rounddown(phdr->p_offset, PAGE_SIZE);
1586		end = roundup(phdr->p_offset + phdr->p_memsz, PAGE_SIZE);
1587		phdr->p_offset = vmcore_off + (phdr->p_offset - start);
1588		vmcore_off = vmcore_off + end - start;
1589	}
1590	set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list);
1591}
1592
1593static int vmcore_add_device_ram_elf64(struct list_head *list, size_t count)
1594{
1595	Elf64_Phdr *phdr_start = (Elf64_Phdr *)(elfcorebuf + sizeof(Elf64_Ehdr));
1596	Elf64_Ehdr *ehdr = (Elf64_Ehdr *)elfcorebuf;
1597	struct vmcore_range *cur;
1598	Elf64_Phdr *phdr;
1599	size_t new_size;
1600	int rc;
1601
1602	if ((Elf32_Half)(ehdr->e_phnum + count) != ehdr->e_phnum + count) {
1603		pr_err("too many device ram ranges\n");
1604		return -ENOSPC;
1605	}
1606
1607	/* elfcorebuf_sz must always cover full pages. */
1608	new_size = sizeof(Elf64_Ehdr) +
1609		   (ehdr->e_phnum + count) * sizeof(Elf64_Phdr);
1610	new_size = roundup(new_size, PAGE_SIZE);
1611
1612	/*
1613	 * Make sure we have sufficient space to include the new PT_LOAD
1614	 * entries.
1615	 */
1616	rc = vmcore_realloc_elfcore_buffer_elf64(new_size);
1617	if (rc) {
1618		pr_err("resizing elfcore failed\n");
1619		return rc;
1620	}
1621
1622	/* Modify our used elfcore buffer size to cover the new entries. */
1623	elfcorebuf_sz = new_size;
1624
1625	/* Fill the added PT_LOAD entries. */
1626	phdr = phdr_start + ehdr->e_phnum;
1627	list_for_each_entry(cur, list, list) {
1628		WARN_ON_ONCE(!IS_ALIGNED(cur->paddr | cur->size, PAGE_SIZE));
1629		elfcorehdr_fill_device_ram_ptload_elf64(phdr, cur->paddr, cur->size);
1630
1631		/* p_offset will be adjusted later. */
1632		phdr++;
1633		ehdr->e_phnum++;
1634	}
1635	list_splice_tail(list, &vmcore_list);
1636
1637	/* We changed elfcorebuf_sz and added new entries; reset all offsets. */
1638	vmcore_reset_offsets_elf64();
1639
1640	/* Finally, recalculate the total vmcore size. */
1641	vmcore_size = get_vmcore_size(elfcorebuf_sz, elfnotes_sz,
1642				      &vmcore_list);
1643	proc_vmcore->size = vmcore_size;
1644	return 0;
1645}
1646
1647static void vmcore_process_device_ram(struct vmcore_cb *cb)
1648{
1649	unsigned char *e_ident = (unsigned char *)elfcorebuf;
1650	struct vmcore_range *first, *m;
1651	LIST_HEAD(list);
1652	int count;
1653
1654	/* We only support Elf64 dumps for now. */
1655	if (WARN_ON_ONCE(e_ident[EI_CLASS] != ELFCLASS64)) {
1656		pr_err("device ram ranges only support Elf64\n");
1657		return;
1658	}
1659
1660	if (cb->get_device_ram(cb, &list)) {
1661		pr_err("obtaining device ram ranges failed\n");
1662		return;
1663	}
1664	count = list_count_nodes(&list);
1665	if (!count)
1666		return;
1667
1668	/*
1669	 * For some reason these ranges are already know? Might happen
1670	 * with unusual register->unregister->register sequences; we'll simply
1671	 * sanity check using the first range.
1672	 */
1673	first = list_first_entry(&list, struct vmcore_range, list);
1674	list_for_each_entry(m, &vmcore_list, list) {
1675		unsigned long long m_end = m->paddr + m->size;
1676		unsigned long long first_end = first->paddr + first->size;
1677
1678		if (first->paddr < m_end && m->paddr < first_end)
1679			goto out_free;
1680	}
1681
1682	/* If adding the mem nodes succeeds, they must not be freed. */
1683	if (!vmcore_add_device_ram_elf64(&list, count))
1684		return;
1685out_free:
1686	vmcore_free_ranges(&list);
1687}
1688#else /* !CONFIG_PROC_VMCORE_DEVICE_RAM */
1689static void vmcore_process_device_ram(struct vmcore_cb *cb)
1690{
1691}
1692#endif /* CONFIG_PROC_VMCORE_DEVICE_RAM */
1693
1694/* Free all dumps in vmcore device dump list */
1695static void vmcore_free_device_dumps(void)
1696{
1697#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP
1698	mutex_lock(&vmcore_mutex);
1699	while (!list_empty(&vmcoredd_list)) {
1700		struct vmcoredd_node *dump;
1701
1702		dump = list_first_entry(&vmcoredd_list, struct vmcoredd_node,
1703					list);
1704		list_del(&dump->list);
1705		vfree(dump->buf);
1706		vfree(dump);
1707	}
1708	mutex_unlock(&vmcore_mutex);
1709#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
1710}
1711
1712/* Init function for vmcore module. */
1713static int __init vmcore_init(void)
1714{
1715	int rc = 0;
1716
1717	/* Allow architectures to allocate ELF header in 2nd kernel */
1718	rc = elfcorehdr_alloc(&elfcorehdr_addr, &elfcorehdr_size);
1719	if (rc)
1720		return rc;
1721	/*
1722	 * If elfcorehdr= has been passed in cmdline or created in 2nd kernel,
1723	 * then capture the dump.
1724	 */
1725	if (!(is_vmcore_usable()))
1726		return rc;
1727	rc = parse_crash_elf_headers();
1728	if (rc) {
1729		elfcorehdr_free(elfcorehdr_addr);
1730		pr_warn("not initialized\n");
1731		return rc;
1732	}
1733	elfcorehdr_free(elfcorehdr_addr);
1734	elfcorehdr_addr = ELFCORE_ADDR_ERR;
1735
1736	proc_vmcore = proc_create("vmcore", S_IRUSR, NULL, &vmcore_proc_ops);
1737	if (proc_vmcore)
1738		proc_vmcore->size = vmcore_size;
1739	return 0;
1740}
1741fs_initcall(vmcore_init);
1742
1743/* Cleanup function for vmcore module. */
1744void vmcore_cleanup(void)
1745{
1746	if (proc_vmcore) {
1747		proc_remove(proc_vmcore);
1748		proc_vmcore = NULL;
1749	}
1750
1751	vmcore_free_ranges(&vmcore_list);
1752	free_elfcorebuf();
1753
1754	/* clear vmcore device dump list */
1755	vmcore_free_device_dumps();
1756}