mmap.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * mm/mmap.c
   4 *
   5 * Written by obz.
   6 *
   7 * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
   8 */
   9
  10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  11
  12#include <linux/kernel.h>
  13#include <linux/slab.h>
  14#include <linux/backing-dev.h>
  15#include <linux/mm.h>
  16#include <linux/mm_inline.h>
  17#include <linux/shm.h>
  18#include <linux/mman.h>
  19#include <linux/pagemap.h>
  20#include <linux/swap.h>
  21#include <linux/syscalls.h>
  22#include <linux/capability.h>
  23#include <linux/init.h>
  24#include <linux/file.h>
  25#include <linux/fs.h>
  26#include <linux/personality.h>
  27#include <linux/security.h>
  28#include <linux/hugetlb.h>
  29#include <linux/shmem_fs.h>
  30#include <linux/profile.h>
  31#include <linux/export.h>
  32#include <linux/mount.h>
  33#include <linux/mempolicy.h>
  34#include <linux/rmap.h>
  35#include <linux/mmu_notifier.h>
  36#include <linux/mmdebug.h>
  37#include <linux/perf_event.h>
  38#include <linux/audit.h>
  39#include <linux/khugepaged.h>
  40#include <linux/uprobes.h>
  41#include <linux/notifier.h>
  42#include <linux/memory.h>
  43#include <linux/printk.h>
  44#include <linux/userfaultfd_k.h>
  45#include <linux/moduleparam.h>
  46#include <linux/pkeys.h>
  47#include <linux/oom.h>
  48#include <linux/sched/mm.h>
  49#include <linux/ksm.h>
  50#include <linux/memfd.h>
  51
  52#include <linux/uaccess.h>
  53#include <asm/cacheflush.h>
  54#include <asm/tlb.h>
  55#include <asm/mmu_context.h>
  56
  57#define CREATE_TRACE_POINTS
  58#include <trace/events/mmap.h>
  59
  60#include "internal.h"
  61
  62#ifndef arch_mmap_check
  63#define arch_mmap_check(addr, len, flags)	(0)
  64#endif
  65
  66#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
  67const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
  68int mmap_rnd_bits_max __ro_after_init = CONFIG_ARCH_MMAP_RND_BITS_MAX;
  69int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
  70#endif
  71#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
  72const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
  73const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
  74int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
  75#endif
  76
  77static bool ignore_rlimit_data;
  78core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
  79
  80/* Update vma->vm_page_prot to reflect vma->vm_flags. */
  81void vma_set_page_prot(struct vm_area_struct *vma)
  82{
  83	vm_flags_t vm_flags = vma->vm_flags;
  84	pgprot_t vm_page_prot;
  85
  86	vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
  87	if (vma_wants_writenotify(vma, vm_page_prot)) {
  88		vm_flags &= ~VM_SHARED;
  89		vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
  90	}
  91	/* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */
  92	WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
  93}
  94
  95/*
  96 * check_brk_limits() - Use platform specific check of range & verify mlock
  97 * limits.
  98 * @addr: The address to check
  99 * @len: The size of increase.
 100 *
 101 * Return: 0 on success.
 102 */
 103static int check_brk_limits(unsigned long addr, unsigned long len)
 104{
 105	unsigned long mapped_addr;
 106
 107	mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
 108	if (IS_ERR_VALUE(mapped_addr))
 109		return mapped_addr;
 110
 111	return mlock_future_ok(current->mm, current->mm->def_flags, len)
 112		? 0 : -EAGAIN;
 113}
 114
 115SYSCALL_DEFINE1(brk, unsigned long, brk)
 116{
 117	unsigned long newbrk, oldbrk, origbrk;
 118	struct mm_struct *mm = current->mm;
 119	struct vm_area_struct *brkvma, *next = NULL;
 120	unsigned long min_brk;
 121	bool populate = false;
 122	LIST_HEAD(uf);
 123	struct vma_iterator vmi;
 124
 125	if (mmap_write_lock_killable(mm))
 126		return -EINTR;
 127
 128	origbrk = mm->brk;
 129
 130	min_brk = mm->start_brk;
 131#ifdef CONFIG_COMPAT_BRK
 132	/*
 133	 * CONFIG_COMPAT_BRK can still be overridden by setting
 134	 * randomize_va_space to 2, which will still cause mm->start_brk
 135	 * to be arbitrarily shifted
 136	 */
 137	if (!current->brk_randomized)
 138		min_brk = mm->end_data;
 139#endif
 140	if (brk < min_brk)
 141		goto out;
 142
 143	/*
 144	 * Check against rlimit here. If this check is done later after the test
 145	 * of oldbrk with newbrk then it can escape the test and let the data
 146	 * segment grow beyond its set limit the in case where the limit is
 147	 * not page aligned -Ram Gupta
 148	 */
 149	if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
 150			      mm->end_data, mm->start_data))
 151		goto out;
 152
 153	newbrk = PAGE_ALIGN(brk);
 154	oldbrk = PAGE_ALIGN(mm->brk);
 155	if (oldbrk == newbrk) {
 156		mm->brk = brk;
 157		goto success;
 158	}
 159
 160	/* Always allow shrinking brk. */
 161	if (brk <= mm->brk) {
 162		/* Search one past newbrk */
 163		vma_iter_init(&vmi, mm, newbrk);
 164		brkvma = vma_find(&vmi, oldbrk);
 165		if (!brkvma || brkvma->vm_start >= oldbrk)
 166			goto out; /* mapping intersects with an existing non-brk vma. */
 167		/*
 168		 * mm->brk must be protected by write mmap_lock.
 169		 * do_vmi_align_munmap() will drop the lock on success,  so
 170		 * update it before calling do_vma_munmap().
 171		 */
 172		mm->brk = brk;
 173		if (do_vmi_align_munmap(&vmi, brkvma, mm, newbrk, oldbrk, &uf,
 174					/* unlock = */ true))
 175			goto out;
 176
 177		goto success_unlocked;
 178	}
 179
 180	if (check_brk_limits(oldbrk, newbrk - oldbrk))
 181		goto out;
 182
 183	/*
 184	 * Only check if the next VMA is within the stack_guard_gap of the
 185	 * expansion area
 186	 */
 187	vma_iter_init(&vmi, mm, oldbrk);
 188	next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap);
 189	if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
 190		goto out;
 191
 192	brkvma = vma_prev_limit(&vmi, mm->start_brk);
 193	/* Ok, looks good - let it rip. */
 194	if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
 195		goto out;
 196
 197	mm->brk = brk;
 198	if (mm->def_flags & VM_LOCKED)
 199		populate = true;
 200
 201success:
 202	mmap_write_unlock(mm);
 203success_unlocked:
 204	userfaultfd_unmap_complete(mm, &uf);
 205	if (populate)
 206		mm_populate(oldbrk, newbrk - oldbrk);
 207	return brk;
 208
 209out:
 210	mm->brk = origbrk;
 211	mmap_write_unlock(mm);
 212	return origbrk;
 213}
 214
 215/*
 216 * If a hint addr is less than mmap_min_addr change hint to be as
 217 * low as possible but still greater than mmap_min_addr
 218 */
 219static inline unsigned long round_hint_to_min(unsigned long hint)
 220{
 221	hint &= PAGE_MASK;
 222	if (((void *)hint != NULL) &&
 223	    (hint < mmap_min_addr))
 224		return PAGE_ALIGN(mmap_min_addr);
 225	return hint;
 226}
 227
 228bool mlock_future_ok(const struct mm_struct *mm, vm_flags_t vm_flags,
 229			unsigned long bytes)
 230{
 231	unsigned long locked_pages, limit_pages;
 232
 233	if (!(vm_flags & VM_LOCKED) || capable(CAP_IPC_LOCK))
 234		return true;
 235
 236	locked_pages = bytes >> PAGE_SHIFT;
 237	locked_pages += mm->locked_vm;
 238
 239	limit_pages = rlimit(RLIMIT_MEMLOCK);
 240	limit_pages >>= PAGE_SHIFT;
 241
 242	return locked_pages <= limit_pages;
 243}
 244
 245static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
 246{
 247	if (S_ISREG(inode->i_mode))
 248		return MAX_LFS_FILESIZE;
 249
 250	if (S_ISBLK(inode->i_mode))
 251		return MAX_LFS_FILESIZE;
 252
 253	if (S_ISSOCK(inode->i_mode))
 254		return MAX_LFS_FILESIZE;
 255
 256	/* Special "we do even unsigned file positions" case */
 257	if (file->f_op->fop_flags & FOP_UNSIGNED_OFFSET)
 258		return 0;
 259
 260	/* Yes, random drivers might want more. But I'm tired of buggy drivers */
 261	return ULONG_MAX;
 262}
 263
 264static inline bool file_mmap_ok(struct file *file, struct inode *inode,
 265				unsigned long pgoff, unsigned long len)
 266{
 267	u64 maxsize = file_mmap_size_max(file, inode);
 268
 269	if (maxsize && len > maxsize)
 270		return false;
 271	maxsize -= len;
 272	if (pgoff > maxsize >> PAGE_SHIFT)
 273		return false;
 274	return true;
 275}
 276
 277/**
 278 * do_mmap() - Perform a userland memory mapping into the current process
 279 * address space of length @len with protection bits @prot, mmap flags @flags
 280 * (from which VMA flags will be inferred), and any additional VMA flags to
 281 * apply @vm_flags. If this is a file-backed mapping then the file is specified
 282 * in @file and page offset into the file via @pgoff.
 283 *
 284 * This function does not perform security checks on the file and assumes, if
 285 * @uf is non-NULL, the caller has provided a list head to track unmap events
 286 * for userfaultfd @uf.
 287 *
 288 * It also simply indicates whether memory population is required by setting
 289 * @populate, which must be non-NULL, expecting the caller to actually perform
 290 * this task itself if appropriate.
 291 *
 292 * This function will invoke architecture-specific (and if provided and
 293 * relevant, file system-specific) logic to determine the most appropriate
 294 * unmapped area in which to place the mapping if not MAP_FIXED.
 295 *
 296 * Callers which require userland mmap() behaviour should invoke vm_mmap(),
 297 * which is also exported for module use.
 298 *
 299 * Those which require this behaviour less security checks, userfaultfd and
 300 * populate behaviour, and who handle the mmap write lock themselves, should
 301 * call this function.
 302 *
 303 * Note that the returned address may reside within a merged VMA if an
 304 * appropriate merge were to take place, so it doesn't necessarily specify the
 305 * start of a VMA, rather only the start of a valid mapped range of length
 306 * @len bytes, rounded down to the nearest page size.
 307 *
 308 * The caller must write-lock current->mm->mmap_lock.
 309 *
 310 * @file: An optional struct file pointer describing the file which is to be
 311 * mapped, if a file-backed mapping.
 312 * @addr: If non-zero, hints at (or if @flags has MAP_FIXED set, specifies) the
 313 * address at which to perform this mapping. See mmap (2) for details. Must be
 314 * page-aligned.
 315 * @len: The length of the mapping. Will be page-aligned and must be at least 1
 316 * page in size.
 317 * @prot: Protection bits describing access required to the mapping. See mmap
 318 * (2) for details.
 319 * @flags: Flags specifying how the mapping should be performed, see mmap (2)
 320 * for details.
 321 * @vm_flags: VMA flags which should be set by default, or 0 otherwise.
 322 * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise.
 323 * @populate: A pointer to a value which will be set to 0 if no population of
 324 * the range is required, or the number of bytes to populate if it is. Must be
 325 * non-NULL. See mmap (2) for details as to under what circumstances population
 326 * of the range occurs.
 327 * @uf: An optional pointer to a list head to track userfaultfd unmap events
 328 * should unmapping events arise. If provided, it is up to the caller to manage
 329 * this.
 330 *
 331 * Returns: Either an error, or the address at which the requested mapping has
 332 * been performed.
 333 */
 334unsigned long do_mmap(struct file *file, unsigned long addr,
 335			unsigned long len, unsigned long prot,
 336			unsigned long flags, vm_flags_t vm_flags,
 337			unsigned long pgoff, unsigned long *populate,
 338			struct list_head *uf)
 339{
 340	struct mm_struct *mm = current->mm;
 341	int pkey = 0;
 342
 343	*populate = 0;
 344
 345	mmap_assert_write_locked(mm);
 346
 347	if (!len)
 348		return -EINVAL;
 349
 350	/*
 351	 * Does the application expect PROT_READ to imply PROT_EXEC?
 352	 *
 353	 * (the exception is when the underlying filesystem is noexec
 354	 *  mounted, in which case we don't add PROT_EXEC.)
 355	 */
 356	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
 357		if (!(file && path_noexec(&file->f_path)))
 358			prot |= PROT_EXEC;
 359
 360	/* force arch specific MAP_FIXED handling in get_unmapped_area */
 361	if (flags & MAP_FIXED_NOREPLACE)
 362		flags |= MAP_FIXED;
 363
 364	if (!(flags & MAP_FIXED))
 365		addr = round_hint_to_min(addr);
 366
 367	/* Careful about overflows.. */
 368	len = PAGE_ALIGN(len);
 369	if (!len)
 370		return -ENOMEM;
 371
 372	/* offset overflow? */
 373	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
 374		return -EOVERFLOW;
 375
 376	/* Too many mappings? */
 377	if (mm->map_count > sysctl_max_map_count)
 378		return -ENOMEM;
 379
 380	/*
 381	 * addr is returned from get_unmapped_area,
 382	 * There are two cases:
 383	 * 1> MAP_FIXED == false
 384	 *	unallocated memory, no need to check sealing.
 385	 * 1> MAP_FIXED == true
 386	 *	sealing is checked inside mmap_region when
 387	 *	do_vmi_munmap is called.
 388	 */
 389
 390	if (prot == PROT_EXEC) {
 391		pkey = execute_only_pkey(mm);
 392		if (pkey < 0)
 393			pkey = 0;
 394	}
 395
 396	/* Do simple checking here so the lower-level routines won't have
 397	 * to. we assume access permissions have been handled by the open
 398	 * of the memory object, so we don't do any here.
 399	 */
 400	vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) |
 401			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
 402
 403	/* Obtain the address to map to. we verify (or select) it and ensure
 404	 * that it represents a valid section of the address space.
 405	 */
 406	addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags);
 407	if (IS_ERR_VALUE(addr))
 408		return addr;
 409
 410	if (flags & MAP_FIXED_NOREPLACE) {
 411		if (find_vma_intersection(mm, addr, addr + len))
 412			return -EEXIST;
 413	}
 414
 415	if (flags & MAP_LOCKED)
 416		if (!can_do_mlock())
 417			return -EPERM;
 418
 419	if (!mlock_future_ok(mm, vm_flags, len))
 420		return -EAGAIN;
 421
 422	if (file) {
 423		struct inode *inode = file_inode(file);
 424		unsigned long flags_mask;
 425		int err;
 426
 427		if (!file_mmap_ok(file, inode, pgoff, len))
 428			return -EOVERFLOW;
 429
 430		flags_mask = LEGACY_MAP_MASK;
 431		if (file->f_op->fop_flags & FOP_MMAP_SYNC)
 432			flags_mask |= MAP_SYNC;
 433
 434		switch (flags & MAP_TYPE) {
 435		case MAP_SHARED:
 436			/*
 437			 * Force use of MAP_SHARED_VALIDATE with non-legacy
 438			 * flags. E.g. MAP_SYNC is dangerous to use with
 439			 * MAP_SHARED as you don't know which consistency model
 440			 * you will get. We silently ignore unsupported flags
 441			 * with MAP_SHARED to preserve backward compatibility.
 442			 */
 443			flags &= LEGACY_MAP_MASK;
 444			fallthrough;
 445		case MAP_SHARED_VALIDATE:
 446			if (flags & ~flags_mask)
 447				return -EOPNOTSUPP;
 448			if (prot & PROT_WRITE) {
 449				if (!(file->f_mode & FMODE_WRITE))
 450					return -EACCES;
 451				if (IS_SWAPFILE(file->f_mapping->host))
 452					return -ETXTBSY;
 453			}
 454
 455			/*
 456			 * Make sure we don't allow writing to an append-only
 457			 * file..
 458			 */
 459			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
 460				return -EACCES;
 461
 462			vm_flags |= VM_SHARED | VM_MAYSHARE;
 463			if (!(file->f_mode & FMODE_WRITE))
 464				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
 465			fallthrough;
 466		case MAP_PRIVATE:
 467			if (!(file->f_mode & FMODE_READ))
 468				return -EACCES;
 469			if (path_noexec(&file->f_path)) {
 470				if (vm_flags & VM_EXEC)
 471					return -EPERM;
 472				vm_flags &= ~VM_MAYEXEC;
 473			}
 474
 475			if (!can_mmap_file(file))
 476				return -ENODEV;
 477			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
 478				return -EINVAL;
 479			break;
 480
 481		default:
 482			return -EINVAL;
 483		}
 484
 485		/*
 486		 * Check to see if we are violating any seals and update VMA
 487		 * flags if necessary to avoid future seal violations.
 488		 */
 489		err = memfd_check_seals_mmap(file, &vm_flags);
 490		if (err)
 491			return (unsigned long)err;
 492	} else {
 493		switch (flags & MAP_TYPE) {
 494		case MAP_SHARED:
 495			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
 496				return -EINVAL;
 497			/*
 498			 * Ignore pgoff.
 499			 */
 500			pgoff = 0;
 501			vm_flags |= VM_SHARED | VM_MAYSHARE;
 502			break;
 503		case MAP_DROPPABLE:
 504			if (VM_DROPPABLE == VM_NONE)
 505				return -ENOTSUPP;
 506			/*
 507			 * A locked or stack area makes no sense to be droppable.
 508			 *
 509			 * Also, since droppable pages can just go away at any time
 510			 * it makes no sense to copy them on fork or dump them.
 511			 *
 512			 * And don't attempt to combine with hugetlb for now.
 513			 */
 514			if (flags & (MAP_LOCKED | MAP_HUGETLB))
 515			        return -EINVAL;
 516			if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP))
 517			        return -EINVAL;
 518
 519			vm_flags |= VM_DROPPABLE;
 520
 521			/*
 522			 * If the pages can be dropped, then it doesn't make
 523			 * sense to reserve them.
 524			 */
 525			vm_flags |= VM_NORESERVE;
 526
 527			/*
 528			 * Likewise, they're volatile enough that they
 529			 * shouldn't survive forks or coredumps.
 530			 */
 531			vm_flags |= VM_WIPEONFORK | VM_DONTDUMP;
 532			fallthrough;
 533		case MAP_PRIVATE:
 534			/*
 535			 * Set pgoff according to addr for anon_vma.
 536			 */
 537			pgoff = addr >> PAGE_SHIFT;
 538			break;
 539		default:
 540			return -EINVAL;
 541		}
 542	}
 543
 544	/*
 545	 * Set 'VM_NORESERVE' if we should not account for the
 546	 * memory use of this mapping.
 547	 */
 548	if (flags & MAP_NORESERVE) {
 549		/* We honor MAP_NORESERVE if allowed to overcommit */
 550		if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
 551			vm_flags |= VM_NORESERVE;
 552
 553		/* hugetlb applies strict overcommit unless MAP_NORESERVE */
 554		if (file && is_file_hugepages(file))
 555			vm_flags |= VM_NORESERVE;
 556	}
 557
 558	addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
 559	if (!IS_ERR_VALUE(addr) &&
 560	    ((vm_flags & VM_LOCKED) ||
 561	     (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
 562		*populate = len;
 563	return addr;
 564}
 565
 566unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
 567			      unsigned long prot, unsigned long flags,
 568			      unsigned long fd, unsigned long pgoff)
 569{
 570	struct file *file = NULL;
 571	unsigned long retval;
 572
 573	if (!(flags & MAP_ANONYMOUS)) {
 574		audit_mmap_fd(fd, flags);
 575		file = fget(fd);
 576		if (!file)
 577			return -EBADF;
 578		if (is_file_hugepages(file)) {
 579			len = ALIGN(len, huge_page_size(hstate_file(file)));
 580		} else if (unlikely(flags & MAP_HUGETLB)) {
 581			retval = -EINVAL;
 582			goto out_fput;
 583		}
 584	} else if (flags & MAP_HUGETLB) {
 585		struct hstate *hs;
 586
 587		hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
 588		if (!hs)
 589			return -EINVAL;
 590
 591		len = ALIGN(len, huge_page_size(hs));
 592		/*
 593		 * VM_NORESERVE is used because the reservations will be
 594		 * taken when vm_ops->mmap() is called
 595		 */
 596		file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
 597				VM_NORESERVE,
 598				HUGETLB_ANONHUGE_INODE,
 599				(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
 600		if (IS_ERR(file))
 601			return PTR_ERR(file);
 602	}
 603
 604	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
 605out_fput:
 606	if (file)
 607		fput(file);
 608	return retval;
 609}
 610
 611SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
 612		unsigned long, prot, unsigned long, flags,
 613		unsigned long, fd, unsigned long, pgoff)
 614{
 615	return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
 616}
 617
 618#ifdef __ARCH_WANT_SYS_OLD_MMAP
 619struct mmap_arg_struct {
 620	unsigned long addr;
 621	unsigned long len;
 622	unsigned long prot;
 623	unsigned long flags;
 624	unsigned long fd;
 625	unsigned long offset;
 626};
 627
 628SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
 629{
 630	struct mmap_arg_struct a;
 631
 632	if (copy_from_user(&a, arg, sizeof(a)))
 633		return -EFAULT;
 634	if (offset_in_page(a.offset))
 635		return -EINVAL;
 636
 637	return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
 638			       a.offset >> PAGE_SHIFT);
 639}
 640#endif /* __ARCH_WANT_SYS_OLD_MMAP */
 641
 642/*
 643 * Determine if the allocation needs to ensure that there is no
 644 * existing mapping within it's guard gaps, for use as start_gap.
 645 */
 646static inline unsigned long stack_guard_placement(vm_flags_t vm_flags)
 647{
 648	if (vm_flags & VM_SHADOW_STACK)
 649		return PAGE_SIZE;
 650
 651	return 0;
 652}
 653
 654/*
 655 * Search for an unmapped address range.
 656 *
 657 * We are looking for a range that:
 658 * - does not intersect with any VMA;
 659 * - is contained within the [low_limit, high_limit) interval;
 660 * - is at least the desired size.
 661 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
 662 */
 663unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
 664{
 665	unsigned long addr;
 666
 667	if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
 668		addr = unmapped_area_topdown(info);
 669	else
 670		addr = unmapped_area(info);
 671
 672	trace_vm_unmapped_area(addr, info);
 673	return addr;
 674}
 675
 676/* Get an address range which is currently unmapped.
 677 * For shmat() with addr=0.
 678 *
 679 * Ugly calling convention alert:
 680 * Return value with the low bits set means error value,
 681 * ie
 682 *	if (ret & ~PAGE_MASK)
 683 *		error = ret;
 684 *
 685 * This function "knows" that -ENOMEM has the bits set.
 686 */
 687unsigned long
 688generic_get_unmapped_area(struct file *filp, unsigned long addr,
 689			  unsigned long len, unsigned long pgoff,
 690			  unsigned long flags, vm_flags_t vm_flags)
 691{
 692	struct mm_struct *mm = current->mm;
 693	struct vm_area_struct *vma, *prev;
 694	struct vm_unmapped_area_info info = {};
 695	const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
 696
 697	if (len > mmap_end - mmap_min_addr)
 698		return -ENOMEM;
 699
 700	if (flags & MAP_FIXED)
 701		return addr;
 702
 703	if (addr) {
 704		addr = PAGE_ALIGN(addr);
 705		vma = find_vma_prev(mm, addr, &prev);
 706		if (mmap_end - len >= addr && addr >= mmap_min_addr &&
 707		    (!vma || addr + len <= vm_start_gap(vma)) &&
 708		    (!prev || addr >= vm_end_gap(prev)))
 709			return addr;
 710	}
 711
 712	info.length = len;
 713	info.low_limit = mm->mmap_base;
 714	info.high_limit = mmap_end;
 715	info.start_gap = stack_guard_placement(vm_flags);
 716	if (filp && is_file_hugepages(filp))
 717		info.align_mask = huge_page_mask_align(filp);
 718	return vm_unmapped_area(&info);
 719}
 720
 721#ifndef HAVE_ARCH_UNMAPPED_AREA
 722unsigned long
 723arch_get_unmapped_area(struct file *filp, unsigned long addr,
 724		       unsigned long len, unsigned long pgoff,
 725		       unsigned long flags, vm_flags_t vm_flags)
 726{
 727	return generic_get_unmapped_area(filp, addr, len, pgoff, flags,
 728					 vm_flags);
 729}
 730#endif
 731
 732/*
 733 * This mmap-allocator allocates new areas top-down from below the
 734 * stack's low limit (the base):
 735 */
 736unsigned long
 737generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 738				  unsigned long len, unsigned long pgoff,
 739				  unsigned long flags, vm_flags_t vm_flags)
 740{
 741	struct vm_area_struct *vma, *prev;
 742	struct mm_struct *mm = current->mm;
 743	struct vm_unmapped_area_info info = {};
 744	const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
 745
 746	/* requested length too big for entire address space */
 747	if (len > mmap_end - mmap_min_addr)
 748		return -ENOMEM;
 749
 750	if (flags & MAP_FIXED)
 751		return addr;
 752
 753	/* requesting a specific address */
 754	if (addr) {
 755		addr = PAGE_ALIGN(addr);
 756		vma = find_vma_prev(mm, addr, &prev);
 757		if (mmap_end - len >= addr && addr >= mmap_min_addr &&
 758				(!vma || addr + len <= vm_start_gap(vma)) &&
 759				(!prev || addr >= vm_end_gap(prev)))
 760			return addr;
 761	}
 762
 763	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
 764	info.length = len;
 765	info.low_limit = PAGE_SIZE;
 766	info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
 767	info.start_gap = stack_guard_placement(vm_flags);
 768	if (filp && is_file_hugepages(filp))
 769		info.align_mask = huge_page_mask_align(filp);
 770	addr = vm_unmapped_area(&info);
 771
 772	/*
 773	 * A failed mmap() very likely causes application failure,
 774	 * so fall back to the bottom-up function here. This scenario
 775	 * can happen with large stack limits and large mmap()
 776	 * allocations.
 777	 */
 778	if (offset_in_page(addr)) {
 779		VM_BUG_ON(addr != -ENOMEM);
 780		info.flags = 0;
 781		info.low_limit = TASK_UNMAPPED_BASE;
 782		info.high_limit = mmap_end;
 783		addr = vm_unmapped_area(&info);
 784	}
 785
 786	return addr;
 787}
 788
 789#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
 790unsigned long
 791arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 792			       unsigned long len, unsigned long pgoff,
 793			       unsigned long flags, vm_flags_t vm_flags)
 794{
 795	return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags,
 796						 vm_flags);
 797}
 798#endif
 799
 800unsigned long mm_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
 801					   unsigned long len, unsigned long pgoff,
 802					   unsigned long flags, vm_flags_t vm_flags)
 803{
 804	if (mm_flags_test(MMF_TOPDOWN, current->mm))
 805		return arch_get_unmapped_area_topdown(filp, addr, len, pgoff,
 806						      flags, vm_flags);
 807	return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags);
 808}
 809
 810unsigned long
 811__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 812		unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags)
 813{
 814	unsigned long (*get_area)(struct file *, unsigned long,
 815				  unsigned long, unsigned long, unsigned long)
 816				  = NULL;
 817
 818	unsigned long error = arch_mmap_check(addr, len, flags);
 819	if (error)
 820		return error;
 821
 822	/* Careful about overflows.. */
 823	if (len > TASK_SIZE)
 824		return -ENOMEM;
 825
 826	if (file) {
 827		if (file->f_op->get_unmapped_area)
 828			get_area = file->f_op->get_unmapped_area;
 829	} else if (flags & MAP_SHARED) {
 830		/*
 831		 * mmap_region() will call shmem_zero_setup() to create a file,
 832		 * so use shmem's get_unmapped_area in case it can be huge.
 833		 */
 834		get_area = shmem_get_unmapped_area;
 835	}
 836
 837	/* Always treat pgoff as zero for anonymous memory. */
 838	if (!file)
 839		pgoff = 0;
 840
 841	if (get_area) {
 842		addr = get_area(file, addr, len, pgoff, flags);
 843	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && !file
 844		   && !addr /* no hint */
 845		   && IS_ALIGNED(len, PMD_SIZE)) {
 846		/* Ensures that larger anonymous mappings are THP aligned. */
 847		addr = thp_get_unmapped_area_vmflags(file, addr, len,
 848						     pgoff, flags, vm_flags);
 849	} else {
 850		addr = mm_get_unmapped_area_vmflags(file, addr, len,
 851						    pgoff, flags, vm_flags);
 852	}
 853	if (IS_ERR_VALUE(addr))
 854		return addr;
 855
 856	if (addr > TASK_SIZE - len)
 857		return -ENOMEM;
 858	if (offset_in_page(addr))
 859		return -EINVAL;
 860
 861	error = security_mmap_addr(addr);
 862	return error ? error : addr;
 863}
 864
 865unsigned long
 866mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 867		     unsigned long pgoff, unsigned long flags)
 868{
 869	return mm_get_unmapped_area_vmflags(file, addr, len, pgoff, flags, 0);
 870}
 871EXPORT_SYMBOL(mm_get_unmapped_area);
 872
 873/**
 874 * find_vma_intersection() - Look up the first VMA which intersects the interval
 875 * @mm: The process address space.
 876 * @start_addr: The inclusive start user address.
 877 * @end_addr: The exclusive end user address.
 878 *
 879 * Returns: The first VMA within the provided range, %NULL otherwise.  Assumes
 880 * start_addr < end_addr.
 881 */
 882struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
 883					     unsigned long start_addr,
 884					     unsigned long end_addr)
 885{
 886	unsigned long index = start_addr;
 887
 888	mmap_assert_locked(mm);
 889	return mt_find(&mm->mm_mt, &index, end_addr - 1);
 890}
 891EXPORT_SYMBOL(find_vma_intersection);
 892
 893/**
 894 * find_vma() - Find the VMA for a given address, or the next VMA.
 895 * @mm: The mm_struct to check
 896 * @addr: The address
 897 *
 898 * Returns: The VMA associated with addr, or the next VMA.
 899 * May return %NULL in the case of no VMA at addr or above.
 900 */
 901struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
 902{
 903	unsigned long index = addr;
 904
 905	mmap_assert_locked(mm);
 906	return mt_find(&mm->mm_mt, &index, ULONG_MAX);
 907}
 908EXPORT_SYMBOL(find_vma);
 909
 910/**
 911 * find_vma_prev() - Find the VMA for a given address, or the next vma and
 912 * set %pprev to the previous VMA, if any.
 913 * @mm: The mm_struct to check
 914 * @addr: The address
 915 * @pprev: The pointer to set to the previous VMA
 916 *
 917 * Note that RCU lock is missing here since the external mmap_lock() is used
 918 * instead.
 919 *
 920 * Returns: The VMA associated with @addr, or the next vma.
 921 * May return %NULL in the case of no vma at addr or above.
 922 */
 923struct vm_area_struct *
 924find_vma_prev(struct mm_struct *mm, unsigned long addr,
 925			struct vm_area_struct **pprev)
 926{
 927	struct vm_area_struct *vma;
 928	VMA_ITERATOR(vmi, mm, addr);
 929
 930	vma = vma_iter_load(&vmi);
 931	*pprev = vma_prev(&vmi);
 932	if (!vma)
 933		vma = vma_next(&vmi);
 934	return vma;
 935}
 936
 937/* enforced gap between the expanding stack and other mappings. */
 938unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
 939
 940static int __init cmdline_parse_stack_guard_gap(char *p)
 941{
 942	unsigned long val;
 943	char *endptr;
 944
 945	val = simple_strtoul(p, &endptr, 10);
 946	if (!*endptr)
 947		stack_guard_gap = val << PAGE_SHIFT;
 948
 949	return 1;
 950}
 951__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
 952
 953#ifdef CONFIG_STACK_GROWSUP
 954int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
 955{
 956	return expand_upwards(vma, address);
 957}
 958
 959struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
 960{
 961	struct vm_area_struct *vma, *prev;
 962
 963	addr &= PAGE_MASK;
 964	vma = find_vma_prev(mm, addr, &prev);
 965	if (vma && (vma->vm_start <= addr))
 966		return vma;
 967	if (!prev)
 968		return NULL;
 969	if (expand_stack_locked(prev, addr))
 970		return NULL;
 971	if (prev->vm_flags & VM_LOCKED)
 972		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
 973	return prev;
 974}
 975#else
 976int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
 977{
 978	return expand_downwards(vma, address);
 979}
 980
 981struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
 982{
 983	struct vm_area_struct *vma;
 984	unsigned long start;
 985
 986	addr &= PAGE_MASK;
 987	vma = find_vma(mm, addr);
 988	if (!vma)
 989		return NULL;
 990	if (vma->vm_start <= addr)
 991		return vma;
 992	start = vma->vm_start;
 993	if (expand_stack_locked(vma, addr))
 994		return NULL;
 995	if (vma->vm_flags & VM_LOCKED)
 996		populate_vma_page_range(vma, addr, start, NULL);
 997	return vma;
 998}
 999#endif
1000
1001#if defined(CONFIG_STACK_GROWSUP)
1002
1003#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
1004#define vma_expand_down(vma, addr) (-EFAULT)
1005
1006#else
1007
1008#define vma_expand_up(vma,addr) (-EFAULT)
1009#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
1010
1011#endif
1012
1013/*
1014 * expand_stack(): legacy interface for page faulting. Don't use unless
1015 * you have to.
1016 *
1017 * This is called with the mm locked for reading, drops the lock, takes
1018 * the lock for writing, tries to look up a vma again, expands it if
1019 * necessary, and downgrades the lock to reading again.
1020 *
1021 * If no vma is found or it can't be expanded, it returns NULL and has
1022 * dropped the lock.
1023 */
1024struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
1025{
1026	struct vm_area_struct *vma, *prev;
1027
1028	mmap_read_unlock(mm);
1029	if (mmap_write_lock_killable(mm))
1030		return NULL;
1031
1032	vma = find_vma_prev(mm, addr, &prev);
1033	if (vma && vma->vm_start <= addr)
1034		goto success;
1035
1036	if (prev && !vma_expand_up(prev, addr)) {
1037		vma = prev;
1038		goto success;
1039	}
1040
1041	if (vma && !vma_expand_down(vma, addr))
1042		goto success;
1043
1044	mmap_write_unlock(mm);
1045	return NULL;
1046
1047success:
1048	mmap_write_downgrade(mm);
1049	return vma;
1050}
1051
1052/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
1053 * @mm: The mm_struct
1054 * @start: The start address to munmap
1055 * @len: The length to be munmapped.
1056 * @uf: The userfaultfd list_head
1057 *
1058 * Return: 0 on success, error otherwise.
1059 */
1060int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
1061	      struct list_head *uf)
1062{
1063	VMA_ITERATOR(vmi, mm, start);
1064
1065	return do_vmi_munmap(&vmi, mm, start, len, uf, false);
1066}
1067
1068int vm_munmap(unsigned long start, size_t len)
1069{
1070	return __vm_munmap(start, len, false);
1071}
1072EXPORT_SYMBOL(vm_munmap);
1073
1074SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1075{
1076	addr = untagged_addr(addr);
1077	return __vm_munmap(addr, len, true);
1078}
1079
1080
1081/*
1082 * Emulation of deprecated remap_file_pages() syscall.
1083 */
1084SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
1085		unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
1086{
1087
1088	struct mm_struct *mm = current->mm;
1089	struct vm_area_struct *vma;
1090	unsigned long populate = 0;
1091	unsigned long ret = -EINVAL;
1092	struct file *file;
1093	vm_flags_t vm_flags;
1094
1095	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/mm/remap_file_pages.rst.\n",
1096		     current->comm, current->pid);
1097
1098	if (prot)
1099		return ret;
1100	start = start & PAGE_MASK;
1101	size = size & PAGE_MASK;
1102
1103	if (start + size <= start)
1104		return ret;
1105
1106	/* Does pgoff wrap? */
1107	if (pgoff + (size >> PAGE_SHIFT) < pgoff)
1108		return ret;
1109
1110	if (mmap_read_lock_killable(mm))
1111		return -EINTR;
1112
1113	/*
1114	 * Look up VMA under read lock first so we can perform the security
1115	 * without holding locks (which can be problematic). We reacquire a
1116	 * write lock later and check nothing changed underneath us.
1117	 */
1118	vma = vma_lookup(mm, start);
1119
1120	if (!vma || !(vma->vm_flags & VM_SHARED)) {
1121		mmap_read_unlock(mm);
1122		return -EINVAL;
1123	}
1124
1125	prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
1126	prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
1127	prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
1128
1129	flags &= MAP_NONBLOCK;
1130	flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
1131	if (vma->vm_flags & VM_LOCKED)
1132		flags |= MAP_LOCKED;
1133
1134	/* Save vm_flags used to calculate prot and flags, and recheck later. */
1135	vm_flags = vma->vm_flags;
1136	file = get_file(vma->vm_file);
1137
1138	mmap_read_unlock(mm);
1139
1140	/* Call outside mmap_lock to be consistent with other callers. */
1141	ret = security_mmap_file(file, prot, flags);
1142	if (ret) {
1143		fput(file);
1144		return ret;
1145	}
1146
1147	ret = -EINVAL;
1148
1149	/* OK security check passed, take write lock + let it rip. */
1150	if (mmap_write_lock_killable(mm)) {
1151		fput(file);
1152		return -EINTR;
1153	}
1154
1155	vma = vma_lookup(mm, start);
1156
1157	if (!vma)
1158		goto out;
1159
1160	/* Make sure things didn't change under us. */
1161	if (vma->vm_flags != vm_flags)
1162		goto out;
1163	if (vma->vm_file != file)
1164		goto out;
1165
1166	if (start + size > vma->vm_end) {
1167		VMA_ITERATOR(vmi, mm, vma->vm_end);
1168		struct vm_area_struct *next, *prev = vma;
1169
1170		for_each_vma_range(vmi, next, start + size) {
1171			/* hole between vmas ? */
1172			if (next->vm_start != prev->vm_end)
1173				goto out;
1174
1175			if (next->vm_file != vma->vm_file)
1176				goto out;
1177
1178			if (next->vm_flags != vma->vm_flags)
1179				goto out;
1180
1181			if (start + size <= next->vm_end)
1182				break;
1183
1184			prev = next;
1185		}
1186
1187		if (!next)
1188			goto out;
1189	}
1190
1191	ret = do_mmap(vma->vm_file, start, size,
1192			prot, flags, 0, pgoff, &populate, NULL);
1193out:
1194	mmap_write_unlock(mm);
1195	fput(file);
1196	if (populate)
1197		mm_populate(ret, populate);
1198	if (!IS_ERR_VALUE(ret))
1199		ret = 0;
1200	return ret;
1201}
1202
1203int vm_brk_flags(unsigned long addr, unsigned long request, vm_flags_t vm_flags)
1204{
1205	struct mm_struct *mm = current->mm;
1206	struct vm_area_struct *vma = NULL;
1207	unsigned long len;
1208	int ret;
1209	bool populate;
1210	LIST_HEAD(uf);
1211	VMA_ITERATOR(vmi, mm, addr);
1212
1213	len = PAGE_ALIGN(request);
1214	if (len < request)
1215		return -ENOMEM;
1216	if (!len)
1217		return 0;
1218
1219	/* Until we need other flags, refuse anything except VM_EXEC. */
1220	if ((vm_flags & (~VM_EXEC)) != 0)
1221		return -EINVAL;
1222
1223	if (mmap_write_lock_killable(mm))
1224		return -EINTR;
1225
1226	ret = check_brk_limits(addr, len);
1227	if (ret)
1228		goto limits_failed;
1229
1230	ret = do_vmi_munmap(&vmi, mm, addr, len, &uf, 0);
1231	if (ret)
1232		goto munmap_failed;
1233
1234	vma = vma_prev(&vmi);
1235	ret = do_brk_flags(&vmi, vma, addr, len, vm_flags);
1236	populate = ((mm->def_flags & VM_LOCKED) != 0);
1237	mmap_write_unlock(mm);
1238	userfaultfd_unmap_complete(mm, &uf);
1239	if (populate && !ret)
1240		mm_populate(addr, len);
1241	return ret;
1242
1243munmap_failed:
1244limits_failed:
1245	mmap_write_unlock(mm);
1246	return ret;
1247}
1248EXPORT_SYMBOL(vm_brk_flags);
1249
1250/* Release all mmaps. */
1251void exit_mmap(struct mm_struct *mm)
1252{
1253	struct mmu_gather tlb;
1254	struct vm_area_struct *vma;
1255	unsigned long nr_accounted = 0;
1256	VMA_ITERATOR(vmi, mm, 0);
1257	int count = 0;
1258
1259	/* mm's last user has gone, and its about to be pulled down */
1260	mmu_notifier_release(mm);
1261
1262	mmap_read_lock(mm);
1263	arch_exit_mmap(mm);
1264
1265	vma = vma_next(&vmi);
1266	if (!vma || unlikely(xa_is_zero(vma))) {
1267		/* Can happen if dup_mmap() received an OOM */
1268		mmap_read_unlock(mm);
1269		mmap_write_lock(mm);
1270		goto destroy;
1271	}
1272
1273	flush_cache_mm(mm);
1274	tlb_gather_mmu_fullmm(&tlb, mm);
1275	/* update_hiwater_rss(mm) here? but nobody should be looking */
1276	/* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */
1277	unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX);
1278	mmap_read_unlock(mm);
1279
1280	/*
1281	 * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper
1282	 * because the memory has been already freed.
1283	 */
1284	mm_flags_set(MMF_OOM_SKIP, mm);
1285	mmap_write_lock(mm);
1286	mt_clear_in_rcu(&mm->mm_mt);
1287	vma_iter_set(&vmi, vma->vm_end);
1288	free_pgtables(&tlb, &vmi.mas, vma, FIRST_USER_ADDRESS,
1289		      USER_PGTABLES_CEILING, true);
1290	tlb_finish_mmu(&tlb);
1291
1292	/*
1293	 * Walk the list again, actually closing and freeing it, with preemption
1294	 * enabled, without holding any MM locks besides the unreachable
1295	 * mmap_write_lock.
1296	 */
1297	vma_iter_set(&vmi, vma->vm_end);
1298	do {
1299		if (vma->vm_flags & VM_ACCOUNT)
1300			nr_accounted += vma_pages(vma);
1301		vma_mark_detached(vma);
1302		remove_vma(vma);
1303		count++;
1304		cond_resched();
1305		vma = vma_next(&vmi);
1306	} while (vma && likely(!xa_is_zero(vma)));
1307
1308	BUG_ON(count != mm->map_count);
1309
1310	trace_exit_mmap(mm);
1311destroy:
1312	__mt_destroy(&mm->mm_mt);
1313	mmap_write_unlock(mm);
1314	vm_unacct_memory(nr_accounted);
1315}
1316
1317/*
1318 * Return true if the calling process may expand its vm space by the passed
1319 * number of pages
1320 */
1321bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
1322{
1323	if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
1324		return false;
1325
1326	if (is_data_mapping(flags) &&
1327	    mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
1328		/* Workaround for Valgrind */
1329		if (rlimit(RLIMIT_DATA) == 0 &&
1330		    mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
1331			return true;
1332
1333		pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
1334			     current->comm, current->pid,
1335			     (mm->data_vm + npages) << PAGE_SHIFT,
1336			     rlimit(RLIMIT_DATA),
1337			     ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");
1338
1339		if (!ignore_rlimit_data)
1340			return false;
1341	}
1342
1343	return true;
1344}
1345
1346void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
1347{
1348	WRITE_ONCE(mm->total_vm, READ_ONCE(mm->total_vm)+npages);
1349
1350	if (is_exec_mapping(flags))
1351		mm->exec_vm += npages;
1352	else if (is_stack_mapping(flags))
1353		mm->stack_vm += npages;
1354	else if (is_data_mapping(flags))
1355		mm->data_vm += npages;
1356}
1357
1358static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
1359
1360/*
1361 * Close hook, called for unmap() and on the old vma for mremap().
1362 *
1363 * Having a close hook prevents vma merging regardless of flags.
1364 */
1365static void special_mapping_close(struct vm_area_struct *vma)
1366{
1367	const struct vm_special_mapping *sm = vma->vm_private_data;
1368
1369	if (sm->close)
1370		sm->close(sm, vma);
1371}
1372
1373static const char *special_mapping_name(struct vm_area_struct *vma)
1374{
1375	return ((struct vm_special_mapping *)vma->vm_private_data)->name;
1376}
1377
1378static int special_mapping_mremap(struct vm_area_struct *new_vma)
1379{
1380	struct vm_special_mapping *sm = new_vma->vm_private_data;
1381
1382	if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
1383		return -EFAULT;
1384
1385	if (sm->mremap)
1386		return sm->mremap(sm, new_vma);
1387
1388	return 0;
1389}
1390
1391static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr)
1392{
1393	/*
1394	 * Forbid splitting special mappings - kernel has expectations over
1395	 * the number of pages in mapping. Together with VM_DONTEXPAND
1396	 * the size of vma should stay the same over the special mapping's
1397	 * lifetime.
1398	 */
1399	return -EINVAL;
1400}
1401
1402static const struct vm_operations_struct special_mapping_vmops = {
1403	.close = special_mapping_close,
1404	.fault = special_mapping_fault,
1405	.mremap = special_mapping_mremap,
1406	.name = special_mapping_name,
1407	/* vDSO code relies that VVAR can't be accessed remotely */
1408	.access = NULL,
1409	.may_split = special_mapping_split,
1410};
1411
1412static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
1413{
1414	struct vm_area_struct *vma = vmf->vma;
1415	pgoff_t pgoff;
1416	struct page **pages;
1417	struct vm_special_mapping *sm = vma->vm_private_data;
1418
1419	if (sm->fault)
1420		return sm->fault(sm, vmf->vma, vmf);
1421
1422	pages = sm->pages;
1423
1424	for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
1425		pgoff--;
1426
1427	if (*pages) {
1428		struct page *page = *pages;
1429		get_page(page);
1430		vmf->page = page;
1431		return 0;
1432	}
1433
1434	return VM_FAULT_SIGBUS;
1435}
1436
1437static struct vm_area_struct *__install_special_mapping(
1438	struct mm_struct *mm,
1439	unsigned long addr, unsigned long len,
1440	vm_flags_t vm_flags, void *priv,
1441	const struct vm_operations_struct *ops)
1442{
1443	int ret;
1444	struct vm_area_struct *vma;
1445
1446	vma = vm_area_alloc(mm);
1447	if (unlikely(vma == NULL))
1448		return ERR_PTR(-ENOMEM);
1449
1450	vma_set_range(vma, addr, addr + len, 0);
1451	vm_flags |= mm->def_flags | VM_DONTEXPAND;
1452	if (pgtable_supports_soft_dirty())
1453		vm_flags |= VM_SOFTDIRTY;
1454	vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK);
1455	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
1456
1457	vma->vm_ops = ops;
1458	vma->vm_private_data = priv;
1459
1460	ret = insert_vm_struct(mm, vma);
1461	if (ret)
1462		goto out;
1463
1464	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
1465
1466	perf_event_mmap(vma);
1467
1468	return vma;
1469
1470out:
1471	vm_area_free(vma);
1472	return ERR_PTR(ret);
1473}
1474
1475bool vma_is_special_mapping(const struct vm_area_struct *vma,
1476	const struct vm_special_mapping *sm)
1477{
1478	return vma->vm_private_data == sm &&
1479		vma->vm_ops == &special_mapping_vmops;
1480}
1481
1482/*
1483 * Called with mm->mmap_lock held for writing.
1484 * Insert a new vma covering the given region, with the given flags.
1485 * Its pages are supplied by the given array of struct page *.
1486 * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
1487 * The region past the last page supplied will always produce SIGBUS.
1488 * The array pointer and the pages it points to are assumed to stay alive
1489 * for as long as this mapping might exist.
1490 */
1491struct vm_area_struct *_install_special_mapping(
1492	struct mm_struct *mm,
1493	unsigned long addr, unsigned long len,
1494	vm_flags_t vm_flags, const struct vm_special_mapping *spec)
1495{
1496	return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
1497					&special_mapping_vmops);
1498}
1499
1500#ifdef CONFIG_SYSCTL
1501#if defined(HAVE_ARCH_PICK_MMAP_LAYOUT) || \
1502		defined(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT)
1503int sysctl_legacy_va_layout;
1504#endif
1505
1506static const struct ctl_table mmap_table[] = {
1507		{
1508				.procname       = "max_map_count",
1509				.data           = &sysctl_max_map_count,
1510				.maxlen         = sizeof(sysctl_max_map_count),
1511				.mode           = 0644,
1512				.proc_handler   = proc_dointvec_minmax,
1513				.extra1         = SYSCTL_ZERO,
1514		},
1515#if defined(HAVE_ARCH_PICK_MMAP_LAYOUT) || \
1516		defined(CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT)
1517		{
1518				.procname       = "legacy_va_layout",
1519				.data           = &sysctl_legacy_va_layout,
1520				.maxlen         = sizeof(sysctl_legacy_va_layout),
1521				.mode           = 0644,
1522				.proc_handler   = proc_dointvec_minmax,
1523				.extra1         = SYSCTL_ZERO,
1524		},
1525#endif
1526#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
1527		{
1528				.procname       = "mmap_rnd_bits",
1529				.data           = &mmap_rnd_bits,
1530				.maxlen         = sizeof(mmap_rnd_bits),
1531				.mode           = 0600,
1532				.proc_handler   = proc_dointvec_minmax,
1533				.extra1         = (void *)&mmap_rnd_bits_min,
1534				.extra2         = (void *)&mmap_rnd_bits_max,
1535		},
1536#endif
1537#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
1538		{
1539				.procname       = "mmap_rnd_compat_bits",
1540				.data           = &mmap_rnd_compat_bits,
1541				.maxlen         = sizeof(mmap_rnd_compat_bits),
1542				.mode           = 0600,
1543				.proc_handler   = proc_dointvec_minmax,
1544				.extra1         = (void *)&mmap_rnd_compat_bits_min,
1545				.extra2         = (void *)&mmap_rnd_compat_bits_max,
1546		},
1547#endif
1548};
1549#endif /* CONFIG_SYSCTL */
1550
1551/*
1552 * initialise the percpu counter for VM, initialise VMA state.
1553 */
1554void __init mmap_init(void)
1555{
1556	int ret;
1557
1558	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
1559	VM_BUG_ON(ret);
1560#ifdef CONFIG_SYSCTL
1561	register_sysctl_init("vm", mmap_table);
1562#endif
1563	vma_state_init();
1564}
1565
1566/*
1567 * Initialise sysctl_user_reserve_kbytes.
1568 *
1569 * This is intended to prevent a user from starting a single memory hogging
1570 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1571 * mode.
1572 *
1573 * The default value is min(3% of free memory, 128MB)
1574 * 128MB is enough to recover with sshd/login, bash, and top/kill.
1575 */
1576static int init_user_reserve(void)
1577{
1578	unsigned long free_kbytes;
1579
1580	free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
1581
1582	sysctl_user_reserve_kbytes = min(free_kbytes / 32, SZ_128K);
1583	return 0;
1584}
1585subsys_initcall(init_user_reserve);
1586
1587/*
1588 * Initialise sysctl_admin_reserve_kbytes.
1589 *
1590 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1591 * to log in and kill a memory hogging process.
1592 *
1593 * Systems with more than 256MB will reserve 8MB, enough to recover
1594 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1595 * only reserve 3% of free pages by default.
1596 */
1597static int init_admin_reserve(void)
1598{
1599	unsigned long free_kbytes;
1600
1601	free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
1602
1603	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, SZ_8K);
1604	return 0;
1605}
1606subsys_initcall(init_admin_reserve);
1607
1608/*
1609 * Reinititalise user and admin reserves if memory is added or removed.
1610 *
1611 * The default user reserve max is 128MB, and the default max for the
1612 * admin reserve is 8MB. These are usually, but not always, enough to
1613 * enable recovery from a memory hogging process using login/sshd, a shell,
1614 * and tools like top. It may make sense to increase or even disable the
1615 * reserve depending on the existence of swap or variations in the recovery
1616 * tools. So, the admin may have changed them.
1617 *
1618 * If memory is added and the reserves have been eliminated or increased above
1619 * the default max, then we'll trust the admin.
1620 *
1621 * If memory is removed and there isn't enough free memory, then we
1622 * need to reset the reserves.
1623 *
1624 * Otherwise keep the reserve set by the admin.
1625 */
1626static int reserve_mem_notifier(struct notifier_block *nb,
1627			     unsigned long action, void *data)
1628{
1629	unsigned long tmp, free_kbytes;
1630
1631	switch (action) {
1632	case MEM_ONLINE:
1633		/* Default max is 128MB. Leave alone if modified by operator. */
1634		tmp = sysctl_user_reserve_kbytes;
1635		if (tmp > 0 && tmp < SZ_128K)
1636			init_user_reserve();
1637
1638		/* Default max is 8MB.  Leave alone if modified by operator. */
1639		tmp = sysctl_admin_reserve_kbytes;
1640		if (tmp > 0 && tmp < SZ_8K)
1641			init_admin_reserve();
1642
1643		break;
1644	case MEM_OFFLINE:
1645		free_kbytes = K(global_zone_page_state(NR_FREE_PAGES));
1646
1647		if (sysctl_user_reserve_kbytes > free_kbytes) {
1648			init_user_reserve();
1649			pr_info("vm.user_reserve_kbytes reset to %lu\n",
1650				sysctl_user_reserve_kbytes);
1651		}
1652
1653		if (sysctl_admin_reserve_kbytes > free_kbytes) {
1654			init_admin_reserve();
1655			pr_info("vm.admin_reserve_kbytes reset to %lu\n",
1656				sysctl_admin_reserve_kbytes);
1657		}
1658		break;
1659	default:
1660		break;
1661	}
1662	return NOTIFY_OK;
1663}
1664
1665static int __meminit init_reserve_notifier(void)
1666{
1667	if (hotplug_memory_notifier(reserve_mem_notifier, DEFAULT_CALLBACK_PRI))
1668		pr_err("Failed registering memory add/remove notifier for admin reserve\n");
1669
1670	return 0;
1671}
1672subsys_initcall(init_reserve_notifier);
1673
1674/*
1675 * Obtain a read lock on mm->mmap_lock, if the specified address is below the
1676 * start of the VMA, the intent is to perform a write, and it is a
1677 * downward-growing stack, then attempt to expand the stack to contain it.
1678 *
1679 * This function is intended only for obtaining an argument page from an ELF
1680 * image, and is almost certainly NOT what you want to use for any other
1681 * purpose.
1682 *
1683 * IMPORTANT - VMA fields are accessed without an mmap lock being held, so the
1684 * VMA referenced must not be linked in any user-visible tree, i.e. it must be a
1685 * new VMA being mapped.
1686 *
1687 * The function assumes that addr is either contained within the VMA or below
1688 * it, and makes no attempt to validate this value beyond that.
1689 *
1690 * Returns true if the read lock was obtained and a stack was perhaps expanded,
1691 * false if the stack expansion failed.
1692 *
1693 * On stack expansion the function temporarily acquires an mmap write lock
1694 * before downgrading it.
1695 */
1696bool mmap_read_lock_maybe_expand(struct mm_struct *mm,
1697				 struct vm_area_struct *new_vma,
1698				 unsigned long addr, bool write)
1699{
1700	if (!write || addr >= new_vma->vm_start) {
1701		mmap_read_lock(mm);
1702		return true;
1703	}
1704
1705	if (!(new_vma->vm_flags & VM_GROWSDOWN))
1706		return false;
1707
1708	mmap_write_lock(mm);
1709	if (expand_downwards(new_vma, addr)) {
1710		mmap_write_unlock(mm);
1711		return false;
1712	}
1713
1714	mmap_write_downgrade(mm);
1715	return true;
1716}
1717
1718__latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
1719{
1720	struct vm_area_struct *mpnt, *tmp;
1721	int retval;
1722	unsigned long charge = 0;
1723	LIST_HEAD(uf);
1724	VMA_ITERATOR(vmi, mm, 0);
1725
1726	if (mmap_write_lock_killable(oldmm))
1727		return -EINTR;
1728	flush_cache_dup_mm(oldmm);
1729	uprobe_dup_mmap(oldmm, mm);
1730	/*
1731	 * Not linked in yet - no deadlock potential:
1732	 */
1733	mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING);
1734
1735	/* No ordering required: file already has been exposed. */
1736	dup_mm_exe_file(mm, oldmm);
1737
1738	mm->total_vm = oldmm->total_vm;
1739	mm->data_vm = oldmm->data_vm;
1740	mm->exec_vm = oldmm->exec_vm;
1741	mm->stack_vm = oldmm->stack_vm;
1742
1743	/* Use __mt_dup() to efficiently build an identical maple tree. */
1744	retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL);
1745	if (unlikely(retval))
1746		goto out;
1747
1748	mt_clear_in_rcu(vmi.mas.tree);
1749	for_each_vma(vmi, mpnt) {
1750		struct file *file;
1751
1752		retval = vma_start_write_killable(mpnt);
1753		if (retval < 0)
1754			goto loop_out;
1755		if (mpnt->vm_flags & VM_DONTCOPY) {
1756			retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start,
1757						    mpnt->vm_end, GFP_KERNEL);
1758			if (retval)
1759				goto loop_out;
1760
1761			vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
1762			continue;
1763		}
1764		charge = 0;
1765		if (mpnt->vm_flags & VM_ACCOUNT) {
1766			unsigned long len = vma_pages(mpnt);
1767
1768			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
1769				goto fail_nomem;
1770			charge = len;
1771		}
1772
1773		tmp = vm_area_dup(mpnt);
1774		if (!tmp)
1775			goto fail_nomem;
1776		retval = vma_dup_policy(mpnt, tmp);
1777		if (retval)
1778			goto fail_nomem_policy;
1779		tmp->vm_mm = mm;
1780		retval = dup_userfaultfd(tmp, &uf);
1781		if (retval)
1782			goto fail_nomem_anon_vma_fork;
1783		if (tmp->vm_flags & VM_WIPEONFORK) {
1784			/*
1785			 * VM_WIPEONFORK gets a clean slate in the child.
1786			 * Don't prepare anon_vma until fault since we don't
1787			 * copy page for current vma.
1788			 */
1789			tmp->anon_vma = NULL;
1790		} else if (anon_vma_fork(tmp, mpnt))
1791			goto fail_nomem_anon_vma_fork;
1792		vm_flags_clear(tmp, VM_LOCKED_MASK);
1793		/*
1794		 * Copy/update hugetlb private vma information.
1795		 */
1796		if (is_vm_hugetlb_page(tmp))
1797			hugetlb_dup_vma_private(tmp);
1798
1799		/*
1800		 * Link the vma into the MT. After using __mt_dup(), memory
1801		 * allocation is not necessary here, so it cannot fail.
1802		 */
1803		vma_iter_bulk_store(&vmi, tmp);
1804
1805		mm->map_count++;
1806
1807		if (tmp->vm_ops && tmp->vm_ops->open)
1808			tmp->vm_ops->open(tmp);
1809
1810		file = tmp->vm_file;
1811		if (file) {
1812			struct address_space *mapping = file->f_mapping;
1813
1814			get_file(file);
1815			i_mmap_lock_write(mapping);
1816			if (vma_is_shared_maywrite(tmp))
1817				mapping_allow_writable(mapping);
1818			flush_dcache_mmap_lock(mapping);
1819			/* insert tmp into the share list, just after mpnt */
1820			vma_interval_tree_insert_after(tmp, mpnt,
1821					&mapping->i_mmap);
1822			flush_dcache_mmap_unlock(mapping);
1823			i_mmap_unlock_write(mapping);
1824		}
1825
1826		if (!(tmp->vm_flags & VM_WIPEONFORK))
1827			retval = copy_page_range(tmp, mpnt);
1828
1829		if (retval) {
1830			mpnt = vma_next(&vmi);
1831			goto loop_out;
1832		}
1833	}
1834	/* a new mm has just been created */
1835	retval = arch_dup_mmap(oldmm, mm);
1836loop_out:
1837	vma_iter_free(&vmi);
1838	if (!retval) {
1839		mt_set_in_rcu(vmi.mas.tree);
1840		ksm_fork(mm, oldmm);
1841		khugepaged_fork(mm, oldmm);
1842	} else {
1843
1844		/*
1845		 * The entire maple tree has already been duplicated. If the
1846		 * mmap duplication fails, mark the failure point with
1847		 * XA_ZERO_ENTRY. In exit_mmap(), if this marker is encountered,
1848		 * stop releasing VMAs that have not been duplicated after this
1849		 * point.
1850		 */
1851		if (mpnt) {
1852			mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1);
1853			mas_store(&vmi.mas, XA_ZERO_ENTRY);
1854			/* Avoid OOM iterating a broken tree */
1855			mm_flags_set(MMF_OOM_SKIP, mm);
1856		}
1857		/*
1858		 * The mm_struct is going to exit, but the locks will be dropped
1859		 * first.  Set the mm_struct as unstable is advisable as it is
1860		 * not fully initialised.
1861		 */
1862		mm_flags_set(MMF_UNSTABLE, mm);
1863	}
1864out:
1865	mmap_write_unlock(mm);
1866	flush_tlb_mm(oldmm);
1867	mmap_write_unlock(oldmm);
1868	if (!retval)
1869		dup_userfaultfd_complete(&uf);
1870	else
1871		dup_userfaultfd_fail(&uf);
1872	return retval;
1873
1874fail_nomem_anon_vma_fork:
1875	mpol_put(vma_policy(tmp));
1876fail_nomem_policy:
1877	vm_area_free(tmp);
1878fail_nomem:
1879	retval = -ENOMEM;
1880	vm_unacct_memory(charge);
1881	goto loop_out;
1882}