mprotect.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  mm/mprotect.c
   4 *
   5 *  (C) Copyright 1994 Linus Torvalds
   6 *  (C) Copyright 2002 Christoph Hellwig
   7 *
   8 *  Address space accounting code	<alan@lxorguk.ukuu.org.uk>
   9 *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
  10 */
  11
  12#include <linux/pagewalk.h>
  13#include <linux/hugetlb.h>
  14#include <linux/shm.h>
  15#include <linux/mman.h>
  16#include <linux/fs.h>
  17#include <linux/highmem.h>
  18#include <linux/security.h>
  19#include <linux/mempolicy.h>
  20#include <linux/personality.h>
  21#include <linux/syscalls.h>
  22#include <linux/swap.h>
  23#include <linux/swapops.h>
  24#include <linux/mmu_notifier.h>
  25#include <linux/migrate.h>
  26#include <linux/perf_event.h>
  27#include <linux/pkeys.h>
  28#include <linux/ksm.h>
  29#include <linux/uaccess.h>
  30#include <linux/mm_inline.h>
  31#include <linux/pgtable.h>
  32#include <linux/userfaultfd_k.h>
  33#include <uapi/linux/mman.h>
  34#include <asm/cacheflush.h>
  35#include <asm/mmu_context.h>
  36#include <asm/tlbflush.h>
  37#include <asm/tlb.h>
  38
  39#include "internal.h"
  40
  41static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte)
  42{
  43	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
  44		return false;
  45
  46	/* Don't touch entries that are not even readable. */
  47	if (pte_protnone(pte))
  48		return false;
  49
  50	/* Do we need write faults for softdirty tracking? */
  51	if (pte_needs_soft_dirty_wp(vma, pte))
  52		return false;
  53
  54	/* Do we need write faults for uffd-wp tracking? */
  55	if (userfaultfd_pte_wp(vma, pte))
  56		return false;
  57
  58	return true;
  59}
  60
  61static bool can_change_private_pte_writable(struct vm_area_struct *vma,
  62					    unsigned long addr, pte_t pte)
  63{
  64	struct page *page;
  65
  66	if (!maybe_change_pte_writable(vma, pte))
  67		return false;
  68
  69	/*
  70	 * Writable MAP_PRIVATE mapping: We can only special-case on
  71	 * exclusive anonymous pages, because we know that our
  72	 * write-fault handler similarly would map them writable without
  73	 * any additional checks while holding the PT lock.
  74	 */
  75	page = vm_normal_page(vma, addr, pte);
  76	return page && PageAnon(page) && PageAnonExclusive(page);
  77}
  78
  79static bool can_change_shared_pte_writable(struct vm_area_struct *vma,
  80					   pte_t pte)
  81{
  82	if (!maybe_change_pte_writable(vma, pte))
  83		return false;
  84
  85	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
  86
  87	/*
  88	 * Writable MAP_SHARED mapping: "clean" might indicate that the FS still
  89	 * needs a real write-fault for writenotify
  90	 * (see vma_wants_writenotify()). If "dirty", the assumption is that the
  91	 * FS was already notified and we can simply mark the PTE writable
  92	 * just like the write-fault handler would do.
  93	 */
  94	return pte_dirty(pte);
  95}
  96
  97bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  98			     pte_t pte)
  99{
 100	if (!(vma->vm_flags & VM_SHARED))
 101		return can_change_private_pte_writable(vma, addr, pte);
 102
 103	return can_change_shared_pte_writable(vma, pte);
 104}
 105
 106static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep,
 107				    pte_t pte, int max_nr_ptes, fpb_t flags)
 108{
 109	/* No underlying folio, so cannot batch */
 110	if (!folio)
 111		return 1;
 112
 113	if (!folio_test_large(folio))
 114		return 1;
 115
 116	return folio_pte_batch_flags(folio, NULL, ptep, &pte, max_nr_ptes, flags);
 117}
 118
 119/* Set nr_ptes number of ptes, starting from idx */
 120static void prot_commit_flush_ptes(struct vm_area_struct *vma, unsigned long addr,
 121		pte_t *ptep, pte_t oldpte, pte_t ptent, int nr_ptes,
 122		int idx, bool set_write, struct mmu_gather *tlb)
 123{
 124	/*
 125	 * Advance the position in the batch by idx; note that if idx > 0,
 126	 * then the nr_ptes passed here is <= batch size - idx.
 127	 */
 128	addr += idx * PAGE_SIZE;
 129	ptep += idx;
 130	oldpte = pte_advance_pfn(oldpte, idx);
 131	ptent = pte_advance_pfn(ptent, idx);
 132
 133	if (set_write)
 134		ptent = pte_mkwrite(ptent, vma);
 135
 136	modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes);
 137	if (pte_needs_flush(oldpte, ptent))
 138		tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
 139}
 140
 141/*
 142 * Get max length of consecutive ptes pointing to PageAnonExclusive() pages or
 143 * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce
 144 * that the ptes point to consecutive pages of the same anon large folio.
 145 */
 146static int page_anon_exclusive_sub_batch(int start_idx, int max_len,
 147		struct page *first_page, bool expected_anon_exclusive)
 148{
 149	int idx;
 150
 151	for (idx = start_idx + 1; idx < start_idx + max_len; ++idx) {
 152		if (expected_anon_exclusive != PageAnonExclusive(first_page + idx))
 153			break;
 154	}
 155	return idx - start_idx;
 156}
 157
 158/*
 159 * This function is a result of trying our very best to retain the
 160 * "avoid the write-fault handler" optimization. In can_change_pte_writable(),
 161 * if the vma is a private vma, and we cannot determine whether to change
 162 * the pte to writable just from the vma and the pte, we then need to look
 163 * at the actual page pointed to by the pte. Unfortunately, if we have a
 164 * batch of ptes pointing to consecutive pages of the same anon large folio,
 165 * the anon-exclusivity (or the negation) of the first page does not guarantee
 166 * the anon-exclusivity (or the negation) of the other pages corresponding to
 167 * the pte batch; hence in this case it is incorrect to decide to change or
 168 * not change the ptes to writable just by using information from the first
 169 * pte of the batch. Therefore, we must individually check all pages and
 170 * retrieve sub-batches.
 171 */
 172static void commit_anon_folio_batch(struct vm_area_struct *vma,
 173		struct folio *folio, struct page *first_page, unsigned long addr, pte_t *ptep,
 174		pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
 175{
 176	bool expected_anon_exclusive;
 177	int sub_batch_idx = 0;
 178	int len;
 179
 180	while (nr_ptes) {
 181		expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx);
 182		len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes,
 183					first_page, expected_anon_exclusive);
 184		prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len,
 185				       sub_batch_idx, expected_anon_exclusive, tlb);
 186		sub_batch_idx += len;
 187		nr_ptes -= len;
 188	}
 189}
 190
 191static void set_write_prot_commit_flush_ptes(struct vm_area_struct *vma,
 192		struct folio *folio, struct page *page, unsigned long addr, pte_t *ptep,
 193		pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
 194{
 195	bool set_write;
 196
 197	if (vma->vm_flags & VM_SHARED) {
 198		set_write = can_change_shared_pte_writable(vma, ptent);
 199		prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
 200				       /* idx = */ 0, set_write, tlb);
 201		return;
 202	}
 203
 204	set_write = maybe_change_pte_writable(vma, ptent) &&
 205		    (folio && folio_test_anon(folio));
 206	if (!set_write) {
 207		prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
 208				       /* idx = */ 0, set_write, tlb);
 209		return;
 210	}
 211	commit_anon_folio_batch(vma, folio, page, addr, ptep, oldpte, ptent, nr_ptes, tlb);
 212}
 213
 214static long change_pte_range(struct mmu_gather *tlb,
 215		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
 216		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
 217{
 218	pte_t *pte, oldpte;
 219	spinlock_t *ptl;
 220	long pages = 0;
 221	bool is_private_single_threaded;
 222	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
 223	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 224	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
 225	int nr_ptes;
 226
 227	tlb_change_page_size(tlb, PAGE_SIZE);
 228	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
 229	if (!pte)
 230		return -EAGAIN;
 231
 232	if (prot_numa)
 233		is_private_single_threaded = vma_is_single_threaded_private(vma);
 234
 235	flush_tlb_batched_pending(vma->vm_mm);
 236	arch_enter_lazy_mmu_mode();
 237	do {
 238		nr_ptes = 1;
 239		oldpte = ptep_get(pte);
 240		if (pte_present(oldpte)) {
 241			const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE;
 242			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
 243			struct folio *folio = NULL;
 244			struct page *page;
 245			pte_t ptent;
 246
 247			/* Already in the desired state. */
 248			if (prot_numa && pte_protnone(oldpte))
 249				continue;
 250
 251			page = vm_normal_page(vma, addr, oldpte);
 252			if (page)
 253				folio = page_folio(page);
 254
 255			/*
 256			 * Avoid trapping faults against the zero or KSM
 257			 * pages. See similar comment in change_huge_pmd.
 258			 */
 259			if (prot_numa &&
 260			    !folio_can_map_prot_numa(folio, vma,
 261						is_private_single_threaded)) {
 262
 263				/* determine batch to skip */
 264				nr_ptes = mprotect_folio_pte_batch(folio,
 265					  pte, oldpte, max_nr_ptes, /* flags = */ 0);
 266				continue;
 267			}
 268
 269			nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags);
 270
 271			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
 272			ptent = pte_modify(oldpte, newprot);
 273
 274			if (uffd_wp)
 275				ptent = pte_mkuffd_wp(ptent);
 276			else if (uffd_wp_resolve)
 277				ptent = pte_clear_uffd_wp(ptent);
 278
 279			/*
 280			 * In some writable, shared mappings, we might want
 281			 * to catch actual write access -- see
 282			 * vma_wants_writenotify().
 283			 *
 284			 * In all writable, private mappings, we have to
 285			 * properly handle COW.
 286			 *
 287			 * In both cases, we can sometimes still change PTEs
 288			 * writable and avoid the write-fault handler, for
 289			 * example, if a PTE is already dirty and no other
 290			 * COW or special handling is required.
 291			 */
 292			if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
 293			     !pte_write(ptent))
 294				set_write_prot_commit_flush_ptes(vma, folio, page,
 295				addr, pte, oldpte, ptent, nr_ptes, tlb);
 296			else
 297				prot_commit_flush_ptes(vma, addr, pte, oldpte, ptent,
 298					nr_ptes, /* idx = */ 0, /* set_write = */ false, tlb);
 299			pages += nr_ptes;
 300		} else if (pte_none(oldpte)) {
 301			/*
 302			 * Nobody plays with any none ptes besides
 303			 * userfaultfd when applying the protections.
 304			 */
 305			if (likely(!uffd_wp))
 306				continue;
 307
 308			if (userfaultfd_wp_use_markers(vma)) {
 309				/*
 310				 * For file-backed mem, we need to be able to
 311				 * wr-protect a none pte, because even if the
 312				 * pte is none, the page/swap cache could
 313				 * exist.  Doing that by install a marker.
 314				 */
 315				set_pte_at(vma->vm_mm, addr, pte,
 316					   make_pte_marker(PTE_MARKER_UFFD_WP));
 317				pages++;
 318			}
 319		} else  {
 320			softleaf_t entry = softleaf_from_pte(oldpte);
 321			pte_t newpte;
 322
 323			if (softleaf_is_migration_write(entry)) {
 324				const struct folio *folio = softleaf_to_folio(entry);
 325
 326				/*
 327				 * A protection check is difficult so
 328				 * just be safe and disable write
 329				 */
 330				if (folio_test_anon(folio))
 331					entry = make_readable_exclusive_migration_entry(
 332							     swp_offset(entry));
 333				else
 334					entry = make_readable_migration_entry(swp_offset(entry));
 335				newpte = swp_entry_to_pte(entry);
 336				if (pte_swp_soft_dirty(oldpte))
 337					newpte = pte_swp_mksoft_dirty(newpte);
 338			} else if (softleaf_is_device_private_write(entry)) {
 339				/*
 340				 * We do not preserve soft-dirtiness. See
 341				 * copy_nonpresent_pte() for explanation.
 342				 */
 343				entry = make_readable_device_private_entry(
 344							swp_offset(entry));
 345				newpte = swp_entry_to_pte(entry);
 346				if (pte_swp_uffd_wp(oldpte))
 347					newpte = pte_swp_mkuffd_wp(newpte);
 348			} else if (softleaf_is_marker(entry)) {
 349				/*
 350				 * Ignore error swap entries unconditionally,
 351				 * because any access should sigbus/sigsegv
 352				 * anyway.
 353				 */
 354				if (softleaf_is_poison_marker(entry) ||
 355				    softleaf_is_guard_marker(entry))
 356					continue;
 357				/*
 358				 * If this is uffd-wp pte marker and we'd like
 359				 * to unprotect it, drop it; the next page
 360				 * fault will trigger without uffd trapping.
 361				 */
 362				if (uffd_wp_resolve) {
 363					pte_clear(vma->vm_mm, addr, pte);
 364					pages++;
 365				}
 366				continue;
 367			} else {
 368				newpte = oldpte;
 369			}
 370
 371			if (uffd_wp)
 372				newpte = pte_swp_mkuffd_wp(newpte);
 373			else if (uffd_wp_resolve)
 374				newpte = pte_swp_clear_uffd_wp(newpte);
 375
 376			if (!pte_same(oldpte, newpte)) {
 377				set_pte_at(vma->vm_mm, addr, pte, newpte);
 378				pages++;
 379			}
 380		}
 381	} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
 382	arch_leave_lazy_mmu_mode();
 383	pte_unmap_unlock(pte - 1, ptl);
 384
 385	return pages;
 386}
 387
 388/*
 389 * Return true if we want to split THPs into PTE mappings in change
 390 * protection procedure, false otherwise.
 391 */
 392static inline bool
 393pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
 394{
 395	/*
 396	 * pte markers only resides in pte level, if we need pte markers,
 397	 * we need to split.  For example, we cannot wr-protect a file thp
 398	 * (e.g. 2M shmem) because file thp is handled differently when
 399	 * split by erasing the pmd so far.
 400	 */
 401	return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);
 402}
 403
 404/*
 405 * Return true if we want to populate pgtables in change protection
 406 * procedure, false otherwise
 407 */
 408static inline bool
 409pgtable_populate_needed(struct vm_area_struct *vma, unsigned long cp_flags)
 410{
 411	/* If not within ioctl(UFFDIO_WRITEPROTECT), then don't bother */
 412	if (!(cp_flags & MM_CP_UFFD_WP))
 413		return false;
 414
 415	/* Populate if the userfaultfd mode requires pte markers */
 416	return userfaultfd_wp_use_markers(vma);
 417}
 418
 419/*
 420 * Populate the pgtable underneath for whatever reason if requested.
 421 * When {pte|pmd|...}_alloc() failed we treat it the same way as pgtable
 422 * allocation failures during page faults by kicking OOM and returning
 423 * error.
 424 */
 425#define  change_pmd_prepare(vma, pmd, cp_flags)				\
 426	({								\
 427		long err = 0;						\
 428		if (unlikely(pgtable_populate_needed(vma, cp_flags))) {	\
 429			if (pte_alloc(vma->vm_mm, pmd))			\
 430				err = -ENOMEM;				\
 431		}							\
 432		err;							\
 433	})
 434
 435/*
 436 * This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to
 437 * have separate change_pmd_prepare() because pte_alloc() returns 0 on success,
 438 * while {pmd|pud|p4d}_alloc() returns the valid pointer on success.
 439 */
 440#define  change_prepare(vma, high, low, addr, cp_flags)			\
 441	  ({								\
 442		long err = 0;						\
 443		if (unlikely(pgtable_populate_needed(vma, cp_flags))) {	\
 444			low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
 445			if (p == NULL)					\
 446				err = -ENOMEM;				\
 447		}							\
 448		err;							\
 449	})
 450
 451static inline long change_pmd_range(struct mmu_gather *tlb,
 452		struct vm_area_struct *vma, pud_t *pud, unsigned long addr,
 453		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
 454{
 455	pmd_t *pmd;
 456	unsigned long next;
 457	long pages = 0;
 458	unsigned long nr_huge_updates = 0;
 459
 460	pmd = pmd_offset(pud, addr);
 461	do {
 462		long ret;
 463		pmd_t _pmd;
 464again:
 465		next = pmd_addr_end(addr, end);
 466
 467		ret = change_pmd_prepare(vma, pmd, cp_flags);
 468		if (ret) {
 469			pages = ret;
 470			break;
 471		}
 472
 473		if (pmd_none(*pmd))
 474			goto next;
 475
 476		_pmd = pmdp_get_lockless(pmd);
 477		if (pmd_is_huge(_pmd)) {
 478			if ((next - addr != HPAGE_PMD_SIZE) ||
 479			    pgtable_split_needed(vma, cp_flags)) {
 480				__split_huge_pmd(vma, pmd, addr, false);
 481				/*
 482				 * For file-backed, the pmd could have been
 483				 * cleared; make sure pmd populated if
 484				 * necessary, then fall-through to pte level.
 485				 */
 486				ret = change_pmd_prepare(vma, pmd, cp_flags);
 487				if (ret) {
 488					pages = ret;
 489					break;
 490				}
 491			} else {
 492				ret = change_huge_pmd(tlb, vma, pmd,
 493						addr, newprot, cp_flags);
 494				if (ret) {
 495					if (ret == HPAGE_PMD_NR) {
 496						pages += HPAGE_PMD_NR;
 497						nr_huge_updates++;
 498					}
 499
 500					/* huge pmd was handled */
 501					goto next;
 502				}
 503			}
 504			/* fall through, the trans huge pmd just split */
 505		}
 506
 507		ret = change_pte_range(tlb, vma, pmd, addr, next, newprot,
 508				       cp_flags);
 509		if (ret < 0)
 510			goto again;
 511		pages += ret;
 512next:
 513		cond_resched();
 514	} while (pmd++, addr = next, addr != end);
 515
 516	if (nr_huge_updates)
 517		count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
 518	return pages;
 519}
 520
 521static inline long change_pud_range(struct mmu_gather *tlb,
 522		struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr,
 523		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
 524{
 525	struct mmu_notifier_range range;
 526	pud_t *pudp, pud;
 527	unsigned long next;
 528	long pages = 0, ret;
 529
 530	range.start = 0;
 531
 532	pudp = pud_offset(p4d, addr);
 533	do {
 534again:
 535		next = pud_addr_end(addr, end);
 536		ret = change_prepare(vma, pudp, pmd, addr, cp_flags);
 537		if (ret) {
 538			pages = ret;
 539			break;
 540		}
 541
 542		pud = pudp_get(pudp);
 543		if (pud_none(pud))
 544			continue;
 545
 546		if (!range.start) {
 547			mmu_notifier_range_init(&range,
 548						MMU_NOTIFY_PROTECTION_VMA, 0,
 549						vma->vm_mm, addr, end);
 550			mmu_notifier_invalidate_range_start(&range);
 551		}
 552
 553		if (pud_leaf(pud)) {
 554			if ((next - addr != PUD_SIZE) ||
 555			    pgtable_split_needed(vma, cp_flags)) {
 556				__split_huge_pud(vma, pudp, addr);
 557				goto again;
 558			} else {
 559				ret = change_huge_pud(tlb, vma, pudp,
 560						      addr, newprot, cp_flags);
 561				if (ret == 0)
 562					goto again;
 563				/* huge pud was handled */
 564				if (ret == HPAGE_PUD_NR)
 565					pages += HPAGE_PUD_NR;
 566				continue;
 567			}
 568		}
 569
 570		pages += change_pmd_range(tlb, vma, pudp, addr, next, newprot,
 571					  cp_flags);
 572	} while (pudp++, addr = next, addr != end);
 573
 574	if (range.start)
 575		mmu_notifier_invalidate_range_end(&range);
 576
 577	return pages;
 578}
 579
 580static inline long change_p4d_range(struct mmu_gather *tlb,
 581		struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr,
 582		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
 583{
 584	p4d_t *p4d;
 585	unsigned long next;
 586	long pages = 0, ret;
 587
 588	p4d = p4d_offset(pgd, addr);
 589	do {
 590		next = p4d_addr_end(addr, end);
 591		ret = change_prepare(vma, p4d, pud, addr, cp_flags);
 592		if (ret)
 593			return ret;
 594		if (p4d_none_or_clear_bad(p4d))
 595			continue;
 596		pages += change_pud_range(tlb, vma, p4d, addr, next, newprot,
 597					  cp_flags);
 598	} while (p4d++, addr = next, addr != end);
 599
 600	return pages;
 601}
 602
 603static long change_protection_range(struct mmu_gather *tlb,
 604		struct vm_area_struct *vma, unsigned long addr,
 605		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
 606{
 607	struct mm_struct *mm = vma->vm_mm;
 608	pgd_t *pgd;
 609	unsigned long next;
 610	long pages = 0, ret;
 611
 612	BUG_ON(addr >= end);
 613	pgd = pgd_offset(mm, addr);
 614	tlb_start_vma(tlb, vma);
 615	do {
 616		next = pgd_addr_end(addr, end);
 617		ret = change_prepare(vma, pgd, p4d, addr, cp_flags);
 618		if (ret) {
 619			pages = ret;
 620			break;
 621		}
 622		if (pgd_none_or_clear_bad(pgd))
 623			continue;
 624		pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot,
 625					  cp_flags);
 626	} while (pgd++, addr = next, addr != end);
 627
 628	tlb_end_vma(tlb, vma);
 629
 630	return pages;
 631}
 632
 633long change_protection(struct mmu_gather *tlb,
 634		       struct vm_area_struct *vma, unsigned long start,
 635		       unsigned long end, unsigned long cp_flags)
 636{
 637	pgprot_t newprot = vma->vm_page_prot;
 638	long pages;
 639
 640	BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL);
 641
 642#ifdef CONFIG_NUMA_BALANCING
 643	/*
 644	 * Ordinary protection updates (mprotect, uffd-wp, softdirty tracking)
 645	 * are expected to reflect their requirements via VMA flags such that
 646	 * vma_set_page_prot() will adjust vma->vm_page_prot accordingly.
 647	 */
 648	if (cp_flags & MM_CP_PROT_NUMA)
 649		newprot = PAGE_NONE;
 650#else
 651	WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA);
 652#endif
 653
 654	if (is_vm_hugetlb_page(vma))
 655		pages = hugetlb_change_protection(vma, start, end, newprot,
 656						  cp_flags);
 657	else
 658		pages = change_protection_range(tlb, vma, start, end, newprot,
 659						cp_flags);
 660
 661	return pages;
 662}
 663
 664static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
 665			       unsigned long next, struct mm_walk *walk)
 666{
 667	return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
 668				  *(pgprot_t *)(walk->private)) ?
 669		0 : -EACCES;
 670}
 671
 672static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
 673				   unsigned long addr, unsigned long next,
 674				   struct mm_walk *walk)
 675{
 676	return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
 677				  *(pgprot_t *)(walk->private)) ?
 678		0 : -EACCES;
 679}
 680
 681static int prot_none_test(unsigned long addr, unsigned long next,
 682			  struct mm_walk *walk)
 683{
 684	return 0;
 685}
 686
 687static const struct mm_walk_ops prot_none_walk_ops = {
 688	.pte_entry		= prot_none_pte_entry,
 689	.hugetlb_entry		= prot_none_hugetlb_entry,
 690	.test_walk		= prot_none_test,
 691	.walk_lock		= PGWALK_WRLOCK,
 692};
 693
 694int
 695mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 696	       struct vm_area_struct *vma, struct vm_area_struct **pprev,
 697	       unsigned long start, unsigned long end, vm_flags_t newflags)
 698{
 699	struct mm_struct *mm = vma->vm_mm;
 700	vm_flags_t oldflags = READ_ONCE(vma->vm_flags);
 701	long nrpages = (end - start) >> PAGE_SHIFT;
 702	unsigned int mm_cp_flags = 0;
 703	unsigned long charged = 0;
 704	int error;
 705
 706	if (vma_is_sealed(vma))
 707		return -EPERM;
 708
 709	if (newflags == oldflags) {
 710		*pprev = vma;
 711		return 0;
 712	}
 713
 714	/*
 715	 * Do PROT_NONE PFN permission checks here when we can still
 716	 * bail out without undoing a lot of state. This is a rather
 717	 * uncommon case, so doesn't need to be very optimized.
 718	 */
 719	if (arch_has_pfn_modify_check() &&
 720	    (oldflags & (VM_PFNMAP|VM_MIXEDMAP)) &&
 721	    (newflags & VM_ACCESS_FLAGS) == 0) {
 722		pgprot_t new_pgprot = vm_get_page_prot(newflags);
 723
 724		error = walk_page_range(current->mm, start, end,
 725				&prot_none_walk_ops, &new_pgprot);
 726		if (error)
 727			return error;
 728	}
 729
 730	/*
 731	 * If we make a private mapping writable we increase our commit;
 732	 * but (without finer accounting) cannot reduce our commit if we
 733	 * make it unwritable again except in the anonymous case where no
 734	 * anon_vma has yet to be assigned.
 735	 *
 736	 * hugetlb mapping were accounted for even if read-only so there is
 737	 * no need to account for them here.
 738	 */
 739	if (newflags & VM_WRITE) {
 740		/* Check space limits when area turns into data. */
 741		if (!may_expand_vm(mm, newflags, nrpages) &&
 742				may_expand_vm(mm, oldflags, nrpages))
 743			return -ENOMEM;
 744		if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
 745						VM_SHARED|VM_NORESERVE))) {
 746			charged = nrpages;
 747			if (security_vm_enough_memory_mm(mm, charged))
 748				return -ENOMEM;
 749			newflags |= VM_ACCOUNT;
 750		}
 751	} else if ((oldflags & VM_ACCOUNT) && vma_is_anonymous(vma) &&
 752		   !vma->anon_vma) {
 753		newflags &= ~VM_ACCOUNT;
 754	}
 755
 756	vma = vma_modify_flags(vmi, *pprev, vma, start, end, &newflags);
 757	if (IS_ERR(vma)) {
 758		error = PTR_ERR(vma);
 759		goto fail;
 760	}
 761
 762	*pprev = vma;
 763
 764	/*
 765	 * vm_flags and vm_page_prot are protected by the mmap_lock
 766	 * held in write mode.
 767	 */
 768	vma_start_write(vma);
 769	vm_flags_reset_once(vma, newflags);
 770	if (vma_wants_manual_pte_write_upgrade(vma))
 771		mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
 772	vma_set_page_prot(vma);
 773
 774	change_protection(tlb, vma, start, end, mm_cp_flags);
 775
 776	if ((oldflags & VM_ACCOUNT) && !(newflags & VM_ACCOUNT))
 777		vm_unacct_memory(nrpages);
 778
 779	/*
 780	 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
 781	 * fault on access.
 782	 */
 783	if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
 784			(newflags & VM_WRITE)) {
 785		populate_vma_page_range(vma, start, end, NULL);
 786	}
 787
 788	vm_stat_account(mm, oldflags, -nrpages);
 789	vm_stat_account(mm, newflags, nrpages);
 790	perf_event_mmap(vma);
 791	return 0;
 792
 793fail:
 794	vm_unacct_memory(charged);
 795	return error;
 796}
 797
 798/*
 799 * pkey==-1 when doing a legacy mprotect()
 800 */
 801static int do_mprotect_pkey(unsigned long start, size_t len,
 802		unsigned long prot, int pkey)
 803{
 804	unsigned long nstart, end, tmp, reqprot;
 805	struct vm_area_struct *vma, *prev;
 806	int error;
 807	const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
 808	const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
 809				(prot & PROT_READ);
 810	struct mmu_gather tlb;
 811	struct vma_iterator vmi;
 812
 813	start = untagged_addr(start);
 814
 815	prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
 816	if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
 817		return -EINVAL;
 818
 819	if (start & ~PAGE_MASK)
 820		return -EINVAL;
 821	if (!len)
 822		return 0;
 823	len = PAGE_ALIGN(len);
 824	end = start + len;
 825	if (end <= start)
 826		return -ENOMEM;
 827	if (!arch_validate_prot(prot, start))
 828		return -EINVAL;
 829
 830	reqprot = prot;
 831
 832	if (mmap_write_lock_killable(current->mm))
 833		return -EINTR;
 834
 835	/*
 836	 * If userspace did not allocate the pkey, do not let
 837	 * them use it here.
 838	 */
 839	error = -EINVAL;
 840	if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
 841		goto out;
 842
 843	vma_iter_init(&vmi, current->mm, start);
 844	vma = vma_find(&vmi, end);
 845	error = -ENOMEM;
 846	if (!vma)
 847		goto out;
 848
 849	if (unlikely(grows & PROT_GROWSDOWN)) {
 850		if (vma->vm_start >= end)
 851			goto out;
 852		start = vma->vm_start;
 853		error = -EINVAL;
 854		if (!(vma->vm_flags & VM_GROWSDOWN))
 855			goto out;
 856	} else {
 857		if (vma->vm_start > start)
 858			goto out;
 859		if (unlikely(grows & PROT_GROWSUP)) {
 860			end = vma->vm_end;
 861			error = -EINVAL;
 862			if (!(vma->vm_flags & VM_GROWSUP))
 863				goto out;
 864		}
 865	}
 866
 867	prev = vma_prev(&vmi);
 868	if (start > vma->vm_start)
 869		prev = vma;
 870
 871	tlb_gather_mmu(&tlb, current->mm);
 872	nstart = start;
 873	tmp = vma->vm_start;
 874	for_each_vma_range(vmi, vma, end) {
 875		vm_flags_t mask_off_old_flags;
 876		vm_flags_t newflags;
 877		int new_vma_pkey;
 878
 879		if (vma->vm_start != tmp) {
 880			error = -ENOMEM;
 881			break;
 882		}
 883
 884		/* Does the application expect PROT_READ to imply PROT_EXEC */
 885		if (rier && (vma->vm_flags & VM_MAYEXEC))
 886			prot |= PROT_EXEC;
 887
 888		/*
 889		 * Each mprotect() call explicitly passes r/w/x permissions.
 890		 * If a permission is not passed to mprotect(), it must be
 891		 * cleared from the VMA.
 892		 */
 893		mask_off_old_flags = VM_ACCESS_FLAGS | VM_FLAGS_CLEAR;
 894
 895		new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
 896		newflags = calc_vm_prot_bits(prot, new_vma_pkey);
 897		newflags |= (vma->vm_flags & ~mask_off_old_flags);
 898
 899		/* newflags >> 4 shift VM_MAY% in place of VM_% */
 900		if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) {
 901			error = -EACCES;
 902			break;
 903		}
 904
 905		if (map_deny_write_exec(vma->vm_flags, newflags)) {
 906			error = -EACCES;
 907			break;
 908		}
 909
 910		/* Allow architectures to sanity-check the new flags */
 911		if (!arch_validate_flags(newflags)) {
 912			error = -EINVAL;
 913			break;
 914		}
 915
 916		error = security_file_mprotect(vma, reqprot, prot);
 917		if (error)
 918			break;
 919
 920		tmp = vma->vm_end;
 921		if (tmp > end)
 922			tmp = end;
 923
 924		if (vma->vm_ops && vma->vm_ops->mprotect) {
 925			error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags);
 926			if (error)
 927				break;
 928		}
 929
 930		error = mprotect_fixup(&vmi, &tlb, vma, &prev, nstart, tmp, newflags);
 931		if (error)
 932			break;
 933
 934		tmp = vma_iter_end(&vmi);
 935		nstart = tmp;
 936		prot = reqprot;
 937	}
 938	tlb_finish_mmu(&tlb);
 939
 940	if (!error && tmp < end)
 941		error = -ENOMEM;
 942
 943out:
 944	mmap_write_unlock(current->mm);
 945	return error;
 946}
 947
 948SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
 949		unsigned long, prot)
 950{
 951	return do_mprotect_pkey(start, len, prot, -1);
 952}
 953
 954#ifdef CONFIG_ARCH_HAS_PKEYS
 955
 956SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
 957		unsigned long, prot, int, pkey)
 958{
 959	return do_mprotect_pkey(start, len, prot, pkey);
 960}
 961
 962SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
 963{
 964	int pkey;
 965	int ret;
 966
 967	/* No flags supported yet. */
 968	if (flags)
 969		return -EINVAL;
 970	/* check for unsupported init values */
 971	if (init_val & ~PKEY_ACCESS_MASK)
 972		return -EINVAL;
 973
 974	mmap_write_lock(current->mm);
 975	pkey = mm_pkey_alloc(current->mm);
 976
 977	ret = -ENOSPC;
 978	if (pkey == -1)
 979		goto out;
 980
 981	ret = arch_set_user_pkey_access(current, pkey, init_val);
 982	if (ret) {
 983		mm_pkey_free(current->mm, pkey);
 984		goto out;
 985	}
 986	ret = pkey;
 987out:
 988	mmap_write_unlock(current->mm);
 989	return ret;
 990}
 991
 992SYSCALL_DEFINE1(pkey_free, int, pkey)
 993{
 994	int ret;
 995
 996	mmap_write_lock(current->mm);
 997	ret = mm_pkey_free(current->mm, pkey);
 998	mmap_write_unlock(current->mm);
 999
1000	/*
1001	 * We could provide warnings or errors if any VMA still
1002	 * has the pkey set here.
1003	 */
1004	return ret;
1005}
1006
1007#endif /* CONFIG_ARCH_HAS_PKEYS */