vma_exec.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2
  3/*
  4 * Functions explicitly implemented for exec functionality which however are
  5 * explicitly VMA-only logic.
  6 */
  7
  8#include "vma_internal.h"
  9#include "vma.h"
 10
 11/*
 12 * Relocate a VMA downwards by shift bytes. There cannot be any VMAs between
 13 * this VMA and its relocated range, which will now reside at [vma->vm_start -
 14 * shift, vma->vm_end - shift).
 15 *
 16 * This function is almost certainly NOT what you want for anything other than
 17 * early executable temporary stack relocation.
 18 */
 19int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift)
 20{
 21	/*
 22	 * The process proceeds as follows:
 23	 *
 24	 * 1) Use shift to calculate the new vma endpoints.
 25	 * 2) Extend vma to cover both the old and new ranges.  This ensures the
 26	 *    arguments passed to subsequent functions are consistent.
 27	 * 3) Move vma's page tables to the new range.
 28	 * 4) Free up any cleared pgd range.
 29	 * 5) Shrink the vma to cover only the new range.
 30	 */
 31
 32	struct mm_struct *mm = vma->vm_mm;
 33	unsigned long old_start = vma->vm_start;
 34	unsigned long old_end = vma->vm_end;
 35	unsigned long length = old_end - old_start;
 36	unsigned long new_start = old_start - shift;
 37	unsigned long new_end = old_end - shift;
 38	VMA_ITERATOR(vmi, mm, new_start);
 39	VMG_STATE(vmg, mm, &vmi, new_start, old_end, 0, vma->vm_pgoff);
 40	struct vm_area_struct *next;
 41	struct mmu_gather tlb;
 42	PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length);
 43
 44	BUG_ON(new_start > new_end);
 45
 46	/*
 47	 * ensure there are no vmas between where we want to go
 48	 * and where we are
 49	 */
 50	if (vma != vma_next(&vmi))
 51		return -EFAULT;
 52
 53	vma_iter_prev_range(&vmi);
 54	/*
 55	 * cover the whole range: [new_start, old_end)
 56	 */
 57	vmg.target = vma;
 58	if (vma_expand(&vmg))
 59		return -ENOMEM;
 60
 61	/*
 62	 * move the page tables downwards, on failure we rely on
 63	 * process cleanup to remove whatever mess we made.
 64	 */
 65	pmc.for_stack = true;
 66	if (length != move_page_tables(&pmc))
 67		return -ENOMEM;
 68
 69	tlb_gather_mmu(&tlb, mm);
 70	next = vma_next(&vmi);
 71	if (new_end > old_start) {
 72		/*
 73		 * when the old and new regions overlap clear from new_end.
 74		 */
 75		free_pgd_range(&tlb, new_end, old_end, new_end,
 76			next ? next->vm_start : USER_PGTABLES_CEILING);
 77	} else {
 78		/*
 79		 * otherwise, clean from old_start; this is done to not touch
 80		 * the address space in [new_end, old_start) some architectures
 81		 * have constraints on va-space that make this illegal (IA64) -
 82		 * for the others its just a little faster.
 83		 */
 84		free_pgd_range(&tlb, old_start, old_end, new_end,
 85			next ? next->vm_start : USER_PGTABLES_CEILING);
 86	}
 87	tlb_finish_mmu(&tlb);
 88
 89	vma_prev(&vmi);
 90	/* Shrink the vma to just the new range */
 91	return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff);
 92}
 93
 94/*
 95 * Establish the stack VMA in an execve'd process, located temporarily at the
 96 * maximum stack address provided by the architecture.
 97 *
 98 * We later relocate this downwards in relocate_vma_down().
 99 *
100 * This function is almost certainly NOT what you want for anything other than
101 * early executable initialisation.
102 *
103 * On success, returns 0 and sets *vmap to the stack VMA and *top_mem_p to the
104 * maximum addressable location in the stack (that is capable of storing a
105 * system word of data).
106 */
107int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
108			  unsigned long *top_mem_p)
109{
110	unsigned long flags = VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP;
111	int err;
112	struct vm_area_struct *vma = vm_area_alloc(mm);
113
114	if (!vma)
115		return -ENOMEM;
116
117	vma_set_anonymous(vma);
118
119	if (mmap_write_lock_killable(mm)) {
120		err = -EINTR;
121		goto err_free;
122	}
123
124	/*
125	 * Need to be called with mmap write lock
126	 * held, to avoid race with ksmd.
127	 */
128	err = ksm_execve(mm);
129	if (err)
130		goto err_ksm;
131
132	/*
133	 * Place the stack at the largest stack address the architecture
134	 * supports. Later, we'll move this to an appropriate place. We don't
135	 * use STACK_TOP because that can depend on attributes which aren't
136	 * configured yet.
137	 */
138	BUILD_BUG_ON(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP);
139	vma->vm_end = STACK_TOP_MAX;
140	vma->vm_start = vma->vm_end - PAGE_SIZE;
141	if (pgtable_supports_soft_dirty())
142		flags |= VM_SOFTDIRTY;
143	vm_flags_init(vma, flags);
144	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
145
146	err = insert_vm_struct(mm, vma);
147	if (err)
148		goto err;
149
150	mm->stack_vm = mm->total_vm = 1;
151	mmap_write_unlock(mm);
152	*vmap = vma;
153	*top_mem_p = vma->vm_end - sizeof(void *);
154	return 0;
155
156err:
157	ksm_exit(mm);
158err_ksm:
159	mmap_write_unlock(mm);
160err_free:
161	*vmap = NULL;
162	vm_area_free(vma);
163	return err;
164}