vma_init.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3/*
  4 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared
  5 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations.
  6 */
  7
  8#include "vma_internal.h"
  9#include "vma.h"
 10
 11/* SLAB cache for vm_area_struct structures */
 12static struct kmem_cache *vm_area_cachep;
 13
 14void __init vma_state_init(void)
 15{
 16	struct kmem_cache_args args = {
 17		.use_freeptr_offset = true,
 18		.freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr),
 19		.sheaf_capacity = 32,
 20	};
 21
 22	vm_area_cachep = kmem_cache_create("vm_area_struct",
 23			sizeof(struct vm_area_struct), &args,
 24			SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
 25			SLAB_ACCOUNT);
 26}
 27
 28struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
 29{
 30	struct vm_area_struct *vma;
 31
 32	vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
 33	if (!vma)
 34		return NULL;
 35
 36	vma_init(vma, mm);
 37
 38	return vma;
 39}
 40
 41static void vm_area_init_from(const struct vm_area_struct *src,
 42			      struct vm_area_struct *dest)
 43{
 44	dest->vm_mm = src->vm_mm;
 45	dest->vm_ops = src->vm_ops;
 46	dest->vm_start = src->vm_start;
 47	dest->vm_end = src->vm_end;
 48	dest->anon_vma = src->anon_vma;
 49	dest->vm_pgoff = src->vm_pgoff;
 50	dest->vm_file = src->vm_file;
 51	dest->vm_private_data = src->vm_private_data;
 52	vm_flags_init(dest, src->vm_flags);
 53	memcpy(&dest->vm_page_prot, &src->vm_page_prot,
 54	       sizeof(dest->vm_page_prot));
 55	/*
 56	 * src->shared.rb may be modified concurrently when called from
 57	 * dup_mmap(), but the clone will reinitialize it.
 58	 */
 59	data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared)));
 60	memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx,
 61	       sizeof(dest->vm_userfaultfd_ctx));
 62#ifdef CONFIG_ANON_VMA_NAME
 63	dest->anon_name = src->anon_name;
 64#endif
 65#ifdef CONFIG_SWAP
 66	memcpy(&dest->swap_readahead_info, &src->swap_readahead_info,
 67	       sizeof(dest->swap_readahead_info));
 68#endif
 69#ifndef CONFIG_MMU
 70	dest->vm_region = src->vm_region;
 71#endif
 72#ifdef CONFIG_NUMA
 73	dest->vm_policy = src->vm_policy;
 74#endif
 75#ifdef __HAVE_PFNMAP_TRACKING
 76	dest->pfnmap_track_ctx = NULL;
 77#endif
 78}
 79
 80#ifdef __HAVE_PFNMAP_TRACKING
 81static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
 82		struct vm_area_struct *new)
 83{
 84	struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx;
 85
 86	if (likely(!ctx))
 87		return 0;
 88
 89	/*
 90	 * We don't expect to ever hit this. If ever required, we would have
 91	 * to duplicate the tracking.
 92	 */
 93	if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX))
 94		return -ENOMEM;
 95	kref_get(&ctx->kref);
 96	new->pfnmap_track_ctx = ctx;
 97	return 0;
 98}
 99
100static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
101{
102	struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx;
103
104	if (likely(!ctx))
105		return;
106
107	kref_put(&ctx->kref, pfnmap_track_ctx_release);
108	vma->pfnmap_track_ctx = NULL;
109}
110#else
111static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
112		struct vm_area_struct *new)
113{
114	return 0;
115}
116static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
117{
118}
119#endif
120
121struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
122{
123	struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
124
125	if (!new)
126		return NULL;
127
128	ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
129	ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
130	vm_area_init_from(orig, new);
131
132	if (vma_pfnmap_track_ctx_dup(orig, new)) {
133		kmem_cache_free(vm_area_cachep, new);
134		return NULL;
135	}
136	vma_lock_init(new, true);
137	INIT_LIST_HEAD(&new->anon_vma_chain);
138	vma_numab_state_init(new);
139	dup_anon_vma_name(orig, new);
140
141	return new;
142}
143
144void vm_area_free(struct vm_area_struct *vma)
145{
146	/* The vma should be detached while being destroyed. */
147	vma_assert_detached(vma);
148	vma_numab_state_free(vma);
149	free_anon_vma_name(vma);
150	vma_pfnmap_track_ctx_release(vma);
151	kmem_cache_free(vm_area_cachep, vma);
152}