internal.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/* Module internals
  3 *
  4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
  5 * Written by David Howells (dhowells@redhat.com)
  6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
  7 */
  8
  9#include <linux/elf.h>
 10#include <linux/compiler.h>
 11#include <linux/module.h>
 12#include <linux/mutex.h>
 13#include <linux/rculist.h>
 14#include <linux/rcupdate.h>
 15#include <linux/mm.h>
 16
 17#ifndef ARCH_SHF_SMALL
 18#define ARCH_SHF_SMALL 0
 19#endif
 20
 21/*
 22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
 23 * section. This leaves 28 bits for offset on 32-bit systems, which is
 24 * about 256 MiB (WARN_ON_ONCE if we exceed that).
 25 */
 26
 27#define SH_ENTSIZE_TYPE_BITS	4
 28#define SH_ENTSIZE_TYPE_SHIFT	(BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
 29#define SH_ENTSIZE_TYPE_MASK	((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
 30#define SH_ENTSIZE_OFFSET_MASK	((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
 31
 32/* Maximum number of characters written by module_flags() */
 33#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
 34
 35struct kernel_symbol {
 36#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
 37	int value_offset;
 38	int name_offset;
 39	int namespace_offset;
 40#else
 41	unsigned long value;
 42	const char *name;
 43	const char *namespace;
 44#endif
 45};
 46
 47extern struct mutex module_mutex;
 48extern struct list_head modules;
 49
 50extern const struct module_attribute *const modinfo_attrs[];
 51extern const size_t modinfo_attrs_count;
 52
 53/* Provided by the linker */
 54extern const struct kernel_symbol __start___ksymtab[];
 55extern const struct kernel_symbol __stop___ksymtab[];
 56extern const struct kernel_symbol __start___ksymtab_gpl[];
 57extern const struct kernel_symbol __stop___ksymtab_gpl[];
 58extern const u32 __start___kcrctab[];
 59extern const u32 __start___kcrctab_gpl[];
 60
 61#define KMOD_PATH_LEN 256
 62extern char modprobe_path[];
 63
 64struct load_info {
 65	const char *name;
 66	/* pointer to module in temporary copy, freed at end of load_module() */
 67	struct module *mod;
 68	Elf_Ehdr *hdr;
 69	unsigned long len;
 70	Elf_Shdr *sechdrs;
 71	char *secstrings, *strtab;
 72	unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
 73	bool sig_ok;
 74#ifdef CONFIG_KALLSYMS
 75	unsigned long mod_kallsyms_init_off;
 76#endif
 77#ifdef CONFIG_MODULE_DECOMPRESS
 78#ifdef CONFIG_MODULE_STATS
 79	unsigned long compressed_len;
 80#endif
 81	struct page **pages;
 82	unsigned int max_pages;
 83	unsigned int used_pages;
 84#endif
 85	struct {
 86		unsigned int sym;
 87		unsigned int str;
 88		unsigned int mod;
 89		unsigned int vers;
 90		unsigned int info;
 91		unsigned int pcpu;
 92		unsigned int vers_ext_crc;
 93		unsigned int vers_ext_name;
 94	} index;
 95};
 96
 97enum mod_license {
 98	NOT_GPL_ONLY,
 99	GPL_ONLY,
100};
101
102struct find_symbol_arg {
103	/* Input */
104	const char *name;
105	bool gplok;
106	bool warn;
107
108	/* Output */
109	struct module *owner;
110	const u32 *crc;
111	const struct kernel_symbol *sym;
112	enum mod_license license;
113};
114
115/* modules using other modules */
116struct module_use {
117	struct list_head source_list;
118	struct list_head target_list;
119	struct module *source, *target;
120};
121
122int mod_verify_sig(const void *mod, struct load_info *info);
123int try_to_force_load(struct module *mod, const char *reason);
124bool find_symbol(struct find_symbol_arg *fsa);
125struct module *find_module_all(const char *name, size_t len, bool even_unformed);
126int cmp_name(const void *name, const void *sym);
127long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
128				Elf_Shdr *sechdr, unsigned int section);
129char *module_flags(struct module *mod, char *buf, bool show_state);
130size_t module_flags_taint(unsigned long taints, char *buf);
131
132char *module_next_tag_pair(char *string, unsigned long *secsize);
133
134#define for_each_modinfo_entry(entry, info, name) \
135	for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
136
137static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
138{
139#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
140	return (unsigned long)offset_to_ptr(&sym->value_offset);
141#else
142	return sym->value;
143#endif
144}
145
146#ifdef CONFIG_LIVEPATCH
147int copy_module_elf(struct module *mod, struct load_info *info);
148void free_module_elf(struct module *mod);
149#else /* !CONFIG_LIVEPATCH */
150static inline int copy_module_elf(struct module *mod, struct load_info *info)
151{
152	return 0;
153}
154
155static inline void free_module_elf(struct module *mod) { }
156#endif /* CONFIG_LIVEPATCH */
157
158static inline bool set_livepatch_module(struct module *mod)
159{
160#ifdef CONFIG_LIVEPATCH
161	mod->klp = true;
162	return true;
163#else
164	return false;
165#endif
166}
167
168/**
169 * enum fail_dup_mod_reason - state at which a duplicate module was detected
170 *
171 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
172 * 	we've determined that another module with the same name is already loaded
173 * 	or being processed on our &modules list. This happens on early_mod_check()
174 * 	right before layout_and_allocate(). The kernel would have already
175 * 	vmalloc()'d space for the entire module through finit_module(). If
176 * 	decompression was used two vmap() spaces were used. These failures can
177 * 	happen when userspace has not seen the module present on the kernel and
178 * 	tries to load the module multiple times at same time.
179 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
180 *	checks and the kernel determines that the module was unique and because
181 *	of this allocated yet another private kernel copy of the module space in
182 *	layout_and_allocate() but after this determined in add_unformed_module()
183 *	that another module with the same name is already loaded or being processed.
184 *	These failures should be mitigated as much as possible and are indicative
185 *	of really fast races in loading modules. Without module decompression
186 *	they waste twice as much vmap space. With module decompression three
187 *	times the module's size vmap space is wasted.
188 */
189enum fail_dup_mod_reason {
190	FAIL_DUP_MOD_BECOMING = 0,
191	FAIL_DUP_MOD_LOAD,
192};
193
194#ifdef CONFIG_MODULE_DEBUGFS
195extern struct dentry *mod_debugfs_root;
196#endif
197
198#ifdef CONFIG_MODULE_STATS
199
200#define mod_stat_add_long(count, var) atomic_long_add(count, var)
201#define mod_stat_inc(name) atomic_inc(name)
202
203extern atomic_long_t total_mod_size;
204extern atomic_long_t total_text_size;
205extern atomic_long_t invalid_kread_bytes;
206extern atomic_long_t invalid_decompress_bytes;
207
208extern atomic_t modcount;
209extern atomic_t failed_kreads;
210extern atomic_t failed_decompress;
211struct mod_fail_load {
212	struct list_head list;
213	char name[MODULE_NAME_LEN];
214	atomic_long_t count;
215	unsigned long dup_fail_mask;
216};
217
218int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
219void mod_stat_bump_invalid(struct load_info *info, int flags);
220void mod_stat_bump_becoming(struct load_info *info, int flags);
221
222#else
223
224#define mod_stat_add_long(name, var)
225#define mod_stat_inc(name)
226
227static inline int try_add_failed_module(const char *name,
228					enum fail_dup_mod_reason reason)
229{
230	return 0;
231}
232
233static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
234{
235}
236
237static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
238{
239}
240
241#endif /* CONFIG_MODULE_STATS */
242
243#ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
244bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
245void kmod_dup_request_announce(char *module_name, int ret);
246#else
247static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
248{
249	return false;
250}
251
252static inline void kmod_dup_request_announce(char *module_name, int ret)
253{
254}
255#endif
256
257#ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
258struct mod_unload_taint {
259	struct list_head list;
260	char name[MODULE_NAME_LEN];
261	unsigned long taints;
262	u64 count;
263};
264
265int try_add_tainted_module(struct module *mod);
266void print_unloaded_tainted_modules(void);
267#else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
268static inline int try_add_tainted_module(struct module *mod)
269{
270	return 0;
271}
272
273static inline void print_unloaded_tainted_modules(void)
274{
275}
276#endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
277
278#ifdef CONFIG_MODULE_DECOMPRESS
279int module_decompress(struct load_info *info, const void *buf, size_t size);
280void module_decompress_cleanup(struct load_info *info);
281#else
282static inline int module_decompress(struct load_info *info,
283				    const void *buf, size_t size)
284{
285	return -EOPNOTSUPP;
286}
287
288static inline void module_decompress_cleanup(struct load_info *info)
289{
290}
291#endif
292
293struct mod_tree_root {
294#ifdef CONFIG_MODULES_TREE_LOOKUP
295	struct latch_tree_root root;
296#endif
297	unsigned long addr_min;
298	unsigned long addr_max;
299#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
300	unsigned long data_addr_min;
301	unsigned long data_addr_max;
302#endif
303};
304
305extern struct mod_tree_root mod_tree;
306
307#ifdef CONFIG_MODULES_TREE_LOOKUP
308void mod_tree_insert(struct module *mod);
309void mod_tree_remove_init(struct module *mod);
310void mod_tree_remove(struct module *mod);
311struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
312#else /* !CONFIG_MODULES_TREE_LOOKUP */
313
314static inline void mod_tree_insert(struct module *mod) { }
315static inline void mod_tree_remove_init(struct module *mod) { }
316static inline void mod_tree_remove(struct module *mod) { }
317static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
318{
319	struct module *mod;
320
321	list_for_each_entry_rcu(mod, &modules, list,
322				lockdep_is_held(&module_mutex)) {
323		if (within_module(addr, mod))
324			return mod;
325	}
326
327	return NULL;
328}
329#endif /* CONFIG_MODULES_TREE_LOOKUP */
330
331int module_enable_rodata_ro(const struct module *mod);
332int module_enable_rodata_ro_after_init(const struct module *mod);
333int module_enable_data_nx(const struct module *mod);
334int module_enable_text_rox(const struct module *mod);
335int module_enforce_rwx_sections(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
336				const char *secstrings,
337				const struct module *mod);
338void module_mark_ro_after_init(const Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
339			       const char *secstrings);
340
341#ifdef CONFIG_MODULE_SIG
342int module_sig_check(struct load_info *info, int flags);
343#else /* !CONFIG_MODULE_SIG */
344static inline int module_sig_check(struct load_info *info, int flags)
345{
346	return 0;
347}
348#endif /* !CONFIG_MODULE_SIG */
349
350#ifdef CONFIG_DEBUG_KMEMLEAK
351void kmemleak_load_module(const struct module *mod, const struct load_info *info);
352#else /* !CONFIG_DEBUG_KMEMLEAK */
353static inline void kmemleak_load_module(const struct module *mod,
354					const struct load_info *info) { }
355#endif /* CONFIG_DEBUG_KMEMLEAK */
356
357#ifdef CONFIG_KALLSYMS
358void init_build_id(struct module *mod, const struct load_info *info);
359void layout_symtab(struct module *mod, struct load_info *info);
360void add_kallsyms(struct module *mod, const struct load_info *info);
361
362static inline bool sect_empty(const Elf_Shdr *sect)
363{
364	return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
365}
366#else /* !CONFIG_KALLSYMS */
367static inline void init_build_id(struct module *mod, const struct load_info *info) { }
368static inline void layout_symtab(struct module *mod, struct load_info *info) { }
369static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
370#endif /* CONFIG_KALLSYMS */
371
372#ifdef CONFIG_SYSFS
373int mod_sysfs_setup(struct module *mod, const struct load_info *info,
374		    struct kernel_param *kparam, unsigned int num_params);
375void mod_sysfs_teardown(struct module *mod);
376void init_param_lock(struct module *mod);
377#else /* !CONFIG_SYSFS */
378static inline int mod_sysfs_setup(struct module *mod,
379			   	  const struct load_info *info,
380			   	  struct kernel_param *kparam,
381			   	  unsigned int num_params)
382{
383	return 0;
384}
385
386static inline void mod_sysfs_teardown(struct module *mod) { }
387static inline void init_param_lock(struct module *mod) { }
388#endif /* CONFIG_SYSFS */
389
390#ifdef CONFIG_MODVERSIONS
391int check_version(const struct load_info *info,
392		  const char *symname, struct module *mod, const u32 *crc);
393void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
394		   struct kernel_symbol *ks, struct tracepoint * const *tp);
395int check_modstruct_version(const struct load_info *info, struct module *mod);
396int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
397struct modversion_info_ext {
398	size_t remaining;
399	const u32 *crc;
400	const char *name;
401};
402void modversion_ext_start(const struct load_info *info, struct modversion_info_ext *ver);
403void modversion_ext_advance(struct modversion_info_ext *ver);
404#define for_each_modversion_info_ext(ver, info) \
405	for (modversion_ext_start(info, &ver); ver.remaining > 0; modversion_ext_advance(&ver))
406#else /* !CONFIG_MODVERSIONS */
407static inline int check_version(const struct load_info *info,
408				const char *symname,
409				struct module *mod,
410				const u32 *crc)
411{
412	return 1;
413}
414
415static inline int check_modstruct_version(const struct load_info *info,
416					  struct module *mod)
417{
418	return 1;
419}
420
421static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
422{
423	return strcmp(amagic, bmagic) == 0;
424}
425#endif /* CONFIG_MODVERSIONS */