hw_tags.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * This file contains core hardware tag-based KASAN code.
  4 *
  5 * Copyright (c) 2020 Google, Inc.
  6 * Author: Andrey Konovalov <andreyknvl@google.com>
  7 */
  8
  9#define pr_fmt(fmt) "kasan: " fmt
 10
 11#include <kunit/visibility.h>
 12#include <linux/init.h>
 13#include <linux/kasan.h>
 14#include <linux/kernel.h>
 15#include <linux/memory.h>
 16#include <linux/mm.h>
 17#include <linux/static_key.h>
 18#include <linux/string.h>
 19#include <linux/string_choices.h>
 20#include <linux/types.h>
 21#include <linux/vmalloc.h>
 22
 23#include "kasan.h"
 24
 25enum kasan_arg {
 26	KASAN_ARG_DEFAULT,
 27	KASAN_ARG_OFF,
 28	KASAN_ARG_ON,
 29};
 30
 31enum kasan_arg_mode {
 32	KASAN_ARG_MODE_DEFAULT,
 33	KASAN_ARG_MODE_SYNC,
 34	KASAN_ARG_MODE_ASYNC,
 35	KASAN_ARG_MODE_ASYMM,
 36};
 37
 38enum kasan_arg_vmalloc {
 39	KASAN_ARG_VMALLOC_DEFAULT,
 40	KASAN_ARG_VMALLOC_OFF,
 41	KASAN_ARG_VMALLOC_ON,
 42};
 43
 44static enum kasan_arg kasan_arg __ro_after_init;
 45static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
 46static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
 47
 48/*
 49 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
 50 * Defaults to KASAN_MODE_SYNC.
 51 */
 52enum kasan_mode kasan_mode __ro_after_init;
 53EXPORT_SYMBOL_GPL(kasan_mode);
 54
 55/* Whether to enable vmalloc tagging. */
 56#ifdef CONFIG_KASAN_VMALLOC
 57DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
 58#else
 59DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc);
 60#endif
 61EXPORT_SYMBOL_GPL(kasan_flag_vmalloc);
 62
 63/* Whether to check write accesses only. */
 64static bool kasan_flag_write_only = false;
 65
 66#define PAGE_ALLOC_SAMPLE_DEFAULT	1
 67#define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT	3
 68
 69/*
 70 * Sampling interval of page_alloc allocation (un)poisoning.
 71 * Defaults to no sampling.
 72 */
 73unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
 74
 75/*
 76 * Minimum order of page_alloc allocations to be affected by sampling.
 77 * The default value is chosen to match both
 78 * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
 79 */
 80unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
 81
 82DEFINE_PER_CPU(long, kasan_page_alloc_skip);
 83
 84/* kasan=off/on */
 85static int __init early_kasan_flag(char *arg)
 86{
 87	if (!arg)
 88		return -EINVAL;
 89
 90	if (!strcmp(arg, "off"))
 91		kasan_arg = KASAN_ARG_OFF;
 92	else if (!strcmp(arg, "on"))
 93		kasan_arg = KASAN_ARG_ON;
 94	else
 95		return -EINVAL;
 96
 97	return 0;
 98}
 99early_param("kasan", early_kasan_flag);
100
101/* kasan.mode=sync/async/asymm */
102static int __init early_kasan_mode(char *arg)
103{
104	if (!arg)
105		return -EINVAL;
106
107	if (!strcmp(arg, "sync"))
108		kasan_arg_mode = KASAN_ARG_MODE_SYNC;
109	else if (!strcmp(arg, "async"))
110		kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
111	else if (!strcmp(arg, "asymm"))
112		kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
113	else
114		return -EINVAL;
115
116	return 0;
117}
118early_param("kasan.mode", early_kasan_mode);
119
120/* kasan.vmalloc=off/on */
121static int __init early_kasan_flag_vmalloc(char *arg)
122{
123	if (!arg)
124		return -EINVAL;
125
126	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
127		return 0;
128
129	if (!strcmp(arg, "off"))
130		kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
131	else if (!strcmp(arg, "on"))
132		kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
133	else
134		return -EINVAL;
135
136	return 0;
137}
138early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
139
140/* kasan.write_only=off/on */
141static int __init early_kasan_flag_write_only(char *arg)
142{
143	if (!arg)
144		return -EINVAL;
145
146	if (!strcmp(arg, "off"))
147		kasan_flag_write_only = false;
148	else if (!strcmp(arg, "on"))
149		kasan_flag_write_only = true;
150	else
151		return -EINVAL;
152
153	return 0;
154}
155early_param("kasan.write_only", early_kasan_flag_write_only);
156
157static inline const char *kasan_mode_info(void)
158{
159	if (kasan_mode == KASAN_MODE_ASYNC)
160		return "async";
161	else if (kasan_mode == KASAN_MODE_ASYMM)
162		return "asymm";
163	else
164		return "sync";
165}
166
167/* kasan.page_alloc.sample=<sampling interval> */
168static int __init early_kasan_flag_page_alloc_sample(char *arg)
169{
170	int rv;
171
172	if (!arg)
173		return -EINVAL;
174
175	rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
176	if (rv)
177		return rv;
178
179	if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
180		kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
181		return -EINVAL;
182	}
183
184	return 0;
185}
186early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
187
188/* kasan.page_alloc.sample.order=<minimum page order> */
189static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
190{
191	int rv;
192
193	if (!arg)
194		return -EINVAL;
195
196	rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
197	if (rv)
198		return rv;
199
200	if (kasan_page_alloc_sample_order > INT_MAX) {
201		kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
202		return -EINVAL;
203	}
204
205	return 0;
206}
207early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
208
209/*
210 * kasan_init_hw_tags_cpu() is called for each CPU.
211 * Not marked as __init as a CPU can be hot-plugged after boot.
212 */
213void kasan_init_hw_tags_cpu(void)
214{
215	/*
216	 * There's no need to check that the hardware is MTE-capable here,
217	 * as this function is only called for MTE-capable hardware.
218	 */
219
220	/*
221	 * If KASAN is disabled via command line, don't initialize it.
222	 * When this function is called, kasan_flag_enabled is not yet
223	 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
224	 */
225	if (kasan_arg == KASAN_ARG_OFF)
226		return;
227
228	/*
229	 * Enable async or asymm modes only when explicitly requested
230	 * through the command line.
231	 */
232	kasan_enable_hw_tags();
233}
234
235/* kasan_init_hw_tags() is called once on boot CPU. */
236void __init kasan_init_hw_tags(void)
237{
238	/* If hardware doesn't support MTE, don't initialize KASAN. */
239	if (!system_supports_mte())
240		return;
241
242	/* If KASAN is disabled via command line, don't initialize it. */
243	if (kasan_arg == KASAN_ARG_OFF)
244		return;
245
246	switch (kasan_arg_mode) {
247	case KASAN_ARG_MODE_DEFAULT:
248		/* Default is specified by kasan_mode definition. */
249		break;
250	case KASAN_ARG_MODE_SYNC:
251		kasan_mode = KASAN_MODE_SYNC;
252		break;
253	case KASAN_ARG_MODE_ASYNC:
254		kasan_mode = KASAN_MODE_ASYNC;
255		break;
256	case KASAN_ARG_MODE_ASYMM:
257		kasan_mode = KASAN_MODE_ASYMM;
258		break;
259	}
260
261	switch (kasan_arg_vmalloc) {
262	case KASAN_ARG_VMALLOC_DEFAULT:
263		/* Default is specified by kasan_flag_vmalloc definition. */
264		break;
265	case KASAN_ARG_VMALLOC_OFF:
266		static_branch_disable(&kasan_flag_vmalloc);
267		break;
268	case KASAN_ARG_VMALLOC_ON:
269		static_branch_enable(&kasan_flag_vmalloc);
270		break;
271	}
272
273	kasan_init_tags();
274
275	/* KASAN is now initialized, enable it. */
276	kasan_enable();
277
278	pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s, write_only=%s)\n",
279		kasan_mode_info(),
280		str_on_off(kasan_vmalloc_enabled()),
281		str_on_off(kasan_stack_collection_enabled()),
282		str_on_off(kasan_flag_write_only));
283}
284
285#ifdef CONFIG_KASAN_VMALLOC
286
287static void unpoison_vmalloc_pages(const void *addr, u8 tag)
288{
289	struct vm_struct *area;
290	int i;
291
292	/*
293	 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
294	 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
295	 * should belong to a single area.
296	 */
297	area = find_vm_area((void *)addr);
298	if (WARN_ON(!area))
299		return;
300
301	for (i = 0; i < area->nr_pages; i++) {
302		struct page *page = area->pages[i];
303
304		page_kasan_tag_set(page, tag);
305	}
306}
307
308static void init_vmalloc_pages(const void *start, unsigned long size)
309{
310	const void *addr;
311
312	for (addr = start; addr < start + size; addr += PAGE_SIZE) {
313		struct page *page = vmalloc_to_page(addr);
314
315		clear_highpage_kasan_tagged(page);
316	}
317}
318
319void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
320				kasan_vmalloc_flags_t flags)
321{
322	u8 tag;
323	unsigned long redzone_start, redzone_size;
324
325	if (!kasan_vmalloc_enabled()) {
326		if (flags & KASAN_VMALLOC_INIT)
327			init_vmalloc_pages(start, size);
328		return (void *)start;
329	}
330
331	/*
332	 * Don't tag non-VM_ALLOC mappings, as:
333	 *
334	 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
335	 *    supports tagging physical memory. Therefore, it can only tag a
336	 *    single mapping of normal physical pages.
337	 * 2. Hardware tag-based KASAN can only tag memory mapped with special
338	 *    mapping protection bits, see arch_vmap_pgprot_tagged().
339	 *    As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
340	 *    providing these bits would require tracking all non-VM_ALLOC
341	 *    mappers.
342	 *
343	 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
344	 * the first virtual mapping, which is created by vmalloc().
345	 * Tagging the page_alloc memory backing that vmalloc() allocation is
346	 * skipped, see ___GFP_SKIP_KASAN.
347	 *
348	 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
349	 */
350	if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
351		WARN_ON(flags & KASAN_VMALLOC_INIT);
352		return (void *)start;
353	}
354
355	/*
356	 * Don't tag executable memory.
357	 * The kernel doesn't tolerate having the PC register tagged.
358	 */
359	if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
360		WARN_ON(flags & KASAN_VMALLOC_INIT);
361		return (void *)start;
362	}
363
364	tag = (flags & KASAN_VMALLOC_KEEP_TAG) ? get_tag(start) : kasan_random_tag();
365	start = set_tag(start, tag);
366
367	/* Unpoison and initialize memory up to size. */
368	kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
369
370	/*
371	 * Explicitly poison and initialize the in-page vmalloc() redzone.
372	 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
373	 * unpoison memory when populating shadow for vmalloc() space.
374	 */
375	redzone_start = round_up((unsigned long)start + size,
376				 KASAN_GRANULE_SIZE);
377	redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
378	kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
379		     flags & KASAN_VMALLOC_INIT);
380
381	/*
382	 * Set per-page tag flags to allow accessing physical memory for the
383	 * vmalloc() mapping through page_address(vmalloc_to_page()).
384	 */
385	unpoison_vmalloc_pages(start, tag);
386
387	return (void *)start;
388}
389
390void __kasan_poison_vmalloc(const void *start, unsigned long size)
391{
392	/*
393	 * No tagging here.
394	 * The physical pages backing the vmalloc() allocation are poisoned
395	 * through the usual page_alloc paths.
396	 */
397}
398
399#endif
400
401void kasan_enable_hw_tags(void)
402{
403	if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
404		hw_enable_tag_checks_async();
405	else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
406		hw_enable_tag_checks_asymm();
407	else
408		hw_enable_tag_checks_sync();
409
410	/*
411	 * CPUs can only be in one of two states:
412	 *   - All CPUs support the write_only feature
413	 *   - No CPUs support the write_only feature
414	 *
415	 * If the first CPU attempts hw_enable_tag_checks_write_only() and
416	 * finds the feature unsupported, kasan_flag_write_only is set to OFF
417	 * to avoid further unnecessary calls on other CPUs.
418	 */
419	if (kasan_flag_write_only && hw_enable_tag_checks_write_only()) {
420		kasan_flag_write_only = false;
421		pr_err_once("write-only mode is not supported and thus not enabled\n");
422	}
423}
424
425#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
426
427EXPORT_SYMBOL_IF_KUNIT(kasan_enable_hw_tags);
428
429VISIBLE_IF_KUNIT void kasan_force_async_fault(void)
430{
431	hw_force_async_tag_fault();
432}
433EXPORT_SYMBOL_IF_KUNIT(kasan_force_async_fault);
434
435VISIBLE_IF_KUNIT bool kasan_write_only_enabled(void)
436{
437	return kasan_flag_write_only;
438}
439EXPORT_SYMBOL_IF_KUNIT(kasan_write_only_enabled);
440
441#endif