report.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * KFENCE reporting.
  4 *
  5 * Copyright (C) 2020, Google LLC.
  6 */
  7
  8#include <linux/stdarg.h>
  9
 10#include <linux/kernel.h>
 11#include <linux/lockdep.h>
 12#include <linux/math.h>
 13#include <linux/printk.h>
 14#include <linux/sched/debug.h>
 15#include <linux/seq_file.h>
 16#include <linux/sprintf.h>
 17#include <linux/stacktrace.h>
 18#include <linux/string.h>
 19#include <linux/string_choices.h>
 20#include <linux/sched/clock.h>
 21#include <trace/events/error_report.h>
 22
 23#include <asm/kfence.h>
 24
 25#include "kfence.h"
 26
 27/* May be overridden by <asm/kfence.h>. */
 28#ifndef ARCH_FUNC_PREFIX
 29#define ARCH_FUNC_PREFIX ""
 30#endif
 31
 32/* Helper function to either print to a seq_file or to console. */
 33__printf(2, 3)
 34static void seq_con_printf(struct seq_file *seq, const char *fmt, ...)
 35{
 36	va_list args;
 37
 38	va_start(args, fmt);
 39	if (seq)
 40		seq_vprintf(seq, fmt, args);
 41	else
 42		vprintk(fmt, args);
 43	va_end(args);
 44}
 45
 46/*
 47 * Get the number of stack entries to skip to get out of MM internals. @type is
 48 * optional, and if set to NULL, assumes an allocation or free stack.
 49 */
 50static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries,
 51			    const enum kfence_error_type *type)
 52{
 53	char buf[64];
 54	int skipnr, fallback = 0;
 55
 56	if (type) {
 57		/* Depending on error type, find different stack entries. */
 58		switch (*type) {
 59		case KFENCE_ERROR_UAF:
 60		case KFENCE_ERROR_OOB:
 61		case KFENCE_ERROR_INVALID:
 62			/*
 63			 * kfence_handle_page_fault() may be called with pt_regs
 64			 * set to NULL; in that case we'll simply show the full
 65			 * stack trace.
 66			 */
 67			return 0;
 68		case KFENCE_ERROR_CORRUPTION:
 69		case KFENCE_ERROR_INVALID_FREE:
 70			break;
 71		}
 72	}
 73
 74	for (skipnr = 0; skipnr < num_entries; skipnr++) {
 75		int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]);
 76
 77		if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") ||
 78		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") ||
 79		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||
 80		    !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) {
 81			/*
 82			 * In case of tail calls from any of the below to any of
 83			 * the above, optimized by the compiler such that the
 84			 * stack trace would omit the initial entry point below.
 85			 */
 86			fallback = skipnr + 1;
 87		}
 88
 89		/*
 90		 * The below list should only include the initial entry points
 91		 * into the slab allocators. Includes the *_bulk() variants by
 92		 * checking prefixes.
 93		 */
 94		if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||
 95		    str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||
 96		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||
 97		    str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))
 98			goto found;
 99	}
100	if (fallback < num_entries)
101		return fallback;
102found:
103	skipnr++;
104	return skipnr < num_entries ? skipnr : 0;
105}
106
107static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta,
108			       bool show_alloc)
109{
110	const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;
111	u64 ts_sec = track->ts_nsec;
112	unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC);
113	u64 interval_nsec = local_clock() - track->ts_nsec;
114	unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC);
115
116	/* Timestamp matches printk timestamp format. */
117	seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n",
118		       show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ?
119		       "rcu freeing" : "freed", track->pid,
120		       track->cpu, (unsigned long)ts_sec, rem_nsec / 1000,
121		       (unsigned long)interval_nsec, rem_interval_nsec / 1000);
122
123	if (track->num_stack_entries) {
124		/* Skip allocation/free internals stack. */
125		int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
126
127		/* stack_trace_seq_print() does not exist; open code our own. */
128		for (; i < track->num_stack_entries; i++)
129			seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]);
130	} else {
131		seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation");
132	}
133}
134
135void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta)
136{
137	const int size = abs(meta->size);
138	const unsigned long start = meta->addr;
139	const struct kmem_cache *const cache = meta->cache;
140
141	lockdep_assert_held(&meta->lock);
142
143	if (meta->state == KFENCE_OBJECT_UNUSED) {
144		seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata);
145		return;
146	}
147
148	seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n",
149		       meta - kfence_metadata, (void *)start, (void *)(start + size - 1),
150		       size, (cache && cache->name) ? cache->name : "<destroyed>");
151
152	kfence_print_stack(seq, meta, true);
153
154	if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) {
155		seq_con_printf(seq, "\n");
156		kfence_print_stack(seq, meta, false);
157	}
158}
159
160/*
161 * Show bytes at @addr that are different from the expected canary values, up to
162 * @max_bytes.
163 */
164static void print_diff_canary(unsigned long address, size_t bytes_to_show,
165			      const struct kfence_metadata *meta)
166{
167	const unsigned long show_until_addr = address + bytes_to_show;
168	const u8 *cur, *end;
169
170	/* Do not show contents of object nor read into following guard page. */
171	end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr)
172						: min(show_until_addr, PAGE_ALIGN(address)));
173
174	pr_cont("[");
175	for (cur = (const u8 *)address; cur < end; cur++) {
176		if (*cur == KFENCE_CANARY_PATTERN_U8(cur))
177			pr_cont(" .");
178		else if (no_hash_pointers)
179			pr_cont(" 0x%02x", *cur);
180		else /* Do not leak kernel memory in non-debug builds. */
181			pr_cont(" !");
182	}
183	pr_cont(" ]");
184}
185
186static const char *get_access_type(bool is_write)
187{
188	return str_write_read(is_write);
189}
190
191void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
192			 const struct kfence_metadata *meta, enum kfence_error_type type)
193{
194	unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 };
195	const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1;
196	int num_stack_entries;
197	int skipnr = 0;
198
199	if (regs) {
200		num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0);
201	} else {
202		num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1);
203		skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type);
204	}
205
206	/* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */
207	if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta))
208		return;
209
210	if (meta)
211		lockdep_assert_held(&meta->lock);
212	/*
213	 * Because we may generate reports in printk-unfriendly parts of the
214	 * kernel, such as scheduler code, the use of printk() could deadlock.
215	 * Until such time that all printing code here is safe in all parts of
216	 * the kernel, accept the risk, and just get our message out (given the
217	 * system might already behave unpredictably due to the memory error).
218	 * As such, also disable lockdep to hide warnings, and avoid disabling
219	 * lockdep for the rest of the kernel.
220	 */
221	lockdep_off();
222
223	pr_err("==================================================================\n");
224	/* Print report header. */
225	switch (type) {
226	case KFENCE_ERROR_OOB: {
227		const bool left_of_object = address < meta->addr;
228
229		pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write),
230		       (void *)stack_entries[skipnr]);
231		pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n",
232		       get_access_type(is_write), (void *)address,
233		       left_of_object ? meta->addr - address : address - meta->addr,
234		       left_of_object ? "left" : "right", object_index);
235		break;
236	}
237	case KFENCE_ERROR_UAF:
238		pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write),
239		       (void *)stack_entries[skipnr]);
240		pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n",
241		       get_access_type(is_write), (void *)address, object_index);
242		break;
243	case KFENCE_ERROR_CORRUPTION:
244		pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]);
245		pr_err("Corrupted memory at 0x%p ", (void *)address);
246		print_diff_canary(address, 16, meta);
247		pr_cont(" (in kfence-#%td):\n", object_index);
248		break;
249	case KFENCE_ERROR_INVALID:
250		pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write),
251		       (void *)stack_entries[skipnr]);
252		pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write),
253		       (void *)address);
254		break;
255	case KFENCE_ERROR_INVALID_FREE:
256		pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]);
257		pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address,
258		       object_index);
259		break;
260	}
261
262	/* Print stack trace and object info. */
263	stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0);
264
265	if (meta) {
266		pr_err("\n");
267		kfence_print_object(NULL, meta);
268	}
269
270	/* Print report footer. */
271	pr_err("\n");
272	if (no_hash_pointers && regs)
273		show_regs(regs);
274	else
275		dump_stack_print_info(KERN_ERR);
276	trace_error_report_end(ERROR_DETECTOR_KFENCE, address);
277	pr_err("==================================================================\n");
278
279	lockdep_on();
280
281	check_panic_on_warn("KFENCE");
282
283	/* We encountered a memory safety error, taint the kernel! */
284	add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK);
285}
286
287#ifdef CONFIG_PRINTK
288static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack)
289{
290	int i, j;
291
292	i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
293	for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j)
294		kp_stack[j] = (void *)track->stack_entries[i];
295	if (j < KS_ADDRS_COUNT)
296		kp_stack[j] = NULL;
297}
298
299bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
300{
301	struct kfence_metadata *meta = addr_to_metadata((unsigned long)object);
302	unsigned long flags;
303
304	if (!meta)
305		return false;
306
307	/*
308	 * If state is UNUSED at least show the pointer requested; the rest
309	 * would be garbage data.
310	 */
311	kpp->kp_ptr = object;
312
313	/* Requesting info an a never-used object is almost certainly a bug. */
314	if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED))
315		return true;
316
317	raw_spin_lock_irqsave(&meta->lock, flags);
318
319	kpp->kp_slab = slab;
320	kpp->kp_slab_cache = meta->cache;
321	kpp->kp_objp = (void *)meta->addr;
322	kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack);
323	if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING)
324		kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack);
325	/* get_stack_skipnr() ensures the first entry is outside allocator. */
326	kpp->kp_ret = kpp->kp_stack[0];
327
328	raw_spin_unlock_irqrestore(&meta->lock, flags);
329
330	return true;
331}
332#endif