stacktrace.c

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * KVM nVHE hypervisor stack tracing support.
  4 *
  5 * The unwinder implementation depends on the nVHE mode:
  6 *
  7 *   1) Non-protected nVHE mode - the host can directly access the
  8 *      HYP stack pages and unwind the HYP stack in EL1. This saves having
  9 *      to allocate shared buffers for the host to read the unwinded
 10 *      stacktrace.
 11 *
 12 *   2) pKVM (protected nVHE) mode - the host cannot directly access
 13 *      the HYP memory. The stack is unwinded in EL2 and dumped to a shared
 14 *      buffer where the host can read and print the stacktrace.
 15 *
 16 * Copyright (C) 2022 Google LLC
 17 */
 18
 19#include <linux/kvm.h>
 20#include <linux/kvm_host.h>
 21
 22#include <asm/kvm_mmu.h>
 23#include <asm/stacktrace/nvhe.h>
 24
 25static struct stack_info stackinfo_get_overflow(void)
 26{
 27	struct kvm_nvhe_stacktrace_info *stacktrace_info
 28				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
 29	unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
 30	unsigned long high = low + OVERFLOW_STACK_SIZE;
 31
 32	return (struct stack_info) {
 33		.low = low,
 34		.high = high,
 35	};
 36}
 37
 38static struct stack_info stackinfo_get_overflow_kern_va(void)
 39{
 40	unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
 41	unsigned long high = low + OVERFLOW_STACK_SIZE;
 42
 43	return (struct stack_info) {
 44		.low = low,
 45		.high = high,
 46	};
 47}
 48
 49static struct stack_info stackinfo_get_hyp(void)
 50{
 51	struct kvm_nvhe_stacktrace_info *stacktrace_info
 52				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
 53	unsigned long low = (unsigned long)stacktrace_info->stack_base;
 54	unsigned long high = low + NVHE_STACK_SIZE;
 55
 56	return (struct stack_info) {
 57		.low = low,
 58		.high = high,
 59	};
 60}
 61
 62static struct stack_info stackinfo_get_hyp_kern_va(void)
 63{
 64	unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_base);
 65	unsigned long high = low + NVHE_STACK_SIZE;
 66
 67	return (struct stack_info) {
 68		.low = low,
 69		.high = high,
 70	};
 71}
 72
 73/*
 74 * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
 75 *
 76 * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to
 77 * allow for guard pages below the stack. Consequently, the fixed offset address
 78 * translation macros won't work here.
 79 *
 80 * The kernel VA is calculated as an offset from the kernel VA of the hypervisor
 81 * stack base.
 82 *
 83 * Returns true on success and updates @addr to its corresponding kernel VA;
 84 * otherwise returns false.
 85 */
 86static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
 87{
 88	struct stack_info stack_hyp, stack_kern;
 89
 90	stack_hyp = stackinfo_get_hyp();
 91	stack_kern = stackinfo_get_hyp_kern_va();
 92	if (stackinfo_on_stack(&stack_hyp, *addr, size))
 93		goto found;
 94
 95	stack_hyp = stackinfo_get_overflow();
 96	stack_kern = stackinfo_get_overflow_kern_va();
 97	if (stackinfo_on_stack(&stack_hyp, *addr, size))
 98		goto found;
 99
100	return false;
101
102found:
103	*addr = *addr - stack_hyp.low + stack_kern.low;
104	return true;
105}
106
107/*
108 * Convert a KVN nVHE HYP frame record address to a kernel VA
109 */
110static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
111{
112	return kvm_nvhe_stack_kern_va(addr, 16);
113}
114
115static int unwind_next(struct unwind_state *state)
116{
117	/*
118	 * The FP is in the hypervisor VA space. Convert it to the kernel VA
119	 * space so it can be unwound by the regular unwind functions.
120	 */
121	if (!kvm_nvhe_stack_kern_record_va(&state->fp))
122		return -EINVAL;
123
124	return unwind_next_frame_record(state);
125}
126
127static void unwind(struct unwind_state *state,
128		   stack_trace_consume_fn consume_entry, void *cookie)
129{
130	while (1) {
131		int ret;
132
133		if (!consume_entry(cookie, state->pc))
134			break;
135		ret = unwind_next(state);
136		if (ret < 0)
137			break;
138	}
139}
140
141/*
142 * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry
143 *
144 * @arg    : the hypervisor offset, used for address translation
145 * @where  : the program counter corresponding to the stack frame
146 */
147static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where)
148{
149	unsigned long va_mask = GENMASK_ULL(__hyp_va_bits - 1, 0);
150	unsigned long hyp_offset = (unsigned long)arg;
151
152	/* Mask tags and convert to kern addr */
153	where = (where & va_mask) + hyp_offset;
154	kvm_err(" [<%016lx>] %pB\n", where, (void *)(where + kaslr_offset()));
155
156	return true;
157}
158
159static void kvm_nvhe_dump_backtrace_start(void)
160{
161	kvm_err("nVHE call trace:\n");
162}
163
164static void kvm_nvhe_dump_backtrace_end(void)
165{
166	kvm_err("---[ end nVHE call trace ]---\n");
167}
168
169/*
170 * hyp_dump_backtrace - Dump the non-protected nVHE backtrace.
171 *
172 * @hyp_offset: hypervisor offset, used for address translation.
173 *
174 * The host can directly access HYP stack pages in non-protected
175 * mode, so the unwinding is done directly from EL1. This removes
176 * the need for shared buffers between host and hypervisor for
177 * the stacktrace.
178 */
179static void hyp_dump_backtrace(unsigned long hyp_offset)
180{
181	struct kvm_nvhe_stacktrace_info *stacktrace_info;
182	struct stack_info stacks[] = {
183		stackinfo_get_overflow_kern_va(),
184		stackinfo_get_hyp_kern_va(),
185	};
186	struct unwind_state state = {
187		.stacks = stacks,
188		.nr_stacks = ARRAY_SIZE(stacks),
189	};
190
191	stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
192
193	kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc);
194
195	kvm_nvhe_dump_backtrace_start();
196	unwind(&state, kvm_nvhe_dump_backtrace_entry, (void *)hyp_offset);
197	kvm_nvhe_dump_backtrace_end();
198}
199
200#ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
201DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
202			 pkvm_stacktrace);
203
204/*
205 * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
206 *
207 * @hyp_offset: hypervisor offset, used for address translation.
208 *
209 * Dumping of the pKVM HYP backtrace is done by reading the
210 * stack addresses from the shared stacktrace buffer, since the
211 * host cannot directly access hypervisor memory in protected
212 * mode.
213 */
214static void pkvm_dump_backtrace(unsigned long hyp_offset)
215{
216	unsigned long *stacktrace
217		= (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace);
218	int i;
219
220	kvm_nvhe_dump_backtrace_start();
221	/* The saved stacktrace is terminated by a null entry */
222	for (i = 0;
223	     i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i];
224	     i++)
225		kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
226	kvm_nvhe_dump_backtrace_end();
227}
228#else	/* !CONFIG_PROTECTED_NVHE_STACKTRACE */
229static void pkvm_dump_backtrace(unsigned long hyp_offset)
230{
231	kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n");
232}
233#endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
234
235/*
236 * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
237 *
238 * @hyp_offset: hypervisor offset, used for address translation.
239 */
240void kvm_nvhe_dump_backtrace(unsigned long hyp_offset)
241{
242	if (is_protected_kvm_enabled())
243		pkvm_dump_backtrace(hyp_offset);
244	else
245		hyp_dump_backtrace(hyp_offset);
246}