unwind_orc.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2#include <linux/objtool.h>
  3#include <linux/module.h>
  4#include <linux/sort.h>
  5#include <linux/bpf.h>
  6#include <asm/ptrace.h>
  7#include <asm/stacktrace.h>
  8#include <asm/unwind.h>
  9#include <asm/orc_types.h>
 10#include <asm/orc_lookup.h>
 11#include <asm/orc_header.h>
 12
 13ORC_HEADER;
 14
 15#define orc_warn(fmt, ...) \
 16	printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
 17
 18#define orc_warn_current(args...)					\
 19({									\
 20	static bool dumped_before;					\
 21	if (state->task == current && !state->error) {			\
 22		orc_warn(args);						\
 23		if (unwind_debug && !dumped_before) {			\
 24			dumped_before = true;				\
 25			unwind_dump(state);				\
 26		}							\
 27	}								\
 28})
 29
 30extern int __start_orc_unwind_ip[];
 31extern int __stop_orc_unwind_ip[];
 32extern struct orc_entry __start_orc_unwind[];
 33extern struct orc_entry __stop_orc_unwind[];
 34
 35static bool orc_init __ro_after_init;
 36static bool unwind_debug __ro_after_init;
 37static unsigned int lookup_num_blocks __ro_after_init;
 38
 39static int __init unwind_debug_cmdline(char *str)
 40{
 41	unwind_debug = true;
 42
 43	return 0;
 44}
 45early_param("unwind_debug", unwind_debug_cmdline);
 46
 47static void unwind_dump(struct unwind_state *state)
 48{
 49	static bool dumped_before;
 50	unsigned long word, *sp;
 51	struct stack_info stack_info = {0};
 52	unsigned long visit_mask = 0;
 53
 54	if (dumped_before)
 55		return;
 56
 57	dumped_before = true;
 58
 59	printk_deferred("unwind stack type:%d next_sp:%p mask:0x%lx graph_idx:%d\n",
 60			state->stack_info.type, state->stack_info.next_sp,
 61			state->stack_mask, state->graph_idx);
 62
 63	for (sp = __builtin_frame_address(0); sp;
 64	     sp = PTR_ALIGN(stack_info.next_sp, sizeof(long))) {
 65		if (get_stack_info(sp, state->task, &stack_info, &visit_mask))
 66			break;
 67
 68		for (; sp < stack_info.end; sp++) {
 69
 70			word = READ_ONCE_NOCHECK(*sp);
 71
 72			printk_deferred("%0*lx: %0*lx (%pB)\n", BITS_PER_LONG/4,
 73					(unsigned long)sp, BITS_PER_LONG/4,
 74					word, (void *)word);
 75		}
 76	}
 77}
 78
 79static inline unsigned long orc_ip(const int *ip)
 80{
 81	return (unsigned long)ip + *ip;
 82}
 83
 84static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
 85				    unsigned int num_entries, unsigned long ip)
 86{
 87	int *first = ip_table;
 88	int *last = ip_table + num_entries - 1;
 89	int *mid, *found = first;
 90
 91	if (!num_entries)
 92		return NULL;
 93
 94	/*
 95	 * Do a binary range search to find the rightmost duplicate of a given
 96	 * starting address.  Some entries are section terminators which are
 97	 * "weak" entries for ensuring there are no gaps.  They should be
 98	 * ignored when they conflict with a real entry.
 99	 */
100	while (first <= last) {
101		mid = first + ((last - first) / 2);
102
103		if (orc_ip(mid) <= ip) {
104			found = mid;
105			first = mid + 1;
106		} else
107			last = mid - 1;
108	}
109
110	return u_table + (found - ip_table);
111}
112
113#ifdef CONFIG_MODULES
114static struct orc_entry *orc_module_find(unsigned long ip)
115{
116	struct module *mod;
117
118	mod = __module_address(ip);
119	if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip)
120		return NULL;
121	return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind,
122			  mod->arch.num_orcs, ip);
123}
124#else
125static struct orc_entry *orc_module_find(unsigned long ip)
126{
127	return NULL;
128}
129#endif
130
131#ifdef CONFIG_DYNAMIC_FTRACE
132static struct orc_entry *orc_find(unsigned long ip);
133
134/*
135 * Ftrace dynamic trampolines do not have orc entries of their own.
136 * But they are copies of the ftrace entries that are static and
137 * defined in ftrace_*.S, which do have orc entries.
138 *
139 * If the unwinder comes across a ftrace trampoline, then find the
140 * ftrace function that was used to create it, and use that ftrace
141 * function's orc entry, as the placement of the return code in
142 * the stack will be identical.
143 */
144static struct orc_entry *orc_ftrace_find(unsigned long ip)
145{
146	struct ftrace_ops *ops;
147	unsigned long tramp_addr, offset;
148
149	ops = ftrace_ops_trampoline(ip);
150	if (!ops)
151		return NULL;
152
153	/* Set tramp_addr to the start of the code copied by the trampoline */
154	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
155		tramp_addr = (unsigned long)ftrace_regs_caller;
156	else
157		tramp_addr = (unsigned long)ftrace_caller;
158
159	/* Now place tramp_addr to the location within the trampoline ip is at */
160	offset = ip - ops->trampoline;
161	tramp_addr += offset;
162
163	/* Prevent unlikely recursion */
164	if (ip == tramp_addr)
165		return NULL;
166
167	return orc_find(tramp_addr);
168}
169#else
170static struct orc_entry *orc_ftrace_find(unsigned long ip)
171{
172	return NULL;
173}
174#endif
175
176/* Fake frame pointer entry -- used as a fallback for generated code */
177static struct orc_entry orc_fp_entry = {
178	.type		= ORC_TYPE_CALL,
179	.sp_reg		= ORC_REG_BP,
180	.sp_offset	= 16,
181	.bp_reg		= ORC_REG_PREV_SP,
182	.bp_offset	= -16,
183};
184
185static struct orc_entry *orc_bpf_find(unsigned long ip)
186{
187#ifdef CONFIG_BPF_JIT
188	if (bpf_has_frame_pointer(ip))
189		return &orc_fp_entry;
190#endif
191
192	return NULL;
193}
194
195/*
196 * If we crash with IP==0, the last successfully executed instruction
197 * was probably an indirect function call with a NULL function pointer,
198 * and we don't have unwind information for NULL.
199 * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
200 * pointer into its parent and then continue normally from there.
201 */
202static struct orc_entry null_orc_entry = {
203	.sp_offset = sizeof(long),
204	.sp_reg = ORC_REG_SP,
205	.bp_reg = ORC_REG_UNDEFINED,
206	.type = ORC_TYPE_CALL
207};
208
209static struct orc_entry *orc_find(unsigned long ip)
210{
211	static struct orc_entry *orc;
212
213	if (ip == 0)
214		return &null_orc_entry;
215
216	/* For non-init vmlinux addresses, use the fast lookup table: */
217	if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) {
218		unsigned int idx, start, stop;
219
220		idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE;
221
222		if (unlikely((idx >= lookup_num_blocks-1))) {
223			orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n",
224				 idx, lookup_num_blocks, (void *)ip);
225			return NULL;
226		}
227
228		start = orc_lookup[idx];
229		stop = orc_lookup[idx + 1] + 1;
230
231		if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) ||
232			     (__start_orc_unwind + stop > __stop_orc_unwind))) {
233			orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n",
234				 idx, lookup_num_blocks, start, stop, (void *)ip);
235			return NULL;
236		}
237
238		return __orc_find(__start_orc_unwind_ip + start,
239				  __start_orc_unwind + start, stop - start, ip);
240	}
241
242	/* vmlinux .init slow lookup: */
243	if (is_kernel_inittext(ip))
244		return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
245				  __stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
246
247	/* Module lookup: */
248	orc = orc_module_find(ip);
249	if (orc)
250		return orc;
251
252	/* BPF lookup: */
253	orc = orc_bpf_find(ip);
254	if (orc)
255		return orc;
256
257	return orc_ftrace_find(ip);
258}
259
260#ifdef CONFIG_MODULES
261
262static DEFINE_MUTEX(sort_mutex);
263static int *cur_orc_ip_table = __start_orc_unwind_ip;
264static struct orc_entry *cur_orc_table = __start_orc_unwind;
265
266static void orc_sort_swap(void *_a, void *_b, int size)
267{
268	struct orc_entry *orc_a, *orc_b;
269	int *a = _a, *b = _b, tmp;
270	int delta = _b - _a;
271
272	/* Swap the .orc_unwind_ip entries: */
273	tmp = *a;
274	*a = *b + delta;
275	*b = tmp - delta;
276
277	/* Swap the corresponding .orc_unwind entries: */
278	orc_a = cur_orc_table + (a - cur_orc_ip_table);
279	orc_b = cur_orc_table + (b - cur_orc_ip_table);
280	swap(*orc_a, *orc_b);
281}
282
283static int orc_sort_cmp(const void *_a, const void *_b)
284{
285	struct orc_entry *orc_a;
286	const int *a = _a, *b = _b;
287	unsigned long a_val = orc_ip(a);
288	unsigned long b_val = orc_ip(b);
289
290	if (a_val > b_val)
291		return 1;
292	if (a_val < b_val)
293		return -1;
294
295	/*
296	 * The "weak" section terminator entries need to always be first
297	 * to ensure the lookup code skips them in favor of real entries.
298	 * These terminator entries exist to handle any gaps created by
299	 * whitelisted .o files which didn't get objtool generation.
300	 */
301	orc_a = cur_orc_table + (a - cur_orc_ip_table);
302	return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
303}
304
305void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
306			void *_orc, size_t orc_size)
307{
308	int *orc_ip = _orc_ip;
309	struct orc_entry *orc = _orc;
310	unsigned int num_entries = orc_ip_size / sizeof(int);
311
312	WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 ||
313		     orc_size % sizeof(*orc) != 0 ||
314		     num_entries != orc_size / sizeof(*orc));
315
316	/*
317	 * The 'cur_orc_*' globals allow the orc_sort_swap() callback to
318	 * associate an .orc_unwind_ip table entry with its corresponding
319	 * .orc_unwind entry so they can both be swapped.
320	 */
321	mutex_lock(&sort_mutex);
322	cur_orc_ip_table = orc_ip;
323	cur_orc_table = orc;
324	sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
325	mutex_unlock(&sort_mutex);
326
327	mod->arch.orc_unwind_ip = orc_ip;
328	mod->arch.orc_unwind = orc;
329	mod->arch.num_orcs = num_entries;
330}
331#endif
332
333void __init unwind_init(void)
334{
335	size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip;
336	size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind;
337	size_t num_entries = orc_ip_size / sizeof(int);
338	struct orc_entry *orc;
339	int i;
340
341	if (!num_entries || orc_ip_size % sizeof(int) != 0 ||
342	    orc_size % sizeof(struct orc_entry) != 0 ||
343	    num_entries != orc_size / sizeof(struct orc_entry)) {
344		orc_warn("WARNING: Bad or missing .orc_unwind table.  Disabling unwinder.\n");
345		return;
346	}
347
348	/*
349	 * Note, the orc_unwind and orc_unwind_ip tables were already
350	 * sorted at build time via the 'sorttable' tool.
351	 * It's ready for binary search straight away, no need to sort it.
352	 */
353
354	/* Initialize the fast lookup table: */
355	lookup_num_blocks = orc_lookup_end - orc_lookup;
356	for (i = 0; i < lookup_num_blocks-1; i++) {
357		orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
358				 num_entries,
359				 LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i));
360		if (!orc) {
361			orc_warn("WARNING: Corrupt .orc_unwind table.  Disabling unwinder.\n");
362			return;
363		}
364
365		orc_lookup[i] = orc - __start_orc_unwind;
366	}
367
368	/* Initialize the ending block: */
369	orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries,
370			 LOOKUP_STOP_IP);
371	if (!orc) {
372		orc_warn("WARNING: Corrupt .orc_unwind table.  Disabling unwinder.\n");
373		return;
374	}
375	orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind;
376
377	orc_init = true;
378}
379
380unsigned long unwind_get_return_address(struct unwind_state *state)
381{
382	if (unwind_done(state))
383		return 0;
384
385	return __kernel_text_address(state->ip) ? state->ip : 0;
386}
387EXPORT_SYMBOL_GPL(unwind_get_return_address);
388
389unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)
390{
391	if (unwind_done(state))
392		return NULL;
393
394	if (state->regs)
395		return &state->regs->ip;
396
397	if (state->sp)
398		return (unsigned long *)state->sp - 1;
399
400	return NULL;
401}
402
403static bool stack_access_ok(struct unwind_state *state, unsigned long _addr,
404			    size_t len)
405{
406	struct stack_info *info = &state->stack_info;
407	void *addr = (void *)_addr;
408
409	if (on_stack(info, addr, len))
410		return true;
411
412	return !get_stack_info(addr, state->task, info, &state->stack_mask) &&
413		on_stack(info, addr, len);
414}
415
416static bool deref_stack_reg(struct unwind_state *state, unsigned long addr,
417			    unsigned long *val)
418{
419	if (!stack_access_ok(state, addr, sizeof(long)))
420		return false;
421
422	*val = READ_ONCE_NOCHECK(*(unsigned long *)addr);
423	return true;
424}
425
426static bool deref_stack_regs(struct unwind_state *state, unsigned long addr,
427			     unsigned long *ip, unsigned long *sp)
428{
429	struct pt_regs *regs = (struct pt_regs *)addr;
430
431	/* x86-32 support will be more complicated due to the &regs->sp hack */
432	BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32));
433
434	if (!stack_access_ok(state, addr, sizeof(struct pt_regs)))
435		return false;
436
437	*ip = READ_ONCE_NOCHECK(regs->ip);
438	*sp = READ_ONCE_NOCHECK(regs->sp);
439	return true;
440}
441
442static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr,
443				  unsigned long *ip, unsigned long *sp)
444{
445	struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET;
446
447	if (!stack_access_ok(state, addr, IRET_FRAME_SIZE))
448		return false;
449
450	*ip = READ_ONCE_NOCHECK(regs->ip);
451	*sp = READ_ONCE_NOCHECK(regs->sp);
452	return true;
453}
454
455/*
456 * If state->regs is non-NULL, and points to a full pt_regs, just get the reg
457 * value from state->regs.
458 *
459 * Otherwise, if state->regs just points to IRET regs, and the previous frame
460 * had full regs, it's safe to get the value from the previous regs.  This can
461 * happen when early/late IRQ entry code gets interrupted by an NMI.
462 */
463static bool get_reg(struct unwind_state *state, unsigned int reg_off,
464		    unsigned long *val)
465{
466	unsigned int reg = reg_off/8;
467
468	if (!state->regs)
469		return false;
470
471	if (state->full_regs) {
472		*val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]);
473		return true;
474	}
475
476	if (state->prev_regs) {
477		*val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]);
478		return true;
479	}
480
481	return false;
482}
483
484bool unwind_next_frame(struct unwind_state *state)
485{
486	unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp;
487	enum stack_type prev_type = state->stack_info.type;
488	struct orc_entry *orc;
489	bool indirect = false;
490
491	if (unwind_done(state))
492		return false;
493
494	/* Don't let modules unload while we're reading their ORC data. */
495	guard(rcu)();
496
497	/* End-of-stack check for user tasks: */
498	if (state->regs && user_mode(state->regs))
499		goto the_end;
500
501	/*
502	 * Find the orc_entry associated with the text address.
503	 *
504	 * For a call frame (as opposed to a signal frame), state->ip points to
505	 * the instruction after the call.  That instruction's stack layout
506	 * could be different from the call instruction's layout, for example
507	 * if the call was to a noreturn function.  So get the ORC data for the
508	 * call instruction itself.
509	 */
510	orc = orc_find(state->signal ? state->ip : state->ip - 1);
511	if (!orc) {
512		/*
513		 * As a fallback, try to assume this code uses a frame pointer.
514		 * This is just a guess, so the rest of the unwind is no longer
515		 * considered reliable.
516		 */
517		orc = &orc_fp_entry;
518		state->error = true;
519	} else {
520		if (orc->type == ORC_TYPE_UNDEFINED)
521			goto err;
522
523		if (orc->type == ORC_TYPE_END_OF_STACK)
524			goto the_end;
525	}
526
527	state->signal = orc->signal;
528
529	/* Find the previous frame's stack: */
530	switch (orc->sp_reg) {
531	case ORC_REG_SP:
532		sp = state->sp + orc->sp_offset;
533		break;
534
535	case ORC_REG_BP:
536		sp = state->bp + orc->sp_offset;
537		break;
538
539	case ORC_REG_SP_INDIRECT:
540		sp = state->sp;
541		indirect = true;
542		break;
543
544	case ORC_REG_BP_INDIRECT:
545		sp = state->bp + orc->sp_offset;
546		indirect = true;
547		break;
548
549	case ORC_REG_R10:
550		if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) {
551			orc_warn_current("missing R10 value at %pB\n",
552					 (void *)state->ip);
553			goto err;
554		}
555		break;
556
557	case ORC_REG_R13:
558		if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) {
559			orc_warn_current("missing R13 value at %pB\n",
560					 (void *)state->ip);
561			goto err;
562		}
563		break;
564
565	case ORC_REG_DI:
566		if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) {
567			orc_warn_current("missing RDI value at %pB\n",
568					 (void *)state->ip);
569			goto err;
570		}
571		break;
572
573	case ORC_REG_DX:
574		if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) {
575			orc_warn_current("missing DX value at %pB\n",
576					 (void *)state->ip);
577			goto err;
578		}
579		break;
580
581	default:
582		orc_warn("unknown SP base reg %d at %pB\n",
583			 orc->sp_reg, (void *)state->ip);
584		goto err;
585	}
586
587	if (indirect) {
588		if (!deref_stack_reg(state, sp, &sp))
589			goto err;
590
591		if (orc->sp_reg == ORC_REG_SP_INDIRECT)
592			sp += orc->sp_offset;
593	}
594
595	/* Find IP, SP and possibly regs: */
596	switch (orc->type) {
597	case ORC_TYPE_CALL:
598		ip_p = sp - sizeof(long);
599
600		if (!deref_stack_reg(state, ip_p, &state->ip))
601			goto err;
602
603		state->ip = unwind_recover_ret_addr(state, state->ip,
604						    (unsigned long *)ip_p);
605		state->sp = sp;
606		state->regs = NULL;
607		state->prev_regs = NULL;
608		break;
609
610	case ORC_TYPE_REGS:
611		if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) {
612			orc_warn_current("can't access registers at %pB\n",
613					 (void *)orig_ip);
614			goto err;
615		}
616		/*
617		 * There is a small chance to interrupt at the entry of
618		 * arch_rethook_trampoline() where the ORC info doesn't exist.
619		 * That point is right after the RET to arch_rethook_trampoline()
620		 * which was modified return address.
621		 * At that point, the @addr_p of the unwind_recover_rethook()
622		 * (this has to point the address of the stack entry storing
623		 * the modified return address) must be "SP - (a stack entry)"
624		 * because SP is incremented by the RET.
625		 */
626		state->ip = unwind_recover_rethook(state, state->ip,
627				(unsigned long *)(state->sp - sizeof(long)));
628		state->regs = (struct pt_regs *)sp;
629		state->prev_regs = NULL;
630		state->full_regs = true;
631		break;
632
633	case ORC_TYPE_REGS_PARTIAL:
634		if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) {
635			orc_warn_current("can't access iret registers at %pB\n",
636					 (void *)orig_ip);
637			goto err;
638		}
639		/* See ORC_TYPE_REGS case comment. */
640		state->ip = unwind_recover_rethook(state, state->ip,
641				(unsigned long *)(state->sp - sizeof(long)));
642
643		if (state->full_regs)
644			state->prev_regs = state->regs;
645		state->regs = (void *)sp - IRET_FRAME_OFFSET;
646		state->full_regs = false;
647		break;
648
649	default:
650		orc_warn("unknown .orc_unwind entry type %d at %pB\n",
651			 orc->type, (void *)orig_ip);
652		goto err;
653	}
654
655	/* Find BP: */
656	switch (orc->bp_reg) {
657	case ORC_REG_UNDEFINED:
658		if (get_reg(state, offsetof(struct pt_regs, bp), &tmp))
659			state->bp = tmp;
660		break;
661
662	case ORC_REG_PREV_SP:
663		if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp))
664			goto err;
665		break;
666
667	case ORC_REG_BP:
668		if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp))
669			goto err;
670		break;
671
672	default:
673		orc_warn("unknown BP base reg %d for ip %pB\n",
674			 orc->bp_reg, (void *)orig_ip);
675		goto err;
676	}
677
678	/* Prevent a recursive loop due to bad ORC data: */
679	if (state->stack_info.type == prev_type &&
680	    on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) &&
681	    state->sp <= prev_sp) {
682		orc_warn_current("stack going in the wrong direction? at %pB\n",
683				 (void *)orig_ip);
684		goto err;
685	}
686
687	return true;
688
689err:
690	state->error = true;
691
692the_end:
693	state->stack_info.type = STACK_TYPE_UNKNOWN;
694	return false;
695}
696EXPORT_SYMBOL_GPL(unwind_next_frame);
697
698void __unwind_start(struct unwind_state *state, struct task_struct *task,
699		    struct pt_regs *regs, unsigned long *first_frame)
700{
701	memset(state, 0, sizeof(*state));
702	state->task = task;
703
704	if (!orc_init)
705		goto err;
706
707	/*
708	 * Refuse to unwind the stack of a task while it's executing on another
709	 * CPU.  This check is racy, but that's ok: the unwinder has other
710	 * checks to prevent it from going off the rails.
711	 */
712	if (task_on_another_cpu(task))
713		goto err;
714
715	if (regs) {
716		if (user_mode(regs))
717			goto the_end;
718
719		state->ip = regs->ip;
720		state->sp = regs->sp;
721		state->bp = regs->bp;
722		state->regs = regs;
723		state->full_regs = true;
724		state->signal = true;
725
726	} else if (task == current) {
727		asm volatile("lea (%%rip), %0\n\t"
728			     "mov %%rsp, %1\n\t"
729			     "mov %%rbp, %2\n\t"
730			     : "=r" (state->ip), "=r" (state->sp),
731			       "=r" (state->bp));
732
733	} else {
734		struct inactive_task_frame *frame = (void *)task->thread.sp;
735
736		state->sp = task->thread.sp + sizeof(*frame);
737		state->bp = READ_ONCE_NOCHECK(frame->bp);
738		state->ip = READ_ONCE_NOCHECK(frame->ret_addr);
739		state->signal = (void *)state->ip == ret_from_fork_asm;
740	}
741
742	if (get_stack_info((unsigned long *)state->sp, state->task,
743			   &state->stack_info, &state->stack_mask)) {
744		/*
745		 * We weren't on a valid stack.  It's possible that
746		 * we overflowed a valid stack into a guard page.
747		 * See if the next page up is valid so that we can
748		 * generate some kind of backtrace if this happens.
749		 */
750		void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp);
751		state->error = true;
752		if (get_stack_info(next_page, state->task, &state->stack_info,
753				   &state->stack_mask))
754			return;
755	}
756
757	/*
758	 * The caller can provide the address of the first frame directly
759	 * (first_frame) or indirectly (regs->sp) to indicate which stack frame
760	 * to start unwinding at.  Skip ahead until we reach it.
761	 */
762
763	/* When starting from regs, skip the regs frame: */
764	if (regs) {
765		unwind_next_frame(state);
766		return;
767	}
768
769	/* Otherwise, skip ahead to the user-specified starting frame: */
770	while (!unwind_done(state) &&
771	       (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||
772			state->sp <= (unsigned long)first_frame))
773		unwind_next_frame(state);
774
775	return;
776
777err:
778	state->error = true;
779the_end:
780	state->stack_info.type = STACK_TYPE_UNKNOWN;
781}
782EXPORT_SYMBOL_GPL(__unwind_start);