interrupt.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2#ifndef _ASM_POWERPC_INTERRUPT_H
  3#define _ASM_POWERPC_INTERRUPT_H
  4
  5/* BookE/4xx */
  6#define INTERRUPT_CRITICAL_INPUT  0x100
  7
  8/* BookE */
  9#define INTERRUPT_DEBUG           0xd00
 10#ifdef CONFIG_BOOKE
 11#define INTERRUPT_PERFMON         0x260
 12#define INTERRUPT_DOORBELL        0x280
 13#endif
 14
 15/* BookS/4xx/8xx */
 16#define INTERRUPT_MACHINE_CHECK   0x200
 17
 18/* BookS/8xx */
 19#define INTERRUPT_SYSTEM_RESET    0x100
 20
 21/* BookS */
 22#define INTERRUPT_DATA_SEGMENT    0x380
 23#define INTERRUPT_INST_SEGMENT    0x480
 24#define INTERRUPT_TRACE           0xd00
 25#define INTERRUPT_H_DATA_STORAGE  0xe00
 26#define INTERRUPT_HMI			0xe60
 27#define INTERRUPT_H_FAC_UNAVAIL   0xf80
 28#ifdef CONFIG_PPC_BOOK3S
 29#define INTERRUPT_DOORBELL        0xa00
 30#define INTERRUPT_PERFMON         0xf00
 31#define INTERRUPT_ALTIVEC_UNAVAIL	0xf20
 32#endif
 33
 34/* BookE/BookS/4xx/8xx */
 35#define INTERRUPT_DATA_STORAGE    0x300
 36#define INTERRUPT_INST_STORAGE    0x400
 37#define INTERRUPT_EXTERNAL		0x500
 38#define INTERRUPT_ALIGNMENT       0x600
 39#define INTERRUPT_PROGRAM         0x700
 40#define INTERRUPT_SYSCALL         0xc00
 41#define INTERRUPT_TRACE			0xd00
 42
 43/* BookE/BookS/44x */
 44#define INTERRUPT_FP_UNAVAIL      0x800
 45
 46/* BookE/BookS/44x/8xx */
 47#define INTERRUPT_DECREMENTER     0x900
 48
 49#ifndef INTERRUPT_PERFMON
 50#define INTERRUPT_PERFMON         0x0
 51#endif
 52
 53/* 8xx */
 54#define INTERRUPT_SOFT_EMU_8xx		0x1000
 55#define INTERRUPT_INST_TLB_MISS_8xx	0x1100
 56#define INTERRUPT_DATA_TLB_MISS_8xx	0x1200
 57#define INTERRUPT_INST_TLB_ERROR_8xx	0x1300
 58#define INTERRUPT_DATA_TLB_ERROR_8xx	0x1400
 59#define INTERRUPT_DATA_BREAKPOINT_8xx	0x1c00
 60#define INTERRUPT_INST_BREAKPOINT_8xx	0x1d00
 61
 62/* 603 */
 63#define INTERRUPT_INST_TLB_MISS_603		0x1000
 64#define INTERRUPT_DATA_LOAD_TLB_MISS_603	0x1100
 65#define INTERRUPT_DATA_STORE_TLB_MISS_603	0x1200
 66
 67#ifndef __ASSEMBLER__
 68
 69#include <linux/context_tracking.h>
 70#include <linux/hardirq.h>
 71#include <asm/cputime.h>
 72#include <asm/firmware.h>
 73#include <asm/ftrace.h>
 74#include <asm/kprobes.h>
 75#include <asm/runlatch.h>
 76
 77#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
 78/*
 79 * WARN/BUG is handled with a program interrupt so minimise checks here to
 80 * avoid recursion and maximise the chance of getting the first oops handled.
 81 */
 82#define INT_SOFT_MASK_BUG_ON(regs, cond)				\
 83do {									\
 84	if ((user_mode(regs) || (TRAP(regs) != INTERRUPT_PROGRAM)))	\
 85		BUG_ON(cond);						\
 86} while (0)
 87#else
 88#define INT_SOFT_MASK_BUG_ON(regs, cond)
 89#endif
 90
 91#ifdef CONFIG_PPC_BOOK3S_64
 92extern char __end_soft_masked[];
 93bool search_kernel_soft_mask_table(unsigned long addr);
 94unsigned long search_kernel_restart_table(unsigned long addr);
 95
 96DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);
 97
 98static inline bool is_implicit_soft_masked(struct pt_regs *regs)
 99{
100	if (user_mode(regs))
101		return false;
102
103	if (regs->nip >= (unsigned long)__end_soft_masked)
104		return false;
105
106	return search_kernel_soft_mask_table(regs->nip);
107}
108
109static inline void srr_regs_clobbered(void)
110{
111	local_paca->srr_valid = 0;
112	local_paca->hsrr_valid = 0;
113}
114#else
115static inline unsigned long search_kernel_restart_table(unsigned long addr)
116{
117	return 0;
118}
119
120static inline bool is_implicit_soft_masked(struct pt_regs *regs)
121{
122	return false;
123}
124
125static inline void srr_regs_clobbered(void)
126{
127}
128#endif
129
130static inline void nap_adjust_return(struct pt_regs *regs)
131{
132#ifdef CONFIG_PPC_970_NAP
133	if (unlikely(test_thread_local_flags(_TLF_NAPPING))) {
134		/* Can avoid a test-and-clear because NMIs do not call this */
135		clear_thread_local_flags(_TLF_NAPPING);
136		regs_set_return_ip(regs, (unsigned long)power4_idle_nap_return);
137	}
138#endif
139}
140
141static inline void booke_restore_dbcr0(void)
142{
143#ifdef CONFIG_PPC_ADV_DEBUG_REGS
144	unsigned long dbcr0 = current->thread.debug.dbcr0;
145
146	if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) {
147		mtspr(SPRN_DBSR, -1);
148		mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]);
149	}
150#endif
151}
152
153static inline void interrupt_enter_prepare(struct pt_regs *regs)
154{
155#ifdef CONFIG_PPC64
156	irq_soft_mask_set(IRQS_ALL_DISABLED);
157
158	/*
159	 * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE].
160	 * Asynchronous interrupts get here with HARD_DIS set (see below), so
161	 * this enables MSR[EE] for synchronous interrupts. IRQs remain
162	 * soft-masked. The interrupt handler may later call
163	 * interrupt_cond_local_irq_enable() to achieve a regular process
164	 * context.
165	 */
166	if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) {
167		INT_SOFT_MASK_BUG_ON(regs, !(regs->msr & MSR_EE));
168		__hard_irq_enable();
169	} else {
170		__hard_RI_enable();
171	}
172	/* Enable MSR[RI] early, to support kernel SLB and hash faults */
173#endif
174
175	if (!arch_irq_disabled_regs(regs))
176		trace_hardirqs_off();
177
178	if (user_mode(regs)) {
179		kuap_lock();
180		CT_WARN_ON(ct_state() != CT_STATE_USER);
181		user_exit_irqoff();
182
183		account_cpu_user_entry();
184		account_stolen_time();
185	} else {
186		kuap_save_and_lock(regs);
187		/*
188		 * CT_WARN_ON comes here via program_check_exception,
189		 * so avoid recursion.
190		 */
191		if (TRAP(regs) != INTERRUPT_PROGRAM)
192			CT_WARN_ON(ct_state() != CT_STATE_KERNEL &&
193				   ct_state() != CT_STATE_IDLE);
194		INT_SOFT_MASK_BUG_ON(regs, is_implicit_soft_masked(regs));
195		INT_SOFT_MASK_BUG_ON(regs, arch_irq_disabled_regs(regs) &&
196					   search_kernel_restart_table(regs->nip));
197	}
198	INT_SOFT_MASK_BUG_ON(regs, !arch_irq_disabled_regs(regs) &&
199				   !(regs->msr & MSR_EE));
200
201	booke_restore_dbcr0();
202}
203
204/*
205 * Care should be taken to note that interrupt_exit_prepare and
206 * interrupt_async_exit_prepare do not necessarily return immediately to
207 * regs context (e.g., if regs is usermode, we don't necessarily return to
208 * user mode). Other interrupts might be taken between here and return,
209 * context switch / preemption may occur in the exit path after this, or a
210 * signal may be delivered, etc.
211 *
212 * The real interrupt exit code is platform specific, e.g.,
213 * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s.
214 *
215 * However interrupt_nmi_exit_prepare does return directly to regs, because
216 * NMIs do not do "exit work" or replay soft-masked interrupts.
217 */
218static inline void interrupt_exit_prepare(struct pt_regs *regs)
219{
220}
221
222static inline void interrupt_async_enter_prepare(struct pt_regs *regs)
223{
224#ifdef CONFIG_PPC64
225	/* Ensure interrupt_enter_prepare does not enable MSR[EE] */
226	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
227#endif
228	interrupt_enter_prepare(regs);
229#ifdef CONFIG_PPC_BOOK3S_64
230	/*
231	 * RI=1 is set by interrupt_enter_prepare, so this thread flags access
232	 * has to come afterward (it can cause SLB faults).
233	 */
234	if (cpu_has_feature(CPU_FTR_CTRL) &&
235	    !test_thread_local_flags(_TLF_RUNLATCH))
236		__ppc64_runlatch_on();
237#endif
238	irq_enter();
239}
240
241static inline void interrupt_async_exit_prepare(struct pt_regs *regs)
242{
243	/*
244	 * Adjust at exit so the main handler sees the true NIA. This must
245	 * come before irq_exit() because irq_exit can enable interrupts, and
246	 * if another interrupt is taken before nap_adjust_return has run
247	 * here, then that interrupt would return directly to idle nap return.
248	 */
249	nap_adjust_return(regs);
250
251	irq_exit();
252	interrupt_exit_prepare(regs);
253}
254
255struct interrupt_nmi_state {
256#ifdef CONFIG_PPC64
257	u8 irq_soft_mask;
258	u8 irq_happened;
259	u8 ftrace_enabled;
260	u64 softe;
261#endif
262};
263
264static inline bool nmi_disables_ftrace(struct pt_regs *regs)
265{
266	/* Allow DEC and PMI to be traced when they are soft-NMI */
267	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
268		if (TRAP(regs) == INTERRUPT_DECREMENTER)
269		       return false;
270		if (TRAP(regs) == INTERRUPT_PERFMON)
271		       return false;
272	}
273	if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) {
274		if (TRAP(regs) == INTERRUPT_PERFMON)
275			return false;
276	}
277
278	return true;
279}
280
281static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
282{
283#ifdef CONFIG_PPC64
284	state->irq_soft_mask = local_paca->irq_soft_mask;
285	state->irq_happened = local_paca->irq_happened;
286	state->softe = regs->softe;
287
288	/*
289	 * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
290	 * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile
291	 * because that goes through irq tracing which we don't want in NMI.
292	 */
293	local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
294	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
295
296	if (!(regs->msr & MSR_EE) || is_implicit_soft_masked(regs)) {
297		/*
298		 * Adjust regs->softe to be soft-masked if it had not been
299		 * reconcied (e.g., interrupt entry with MSR[EE]=0 but softe
300		 * not yet set disabled), or if it was in an implicit soft
301		 * masked state. This makes arch_irq_disabled_regs(regs)
302		 * behave as expected.
303		 */
304		regs->softe = IRQS_ALL_DISABLED;
305	}
306
307	__hard_RI_enable();
308
309	/* Don't do any per-CPU operations until interrupt state is fixed */
310
311	if (nmi_disables_ftrace(regs)) {
312		state->ftrace_enabled = this_cpu_get_ftrace_enabled();
313		this_cpu_set_ftrace_enabled(0);
314	}
315#endif
316
317	/* If data relocations are enabled, it's safe to use nmi_enter() */
318	if (mfmsr() & MSR_DR) {
319		nmi_enter();
320		return;
321	}
322
323	/*
324	 * But do not use nmi_enter() for pseries hash guest taking a real-mode
325	 * NMI because not everything it touches is within the RMA limit.
326	 */
327	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
328	    firmware_has_feature(FW_FEATURE_LPAR) &&
329	    !radix_enabled())
330		return;
331
332	/*
333	 * Likewise, don't use it if we have some form of instrumentation (like
334	 * KASAN shadow) that is not safe to access in real mode (even on radix)
335	 */
336	if (IS_ENABLED(CONFIG_KASAN))
337		return;
338
339	/*
340	 * Likewise, do not use it in real mode if percpu first chunk is not
341	 * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there
342	 * are chances where percpu allocation can come from vmalloc area.
343	 */
344	if (percpu_first_chunk_is_paged)
345		return;
346
347	/* Otherwise, it should be safe to call it */
348	nmi_enter();
349}
350
351static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
352{
353	if (mfmsr() & MSR_DR) {
354		// nmi_exit if relocations are on
355		nmi_exit();
356	} else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
357		   firmware_has_feature(FW_FEATURE_LPAR) &&
358		   !radix_enabled()) {
359		// no nmi_exit for a pseries hash guest taking a real mode exception
360	} else if (IS_ENABLED(CONFIG_KASAN)) {
361		// no nmi_exit for KASAN in real mode
362	} else if (percpu_first_chunk_is_paged) {
363		// no nmi_exit if percpu first chunk is not embedded
364	} else {
365		nmi_exit();
366	}
367
368	/*
369	 * nmi does not call nap_adjust_return because nmi should not create
370	 * new work to do (must use irq_work for that).
371	 */
372
373#ifdef CONFIG_PPC64
374#ifdef CONFIG_PPC_BOOK3S
375	if (arch_irq_disabled_regs(regs)) {
376		unsigned long rst = search_kernel_restart_table(regs->nip);
377		if (rst)
378			regs_set_return_ip(regs, rst);
379	}
380#endif
381
382	if (nmi_disables_ftrace(regs))
383		this_cpu_set_ftrace_enabled(state->ftrace_enabled);
384
385	/* Check we didn't change the pending interrupt mask. */
386	WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened);
387	regs->softe = state->softe;
388	local_paca->irq_happened = state->irq_happened;
389	local_paca->irq_soft_mask = state->irq_soft_mask;
390#endif
391}
392
393/*
394 * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each
395 * function definition. The reason for this is the noinstr section is placed
396 * after the main text section, i.e., very far away from the interrupt entry
397 * asm. That creates problems with fitting linker stubs when building large
398 * kernels.
399 */
400#define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address
401
402/**
403 * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function
404 * @func:	Function name of the entry point
405 * @returns:	Returns a value back to asm caller
406 */
407#define DECLARE_INTERRUPT_HANDLER_RAW(func)				\
408	__visible long func(struct pt_regs *regs)
409
410/**
411 * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function
412 * @func:	Function name of the entry point
413 * @returns:	Returns a value back to asm caller
414 *
415 * @func is called from ASM entry code.
416 *
417 * This is a plain function which does no tracing, reconciling, etc.
418 * The macro is written so it acts as function definition. Append the
419 * body with a pair of curly brackets.
420 *
421 * raw interrupt handlers must not enable or disable interrupts, or
422 * schedule, tracing and instrumentation (ftrace, lockdep, etc) would
423 * not be advisable either, although may be possible in a pinch, the
424 * trace will look odd at least.
425 *
426 * A raw handler may call one of the other interrupt handler functions
427 * to be converted into that interrupt context without these restrictions.
428 *
429 * On PPC64, _RAW handlers may return with fast_interrupt_return.
430 *
431 * Specific handlers may have additional restrictions.
432 */
433#define DEFINE_INTERRUPT_HANDLER_RAW(func)				\
434static __always_inline __no_sanitize_address __no_kcsan long		\
435____##func(struct pt_regs *regs);					\
436									\
437interrupt_handler long func(struct pt_regs *regs)			\
438{									\
439	long ret;							\
440									\
441	__hard_RI_enable();						\
442									\
443	ret = ____##func (regs);					\
444									\
445	return ret;							\
446}									\
447NOKPROBE_SYMBOL(func);							\
448									\
449static __always_inline __no_sanitize_address __no_kcsan long		\
450____##func(struct pt_regs *regs)
451
452/**
453 * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function
454 * @func:	Function name of the entry point
455 */
456#define DECLARE_INTERRUPT_HANDLER(func)					\
457	__visible void func(struct pt_regs *regs)
458
459/**
460 * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function
461 * @func:	Function name of the entry point
462 *
463 * @func is called from ASM entry code.
464 *
465 * The macro is written so it acts as function definition. Append the
466 * body with a pair of curly brackets.
467 */
468#define DEFINE_INTERRUPT_HANDLER(func)					\
469static __always_inline void ____##func(struct pt_regs *regs);		\
470									\
471interrupt_handler void func(struct pt_regs *regs)			\
472{									\
473	interrupt_enter_prepare(regs);					\
474									\
475	____##func (regs);						\
476									\
477	interrupt_exit_prepare(regs);					\
478}									\
479NOKPROBE_SYMBOL(func);							\
480									\
481static __always_inline void ____##func(struct pt_regs *regs)
482
483/**
484 * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function
485 * @func:	Function name of the entry point
486 * @returns:	Returns a value back to asm caller
487 */
488#define DECLARE_INTERRUPT_HANDLER_RET(func)				\
489	__visible long func(struct pt_regs *regs)
490
491/**
492 * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function
493 * @func:	Function name of the entry point
494 * @returns:	Returns a value back to asm caller
495 *
496 * @func is called from ASM entry code.
497 *
498 * The macro is written so it acts as function definition. Append the
499 * body with a pair of curly brackets.
500 */
501#define DEFINE_INTERRUPT_HANDLER_RET(func)				\
502static __always_inline long ____##func(struct pt_regs *regs);		\
503									\
504interrupt_handler long func(struct pt_regs *regs)			\
505{									\
506	long ret;							\
507									\
508	interrupt_enter_prepare(regs);					\
509									\
510	ret = ____##func (regs);					\
511									\
512	interrupt_exit_prepare(regs);					\
513									\
514	return ret;							\
515}									\
516NOKPROBE_SYMBOL(func);							\
517									\
518static __always_inline long ____##func(struct pt_regs *regs)
519
520/**
521 * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function
522 * @func:	Function name of the entry point
523 */
524#define DECLARE_INTERRUPT_HANDLER_ASYNC(func)				\
525	__visible void func(struct pt_regs *regs)
526
527/**
528 * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function
529 * @func:	Function name of the entry point
530 *
531 * @func is called from ASM entry code.
532 *
533 * The macro is written so it acts as function definition. Append the
534 * body with a pair of curly brackets.
535 */
536#define DEFINE_INTERRUPT_HANDLER_ASYNC(func)				\
537static __always_inline void ____##func(struct pt_regs *regs);		\
538									\
539interrupt_handler void func(struct pt_regs *regs)			\
540{									\
541	interrupt_async_enter_prepare(regs);				\
542									\
543	____##func (regs);						\
544									\
545	interrupt_async_exit_prepare(regs);				\
546}									\
547NOKPROBE_SYMBOL(func);							\
548									\
549static __always_inline void ____##func(struct pt_regs *regs)
550
551/**
552 * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function
553 * @func:	Function name of the entry point
554 * @returns:	Returns a value back to asm caller
555 */
556#define DECLARE_INTERRUPT_HANDLER_NMI(func)				\
557	__visible long func(struct pt_regs *regs)
558
559/**
560 * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function
561 * @func:	Function name of the entry point
562 * @returns:	Returns a value back to asm caller
563 *
564 * @func is called from ASM entry code.
565 *
566 * The macro is written so it acts as function definition. Append the
567 * body with a pair of curly brackets.
568 */
569#define DEFINE_INTERRUPT_HANDLER_NMI(func)				\
570static __always_inline __no_sanitize_address __no_kcsan long		\
571____##func(struct pt_regs *regs);					\
572									\
573interrupt_handler long func(struct pt_regs *regs)			\
574{									\
575	struct interrupt_nmi_state state;				\
576	long ret;							\
577									\
578	interrupt_nmi_enter_prepare(regs, &state);			\
579									\
580	ret = ____##func (regs);					\
581									\
582	interrupt_nmi_exit_prepare(regs, &state);			\
583									\
584	return ret;							\
585}									\
586NOKPROBE_SYMBOL(func);							\
587									\
588static __always_inline  __no_sanitize_address __no_kcsan long		\
589____##func(struct pt_regs *regs)
590
591
592/* Interrupt handlers */
593/* kernel/traps.c */
594DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception);
595#ifdef CONFIG_PPC_BOOK3S_64
596DECLARE_INTERRUPT_HANDLER_RAW(machine_check_early_boot);
597DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async);
598#endif
599DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception);
600DECLARE_INTERRUPT_HANDLER(SMIException);
601DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);
602DECLARE_INTERRUPT_HANDLER(unknown_exception);
603DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
604DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception);
605DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);
606DECLARE_INTERRUPT_HANDLER(RunModeException);
607DECLARE_INTERRUPT_HANDLER(single_step_exception);
608DECLARE_INTERRUPT_HANDLER(program_check_exception);
609DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt);
610DECLARE_INTERRUPT_HANDLER(alignment_exception);
611DECLARE_INTERRUPT_HANDLER(StackOverflow);
612DECLARE_INTERRUPT_HANDLER(stack_overflow_exception);
613DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception);
614DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception);
615DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception);
616DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception);
617DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm);
618DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm);
619DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm);
620DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi);
621DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async);
622DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception);
623DECLARE_INTERRUPT_HANDLER(DebugException);
624DECLARE_INTERRUPT_HANDLER(altivec_assist_exception);
625DECLARE_INTERRUPT_HANDLER(CacheLockingException);
626DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException);
627DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException);
628DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException);
629DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);
630
631/* slb.c */
632DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault);
633DECLARE_INTERRUPT_HANDLER(do_bad_segment_interrupt);
634
635/* hash_utils.c */
636DECLARE_INTERRUPT_HANDLER(do_hash_fault);
637
638/* fault.c */
639DECLARE_INTERRUPT_HANDLER(do_page_fault);
640DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv);
641
642/* process.c */
643DECLARE_INTERRUPT_HANDLER(do_break);
644
645/* time.c */
646DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt);
647
648/* mce.c */
649DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early);
650DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode);
651
652DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException);
653
654/* irq.c */
655DECLARE_INTERRUPT_HANDLER_ASYNC(do_IRQ);
656
657void __noreturn unrecoverable_exception(struct pt_regs *regs);
658
659void replay_system_reset(void);
660void replay_soft_interrupts(void);
661
662static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs)
663{
664	if (!arch_irq_disabled_regs(regs))
665		local_irq_enable();
666}
667
668long system_call_exception(struct pt_regs *regs, unsigned long r0);
669notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv);
670notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs);
671notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs);
672#ifdef CONFIG_PPC64
673unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs);
674unsigned long interrupt_exit_user_restart(struct pt_regs *regs);
675unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs);
676#endif
677
678#endif /* __ASSEMBLER__ */
679
680#endif /* _ASM_POWERPC_INTERRUPT_H */