idtentry.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _ASM_X86_IDTENTRY_H
  3#define _ASM_X86_IDTENTRY_H
  4
  5/* Interrupts/Exceptions */
  6#include <asm/trapnr.h>
  7
  8#define IDT_ALIGN	(8 * (1 + HAS_KERNEL_IBT))
  9
 10#ifndef __ASSEMBLER__
 11#include <linux/entry-common.h>
 12#include <linux/hardirq.h>
 13
 14#include <asm/irq_stack.h>
 15
 16typedef void (*idtentry_t)(struct pt_regs *regs);
 17
 18/**
 19 * DECLARE_IDTENTRY - Declare functions for simple IDT entry points
 20 *		      No error code pushed by hardware
 21 * @vector:	Vector number (ignored for C)
 22 * @func:	Function name of the entry point
 23 *
 24 * Declares four functions:
 25 * - The ASM entry point: asm_##func
 26 * - The XEN PV trap entry point: xen_##func (maybe unused)
 27 * - The C handler called from the FRED event dispatcher (maybe unused)
 28 * - The C handler called from the ASM entry point
 29 *
 30 * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it
 31 * declares the entry points for usage in C code. There is an ASM variant
 32 * as well which is used to emit the entry stubs in entry_32/64.S.
 33 */
 34#define DECLARE_IDTENTRY(vector, func)					\
 35	asmlinkage void asm_##func(void);				\
 36	asmlinkage void xen_asm_##func(void);				\
 37	void fred_##func(struct pt_regs *regs);				\
 38	__visible void func(struct pt_regs *regs)
 39
 40/**
 41 * DEFINE_IDTENTRY - Emit code for simple IDT entry points
 42 * @func:	Function name of the entry point
 43 *
 44 * @func is called from ASM entry code with interrupts disabled.
 45 *
 46 * The macro is written so it acts as function definition. Append the
 47 * body with a pair of curly brackets.
 48 *
 49 * irqentry_enter() contains common code which has to be invoked before
 50 * arbitrary code in the body. irqentry_exit() contains common code
 51 * which has to run before returning to the low level assembly code.
 52 */
 53#define DEFINE_IDTENTRY(func)						\
 54static __always_inline void __##func(struct pt_regs *regs);		\
 55									\
 56__visible noinstr void func(struct pt_regs *regs)			\
 57{									\
 58	irqentry_state_t state = irqentry_enter(regs);			\
 59									\
 60	instrumentation_begin();					\
 61	__##func (regs);						\
 62	instrumentation_end();						\
 63	irqentry_exit(regs, state);					\
 64}									\
 65									\
 66static __always_inline void __##func(struct pt_regs *regs)
 67
 68/* Special case for 32bit IRET 'trap' */
 69#define DECLARE_IDTENTRY_SW	DECLARE_IDTENTRY
 70#define DEFINE_IDTENTRY_SW	DEFINE_IDTENTRY
 71
 72/**
 73 * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points
 74 *				Error code pushed by hardware
 75 * @vector:	Vector number (ignored for C)
 76 * @func:	Function name of the entry point
 77 *
 78 * Declares three functions:
 79 * - The ASM entry point: asm_##func
 80 * - The XEN PV trap entry point: xen_##func (maybe unused)
 81 * - The C handler called from the ASM entry point
 82 *
 83 * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the
 84 * C-handler.
 85 */
 86#define DECLARE_IDTENTRY_ERRORCODE(vector, func)			\
 87	asmlinkage void asm_##func(void);				\
 88	asmlinkage void xen_asm_##func(void);				\
 89	__visible void func(struct pt_regs *regs, unsigned long error_code)
 90
 91/**
 92 * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points
 93 *			       Error code pushed by hardware
 94 * @func:	Function name of the entry point
 95 *
 96 * Same as DEFINE_IDTENTRY, but has an extra error_code argument
 97 */
 98#define DEFINE_IDTENTRY_ERRORCODE(func)					\
 99static __always_inline void __##func(struct pt_regs *regs,		\
100				     unsigned long error_code);		\
101									\
102__visible noinstr void func(struct pt_regs *regs,			\
103			    unsigned long error_code)			\
104{									\
105	irqentry_state_t state = irqentry_enter(regs);			\
106									\
107	instrumentation_begin();					\
108	__##func (regs, error_code);					\
109	instrumentation_end();						\
110	irqentry_exit(regs, state);					\
111}									\
112									\
113static __always_inline void __##func(struct pt_regs *regs,		\
114				     unsigned long error_code)
115
116/**
117 * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points
118 *		      No error code pushed by hardware
119 * @vector:	Vector number (ignored for C)
120 * @func:	Function name of the entry point
121 *
122 * Maps to DECLARE_IDTENTRY().
123 */
124#define DECLARE_IDTENTRY_RAW(vector, func)				\
125	DECLARE_IDTENTRY(vector, func)
126
127/**
128 * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points
129 * @func:	Function name of the entry point
130 *
131 * @func is called from ASM entry code with interrupts disabled.
132 *
133 * The macro is written so it acts as function definition. Append the
134 * body with a pair of curly brackets.
135 *
136 * Contrary to DEFINE_IDTENTRY() this does not invoke the
137 * idtentry_enter/exit() helpers before and after the body invocation. This
138 * needs to be done in the body itself if applicable. Use if extra work
139 * is required before the enter/exit() helpers are invoked.
140 */
141#define DEFINE_IDTENTRY_RAW(func)					\
142__visible noinstr void func(struct pt_regs *regs)
143
144/**
145 * DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points
146 * @func:	Function name of the entry point
147 *
148 * @func is called from the FRED event dispatcher with interrupts disabled.
149 *
150 * See @DEFINE_IDTENTRY_RAW for further details.
151 */
152#define DEFINE_FREDENTRY_RAW(func)					\
153noinstr void fred_##func(struct pt_regs *regs)
154
155/**
156 * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points
157 *				    Error code pushed by hardware
158 * @vector:	Vector number (ignored for C)
159 * @func:	Function name of the entry point
160 *
161 * Maps to DECLARE_IDTENTRY_ERRORCODE()
162 */
163#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)			\
164	DECLARE_IDTENTRY_ERRORCODE(vector, func)
165
166/**
167 * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points
168 * @func:	Function name of the entry point
169 *
170 * @func is called from ASM entry code with interrupts disabled.
171 *
172 * The macro is written so it acts as function definition. Append the
173 * body with a pair of curly brackets.
174 *
175 * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the
176 * irqentry_enter/exit() helpers before and after the body invocation. This
177 * needs to be done in the body itself if applicable. Use if extra work
178 * is required before the enter/exit() helpers are invoked.
179 */
180#define DEFINE_IDTENTRY_RAW_ERRORCODE(func)				\
181__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
182
183/**
184 * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
185 *			  points (common/spurious)
186 * @vector:	Vector number (ignored for C)
187 * @func:	Function name of the entry point
188 *
189 * Maps to DECLARE_IDTENTRY_ERRORCODE()
190 */
191#define DECLARE_IDTENTRY_IRQ(vector, func)				\
192	DECLARE_IDTENTRY_ERRORCODE(vector, func)
193
194/**
195 * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points
196 * @func:	Function name of the entry point
197 *
198 * The vector number is pushed by the low level entry stub and handed
199 * to the function as error_code argument which needs to be truncated
200 * to an u8 because the push is sign extending.
201 *
202 * irq_enter/exit_rcu() are invoked before the function body and the
203 * KVM L1D flush request is set. Stack switching to the interrupt stack
204 * has to be done in the function body if necessary.
205 */
206#define DEFINE_IDTENTRY_IRQ(func)					\
207static void __##func(struct pt_regs *regs, u32 vector);			\
208									\
209__visible noinstr void func(struct pt_regs *regs,			\
210			    unsigned long error_code)			\
211{									\
212	irqentry_state_t state = irqentry_enter(regs);			\
213	u32 vector = (u32)(u8)error_code;				\
214									\
215	kvm_set_cpu_l1tf_flush_l1d();                                   \
216	instrumentation_begin();					\
217	run_irq_on_irqstack_cond(__##func, regs, vector);		\
218	instrumentation_end();						\
219	irqentry_exit(regs, state);					\
220}									\
221									\
222static noinline void __##func(struct pt_regs *regs, u32 vector)
223
224/**
225 * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points
226 * @vector:	Vector number (ignored for C)
227 * @func:	Function name of the entry point
228 *
229 * Declares three functions:
230 * - The ASM entry point: asm_##func
231 * - The XEN PV trap entry point: xen_##func (maybe unused)
232 * - The C handler called from the ASM entry point
233 *
234 * Maps to DECLARE_IDTENTRY().
235 */
236#define DECLARE_IDTENTRY_SYSVEC(vector, func)				\
237	DECLARE_IDTENTRY(vector, func)
238
239/**
240 * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points
241 * @func:	Function name of the entry point
242 *
243 * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the
244 * function body. KVM L1D flush request is set.
245 *
246 * Runs the function on the interrupt stack if the entry hit kernel mode
247 */
248#define DEFINE_IDTENTRY_SYSVEC(func)					\
249static void __##func(struct pt_regs *regs);				\
250									\
251static __always_inline void instr_##func(struct pt_regs *regs)		\
252{									\
253	run_sysvec_on_irqstack_cond(__##func, regs);			\
254}									\
255									\
256__visible noinstr void func(struct pt_regs *regs)			\
257{									\
258	irqentry_state_t state = irqentry_enter(regs);			\
259									\
260	kvm_set_cpu_l1tf_flush_l1d();                                   \
261	instrumentation_begin();					\
262	instr_##func (regs);						\
263	instrumentation_end();						\
264	irqentry_exit(regs, state);					\
265}									\
266									\
267void fred_##func(struct pt_regs *regs)					\
268{									\
269	instr_##func (regs);						\
270}									\
271									\
272static noinline void __##func(struct pt_regs *regs)
273
274/**
275 * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT
276 *				   entry points
277 * @func:	Function name of the entry point
278 *
279 * Runs the function on the interrupted stack. No switch to IRQ stack and
280 * only the minimal __irq_enter/exit() handling.
281 *
282 * Only use for 'empty' vectors like reschedule IPI and KVM posted
283 * interrupt vectors.
284 */
285#define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func)				\
286static __always_inline void __##func(struct pt_regs *regs);		\
287									\
288static __always_inline void instr_##func(struct pt_regs *regs)		\
289{									\
290	__irq_enter_raw();						\
291	__##func (regs);						\
292	__irq_exit_raw();						\
293}									\
294									\
295__visible noinstr void func(struct pt_regs *regs)			\
296{									\
297	irqentry_state_t state = irqentry_enter(regs);			\
298									\
299	kvm_set_cpu_l1tf_flush_l1d();                                   \
300	instrumentation_begin();					\
301	instr_##func (regs);						\
302	instrumentation_end();						\
303	irqentry_exit(regs, state);					\
304}									\
305									\
306void fred_##func(struct pt_regs *regs)					\
307{									\
308	instr_##func (regs);						\
309}									\
310									\
311static __always_inline void __##func(struct pt_regs *regs)
312
313/**
314 * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point
315 * @vector:	Vector number (ignored for C)
316 * @func:	Function name of the entry point
317 *
318 * Declares three functions:
319 * - The ASM entry point: asm_##func
320 * - The XEN PV trap entry point: xen_##func (maybe unused)
321 * - The C handler called from the ASM entry point
322 *
323 * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit
324 * difference
325 */
326#define DECLARE_IDTENTRY_XENCB(vector, func)				\
327	DECLARE_IDTENTRY(vector, func)
328
329#ifdef CONFIG_X86_64
330/**
331 * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points
332 * @vector:	Vector number (ignored for C)
333 * @func:	Function name of the entry point
334 *
335 * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler
336 * which is called from the ASM entry point on user mode entry
337 */
338#define DECLARE_IDTENTRY_IST(vector, func)				\
339	DECLARE_IDTENTRY_RAW(vector, func);				\
340	__visible void noist_##func(struct pt_regs *regs)
341
342/**
343 * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
344 * @vector:	Vector number (ignored for C)
345 * @func:	Function name of the entry point
346 *
347 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the
348 * safe_stack C handler.
349 */
350#define DECLARE_IDTENTRY_VC(vector, func)				\
351	DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func);			\
352	__visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code);	\
353	__visible noinstr void   user_##func(struct pt_regs *regs, unsigned long error_code)
354
355/**
356 * DEFINE_IDTENTRY_IST - Emit code for IST entry points
357 * @func:	Function name of the entry point
358 *
359 * Maps to DEFINE_IDTENTRY_RAW
360 */
361#define DEFINE_IDTENTRY_IST(func)					\
362	DEFINE_IDTENTRY_RAW(func)
363
364/**
365 * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which
366 *			   belong to a IST entry point (MCE, DB)
367 * @func:	Function name of the entry point. Must be the same as
368 *		the function name of the corresponding IST variant
369 *
370 * Maps to DEFINE_IDTENTRY_RAW().
371 */
372#define DEFINE_IDTENTRY_NOIST(func)					\
373	DEFINE_IDTENTRY_RAW(noist_##func)
374
375/**
376 * DECLARE_IDTENTRY_DF - Declare functions for double fault
377 * @vector:	Vector number (ignored for C)
378 * @func:	Function name of the entry point
379 *
380 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE
381 */
382#define DECLARE_IDTENTRY_DF(vector, func)				\
383	DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)
384
385/**
386 * DEFINE_IDTENTRY_DF - Emit code for double fault
387 * @func:	Function name of the entry point
388 *
389 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
390 */
391#define DEFINE_IDTENTRY_DF(func)					\
392	DEFINE_IDTENTRY_RAW_ERRORCODE(func)
393
394/**
395 * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
396 *			       when raised from kernel mode
397 * @func:	Function name of the entry point
398 *
399 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
400 */
401#define DEFINE_IDTENTRY_VC_KERNEL(func)				\
402	DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func)
403
404/**
405 * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
406 *			     when raised from user mode
407 * @func:	Function name of the entry point
408 *
409 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
410 */
411#define DEFINE_IDTENTRY_VC_USER(func)				\
412	DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func)
413
414#else	/* CONFIG_X86_64 */
415
416/**
417 * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant
418 * @vector:	Vector number (ignored for C)
419 * @func:	Function name of the entry point
420 *
421 * Declares two functions:
422 * - The ASM entry point: asm_##func
423 * - The C handler called from the C shim
424 */
425#define DECLARE_IDTENTRY_DF(vector, func)				\
426	asmlinkage void asm_##func(void);				\
427	__visible void func(struct pt_regs *regs,			\
428			    unsigned long error_code,			\
429			    unsigned long address)
430
431/**
432 * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit
433 * @func:	Function name of the entry point
434 *
435 * This is called through the doublefault shim which already provides
436 * cr2 in the address argument.
437 */
438#define DEFINE_IDTENTRY_DF(func)					\
439__visible noinstr void func(struct pt_regs *regs,			\
440			    unsigned long error_code,			\
441			    unsigned long address)
442
443#endif	/* !CONFIG_X86_64 */
444
445/* C-Code mapping */
446#define DECLARE_IDTENTRY_NMI		DECLARE_IDTENTRY_RAW
447#define DEFINE_IDTENTRY_NMI		DEFINE_IDTENTRY_RAW
448#define DEFINE_FREDENTRY_NMI		DEFINE_FREDENTRY_RAW
449
450#ifdef CONFIG_X86_64
451#define DECLARE_IDTENTRY_MCE		DECLARE_IDTENTRY_IST
452#define DEFINE_IDTENTRY_MCE		DEFINE_IDTENTRY_IST
453#define DEFINE_IDTENTRY_MCE_USER	DEFINE_IDTENTRY_NOIST
454#define DEFINE_FREDENTRY_MCE		DEFINE_FREDENTRY_RAW
455
456#define DECLARE_IDTENTRY_DEBUG		DECLARE_IDTENTRY_IST
457#define DEFINE_IDTENTRY_DEBUG		DEFINE_IDTENTRY_IST
458#define DEFINE_IDTENTRY_DEBUG_USER	DEFINE_IDTENTRY_NOIST
459#define DEFINE_FREDENTRY_DEBUG		DEFINE_FREDENTRY_RAW
460#endif
461
462void idt_install_sysvec(unsigned int n, const void *function);
463void fred_install_sysvec(unsigned int vector, const idtentry_t function);
464
465#define sysvec_install(vector, function) {				\
466	if (IS_ENABLED(CONFIG_X86_FRED))				\
467		fred_install_sysvec(vector, function);			\
468	if (!cpu_feature_enabled(X86_FEATURE_FRED))			\
469		idt_install_sysvec(vector, asm_##function);		\
470}
471
472#else /* !__ASSEMBLER__ */
473
474/*
475 * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs.
476 */
477#define DECLARE_IDTENTRY(vector, func)					\
478	idtentry vector asm_##func func has_error_code=0
479
480#define DECLARE_IDTENTRY_ERRORCODE(vector, func)			\
481	idtentry vector asm_##func func has_error_code=1
482
483/* Special case for 32bit IRET 'trap'. Do not emit ASM code */
484#define DECLARE_IDTENTRY_SW(vector, func)
485
486#define DECLARE_IDTENTRY_RAW(vector, func)				\
487	DECLARE_IDTENTRY(vector, func)
488
489#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)			\
490	DECLARE_IDTENTRY_ERRORCODE(vector, func)
491
492/* Entries for common/spurious (device) interrupts */
493#define DECLARE_IDTENTRY_IRQ(vector, func)				\
494	idtentry_irq vector func
495
496/* System vector entries */
497#define DECLARE_IDTENTRY_SYSVEC(vector, func)				\
498	DECLARE_IDTENTRY(vector, func)
499
500#ifdef CONFIG_X86_64
501# define DECLARE_IDTENTRY_MCE(vector, func)				\
502	idtentry_mce_db vector asm_##func func
503
504# define DECLARE_IDTENTRY_DEBUG(vector, func)				\
505	idtentry_mce_db vector asm_##func func
506
507# define DECLARE_IDTENTRY_DF(vector, func)				\
508	idtentry_df vector asm_##func func
509
510# define DECLARE_IDTENTRY_XENCB(vector, func)				\
511	DECLARE_IDTENTRY(vector, func)
512
513# define DECLARE_IDTENTRY_VC(vector, func)				\
514	idtentry_vc vector asm_##func func
515
516#else
517# define DECLARE_IDTENTRY_MCE(vector, func)				\
518	DECLARE_IDTENTRY(vector, func)
519
520/* No ASM emitted for DF as this goes through a C shim */
521# define DECLARE_IDTENTRY_DF(vector, func)
522
523/* No ASM emitted for XEN hypervisor callback */
524# define DECLARE_IDTENTRY_XENCB(vector, func)
525
526#endif
527
528/* No ASM code emitted for NMI */
529#define DECLARE_IDTENTRY_NMI(vector, func)
530
531/*
532 * ASM code to emit the common vector entry stubs where each stub is
533 * packed into IDT_ALIGN bytes.
534 *
535 * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
536 * GCC treats the local vector variable as unsigned int and would expand
537 * all vectors above 0x7F to a 5 byte push. The original code did an
538 * adjustment of the vector number to be in the signed byte range to avoid
539 * this. While clever it's mindboggling counterintuitive and requires the
540 * odd conversion back to a real vector number in the C entry points. Using
541 * .byte achieves the same thing and the only fixup needed in the C entry
542 * point is to mask off the bits above bit 7 because the push is sign
543 * extending.
544 */
545	.align IDT_ALIGN
546SYM_CODE_START(irq_entries_start)
547    vector=FIRST_EXTERNAL_VECTOR
548    .rept NR_EXTERNAL_VECTORS
549	UNWIND_HINT_IRET_REGS
5500 :
551	ENDBR
552	.byte	0x6a, vector
553	jmp	asm_common_interrupt
554	/* Ensure that the above is IDT_ALIGN bytes max */
555	.fill 0b + IDT_ALIGN - ., 1, 0xcc
556	vector = vector+1
557    .endr
558SYM_CODE_END(irq_entries_start)
559
560#ifdef CONFIG_X86_LOCAL_APIC
561	.align IDT_ALIGN
562SYM_CODE_START(spurious_entries_start)
563    vector=FIRST_SYSTEM_VECTOR
564    .rept NR_SYSTEM_VECTORS
565	UNWIND_HINT_IRET_REGS
5660 :
567	ENDBR
568	.byte	0x6a, vector
569	jmp	asm_spurious_interrupt
570	/* Ensure that the above is IDT_ALIGN bytes max */
571	.fill 0b + IDT_ALIGN - ., 1, 0xcc
572	vector = vector+1
573    .endr
574SYM_CODE_END(spurious_entries_start)
575#endif
576
577#endif /* __ASSEMBLER__ */
578
579/*
580 * The actual entry points. Note that DECLARE_IDTENTRY*() serves two
581 * purposes:
582 *  - provide the function declarations when included from C-Code
583 *  - emit the ASM stubs when included from entry_32/64.S
584 *
585 * This avoids duplicate defines and ensures that everything is consistent.
586 */
587
588/*
589 * Dummy trap number so the low level ASM macro vector number checks do not
590 * match which results in emitting plain IDTENTRY stubs without bells and
591 * whistles.
592 */
593#define X86_TRAP_OTHER		0xFFFF
594
595/* Simple exception entry points. No hardware error code */
596DECLARE_IDTENTRY(X86_TRAP_DE,		exc_divide_error);
597DECLARE_IDTENTRY(X86_TRAP_OF,		exc_overflow);
598DECLARE_IDTENTRY(X86_TRAP_BR,		exc_bounds);
599DECLARE_IDTENTRY(X86_TRAP_NM,		exc_device_not_available);
600DECLARE_IDTENTRY(X86_TRAP_OLD_MF,	exc_coproc_segment_overrun);
601DECLARE_IDTENTRY(X86_TRAP_SPURIOUS,	exc_spurious_interrupt_bug);
602DECLARE_IDTENTRY(X86_TRAP_MF,		exc_coprocessor_error);
603DECLARE_IDTENTRY(X86_TRAP_XF,		exc_simd_coprocessor_error);
604
605/* 32bit software IRET trap. Do not emit ASM code */
606DECLARE_IDTENTRY_SW(X86_TRAP_IRET,	iret_error);
607
608/* Simple exception entries with error code pushed by hardware */
609DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS,	exc_invalid_tss);
610DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP,	exc_segment_not_present);
611DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS,	exc_stack_segment);
612DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP,	exc_general_protection);
613DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC,	exc_alignment_check);
614
615/* Raw exception entries which need extra work */
616DECLARE_IDTENTRY_RAW(X86_TRAP_UD,		exc_invalid_op);
617DECLARE_IDTENTRY_RAW(X86_TRAP_BP,		exc_int3);
618DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF,	exc_page_fault);
619
620#if defined(CONFIG_IA32_EMULATION)
621DECLARE_IDTENTRY_RAW(IA32_SYSCALL_VECTOR,	int80_emulation);
622#endif
623
624#ifdef CONFIG_X86_MCE
625#ifdef CONFIG_X86_64
626DECLARE_IDTENTRY_MCE(X86_TRAP_MC,	exc_machine_check);
627#else
628DECLARE_IDTENTRY_RAW(X86_TRAP_MC,	exc_machine_check);
629#endif
630#ifdef CONFIG_XEN_PV
631DECLARE_IDTENTRY_RAW(X86_TRAP_MC,	xenpv_exc_machine_check);
632#endif
633#endif
634
635/* NMI */
636
637#if IS_ENABLED(CONFIG_KVM_INTEL)
638/*
639 * Special entry point for VMX which invokes this on the kernel stack, even for
640 * 64-bit, i.e. without using an IST.  asm_exc_nmi() requires an IST to work
641 * correctly vs. the NMI 'executing' marker.  Used for 32-bit kernels as well
642 * to avoid more ifdeffery.
643 */
644DECLARE_IDTENTRY(X86_TRAP_NMI,		exc_nmi_kvm_vmx);
645#endif
646
647DECLARE_IDTENTRY_NMI(X86_TRAP_NMI,	exc_nmi);
648#ifdef CONFIG_XEN_PV
649DECLARE_IDTENTRY_RAW(X86_TRAP_NMI,	xenpv_exc_nmi);
650#endif
651
652/* #DB */
653#ifdef CONFIG_X86_64
654DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB,	exc_debug);
655#else
656DECLARE_IDTENTRY_RAW(X86_TRAP_DB,	exc_debug);
657#endif
658#ifdef CONFIG_XEN_PV
659DECLARE_IDTENTRY_RAW(X86_TRAP_DB,	xenpv_exc_debug);
660#endif
661
662/* #DF */
663DECLARE_IDTENTRY_DF(X86_TRAP_DF,	exc_double_fault);
664#ifdef CONFIG_XEN_PV
665DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF,	xenpv_exc_double_fault);
666#endif
667
668/* #CP */
669#ifdef CONFIG_X86_CET
670DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP,	exc_control_protection);
671#endif
672
673/* #VC */
674#ifdef CONFIG_AMD_MEM_ENCRYPT
675DECLARE_IDTENTRY_VC(X86_TRAP_VC,	exc_vmm_communication);
676#endif
677
678#ifdef CONFIG_XEN_PV
679DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER,	exc_xen_hypervisor_callback);
680DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER,	exc_xen_unknown_trap);
681#endif
682
683#ifdef CONFIG_INTEL_TDX_GUEST
684DECLARE_IDTENTRY(X86_TRAP_VE,		exc_virtualization_exception);
685#endif
686
687/* Device interrupts common/spurious */
688DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,	common_interrupt);
689#ifdef CONFIG_X86_LOCAL_APIC
690DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,	spurious_interrupt);
691#endif
692
693/* System vector entry points */
694#ifdef CONFIG_X86_LOCAL_APIC
695DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR,		sysvec_error_interrupt);
696DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR,		sysvec_spurious_apic_interrupt);
697DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR,		sysvec_apic_timer_interrupt);
698DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR,	sysvec_x86_platform_ipi);
699#endif
700
701#ifdef CONFIG_SMP
702DECLARE_IDTENTRY(RESCHEDULE_VECTOR,			sysvec_reschedule_ipi);
703DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR,			sysvec_reboot);
704DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR,	sysvec_call_function_single);
705DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR,		sysvec_call_function);
706#else
707# define fred_sysvec_reschedule_ipi			NULL
708# define fred_sysvec_reboot				NULL
709# define fred_sysvec_call_function_single		NULL
710# define fred_sysvec_call_function			NULL
711#endif
712
713#ifdef CONFIG_X86_LOCAL_APIC
714# ifdef CONFIG_X86_MCE_THRESHOLD
715DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR,		sysvec_threshold);
716# else
717# define fred_sysvec_threshold				NULL
718# endif
719
720# ifdef CONFIG_X86_MCE_AMD
721DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR,		sysvec_deferred_error);
722# else
723# define fred_sysvec_deferred_error			NULL
724# endif
725
726# ifdef CONFIG_X86_THERMAL_VECTOR
727DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR,		sysvec_thermal);
728# else
729# define fred_sysvec_thermal				NULL
730# endif
731
732# ifdef CONFIG_IRQ_WORK
733DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR,		sysvec_irq_work);
734# else
735# define fred_sysvec_irq_work				NULL
736# endif
737#endif
738
739#if IS_ENABLED(CONFIG_KVM)
740DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR,		sysvec_kvm_posted_intr_ipi);
741DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR,	sysvec_kvm_posted_intr_wakeup_ipi);
742DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR,	sysvec_kvm_posted_intr_nested_ipi);
743#else
744# define fred_sysvec_kvm_posted_intr_ipi		NULL
745# define fred_sysvec_kvm_posted_intr_wakeup_ipi		NULL
746# define fred_sysvec_kvm_posted_intr_nested_ipi		NULL
747#endif
748
749# ifdef CONFIG_X86_POSTED_MSI
750DECLARE_IDTENTRY_SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR,	sysvec_posted_msi_notification);
751#else
752# define fred_sysvec_posted_msi_notification		NULL
753# endif
754
755#if IS_ENABLED(CONFIG_HYPERV)
756DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_hyperv_callback);
757DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR,	sysvec_hyperv_reenlightenment);
758DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR,		sysvec_hyperv_stimer0);
759#endif
760
761#if IS_ENABLED(CONFIG_ACRN_GUEST)
762DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_acrn_hv_callback);
763#endif
764
765#ifdef CONFIG_XEN_PVHVM
766DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_xen_hvm_callback);
767#endif
768
769#ifdef CONFIG_KVM_GUEST
770DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR,	sysvec_kvm_asyncpf_interrupt);
771#endif
772
773#undef X86_TRAP_OTHER
774
775#endif