entry_fred.c

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * The FRED specific kernel/user entry functions which are invoked from
  4 * assembly code and dispatch to the associated handlers.
  5 */
  6#include <linux/kernel.h>
  7#include <linux/kdebug.h>
  8#include <linux/nospec.h>
  9
 10#include <asm/desc.h>
 11#include <asm/fred.h>
 12#include <asm/idtentry.h>
 13#include <asm/syscall.h>
 14#include <asm/trapnr.h>
 15#include <asm/traps.h>
 16
 17/* FRED EVENT_TYPE_OTHER vector numbers */
 18#define FRED_SYSCALL			1
 19#define FRED_SYSENTER			2
 20
 21static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
 22{
 23	irqentry_state_t irq_state = irqentry_nmi_enter(regs);
 24
 25	instrumentation_begin();
 26
 27	/* Panic on events from a high stack level */
 28	if (regs->fred_cs.sl > 0) {
 29		pr_emerg("PANIC: invalid or fatal FRED event; event type %u "
 30			 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
 31			 regs->fred_ss.type, regs->fred_ss.vector, error_code,
 32			 fred_event_data(regs), regs->cs, regs->ip);
 33		die("invalid or fatal FRED event", regs, error_code);
 34		panic("invalid or fatal FRED event");
 35	} else {
 36		unsigned long flags = oops_begin();
 37		int sig = SIGKILL;
 38
 39		pr_alert("BUG: invalid or fatal FRED event; event type %u "
 40			 "vector %u error 0x%lx aux 0x%lx at %04x:%016lx\n",
 41			 regs->fred_ss.type, regs->fred_ss.vector, error_code,
 42			 fred_event_data(regs), regs->cs, regs->ip);
 43
 44		if (__die("Invalid or fatal FRED event", regs, error_code))
 45			sig = 0;
 46
 47		oops_end(flags, regs, sig);
 48	}
 49
 50	instrumentation_end();
 51	irqentry_nmi_exit(regs, irq_state);
 52}
 53
 54static noinstr void fred_intx(struct pt_regs *regs)
 55{
 56	switch (regs->fred_ss.vector) {
 57	/* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */
 58	case X86_TRAP_BP:
 59		return exc_int3(regs);
 60
 61	/* Opcode 0xcd, 0x4, NOT INTO (opcode 0xce) */
 62	case X86_TRAP_OF:
 63		return exc_overflow(regs);
 64
 65#ifdef CONFIG_IA32_EMULATION
 66	/* INT80 */
 67	case IA32_SYSCALL_VECTOR:
 68		if (ia32_enabled())
 69			return fred_int80_emulation(regs);
 70		fallthrough;
 71#endif
 72
 73	default:
 74		return exc_general_protection(regs, 0);
 75	}
 76}
 77
 78static __always_inline void fred_other(struct pt_regs *regs)
 79{
 80	/* The compiler can fold these conditions into a single test */
 81	if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.l)) {
 82		regs->orig_ax = regs->ax;
 83		regs->ax = -ENOSYS;
 84		do_syscall_64(regs, regs->orig_ax);
 85		return;
 86	} else if (ia32_enabled() &&
 87		   likely(regs->fred_ss.vector == FRED_SYSENTER && !regs->fred_ss.l)) {
 88		regs->orig_ax = regs->ax;
 89		regs->ax = -ENOSYS;
 90		do_fast_syscall_32(regs);
 91		return;
 92	} else {
 93		exc_invalid_op(regs);
 94		return;
 95	}
 96}
 97
 98#define SYSVEC(_vector, _function) [_vector - FIRST_SYSTEM_VECTOR] = fred_sysvec_##_function
 99
100static idtentry_t sysvec_table[NR_SYSTEM_VECTORS] __ro_after_init = {
101	SYSVEC(ERROR_APIC_VECTOR,		error_interrupt),
102	SYSVEC(SPURIOUS_APIC_VECTOR,		spurious_apic_interrupt),
103	SYSVEC(LOCAL_TIMER_VECTOR,		apic_timer_interrupt),
104	SYSVEC(X86_PLATFORM_IPI_VECTOR,		x86_platform_ipi),
105
106	SYSVEC(RESCHEDULE_VECTOR,		reschedule_ipi),
107	SYSVEC(CALL_FUNCTION_SINGLE_VECTOR,	call_function_single),
108	SYSVEC(CALL_FUNCTION_VECTOR,		call_function),
109	SYSVEC(REBOOT_VECTOR,			reboot),
110
111	SYSVEC(THRESHOLD_APIC_VECTOR,		threshold),
112	SYSVEC(DEFERRED_ERROR_VECTOR,		deferred_error),
113	SYSVEC(THERMAL_APIC_VECTOR,		thermal),
114
115	SYSVEC(IRQ_WORK_VECTOR,			irq_work),
116
117	SYSVEC(POSTED_INTR_VECTOR,		kvm_posted_intr_ipi),
118	SYSVEC(POSTED_INTR_WAKEUP_VECTOR,	kvm_posted_intr_wakeup_ipi),
119	SYSVEC(POSTED_INTR_NESTED_VECTOR,	kvm_posted_intr_nested_ipi),
120
121	SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR,	posted_msi_notification),
122};
123
124static bool fred_setup_done __initdata;
125
126void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler)
127{
128	if (WARN_ON_ONCE(sysvec < FIRST_SYSTEM_VECTOR))
129		return;
130
131	if (WARN_ON_ONCE(fred_setup_done))
132		return;
133
134	if (!WARN_ON_ONCE(sysvec_table[sysvec - FIRST_SYSTEM_VECTOR]))
135		 sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler;
136}
137
138static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs)
139{
140	spurious_interrupt(regs, regs->fred_ss.vector);
141}
142
143void __init fred_complete_exception_setup(void)
144{
145	unsigned int vector;
146
147	for (vector = 0; vector < FIRST_EXTERNAL_VECTOR; vector++)
148		set_bit(vector, system_vectors);
149
150	for (vector = 0; vector < NR_SYSTEM_VECTORS; vector++) {
151		if (sysvec_table[vector])
152			set_bit(vector + FIRST_SYSTEM_VECTOR, system_vectors);
153		else
154			sysvec_table[vector] = fred_handle_spurious_interrupt;
155	}
156	fred_setup_done = true;
157}
158
159static noinstr void fred_extint(struct pt_regs *regs)
160{
161	unsigned int vector = regs->fred_ss.vector;
162
163	if (WARN_ON_ONCE(vector < FIRST_EXTERNAL_VECTOR))
164		return;
165
166	if (likely(vector >= FIRST_SYSTEM_VECTOR)) {
167		irqentry_state_t state = irqentry_enter(regs);
168
169		instrumentation_begin();
170		sysvec_table[array_index_nospec(vector - FIRST_SYSTEM_VECTOR,
171						NR_SYSTEM_VECTORS)](regs);
172		instrumentation_end();
173		irqentry_exit(regs, state);
174	} else {
175		common_interrupt(regs, vector);
176	}
177}
178
179#ifdef CONFIG_AMD_MEM_ENCRYPT
180noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
181{
182	if (user_mode(regs))
183		return user_exc_vmm_communication(regs, error_code);
184	else
185		return kernel_exc_vmm_communication(regs, error_code);
186}
187#endif
188
189static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
190{
191	/* Optimize for #PF. That's the only exception which matters performance wise */
192	if (likely(regs->fred_ss.vector == X86_TRAP_PF))
193		return exc_page_fault(regs, error_code);
194
195	switch (regs->fred_ss.vector) {
196	case X86_TRAP_DE: return exc_divide_error(regs);
197	case X86_TRAP_DB: return fred_exc_debug(regs);
198	case X86_TRAP_BR: return exc_bounds(regs);
199	case X86_TRAP_UD: return exc_invalid_op(regs);
200	case X86_TRAP_NM: return exc_device_not_available(regs);
201	case X86_TRAP_DF: return exc_double_fault(regs, error_code);
202	case X86_TRAP_TS: return exc_invalid_tss(regs, error_code);
203	case X86_TRAP_NP: return exc_segment_not_present(regs, error_code);
204	case X86_TRAP_SS: return exc_stack_segment(regs, error_code);
205	case X86_TRAP_GP: return exc_general_protection(regs, error_code);
206	case X86_TRAP_MF: return exc_coprocessor_error(regs);
207	case X86_TRAP_AC: return exc_alignment_check(regs, error_code);
208	case X86_TRAP_XF: return exc_simd_coprocessor_error(regs);
209
210#ifdef CONFIG_X86_MCE
211	case X86_TRAP_MC: return fred_exc_machine_check(regs);
212#endif
213#ifdef CONFIG_INTEL_TDX_GUEST
214	case X86_TRAP_VE: return exc_virtualization_exception(regs);
215#endif
216#ifdef CONFIG_X86_CET
217	case X86_TRAP_CP: return exc_control_protection(regs, error_code);
218#endif
219#ifdef CONFIG_AMD_MEM_ENCRYPT
220	case X86_TRAP_VC: return exc_vmm_communication(regs, error_code);
221#endif
222
223	default: return fred_bad_type(regs, error_code);
224	}
225
226}
227
228static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
229{
230	switch (regs->fred_ss.vector) {
231	case X86_TRAP_BP: return exc_int3(regs);
232	case X86_TRAP_OF: return exc_overflow(regs);
233	default: return fred_bad_type(regs, error_code);
234	}
235}
236
237__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
238{
239	unsigned long error_code = regs->orig_ax;
240
241	/* Invalidate orig_ax so that syscall_get_nr() works correctly */
242	regs->orig_ax = -1;
243
244	switch (regs->fred_ss.type) {
245	case EVENT_TYPE_EXTINT:
246		return fred_extint(regs);
247	case EVENT_TYPE_NMI:
248		if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
249			return fred_exc_nmi(regs);
250		break;
251	case EVENT_TYPE_HWEXC:
252		return fred_hwexc(regs, error_code);
253	case EVENT_TYPE_SWINT:
254		return fred_intx(regs);
255	case EVENT_TYPE_PRIV_SWEXC:
256		if (likely(regs->fred_ss.vector == X86_TRAP_DB))
257			return fred_exc_debug(regs);
258		break;
259	case EVENT_TYPE_SWEXC:
260		return fred_swexc(regs, error_code);
261	case EVENT_TYPE_OTHER:
262		return fred_other(regs);
263	default: break;
264	}
265
266	return fred_bad_type(regs, error_code);
267}
268
269__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
270{
271	unsigned long error_code = regs->orig_ax;
272
273	/* Invalidate orig_ax so that syscall_get_nr() works correctly */
274	regs->orig_ax = -1;
275
276	switch (regs->fred_ss.type) {
277	case EVENT_TYPE_EXTINT:
278		return fred_extint(regs);
279	case EVENT_TYPE_NMI:
280		if (likely(regs->fred_ss.vector == X86_TRAP_NMI))
281			return fred_exc_nmi(regs);
282		break;
283	case EVENT_TYPE_HWEXC:
284		return fred_hwexc(regs, error_code);
285	case EVENT_TYPE_PRIV_SWEXC:
286		if (likely(regs->fred_ss.vector == X86_TRAP_DB))
287			return fred_exc_debug(regs);
288		break;
289	case EVENT_TYPE_SWEXC:
290		return fred_swexc(regs, error_code);
291	default: break;
292	}
293
294	return fred_bad_type(regs, error_code);
295}
296
297#if IS_ENABLED(CONFIG_KVM_INTEL)
298__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
299{
300	switch (regs->fred_ss.type) {
301	case EVENT_TYPE_EXTINT:
302		return fred_extint(regs);
303	case EVENT_TYPE_NMI:
304		return fred_exc_nmi(regs);
305	default:
306		WARN_ON_ONCE(1);
307	}
308}
309#endif