entry_64_fred.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * The actual FRED entry points.
  4 */
  5
  6#include <linux/export.h>
  7#include <linux/kvm_types.h>
  8
  9#include <asm/asm.h>
 10#include <asm/fred.h>
 11#include <asm/segment.h>
 12
 13#include "calling.h"
 14
 15	.code64
 16	.section .noinstr.text, "ax"
 17
 18.macro FRED_ENTER
 19	UNWIND_HINT_END_OF_STACK
 20	ANNOTATE_NOENDBR
 21	PUSH_AND_CLEAR_REGS
 22	movq	%rsp, %rdi	/* %rdi -> pt_regs */
 23.endm
 24
 25.macro FRED_EXIT
 26	UNWIND_HINT_REGS
 27	POP_REGS
 28.endm
 29
 30/*
 31 * The new RIP value that FRED event delivery establishes is
 32 * IA32_FRED_CONFIG & ~FFFH for events that occur in ring 3.
 33 * Thus the FRED ring 3 entry point must be 4K page aligned.
 34 */
 35	.align 4096
 36
 37SYM_CODE_START_NOALIGN(asm_fred_entrypoint_user)
 38	FRED_ENTER
 39	call	fred_entry_from_user
 40SYM_INNER_LABEL(asm_fred_exit_user, SYM_L_GLOBAL)
 41	FRED_EXIT
 421:	ERETU
 43
 44	_ASM_EXTABLE_TYPE(1b, asm_fred_entrypoint_user, EX_TYPE_ERETU)
 45SYM_CODE_END(asm_fred_entrypoint_user)
 46
 47/*
 48 * The new RIP value that FRED event delivery establishes is
 49 * (IA32_FRED_CONFIG & ~FFFH) + 256 for events that occur in
 50 * ring 0, i.e., asm_fred_entrypoint_user + 256.
 51 */
 52	.org asm_fred_entrypoint_user + 256, 0xcc
 53SYM_CODE_START_NOALIGN(asm_fred_entrypoint_kernel)
 54	FRED_ENTER
 55	call	fred_entry_from_kernel
 56	FRED_EXIT
 57	ERETS
 58SYM_CODE_END(asm_fred_entrypoint_kernel)
 59
 60#if IS_ENABLED(CONFIG_KVM_INTEL)
 61SYM_FUNC_START(asm_fred_entry_from_kvm)
 62	ANNOTATE_NOENDBR
 63	push %rbp
 64	mov %rsp, %rbp
 65
 66	UNWIND_HINT_SAVE
 67
 68	/*
 69	 * Both IRQ and NMI from VMX can be handled on current task stack
 70	 * because there is no need to protect from reentrancy and the call
 71	 * stack leading to this helper is effectively constant and shallow
 72	 * (relatively speaking). Do the same when FRED is active, i.e., no
 73	 * need to check current stack level for a stack switch.
 74	 *
 75	 * Emulate the FRED-defined redzone and stack alignment.
 76	 */
 77	sub $(FRED_CONFIG_REDZONE_AMOUNT << 6), %rsp
 78	and $FRED_STACK_FRAME_RSP_MASK, %rsp
 79
 80	/*
 81	 * Start to push a FRED stack frame, which is always 64 bytes:
 82	 *
 83	 * +--------+-----------------+
 84	 * | Bytes  | Usage           |
 85	 * +--------+-----------------+
 86	 * | 63:56  | Reserved        |
 87	 * | 55:48  | Event Data      |
 88	 * | 47:40  | SS + Event Info |
 89	 * | 39:32  | RSP             |
 90	 * | 31:24  | RFLAGS          |
 91	 * | 23:16  | CS + Aux Info   |
 92	 * |  15:8  | RIP             |
 93	 * |   7:0  | Error Code      |
 94	 * +--------+-----------------+
 95	 */
 96	push $0				/* Reserved, must be 0 */
 97	push $0				/* Event data, 0 for IRQ/NMI */
 98	push %rdi			/* fred_ss handed in by the caller */
 99	push %rbp
100	pushf
101	push $__KERNEL_CS
102
103	/*
104	 * Unlike the IDT event delivery, FRED _always_ pushes an error code
105	 * after pushing the return RIP, thus the CALL instruction CANNOT be
106	 * used here to push the return RIP, otherwise there is no chance to
107	 * push an error code before invoking the IRQ/NMI handler.
108	 *
109	 * Use LEA to get the return RIP and push it, then push an error code.
110	 */
111	lea 1f(%rip), %rax
112	push %rax				/* Return RIP */
113	push $0					/* Error code, 0 for IRQ/NMI */
114
115	PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0
116
117	movq %rsp, %rdi				/* %rdi -> pt_regs */
118	/*
119	 * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx}
120	 * are clobbered, which corresponds to: arguments, extra caller-saved
121	 * and return. All registers a C function is allowed to clobber.
122	 *
123	 * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15}
124	 * are untouched, with the exception of rbp, which carries the stack
125	 * frame and will be restored before exit.
126	 *
127	 * Further calling another C function will not alter this state.
128	 */
129	call __fred_entry_from_kvm		/* Call the C entry point */
130
131	/*
132	 * When FRED, use ERETS to potentially clear NMIs, otherwise simply
133	 * restore the stack pointer.
134	 */
135	ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
136	            __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \
137		    X86_FEATURE_FRED
138
1391:	/*
140	 * Objtool doesn't understand ERETS, and the cfi register state is
141	 * different from initial_func_cfi due to PUSH_REGS. Tell it the state
142	 * is similar to where UNWIND_HINT_SAVE is.
143	 */
144	UNWIND_HINT_RESTORE
145
146	pop %rbp
147	RET
148
149SYM_FUNC_END(asm_fred_entry_from_kvm)
150EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
151#endif