vmenter.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#include <linux/linkage.h>
  3#include <asm/asm.h>
  4#include <asm/asm-offsets.h>
  5#include <asm/bitsperlong.h>
  6#include <asm/frame.h>
  7#include <asm/kvm_vcpu_regs.h>
  8#include <asm/nospec-branch.h>
  9#include "kvm-asm-offsets.h"
 10
 11#define WORD_SIZE (BITS_PER_LONG / 8)
 12
 13/* Intentionally omit RAX as it's context switched by hardware */
 14#define VCPU_RCX	(SVM_vcpu_arch_regs + __VCPU_REGS_RCX * WORD_SIZE)
 15#define VCPU_RDX	(SVM_vcpu_arch_regs + __VCPU_REGS_RDX * WORD_SIZE)
 16#define VCPU_RBX	(SVM_vcpu_arch_regs + __VCPU_REGS_RBX * WORD_SIZE)
 17/* Intentionally omit RSP as it's context switched by hardware */
 18#define VCPU_RBP	(SVM_vcpu_arch_regs + __VCPU_REGS_RBP * WORD_SIZE)
 19#define VCPU_RSI	(SVM_vcpu_arch_regs + __VCPU_REGS_RSI * WORD_SIZE)
 20#define VCPU_RDI	(SVM_vcpu_arch_regs + __VCPU_REGS_RDI * WORD_SIZE)
 21
 22#ifdef CONFIG_X86_64
 23#define VCPU_R8		(SVM_vcpu_arch_regs + __VCPU_REGS_R8  * WORD_SIZE)
 24#define VCPU_R9		(SVM_vcpu_arch_regs + __VCPU_REGS_R9  * WORD_SIZE)
 25#define VCPU_R10	(SVM_vcpu_arch_regs + __VCPU_REGS_R10 * WORD_SIZE)
 26#define VCPU_R11	(SVM_vcpu_arch_regs + __VCPU_REGS_R11 * WORD_SIZE)
 27#define VCPU_R12	(SVM_vcpu_arch_regs + __VCPU_REGS_R12 * WORD_SIZE)
 28#define VCPU_R13	(SVM_vcpu_arch_regs + __VCPU_REGS_R13 * WORD_SIZE)
 29#define VCPU_R14	(SVM_vcpu_arch_regs + __VCPU_REGS_R14 * WORD_SIZE)
 30#define VCPU_R15	(SVM_vcpu_arch_regs + __VCPU_REGS_R15 * WORD_SIZE)
 31#endif
 32
 33#define SVM_vmcb01_pa	(SVM_vmcb01 + KVM_VMCB_pa)
 34
 35.section .noinstr.text, "ax"
 36
 37.macro RESTORE_GUEST_SPEC_CTRL
 38	/* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */
 39	ALTERNATIVE_2 "", \
 40		"jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \
 41		"", X86_FEATURE_V_SPEC_CTRL
 42801:
 43.endm
 44.macro RESTORE_GUEST_SPEC_CTRL_BODY
 45800:
 46	/*
 47	 * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
 48	 * host's, write the MSR.  This is kept out-of-line so that the common
 49	 * case does not have to jump.
 50	 *
 51	 * IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
 52	 * there must not be any returns or indirect branches between this code
 53	 * and vmentry.
 54	 */
 55#ifdef CONFIG_X86_64
 56	mov SVM_spec_ctrl(%rdi), %rdx
 57	cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx
 58	je 801b
 59	movl %edx, %eax
 60	shr $32, %rdx
 61#else
 62	mov SVM_spec_ctrl(%edi), %eax
 63	mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx
 64	xor %eax, %ecx
 65	mov SVM_spec_ctrl + 4(%edi), %edx
 66	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi
 67	xor %edx, %esi
 68	or %esi, %ecx
 69	je 801b
 70#endif
 71	mov $MSR_IA32_SPEC_CTRL, %ecx
 72	wrmsr
 73	jmp 801b
 74.endm
 75
 76.macro RESTORE_HOST_SPEC_CTRL
 77	/* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */
 78	ALTERNATIVE_2 "", \
 79		"jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \
 80		"", X86_FEATURE_V_SPEC_CTRL
 81901:
 82.endm
 83.macro RESTORE_HOST_SPEC_CTRL_BODY spec_ctrl_intercepted:req
 84900:
 85	/* Same for after vmexit.  */
 86	mov $MSR_IA32_SPEC_CTRL, %ecx
 87
 88	/*
 89	 * Load the value that the guest had written into MSR_IA32_SPEC_CTRL,
 90	 * if it was not intercepted during guest execution.
 91	 */
 92	cmpb $0, \spec_ctrl_intercepted
 93	jnz 998f
 94	rdmsr
 95	movl %eax, SVM_spec_ctrl(%_ASM_DI)
 96	movl %edx, SVM_spec_ctrl + 4(%_ASM_DI)
 97998:
 98	/* Now restore the host value of the MSR if different from the guest's.  */
 99#ifdef CONFIG_X86_64
100	mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx
101	cmp SVM_spec_ctrl(%rdi), %rdx
102	je 901b
103	movl %edx, %eax
104	shr $32, %rdx
105#else
106	mov PER_CPU_VAR(x86_spec_ctrl_current), %eax
107	mov SVM_spec_ctrl(%edi), %esi
108	xor %eax, %esi
109	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edx
110	mov SVM_spec_ctrl + 4(%edi), %edi
111	xor %edx, %edi
112	or %edi, %esi
113	je 901b
114#endif
115	wrmsr
116	jmp 901b
117.endm
118
119#define SVM_CLEAR_CPU_BUFFERS \
120	ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF_VM
121
122/**
123 * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
124 * @svm:	struct vcpu_svm *
125 * @spec_ctrl_intercepted: bool
126 */
127SYM_FUNC_START(__svm_vcpu_run)
128	push %_ASM_BP
129	mov  %_ASM_SP, %_ASM_BP
130#ifdef CONFIG_X86_64
131	push %r15
132	push %r14
133	push %r13
134	push %r12
135#else
136	push %edi
137	push %esi
138#endif
139	push %_ASM_BX
140
141	/*
142	 * Save variables needed after vmexit on the stack, in inverse
143	 * order compared to when they are needed.
144	 */
145
146	/* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL.  */
147	push %_ASM_ARG2
148
149	/* Needed to restore access to percpu variables.  */
150	__ASM_SIZE(push) PER_CPU_VAR(svm_data + SD_save_area_pa)
151
152	/* Finally save @svm. */
153	push %_ASM_ARG1
154
155.ifnc _ASM_ARG1, _ASM_DI
156	/*
157	 * Stash @svm in RDI early. On 32-bit, arguments are in RAX, RCX
158	 * and RDX which are clobbered by RESTORE_GUEST_SPEC_CTRL.
159	 */
160	mov %_ASM_ARG1, %_ASM_DI
161.endif
162
163	/* Clobbers RAX, RCX, RDX (and ESI on 32-bit), consumes RDI (@svm). */
164	RESTORE_GUEST_SPEC_CTRL
165
166	/*
167	 * Use a single vmcb (vmcb01 because it's always valid) for
168	 * context switching guest state via VMLOAD/VMSAVE, that way
169	 * the state doesn't need to be copied between vmcb01 and
170	 * vmcb02 when switching vmcbs for nested virtualization.
171	 */
172	mov SVM_vmcb01_pa(%_ASM_DI), %_ASM_AX
1731:	vmload %_ASM_AX
1742:
175
176	/* Get svm->current_vmcb->pa into RAX. */
177	mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX
178	mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX
179
180	/* Load guest registers. */
181	mov VCPU_RCX(%_ASM_DI), %_ASM_CX
182	mov VCPU_RDX(%_ASM_DI), %_ASM_DX
183	mov VCPU_RBX(%_ASM_DI), %_ASM_BX
184	mov VCPU_RBP(%_ASM_DI), %_ASM_BP
185	mov VCPU_RSI(%_ASM_DI), %_ASM_SI
186#ifdef CONFIG_X86_64
187	mov VCPU_R8 (%_ASM_DI),  %r8
188	mov VCPU_R9 (%_ASM_DI),  %r9
189	mov VCPU_R10(%_ASM_DI), %r10
190	mov VCPU_R11(%_ASM_DI), %r11
191	mov VCPU_R12(%_ASM_DI), %r12
192	mov VCPU_R13(%_ASM_DI), %r13
193	mov VCPU_R14(%_ASM_DI), %r14
194	mov VCPU_R15(%_ASM_DI), %r15
195#endif
196	mov VCPU_RDI(%_ASM_DI), %_ASM_DI
197
198	/* Clobbers EFLAGS.ZF */
199	SVM_CLEAR_CPU_BUFFERS
200
201	/* Enter guest mode */
2023:	vmrun %_ASM_AX
2034:
204	/* Pop @svm to RAX while it's the only available register. */
205	pop %_ASM_AX
206
207	/* Save all guest registers.  */
208	mov %_ASM_CX,   VCPU_RCX(%_ASM_AX)
209	mov %_ASM_DX,   VCPU_RDX(%_ASM_AX)
210	mov %_ASM_BX,   VCPU_RBX(%_ASM_AX)
211	mov %_ASM_BP,   VCPU_RBP(%_ASM_AX)
212	mov %_ASM_SI,   VCPU_RSI(%_ASM_AX)
213	mov %_ASM_DI,   VCPU_RDI(%_ASM_AX)
214#ifdef CONFIG_X86_64
215	mov %r8,  VCPU_R8 (%_ASM_AX)
216	mov %r9,  VCPU_R9 (%_ASM_AX)
217	mov %r10, VCPU_R10(%_ASM_AX)
218	mov %r11, VCPU_R11(%_ASM_AX)
219	mov %r12, VCPU_R12(%_ASM_AX)
220	mov %r13, VCPU_R13(%_ASM_AX)
221	mov %r14, VCPU_R14(%_ASM_AX)
222	mov %r15, VCPU_R15(%_ASM_AX)
223#endif
224
225	/* @svm can stay in RDI from now on.  */
226	mov %_ASM_AX, %_ASM_DI
227
228	mov SVM_vmcb01_pa(%_ASM_DI), %_ASM_AX
2295:	vmsave %_ASM_AX
2306:
231
232	/* Restores GSBASE among other things, allowing access to percpu data.  */
233	pop %_ASM_AX
2347:	vmload %_ASM_AX
2358:
236
237	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
238	FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
239
240	/*
241	 * Clobbers RAX, RCX, RDX (and ESI, EDI on 32-bit), consumes RDI (@svm)
242	 * and RSP (pointer to @spec_ctrl_intercepted).
243	 */
244	RESTORE_HOST_SPEC_CTRL
245
246	/*
247	 * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
248	 * untrained as soon as we exit the VM and are back to the
249	 * kernel. This should be done before re-enabling interrupts
250	 * because interrupt handlers won't sanitize 'ret' if the return is
251	 * from the kernel.
252	 */
253	UNTRAIN_RET_VM
254
255	/*
256	 * Clear all general purpose registers except RSP and RAX to prevent
257	 * speculative use of the guest's values, even those that are reloaded
258	 * via the stack.  In theory, an L1 cache miss when restoring registers
259	 * could lead to speculative execution with the guest's values.
260	 * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
261	 * free.  RSP and RAX are exempt as they are restored by hardware
262	 * during VM-Exit.
263	 */
264	xor %ecx, %ecx
265	xor %edx, %edx
266	xor %ebx, %ebx
267	xor %ebp, %ebp
268	xor %esi, %esi
269	xor %edi, %edi
270#ifdef CONFIG_X86_64
271	xor %r8d,  %r8d
272	xor %r9d,  %r9d
273	xor %r10d, %r10d
274	xor %r11d, %r11d
275	xor %r12d, %r12d
276	xor %r13d, %r13d
277	xor %r14d, %r14d
278	xor %r15d, %r15d
279#endif
280
281	/* "Pop" @spec_ctrl_intercepted.  */
282	pop %_ASM_BX
283
284	pop %_ASM_BX
285
286#ifdef CONFIG_X86_64
287	pop %r12
288	pop %r13
289	pop %r14
290	pop %r15
291#else
292	pop %esi
293	pop %edi
294#endif
295	pop %_ASM_BP
296	RET
297
298	RESTORE_GUEST_SPEC_CTRL_BODY
299	RESTORE_HOST_SPEC_CTRL_BODY (%_ASM_SP)
300
30110:	cmpb $0, _ASM_RIP(kvm_rebooting)
302	jne 2b
303	ud2
30430:	cmpb $0, _ASM_RIP(kvm_rebooting)
305	jne 4b
306	ud2
30750:	cmpb $0, _ASM_RIP(kvm_rebooting)
308	jne 6b
309	ud2
31070:	cmpb $0, _ASM_RIP(kvm_rebooting)
311	jne 8b
312	ud2
313
314	_ASM_EXTABLE(1b, 10b)
315	_ASM_EXTABLE(3b, 30b)
316	_ASM_EXTABLE(5b, 50b)
317	_ASM_EXTABLE(7b, 70b)
318
319SYM_FUNC_END(__svm_vcpu_run)
320
321#ifdef CONFIG_KVM_AMD_SEV
322
323
324#ifdef CONFIG_X86_64
325#define SEV_ES_GPRS_BASE 0x300
326#define SEV_ES_RBX	(SEV_ES_GPRS_BASE + __VCPU_REGS_RBX * WORD_SIZE)
327#define SEV_ES_RBP	(SEV_ES_GPRS_BASE + __VCPU_REGS_RBP * WORD_SIZE)
328#define SEV_ES_RSI	(SEV_ES_GPRS_BASE + __VCPU_REGS_RSI * WORD_SIZE)
329#define SEV_ES_RDI	(SEV_ES_GPRS_BASE + __VCPU_REGS_RDI * WORD_SIZE)
330#define SEV_ES_R12	(SEV_ES_GPRS_BASE + __VCPU_REGS_R12 * WORD_SIZE)
331#define SEV_ES_R13	(SEV_ES_GPRS_BASE + __VCPU_REGS_R13 * WORD_SIZE)
332#define SEV_ES_R14	(SEV_ES_GPRS_BASE + __VCPU_REGS_R14 * WORD_SIZE)
333#define SEV_ES_R15	(SEV_ES_GPRS_BASE + __VCPU_REGS_R15 * WORD_SIZE)
334#endif
335
336/**
337 * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode
338 * @svm:	struct vcpu_svm *
339 * @spec_ctrl_intercepted: bool
340 */
341SYM_FUNC_START(__svm_sev_es_vcpu_run)
342	FRAME_BEGIN
343
344	/*
345	 * Save non-volatile (callee-saved) registers to the host save area.
346	 * Except for RAX and RSP, all GPRs are restored on #VMEXIT, but not
347	 * saved on VMRUN.
348	 */
349	mov %rbp, SEV_ES_RBP (%rdx)
350	mov %r15, SEV_ES_R15 (%rdx)
351	mov %r14, SEV_ES_R14 (%rdx)
352	mov %r13, SEV_ES_R13 (%rdx)
353	mov %r12, SEV_ES_R12 (%rdx)
354	mov %rbx, SEV_ES_RBX (%rdx)
355
356	/*
357	 * Save volatile registers that hold arguments that are needed after
358	 * #VMEXIT (RDI=@svm and RSI=@spec_ctrl_intercepted).
359	 */
360	mov %rdi, SEV_ES_RDI (%rdx)
361	mov %rsi, SEV_ES_RSI (%rdx)
362
363	/* Clobbers RAX, RCX, and RDX (@hostsa), consumes RDI (@svm). */
364	RESTORE_GUEST_SPEC_CTRL
365
366	/* Get svm->current_vmcb->pa into RAX. */
367	mov SVM_current_vmcb(%rdi), %rax
368	mov KVM_VMCB_pa(%rax), %rax
369
370	/* Clobbers EFLAGS.ZF */
371	SVM_CLEAR_CPU_BUFFERS
372
373	/* Enter guest mode */
3741:	vmrun %rax
3752:
376	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
377	FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
378
379	/* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@spec_ctrl_intercepted). */
380	RESTORE_HOST_SPEC_CTRL
381
382	/*
383	 * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
384	 * untrained as soon as we exit the VM and are back to the
385	 * kernel. This should be done before re-enabling interrupts
386	 * because interrupt handlers won't sanitize RET if the return is
387	 * from the kernel.
388	 */
389	UNTRAIN_RET_VM
390
391	FRAME_END
392	RET
393
394	RESTORE_GUEST_SPEC_CTRL_BODY
395	RESTORE_HOST_SPEC_CTRL_BODY %sil
396
3973:	cmpb $0, kvm_rebooting(%rip)
398	jne 2b
399	ud2
400
401	_ASM_EXTABLE(1b, 3b)
402
403SYM_FUNC_END(__svm_sev_es_vcpu_run)
404#endif /* CONFIG_KVM_AMD_SEV */