la57toggle.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2
  3#include <linux/linkage.h>
  4#include <asm/segment.h>
  5#include <asm/boot.h>
  6#include <asm/msr.h>
  7#include <asm/processor-flags.h>
  8
  9/*
 10 * This is the 32-bit trampoline that will be copied over to low memory. It
 11 * will be called using the ordinary 64-bit calling convention from code
 12 * running in 64-bit mode.
 13 *
 14 * Return address is at the top of the stack (might be above 4G).
 15 * The first argument (EDI) contains the address of the temporary PGD level
 16 * page table in 32-bit addressable memory which will be programmed into
 17 * register CR3.
 18 */
 19
 20	.section ".rodata", "a", @progbits
 21SYM_CODE_START(trampoline_32bit_src)
 22	/*
 23	 * Preserve callee save 64-bit registers on the stack: this is
 24	 * necessary because the architecture does not guarantee that GPRs will
 25	 * retain their full 64-bit values across a 32-bit mode switch.
 26	 */
 27	pushq	%r15
 28	pushq	%r14
 29	pushq	%r13
 30	pushq	%r12
 31	pushq	%rbp
 32	pushq	%rbx
 33
 34	/* Preserve top half of RSP in a legacy mode GPR to avoid truncation */
 35	movq	%rsp, %rbx
 36	shrq	$32, %rbx
 37
 38	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
 39	pushq	$__KERNEL32_CS
 40	leaq	0f(%rip), %rax
 41	pushq	%rax
 42	lretq
 43
 44	/*
 45	 * The 32-bit code below will do a far jump back to long mode and end
 46	 * up here after reconfiguring the number of paging levels. First, the
 47	 * stack pointer needs to be restored to its full 64-bit value before
 48	 * the callee save register contents can be popped from the stack.
 49	 */
 50.Lret:
 51	shlq	$32, %rbx
 52	orq	%rbx, %rsp
 53
 54	/* Restore the preserved 64-bit registers */
 55	popq	%rbx
 56	popq	%rbp
 57	popq	%r12
 58	popq	%r13
 59	popq	%r14
 60	popq	%r15
 61	retq
 62
 63	.code32
 640:
 65	/* Disable paging */
 66	movl	%cr0, %eax
 67	btrl	$X86_CR0_PG_BIT, %eax
 68	movl	%eax, %cr0
 69
 70	/* Point CR3 to the trampoline's new top level page table */
 71	movl	%edi, %cr3
 72
 73	/* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
 74	movl	$MSR_EFER, %ecx
 75	rdmsr
 76	btsl	$_EFER_LME, %eax
 77	/* Avoid writing EFER if no change was made (for TDX guest) */
 78	jc	1f
 79	wrmsr
 801:
 81	/* Toggle CR4.LA57 */
 82	movl	%cr4, %eax
 83	btcl	$X86_CR4_LA57_BIT, %eax
 84	movl	%eax, %cr4
 85
 86	/* Enable paging again. */
 87	movl	%cr0, %eax
 88	btsl	$X86_CR0_PG_BIT, %eax
 89	movl	%eax, %cr0
 90
 91	/*
 92	 * Return to the 64-bit calling code using LJMP rather than LRET, to
 93	 * avoid the need for a 32-bit addressable stack. The destination
 94	 * address will be adjusted after the template code is copied into a
 95	 * 32-bit addressable buffer.
 96	 */
 97.Ljmp:	ljmpl	$__KERNEL_CS, $(.Lret - trampoline_32bit_src)
 98SYM_CODE_END(trampoline_32bit_src)
 99
100/*
101 * This symbol is placed right after trampoline_32bit_src() so its address can
102 * be used to infer the size of the trampoline code.
103 */
104SYM_DATA(trampoline_ljmp_imm_offset, .word  .Ljmp + 1 - trampoline_32bit_src)
105
106	/*
107         * The trampoline code has a size limit.
108         * Make sure we fail to compile if the trampoline code grows
109         * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes.
110	 */
111	.org	trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE