getuser.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * __get_user functions.
  4 *
  5 * (C) Copyright 1998 Linus Torvalds
  6 * (C) Copyright 2005 Andi Kleen
  7 * (C) Copyright 2008 Glauber Costa
  8 *
  9 * These functions have a non-standard call interface
 10 * to make them more efficient, especially as they
 11 * return an error value in addition to the "real"
 12 * return value.
 13 */
 14
 15/*
 16 * __get_user_X
 17 *
 18 * Inputs:	%[r|e]ax contains the address.
 19 *
 20 * Outputs:	%[r|e]ax is error code (0 or -EFAULT)
 21 *		%[r|e]dx contains zero-extended value
 22 *		%ecx contains the high half for 32-bit __get_user_8
 23 *
 24 *
 25 * These functions should not modify any other registers,
 26 * as they get called from within inline assembly.
 27 */
 28
 29#include <linux/export.h>
 30#include <linux/linkage.h>
 31#include <linux/objtool.h>
 32#include <asm/page_types.h>
 33#include <asm/errno.h>
 34#include <asm/asm-offsets.h>
 35#include <asm/thread_info.h>
 36#include <asm/asm.h>
 37#include <asm/smap.h>
 38#include <asm/runtime-const.h>
 39
 40#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
 41
 42.macro check_range size:req
 43.if IS_ENABLED(CONFIG_X86_64)
 44	RUNTIME_CONST_PTR USER_PTR_MAX, rdx
 45	cmp %rdx, %rax
 46	cmova %rdx, %rax
 47.else
 48	cmp $TASK_SIZE_MAX-\size+1, %eax
 49	jae .Lbad_get_user
 50	sbb %edx, %edx		/* array_index_mask_nospec() */
 51	and %edx, %eax
 52.endif
 53.endm
 54
 55.macro UACCESS op src dst
 561:	\op \src,\dst
 57	_ASM_EXTABLE_UA(1b, __get_user_handle_exception)
 58.endm
 59
 60
 61	.text
 62SYM_FUNC_START(__get_user_1)
 63	ANNOTATE_NOENDBR
 64	check_range size=1
 65	ASM_STAC
 66	UACCESS movzbl (%_ASM_AX),%edx
 67	xor %eax,%eax
 68	ASM_CLAC
 69	RET
 70SYM_FUNC_END(__get_user_1)
 71EXPORT_SYMBOL(__get_user_1)
 72
 73SYM_FUNC_START(__get_user_2)
 74	ANNOTATE_NOENDBR
 75	check_range size=2
 76	ASM_STAC
 77	UACCESS movzwl (%_ASM_AX),%edx
 78	xor %eax,%eax
 79	ASM_CLAC
 80	RET
 81SYM_FUNC_END(__get_user_2)
 82EXPORT_SYMBOL(__get_user_2)
 83
 84SYM_FUNC_START(__get_user_4)
 85	ANNOTATE_NOENDBR
 86	check_range size=4
 87	ASM_STAC
 88	UACCESS movl (%_ASM_AX),%edx
 89	xor %eax,%eax
 90	ASM_CLAC
 91	RET
 92SYM_FUNC_END(__get_user_4)
 93EXPORT_SYMBOL(__get_user_4)
 94
 95SYM_FUNC_START(__get_user_8)
 96	ANNOTATE_NOENDBR
 97#ifndef CONFIG_X86_64
 98	xor %ecx,%ecx
 99#endif
100	check_range size=8
101	ASM_STAC
102#ifdef CONFIG_X86_64
103	UACCESS movq (%_ASM_AX),%rdx
104#else
105	UACCESS movl (%_ASM_AX),%edx
106	UACCESS movl 4(%_ASM_AX),%ecx
107#endif
108	xor %eax,%eax
109	ASM_CLAC
110	RET
111SYM_FUNC_END(__get_user_8)
112EXPORT_SYMBOL(__get_user_8)
113
114/* .. and the same for __get_user, just without the range checks */
115SYM_FUNC_START(__get_user_nocheck_1)
116	ANNOTATE_NOENDBR
117	ASM_STAC
118	ASM_BARRIER_NOSPEC
119	UACCESS movzbl (%_ASM_AX),%edx
120	xor %eax,%eax
121	ASM_CLAC
122	RET
123SYM_FUNC_END(__get_user_nocheck_1)
124EXPORT_SYMBOL(__get_user_nocheck_1)
125
126SYM_FUNC_START(__get_user_nocheck_2)
127	ANNOTATE_NOENDBR
128	ASM_STAC
129	ASM_BARRIER_NOSPEC
130	UACCESS movzwl (%_ASM_AX),%edx
131	xor %eax,%eax
132	ASM_CLAC
133	RET
134SYM_FUNC_END(__get_user_nocheck_2)
135EXPORT_SYMBOL(__get_user_nocheck_2)
136
137SYM_FUNC_START(__get_user_nocheck_4)
138	ANNOTATE_NOENDBR
139	ASM_STAC
140	ASM_BARRIER_NOSPEC
141	UACCESS movl (%_ASM_AX),%edx
142	xor %eax,%eax
143	ASM_CLAC
144	RET
145SYM_FUNC_END(__get_user_nocheck_4)
146EXPORT_SYMBOL(__get_user_nocheck_4)
147
148SYM_FUNC_START(__get_user_nocheck_8)
149	ANNOTATE_NOENDBR
150	ASM_STAC
151	ASM_BARRIER_NOSPEC
152#ifdef CONFIG_X86_64
153	UACCESS movq (%_ASM_AX),%rdx
154#else
155	xor %ecx,%ecx
156	UACCESS movl (%_ASM_AX),%edx
157	UACCESS movl 4(%_ASM_AX),%ecx
158#endif
159	xor %eax,%eax
160	ASM_CLAC
161	RET
162SYM_FUNC_END(__get_user_nocheck_8)
163EXPORT_SYMBOL(__get_user_nocheck_8)
164
165
166SYM_CODE_START_LOCAL(__get_user_handle_exception)
167	ASM_CLAC
168.Lbad_get_user:
169	xor %edx,%edx
170	mov $(-EFAULT),%_ASM_AX
171	RET
172SYM_CODE_END(__get_user_handle_exception)