clear_user.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 */
  5
  6#include <linux/export.h>
  7#include <asm/alternative-asm.h>
  8#include <asm/asm.h>
  9#include <asm/asmmacro.h>
 10#include <asm/asm-extable.h>
 11#include <asm/cpu.h>
 12#include <asm/regdef.h>
 13#include <asm/unwind_hints.h>
 14
 15SYM_FUNC_START(__clear_user)
 16#ifdef CONFIG_32BIT
 17	b		__clear_user_generic
 18#else
 19	/*
 20	 * Some CPUs support hardware unaligned access
 21	 */
 22	ALTERNATIVE	"b __clear_user_generic",	\
 23			"b __clear_user_fast", CPU_FEATURE_UAL
 24#endif
 25SYM_FUNC_END(__clear_user)
 26
 27EXPORT_SYMBOL(__clear_user)
 28
 29/*
 30 * unsigned long __clear_user_generic(void *addr, size_t size)
 31 *
 32 * a0: addr
 33 * a1: size
 34 */
 35SYM_FUNC_START(__clear_user_generic)
 36	beqz		a1, 2f
 37
 381:	st.b		zero, a0, 0
 39	PTR_ADDI	a0, a0, 1
 40	PTR_ADDI	a1, a1, -1
 41	bgtz		a1, 1b
 42
 432:	move		a0, a1
 44	jr		ra
 45
 46	_asm_extable 	1b, 2b
 47SYM_FUNC_END(__clear_user_generic)
 48
 49#ifdef CONFIG_64BIT
 50/*
 51 * unsigned long __clear_user_fast(void *addr, unsigned long size)
 52 *
 53 * a0: addr
 54 * a1: size
 55 */
 56SYM_FUNC_START(__clear_user_fast)
 57	sltui	t0, a1, 9
 58	bnez	t0, .Lsmall
 59
 60	add.d	a2, a0, a1
 610:	st.d	zero, a0, 0
 62
 63	/* align up address */
 64	addi.d	a0, a0, 8
 65	bstrins.d	a0, zero, 2, 0
 66
 67	addi.d	a3, a2, -64
 68	bgeu	a0, a3, .Llt64
 69
 70	/* set 64 bytes at a time */
 71.Lloop64:
 721:	st.d	zero, a0, 0
 732:	st.d	zero, a0, 8
 743:	st.d	zero, a0, 16
 754:	st.d	zero, a0, 24
 765:	st.d	zero, a0, 32
 776:	st.d	zero, a0, 40
 787:	st.d	zero, a0, 48
 798:	st.d	zero, a0, 56
 80	addi.d	a0, a0, 64
 81	bltu	a0, a3, .Lloop64
 82
 83	/* set the remaining bytes */
 84.Llt64:
 85	addi.d	a3, a2, -32
 86	bgeu	a0, a3, .Llt32
 879:	st.d	zero, a0, 0
 8810:	st.d	zero, a0, 8
 8911:	st.d	zero, a0, 16
 9012:	st.d	zero, a0, 24
 91	addi.d	a0, a0, 32
 92
 93.Llt32:
 94	addi.d	a3, a2, -16
 95	bgeu	a0, a3, .Llt16
 9613:	st.d	zero, a0, 0
 9714:	st.d	zero, a0, 8
 98	addi.d	a0, a0, 16
 99
100.Llt16:
101	addi.d	a3, a2, -8
102	bgeu	a0, a3, .Llt8
10315:	st.d	zero, a0, 0
104	addi.d	a0, a0, 8
105
106.Llt8:
10716:	st.d	zero, a2, -8
108
109	/* return */
110	move	a0, zero
111	jr	ra
112
113	.align	4
114.Lsmall:
115	pcaddi	t0, 4
116	slli.d	a2, a1, 4
117	add.d	t0, t0, a2
118	jr	t0
119
120	.align	4
121	move	a0, zero
122	jr	ra
123
124	.align	4
12517:	st.b	zero, a0, 0
126	move	a0, zero
127	jr	ra
128
129	.align	4
13018:	st.h	zero, a0, 0
131	move	a0, zero
132	jr	ra
133
134	.align	4
13519:	st.h	zero, a0, 0
13620:	st.b	zero, a0, 2
137	move	a0, zero
138	jr	ra
139
140	.align	4
14121:	st.w	zero, a0, 0
142	move	a0, zero
143	jr	ra
144
145	.align	4
14622:	st.w	zero, a0, 0
14723:	st.b	zero, a0, 4
148	move	a0, zero
149	jr	ra
150
151	.align	4
15224:	st.w	zero, a0, 0
15325:	st.h	zero, a0, 4
154	move	a0, zero
155	jr	ra
156
157	.align	4
15826:	st.w	zero, a0, 0
15927:	st.w	zero, a0, 3
160	move	a0, zero
161	jr	ra
162
163	.align	4
16428:	st.d	zero, a0, 0
165	move	a0, zero
166	jr	ra
167
168	/* fixup and ex_table */
169.Llarge_fixup:
170	sub.d	a1, a2, a0
171
172.Lsmall_fixup:
17329:	st.b	zero, a0, 0
174	addi.d	a0, a0, 1
175	addi.d	a1, a1, -1
176	bgt	a1, zero, 29b
177
178.Lexit:
179	move	a0, a1
180	jr	ra
181
182	_asm_extable 0b, .Lsmall_fixup
183	_asm_extable 1b, .Llarge_fixup
184	_asm_extable 2b, .Llarge_fixup
185	_asm_extable 3b, .Llarge_fixup
186	_asm_extable 4b, .Llarge_fixup
187	_asm_extable 5b, .Llarge_fixup
188	_asm_extable 6b, .Llarge_fixup
189	_asm_extable 7b, .Llarge_fixup
190	_asm_extable 8b, .Llarge_fixup
191	_asm_extable 9b, .Llarge_fixup
192	_asm_extable 10b, .Llarge_fixup
193	_asm_extable 11b, .Llarge_fixup
194	_asm_extable 12b, .Llarge_fixup
195	_asm_extable 13b, .Llarge_fixup
196	_asm_extable 14b, .Llarge_fixup
197	_asm_extable 15b, .Llarge_fixup
198	_asm_extable 16b, .Llarge_fixup
199	_asm_extable 17b, .Lexit
200	_asm_extable 18b, .Lsmall_fixup
201	_asm_extable 19b, .Lsmall_fixup
202	_asm_extable 20b, .Lsmall_fixup
203	_asm_extable 21b, .Lsmall_fixup
204	_asm_extable 22b, .Lsmall_fixup
205	_asm_extable 23b, .Lsmall_fixup
206	_asm_extable 24b, .Lsmall_fixup
207	_asm_extable 25b, .Lsmall_fixup
208	_asm_extable 26b, .Lsmall_fixup
209	_asm_extable 27b, .Lsmall_fixup
210	_asm_extable 28b, .Lsmall_fixup
211	_asm_extable 29b, .Lexit
212SYM_FUNC_END(__clear_user_fast)
213
214STACK_FRAME_NON_STANDARD __clear_user_fast
215#endif