uaccess.S

  1#include <linux/linkage.h>
  2#include <linux/export.h>
  3#include <asm/asm.h>
  4#include <asm/asm-extable.h>
  5#include <asm/csr.h>
  6#include <asm/hwcap.h>
  7#include <asm/alternative-macros.h>
  8
  9	.macro fixup op reg addr lbl
 10100:
 11	\op \reg, \addr
 12	_asm_extable	100b, \lbl
 13	.endm
 14
 15SYM_FUNC_START(__asm_copy_to_user)
 16#ifdef CONFIG_RISCV_ISA_V
 17	ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V)
 18	REG_L	t0, riscv_v_usercopy_threshold
 19	bltu	a2, t0, fallback_scalar_usercopy
 20	li	a3, 1
 21	tail 	enter_vector_usercopy
 22#endif
 23SYM_FUNC_END(__asm_copy_to_user)
 24EXPORT_SYMBOL(__asm_copy_to_user)
 25SYM_FUNC_ALIAS(__asm_copy_from_user, __asm_copy_to_user)
 26EXPORT_SYMBOL(__asm_copy_from_user)
 27
 28SYM_FUNC_START(fallback_scalar_usercopy)
 29	/* Enable access to user memory */
 30	li	t6, SR_SUM
 31	csrs 	CSR_STATUS, t6
 32	mv 	t6, ra
 33
 34	call 	fallback_scalar_usercopy_sum_enabled
 35
 36	/* Disable access to user memory */
 37	mv 	ra, t6
 38	li 	t6, SR_SUM
 39	csrc 	CSR_STATUS, t6
 40	ret
 41SYM_FUNC_END(fallback_scalar_usercopy)
 42
 43SYM_FUNC_START(__asm_copy_to_user_sum_enabled)
 44#ifdef CONFIG_RISCV_ISA_V
 45	ALTERNATIVE("j fallback_scalar_usercopy_sum_enabled", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V)
 46	REG_L	t0, riscv_v_usercopy_threshold
 47	bltu	a2, t0, fallback_scalar_usercopy_sum_enabled
 48	li	a3, 0
 49	tail 	enter_vector_usercopy
 50#endif
 51SYM_FUNC_END(__asm_copy_to_user_sum_enabled)
 52SYM_FUNC_ALIAS(__asm_copy_from_user_sum_enabled, __asm_copy_to_user_sum_enabled)
 53EXPORT_SYMBOL(__asm_copy_from_user_sum_enabled)
 54EXPORT_SYMBOL(__asm_copy_to_user_sum_enabled)
 55
 56SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled)
 57	/*
 58	 * Save the terminal address which will be used to compute the number
 59	 * of bytes copied in case of a fixup exception.
 60	 */
 61	add	t5, a0, a2
 62
 63	/*
 64	 * Register allocation for code below:
 65	 * a0 - start of uncopied dst
 66	 * a1 - start of uncopied src
 67	 * a2 - size
 68	 * t0 - end of uncopied dst
 69	 */
 70	add	t0, a0, a2
 71
 72	/*
 73	 * Use byte copy only if too small.
 74	 * SZREG holds 4 for RV32 and 8 for RV64
 75	 */
 76	li	a3, 9*SZREG-1 /* size must >= (word_copy stride + SZREG-1) */
 77	bltu	a2, a3, .Lbyte_copy_tail
 78
 79	/*
 80	 * Copy first bytes until dst is aligned to word boundary.
 81	 * a0 - start of dst
 82	 * t1 - start of aligned dst
 83	 */
 84	addi	t1, a0, SZREG-1
 85	andi	t1, t1, ~(SZREG-1)
 86	/* dst is already aligned, skip */
 87	beq	a0, t1, .Lskip_align_dst
 881:
 89	/* a5 - one byte for copying data */
 90	fixup lb      a5, 0(a1), 10f
 91	addi	a1, a1, 1	/* src */
 92	fixup sb      a5, 0(a0), 10f
 93	addi	a0, a0, 1	/* dst */
 94	bltu	a0, t1, 1b	/* t1 - start of aligned dst */
 95
 96.Lskip_align_dst:
 97	/*
 98	 * Now dst is aligned.
 99	 * Use shift-copy if src is misaligned.
100	 * Use word-copy if both src and dst are aligned because
101	 * can not use shift-copy which do not require shifting
102	 */
103	/* a1 - start of src */
104	andi	a3, a1, SZREG-1
105	bnez	a3, .Lshift_copy
106
107.Lword_copy:
108        /*
109	 * Both src and dst are aligned, unrolled word copy
110	 *
111	 * a0 - start of aligned dst
112	 * a1 - start of aligned src
113	 * t0 - end of aligned dst
114	 */
115	addi	t0, t0, -(8*SZREG) /* not to over run */
1162:
117	fixup REG_L   a4,        0(a1), 10f
118	fixup REG_L   a5,    SZREG(a1), 10f
119	fixup REG_L   a6,  2*SZREG(a1), 10f
120	fixup REG_L   a7,  3*SZREG(a1), 10f
121	fixup REG_L   t1,  4*SZREG(a1), 10f
122	fixup REG_L   t2,  5*SZREG(a1), 10f
123	fixup REG_L   t3,  6*SZREG(a1), 10f
124	fixup REG_L   t4,  7*SZREG(a1), 10f
125	fixup REG_S   a4,        0(a0), 10f
126	fixup REG_S   a5,    SZREG(a0), 10f
127	fixup REG_S   a6,  2*SZREG(a0), 10f
128	fixup REG_S   a7,  3*SZREG(a0), 10f
129	fixup REG_S   t1,  4*SZREG(a0), 10f
130	fixup REG_S   t2,  5*SZREG(a0), 10f
131	fixup REG_S   t3,  6*SZREG(a0), 10f
132	fixup REG_S   t4,  7*SZREG(a0), 10f
133	addi	a0, a0, 8*SZREG
134	addi	a1, a1, 8*SZREG
135	bleu	a0, t0, 2b
136
137	addi	t0, t0, 8*SZREG /* revert to original value */
138	j	.Lbyte_copy_tail
139
140.Lshift_copy:
141
142	/*
143	 * Word copy with shifting.
144	 * For misaligned copy we still perform aligned word copy, but
145	 * we need to use the value fetched from the previous iteration and
146	 * do some shifts.
147	 * This is safe because reading is less than a word size.
148	 *
149	 * a0 - start of aligned dst
150	 * a1 - start of src
151	 * a3 - a1 & mask:(SZREG-1)
152	 * t0 - end of uncopied dst
153	 * t1 - end of aligned dst
154	 */
155	/* calculating aligned word boundary for dst */
156	andi	t1, t0, ~(SZREG-1)
157	/* Converting unaligned src to aligned src */
158	andi	a1, a1, ~(SZREG-1)
159
160	/*
161	 * Calculate shifts
162	 * t3 - prev shift
163	 * t4 - current shift
164	 */
165	slli	t3, a3, 3 /* converting bytes in a3 to bits */
166	li	a5, SZREG*8
167	sub	t4, a5, t3
168
169	/* Load the first word to combine with second word */
170	fixup REG_L   a5, 0(a1), 10f
171
1723:
173	/* Main shifting copy
174	 *
175	 * a0 - start of aligned dst
176	 * a1 - start of aligned src
177	 * t1 - end of aligned dst
178	 */
179
180	/* At least one iteration will be executed */
181	srl	a4, a5, t3
182	fixup REG_L   a5, SZREG(a1), 10f
183	addi	a1, a1, SZREG
184	sll	a2, a5, t4
185	or	a2, a2, a4
186	fixup REG_S   a2, 0(a0), 10f
187	addi	a0, a0, SZREG
188	bltu	a0, t1, 3b
189
190	/* Revert src to original unaligned value  */
191	add	a1, a1, a3
192
193.Lbyte_copy_tail:
194	/*
195	 * Byte copy anything left.
196	 *
197	 * a0 - start of remaining dst
198	 * a1 - start of remaining src
199	 * t0 - end of remaining dst
200	 */
201	bgeu	a0, t0, .Lout_copy_user  /* check if end of copy */
2024:
203	fixup lb      a5, 0(a1), 10f
204	addi	a1, a1, 1	/* src */
205	fixup sb      a5, 0(a0), 10f
206	addi	a0, a0, 1	/* dst */
207	bltu	a0, t0, 4b	/* t0 - end of dst */
208
209.Lout_copy_user:
210	li	a0, 0
211	ret
21210:
213	sub a0, t5, a0
214	ret
215SYM_FUNC_END(fallback_scalar_usercopy_sum_enabled)
216
217SYM_FUNC_START(__clear_user)
218
219	/* Enable access to user memory */
220	li t6, SR_SUM
221	csrs CSR_STATUS, t6
222
223	add a3, a0, a1
224	addi t0, a0, SZREG-1
225	andi t1, a3, ~(SZREG-1)
226	andi t0, t0, ~(SZREG-1)
227	/*
228	 * a3: terminal address of target region
229	 * t0: lowest doubleword-aligned address in target region
230	 * t1: highest doubleword-aligned address in target region
231	 */
232	bgeu t0, t1, 2f
233	bltu a0, t0, 4f
2341:
235	fixup REG_S, zero, (a0), 11f
236	addi a0, a0, SZREG
237	bltu a0, t1, 1b
2382:
239	bltu a0, a3, 5f
240
2413:
242	/* Disable access to user memory */
243	csrc CSR_STATUS, t6
244	li a0, 0
245	ret
2464: /* Edge case: unalignment */
247	fixup sb, zero, (a0), 11f
248	addi a0, a0, 1
249	bltu a0, t0, 4b
250	j 1b
2515: /* Edge case: remainder */
252	fixup sb, zero, (a0), 11f
253	addi a0, a0, 1
254	bltu a0, a3, 5b
255	j 3b
256
257	/* Exception fixup code */
25811:
259	/* Disable access to user memory */
260	csrc CSR_STATUS, t6
261	sub a0, a3, a0
262	ret
263SYM_FUNC_END(__clear_user)
264EXPORT_SYMBOL(__clear_user)