mem.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * String handling functions.
  4 *
  5 * Copyright IBM Corp. 2012
  6 */
  7
  8#include <linux/export.h>
  9#include <linux/linkage.h>
 10#include <asm/nospec-insn.h>
 11
 12	GEN_BR_THUNK %r14
 13
 14/*
 15 * void *memmove(void *dest, const void *src, size_t n)
 16 */
 17SYM_FUNC_START(__memmove)
 18	ltgr	%r4,%r4
 19	lgr	%r1,%r2
 20	jz	.Lmemmove_exit
 21	aghi	%r4,-1
 22	clgr	%r2,%r3
 23	jnh	.Lmemmove_forward
 24	la	%r5,1(%r4,%r3)
 25	clgr	%r2,%r5
 26	jl	.Lmemmove_reverse
 27.Lmemmove_forward:
 28	srlg	%r0,%r4,8
 29	ltgr	%r0,%r0
 30	jz	.Lmemmove_forward_remainder
 31.Lmemmove_forward_loop:
 32	mvc	0(256,%r1),0(%r3)
 33	la	%r1,256(%r1)
 34	la	%r3,256(%r3)
 35	brctg	%r0,.Lmemmove_forward_loop
 36.Lmemmove_forward_remainder:
 37	exrl	%r4,.Lmemmove_mvc
 38.Lmemmove_exit:
 39	BR_EX	%r14
 40.Lmemmove_reverse:
 41	ic	%r0,0(%r4,%r3)
 42	stc	%r0,0(%r4,%r1)
 43	brctg	%r4,.Lmemmove_reverse
 44	ic	%r0,0(%r4,%r3)
 45	stc	%r0,0(%r4,%r1)
 46	BR_EX	%r14
 47.Lmemmove_mvc:
 48	mvc	0(1,%r1),0(%r3)
 49SYM_FUNC_END(__memmove)
 50EXPORT_SYMBOL(__memmove)
 51
 52SYM_FUNC_ALIAS(memmove, __memmove)
 53EXPORT_SYMBOL(memmove)
 54
 55/*
 56 * memset implementation
 57 *
 58 * This code corresponds to the C construct below. We do distinguish
 59 * between clearing (c == 0) and setting a memory array (c != 0) simply
 60 * because nearly all memset invocations in the kernel clear memory and
 61 * the xc instruction is preferred in such cases.
 62 *
 63 * void *memset(void *s, int c, size_t n)
 64 * {
 65 *	if (likely(c == 0))
 66 *		return __builtin_memset(s, 0, n);
 67 *	return __builtin_memset(s, c, n);
 68 * }
 69 */
 70SYM_FUNC_START(__memset)
 71	ltgr	%r4,%r4
 72	jz	.Lmemset_exit
 73	ltgr	%r3,%r3
 74	jnz	.Lmemset_fill
 75	aghi	%r4,-1
 76	srlg	%r3,%r4,8
 77	ltgr	%r3,%r3
 78	lgr	%r1,%r2
 79	jz	.Lmemset_clear_remainder
 80.Lmemset_clear_loop:
 81	xc	0(256,%r1),0(%r1)
 82	la	%r1,256(%r1)
 83	brctg	%r3,.Lmemset_clear_loop
 84.Lmemset_clear_remainder:
 85	exrl	%r4,.Lmemset_xc
 86.Lmemset_exit:
 87	BR_EX	%r14
 88.Lmemset_fill:
 89	cghi	%r4,1
 90	lgr	%r1,%r2
 91	je	.Lmemset_fill_exit
 92	aghi	%r4,-2
 93	srlg	%r5,%r4,8
 94	ltgr	%r5,%r5
 95	jz	.Lmemset_fill_remainder
 96.Lmemset_fill_loop:
 97	stc	%r3,0(%r1)
 98	mvc	1(255,%r1),0(%r1)
 99	la	%r1,256(%r1)
100	brctg	%r5,.Lmemset_fill_loop
101.Lmemset_fill_remainder:
102	stc	%r3,0(%r1)
103	exrl	%r4,.Lmemset_mvc
104	BR_EX	%r14
105.Lmemset_fill_exit:
106	stc	%r3,0(%r1)
107	BR_EX	%r14
108.Lmemset_xc:
109	xc	0(1,%r1),0(%r1)
110.Lmemset_mvc:
111	mvc	1(1,%r1),0(%r1)
112SYM_FUNC_END(__memset)
113EXPORT_SYMBOL(__memset)
114
115SYM_FUNC_ALIAS(memset, __memset)
116EXPORT_SYMBOL(memset)
117
118/*
119 * memcpy implementation
120 *
121 * void *memcpy(void *dest, const void *src, size_t n)
122 */
123SYM_FUNC_START(__memcpy)
124	ltgr	%r4,%r4
125	jz	.Lmemcpy_exit
126	aghi	%r4,-1
127	srlg	%r5,%r4,8
128	ltgr	%r5,%r5
129	lgr	%r1,%r2
130	jnz	.Lmemcpy_loop
131.Lmemcpy_remainder:
132	exrl	%r4,.Lmemcpy_mvc
133.Lmemcpy_exit:
134	BR_EX	%r14
135.Lmemcpy_loop:
136	mvc	0(256,%r1),0(%r3)
137	la	%r1,256(%r1)
138	la	%r3,256(%r3)
139	brctg	%r5,.Lmemcpy_loop
140	j	.Lmemcpy_remainder
141.Lmemcpy_mvc:
142	mvc	0(1,%r1),0(%r3)
143SYM_FUNC_END(__memcpy)
144EXPORT_SYMBOL(__memcpy)
145
146SYM_FUNC_ALIAS(memcpy, __memcpy)
147EXPORT_SYMBOL(memcpy)
148
149/*
150 * __memset16/32/64
151 *
152 * void *__memset16(uint16_t *s, uint16_t v, size_t count)
153 * void *__memset32(uint32_t *s, uint32_t v, size_t count)
154 * void *__memset64(uint64_t *s, uint64_t v, size_t count)
155 */
156.macro __MEMSET bits,bytes,insn
157SYM_FUNC_START(__memset\bits)
158	ltgr	%r4,%r4
159	jz	.L__memset_exit\bits
160	cghi	%r4,\bytes
161	je	.L__memset_store\bits
162	aghi	%r4,-(\bytes+1)
163	srlg	%r5,%r4,8
164	ltgr	%r5,%r5
165	lgr	%r1,%r2
166	jz	.L__memset_remainder\bits
167.L__memset_loop\bits:
168	\insn	%r3,0(%r1)
169	mvc	\bytes(256-\bytes,%r1),0(%r1)
170	la	%r1,256(%r1)
171	brctg	%r5,.L__memset_loop\bits
172.L__memset_remainder\bits:
173	\insn	%r3,0(%r1)
174	exrl	%r4,.L__memset_mvc\bits
175	BR_EX	%r14
176.L__memset_store\bits:
177	\insn	%r3,0(%r2)
178.L__memset_exit\bits:
179	BR_EX	%r14
180.L__memset_mvc\bits:
181	mvc	\bytes(1,%r1),0(%r1)
182SYM_FUNC_END(__memset\bits)
183.endm
184
185__MEMSET 16,2,sth
186EXPORT_SYMBOL(__memset16)
187
188__MEMSET 32,4,st
189EXPORT_SYMBOL(__memset32)
190
191__MEMSET 64,8,stg
192EXPORT_SYMBOL(__memset64)