memmove.S

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 */
  5
  6#include <linux/export.h>
  7#include <asm/alternative-asm.h>
  8#include <asm/asm.h>
  9#include <asm/asmmacro.h>
 10#include <asm/cpu.h>
 11#include <asm/regdef.h>
 12
 13.section .noinstr.text, "ax"
 14
 15SYM_FUNC_START(memmove)
 16	blt	a0, a1, __memcpy	/* dst < src, memcpy */
 17	blt	a1, a0, __rmemcpy	/* src < dst, rmemcpy */
 18	jr	ra			/* dst == src, return */
 19SYM_FUNC_END(memmove)
 20SYM_FUNC_ALIAS(__memmove, memmove)
 21
 22EXPORT_SYMBOL(memmove)
 23EXPORT_SYMBOL(__memmove)
 24
 25_ASM_NOKPROBE(memmove)
 26_ASM_NOKPROBE(__memmove)
 27
 28SYM_FUNC_START(__rmemcpy)
 29	/*
 30	 * Some CPUs support hardware unaligned access
 31	 */
 32	ALTERNATIVE	"b __rmemcpy_generic", \
 33			"b __rmemcpy_fast", CPU_FEATURE_UAL
 34SYM_FUNC_END(__rmemcpy)
 35_ASM_NOKPROBE(__rmemcpy)
 36
 37/*
 38 * void *__rmemcpy_generic(void *dst, const void *src, size_t n)
 39 *
 40 * a0: dst
 41 * a1: src
 42 * a2: n
 43 */
 44SYM_FUNC_START(__rmemcpy_generic)
 45	move	a3, a0
 46	beqz	a2, 2f
 47
 48	add.d	a0, a0, a2
 49	add.d	a1, a1, a2
 50
 511:	ld.b	t0, a1, -1
 52	st.b	t0, a0, -1
 53	addi.d	a0, a0, -1
 54	addi.d	a1, a1, -1
 55	addi.d	a2, a2, -1
 56	bgt	a2, zero, 1b
 57
 582:	move	a0, a3
 59	jr	ra
 60SYM_FUNC_END(__rmemcpy_generic)
 61_ASM_NOKPROBE(__rmemcpy_generic)
 62
 63/*
 64 * void *__rmemcpy_fast(void *dst, const void *src, size_t n)
 65 *
 66 * a0: dst
 67 * a1: src
 68 * a2: n
 69 */
 70SYM_FUNC_START(__rmemcpy_fast)
 71	sltui	t0, a2, 9
 72	bnez	t0, __memcpy_small
 73
 74	add.d	a3, a1, a2
 75	add.d	a2, a0, a2
 76	ld.d	a6, a1, 0
 77	ld.d	a7, a3, -8
 78
 79	/* align up destination address */
 80	andi	t1, a2, 7
 81	sub.d	a3, a3, t1
 82	sub.d	a5, a2, t1
 83
 84	addi.d	a4, a1, 64
 85	bgeu	a4, a3, .Llt64
 86
 87	/* copy 64 bytes at a time */
 88.Lloop64:
 89	ld.d	t0, a3, -8
 90	ld.d	t1, a3, -16
 91	ld.d	t2, a3, -24
 92	ld.d	t3, a3, -32
 93	ld.d	t4, a3, -40
 94	ld.d	t5, a3, -48
 95	ld.d	t6, a3, -56
 96	ld.d	t7, a3, -64
 97	addi.d	a3, a3, -64
 98	st.d	t0, a5, -8
 99	st.d	t1, a5, -16
100	st.d	t2, a5, -24
101	st.d	t3, a5, -32
102	st.d	t4, a5, -40
103	st.d	t5, a5, -48
104	st.d	t6, a5, -56
105	st.d	t7, a5, -64
106	addi.d	a5, a5, -64
107	bltu	a4, a3, .Lloop64
108
109	/* copy the remaining bytes */
110.Llt64:
111	addi.d	a4, a1, 32
112	bgeu	a4, a3, .Llt32
113	ld.d	t0, a3, -8
114	ld.d	t1, a3, -16
115	ld.d	t2, a3, -24
116	ld.d	t3, a3, -32
117	addi.d	a3, a3, -32
118	st.d	t0, a5, -8
119	st.d	t1, a5, -16
120	st.d	t2, a5, -24
121	st.d	t3, a5, -32
122	addi.d	a5, a5, -32
123
124.Llt32:
125	addi.d	a4, a1, 16
126	bgeu	a4, a3, .Llt16
127	ld.d	t0, a3, -8
128	ld.d	t1, a3, -16
129	addi.d	a3, a3, -16
130	st.d	t0, a5, -8
131	st.d	t1, a5, -16
132	addi.d	a5, a5, -16
133
134.Llt16:
135	addi.d	a4, a1, 8
136	bgeu	a4, a3, .Llt8
137	ld.d	t0, a3, -8
138	st.d	t0, a5, -8
139
140.Llt8:
141	st.d	a6, a0, 0
142	st.d	a7, a2, -8
143
144	/* return */
145	jr	ra
146SYM_FUNC_END(__rmemcpy_fast)
147_ASM_NOKPROBE(__rmemcpy_fast)