riscv_v_helpers.c

 1// SPDX-License-Identifier: GPL-2.0-or-later
 2/*
 3 * Copyright (C) 2023 SiFive
 4 * Author: Andy Chiu <andy.chiu@sifive.com>
 5 */
 6#include <linux/linkage.h>
 7#include <asm/asm.h>
 8
 9#include <asm/vector.h>
10#include <asm/simd.h>
11
12#ifdef CONFIG_MMU
13#include <asm/asm-prototypes.h>
14#endif
15
16#ifdef CONFIG_MMU
17size_t riscv_v_usercopy_threshold = CONFIG_RISCV_ISA_V_UCOPY_THRESHOLD;
18int __asm_vector_usercopy(void *dst, void *src, size_t n);
19int __asm_vector_usercopy_sum_enabled(void *dst, void *src, size_t n);
20int fallback_scalar_usercopy(void *dst, void *src, size_t n);
21int fallback_scalar_usercopy_sum_enabled(void *dst, void *src, size_t n);
22asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n,
23				     bool enable_sum)
24{
25	size_t remain, copied;
26
27	/* skip has_vector() check because it has been done by the asm  */
28	if (!may_use_simd())
29		goto fallback;
30
31	kernel_vector_begin();
32	remain = enable_sum ? __asm_vector_usercopy(dst, src, n) :
33			      __asm_vector_usercopy_sum_enabled(dst, src, n);
34	kernel_vector_end();
35
36	if (remain) {
37		copied = n - remain;
38		dst += copied;
39		src += copied;
40		n = remain;
41		goto fallback;
42	}
43
44	return remain;
45
46fallback:
47	return enable_sum ? fallback_scalar_usercopy(dst, src, n) :
48			    fallback_scalar_usercopy_sum_enabled(dst, src, n);
49}
50#endif