neon.c

 1// SPDX-License-Identifier: GPL-2.0-only
 2/*
 3 * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
 4 *
 5 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
 6 */
 7
 8#include <linux/raid/pq.h>
 9
10#ifdef __KERNEL__
11#include <asm/simd.h>
12#else
13#define scoped_ksimd()
14#define cpu_has_neon()		(1)
15#endif
16
17/*
18 * There are 2 reasons these wrappers are kept in a separate compilation unit
19 * from the actual implementations in neonN.c (generated from neon.uc by
20 * unroll.awk):
21 * - the actual implementations use NEON intrinsics, and the GCC support header
22 *   (arm_neon.h) is not fully compatible (type wise) with the kernel;
23 * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
24 *   and we have to make sure that we never use *any* NEON/VFP instructions
25 *   outside a kernel_neon_begin()/kernel_neon_end() pair.
26 */
27
28#define RAID6_NEON_WRAPPER(_n)						\
29	static void raid6_neon ## _n ## _gen_syndrome(int disks,	\
30					size_t bytes, void **ptrs)	\
31	{								\
32		void raid6_neon ## _n  ## _gen_syndrome_real(int,	\
33						unsigned long, void**);	\
34		scoped_ksimd()						\
35			raid6_neon ## _n ## _gen_syndrome_real(disks,	\
36					(unsigned long)bytes, ptrs);	\
37	}								\
38	static void raid6_neon ## _n ## _xor_syndrome(int disks,	\
39					int start, int stop, 		\
40					size_t bytes, void **ptrs)	\
41	{								\
42		void raid6_neon ## _n  ## _xor_syndrome_real(int,	\
43				int, int, unsigned long, void**);	\
44		scoped_ksimd()						\
45			raid6_neon ## _n ## _xor_syndrome_real(disks,	\
46				start, stop, (unsigned long)bytes, ptrs);\
47	}								\
48	struct raid6_calls const raid6_neonx ## _n = {			\
49		raid6_neon ## _n ## _gen_syndrome,			\
50		raid6_neon ## _n ## _xor_syndrome,			\
51		raid6_have_neon,					\
52		"neonx" #_n,						\
53		0							\
54	}
55
56static int raid6_have_neon(void)
57{
58	return cpu_has_neon();
59}
60
61RAID6_NEON_WRAPPER(1);
62RAID6_NEON_WRAPPER(2);
63RAID6_NEON_WRAPPER(4);
64RAID6_NEON_WRAPPER(8);