crc32.h

 1// SPDX-License-Identifier: GPL-2.0-only
 2
 3#include <asm/alternative.h>
 4#include <asm/cpufeature.h>
 5#include <asm/simd.h>
 6
 7// The minimum input length to consider the 4-way interleaved code path
 8static const size_t min_len = 1024;
 9
10asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len);
11asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len);
12asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len);
13
14asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
15asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
16asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);
17
18static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
19{
20	if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
21		return crc32_le_base(crc, p, len);
22
23	if (len >= min_len && cpu_have_named_feature(PMULL) &&
24	    likely(may_use_simd())) {
25		scoped_ksimd()
26			crc = crc32_le_arm64_4way(crc, p, len);
27
28		p += round_down(len, 64);
29		len %= 64;
30
31		if (!len)
32			return crc;
33	}
34
35	return crc32_le_arm64(crc, p, len);
36}
37
38static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
39{
40	if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
41		return crc32c_base(crc, p, len);
42
43	if (len >= min_len && cpu_have_named_feature(PMULL) &&
44	    likely(may_use_simd())) {
45		scoped_ksimd()
46			crc = crc32c_le_arm64_4way(crc, p, len);
47
48		p += round_down(len, 64);
49		len %= 64;
50
51		if (!len)
52			return crc;
53	}
54
55	return crc32c_le_arm64(crc, p, len);
56}
57
58static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
59{
60	if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
61		return crc32_be_base(crc, p, len);
62
63	if (len >= min_len && cpu_have_named_feature(PMULL) &&
64	    likely(may_use_simd())) {
65		scoped_ksimd()
66			crc = crc32_be_arm64_4way(crc, p, len);
67
68		p += round_down(len, 64);
69		len %= 64;
70
71		if (!len)
72			return crc;
73	}
74
75	return crc32_be_arm64(crc, p, len);
76}
77
78static inline u32 crc32_optimizations_arch(void)
79{
80	if (alternative_has_cap_likely(ARM64_HAS_CRC32))
81		return CRC32_LE_OPTIMIZATION |
82		       CRC32_BE_OPTIMIZATION |
83		       CRC32C_OPTIMIZATION;
84	return 0;
85}