crc32.h

 1// SPDX-License-Identifier: GPL-2.0-only
 2/*
 3 * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
 4 *
 5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
 6 */
 7
 8#include <linux/cpufeature.h>
 9
10#include <asm/hwcap.h>
11#include <asm/simd.h>
12
13static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
14static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
15
16#define PMULL_MIN_LEN	64	/* min size of buffer for pmull functions */
17
18asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
19asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
20
21asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
22asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
23
24static inline u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len)
25{
26	if (static_branch_likely(&have_crc32))
27		return crc32_armv8_le(crc, p, len);
28	return crc32_le_base(crc, p, len);
29}
30
31static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
32{
33	if (len >= PMULL_MIN_LEN + 15 &&
34	    static_branch_likely(&have_pmull) && likely(may_use_simd())) {
35		size_t n = -(uintptr_t)p & 15;
36
37		/* align p to 16-byte boundary */
38		if (n) {
39			crc = crc32_le_scalar(crc, p, n);
40			p += n;
41			len -= n;
42		}
43		n = round_down(len, 16);
44		scoped_ksimd()
45			crc = crc32_pmull_le(p, n, crc);
46		p += n;
47		len -= n;
48	}
49	return crc32_le_scalar(crc, p, len);
50}
51
52static inline u32 crc32c_scalar(u32 crc, const u8 *p, size_t len)
53{
54	if (static_branch_likely(&have_crc32))
55		return crc32c_armv8_le(crc, p, len);
56	return crc32c_base(crc, p, len);
57}
58
59static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
60{
61	if (len >= PMULL_MIN_LEN + 15 &&
62	    static_branch_likely(&have_pmull) && likely(may_use_simd())) {
63		size_t n = -(uintptr_t)p & 15;
64
65		/* align p to 16-byte boundary */
66		if (n) {
67			crc = crc32c_scalar(crc, p, n);
68			p += n;
69			len -= n;
70		}
71		n = round_down(len, 16);
72		scoped_ksimd()
73			crc = crc32c_pmull_le(p, n, crc);
74		p += n;
75		len -= n;
76	}
77	return crc32c_scalar(crc, p, len);
78}
79
80#define crc32_be_arch crc32_be_base /* not implemented on this arch */
81
82#define crc32_mod_init_arch crc32_mod_init_arch
83static void crc32_mod_init_arch(void)
84{
85	if (elf_hwcap2 & HWCAP2_CRC32)
86		static_branch_enable(&have_crc32);
87	if (elf_hwcap2 & HWCAP2_PMULL)
88		static_branch_enable(&have_pmull);
89}
90
91static inline u32 crc32_optimizations_arch(void)
92{
93	if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL))
94		return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
95	return 0;
96}