sha1.h

 1/* SPDX-License-Identifier: GPL-2.0-or-later */
 2/*
 3 * SHA-1 optimized for x86_64
 4 *
 5 * Copyright 2025 Google LLC
 6 */
 7#include <asm/fpu/api.h>
 8#include <linux/static_call.h>
 9
10DEFINE_STATIC_CALL(sha1_blocks_x86, sha1_blocks_generic);
11
12#define DEFINE_X86_SHA1_FN(c_fn, asm_fn)                           \
13	asmlinkage void asm_fn(struct sha1_block_state *state,     \
14			       const u8 *data, size_t nblocks);    \
15	static void c_fn(struct sha1_block_state *state,           \
16			 const u8 *data, size_t nblocks)           \
17	{                                                          \
18		if (likely(irq_fpu_usable())) {                    \
19			kernel_fpu_begin();                        \
20			asm_fn(state, data, nblocks);              \
21			kernel_fpu_end();                          \
22		} else {                                           \
23			sha1_blocks_generic(state, data, nblocks); \
24		}                                                  \
25	}
26
27DEFINE_X86_SHA1_FN(sha1_blocks_ssse3, sha1_transform_ssse3);
28DEFINE_X86_SHA1_FN(sha1_blocks_avx, sha1_transform_avx);
29DEFINE_X86_SHA1_FN(sha1_blocks_ni, sha1_ni_transform);
30
31#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */
32
33asmlinkage void sha1_transform_avx2(struct sha1_block_state *state,
34				    const u8 *data, size_t nblocks);
35static void sha1_blocks_avx2(struct sha1_block_state *state,
36			     const u8 *data, size_t nblocks)
37{
38	if (likely(irq_fpu_usable())) {
39		kernel_fpu_begin();
40		/* Select the optimal transform based on the number of blocks */
41		if (nblocks >= SHA1_AVX2_BLOCK_OPTSIZE)
42			sha1_transform_avx2(state, data, nblocks);
43		else
44			sha1_transform_avx(state, data, nblocks);
45		kernel_fpu_end();
46	} else {
47		sha1_blocks_generic(state, data, nblocks);
48	}
49}
50
51static void sha1_blocks(struct sha1_block_state *state,
52			const u8 *data, size_t nblocks)
53{
54	static_call(sha1_blocks_x86)(state, data, nblocks);
55}
56
57#define sha1_mod_init_arch sha1_mod_init_arch
58static void sha1_mod_init_arch(void)
59{
60	if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
61		static_call_update(sha1_blocks_x86, sha1_blocks_ni);
62	} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
63				     NULL) &&
64		   boot_cpu_has(X86_FEATURE_AVX)) {
65		if (boot_cpu_has(X86_FEATURE_AVX2) &&
66		    boot_cpu_has(X86_FEATURE_BMI1) &&
67		    boot_cpu_has(X86_FEATURE_BMI2))
68			static_call_update(sha1_blocks_x86, sha1_blocks_avx2);
69		else
70			static_call_update(sha1_blocks_x86, sha1_blocks_avx);
71	} else if (boot_cpu_has(X86_FEATURE_SSSE3)) {
72		static_call_update(sha1_blocks_x86, sha1_blocks_ssse3);
73	}
74}