sha512.h

 1/* SPDX-License-Identifier: GPL-2.0-or-later */
 2/*
 3 * x86-optimized SHA-512 block function
 4 *
 5 * Copyright 2025 Google LLC
 6 */
 7#include <asm/fpu/api.h>
 8#include <linux/static_call.h>
 9
10DEFINE_STATIC_CALL(sha512_blocks_x86, sha512_blocks_generic);
11
12#define DEFINE_X86_SHA512_FN(c_fn, asm_fn)                                 \
13	asmlinkage void asm_fn(struct sha512_block_state *state,           \
14			       const u8 *data, size_t nblocks);            \
15	static void c_fn(struct sha512_block_state *state, const u8 *data, \
16			 size_t nblocks)                                   \
17	{                                                                  \
18		if (likely(irq_fpu_usable())) {                            \
19			kernel_fpu_begin();                                \
20			asm_fn(state, data, nblocks);                      \
21			kernel_fpu_end();                                  \
22		} else {                                                   \
23			sha512_blocks_generic(state, data, nblocks);       \
24		}                                                          \
25	}
26
27DEFINE_X86_SHA512_FN(sha512_blocks_ssse3, sha512_transform_ssse3);
28DEFINE_X86_SHA512_FN(sha512_blocks_avx, sha512_transform_avx);
29DEFINE_X86_SHA512_FN(sha512_blocks_avx2, sha512_transform_rorx);
30
31static void sha512_blocks(struct sha512_block_state *state,
32			  const u8 *data, size_t nblocks)
33{
34	static_call(sha512_blocks_x86)(state, data, nblocks);
35}
36
37#define sha512_mod_init_arch sha512_mod_init_arch
38static void sha512_mod_init_arch(void)
39{
40	if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) &&
41	    boot_cpu_has(X86_FEATURE_AVX)) {
42		if (boot_cpu_has(X86_FEATURE_AVX2) &&
43		    boot_cpu_has(X86_FEATURE_BMI2))
44			static_call_update(sha512_blocks_x86,
45					   sha512_blocks_avx2);
46		else
47			static_call_update(sha512_blocks_x86,
48					   sha512_blocks_avx);
49	} else if (boot_cpu_has(X86_FEATURE_SSSE3)) {
50		static_call_update(sha512_blocks_x86, sha512_blocks_ssse3);
51	}
52}