blake2s.h

 1/* SPDX-License-Identifier: GPL-2.0 OR MIT */
 2/*
 3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 4 */
 5
 6#include <asm/cpufeature.h>
 7#include <asm/fpu/api.h>
 8#include <asm/processor.h>
 9#include <asm/simd.h>
10#include <linux/jump_label.h>
11#include <linux/kernel.h>
12#include <linux/sizes.h>
13
14asmlinkage void blake2s_compress_ssse3(struct blake2s_ctx *ctx,
15				       const u8 *data, size_t nblocks, u32 inc);
16asmlinkage void blake2s_compress_avx512(struct blake2s_ctx *ctx,
17					const u8 *data, size_t nblocks, u32 inc);
18
19static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
20static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
21
22static void blake2s_compress(struct blake2s_ctx *ctx,
23			     const u8 *data, size_t nblocks, u32 inc)
24{
25	/* SIMD disables preemption, so relax after processing each page. */
26	BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
27
28	if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) {
29		blake2s_compress_generic(ctx, data, nblocks, inc);
30		return;
31	}
32
33	do {
34		const size_t blocks = min_t(size_t, nblocks,
35					    SZ_4K / BLAKE2S_BLOCK_SIZE);
36
37		kernel_fpu_begin();
38		if (static_branch_likely(&blake2s_use_avx512))
39			blake2s_compress_avx512(ctx, data, blocks, inc);
40		else
41			blake2s_compress_ssse3(ctx, data, blocks, inc);
42		kernel_fpu_end();
43
44		data += blocks * BLAKE2S_BLOCK_SIZE;
45		nblocks -= blocks;
46	} while (nblocks);
47}
48
49#define blake2s_mod_init_arch blake2s_mod_init_arch
50static void blake2s_mod_init_arch(void)
51{
52	if (boot_cpu_has(X86_FEATURE_SSSE3))
53		static_branch_enable(&blake2s_use_ssse3);
54
55	if (boot_cpu_has(X86_FEATURE_AVX) &&
56	    boot_cpu_has(X86_FEATURE_AVX2) &&
57	    boot_cpu_has(X86_FEATURE_AVX512F) &&
58	    boot_cpu_has(X86_FEATURE_AVX512VL) &&
59	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
60			      XFEATURE_MASK_AVX512, NULL))
61		static_branch_enable(&blake2s_use_avx512);
62}