polyval.h

 1/* SPDX-License-Identifier: GPL-2.0-or-later */
 2/*
 3 * POLYVAL library functions, x86_64 optimized
 4 *
 5 * Copyright 2025 Google LLC
 6 */
 7#include <asm/fpu/api.h>
 8#include <linux/cpufeature.h>
 9
10#define NUM_H_POWERS 8
11
12static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx);
13
14asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
15				       const struct polyval_elem *b);
16asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
17					  const struct polyval_key *key,
18					  const u8 *data, size_t nblocks);
19
20static void polyval_preparekey_arch(struct polyval_key *key,
21				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
22{
23	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
24	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
25	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
26		kernel_fpu_begin();
27		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
28			key->h_powers[i] = key->h_powers[i + 1];
29			polyval_mul_pclmul_avx(
30				&key->h_powers[i],
31				&key->h_powers[NUM_H_POWERS - 1]);
32		}
33		kernel_fpu_end();
34	} else {
35		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
36			key->h_powers[i] = key->h_powers[i + 1];
37			polyval_mul_generic(&key->h_powers[i],
38					    &key->h_powers[NUM_H_POWERS - 1]);
39		}
40	}
41}
42
43static void polyval_mul_arch(struct polyval_elem *acc,
44			     const struct polyval_key *key)
45{
46	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
47		kernel_fpu_begin();
48		polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]);
49		kernel_fpu_end();
50	} else {
51		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
52	}
53}
54
55static void polyval_blocks_arch(struct polyval_elem *acc,
56				const struct polyval_key *key,
57				const u8 *data, size_t nblocks)
58{
59	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
60		do {
61			/* Allow rescheduling every 4 KiB. */
62			size_t n = min_t(size_t, nblocks,
63					 4096 / POLYVAL_BLOCK_SIZE);
64
65			kernel_fpu_begin();
66			polyval_blocks_pclmul_avx(acc, key, data, n);
67			kernel_fpu_end();
68			data += n * POLYVAL_BLOCK_SIZE;
69			nblocks -= n;
70		} while (nblocks);
71	} else {
72		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
73				       data, nblocks);
74	}
75}
76
77#define polyval_mod_init_arch polyval_mod_init_arch
78static void polyval_mod_init_arch(void)
79{
80	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) &&
81	    boot_cpu_has(X86_FEATURE_AVX))
82		static_branch_enable(&have_pclmul_avx);
83}