polyval.h

 1/* SPDX-License-Identifier: GPL-2.0-or-later */
 2/*
 3 * POLYVAL library functions, arm64 optimized
 4 *
 5 * Copyright 2025 Google LLC
 6 */
 7#include <asm/simd.h>
 8#include <linux/cpufeature.h>
 9
10#define NUM_H_POWERS 8
11
12static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
13
14asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
15				  const struct polyval_elem *b);
16asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
17				     const struct polyval_key *key,
18				     const u8 *data, size_t nblocks);
19
20static void polyval_preparekey_arch(struct polyval_key *key,
21				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
22{
23	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
24	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
25	if (static_branch_likely(&have_pmull) && may_use_simd()) {
26		scoped_ksimd() {
27			for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
28				key->h_powers[i] = key->h_powers[i + 1];
29				polyval_mul_pmull(
30					&key->h_powers[i],
31					&key->h_powers[NUM_H_POWERS - 1]);
32			}
33		}
34	} else {
35		for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
36			key->h_powers[i] = key->h_powers[i + 1];
37			polyval_mul_generic(&key->h_powers[i],
38					    &key->h_powers[NUM_H_POWERS - 1]);
39		}
40	}
41}
42
43static void polyval_mul_arch(struct polyval_elem *acc,
44			     const struct polyval_key *key)
45{
46	if (static_branch_likely(&have_pmull) && may_use_simd()) {
47		scoped_ksimd()
48			polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]);
49	} else {
50		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
51	}
52}
53
54static void polyval_blocks_arch(struct polyval_elem *acc,
55				const struct polyval_key *key,
56				const u8 *data, size_t nblocks)
57{
58	if (static_branch_likely(&have_pmull) && may_use_simd()) {
59		do {
60			/* Allow rescheduling every 4 KiB. */
61			size_t n = min_t(size_t, nblocks,
62					 4096 / POLYVAL_BLOCK_SIZE);
63
64			scoped_ksimd()
65				polyval_blocks_pmull(acc, key, data, n);
66			data += n * POLYVAL_BLOCK_SIZE;
67			nblocks -= n;
68		} while (nblocks);
69	} else {
70		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
71				       data, nblocks);
72	}
73}
74
75#define polyval_mod_init_arch polyval_mod_init_arch
76static void polyval_mod_init_arch(void)
77{
78	if (cpu_have_named_feature(PMULL))
79		static_branch_enable(&have_pmull);
80}