sha3.h

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/*
  3 * SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF)
  4 *
  5 * Copyright 2025 Google LLC
  6 */
  7#include <asm/cpacf.h>
  8#include <linux/cpufeature.h>
  9
 10static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
 11static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3_init_optim);
 12
 13static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
 14			       size_t nblocks, size_t block_size)
 15{
 16	if (static_branch_likely(&have_sha3)) {
 17		/*
 18		 * Note that KIMD assumes little-endian order of the state
 19		 * words.  sha3_state already uses that order, though, so
 20		 * there's no need for a byteswap.
 21		 */
 22		switch (block_size) {
 23		case SHA3_224_BLOCK_SIZE:
 24			cpacf_kimd(CPACF_KIMD_SHA3_224, state,
 25				   data, nblocks * block_size);
 26			return;
 27		case SHA3_256_BLOCK_SIZE:
 28			/*
 29			 * This case handles both SHA3-256 and SHAKE256, since
 30			 * they have the same block size.
 31			 */
 32			cpacf_kimd(CPACF_KIMD_SHA3_256, state,
 33				   data, nblocks * block_size);
 34			return;
 35		case SHA3_384_BLOCK_SIZE:
 36			cpacf_kimd(CPACF_KIMD_SHA3_384, state,
 37				   data, nblocks * block_size);
 38			return;
 39		case SHA3_512_BLOCK_SIZE:
 40			cpacf_kimd(CPACF_KIMD_SHA3_512, state,
 41				   data, nblocks * block_size);
 42			return;
 43		}
 44	}
 45	sha3_absorb_blocks_generic(state, data, nblocks, block_size);
 46}
 47
 48static void sha3_keccakf(struct sha3_state *state)
 49{
 50	if (static_branch_likely(&have_sha3)) {
 51		/*
 52		 * Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain
 53		 * Keccak-f permutation, which is what we want here.  Use
 54		 * SHA3-512 since it has the smallest block size.
 55		 */
 56		static const u8 zeroes[SHA3_512_BLOCK_SIZE];
 57
 58		cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes));
 59	} else {
 60		sha3_keccakf_generic(state);
 61	}
 62}
 63
 64static inline bool s390_sha3(int func, const u8 *in, size_t in_len,
 65			     u8 *out, size_t out_len)
 66{
 67	struct sha3_state state;
 68
 69	if (!static_branch_likely(&have_sha3))
 70		return false;
 71
 72	if (static_branch_likely(&have_sha3_init_optim))
 73		func |= CPACF_KLMD_NIP | CPACF_KLMD_DUFOP;
 74	else
 75		memset(&state, 0, sizeof(state));
 76
 77	cpacf_klmd(func, &state, in, in_len);
 78
 79	if (static_branch_likely(&have_sha3_init_optim))
 80		kmsan_unpoison_memory(&state, out_len);
 81
 82	memcpy(out, &state, out_len);
 83	memzero_explicit(&state, sizeof(state));
 84	return true;
 85}
 86
 87#define sha3_224_arch sha3_224_arch
 88static bool sha3_224_arch(const u8 *in, size_t in_len,
 89			  u8 out[SHA3_224_DIGEST_SIZE])
 90{
 91	return s390_sha3(CPACF_KLMD_SHA3_224, in, in_len,
 92			 out, SHA3_224_DIGEST_SIZE);
 93}
 94
 95#define sha3_256_arch sha3_256_arch
 96static bool sha3_256_arch(const u8 *in, size_t in_len,
 97			  u8 out[SHA3_256_DIGEST_SIZE])
 98{
 99	return s390_sha3(CPACF_KLMD_SHA3_256, in, in_len,
100			 out, SHA3_256_DIGEST_SIZE);
101}
102
103#define sha3_384_arch sha3_384_arch
104static bool sha3_384_arch(const u8 *in, size_t in_len,
105			  u8 out[SHA3_384_DIGEST_SIZE])
106{
107	return s390_sha3(CPACF_KLMD_SHA3_384, in, in_len,
108			 out, SHA3_384_DIGEST_SIZE);
109}
110
111#define sha3_512_arch sha3_512_arch
112static bool sha3_512_arch(const u8 *in, size_t in_len,
113			  u8 out[SHA3_512_DIGEST_SIZE])
114{
115	return s390_sha3(CPACF_KLMD_SHA3_512, in, in_len,
116			 out, SHA3_512_DIGEST_SIZE);
117}
118
119#define sha3_mod_init_arch sha3_mod_init_arch
120static void sha3_mod_init_arch(void)
121{
122	int num_present = 0;
123	int num_possible = 0;
124
125	if (!cpu_have_feature(S390_CPU_FEATURE_MSA))
126		return;
127	/*
128	 * Since all the SHA-3 functions are in Message-Security-Assist
129	 * Extension 6, just treat them as all or nothing.  This way we need
130	 * only one static_key.
131	 */
132#define QUERY(opcode, func) \
133	({ num_present += !!cpacf_query_func(opcode, func); num_possible++; })
134	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224);
135	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256);
136	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384);
137	QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512);
138	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_224);
139	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_256);
140	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_384);
141	QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_512);
142#undef QUERY
143
144	if (num_present == num_possible) {
145		static_branch_enable(&have_sha3);
146		if (test_facility(86))
147			static_branch_enable(&have_sha3_init_optim);
148	} else if (num_present != 0) {
149		pr_warn("Unsupported combination of SHA-3 facilities\n");
150	}
151}