ghash-riscv64-glue.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * GHASH using the RISC-V vector crypto extensions
  4 *
  5 * Copyright (C) 2023 VRULL GmbH
  6 * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
  7 *
  8 * Copyright (C) 2023 SiFive, Inc.
  9 * Author: Jerry Shih <jerry.shih@sifive.com>
 10 */
 11
 12#include <asm/simd.h>
 13#include <asm/vector.h>
 14#include <crypto/b128ops.h>
 15#include <crypto/gf128mul.h>
 16#include <crypto/ghash.h>
 17#include <crypto/internal/hash.h>
 18#include <crypto/internal/simd.h>
 19#include <crypto/utils.h>
 20#include <linux/errno.h>
 21#include <linux/kernel.h>
 22#include <linux/module.h>
 23#include <linux/string.h>
 24
 25asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data,
 26			   size_t len);
 27
 28struct riscv64_ghash_tfm_ctx {
 29	be128 key;
 30};
 31
 32struct riscv64_ghash_desc_ctx {
 33	be128 accumulator;
 34};
 35
 36static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
 37				unsigned int keylen)
 38{
 39	struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm);
 40
 41	if (keylen != GHASH_BLOCK_SIZE)
 42		return -EINVAL;
 43
 44	memcpy(&tctx->key, key, GHASH_BLOCK_SIZE);
 45
 46	return 0;
 47}
 48
 49static int riscv64_ghash_init(struct shash_desc *desc)
 50{
 51	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 52
 53	*dctx = (struct riscv64_ghash_desc_ctx){};
 54
 55	return 0;
 56}
 57
 58static inline void
 59riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx,
 60		     struct riscv64_ghash_desc_ctx *dctx,
 61		     const u8 *src, size_t srclen)
 62{
 63	/* The srclen is nonzero and a multiple of 16. */
 64	if (crypto_simd_usable()) {
 65		kernel_vector_begin();
 66		ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen);
 67		kernel_vector_end();
 68	} else {
 69		do {
 70			crypto_xor((u8 *)&dctx->accumulator, src,
 71				   GHASH_BLOCK_SIZE);
 72			gf128mul_lle(&dctx->accumulator, &tctx->key);
 73			src += GHASH_BLOCK_SIZE;
 74			srclen -= GHASH_BLOCK_SIZE;
 75		} while (srclen);
 76	}
 77}
 78
 79static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src,
 80				unsigned int srclen)
 81{
 82	const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 83	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 84
 85	riscv64_ghash_blocks(tctx, dctx, src,
 86			     round_down(srclen, GHASH_BLOCK_SIZE));
 87	return srclen - round_down(srclen, GHASH_BLOCK_SIZE);
 88}
 89
 90static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src,
 91			       unsigned int len, u8 *out)
 92{
 93	const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 94	struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 95
 96	if (len) {
 97		u8 buf[GHASH_BLOCK_SIZE] = {};
 98
 99		memcpy(buf, src, len);
100		riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE);
101		memzero_explicit(buf, sizeof(buf));
102	}
103
104	memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE);
105	return 0;
106}
107
108static struct shash_alg riscv64_ghash_alg = {
109	.init = riscv64_ghash_init,
110	.update = riscv64_ghash_update,
111	.finup = riscv64_ghash_finup,
112	.setkey = riscv64_ghash_setkey,
113	.descsize = sizeof(struct riscv64_ghash_desc_ctx),
114	.digestsize = GHASH_DIGEST_SIZE,
115	.base = {
116		.cra_blocksize = GHASH_BLOCK_SIZE,
117		.cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx),
118		.cra_priority = 300,
119		.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
120		.cra_name = "ghash",
121		.cra_driver_name = "ghash-riscv64-zvkg",
122		.cra_module = THIS_MODULE,
123	},
124};
125
126static int __init riscv64_ghash_mod_init(void)
127{
128	if (riscv_isa_extension_available(NULL, ZVKG) &&
129	    riscv_vector_vlen() >= 128)
130		return crypto_register_shash(&riscv64_ghash_alg);
131
132	return -ENODEV;
133}
134
135static void __exit riscv64_ghash_mod_exit(void)
136{
137	crypto_unregister_shash(&riscv64_ghash_alg);
138}
139
140module_init(riscv64_ghash_mod_init);
141module_exit(riscv64_ghash_mod_exit);
142
143MODULE_DESCRIPTION("GHASH (RISC-V accelerated)");
144MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
145MODULE_LICENSE("GPL");
146MODULE_ALIAS_CRYPTO("ghash");