ghash.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * GHASH routines supporting VMX instructions on the Power 8
  4 *
  5 * Copyright (C) 2015, 2019 International Business Machines Inc.
  6 *
  7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
  8 *
  9 * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback
 10 * mechanism. The new approach is based on arm64 code, which is:
 11 *   Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
 12 */
 13
 14#include "aesp8-ppc.h"
 15#include <asm/switch_to.h>
 16#include <crypto/aes.h>
 17#include <crypto/gf128mul.h>
 18#include <crypto/ghash.h>
 19#include <crypto/internal/hash.h>
 20#include <crypto/internal/simd.h>
 21#include <linux/err.h>
 22#include <linux/kernel.h>
 23#include <linux/module.h>
 24#include <linux/string.h>
 25#include <linux/uaccess.h>
 26
 27void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
 28void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
 29void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
 30		  const u8 *in, size_t len);
 31
 32struct p8_ghash_ctx {
 33	/* key used by vector asm */
 34	u128 htable[16];
 35	/* key used by software fallback */
 36	be128 key;
 37};
 38
 39struct p8_ghash_desc_ctx {
 40	u64 shash[2];
 41};
 42
 43static int p8_ghash_init(struct shash_desc *desc)
 44{
 45	struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
 46
 47	memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
 48	return 0;
 49}
 50
 51static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
 52			   unsigned int keylen)
 53{
 54	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
 55
 56	if (keylen != GHASH_BLOCK_SIZE)
 57		return -EINVAL;
 58
 59	preempt_disable();
 60	pagefault_disable();
 61	enable_kernel_vsx();
 62	gcm_init_p8(ctx->htable, (const u64 *) key);
 63	disable_kernel_vsx();
 64	pagefault_enable();
 65	preempt_enable();
 66
 67	memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
 68
 69	return 0;
 70}
 71
 72static inline void __ghash_block(struct p8_ghash_ctx *ctx,
 73				 struct p8_ghash_desc_ctx *dctx,
 74				 const u8 *src)
 75{
 76	if (crypto_simd_usable()) {
 77		preempt_disable();
 78		pagefault_disable();
 79		enable_kernel_vsx();
 80		gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE);
 81		disable_kernel_vsx();
 82		pagefault_enable();
 83		preempt_enable();
 84	} else {
 85		crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
 86		gf128mul_lle((be128 *)dctx->shash, &ctx->key);
 87	}
 88}
 89
 90static inline int __ghash_blocks(struct p8_ghash_ctx *ctx,
 91				 struct p8_ghash_desc_ctx *dctx,
 92				 const u8 *src, unsigned int srclen)
 93{
 94	int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE);
 95
 96	srclen -= remain;
 97	if (crypto_simd_usable()) {
 98		preempt_disable();
 99		pagefault_disable();
100		enable_kernel_vsx();
101		gcm_ghash_p8(dctx->shash, ctx->htable,
102				src, srclen);
103		disable_kernel_vsx();
104		pagefault_enable();
105		preempt_enable();
106	} else {
107		do {
108			crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
109			gf128mul_lle((be128 *)dctx->shash, &ctx->key);
110			srclen -= GHASH_BLOCK_SIZE;
111			src += GHASH_BLOCK_SIZE;
112		} while (srclen);
113	}
114
115	return remain;
116}
117
118static int p8_ghash_update(struct shash_desc *desc,
119			   const u8 *src, unsigned int srclen)
120{
121	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
122	struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
123
124	return __ghash_blocks(ctx, dctx, src, srclen);
125}
126
127static int p8_ghash_finup(struct shash_desc *desc, const u8 *src,
128			  unsigned int len, u8 *out)
129{
130	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
131	struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
132
133	if (len) {
134		u8 buf[GHASH_BLOCK_SIZE] = {};
135
136		memcpy(buf, src, len);
137		__ghash_block(ctx, dctx, buf);
138		memzero_explicit(buf, sizeof(buf));
139	}
140	memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
141	return 0;
142}
143
144struct shash_alg p8_ghash_alg = {
145	.digestsize = GHASH_DIGEST_SIZE,
146	.init = p8_ghash_init,
147	.update = p8_ghash_update,
148	.finup = p8_ghash_finup,
149	.setkey = p8_ghash_setkey,
150	.descsize = sizeof(struct p8_ghash_desc_ctx),
151	.base = {
152		 .cra_name = "ghash",
153		 .cra_driver_name = "p8_ghash",
154		 .cra_priority = 1000,
155		 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
156		 .cra_blocksize = GHASH_BLOCK_SIZE,
157		 .cra_ctxsize = sizeof(struct p8_ghash_ctx),
158		 .cra_module = THIS_MODULE,
159	},
160};