aegis128-aesni-glue.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * The AEGIS-128 Authenticated-Encryption Algorithm
  4 *   Glue for AES-NI + SSE4.1 implementation
  5 *
  6 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
  7 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
  8 */
  9
 10#include <crypto/internal/aead.h>
 11#include <crypto/internal/skcipher.h>
 12#include <crypto/scatterwalk.h>
 13#include <linux/module.h>
 14#include <asm/fpu/api.h>
 15#include <asm/cpu_device_id.h>
 16
 17#define AEGIS128_BLOCK_ALIGN 16
 18#define AEGIS128_BLOCK_SIZE 16
 19#define AEGIS128_NONCE_SIZE 16
 20#define AEGIS128_STATE_BLOCKS 5
 21#define AEGIS128_KEY_SIZE 16
 22#define AEGIS128_MIN_AUTH_SIZE 8
 23#define AEGIS128_MAX_AUTH_SIZE 16
 24
 25struct aegis_block {
 26	u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN);
 27};
 28
 29struct aegis_state {
 30	struct aegis_block blocks[AEGIS128_STATE_BLOCKS];
 31};
 32
 33struct aegis_ctx {
 34	struct aegis_block key;
 35};
 36
 37asmlinkage void aegis128_aesni_init(struct aegis_state *state,
 38				    const struct aegis_block *key,
 39				    const u8 iv[AEGIS128_NONCE_SIZE]);
 40
 41asmlinkage void aegis128_aesni_ad(struct aegis_state *state, const u8 *data,
 42				  unsigned int len);
 43
 44asmlinkage void aegis128_aesni_enc(struct aegis_state *state, const u8 *src,
 45				   u8 *dst, unsigned int len);
 46
 47asmlinkage void aegis128_aesni_dec(struct aegis_state *state, const u8 *src,
 48				   u8 *dst, unsigned int len);
 49
 50asmlinkage void aegis128_aesni_enc_tail(struct aegis_state *state,
 51					const u8 *src, u8 *dst,
 52					unsigned int len);
 53
 54asmlinkage void aegis128_aesni_dec_tail(struct aegis_state *state,
 55					const u8 *src, u8 *dst,
 56					unsigned int len);
 57
 58asmlinkage void aegis128_aesni_final(struct aegis_state *state,
 59				     struct aegis_block *tag_xor,
 60				     unsigned int assoclen,
 61				     unsigned int cryptlen);
 62
 63static void crypto_aegis128_aesni_process_ad(
 64		struct aegis_state *state, struct scatterlist *sg_src,
 65		unsigned int assoclen)
 66{
 67	struct scatter_walk walk;
 68	struct aegis_block buf;
 69	unsigned int pos = 0;
 70
 71	scatterwalk_start(&walk, sg_src);
 72	while (assoclen != 0) {
 73		unsigned int size = scatterwalk_next(&walk, assoclen);
 74		const u8 *src = walk.addr;
 75		unsigned int left = size;
 76
 77		if (pos + size >= AEGIS128_BLOCK_SIZE) {
 78			if (pos > 0) {
 79				unsigned int fill = AEGIS128_BLOCK_SIZE - pos;
 80				memcpy(buf.bytes + pos, src, fill);
 81				aegis128_aesni_ad(state, buf.bytes,
 82						  AEGIS128_BLOCK_SIZE);
 83				pos = 0;
 84				left -= fill;
 85				src += fill;
 86			}
 87
 88			aegis128_aesni_ad(state, src,
 89					  left & ~(AEGIS128_BLOCK_SIZE - 1));
 90			src += left & ~(AEGIS128_BLOCK_SIZE - 1);
 91			left &= AEGIS128_BLOCK_SIZE - 1;
 92		}
 93
 94		memcpy(buf.bytes + pos, src, left);
 95		pos += left;
 96		assoclen -= size;
 97
 98		scatterwalk_done_src(&walk, size);
 99	}
100
101	if (pos > 0) {
102		memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos);
103		aegis128_aesni_ad(state, buf.bytes, AEGIS128_BLOCK_SIZE);
104	}
105}
106
107static __always_inline int
108crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
109				    struct skcipher_walk *walk, bool enc)
110{
111	int err = 0;
112
113	while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
114		if (enc)
115			aegis128_aesni_enc(state, walk->src.virt.addr,
116					   walk->dst.virt.addr,
117					   round_down(walk->nbytes,
118						      AEGIS128_BLOCK_SIZE));
119		else
120			aegis128_aesni_dec(state, walk->src.virt.addr,
121					   walk->dst.virt.addr,
122					   round_down(walk->nbytes,
123						      AEGIS128_BLOCK_SIZE));
124		kernel_fpu_end();
125		err = skcipher_walk_done(walk,
126					 walk->nbytes % AEGIS128_BLOCK_SIZE);
127		kernel_fpu_begin();
128	}
129
130	if (walk->nbytes) {
131		if (enc)
132			aegis128_aesni_enc_tail(state, walk->src.virt.addr,
133						walk->dst.virt.addr,
134						walk->nbytes);
135		else
136			aegis128_aesni_dec_tail(state, walk->src.virt.addr,
137						walk->dst.virt.addr,
138						walk->nbytes);
139		kernel_fpu_end();
140		err = skcipher_walk_done(walk, 0);
141		kernel_fpu_begin();
142	}
143	return err;
144}
145
146static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
147{
148	u8 *ctx = crypto_aead_ctx(aead);
149	ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
150	return (void *)ctx;
151}
152
153static int crypto_aegis128_aesni_setkey(struct crypto_aead *aead, const u8 *key,
154					unsigned int keylen)
155{
156	struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(aead);
157
158	if (keylen != AEGIS128_KEY_SIZE)
159		return -EINVAL;
160
161	memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);
162
163	return 0;
164}
165
166static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
167						unsigned int authsize)
168{
169	if (authsize > AEGIS128_MAX_AUTH_SIZE)
170		return -EINVAL;
171	if (authsize < AEGIS128_MIN_AUTH_SIZE)
172		return -EINVAL;
173	return 0;
174}
175
176static __always_inline int
177crypto_aegis128_aesni_crypt(struct aead_request *req,
178			    struct aegis_block *tag_xor,
179			    unsigned int cryptlen, bool enc)
180{
181	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
182	struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
183	struct skcipher_walk walk;
184	struct aegis_state state;
185	int err;
186
187	if (enc)
188		err = skcipher_walk_aead_encrypt(&walk, req, false);
189	else
190		err = skcipher_walk_aead_decrypt(&walk, req, false);
191	if (err)
192		return err;
193
194	kernel_fpu_begin();
195
196	aegis128_aesni_init(&state, &ctx->key, req->iv);
197	crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
198	err = crypto_aegis128_aesni_process_crypt(&state, &walk, enc);
199	if (err == 0)
200		aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
201	kernel_fpu_end();
202	return err;
203}
204
205static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
206{
207	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
208	struct aegis_block tag = {};
209	unsigned int authsize = crypto_aead_authsize(tfm);
210	unsigned int cryptlen = req->cryptlen;
211	int err;
212
213	err = crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true);
214	if (err)
215		return err;
216
217	scatterwalk_map_and_copy(tag.bytes, req->dst,
218				 req->assoclen + cryptlen, authsize, 1);
219	return 0;
220}
221
222static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
223{
224	static const struct aegis_block zeros = {};
225
226	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
227	struct aegis_block tag;
228	unsigned int authsize = crypto_aead_authsize(tfm);
229	unsigned int cryptlen = req->cryptlen - authsize;
230	int err;
231
232	scatterwalk_map_and_copy(tag.bytes, req->src,
233				 req->assoclen + cryptlen, authsize, 0);
234
235	err = crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false);
236	if (err)
237		return err;
238
239	return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
240}
241
242static struct aead_alg crypto_aegis128_aesni_alg = {
243	.setkey = crypto_aegis128_aesni_setkey,
244	.setauthsize = crypto_aegis128_aesni_setauthsize,
245	.encrypt = crypto_aegis128_aesni_encrypt,
246	.decrypt = crypto_aegis128_aesni_decrypt,
247
248	.ivsize = AEGIS128_NONCE_SIZE,
249	.maxauthsize = AEGIS128_MAX_AUTH_SIZE,
250	.chunksize = AEGIS128_BLOCK_SIZE,
251
252	.base = {
253		.cra_blocksize = 1,
254		.cra_ctxsize = sizeof(struct aegis_ctx) +
255			       __alignof__(struct aegis_ctx),
256		.cra_priority = 400,
257
258		.cra_name = "aegis128",
259		.cra_driver_name = "aegis128-aesni",
260
261		.cra_module = THIS_MODULE,
262	}
263};
264
265static int __init crypto_aegis128_aesni_module_init(void)
266{
267	if (!boot_cpu_has(X86_FEATURE_XMM4_1) ||
268	    !boot_cpu_has(X86_FEATURE_AES) ||
269	    !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
270		return -ENODEV;
271
272	return crypto_register_aead(&crypto_aegis128_aesni_alg);
273}
274
275static void __exit crypto_aegis128_aesni_module_exit(void)
276{
277	crypto_unregister_aead(&crypto_aegis128_aesni_alg);
278}
279
280module_init(crypto_aegis128_aesni_module_init);
281module_exit(crypto_aegis128_aesni_module_exit);
282
283MODULE_LICENSE("GPL");
284MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
285MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE4.1 implementation");
286MODULE_ALIAS_CRYPTO("aegis128");
287MODULE_ALIAS_CRYPTO("aegis128-aesni");