aegis128-core.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * The AEGIS-128 Authenticated-Encryption Algorithm
  4 *
  5 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
  6 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
  7 */
  8
  9#include <crypto/algapi.h>
 10#include <crypto/internal/aead.h>
 11#include <crypto/internal/simd.h>
 12#include <crypto/internal/skcipher.h>
 13#include <crypto/scatterwalk.h>
 14#include <linux/err.h>
 15#include <linux/init.h>
 16#include <linux/jump_label.h>
 17#include <linux/kernel.h>
 18#include <linux/module.h>
 19#include <linux/scatterlist.h>
 20
 21#include <asm/simd.h>
 22
 23#include "aegis.h"
 24
 25#define AEGIS128_NONCE_SIZE 16
 26#define AEGIS128_STATE_BLOCKS 5
 27#define AEGIS128_KEY_SIZE 16
 28#define AEGIS128_MIN_AUTH_SIZE 8
 29#define AEGIS128_MAX_AUTH_SIZE 16
 30
 31struct aegis_state {
 32	union aegis_block blocks[AEGIS128_STATE_BLOCKS];
 33};
 34
 35struct aegis_ctx {
 36	union aegis_block key;
 37};
 38
 39static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_simd);
 40
 41static const union aegis_block crypto_aegis_const[2] = {
 42	{ .words64 = {
 43		cpu_to_le64(U64_C(0x0d08050302010100)),
 44		cpu_to_le64(U64_C(0x6279e99059372215)),
 45	} },
 46	{ .words64 = {
 47		cpu_to_le64(U64_C(0xf12fc26d55183ddb)),
 48		cpu_to_le64(U64_C(0xdd28b57342311120)),
 49	} },
 50};
 51
 52static bool aegis128_do_simd(void)
 53{
 54#ifdef CONFIG_CRYPTO_AEGIS128_SIMD
 55	if (static_branch_likely(&have_simd))
 56		return crypto_simd_usable();
 57#endif
 58	return false;
 59}
 60
 61static void crypto_aegis128_update(struct aegis_state *state)
 62{
 63	union aegis_block tmp;
 64	unsigned int i;
 65
 66	tmp = state->blocks[AEGIS128_STATE_BLOCKS - 1];
 67	for (i = AEGIS128_STATE_BLOCKS - 1; i > 0; i--)
 68		crypto_aegis_aesenc(&state->blocks[i], &state->blocks[i - 1],
 69				    &state->blocks[i]);
 70	crypto_aegis_aesenc(&state->blocks[0], &tmp, &state->blocks[0]);
 71}
 72
 73static void crypto_aegis128_update_a(struct aegis_state *state,
 74				     const union aegis_block *msg,
 75				     bool do_simd)
 76{
 77	if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) {
 78		crypto_aegis128_update_simd(state, msg);
 79		return;
 80	}
 81
 82	crypto_aegis128_update(state);
 83	crypto_aegis_block_xor(&state->blocks[0], msg);
 84}
 85
 86static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg,
 87				     bool do_simd)
 88{
 89	if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) {
 90		crypto_aegis128_update_simd(state, msg);
 91		return;
 92	}
 93
 94	crypto_aegis128_update(state);
 95	crypto_xor(state->blocks[0].bytes, msg, AEGIS_BLOCK_SIZE);
 96}
 97
 98static void crypto_aegis128_init(struct aegis_state *state,
 99				 const union aegis_block *key,
100				 const u8 *iv)
101{
102	union aegis_block key_iv;
103	unsigned int i;
104
105	key_iv = *key;
106	crypto_xor(key_iv.bytes, iv, AEGIS_BLOCK_SIZE);
107
108	state->blocks[0] = key_iv;
109	state->blocks[1] = crypto_aegis_const[1];
110	state->blocks[2] = crypto_aegis_const[0];
111	state->blocks[3] = *key;
112	state->blocks[4] = *key;
113
114	crypto_aegis_block_xor(&state->blocks[3], &crypto_aegis_const[0]);
115	crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[1]);
116
117	for (i = 0; i < 5; i++) {
118		crypto_aegis128_update_a(state, key, false);
119		crypto_aegis128_update_a(state, &key_iv, false);
120	}
121}
122
123static void crypto_aegis128_ad(struct aegis_state *state,
124			       const u8 *src, unsigned int size,
125			       bool do_simd)
126{
127	if (AEGIS_ALIGNED(src)) {
128		const union aegis_block *src_blk =
129				(const union aegis_block *)src;
130
131		while (size >= AEGIS_BLOCK_SIZE) {
132			crypto_aegis128_update_a(state, src_blk, do_simd);
133
134			size -= AEGIS_BLOCK_SIZE;
135			src_blk++;
136		}
137	} else {
138		while (size >= AEGIS_BLOCK_SIZE) {
139			crypto_aegis128_update_u(state, src, do_simd);
140
141			size -= AEGIS_BLOCK_SIZE;
142			src += AEGIS_BLOCK_SIZE;
143		}
144	}
145}
146
147static void crypto_aegis128_wipe_chunk(struct aegis_state *state, u8 *dst,
148				       const u8 *src, unsigned int size)
149{
150	memzero_explicit(dst, size);
151}
152
153static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst,
154					  const u8 *src, unsigned int size)
155{
156	union aegis_block tmp;
157
158	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
159		while (size >= AEGIS_BLOCK_SIZE) {
160			union aegis_block *dst_blk =
161					(union aegis_block *)dst;
162			const union aegis_block *src_blk =
163					(const union aegis_block *)src;
164
165			tmp = state->blocks[2];
166			crypto_aegis_block_and(&tmp, &state->blocks[3]);
167			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
168			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
169			crypto_aegis_block_xor(&tmp, src_blk);
170
171			crypto_aegis128_update_a(state, src_blk, false);
172
173			*dst_blk = tmp;
174
175			size -= AEGIS_BLOCK_SIZE;
176			src += AEGIS_BLOCK_SIZE;
177			dst += AEGIS_BLOCK_SIZE;
178		}
179	} else {
180		while (size >= AEGIS_BLOCK_SIZE) {
181			tmp = state->blocks[2];
182			crypto_aegis_block_and(&tmp, &state->blocks[3]);
183			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
184			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
185			crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);
186
187			crypto_aegis128_update_u(state, src, false);
188
189			memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);
190
191			size -= AEGIS_BLOCK_SIZE;
192			src += AEGIS_BLOCK_SIZE;
193			dst += AEGIS_BLOCK_SIZE;
194		}
195	}
196
197	if (size > 0) {
198		union aegis_block msg = {};
199		memcpy(msg.bytes, src, size);
200
201		tmp = state->blocks[2];
202		crypto_aegis_block_and(&tmp, &state->blocks[3]);
203		crypto_aegis_block_xor(&tmp, &state->blocks[4]);
204		crypto_aegis_block_xor(&tmp, &state->blocks[1]);
205
206		crypto_aegis128_update_a(state, &msg, false);
207
208		crypto_aegis_block_xor(&msg, &tmp);
209
210		memcpy(dst, msg.bytes, size);
211	}
212}
213
214static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst,
215					  const u8 *src, unsigned int size)
216{
217	union aegis_block tmp;
218
219	if (AEGIS_ALIGNED(src) && AEGIS_ALIGNED(dst)) {
220		while (size >= AEGIS_BLOCK_SIZE) {
221			union aegis_block *dst_blk =
222					(union aegis_block *)dst;
223			const union aegis_block *src_blk =
224					(const union aegis_block *)src;
225
226			tmp = state->blocks[2];
227			crypto_aegis_block_and(&tmp, &state->blocks[3]);
228			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
229			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
230			crypto_aegis_block_xor(&tmp, src_blk);
231
232			crypto_aegis128_update_a(state, &tmp, false);
233
234			*dst_blk = tmp;
235
236			size -= AEGIS_BLOCK_SIZE;
237			src += AEGIS_BLOCK_SIZE;
238			dst += AEGIS_BLOCK_SIZE;
239		}
240	} else {
241		while (size >= AEGIS_BLOCK_SIZE) {
242			tmp = state->blocks[2];
243			crypto_aegis_block_and(&tmp, &state->blocks[3]);
244			crypto_aegis_block_xor(&tmp, &state->blocks[4]);
245			crypto_aegis_block_xor(&tmp, &state->blocks[1]);
246			crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE);
247
248			crypto_aegis128_update_a(state, &tmp, false);
249
250			memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE);
251
252			size -= AEGIS_BLOCK_SIZE;
253			src += AEGIS_BLOCK_SIZE;
254			dst += AEGIS_BLOCK_SIZE;
255		}
256	}
257
258	if (size > 0) {
259		union aegis_block msg = {};
260		memcpy(msg.bytes, src, size);
261
262		tmp = state->blocks[2];
263		crypto_aegis_block_and(&tmp, &state->blocks[3]);
264		crypto_aegis_block_xor(&tmp, &state->blocks[4]);
265		crypto_aegis_block_xor(&tmp, &state->blocks[1]);
266		crypto_aegis_block_xor(&msg, &tmp);
267
268		memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size);
269
270		crypto_aegis128_update_a(state, &msg, false);
271
272		memcpy(dst, msg.bytes, size);
273	}
274}
275
276static void crypto_aegis128_process_ad(struct aegis_state *state,
277				       struct scatterlist *sg_src,
278				       unsigned int assoclen,
279				       bool do_simd)
280{
281	struct scatter_walk walk;
282	union aegis_block buf;
283	unsigned int pos = 0;
284
285	scatterwalk_start(&walk, sg_src);
286	while (assoclen != 0) {
287		unsigned int size = scatterwalk_next(&walk, assoclen);
288		const u8 *src = walk.addr;
289		unsigned int left = size;
290
291		if (pos + size >= AEGIS_BLOCK_SIZE) {
292			if (pos > 0) {
293				unsigned int fill = AEGIS_BLOCK_SIZE - pos;
294				memcpy(buf.bytes + pos, src, fill);
295				crypto_aegis128_update_a(state, &buf, do_simd);
296				pos = 0;
297				left -= fill;
298				src += fill;
299			}
300
301			crypto_aegis128_ad(state, src, left, do_simd);
302			src += left & ~(AEGIS_BLOCK_SIZE - 1);
303			left &= AEGIS_BLOCK_SIZE - 1;
304		}
305
306		memcpy(buf.bytes + pos, src, left);
307
308		pos += left;
309		assoclen -= size;
310		scatterwalk_done_src(&walk, size);
311	}
312
313	if (pos > 0) {
314		memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos);
315		crypto_aegis128_update_a(state, &buf, do_simd);
316	}
317}
318
319static __always_inline
320int crypto_aegis128_process_crypt(struct aegis_state *state,
321				  struct skcipher_walk *walk,
322				  void (*crypt)(struct aegis_state *state,
323						u8 *dst,
324						const u8 *src,
325						unsigned int size))
326{
327	int err = 0;
328
329	while (walk->nbytes) {
330		unsigned int nbytes = walk->nbytes;
331
332		if (nbytes < walk->total)
333			nbytes = round_down(nbytes, walk->stride);
334
335		crypt(state, walk->dst.virt.addr, walk->src.virt.addr, nbytes);
336
337		err = skcipher_walk_done(walk, walk->nbytes - nbytes);
338	}
339	return err;
340}
341
342static void crypto_aegis128_final(struct aegis_state *state,
343				  union aegis_block *tag_xor,
344				  u64 assoclen, u64 cryptlen)
345{
346	u64 assocbits = assoclen * 8;
347	u64 cryptbits = cryptlen * 8;
348
349	union aegis_block tmp;
350	unsigned int i;
351
352	tmp.words64[0] = cpu_to_le64(assocbits);
353	tmp.words64[1] = cpu_to_le64(cryptbits);
354
355	crypto_aegis_block_xor(&tmp, &state->blocks[3]);
356
357	for (i = 0; i < 7; i++)
358		crypto_aegis128_update_a(state, &tmp, false);
359
360	for (i = 0; i < AEGIS128_STATE_BLOCKS; i++)
361		crypto_aegis_block_xor(tag_xor, &state->blocks[i]);
362}
363
364static int crypto_aegis128_setkey(struct crypto_aead *aead, const u8 *key,
365				  unsigned int keylen)
366{
367	struct aegis_ctx *ctx = crypto_aead_ctx(aead);
368
369	if (keylen != AEGIS128_KEY_SIZE)
370		return -EINVAL;
371
372	memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);
373	return 0;
374}
375
376static int crypto_aegis128_setauthsize(struct crypto_aead *tfm,
377				       unsigned int authsize)
378{
379	if (authsize > AEGIS128_MAX_AUTH_SIZE)
380		return -EINVAL;
381	if (authsize < AEGIS128_MIN_AUTH_SIZE)
382		return -EINVAL;
383	return 0;
384}
385
386static int crypto_aegis128_encrypt_generic(struct aead_request *req)
387{
388	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
389	union aegis_block tag = {};
390	unsigned int authsize = crypto_aead_authsize(tfm);
391	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
392	unsigned int cryptlen = req->cryptlen;
393	struct skcipher_walk walk;
394	struct aegis_state state;
395
396	skcipher_walk_aead_encrypt(&walk, req, false);
397	crypto_aegis128_init(&state, &ctx->key, req->iv);
398	crypto_aegis128_process_ad(&state, req->src, req->assoclen, false);
399	crypto_aegis128_process_crypt(&state, &walk,
400				      crypto_aegis128_encrypt_chunk);
401	crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen);
402
403	scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
404				 authsize, 1);
405	return 0;
406}
407
408static int crypto_aegis128_decrypt_generic(struct aead_request *req)
409{
410	static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {};
411	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
412	union aegis_block tag;
413	unsigned int authsize = crypto_aead_authsize(tfm);
414	unsigned int cryptlen = req->cryptlen - authsize;
415	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
416	struct skcipher_walk walk;
417	struct aegis_state state;
418
419	scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
420				 authsize, 0);
421
422	skcipher_walk_aead_decrypt(&walk, req, false);
423	crypto_aegis128_init(&state, &ctx->key, req->iv);
424	crypto_aegis128_process_ad(&state, req->src, req->assoclen, false);
425	crypto_aegis128_process_crypt(&state, &walk,
426				      crypto_aegis128_decrypt_chunk);
427	crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen);
428
429	if (unlikely(crypto_memneq(tag.bytes, zeros, authsize))) {
430		/*
431		 * From Chapter 4. 'Security Analysis' of the AEGIS spec [0]
432		 *
433		 * "3. If verification fails, the decrypted plaintext and the
434		 *     wrong authentication tag should not be given as output."
435		 *
436		 * [0] https://competitions.cr.yp.to/round3/aegisv11.pdf
437		 */
438		skcipher_walk_aead_decrypt(&walk, req, false);
439		crypto_aegis128_process_crypt(NULL, &walk,
440					      crypto_aegis128_wipe_chunk);
441		memzero_explicit(&tag, sizeof(tag));
442		return -EBADMSG;
443	}
444	return 0;
445}
446
447static int crypto_aegis128_encrypt_simd(struct aead_request *req)
448{
449	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
450	union aegis_block tag = {};
451	unsigned int authsize = crypto_aead_authsize(tfm);
452	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
453	unsigned int cryptlen = req->cryptlen;
454	struct skcipher_walk walk;
455	struct aegis_state state;
456
457	if (!aegis128_do_simd())
458		return crypto_aegis128_encrypt_generic(req);
459
460	skcipher_walk_aead_encrypt(&walk, req, false);
461	crypto_aegis128_init_simd(&state, &ctx->key, req->iv);
462	crypto_aegis128_process_ad(&state, req->src, req->assoclen, true);
463	crypto_aegis128_process_crypt(&state, &walk,
464				      crypto_aegis128_encrypt_chunk_simd);
465	crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen, 0);
466
467	scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
468				 authsize, 1);
469	return 0;
470}
471
472static int crypto_aegis128_decrypt_simd(struct aead_request *req)
473{
474	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
475	union aegis_block tag;
476	unsigned int authsize = crypto_aead_authsize(tfm);
477	unsigned int cryptlen = req->cryptlen - authsize;
478	struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
479	struct skcipher_walk walk;
480	struct aegis_state state;
481
482	if (!aegis128_do_simd())
483		return crypto_aegis128_decrypt_generic(req);
484
485	scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
486				 authsize, 0);
487
488	skcipher_walk_aead_decrypt(&walk, req, false);
489	crypto_aegis128_init_simd(&state, &ctx->key, req->iv);
490	crypto_aegis128_process_ad(&state, req->src, req->assoclen, true);
491	crypto_aegis128_process_crypt(&state, &walk,
492				      crypto_aegis128_decrypt_chunk_simd);
493
494	if (unlikely(crypto_aegis128_final_simd(&state, &tag, req->assoclen,
495						cryptlen, authsize))) {
496		skcipher_walk_aead_decrypt(&walk, req, false);
497		crypto_aegis128_process_crypt(NULL, &walk,
498					      crypto_aegis128_wipe_chunk);
499		return -EBADMSG;
500	}
501	return 0;
502}
503
504static struct aead_alg crypto_aegis128_alg_generic = {
505	.setkey			= crypto_aegis128_setkey,
506	.setauthsize		= crypto_aegis128_setauthsize,
507	.encrypt		= crypto_aegis128_encrypt_generic,
508	.decrypt		= crypto_aegis128_decrypt_generic,
509
510	.ivsize			= AEGIS128_NONCE_SIZE,
511	.maxauthsize		= AEGIS128_MAX_AUTH_SIZE,
512	.chunksize		= AEGIS_BLOCK_SIZE,
513
514	.base.cra_blocksize	= 1,
515	.base.cra_ctxsize	= sizeof(struct aegis_ctx),
516	.base.cra_priority	= 100,
517	.base.cra_name		= "aegis128",
518	.base.cra_driver_name	= "aegis128-generic",
519	.base.cra_module	= THIS_MODULE,
520};
521
522static struct aead_alg crypto_aegis128_alg_simd = {
523	.setkey			= crypto_aegis128_setkey,
524	.setauthsize		= crypto_aegis128_setauthsize,
525	.encrypt		= crypto_aegis128_encrypt_simd,
526	.decrypt		= crypto_aegis128_decrypt_simd,
527
528	.ivsize			= AEGIS128_NONCE_SIZE,
529	.maxauthsize		= AEGIS128_MAX_AUTH_SIZE,
530	.chunksize		= AEGIS_BLOCK_SIZE,
531
532	.base.cra_blocksize	= 1,
533	.base.cra_ctxsize	= sizeof(struct aegis_ctx),
534	.base.cra_priority	= 200,
535	.base.cra_name		= "aegis128",
536	.base.cra_driver_name	= "aegis128-simd",
537	.base.cra_module	= THIS_MODULE,
538};
539
540static int __init crypto_aegis128_module_init(void)
541{
542	int ret;
543
544	ret = crypto_register_aead(&crypto_aegis128_alg_generic);
545	if (ret)
546		return ret;
547
548	if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) &&
549	    crypto_aegis128_have_simd()) {
550		ret = crypto_register_aead(&crypto_aegis128_alg_simd);
551		if (ret) {
552			crypto_unregister_aead(&crypto_aegis128_alg_generic);
553			return ret;
554		}
555		static_branch_enable(&have_simd);
556	}
557	return 0;
558}
559
560static void __exit crypto_aegis128_module_exit(void)
561{
562	if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) &&
563	    crypto_aegis128_have_simd())
564		crypto_unregister_aead(&crypto_aegis128_alg_simd);
565
566	crypto_unregister_aead(&crypto_aegis128_alg_generic);
567}
568
569module_init(crypto_aegis128_module_init);
570module_exit(crypto_aegis128_module_exit);
571
572MODULE_LICENSE("GPL");
573MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
574MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm");
575MODULE_ALIAS_CRYPTO("aegis128");
576MODULE_ALIAS_CRYPTO("aegis128-generic");
577MODULE_ALIAS_CRYPTO("aegis128-simd");