aes-glue.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
  4 *
  5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
  6 */
  7
  8#include <crypto/aes.h>
  9#include <crypto/ctr.h>
 10#include <crypto/internal/hash.h>
 11#include <crypto/internal/skcipher.h>
 12#include <crypto/scatterwalk.h>
 13#include <crypto/sha2.h>
 14#include <crypto/utils.h>
 15#include <crypto/xts.h>
 16#include <linux/cpufeature.h>
 17#include <linux/kernel.h>
 18#include <linux/module.h>
 19#include <linux/string.h>
 20
 21#include <asm/hwcap.h>
 22#include <asm/simd.h>
 23
 24#include "aes-ce-setkey.h"
 25
 26#ifdef USE_V8_CRYPTO_EXTENSIONS
 27#define MODE			"ce"
 28#define PRIO			300
 29#define aes_expandkey		ce_aes_expandkey
 30#define aes_ecb_encrypt		ce_aes_ecb_encrypt
 31#define aes_ecb_decrypt		ce_aes_ecb_decrypt
 32#define aes_cbc_encrypt		ce_aes_cbc_encrypt
 33#define aes_cbc_decrypt		ce_aes_cbc_decrypt
 34#define aes_cbc_cts_encrypt	ce_aes_cbc_cts_encrypt
 35#define aes_cbc_cts_decrypt	ce_aes_cbc_cts_decrypt
 36#define aes_essiv_cbc_encrypt	ce_aes_essiv_cbc_encrypt
 37#define aes_essiv_cbc_decrypt	ce_aes_essiv_cbc_decrypt
 38#define aes_ctr_encrypt		ce_aes_ctr_encrypt
 39#define aes_xctr_encrypt	ce_aes_xctr_encrypt
 40#define aes_xts_encrypt		ce_aes_xts_encrypt
 41#define aes_xts_decrypt		ce_aes_xts_decrypt
 42#define aes_mac_update		ce_aes_mac_update
 43MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions");
 44#else
 45#define MODE			"neon"
 46#define PRIO			200
 47#define aes_ecb_encrypt		neon_aes_ecb_encrypt
 48#define aes_ecb_decrypt		neon_aes_ecb_decrypt
 49#define aes_cbc_encrypt		neon_aes_cbc_encrypt
 50#define aes_cbc_decrypt		neon_aes_cbc_decrypt
 51#define aes_cbc_cts_encrypt	neon_aes_cbc_cts_encrypt
 52#define aes_cbc_cts_decrypt	neon_aes_cbc_cts_decrypt
 53#define aes_essiv_cbc_encrypt	neon_aes_essiv_cbc_encrypt
 54#define aes_essiv_cbc_decrypt	neon_aes_essiv_cbc_decrypt
 55#define aes_ctr_encrypt		neon_aes_ctr_encrypt
 56#define aes_xctr_encrypt	neon_aes_xctr_encrypt
 57#define aes_xts_encrypt		neon_aes_xts_encrypt
 58#define aes_xts_decrypt		neon_aes_xts_decrypt
 59#define aes_mac_update		neon_aes_mac_update
 60MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON");
 61#endif
 62#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
 63MODULE_ALIAS_CRYPTO("ecb(aes)");
 64MODULE_ALIAS_CRYPTO("cbc(aes)");
 65MODULE_ALIAS_CRYPTO("ctr(aes)");
 66MODULE_ALIAS_CRYPTO("xts(aes)");
 67MODULE_ALIAS_CRYPTO("xctr(aes)");
 68#endif
 69MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
 70MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
 71MODULE_ALIAS_CRYPTO("cmac(aes)");
 72MODULE_ALIAS_CRYPTO("xcbc(aes)");
 73MODULE_ALIAS_CRYPTO("cbcmac(aes)");
 74
 75MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 76MODULE_LICENSE("GPL v2");
 77
 78/* defined in aes-modes.S */
 79asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
 80				int rounds, int blocks);
 81asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
 82				int rounds, int blocks);
 83
 84asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
 85				int rounds, int blocks, u8 iv[]);
 86asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
 87				int rounds, int blocks, u8 iv[]);
 88
 89asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
 90				int rounds, int bytes, u8 const iv[]);
 91asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
 92				int rounds, int bytes, u8 const iv[]);
 93
 94asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
 95				int rounds, int bytes, u8 ctr[]);
 96
 97asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
 98				 int rounds, int bytes, u8 ctr[], int byte_ctr);
 99
100asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
101				int rounds, int bytes, u32 const rk2[], u8 iv[],
102				int first);
103asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
104				int rounds, int bytes, u32 const rk2[], u8 iv[],
105				int first);
106
107asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
108				      int rounds, int blocks, u8 iv[],
109				      u32 const rk2[]);
110asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
111				      int rounds, int blocks, u8 iv[],
112				      u32 const rk2[]);
113
114asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds,
115			      int blocks, u8 dg[], int enc_before,
116			      int enc_after);
117
118struct crypto_aes_xts_ctx {
119	struct crypto_aes_ctx key1;
120	struct crypto_aes_ctx __aligned(8) key2;
121};
122
123struct crypto_aes_essiv_cbc_ctx {
124	struct crypto_aes_ctx key1;
125	struct crypto_aes_ctx __aligned(8) key2;
126};
127
128struct mac_tfm_ctx {
129	struct crypto_aes_ctx key;
130	u8 __aligned(8) consts[];
131};
132
133struct mac_desc_ctx {
134	u8 dg[AES_BLOCK_SIZE];
135};
136
137static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
138			       unsigned int key_len)
139{
140	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
141
142	return aes_expandkey(ctx, in_key, key_len);
143}
144
145static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm,
146				      const u8 *in_key, unsigned int key_len)
147{
148	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
149	int ret;
150
151	ret = xts_verify_key(tfm, in_key, key_len);
152	if (ret)
153		return ret;
154
155	ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
156	if (!ret)
157		ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
158				    key_len / 2);
159	return ret;
160}
161
162static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm,
163					    const u8 *in_key,
164					    unsigned int key_len)
165{
166	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
167	u8 digest[SHA256_DIGEST_SIZE];
168	int ret;
169
170	ret = aes_expandkey(&ctx->key1, in_key, key_len);
171	if (ret)
172		return ret;
173
174	sha256(in_key, key_len, digest);
175
176	return aes_expandkey(&ctx->key2, digest, sizeof(digest));
177}
178
179static int __maybe_unused ecb_encrypt(struct skcipher_request *req)
180{
181	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
182	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
183	int err, rounds = 6 + ctx->key_length / 4;
184	struct skcipher_walk walk;
185	unsigned int blocks;
186
187	err = skcipher_walk_virt(&walk, req, false);
188
189	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
190		scoped_ksimd()
191			aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
192					ctx->key_enc, rounds, blocks);
193		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
194	}
195	return err;
196}
197
198static int __maybe_unused ecb_decrypt(struct skcipher_request *req)
199{
200	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
201	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
202	int err, rounds = 6 + ctx->key_length / 4;
203	struct skcipher_walk walk;
204	unsigned int blocks;
205
206	err = skcipher_walk_virt(&walk, req, false);
207
208	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
209		scoped_ksimd()
210			aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
211					ctx->key_dec, rounds, blocks);
212		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
213	}
214	return err;
215}
216
217static int cbc_encrypt_walk(struct skcipher_request *req,
218			    struct skcipher_walk *walk)
219{
220	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
221	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
222	int err = 0, rounds = 6 + ctx->key_length / 4;
223	unsigned int blocks;
224
225	while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
226		scoped_ksimd()
227			aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
228					ctx->key_enc, rounds, blocks, walk->iv);
229		err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
230	}
231	return err;
232}
233
234static int __maybe_unused cbc_encrypt(struct skcipher_request *req)
235{
236	struct skcipher_walk walk;
237	int err;
238
239	err = skcipher_walk_virt(&walk, req, false);
240	if (err)
241		return err;
242	return cbc_encrypt_walk(req, &walk);
243}
244
245static int cbc_decrypt_walk(struct skcipher_request *req,
246			    struct skcipher_walk *walk)
247{
248	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
249	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
250	int err = 0, rounds = 6 + ctx->key_length / 4;
251	unsigned int blocks;
252
253	while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
254		scoped_ksimd()
255			aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
256					ctx->key_dec, rounds, blocks, walk->iv);
257		err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
258	}
259	return err;
260}
261
262static int __maybe_unused cbc_decrypt(struct skcipher_request *req)
263{
264	struct skcipher_walk walk;
265	int err;
266
267	err = skcipher_walk_virt(&walk, req, false);
268	if (err)
269		return err;
270	return cbc_decrypt_walk(req, &walk);
271}
272
273static int cts_cbc_encrypt(struct skcipher_request *req)
274{
275	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
276	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
277	int err, rounds = 6 + ctx->key_length / 4;
278	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
279	struct scatterlist *src = req->src, *dst = req->dst;
280	struct scatterlist sg_src[2], sg_dst[2];
281	struct skcipher_request subreq;
282	struct skcipher_walk walk;
283
284	skcipher_request_set_tfm(&subreq, tfm);
285	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
286				      NULL, NULL);
287
288	if (req->cryptlen <= AES_BLOCK_SIZE) {
289		if (req->cryptlen < AES_BLOCK_SIZE)
290			return -EINVAL;
291		cbc_blocks = 1;
292	}
293
294	if (cbc_blocks > 0) {
295		skcipher_request_set_crypt(&subreq, req->src, req->dst,
296					   cbc_blocks * AES_BLOCK_SIZE,
297					   req->iv);
298
299		err = skcipher_walk_virt(&walk, &subreq, false) ?:
300		      cbc_encrypt_walk(&subreq, &walk);
301		if (err)
302			return err;
303
304		if (req->cryptlen == AES_BLOCK_SIZE)
305			return 0;
306
307		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
308		if (req->dst != req->src)
309			dst = scatterwalk_ffwd(sg_dst, req->dst,
310					       subreq.cryptlen);
311	}
312
313	/* handle ciphertext stealing */
314	skcipher_request_set_crypt(&subreq, src, dst,
315				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
316				   req->iv);
317
318	err = skcipher_walk_virt(&walk, &subreq, false);
319	if (err)
320		return err;
321
322	scoped_ksimd()
323		aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
324				    ctx->key_enc, rounds, walk.nbytes, walk.iv);
325
326	return skcipher_walk_done(&walk, 0);
327}
328
329static int cts_cbc_decrypt(struct skcipher_request *req)
330{
331	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
332	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
333	int err, rounds = 6 + ctx->key_length / 4;
334	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
335	struct scatterlist *src = req->src, *dst = req->dst;
336	struct scatterlist sg_src[2], sg_dst[2];
337	struct skcipher_request subreq;
338	struct skcipher_walk walk;
339
340	skcipher_request_set_tfm(&subreq, tfm);
341	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
342				      NULL, NULL);
343
344	if (req->cryptlen <= AES_BLOCK_SIZE) {
345		if (req->cryptlen < AES_BLOCK_SIZE)
346			return -EINVAL;
347		cbc_blocks = 1;
348	}
349
350	if (cbc_blocks > 0) {
351		skcipher_request_set_crypt(&subreq, req->src, req->dst,
352					   cbc_blocks * AES_BLOCK_SIZE,
353					   req->iv);
354
355		err = skcipher_walk_virt(&walk, &subreq, false) ?:
356		      cbc_decrypt_walk(&subreq, &walk);
357		if (err)
358			return err;
359
360		if (req->cryptlen == AES_BLOCK_SIZE)
361			return 0;
362
363		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
364		if (req->dst != req->src)
365			dst = scatterwalk_ffwd(sg_dst, req->dst,
366					       subreq.cryptlen);
367	}
368
369	/* handle ciphertext stealing */
370	skcipher_request_set_crypt(&subreq, src, dst,
371				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
372				   req->iv);
373
374	err = skcipher_walk_virt(&walk, &subreq, false);
375	if (err)
376		return err;
377
378	scoped_ksimd()
379		aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
380				    ctx->key_dec, rounds, walk.nbytes, walk.iv);
381
382	return skcipher_walk_done(&walk, 0);
383}
384
385static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req)
386{
387	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
388	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
389	int err, rounds = 6 + ctx->key1.key_length / 4;
390	struct skcipher_walk walk;
391	unsigned int blocks;
392
393	err = skcipher_walk_virt(&walk, req, false);
394
395	blocks = walk.nbytes / AES_BLOCK_SIZE;
396	if (blocks) {
397		scoped_ksimd()
398			aes_essiv_cbc_encrypt(walk.dst.virt.addr,
399					      walk.src.virt.addr,
400					      ctx->key1.key_enc, rounds, blocks,
401					      req->iv, ctx->key2.key_enc);
402		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
403	}
404	return err ?: cbc_encrypt_walk(req, &walk);
405}
406
407static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req)
408{
409	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
410	struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
411	int err, rounds = 6 + ctx->key1.key_length / 4;
412	struct skcipher_walk walk;
413	unsigned int blocks;
414
415	err = skcipher_walk_virt(&walk, req, false);
416
417	blocks = walk.nbytes / AES_BLOCK_SIZE;
418	if (blocks) {
419		scoped_ksimd()
420			aes_essiv_cbc_decrypt(walk.dst.virt.addr,
421					      walk.src.virt.addr,
422					      ctx->key1.key_dec, rounds, blocks,
423					      req->iv, ctx->key2.key_enc);
424		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
425	}
426	return err ?: cbc_decrypt_walk(req, &walk);
427}
428
429static int __maybe_unused xctr_encrypt(struct skcipher_request *req)
430{
431	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
432	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
433	int err, rounds = 6 + ctx->key_length / 4;
434	struct skcipher_walk walk;
435	unsigned int byte_ctr = 0;
436
437	err = skcipher_walk_virt(&walk, req, false);
438
439	while (walk.nbytes > 0) {
440		const u8 *src = walk.src.virt.addr;
441		unsigned int nbytes = walk.nbytes;
442		u8 *dst = walk.dst.virt.addr;
443		u8 buf[AES_BLOCK_SIZE];
444
445		/*
446		 * If given less than 16 bytes, we must copy the partial block
447		 * into a temporary buffer of 16 bytes to avoid out of bounds
448		 * reads and writes.  Furthermore, this code is somewhat unusual
449		 * in that it expects the end of the data to be at the end of
450		 * the temporary buffer, rather than the start of the data at
451		 * the start of the temporary buffer.
452		 */
453		if (unlikely(nbytes < AES_BLOCK_SIZE))
454			src = dst = memcpy(buf + sizeof(buf) - nbytes,
455					   src, nbytes);
456		else if (nbytes < walk.total)
457			nbytes &= ~(AES_BLOCK_SIZE - 1);
458
459		scoped_ksimd()
460			aes_xctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
461							 walk.iv, byte_ctr);
462
463		if (unlikely(nbytes < AES_BLOCK_SIZE))
464			memcpy(walk.dst.virt.addr,
465			       buf + sizeof(buf) - nbytes, nbytes);
466		byte_ctr += nbytes;
467
468		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
469	}
470
471	return err;
472}
473
474static int __maybe_unused ctr_encrypt(struct skcipher_request *req)
475{
476	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
477	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
478	int err, rounds = 6 + ctx->key_length / 4;
479	struct skcipher_walk walk;
480
481	err = skcipher_walk_virt(&walk, req, false);
482
483	while (walk.nbytes > 0) {
484		const u8 *src = walk.src.virt.addr;
485		unsigned int nbytes = walk.nbytes;
486		u8 *dst = walk.dst.virt.addr;
487		u8 buf[AES_BLOCK_SIZE];
488
489		/*
490		 * If given less than 16 bytes, we must copy the partial block
491		 * into a temporary buffer of 16 bytes to avoid out of bounds
492		 * reads and writes.  Furthermore, this code is somewhat unusual
493		 * in that it expects the end of the data to be at the end of
494		 * the temporary buffer, rather than the start of the data at
495		 * the start of the temporary buffer.
496		 */
497		if (unlikely(nbytes < AES_BLOCK_SIZE))
498			src = dst = memcpy(buf + sizeof(buf) - nbytes,
499					   src, nbytes);
500		else if (nbytes < walk.total)
501			nbytes &= ~(AES_BLOCK_SIZE - 1);
502
503		scoped_ksimd()
504			aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
505					walk.iv);
506
507		if (unlikely(nbytes < AES_BLOCK_SIZE))
508			memcpy(walk.dst.virt.addr,
509			       buf + sizeof(buf) - nbytes, nbytes);
510
511		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
512	}
513
514	return err;
515}
516
517static int __maybe_unused xts_encrypt(struct skcipher_request *req)
518{
519	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
520	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
521	int err, first, rounds = 6 + ctx->key1.key_length / 4;
522	int tail = req->cryptlen % AES_BLOCK_SIZE;
523	struct scatterlist sg_src[2], sg_dst[2];
524	struct skcipher_request subreq;
525	struct scatterlist *src, *dst;
526	struct skcipher_walk walk;
527
528	if (req->cryptlen < AES_BLOCK_SIZE)
529		return -EINVAL;
530
531	err = skcipher_walk_virt(&walk, req, false);
532
533	if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
534		int xts_blocks = DIV_ROUND_UP(req->cryptlen,
535					      AES_BLOCK_SIZE) - 2;
536
537		skcipher_walk_abort(&walk);
538
539		skcipher_request_set_tfm(&subreq, tfm);
540		skcipher_request_set_callback(&subreq,
541					      skcipher_request_flags(req),
542					      NULL, NULL);
543		skcipher_request_set_crypt(&subreq, req->src, req->dst,
544					   xts_blocks * AES_BLOCK_SIZE,
545					   req->iv);
546		req = &subreq;
547		err = skcipher_walk_virt(&walk, req, false);
548	} else {
549		tail = 0;
550	}
551
552	scoped_ksimd() {
553		for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
554			int nbytes = walk.nbytes;
555
556			if (walk.nbytes < walk.total)
557				nbytes &= ~(AES_BLOCK_SIZE - 1);
558
559			aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
560					ctx->key1.key_enc, rounds, nbytes,
561					ctx->key2.key_enc, walk.iv, first);
562			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
563		}
564
565		if (err || likely(!tail))
566			return err;
567
568		dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
569		if (req->dst != req->src)
570			dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
571
572		skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
573					   req->iv);
574
575		err = skcipher_walk_virt(&walk, &subreq, false);
576		if (err)
577			return err;
578
579		aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
580				ctx->key1.key_enc, rounds, walk.nbytes,
581				ctx->key2.key_enc, walk.iv, first);
582	}
583	return skcipher_walk_done(&walk, 0);
584}
585
586static int __maybe_unused xts_decrypt(struct skcipher_request *req)
587{
588	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
589	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
590	int err, first, rounds = 6 + ctx->key1.key_length / 4;
591	int tail = req->cryptlen % AES_BLOCK_SIZE;
592	struct scatterlist sg_src[2], sg_dst[2];
593	struct skcipher_request subreq;
594	struct scatterlist *src, *dst;
595	struct skcipher_walk walk;
596
597	if (req->cryptlen < AES_BLOCK_SIZE)
598		return -EINVAL;
599
600	err = skcipher_walk_virt(&walk, req, false);
601
602	if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
603		int xts_blocks = DIV_ROUND_UP(req->cryptlen,
604					      AES_BLOCK_SIZE) - 2;
605
606		skcipher_walk_abort(&walk);
607
608		skcipher_request_set_tfm(&subreq, tfm);
609		skcipher_request_set_callback(&subreq,
610					      skcipher_request_flags(req),
611					      NULL, NULL);
612		skcipher_request_set_crypt(&subreq, req->src, req->dst,
613					   xts_blocks * AES_BLOCK_SIZE,
614					   req->iv);
615		req = &subreq;
616		err = skcipher_walk_virt(&walk, req, false);
617	} else {
618		tail = 0;
619	}
620
621	scoped_ksimd() {
622		for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
623			int nbytes = walk.nbytes;
624
625			if (walk.nbytes < walk.total)
626				nbytes &= ~(AES_BLOCK_SIZE - 1);
627
628			aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
629					ctx->key1.key_dec, rounds, nbytes,
630					ctx->key2.key_enc, walk.iv, first);
631			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
632		}
633
634		if (err || likely(!tail))
635			return err;
636
637		dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
638		if (req->dst != req->src)
639			dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
640
641		skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
642					   req->iv);
643
644		err = skcipher_walk_virt(&walk, &subreq, false);
645		if (err)
646			return err;
647
648		aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
649				ctx->key1.key_dec, rounds, walk.nbytes,
650				ctx->key2.key_enc, walk.iv, first);
651	}
652	return skcipher_walk_done(&walk, 0);
653}
654
655static struct skcipher_alg aes_algs[] = { {
656#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
657	.base = {
658		.cra_name		= "ecb(aes)",
659		.cra_driver_name	= "ecb-aes-" MODE,
660		.cra_priority		= PRIO,
661		.cra_blocksize		= AES_BLOCK_SIZE,
662		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
663		.cra_module		= THIS_MODULE,
664	},
665	.min_keysize	= AES_MIN_KEY_SIZE,
666	.max_keysize	= AES_MAX_KEY_SIZE,
667	.setkey		= skcipher_aes_setkey,
668	.encrypt	= ecb_encrypt,
669	.decrypt	= ecb_decrypt,
670}, {
671	.base = {
672		.cra_name		= "cbc(aes)",
673		.cra_driver_name	= "cbc-aes-" MODE,
674		.cra_priority		= PRIO,
675		.cra_blocksize		= AES_BLOCK_SIZE,
676		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
677		.cra_module		= THIS_MODULE,
678	},
679	.min_keysize	= AES_MIN_KEY_SIZE,
680	.max_keysize	= AES_MAX_KEY_SIZE,
681	.ivsize		= AES_BLOCK_SIZE,
682	.setkey		= skcipher_aes_setkey,
683	.encrypt	= cbc_encrypt,
684	.decrypt	= cbc_decrypt,
685}, {
686	.base = {
687		.cra_name		= "ctr(aes)",
688		.cra_driver_name	= "ctr-aes-" MODE,
689		.cra_priority		= PRIO,
690		.cra_blocksize		= 1,
691		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
692		.cra_module		= THIS_MODULE,
693	},
694	.min_keysize	= AES_MIN_KEY_SIZE,
695	.max_keysize	= AES_MAX_KEY_SIZE,
696	.ivsize		= AES_BLOCK_SIZE,
697	.chunksize	= AES_BLOCK_SIZE,
698	.setkey		= skcipher_aes_setkey,
699	.encrypt	= ctr_encrypt,
700	.decrypt	= ctr_encrypt,
701}, {
702	.base = {
703		.cra_name		= "xctr(aes)",
704		.cra_driver_name	= "xctr-aes-" MODE,
705		.cra_priority		= PRIO,
706		.cra_blocksize		= 1,
707		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
708		.cra_module		= THIS_MODULE,
709	},
710	.min_keysize	= AES_MIN_KEY_SIZE,
711	.max_keysize	= AES_MAX_KEY_SIZE,
712	.ivsize		= AES_BLOCK_SIZE,
713	.chunksize	= AES_BLOCK_SIZE,
714	.setkey		= skcipher_aes_setkey,
715	.encrypt	= xctr_encrypt,
716	.decrypt	= xctr_encrypt,
717}, {
718	.base = {
719		.cra_name		= "xts(aes)",
720		.cra_driver_name	= "xts-aes-" MODE,
721		.cra_priority		= PRIO,
722		.cra_blocksize		= AES_BLOCK_SIZE,
723		.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
724		.cra_module		= THIS_MODULE,
725	},
726	.min_keysize	= 2 * AES_MIN_KEY_SIZE,
727	.max_keysize	= 2 * AES_MAX_KEY_SIZE,
728	.ivsize		= AES_BLOCK_SIZE,
729	.walksize	= 2 * AES_BLOCK_SIZE,
730	.setkey		= xts_set_key,
731	.encrypt	= xts_encrypt,
732	.decrypt	= xts_decrypt,
733}, {
734#endif
735	.base = {
736		.cra_name		= "cts(cbc(aes))",
737		.cra_driver_name	= "cts-cbc-aes-" MODE,
738		.cra_priority		= PRIO,
739		.cra_blocksize		= AES_BLOCK_SIZE,
740		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
741		.cra_module		= THIS_MODULE,
742	},
743	.min_keysize	= AES_MIN_KEY_SIZE,
744	.max_keysize	= AES_MAX_KEY_SIZE,
745	.ivsize		= AES_BLOCK_SIZE,
746	.walksize	= 2 * AES_BLOCK_SIZE,
747	.setkey		= skcipher_aes_setkey,
748	.encrypt	= cts_cbc_encrypt,
749	.decrypt	= cts_cbc_decrypt,
750}, {
751	.base = {
752		.cra_name		= "essiv(cbc(aes),sha256)",
753		.cra_driver_name	= "essiv-cbc-aes-sha256-" MODE,
754		.cra_priority		= PRIO + 1,
755		.cra_blocksize		= AES_BLOCK_SIZE,
756		.cra_ctxsize		= sizeof(struct crypto_aes_essiv_cbc_ctx),
757		.cra_module		= THIS_MODULE,
758	},
759	.min_keysize	= AES_MIN_KEY_SIZE,
760	.max_keysize	= AES_MAX_KEY_SIZE,
761	.ivsize		= AES_BLOCK_SIZE,
762	.setkey		= essiv_cbc_set_key,
763	.encrypt	= essiv_cbc_encrypt,
764	.decrypt	= essiv_cbc_decrypt,
765} };
766
767static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
768			 unsigned int key_len)
769{
770	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
771
772	return aes_expandkey(&ctx->key, in_key, key_len);
773}
774
775static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
776{
777	u64 a = be64_to_cpu(x->a);
778	u64 b = be64_to_cpu(x->b);
779
780	y->a = cpu_to_be64((a << 1) | (b >> 63));
781	y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
782}
783
784static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
785		       unsigned int key_len)
786{
787	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
788	be128 *consts = (be128 *)ctx->consts;
789	int rounds = 6 + key_len / 4;
790	int err;
791
792	err = cbcmac_setkey(tfm, in_key, key_len);
793	if (err)
794		return err;
795
796	/* encrypt the zero vector */
797	scoped_ksimd()
798		aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){},
799				ctx->key.key_enc, rounds, 1);
800
801	cmac_gf128_mul_by_x(consts, consts);
802	cmac_gf128_mul_by_x(consts + 1, consts);
803
804	return 0;
805}
806
807static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
808		       unsigned int key_len)
809{
810	static u8 const ks[3][AES_BLOCK_SIZE] = {
811		{ [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
812		{ [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
813		{ [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
814	};
815
816	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
817	int rounds = 6 + key_len / 4;
818	u8 key[AES_BLOCK_SIZE];
819	int err;
820
821	err = cbcmac_setkey(tfm, in_key, key_len);
822	if (err)
823		return err;
824
825	scoped_ksimd() {
826		aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
827		aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
828	}
829
830	return cbcmac_setkey(tfm, key, sizeof(key));
831}
832
833static int mac_init(struct shash_desc *desc)
834{
835	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
836
837	memset(ctx->dg, 0, AES_BLOCK_SIZE);
838	return 0;
839}
840
841static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
842			  u8 dg[], int enc_before)
843{
844	int rounds = 6 + ctx->key_length / 4;
845	int rem;
846
847	do {
848		scoped_ksimd()
849			rem = aes_mac_update(in, ctx->key_enc, rounds, blocks,
850					     dg, enc_before, !enc_before);
851		in += (blocks - rem) * AES_BLOCK_SIZE;
852		blocks = rem;
853	} while (blocks);
854}
855
856static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
857{
858	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
859	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
860	int blocks = len / AES_BLOCK_SIZE;
861
862	len %= AES_BLOCK_SIZE;
863	mac_do_update(&tctx->key, p, blocks, ctx->dg, 0);
864	return len;
865}
866
867static int cbcmac_finup(struct shash_desc *desc, const u8 *src,
868			unsigned int len, u8 *out)
869{
870	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
871	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
872
873	if (len) {
874		crypto_xor(ctx->dg, src, len);
875		mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1);
876	}
877	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
878	return 0;
879}
880
881static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len,
882		      u8 *out)
883{
884	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
885	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
886	u8 *consts = tctx->consts;
887
888	crypto_xor(ctx->dg, src, len);
889	if (len != AES_BLOCK_SIZE) {
890		ctx->dg[len] ^= 0x80;
891		consts += AES_BLOCK_SIZE;
892	}
893	mac_do_update(&tctx->key, consts, 1, ctx->dg, 0);
894	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
895	return 0;
896}
897
898static struct shash_alg mac_algs[] = { {
899	.base.cra_name		= "cmac(aes)",
900	.base.cra_driver_name	= "cmac-aes-" MODE,
901	.base.cra_priority	= PRIO,
902	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
903				  CRYPTO_AHASH_ALG_FINAL_NONZERO,
904	.base.cra_blocksize	= AES_BLOCK_SIZE,
905	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx) +
906				  2 * AES_BLOCK_SIZE,
907	.base.cra_module	= THIS_MODULE,
908
909	.digestsize		= AES_BLOCK_SIZE,
910	.init			= mac_init,
911	.update			= mac_update,
912	.finup			= cmac_finup,
913	.setkey			= cmac_setkey,
914	.descsize		= sizeof(struct mac_desc_ctx),
915}, {
916	.base.cra_name		= "xcbc(aes)",
917	.base.cra_driver_name	= "xcbc-aes-" MODE,
918	.base.cra_priority	= PRIO,
919	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
920				  CRYPTO_AHASH_ALG_FINAL_NONZERO,
921	.base.cra_blocksize	= AES_BLOCK_SIZE,
922	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx) +
923				  2 * AES_BLOCK_SIZE,
924	.base.cra_module	= THIS_MODULE,
925
926	.digestsize		= AES_BLOCK_SIZE,
927	.init			= mac_init,
928	.update			= mac_update,
929	.finup			= cmac_finup,
930	.setkey			= xcbc_setkey,
931	.descsize		= sizeof(struct mac_desc_ctx),
932}, {
933	.base.cra_name		= "cbcmac(aes)",
934	.base.cra_driver_name	= "cbcmac-aes-" MODE,
935	.base.cra_priority	= PRIO,
936	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
937	.base.cra_blocksize	= AES_BLOCK_SIZE,
938	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx),
939	.base.cra_module	= THIS_MODULE,
940
941	.digestsize		= AES_BLOCK_SIZE,
942	.init			= mac_init,
943	.update			= mac_update,
944	.finup			= cbcmac_finup,
945	.setkey			= cbcmac_setkey,
946	.descsize		= sizeof(struct mac_desc_ctx),
947} };
948
949static void aes_exit(void)
950{
951	crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
952	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
953}
954
955static int __init aes_init(void)
956{
957	int err;
958
959	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
960	if (err)
961		return err;
962
963	err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
964	if (err)
965		goto unregister_ciphers;
966
967	return 0;
968
969unregister_ciphers:
970	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
971	return err;
972}
973
974#ifdef USE_V8_CRYPTO_EXTENSIONS
975module_cpu_feature_match(AES, aes_init);
976EXPORT_SYMBOL_NS(ce_aes_mac_update, "CRYPTO_INTERNAL");
977#else
978module_init(aes_init);
979EXPORT_SYMBOL(neon_aes_ecb_encrypt);
980EXPORT_SYMBOL(neon_aes_cbc_encrypt);
981EXPORT_SYMBOL(neon_aes_ctr_encrypt);
982EXPORT_SYMBOL(neon_aes_xts_encrypt);
983EXPORT_SYMBOL(neon_aes_xts_decrypt);
984#endif
985module_exit(aes_exit);