ghash-ce-glue.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
  4 *
  5 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
  6 */
  7
  8#include <crypto/aes.h>
  9#include <crypto/b128ops.h>
 10#include <crypto/gcm.h>
 11#include <crypto/ghash.h>
 12#include <crypto/gf128mul.h>
 13#include <crypto/internal/aead.h>
 14#include <crypto/internal/hash.h>
 15#include <crypto/internal/skcipher.h>
 16#include <crypto/scatterwalk.h>
 17#include <linux/cpufeature.h>
 18#include <linux/errno.h>
 19#include <linux/kernel.h>
 20#include <linux/module.h>
 21#include <linux/string.h>
 22#include <linux/unaligned.h>
 23
 24#include <asm/simd.h>
 25
 26MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
 27MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 28MODULE_LICENSE("GPL v2");
 29MODULE_ALIAS_CRYPTO("ghash");
 30
 31#define RFC4106_NONCE_SIZE	4
 32
 33struct ghash_key {
 34	be128			k;
 35	u64			h[][2];
 36};
 37
 38struct arm_ghash_desc_ctx {
 39	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
 40};
 41
 42struct gcm_aes_ctx {
 43	struct crypto_aes_ctx	aes_key;
 44	u8			nonce[RFC4106_NONCE_SIZE];
 45	struct ghash_key	ghash_key;
 46};
 47
 48asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
 49				       u64 const h[][2], const char *head);
 50
 51asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
 52				      u64 const h[][2], const char *head);
 53
 54asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
 55				  u64 const h[][2], u64 dg[], u8 ctr[],
 56				  u32 const rk[], int rounds, u8 tag[]);
 57asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
 58				 u64 const h[][2], u64 dg[], u8 ctr[],
 59				 u32 const rk[], int rounds, const u8 l[],
 60				 const u8 tag[], u64 authsize);
 61
 62static int ghash_init(struct shash_desc *desc)
 63{
 64	struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 65
 66	*ctx = (struct arm_ghash_desc_ctx){};
 67	return 0;
 68}
 69
 70static __always_inline
 71void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
 72			  struct ghash_key *key, const char *head,
 73			  void (*simd_update)(int blocks, u64 dg[],
 74					      const char *src,
 75					      u64 const h[][2],
 76					      const char *head))
 77{
 78	scoped_ksimd()
 79		simd_update(blocks, dg, src, key->h, head);
 80}
 81
 82/* avoid hogging the CPU for too long */
 83#define MAX_BLOCKS	(SZ_64K / GHASH_BLOCK_SIZE)
 84
 85static int ghash_update(struct shash_desc *desc, const u8 *src,
 86			unsigned int len)
 87{
 88	struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 89	struct ghash_key *key = crypto_shash_ctx(desc->tfm);
 90	int blocks;
 91
 92	blocks = len / GHASH_BLOCK_SIZE;
 93	len -= blocks * GHASH_BLOCK_SIZE;
 94
 95	do {
 96		int chunk = min(blocks, MAX_BLOCKS);
 97
 98		ghash_do_simd_update(chunk, ctx->digest, src, key, NULL,
 99				     pmull_ghash_update_p8);
100		blocks -= chunk;
101		src += chunk * GHASH_BLOCK_SIZE;
102	} while (unlikely(blocks > 0));
103	return len;
104}
105
106static int ghash_export(struct shash_desc *desc, void *out)
107{
108	struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
109	u8 *dst = out;
110
111	put_unaligned_be64(ctx->digest[1], dst);
112	put_unaligned_be64(ctx->digest[0], dst + 8);
113	return 0;
114}
115
116static int ghash_import(struct shash_desc *desc, const void *in)
117{
118	struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
119	const u8 *src = in;
120
121	ctx->digest[1] = get_unaligned_be64(src);
122	ctx->digest[0] = get_unaligned_be64(src + 8);
123	return 0;
124}
125
126static int ghash_finup(struct shash_desc *desc, const u8 *src,
127		       unsigned int len, u8 *dst)
128{
129	struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
130	struct ghash_key *key = crypto_shash_ctx(desc->tfm);
131
132	if (len) {
133		u8 buf[GHASH_BLOCK_SIZE] = {};
134
135		memcpy(buf, src, len);
136		ghash_do_simd_update(1, ctx->digest, buf, key, NULL,
137				     pmull_ghash_update_p8);
138		memzero_explicit(buf, sizeof(buf));
139	}
140	return ghash_export(desc, dst);
141}
142
143static void ghash_reflect(u64 h[], const be128 *k)
144{
145	u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0;
146
147	h[0] = (be64_to_cpu(k->b) << 1) | carry;
148	h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
149
150	if (carry)
151		h[1] ^= 0xc200000000000000UL;
152}
153
154static int ghash_setkey(struct crypto_shash *tfm,
155			const u8 *inkey, unsigned int keylen)
156{
157	struct ghash_key *key = crypto_shash_ctx(tfm);
158
159	if (keylen != GHASH_BLOCK_SIZE)
160		return -EINVAL;
161
162	/* needed for the fallback */
163	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
164
165	ghash_reflect(key->h[0], &key->k);
166	return 0;
167}
168
169static struct shash_alg ghash_alg = {
170	.base.cra_name		= "ghash",
171	.base.cra_driver_name	= "ghash-neon",
172	.base.cra_priority	= 150,
173	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
174	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
175	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
176	.base.cra_module	= THIS_MODULE,
177
178	.digestsize		= GHASH_DIGEST_SIZE,
179	.init			= ghash_init,
180	.update			= ghash_update,
181	.finup			= ghash_finup,
182	.setkey			= ghash_setkey,
183	.export			= ghash_export,
184	.import			= ghash_import,
185	.descsize		= sizeof(struct arm_ghash_desc_ctx),
186	.statesize		= sizeof(struct ghash_desc_ctx),
187};
188
189static int num_rounds(struct crypto_aes_ctx *ctx)
190{
191	/*
192	 * # of rounds specified by AES:
193	 * 128 bit key		10 rounds
194	 * 192 bit key		12 rounds
195	 * 256 bit key		14 rounds
196	 * => n byte key	=> 6 + (n/4) rounds
197	 */
198	return 6 + ctx->key_length / 4;
199}
200
201static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
202			  unsigned int keylen)
203{
204	struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
205	u8 key[GHASH_BLOCK_SIZE];
206	be128 h;
207	int ret;
208
209	ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
210	if (ret)
211		return -EINVAL;
212
213	aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});
214
215	/* needed for the fallback */
216	memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE);
217
218	ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k);
219
220	h = ctx->ghash_key.k;
221	gf128mul_lle(&h, &ctx->ghash_key.k);
222	ghash_reflect(ctx->ghash_key.h[1], &h);
223
224	gf128mul_lle(&h, &ctx->ghash_key.k);
225	ghash_reflect(ctx->ghash_key.h[2], &h);
226
227	gf128mul_lle(&h, &ctx->ghash_key.k);
228	ghash_reflect(ctx->ghash_key.h[3], &h);
229
230	return 0;
231}
232
233static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
234{
235	return crypto_gcm_check_authsize(authsize);
236}
237
238static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
239			   int *buf_count, struct gcm_aes_ctx *ctx)
240{
241	if (*buf_count > 0) {
242		int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
243
244		memcpy(&buf[*buf_count], src, buf_added);
245
246		*buf_count += buf_added;
247		src += buf_added;
248		count -= buf_added;
249	}
250
251	if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
252		int blocks = count / GHASH_BLOCK_SIZE;
253
254		ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,
255				     *buf_count ? buf : NULL,
256				     pmull_ghash_update_p64);
257
258		src += blocks * GHASH_BLOCK_SIZE;
259		count %= GHASH_BLOCK_SIZE;
260		*buf_count = 0;
261	}
262
263	if (count > 0) {
264		memcpy(buf, src, count);
265		*buf_count = count;
266	}
267}
268
269static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len)
270{
271	struct crypto_aead *aead = crypto_aead_reqtfm(req);
272	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
273	u8 buf[GHASH_BLOCK_SIZE];
274	struct scatter_walk walk;
275	int buf_count = 0;
276
277	scatterwalk_start(&walk, req->src);
278
279	do {
280		unsigned int n;
281
282		n = scatterwalk_next(&walk, len);
283		gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx);
284		scatterwalk_done_src(&walk, n);
285		len -= n;
286	} while (len);
287
288	if (buf_count) {
289		memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
290		ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,
291				     pmull_ghash_update_p64);
292	}
293}
294
295static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen)
296{
297	struct crypto_aead *aead = crypto_aead_reqtfm(req);
298	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
299	int nrounds = num_rounds(&ctx->aes_key);
300	struct skcipher_walk walk;
301	u8 buf[AES_BLOCK_SIZE];
302	u64 dg[2] = {};
303	be128 lengths;
304	u8 *tag;
305	int err;
306
307	lengths.a = cpu_to_be64(assoclen * 8);
308	lengths.b = cpu_to_be64(req->cryptlen * 8);
309
310	if (assoclen)
311		gcm_calculate_auth_mac(req, dg, assoclen);
312
313	put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);
314
315	err = skcipher_walk_aead_encrypt(&walk, req, false);
316
317	do {
318		const u8 *src = walk.src.virt.addr;
319		u8 *dst = walk.dst.virt.addr;
320		int nbytes = walk.nbytes;
321
322		tag = (u8 *)&lengths;
323
324		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
325			src = dst = memcpy(buf + sizeof(buf) - nbytes,
326					   src, nbytes);
327		} else if (nbytes < walk.total) {
328			nbytes &= ~(AES_BLOCK_SIZE - 1);
329			tag = NULL;
330		}
331
332		scoped_ksimd()
333			pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,
334					  dg, iv, ctx->aes_key.key_enc, nrounds,
335					  tag);
336
337		if (unlikely(!nbytes))
338			break;
339
340		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
341			memcpy(walk.dst.virt.addr,
342			       buf + sizeof(buf) - nbytes, nbytes);
343
344		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
345	} while (walk.nbytes);
346
347	if (err)
348		return err;
349
350	/* copy authtag to end of dst */
351	scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
352				 crypto_aead_authsize(aead), 1);
353
354	return 0;
355}
356
357static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen)
358{
359	struct crypto_aead *aead = crypto_aead_reqtfm(req);
360	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
361	unsigned int authsize = crypto_aead_authsize(aead);
362	int nrounds = num_rounds(&ctx->aes_key);
363	struct skcipher_walk walk;
364	u8 otag[AES_BLOCK_SIZE];
365	u8 buf[AES_BLOCK_SIZE];
366	u64 dg[2] = {};
367	be128 lengths;
368	u8 *tag;
369	int ret;
370	int err;
371
372	lengths.a = cpu_to_be64(assoclen * 8);
373	lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
374
375	if (assoclen)
376		gcm_calculate_auth_mac(req, dg, assoclen);
377
378	put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);
379
380	scatterwalk_map_and_copy(otag, req->src,
381				 req->assoclen + req->cryptlen - authsize,
382				 authsize, 0);
383
384	err = skcipher_walk_aead_decrypt(&walk, req, false);
385
386	do {
387		const u8 *src = walk.src.virt.addr;
388		u8 *dst = walk.dst.virt.addr;
389		int nbytes = walk.nbytes;
390
391		tag = (u8 *)&lengths;
392
393		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
394			src = dst = memcpy(buf + sizeof(buf) - nbytes,
395					   src, nbytes);
396		} else if (nbytes < walk.total) {
397			nbytes &= ~(AES_BLOCK_SIZE - 1);
398			tag = NULL;
399		}
400
401		scoped_ksimd()
402			ret = pmull_gcm_decrypt(nbytes, dst, src,
403						ctx->ghash_key.h,
404						dg, iv, ctx->aes_key.key_enc,
405						nrounds, tag, otag, authsize);
406
407		if (unlikely(!nbytes))
408			break;
409
410		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
411			memcpy(walk.dst.virt.addr,
412			       buf + sizeof(buf) - nbytes, nbytes);
413
414		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
415	} while (walk.nbytes);
416
417	if (err)
418		return err;
419
420	return ret ? -EBADMSG : 0;
421}
422
423static int gcm_aes_encrypt(struct aead_request *req)
424{
425	u8 iv[AES_BLOCK_SIZE];
426
427	memcpy(iv, req->iv, GCM_AES_IV_SIZE);
428	return gcm_encrypt(req, iv, req->assoclen);
429}
430
431static int gcm_aes_decrypt(struct aead_request *req)
432{
433	u8 iv[AES_BLOCK_SIZE];
434
435	memcpy(iv, req->iv, GCM_AES_IV_SIZE);
436	return gcm_decrypt(req, iv, req->assoclen);
437}
438
439static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
440			  unsigned int keylen)
441{
442	struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
443	int err;
444
445	keylen -= RFC4106_NONCE_SIZE;
446	err = gcm_aes_setkey(tfm, inkey, keylen);
447	if (err)
448		return err;
449
450	memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE);
451	return 0;
452}
453
454static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
455{
456	return crypto_rfc4106_check_authsize(authsize);
457}
458
459static int rfc4106_encrypt(struct aead_request *req)
460{
461	struct crypto_aead *aead = crypto_aead_reqtfm(req);
462	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
463	u8 iv[AES_BLOCK_SIZE];
464
465	memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
466	memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
467
468	return crypto_ipsec_check_assoclen(req->assoclen) ?:
469	       gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
470}
471
472static int rfc4106_decrypt(struct aead_request *req)
473{
474	struct crypto_aead *aead = crypto_aead_reqtfm(req);
475	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
476	u8 iv[AES_BLOCK_SIZE];
477
478	memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
479	memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
480
481	return crypto_ipsec_check_assoclen(req->assoclen) ?:
482	       gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
483}
484
485static struct aead_alg gcm_aes_algs[] = {{
486	.ivsize			= GCM_AES_IV_SIZE,
487	.chunksize		= AES_BLOCK_SIZE,
488	.maxauthsize		= AES_BLOCK_SIZE,
489	.setkey			= gcm_aes_setkey,
490	.setauthsize		= gcm_aes_setauthsize,
491	.encrypt		= gcm_aes_encrypt,
492	.decrypt		= gcm_aes_decrypt,
493
494	.base.cra_name		= "gcm(aes)",
495	.base.cra_driver_name	= "gcm-aes-ce",
496	.base.cra_priority	= 300,
497	.base.cra_blocksize	= 1,
498	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx) +
499				  4 * sizeof(u64[2]),
500	.base.cra_module	= THIS_MODULE,
501}, {
502	.ivsize			= GCM_RFC4106_IV_SIZE,
503	.chunksize		= AES_BLOCK_SIZE,
504	.maxauthsize		= AES_BLOCK_SIZE,
505	.setkey			= rfc4106_setkey,
506	.setauthsize		= rfc4106_setauthsize,
507	.encrypt		= rfc4106_encrypt,
508	.decrypt		= rfc4106_decrypt,
509
510	.base.cra_name		= "rfc4106(gcm(aes))",
511	.base.cra_driver_name	= "rfc4106-gcm-aes-ce",
512	.base.cra_priority	= 300,
513	.base.cra_blocksize	= 1,
514	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx) +
515				  4 * sizeof(u64[2]),
516	.base.cra_module	= THIS_MODULE,
517}};
518
519static int __init ghash_ce_mod_init(void)
520{
521	if (!cpu_have_named_feature(ASIMD))
522		return -ENODEV;
523
524	if (cpu_have_named_feature(PMULL))
525		return crypto_register_aeads(gcm_aes_algs,
526					     ARRAY_SIZE(gcm_aes_algs));
527
528	return crypto_register_shash(&ghash_alg);
529}
530
531static void __exit ghash_ce_mod_exit(void)
532{
533	if (cpu_have_named_feature(PMULL))
534		crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
535	else
536		crypto_unregister_shash(&ghash_alg);
537}
538
539static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = {
540	{ cpu_feature(PMULL) }, { }
541};
542MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);
543
544module_init(ghash_ce_mod_init);
545module_exit(ghash_ce_mod_exit);