aes-gcm-p10-glue.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Glue code for accelerated AES-GCM stitched implementation for ppc64le.
  4 *
  5 * Copyright 2022- IBM Inc. All rights reserved
  6 */
  7
  8#include <linux/unaligned.h>
  9#include <asm/simd.h>
 10#include <asm/switch_to.h>
 11#include <crypto/gcm.h>
 12#include <crypto/aes.h>
 13#include <crypto/algapi.h>
 14#include <crypto/b128ops.h>
 15#include <crypto/gf128mul.h>
 16#include <crypto/internal/simd.h>
 17#include <crypto/internal/aead.h>
 18#include <crypto/internal/hash.h>
 19#include <crypto/internal/skcipher.h>
 20#include <crypto/scatterwalk.h>
 21#include <linux/cpufeature.h>
 22#include <linux/crypto.h>
 23#include <linux/module.h>
 24#include <linux/types.h>
 25
 26#define	PPC_ALIGN		16
 27#define GCM_IV_SIZE		12
 28#define RFC4106_NONCE_SIZE	4
 29
 30MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation");
 31MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com");
 32MODULE_LICENSE("GPL v2");
 33MODULE_ALIAS_CRYPTO("aes");
 34
 35asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits,
 36				       void *key);
 37asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key);
 38asmlinkage void aes_p10_gcm_encrypt(const u8 *in, u8 *out, size_t len,
 39				    void *rkey, u8 *iv, void *Xi);
 40asmlinkage void aes_p10_gcm_decrypt(const u8 *in, u8 *out, size_t len,
 41				    void *rkey, u8 *iv, void *Xi);
 42asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]);
 43asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable,
 44			      unsigned char *aad, unsigned int alen);
 45asmlinkage void gcm_update(u8 *iv, void *Xi);
 46
 47struct aes_key {
 48	u8 key[AES_MAX_KEYLENGTH];
 49	u64 rounds;
 50};
 51
 52struct gcm_ctx {
 53	u8 iv[16];
 54	u8 ivtag[16];
 55	u8 aad_hash[16];
 56	u64 aadLen;
 57	u64 Plen;	/* offset 56 - used in aes_p10_gcm_{en/de}crypt */
 58	u8 pblock[16];
 59};
 60struct Hash_ctx {
 61	u8 H[16];	/* subkey */
 62	u8 Htable[256];	/* Xi, Hash table(offset 32) */
 63};
 64
 65struct p10_aes_gcm_ctx {
 66	struct aes_key enc_key;
 67	u8 nonce[RFC4106_NONCE_SIZE];
 68};
 69
 70static void vsx_begin(void)
 71{
 72	preempt_disable();
 73	pagefault_disable();
 74	enable_kernel_vsx();
 75}
 76
 77static void vsx_end(void)
 78{
 79	disable_kernel_vsx();
 80	pagefault_enable();
 81	preempt_enable();
 82}
 83
 84static void set_subkey(unsigned char *hash)
 85{
 86	*(u64 *)&hash[0] = be64_to_cpup((__be64 *)&hash[0]);
 87	*(u64 *)&hash[8] = be64_to_cpup((__be64 *)&hash[8]);
 88}
 89
 90/*
 91 * Compute aad if any.
 92 *   - Hash aad and copy to Xi.
 93 */
 94static void set_aad(struct gcm_ctx *gctx, struct Hash_ctx *hash,
 95		    unsigned char *aad, int alen)
 96{
 97	int i;
 98	u8 nXi[16] = {0, };
 99
100	gctx->aadLen = alen;
101	i = alen & ~0xf;
102	if (i) {
103		gcm_ghash_p10(nXi, hash->Htable+32, aad, i);
104		aad += i;
105		alen -= i;
106	}
107	if (alen) {
108		for (i = 0; i < alen; i++)
109			nXi[i] ^= aad[i];
110
111		memset(gctx->aad_hash, 0, 16);
112		gcm_ghash_p10(gctx->aad_hash, hash->Htable+32, nXi, 16);
113	} else {
114		memcpy(gctx->aad_hash, nXi, 16);
115	}
116
117	memcpy(hash->Htable, gctx->aad_hash, 16);
118}
119
120static void gcmp10_init(struct gcm_ctx *gctx, u8 *iv, unsigned char *rdkey,
121			struct Hash_ctx *hash, u8 *assoc, unsigned int assoclen)
122{
123	__be32 counter = cpu_to_be32(1);
124
125	aes_p10_encrypt(hash->H, hash->H, rdkey);
126	set_subkey(hash->H);
127	gcm_init_htable(hash->Htable+32, hash->H);
128
129	*((__be32 *)(iv+12)) = counter;
130
131	gctx->Plen = 0;
132
133	/*
134	 * Encrypt counter vector as iv tag and increment counter.
135	 */
136	aes_p10_encrypt(iv, gctx->ivtag, rdkey);
137
138	counter = cpu_to_be32(2);
139	*((__be32 *)(iv+12)) = counter;
140	memcpy(gctx->iv, iv, 16);
141
142	gctx->aadLen = assoclen;
143	memset(gctx->aad_hash, 0, 16);
144	if (assoclen)
145		set_aad(gctx, hash, assoc, assoclen);
146}
147
148static void finish_tag(struct gcm_ctx *gctx, struct Hash_ctx *hash, int len)
149{
150	int i;
151	unsigned char len_ac[16 + PPC_ALIGN];
152	unsigned char *aclen = PTR_ALIGN((void *)len_ac, PPC_ALIGN);
153	__be64 clen = cpu_to_be64(len << 3);
154	__be64 alen = cpu_to_be64(gctx->aadLen << 3);
155
156	if (len == 0 && gctx->aadLen == 0) {
157		memcpy(hash->Htable, gctx->ivtag, 16);
158		return;
159	}
160
161	/*
162	 * Len is in bits.
163	 */
164	*((__be64 *)(aclen)) = alen;
165	*((__be64 *)(aclen+8)) = clen;
166
167	/*
168	 * hash (AAD len and len)
169	 */
170	gcm_ghash_p10(hash->Htable, hash->Htable+32, aclen, 16);
171
172	for (i = 0; i < 16; i++)
173		hash->Htable[i] ^= gctx->ivtag[i];
174}
175
176static int set_authsize(struct crypto_aead *tfm, unsigned int authsize)
177{
178	switch (authsize) {
179	case 4:
180	case 8:
181	case 12:
182	case 13:
183	case 14:
184	case 15:
185	case 16:
186		break;
187	default:
188		return -EINVAL;
189	}
190
191	return 0;
192}
193
194static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
195			      unsigned int keylen)
196{
197	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
198	struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
199	int ret;
200
201	vsx_begin();
202	ret = aes_p10_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
203	vsx_end();
204
205	return ret ? -EINVAL : 0;
206}
207
208static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv,
209			     int assoclen, int enc)
210{
211	struct crypto_tfm *tfm = req->base.tfm;
212	struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
213	u8 databuf[sizeof(struct gcm_ctx) + PPC_ALIGN];
214	struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN);
215	u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN];
216	struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN);
217	struct skcipher_walk walk;
218	u8 *assocmem = NULL;
219	u8 *assoc;
220	unsigned int cryptlen = req->cryptlen;
221	unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN];
222	unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN);
223	int ret;
224	unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm));
225	u8 otag[16];
226	int total_processed = 0;
227	int nbytes;
228
229	memset(databuf, 0, sizeof(databuf));
230	memset(hashbuf, 0, sizeof(hashbuf));
231	memset(ivbuf, 0, sizeof(ivbuf));
232	memcpy(iv, riv, GCM_IV_SIZE);
233
234	/* Linearize assoc, if not already linear */
235	if (req->src->length >= assoclen && req->src->length) {
236		assoc = sg_virt(req->src); /* ppc64 is !HIGHMEM */
237	} else {
238		gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
239			      GFP_KERNEL : GFP_ATOMIC;
240
241		/* assoc can be any length, so must be on heap */
242		assocmem = kmalloc(assoclen, flags);
243		if (unlikely(!assocmem))
244			return -ENOMEM;
245		assoc = assocmem;
246
247		scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
248	}
249
250	vsx_begin();
251	gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen);
252	vsx_end();
253
254	kfree(assocmem);
255
256	if (enc)
257		ret = skcipher_walk_aead_encrypt(&walk, req, false);
258	else
259		ret = skcipher_walk_aead_decrypt(&walk, req, false);
260	if (ret)
261		return ret;
262
263	while ((nbytes = walk.nbytes) > 0 && ret == 0) {
264		const u8 *src = walk.src.virt.addr;
265		u8 *dst = walk.dst.virt.addr;
266		u8 buf[AES_BLOCK_SIZE];
267
268		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
269			src = dst = memcpy(buf, src, nbytes);
270
271		vsx_begin();
272		if (enc)
273			aes_p10_gcm_encrypt(src, dst, nbytes,
274					    &ctx->enc_key, gctx->iv, hash->Htable);
275		else
276			aes_p10_gcm_decrypt(src, dst, nbytes,
277					    &ctx->enc_key, gctx->iv, hash->Htable);
278
279		if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
280			memcpy(walk.dst.virt.addr, buf, nbytes);
281
282		vsx_end();
283
284		total_processed += walk.nbytes;
285		ret = skcipher_walk_done(&walk, 0);
286	}
287
288	if (ret)
289		return ret;
290
291	/* Finalize hash */
292	vsx_begin();
293	gcm_update(gctx->iv, hash->Htable);
294	finish_tag(gctx, hash, total_processed);
295	vsx_end();
296
297	/* copy Xi to end of dst */
298	if (enc)
299		scatterwalk_map_and_copy(hash->Htable, req->dst, req->assoclen + cryptlen,
300					 auth_tag_len, 1);
301	else {
302		scatterwalk_map_and_copy(otag, req->src,
303					 req->assoclen + cryptlen - auth_tag_len,
304					 auth_tag_len, 0);
305
306		if (crypto_memneq(otag, hash->Htable, auth_tag_len)) {
307			memzero_explicit(hash->Htable, 16);
308			return -EBADMSG;
309		}
310	}
311
312	return 0;
313}
314
315static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
316			  unsigned int keylen)
317{
318	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
319	int err;
320
321	keylen -= RFC4106_NONCE_SIZE;
322	err = p10_aes_gcm_setkey(tfm, inkey, keylen);
323	if (err)
324		return err;
325
326	memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE);
327	return 0;
328}
329
330static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
331{
332	return crypto_rfc4106_check_authsize(authsize);
333}
334
335static int rfc4106_encrypt(struct aead_request *req)
336{
337	struct crypto_aead *aead = crypto_aead_reqtfm(req);
338	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
339	u8 iv[AES_BLOCK_SIZE];
340
341	memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
342	memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
343
344	return crypto_ipsec_check_assoclen(req->assoclen) ?:
345	       p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 1);
346}
347
348static int rfc4106_decrypt(struct aead_request *req)
349{
350	struct crypto_aead *aead = crypto_aead_reqtfm(req);
351	struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
352	u8 iv[AES_BLOCK_SIZE];
353
354	memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
355	memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
356
357	return crypto_ipsec_check_assoclen(req->assoclen) ?:
358	       p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 0);
359}
360
361static int p10_aes_gcm_encrypt(struct aead_request *req)
362{
363	return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 1);
364}
365
366static int p10_aes_gcm_decrypt(struct aead_request *req)
367{
368	return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 0);
369}
370
371static struct aead_alg gcm_aes_algs[] = {{
372	.ivsize			= GCM_IV_SIZE,
373	.maxauthsize		= 16,
374
375	.setauthsize		= set_authsize,
376	.setkey			= p10_aes_gcm_setkey,
377	.encrypt		= p10_aes_gcm_encrypt,
378	.decrypt		= p10_aes_gcm_decrypt,
379
380	.base.cra_name		= "__gcm(aes)",
381	.base.cra_driver_name	= "__aes_gcm_p10",
382	.base.cra_priority	= 2100,
383	.base.cra_blocksize	= 1,
384	.base.cra_ctxsize	= sizeof(struct p10_aes_gcm_ctx)+
385				  4 * sizeof(u64[2]),
386	.base.cra_module	= THIS_MODULE,
387	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
388}, {
389	.ivsize			= GCM_RFC4106_IV_SIZE,
390	.maxauthsize		= 16,
391	.setkey			= rfc4106_setkey,
392	.setauthsize		= rfc4106_setauthsize,
393	.encrypt		= rfc4106_encrypt,
394	.decrypt		= rfc4106_decrypt,
395
396	.base.cra_name		= "__rfc4106(gcm(aes))",
397	.base.cra_driver_name	= "__rfc4106_aes_gcm_p10",
398	.base.cra_priority	= 2100,
399	.base.cra_blocksize	= 1,
400	.base.cra_ctxsize	= sizeof(struct p10_aes_gcm_ctx) +
401				  4 * sizeof(u64[2]),
402	.base.cra_module	= THIS_MODULE,
403	.base.cra_flags		= CRYPTO_ALG_INTERNAL,
404}};
405
406static struct simd_aead_alg *p10_simd_aeads[ARRAY_SIZE(gcm_aes_algs)];
407
408static int __init p10_init(void)
409{
410	int ret;
411
412	if (!cpu_has_feature(CPU_FTR_ARCH_31))
413		return 0;
414
415	ret = simd_register_aeads_compat(gcm_aes_algs,
416					 ARRAY_SIZE(gcm_aes_algs),
417					 p10_simd_aeads);
418	if (ret) {
419		simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs),
420				      p10_simd_aeads);
421		return ret;
422	}
423	return 0;
424}
425
426static void __exit p10_exit(void)
427{
428	simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs),
429			      p10_simd_aeads);
430}
431
432module_init(p10_init);
433module_exit(p10_exit);