sm4-ce-glue.c

  1/* SPDX-License-Identifier: GPL-2.0-or-later */
  2/*
  3 * SM4 Cipher Algorithm, using ARMv8 Crypto Extensions
  4 * as specified in
  5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
  6 *
  7 * Copyright (C) 2022, Alibaba Group.
  8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
  9 */
 10
 11#include <asm/simd.h>
 12#include <crypto/b128ops.h>
 13#include <crypto/internal/hash.h>
 14#include <crypto/internal/skcipher.h>
 15#include <crypto/scatterwalk.h>
 16#include <crypto/sm4.h>
 17#include <crypto/utils.h>
 18#include <crypto/xts.h>
 19#include <linux/cpufeature.h>
 20#include <linux/kernel.h>
 21#include <linux/module.h>
 22#include <linux/string.h>
 23
 24#define BYTES2BLKS(nbytes)	((nbytes) >> 4)
 25
 26asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec,
 27				  const u32 *fk, const u32 *ck);
 28asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src);
 29asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src,
 30			     unsigned int nblks);
 31asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src,
 32			       u8 *iv, unsigned int nblocks);
 33asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src,
 34			       u8 *iv, unsigned int nblocks);
 35asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src,
 36				   u8 *iv, unsigned int nbytes);
 37asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src,
 38				   u8 *iv, unsigned int nbytes);
 39asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src,
 40			       u8 *iv, unsigned int nblks);
 41asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src,
 42			       u8 *tweak, unsigned int nbytes,
 43			       const u32 *rkey2_enc);
 44asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src,
 45			       u8 *tweak, unsigned int nbytes,
 46			       const u32 *rkey2_enc);
 47asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest,
 48				  const u8 *src, unsigned int nblocks,
 49				  bool enc_before, bool enc_after);
 50
 51EXPORT_SYMBOL(sm4_ce_expand_key);
 52EXPORT_SYMBOL(sm4_ce_crypt_block);
 53EXPORT_SYMBOL(sm4_ce_cbc_enc);
 54
 55struct sm4_xts_ctx {
 56	struct sm4_ctx key1;
 57	struct sm4_ctx key2;
 58};
 59
 60struct sm4_mac_tfm_ctx {
 61	struct sm4_ctx key;
 62	u8 __aligned(8) consts[];
 63};
 64
 65struct sm4_mac_desc_ctx {
 66	u8 digest[SM4_BLOCK_SIZE];
 67};
 68
 69static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
 70		      unsigned int key_len)
 71{
 72	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
 73
 74	if (key_len != SM4_KEY_SIZE)
 75		return -EINVAL;
 76
 77	scoped_ksimd()
 78		sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec,
 79				  crypto_sm4_fk, crypto_sm4_ck);
 80	return 0;
 81}
 82
 83static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
 84			  unsigned int key_len)
 85{
 86	struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 87	int ret;
 88
 89	if (key_len != SM4_KEY_SIZE * 2)
 90		return -EINVAL;
 91
 92	ret = xts_verify_key(tfm, key, key_len);
 93	if (ret)
 94		return ret;
 95
 96	scoped_ksimd() {
 97		sm4_ce_expand_key(key, ctx->key1.rkey_enc,
 98				ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
 99		sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc,
100				ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
101	}
102
103	return 0;
104}
105
106static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
107{
108	struct skcipher_walk walk;
109	unsigned int nbytes;
110	int err;
111
112	err = skcipher_walk_virt(&walk, req, false);
113
114	while ((nbytes = walk.nbytes) > 0) {
115		const u8 *src = walk.src.virt.addr;
116		u8 *dst = walk.dst.virt.addr;
117		unsigned int nblks;
118
119		scoped_ksimd() {
120			nblks = BYTES2BLKS(nbytes);
121			if (nblks) {
122				sm4_ce_crypt(rkey, dst, src, nblks);
123				nbytes -= nblks * SM4_BLOCK_SIZE;
124			}
125		}
126
127		err = skcipher_walk_done(&walk, nbytes);
128	}
129
130	return err;
131}
132
133static int sm4_ecb_encrypt(struct skcipher_request *req)
134{
135	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
136	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
137
138	return sm4_ecb_do_crypt(req, ctx->rkey_enc);
139}
140
141static int sm4_ecb_decrypt(struct skcipher_request *req)
142{
143	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
144	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
145
146	return sm4_ecb_do_crypt(req, ctx->rkey_dec);
147}
148
149static int sm4_cbc_crypt(struct skcipher_request *req,
150			 struct sm4_ctx *ctx, bool encrypt)
151{
152	struct skcipher_walk walk;
153	unsigned int nbytes;
154	int err;
155
156	err = skcipher_walk_virt(&walk, req, false);
157	if (err)
158		return err;
159
160	while ((nbytes = walk.nbytes) > 0) {
161		const u8 *src = walk.src.virt.addr;
162		u8 *dst = walk.dst.virt.addr;
163		unsigned int nblocks;
164
165		nblocks = nbytes / SM4_BLOCK_SIZE;
166		if (nblocks) {
167			scoped_ksimd() {
168				if (encrypt)
169					sm4_ce_cbc_enc(ctx->rkey_enc, dst, src,
170						       walk.iv, nblocks);
171				else
172					sm4_ce_cbc_dec(ctx->rkey_dec, dst, src,
173						       walk.iv, nblocks);
174			}
175		}
176
177		err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
178	}
179
180	return err;
181}
182
183static int sm4_cbc_encrypt(struct skcipher_request *req)
184{
185	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
186	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
187
188	return sm4_cbc_crypt(req, ctx, true);
189}
190
191static int sm4_cbc_decrypt(struct skcipher_request *req)
192{
193	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
194	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
195
196	return sm4_cbc_crypt(req, ctx, false);
197}
198
199static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt)
200{
201	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
202	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
203	struct scatterlist *src = req->src;
204	struct scatterlist *dst = req->dst;
205	struct scatterlist sg_src[2], sg_dst[2];
206	struct skcipher_request subreq;
207	struct skcipher_walk walk;
208	int cbc_blocks;
209	int err;
210
211	if (req->cryptlen < SM4_BLOCK_SIZE)
212		return -EINVAL;
213
214	if (req->cryptlen == SM4_BLOCK_SIZE)
215		return sm4_cbc_crypt(req, ctx, encrypt);
216
217	skcipher_request_set_tfm(&subreq, tfm);
218	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
219				      NULL, NULL);
220
221	/* handle the CBC cryption part */
222	cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
223	if (cbc_blocks) {
224		skcipher_request_set_crypt(&subreq, src, dst,
225					   cbc_blocks * SM4_BLOCK_SIZE,
226					   req->iv);
227
228		err = sm4_cbc_crypt(&subreq, ctx, encrypt);
229		if (err)
230			return err;
231
232		dst = src = scatterwalk_ffwd(sg_src, src, subreq.cryptlen);
233		if (req->dst != req->src)
234			dst = scatterwalk_ffwd(sg_dst, req->dst,
235					       subreq.cryptlen);
236	}
237
238	/* handle ciphertext stealing */
239	skcipher_request_set_crypt(&subreq, src, dst,
240				   req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE,
241				   req->iv);
242
243	err = skcipher_walk_virt(&walk, &subreq, false);
244	if (err)
245		return err;
246
247	scoped_ksimd() {
248		if (encrypt)
249			sm4_ce_cbc_cts_enc(ctx->rkey_enc, walk.dst.virt.addr,
250					   walk.src.virt.addr, walk.iv, walk.nbytes);
251		else
252			sm4_ce_cbc_cts_dec(ctx->rkey_dec, walk.dst.virt.addr,
253					   walk.src.virt.addr, walk.iv, walk.nbytes);
254	}
255
256	return skcipher_walk_done(&walk, 0);
257}
258
259static int sm4_cbc_cts_encrypt(struct skcipher_request *req)
260{
261	return sm4_cbc_cts_crypt(req, true);
262}
263
264static int sm4_cbc_cts_decrypt(struct skcipher_request *req)
265{
266	return sm4_cbc_cts_crypt(req, false);
267}
268
269static int sm4_ctr_crypt(struct skcipher_request *req)
270{
271	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
272	struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
273	struct skcipher_walk walk;
274	unsigned int nbytes;
275	int err;
276
277	err = skcipher_walk_virt(&walk, req, false);
278
279	while ((nbytes = walk.nbytes) > 0) {
280		const u8 *src = walk.src.virt.addr;
281		u8 *dst = walk.dst.virt.addr;
282		unsigned int nblks;
283
284		scoped_ksimd() {
285			nblks = BYTES2BLKS(nbytes);
286			if (nblks) {
287				sm4_ce_ctr_enc(ctx->rkey_enc, dst, src, walk.iv, nblks);
288				dst += nblks * SM4_BLOCK_SIZE;
289				src += nblks * SM4_BLOCK_SIZE;
290				nbytes -= nblks * SM4_BLOCK_SIZE;
291			}
292
293			/* tail */
294			if (walk.nbytes == walk.total && nbytes > 0) {
295				u8 keystream[SM4_BLOCK_SIZE];
296
297				sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
298				crypto_inc(walk.iv, SM4_BLOCK_SIZE);
299				crypto_xor_cpy(dst, src, keystream, nbytes);
300				nbytes = 0;
301			}
302		}
303
304		err = skcipher_walk_done(&walk, nbytes);
305	}
306
307	return err;
308}
309
310static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt)
311{
312	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
313	struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
314	int tail = req->cryptlen % SM4_BLOCK_SIZE;
315	const u32 *rkey2_enc = ctx->key2.rkey_enc;
316	struct scatterlist sg_src[2], sg_dst[2];
317	struct skcipher_request subreq;
318	struct scatterlist *src, *dst;
319	struct skcipher_walk walk;
320	unsigned int nbytes;
321	int err;
322
323	if (req->cryptlen < SM4_BLOCK_SIZE)
324		return -EINVAL;
325
326	err = skcipher_walk_virt(&walk, req, false);
327	if (err)
328		return err;
329
330	if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
331		int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
332
333		skcipher_walk_abort(&walk);
334
335		skcipher_request_set_tfm(&subreq, tfm);
336		skcipher_request_set_callback(&subreq,
337					      skcipher_request_flags(req),
338					      NULL, NULL);
339		skcipher_request_set_crypt(&subreq, req->src, req->dst,
340					   nblocks * SM4_BLOCK_SIZE, req->iv);
341
342		err = skcipher_walk_virt(&walk, &subreq, false);
343		if (err)
344			return err;
345	} else {
346		tail = 0;
347	}
348
349	scoped_ksimd() {
350		while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) {
351			if (nbytes < walk.total)
352				nbytes &= ~(SM4_BLOCK_SIZE - 1);
353
354			if (encrypt)
355				sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
356						walk.src.virt.addr, walk.iv, nbytes,
357						rkey2_enc);
358			else
359				sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
360						walk.src.virt.addr, walk.iv, nbytes,
361						rkey2_enc);
362
363			rkey2_enc = NULL;
364
365			err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
366			if (err)
367				return err;
368		}
369
370		if (likely(tail == 0))
371			return 0;
372
373		/* handle ciphertext stealing */
374
375		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
376		if (req->dst != req->src)
377			dst = scatterwalk_ffwd(sg_dst, req->dst, subreq.cryptlen);
378
379		skcipher_request_set_crypt(&subreq, src, dst,
380					   SM4_BLOCK_SIZE + tail, req->iv);
381
382		err = skcipher_walk_virt(&walk, &subreq, false);
383		if (err)
384			return err;
385
386		if (encrypt)
387			sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
388					walk.src.virt.addr, walk.iv, walk.nbytes,
389					rkey2_enc);
390		else
391			sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
392					walk.src.virt.addr, walk.iv, walk.nbytes,
393					rkey2_enc);
394	}
395
396	return skcipher_walk_done(&walk, 0);
397}
398
399static int sm4_xts_encrypt(struct skcipher_request *req)
400{
401	return sm4_xts_crypt(req, true);
402}
403
404static int sm4_xts_decrypt(struct skcipher_request *req)
405{
406	return sm4_xts_crypt(req, false);
407}
408
409static struct skcipher_alg sm4_algs[] = {
410	{
411		.base = {
412			.cra_name		= "ecb(sm4)",
413			.cra_driver_name	= "ecb-sm4-ce",
414			.cra_priority		= 400,
415			.cra_blocksize		= SM4_BLOCK_SIZE,
416			.cra_ctxsize		= sizeof(struct sm4_ctx),
417			.cra_module		= THIS_MODULE,
418		},
419		.min_keysize	= SM4_KEY_SIZE,
420		.max_keysize	= SM4_KEY_SIZE,
421		.setkey		= sm4_setkey,
422		.encrypt	= sm4_ecb_encrypt,
423		.decrypt	= sm4_ecb_decrypt,
424	}, {
425		.base = {
426			.cra_name		= "cbc(sm4)",
427			.cra_driver_name	= "cbc-sm4-ce",
428			.cra_priority		= 400,
429			.cra_blocksize		= SM4_BLOCK_SIZE,
430			.cra_ctxsize		= sizeof(struct sm4_ctx),
431			.cra_module		= THIS_MODULE,
432		},
433		.min_keysize	= SM4_KEY_SIZE,
434		.max_keysize	= SM4_KEY_SIZE,
435		.ivsize		= SM4_BLOCK_SIZE,
436		.setkey		= sm4_setkey,
437		.encrypt	= sm4_cbc_encrypt,
438		.decrypt	= sm4_cbc_decrypt,
439	}, {
440		.base = {
441			.cra_name		= "ctr(sm4)",
442			.cra_driver_name	= "ctr-sm4-ce",
443			.cra_priority		= 400,
444			.cra_blocksize		= 1,
445			.cra_ctxsize		= sizeof(struct sm4_ctx),
446			.cra_module		= THIS_MODULE,
447		},
448		.min_keysize	= SM4_KEY_SIZE,
449		.max_keysize	= SM4_KEY_SIZE,
450		.ivsize		= SM4_BLOCK_SIZE,
451		.chunksize	= SM4_BLOCK_SIZE,
452		.setkey		= sm4_setkey,
453		.encrypt	= sm4_ctr_crypt,
454		.decrypt	= sm4_ctr_crypt,
455	}, {
456		.base = {
457			.cra_name		= "cts(cbc(sm4))",
458			.cra_driver_name	= "cts-cbc-sm4-ce",
459			.cra_priority		= 400,
460			.cra_blocksize		= SM4_BLOCK_SIZE,
461			.cra_ctxsize		= sizeof(struct sm4_ctx),
462			.cra_module		= THIS_MODULE,
463		},
464		.min_keysize	= SM4_KEY_SIZE,
465		.max_keysize	= SM4_KEY_SIZE,
466		.ivsize		= SM4_BLOCK_SIZE,
467		.walksize	= SM4_BLOCK_SIZE * 2,
468		.setkey		= sm4_setkey,
469		.encrypt	= sm4_cbc_cts_encrypt,
470		.decrypt	= sm4_cbc_cts_decrypt,
471	}, {
472		.base = {
473			.cra_name		= "xts(sm4)",
474			.cra_driver_name	= "xts-sm4-ce",
475			.cra_priority		= 400,
476			.cra_blocksize		= SM4_BLOCK_SIZE,
477			.cra_ctxsize		= sizeof(struct sm4_xts_ctx),
478			.cra_module		= THIS_MODULE,
479		},
480		.min_keysize	= SM4_KEY_SIZE * 2,
481		.max_keysize	= SM4_KEY_SIZE * 2,
482		.ivsize		= SM4_BLOCK_SIZE,
483		.walksize	= SM4_BLOCK_SIZE * 2,
484		.setkey		= sm4_xts_setkey,
485		.encrypt	= sm4_xts_encrypt,
486		.decrypt	= sm4_xts_decrypt,
487	}
488};
489
490static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key,
491			     unsigned int key_len)
492{
493	struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
494
495	if (key_len != SM4_KEY_SIZE)
496		return -EINVAL;
497
498	scoped_ksimd()
499		sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
500				crypto_sm4_fk, crypto_sm4_ck);
501	return 0;
502}
503
504static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key,
505			   unsigned int key_len)
506{
507	struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
508	be128 *consts = (be128 *)ctx->consts;
509	u64 a, b;
510
511	if (key_len != SM4_KEY_SIZE)
512		return -EINVAL;
513
514	memset(consts, 0, SM4_BLOCK_SIZE);
515
516	scoped_ksimd() {
517		sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
518				crypto_sm4_fk, crypto_sm4_ck);
519
520		/* encrypt the zero block */
521		sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts);
522	}
523
524	/* gf(2^128) multiply zero-ciphertext with u and u^2 */
525	a = be64_to_cpu(consts[0].a);
526	b = be64_to_cpu(consts[0].b);
527	consts[0].a = cpu_to_be64((a << 1) | (b >> 63));
528	consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
529
530	a = be64_to_cpu(consts[0].a);
531	b = be64_to_cpu(consts[0].b);
532	consts[1].a = cpu_to_be64((a << 1) | (b >> 63));
533	consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
534
535	return 0;
536}
537
538static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key,
539			   unsigned int key_len)
540{
541	struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
542	u8 __aligned(8) key2[SM4_BLOCK_SIZE];
543	static u8 const ks[3][SM4_BLOCK_SIZE] = {
544		{ [0 ... SM4_BLOCK_SIZE - 1] = 0x1},
545		{ [0 ... SM4_BLOCK_SIZE - 1] = 0x2},
546		{ [0 ... SM4_BLOCK_SIZE - 1] = 0x3},
547	};
548
549	if (key_len != SM4_KEY_SIZE)
550		return -EINVAL;
551
552	scoped_ksimd() {
553		sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
554				crypto_sm4_fk, crypto_sm4_ck);
555
556		sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]);
557		sm4_ce_crypt(ctx->key.rkey_enc, ctx->consts, ks[1], 2);
558
559		sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec,
560				crypto_sm4_fk, crypto_sm4_ck);
561	}
562
563	return 0;
564}
565
566static int sm4_mac_init(struct shash_desc *desc)
567{
568	struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
569
570	memset(ctx->digest, 0, SM4_BLOCK_SIZE);
571	return 0;
572}
573
574static int sm4_mac_update(struct shash_desc *desc, const u8 *p,
575			  unsigned int len)
576{
577	struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
578	struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
579	unsigned int nblocks = len / SM4_BLOCK_SIZE;
580
581	len %= SM4_BLOCK_SIZE;
582	scoped_ksimd()
583		sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p,
584				nblocks, false, true);
585	return len;
586}
587
588static int sm4_cmac_finup(struct shash_desc *desc, const u8 *src,
589			  unsigned int len, u8 *out)
590{
591	struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
592	struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
593	const u8 *consts = tctx->consts;
594
595	crypto_xor(ctx->digest, src, len);
596	if (len != SM4_BLOCK_SIZE) {
597		ctx->digest[len] ^= 0x80;
598		consts += SM4_BLOCK_SIZE;
599	}
600	scoped_ksimd()
601		sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1,
602				  false, true);
603	memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
604	return 0;
605}
606
607static int sm4_cbcmac_finup(struct shash_desc *desc, const u8 *src,
608			    unsigned int len, u8 *out)
609{
610	struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
611	struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
612
613	if (len) {
614		crypto_xor(ctx->digest, src, len);
615		scoped_ksimd()
616			sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest,
617					   ctx->digest);
618	}
619	memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
620	return 0;
621}
622
623static struct shash_alg sm4_mac_algs[] = {
624	{
625		.base = {
626			.cra_name		= "cmac(sm4)",
627			.cra_driver_name	= "cmac-sm4-ce",
628			.cra_priority		= 400,
629			.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
630						  CRYPTO_AHASH_ALG_FINAL_NONZERO,
631			.cra_blocksize		= SM4_BLOCK_SIZE,
632			.cra_ctxsize		= sizeof(struct sm4_mac_tfm_ctx)
633							+ SM4_BLOCK_SIZE * 2,
634			.cra_module		= THIS_MODULE,
635		},
636		.digestsize	= SM4_BLOCK_SIZE,
637		.init		= sm4_mac_init,
638		.update		= sm4_mac_update,
639		.finup		= sm4_cmac_finup,
640		.setkey		= sm4_cmac_setkey,
641		.descsize	= sizeof(struct sm4_mac_desc_ctx),
642	}, {
643		.base = {
644			.cra_name		= "xcbc(sm4)",
645			.cra_driver_name	= "xcbc-sm4-ce",
646			.cra_priority		= 400,
647			.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
648						  CRYPTO_AHASH_ALG_FINAL_NONZERO,
649			.cra_blocksize		= SM4_BLOCK_SIZE,
650			.cra_ctxsize		= sizeof(struct sm4_mac_tfm_ctx)
651							+ SM4_BLOCK_SIZE * 2,
652			.cra_module		= THIS_MODULE,
653		},
654		.digestsize	= SM4_BLOCK_SIZE,
655		.init		= sm4_mac_init,
656		.update		= sm4_mac_update,
657		.finup		= sm4_cmac_finup,
658		.setkey		= sm4_xcbc_setkey,
659		.descsize	= sizeof(struct sm4_mac_desc_ctx),
660	}, {
661		.base = {
662			.cra_name		= "cbcmac(sm4)",
663			.cra_driver_name	= "cbcmac-sm4-ce",
664			.cra_priority		= 400,
665			.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
666			.cra_blocksize		= SM4_BLOCK_SIZE,
667			.cra_ctxsize		= sizeof(struct sm4_mac_tfm_ctx),
668			.cra_module		= THIS_MODULE,
669		},
670		.digestsize	= SM4_BLOCK_SIZE,
671		.init		= sm4_mac_init,
672		.update		= sm4_mac_update,
673		.finup		= sm4_cbcmac_finup,
674		.setkey		= sm4_cbcmac_setkey,
675		.descsize	= sizeof(struct sm4_mac_desc_ctx),
676	}
677};
678
679static int __init sm4_init(void)
680{
681	int err;
682
683	err = crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
684	if (err)
685		return err;
686
687	err = crypto_register_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
688	if (err)
689		goto out_err;
690
691	return 0;
692
693out_err:
694	crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
695	return err;
696}
697
698static void __exit sm4_exit(void)
699{
700	crypto_unregister_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
701	crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
702}
703
704module_cpu_feature_match(SM4, sm4_init);
705module_exit(sm4_exit);
706
707MODULE_DESCRIPTION("SM4 ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
708MODULE_ALIAS_CRYPTO("sm4-ce");
709MODULE_ALIAS_CRYPTO("sm4");
710MODULE_ALIAS_CRYPTO("ecb(sm4)");
711MODULE_ALIAS_CRYPTO("cbc(sm4)");
712MODULE_ALIAS_CRYPTO("ctr(sm4)");
713MODULE_ALIAS_CRYPTO("cts(cbc(sm4))");
714MODULE_ALIAS_CRYPTO("xts(sm4)");
715MODULE_ALIAS_CRYPTO("cmac(sm4)");
716MODULE_ALIAS_CRYPTO("xcbc(sm4)");
717MODULE_ALIAS_CRYPTO("cbcmac(sm4)");
718MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
719MODULE_LICENSE("GPL v2");