serpent_avx2_glue.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
  4 *
  5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
  6 */
  7
  8#include <linux/module.h>
  9#include <linux/types.h>
 10#include <linux/crypto.h>
 11#include <linux/err.h>
 12#include <crypto/algapi.h>
 13#include <crypto/serpent.h>
 14
 15#include "serpent-avx.h"
 16#include "ecb_cbc_helpers.h"
 17
 18#define SERPENT_AVX2_PARALLEL_BLOCKS 16
 19
 20/* 16-way AVX2 parallel cipher functions */
 21asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
 22asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
 23asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
 24
 25static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
 26				   const u8 *key, unsigned int keylen)
 27{
 28	return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
 29}
 30
 31static int ecb_encrypt(struct skcipher_request *req)
 32{
 33	ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
 34	ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way);
 35	ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx);
 36	ECB_BLOCK(1, __serpent_encrypt);
 37	ECB_WALK_END();
 38}
 39
 40static int ecb_decrypt(struct skcipher_request *req)
 41{
 42	ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
 43	ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way);
 44	ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx);
 45	ECB_BLOCK(1, __serpent_decrypt);
 46	ECB_WALK_END();
 47}
 48
 49static int cbc_encrypt(struct skcipher_request *req)
 50{
 51	CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
 52	CBC_ENC_BLOCK(__serpent_encrypt);
 53	CBC_WALK_END();
 54}
 55
 56static int cbc_decrypt(struct skcipher_request *req)
 57{
 58	CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
 59	CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way);
 60	CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx);
 61	CBC_DEC_BLOCK(1, __serpent_decrypt);
 62	CBC_WALK_END();
 63}
 64
 65static struct skcipher_alg serpent_algs[] = {
 66	{
 67		.base.cra_name		= "ecb(serpent)",
 68		.base.cra_driver_name	= "ecb-serpent-avx2",
 69		.base.cra_priority	= 600,
 70		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 71		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 72		.base.cra_module	= THIS_MODULE,
 73		.min_keysize		= SERPENT_MIN_KEY_SIZE,
 74		.max_keysize		= SERPENT_MAX_KEY_SIZE,
 75		.setkey			= serpent_setkey_skcipher,
 76		.encrypt		= ecb_encrypt,
 77		.decrypt		= ecb_decrypt,
 78	}, {
 79		.base.cra_name		= "cbc(serpent)",
 80		.base.cra_driver_name	= "cbc-serpent-avx2",
 81		.base.cra_priority	= 600,
 82		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
 83		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
 84		.base.cra_module	= THIS_MODULE,
 85		.min_keysize		= SERPENT_MIN_KEY_SIZE,
 86		.max_keysize		= SERPENT_MAX_KEY_SIZE,
 87		.ivsize			= SERPENT_BLOCK_SIZE,
 88		.setkey			= serpent_setkey_skcipher,
 89		.encrypt		= cbc_encrypt,
 90		.decrypt		= cbc_decrypt,
 91	},
 92};
 93
 94static int __init serpent_avx2_init(void)
 95{
 96	const char *feature_name;
 97
 98	if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 99		pr_info("AVX2 instructions are not detected.\n");
100		return -ENODEV;
101	}
102	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
103				&feature_name)) {
104		pr_info("CPU feature '%s' is not supported.\n", feature_name);
105		return -ENODEV;
106	}
107
108	return crypto_register_skciphers(serpent_algs,
109					 ARRAY_SIZE(serpent_algs));
110}
111
112static void __exit serpent_avx2_fini(void)
113{
114	crypto_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs));
115}
116
117module_init(serpent_avx2_init);
118module_exit(serpent_avx2_fini);
119
120MODULE_LICENSE("GPL");
121MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
122MODULE_ALIAS_CRYPTO("serpent");
123MODULE_ALIAS_CRYPTO("serpent-asm");