des_glue.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
  3 *
  4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
  5 */
  6
  7#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
  8
  9#include <linux/crypto.h>
 10#include <linux/init.h>
 11#include <linux/module.h>
 12#include <linux/mm.h>
 13#include <linux/types.h>
 14#include <crypto/algapi.h>
 15#include <crypto/internal/des.h>
 16#include <crypto/internal/skcipher.h>
 17
 18#include <asm/fpumacro.h>
 19#include <asm/opcodes.h>
 20#include <asm/pstate.h>
 21#include <asm/elf.h>
 22
 23struct des_sparc64_ctx {
 24	u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
 25	u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
 26};
 27
 28struct des3_ede_sparc64_ctx {
 29	u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
 30	u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
 31};
 32
 33static void encrypt_to_decrypt(u64 *d, const u64 *e)
 34{
 35	const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
 36	int i;
 37
 38	for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
 39		*d++ = *s--;
 40}
 41
 42extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
 43
 44static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
 45		       unsigned int keylen)
 46{
 47	struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
 48	int err;
 49
 50	/* Even though we have special instructions for key expansion,
 51	 * we call des_verify_key() so that we don't have to write our own
 52	 * weak key detection code.
 53	 */
 54	err = crypto_des_verify_key(tfm, key);
 55	if (err)
 56		return err;
 57
 58	des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
 59	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
 60
 61	return 0;
 62}
 63
 64static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
 65				unsigned int keylen)
 66{
 67	return des_set_key(crypto_skcipher_tfm(tfm), key, keylen);
 68}
 69
 70extern void des_sparc64_crypt(const u64 *key, const u64 *input,
 71			      u64 *output);
 72
 73static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 74{
 75	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
 76	const u64 *K = ctx->encrypt_expkey;
 77
 78	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
 79}
 80
 81static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 82{
 83	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
 84	const u64 *K = ctx->decrypt_expkey;
 85
 86	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
 87}
 88
 89extern void des_sparc64_load_keys(const u64 *key);
 90
 91extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
 92				  unsigned int len);
 93
 94static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
 95{
 96	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 97	const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
 98	struct skcipher_walk walk;
 99	unsigned int nbytes;
100	int err;
101
102	err = skcipher_walk_virt(&walk, req, true);
103	if (err)
104		return err;
105
106	if (encrypt)
107		des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
108	else
109		des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
110	while ((nbytes = walk.nbytes) != 0) {
111		des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr,
112				      round_down(nbytes, DES_BLOCK_SIZE));
113		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
114	}
115	fprs_write(0);
116	return err;
117}
118
119static int ecb_encrypt(struct skcipher_request *req)
120{
121	return __ecb_crypt(req, true);
122}
123
124static int ecb_decrypt(struct skcipher_request *req)
125{
126	return __ecb_crypt(req, false);
127}
128
129extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
130				    unsigned int len, u64 *iv);
131
132extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
133				    unsigned int len, u64 *iv);
134
135static int __cbc_crypt(struct skcipher_request *req, bool encrypt)
136{
137	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
138	const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
139	struct skcipher_walk walk;
140	unsigned int nbytes;
141	int err;
142
143	err = skcipher_walk_virt(&walk, req, true);
144	if (err)
145		return err;
146
147	if (encrypt)
148		des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
149	else
150		des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
151	while ((nbytes = walk.nbytes) != 0) {
152		if (encrypt)
153			des_sparc64_cbc_encrypt(walk.src.virt.addr,
154						walk.dst.virt.addr,
155						round_down(nbytes,
156							   DES_BLOCK_SIZE),
157						walk.iv);
158		else
159			des_sparc64_cbc_decrypt(walk.src.virt.addr,
160						walk.dst.virt.addr,
161						round_down(nbytes,
162							   DES_BLOCK_SIZE),
163						walk.iv);
164		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
165	}
166	fprs_write(0);
167	return err;
168}
169
170static int cbc_encrypt(struct skcipher_request *req)
171{
172	return __cbc_crypt(req, true);
173}
174
175static int cbc_decrypt(struct skcipher_request *req)
176{
177	return __cbc_crypt(req, false);
178}
179
180static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
181			    unsigned int keylen)
182{
183	struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
184	u64 k1[DES_EXPKEY_WORDS / 2];
185	u64 k2[DES_EXPKEY_WORDS / 2];
186	u64 k3[DES_EXPKEY_WORDS / 2];
187	int err;
188
189	err = crypto_des3_ede_verify_key(tfm, key);
190	if (err)
191		return err;
192
193	des_sparc64_key_expand((const u32 *)key, k1);
194	key += DES_KEY_SIZE;
195	des_sparc64_key_expand((const u32 *)key, k2);
196	key += DES_KEY_SIZE;
197	des_sparc64_key_expand((const u32 *)key, k3);
198
199	memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
200	encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
201	memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
202	       &k3[0], sizeof(k3));
203
204	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
205	memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
206	       &k2[0], sizeof(k2));
207	encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
208			   &k1[0]);
209
210	return 0;
211}
212
213static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
214				     unsigned int keylen)
215{
216	return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen);
217}
218
219extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
220				   u64 *output);
221
222static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
223{
224	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
225	const u64 *K = ctx->encrypt_expkey;
226
227	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
228}
229
230static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
231{
232	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
233	const u64 *K = ctx->decrypt_expkey;
234
235	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
236}
237
238extern void des3_ede_sparc64_load_keys(const u64 *key);
239
240extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
241				       u64 *output, unsigned int len);
242
243static int __ecb3_crypt(struct skcipher_request *req, bool encrypt)
244{
245	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
246	const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
247	struct skcipher_walk walk;
248	const u64 *K;
249	unsigned int nbytes;
250	int err;
251
252	err = skcipher_walk_virt(&walk, req, true);
253	if (err)
254		return err;
255
256	if (encrypt)
257		K = &ctx->encrypt_expkey[0];
258	else
259		K = &ctx->decrypt_expkey[0];
260	des3_ede_sparc64_load_keys(K);
261	while ((nbytes = walk.nbytes) != 0) {
262		des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr,
263					   walk.dst.virt.addr,
264					   round_down(nbytes, DES_BLOCK_SIZE));
265		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
266	}
267	fprs_write(0);
268	return err;
269}
270
271static int ecb3_encrypt(struct skcipher_request *req)
272{
273	return __ecb3_crypt(req, true);
274}
275
276static int ecb3_decrypt(struct skcipher_request *req)
277{
278	return __ecb3_crypt(req, false);
279}
280
281extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
282					 u64 *output, unsigned int len,
283					 u64 *iv);
284
285extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
286					 u64 *output, unsigned int len,
287					 u64 *iv);
288
289static int __cbc3_crypt(struct skcipher_request *req, bool encrypt)
290{
291	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
292	const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
293	struct skcipher_walk walk;
294	const u64 *K;
295	unsigned int nbytes;
296	int err;
297
298	err = skcipher_walk_virt(&walk, req, true);
299	if (err)
300		return err;
301
302	if (encrypt)
303		K = &ctx->encrypt_expkey[0];
304	else
305		K = &ctx->decrypt_expkey[0];
306	des3_ede_sparc64_load_keys(K);
307	while ((nbytes = walk.nbytes) != 0) {
308		if (encrypt)
309			des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr,
310						     walk.dst.virt.addr,
311						     round_down(nbytes,
312								DES_BLOCK_SIZE),
313						     walk.iv);
314		else
315			des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr,
316						     walk.dst.virt.addr,
317						     round_down(nbytes,
318								DES_BLOCK_SIZE),
319						     walk.iv);
320		err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
321	}
322	fprs_write(0);
323	return err;
324}
325
326static int cbc3_encrypt(struct skcipher_request *req)
327{
328	return __cbc3_crypt(req, true);
329}
330
331static int cbc3_decrypt(struct skcipher_request *req)
332{
333	return __cbc3_crypt(req, false);
334}
335
336static struct crypto_alg cipher_algs[] = {
337	{
338		.cra_name		= "des",
339		.cra_driver_name	= "des-sparc64",
340		.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
341		.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
342		.cra_blocksize		= DES_BLOCK_SIZE,
343		.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
344		.cra_alignmask		= 7,
345		.cra_module		= THIS_MODULE,
346		.cra_u	= {
347			.cipher	= {
348				.cia_min_keysize	= DES_KEY_SIZE,
349				.cia_max_keysize	= DES_KEY_SIZE,
350				.cia_setkey		= des_set_key,
351				.cia_encrypt		= sparc_des_encrypt,
352				.cia_decrypt		= sparc_des_decrypt
353			}
354		}
355	}, {
356		.cra_name		= "des3_ede",
357		.cra_driver_name	= "des3_ede-sparc64",
358		.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
359		.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
360		.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
361		.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
362		.cra_alignmask		= 7,
363		.cra_module		= THIS_MODULE,
364		.cra_u	= {
365			.cipher	= {
366				.cia_min_keysize	= DES3_EDE_KEY_SIZE,
367				.cia_max_keysize	= DES3_EDE_KEY_SIZE,
368				.cia_setkey		= des3_ede_set_key,
369				.cia_encrypt		= sparc_des3_ede_encrypt,
370				.cia_decrypt		= sparc_des3_ede_decrypt
371			}
372		}
373	}
374};
375
376static struct skcipher_alg skcipher_algs[] = {
377	{
378		.base.cra_name		= "ecb(des)",
379		.base.cra_driver_name	= "ecb-des-sparc64",
380		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
381		.base.cra_blocksize	= DES_BLOCK_SIZE,
382		.base.cra_ctxsize	= sizeof(struct des_sparc64_ctx),
383		.base.cra_alignmask	= 7,
384		.base.cra_module	= THIS_MODULE,
385		.min_keysize		= DES_KEY_SIZE,
386		.max_keysize		= DES_KEY_SIZE,
387		.setkey			= des_set_key_skcipher,
388		.encrypt		= ecb_encrypt,
389		.decrypt		= ecb_decrypt,
390	}, {
391		.base.cra_name		= "cbc(des)",
392		.base.cra_driver_name	= "cbc-des-sparc64",
393		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
394		.base.cra_blocksize	= DES_BLOCK_SIZE,
395		.base.cra_ctxsize	= sizeof(struct des_sparc64_ctx),
396		.base.cra_alignmask	= 7,
397		.base.cra_module	= THIS_MODULE,
398		.min_keysize		= DES_KEY_SIZE,
399		.max_keysize		= DES_KEY_SIZE,
400		.ivsize			= DES_BLOCK_SIZE,
401		.setkey			= des_set_key_skcipher,
402		.encrypt		= cbc_encrypt,
403		.decrypt		= cbc_decrypt,
404	}, {
405		.base.cra_name		= "ecb(des3_ede)",
406		.base.cra_driver_name	= "ecb-des3_ede-sparc64",
407		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
408		.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
409		.base.cra_ctxsize	= sizeof(struct des3_ede_sparc64_ctx),
410		.base.cra_alignmask	= 7,
411		.base.cra_module	= THIS_MODULE,
412		.min_keysize		= DES3_EDE_KEY_SIZE,
413		.max_keysize		= DES3_EDE_KEY_SIZE,
414		.setkey			= des3_ede_set_key_skcipher,
415		.encrypt		= ecb3_encrypt,
416		.decrypt		= ecb3_decrypt,
417	}, {
418		.base.cra_name		= "cbc(des3_ede)",
419		.base.cra_driver_name	= "cbc-des3_ede-sparc64",
420		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
421		.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
422		.base.cra_ctxsize	= sizeof(struct des3_ede_sparc64_ctx),
423		.base.cra_alignmask	= 7,
424		.base.cra_module	= THIS_MODULE,
425		.min_keysize		= DES3_EDE_KEY_SIZE,
426		.max_keysize		= DES3_EDE_KEY_SIZE,
427		.ivsize			= DES3_EDE_BLOCK_SIZE,
428		.setkey			= des3_ede_set_key_skcipher,
429		.encrypt		= cbc3_encrypt,
430		.decrypt		= cbc3_decrypt,
431	}
432};
433
434static bool __init sparc64_has_des_opcode(void)
435{
436	unsigned long cfr;
437
438	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
439		return false;
440
441	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
442	if (!(cfr & CFR_DES))
443		return false;
444
445	return true;
446}
447
448static int __init des_sparc64_mod_init(void)
449{
450	int err;
451
452	if (!sparc64_has_des_opcode()) {
453		pr_info("sparc64 des opcodes not available.\n");
454		return -ENODEV;
455	}
456	pr_info("Using sparc64 des opcodes optimized DES implementation\n");
457	err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
458	if (err)
459		return err;
460	err = crypto_register_skciphers(skcipher_algs,
461					ARRAY_SIZE(skcipher_algs));
462	if (err)
463		crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
464	return err;
465}
466
467static void __exit des_sparc64_mod_fini(void)
468{
469	crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
470	crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
471}
472
473module_init(des_sparc64_mod_init);
474module_exit(des_sparc64_mod_fini);
475
476MODULE_LICENSE("GPL");
477MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
478
479MODULE_ALIAS_CRYPTO("des");
480MODULE_ALIAS_CRYPTO("des3_ede");
481
482#include "crop_devid.c"