aes_ctr.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * AES CTR routines supporting VMX instructions on the Power 8
  4 *
  5 * Copyright (C) 2015 International Business Machines Inc.
  6 *
  7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
  8 */
  9
 10#include <asm/simd.h>
 11#include <asm/switch_to.h>
 12#include <crypto/aes.h>
 13#include <crypto/internal/simd.h>
 14#include <crypto/internal/skcipher.h>
 15#include <linux/err.h>
 16#include <linux/kernel.h>
 17#include <linux/module.h>
 18#include <linux/uaccess.h>
 19
 20#include "aesp8-ppc.h"
 21
 22struct p8_aes_ctr_ctx {
 23	struct crypto_skcipher *fallback;
 24	struct aes_key enc_key;
 25};
 26
 27static int p8_aes_ctr_init(struct crypto_skcipher *tfm)
 28{
 29	struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
 30	struct crypto_skcipher *fallback;
 31
 32	fallback = crypto_alloc_skcipher("ctr(aes)", 0,
 33					 CRYPTO_ALG_NEED_FALLBACK |
 34					 CRYPTO_ALG_ASYNC);
 35	if (IS_ERR(fallback)) {
 36		pr_err("Failed to allocate ctr(aes) fallback: %ld\n",
 37		       PTR_ERR(fallback));
 38		return PTR_ERR(fallback);
 39	}
 40
 41	crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
 42				    crypto_skcipher_reqsize(fallback));
 43	ctx->fallback = fallback;
 44	return 0;
 45}
 46
 47static void p8_aes_ctr_exit(struct crypto_skcipher *tfm)
 48{
 49	struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
 50
 51	crypto_free_skcipher(ctx->fallback);
 52}
 53
 54static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key,
 55			     unsigned int keylen)
 56{
 57	struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
 58	int ret;
 59
 60	preempt_disable();
 61	pagefault_disable();
 62	enable_kernel_vsx();
 63	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
 64	disable_kernel_vsx();
 65	pagefault_enable();
 66	preempt_enable();
 67
 68	ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
 69
 70	return ret ? -EINVAL : 0;
 71}
 72
 73static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx,
 74			     struct skcipher_walk *walk)
 75{
 76	const u8 *src = walk->src.virt.addr;
 77	u8 *ctrblk = walk->iv;
 78	u8 keystream[AES_BLOCK_SIZE];
 79	u8 *dst = walk->dst.virt.addr;
 80	unsigned int nbytes = walk->nbytes;
 81
 82	preempt_disable();
 83	pagefault_disable();
 84	enable_kernel_vsx();
 85	aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key);
 86	disable_kernel_vsx();
 87	pagefault_enable();
 88	preempt_enable();
 89
 90	crypto_xor_cpy(dst, keystream, src, nbytes);
 91	crypto_inc(ctrblk, AES_BLOCK_SIZE);
 92}
 93
 94static int p8_aes_ctr_crypt(struct skcipher_request *req)
 95{
 96	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 97	const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
 98	struct skcipher_walk walk;
 99	unsigned int nbytes;
100	int ret;
101
102	if (!crypto_simd_usable()) {
103		struct skcipher_request *subreq = skcipher_request_ctx(req);
104
105		*subreq = *req;
106		skcipher_request_set_tfm(subreq, ctx->fallback);
107		return crypto_skcipher_encrypt(subreq);
108	}
109
110	ret = skcipher_walk_virt(&walk, req, false);
111	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
112		preempt_disable();
113		pagefault_disable();
114		enable_kernel_vsx();
115		aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
116					    walk.dst.virt.addr,
117					    nbytes / AES_BLOCK_SIZE,
118					    &ctx->enc_key, walk.iv);
119		disable_kernel_vsx();
120		pagefault_enable();
121		preempt_enable();
122
123		do {
124			crypto_inc(walk.iv, AES_BLOCK_SIZE);
125		} while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE);
126
127		ret = skcipher_walk_done(&walk, nbytes);
128	}
129	if (nbytes) {
130		p8_aes_ctr_final(ctx, &walk);
131		ret = skcipher_walk_done(&walk, 0);
132	}
133	return ret;
134}
135
136struct skcipher_alg p8_aes_ctr_alg = {
137	.base.cra_name = "ctr(aes)",
138	.base.cra_driver_name = "p8_aes_ctr",
139	.base.cra_module = THIS_MODULE,
140	.base.cra_priority = 2000,
141	.base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
142	.base.cra_blocksize = 1,
143	.base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
144	.setkey = p8_aes_ctr_setkey,
145	.encrypt = p8_aes_ctr_crypt,
146	.decrypt = p8_aes_ctr_crypt,
147	.init = p8_aes_ctr_init,
148	.exit = p8_aes_ctr_exit,
149	.min_keysize = AES_MIN_KEY_SIZE,
150	.max_keysize = AES_MAX_KEY_SIZE,
151	.ivsize = AES_BLOCK_SIZE,
152	.chunksize = AES_BLOCK_SIZE,
153};