aesni-intel_glue.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Support for AES-NI and VAES instructions.  This file contains glue code.
   4 * The real AES implementations are in aesni-intel_asm.S and other .S files.
   5 *
   6 * Copyright (C) 2008, Intel Corp.
   7 *    Author: Huang Ying <ying.huang@intel.com>
   8 *
   9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10 * interface for 64-bit kernels.
  11 *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13 *             Tadeusz Struk (tadeusz.struk@intel.com)
  14 *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15 *    Copyright (c) 2010, Intel Corporation.
  16 *
  17 * Copyright 2024 Google LLC
  18 */
  19
  20#include <linux/hardirq.h>
  21#include <linux/types.h>
  22#include <linux/module.h>
  23#include <linux/err.h>
  24#include <crypto/algapi.h>
  25#include <crypto/aes.h>
  26#include <crypto/b128ops.h>
  27#include <crypto/gcm.h>
  28#include <crypto/xts.h>
  29#include <asm/cpu_device_id.h>
  30#include <asm/simd.h>
  31#include <crypto/scatterwalk.h>
  32#include <crypto/internal/aead.h>
  33#include <crypto/internal/simd.h>
  34#include <crypto/internal/skcipher.h>
  35#include <linux/jump_label.h>
  36#include <linux/workqueue.h>
  37#include <linux/spinlock.h>
  38#include <linux/static_call.h>
  39
  40
  41#define AESNI_ALIGN	16
  42#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  43#define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE - 1))
  44#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  45#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  46#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  47
  48struct aesni_xts_ctx {
  49	struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
  50	struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
  51};
  52
  53static inline void *aes_align_addr(void *addr)
  54{
  55	if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
  56		return addr;
  57	return PTR_ALIGN(addr, AESNI_ALIGN);
  58}
  59
  60asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  61			      unsigned int key_len);
  62asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
  63asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
  64asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  65			      const u8 *in, unsigned int len);
  66asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  67			      const u8 *in, unsigned int len);
  68asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  69			      const u8 *in, unsigned int len, u8 *iv);
  70asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  71			      const u8 *in, unsigned int len, u8 *iv);
  72asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  73				  const u8 *in, unsigned int len, u8 *iv);
  74asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  75				  const u8 *in, unsigned int len, u8 *iv);
  76
  77asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
  78			      const u8 *in, unsigned int len, u8 *iv);
  79
  80asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
  81			      const u8 *in, unsigned int len, u8 *iv);
  82
  83#ifdef CONFIG_X86_64
  84asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
  85			      const u8 *in, unsigned int len, u8 *iv);
  86#endif
  87
  88static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
  89{
  90	return aes_align_addr(raw_ctx);
  91}
  92
  93static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
  94{
  95	return aes_align_addr(crypto_skcipher_ctx(tfm));
  96}
  97
  98static int aes_set_key_common(struct crypto_aes_ctx *ctx,
  99			      const u8 *in_key, unsigned int key_len)
 100{
 101	int err;
 102
 103	if (!crypto_simd_usable())
 104		return aes_expandkey(ctx, in_key, key_len);
 105
 106	err = aes_check_keylen(key_len);
 107	if (err)
 108		return err;
 109
 110	kernel_fpu_begin();
 111	aesni_set_key(ctx, in_key, key_len);
 112	kernel_fpu_end();
 113	return 0;
 114}
 115
 116static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 117		       unsigned int key_len)
 118{
 119	return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
 120				  key_len);
 121}
 122
 123static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 124{
 125	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 126
 127	if (!crypto_simd_usable()) {
 128		aes_encrypt(ctx, dst, src);
 129	} else {
 130		kernel_fpu_begin();
 131		aesni_enc(ctx, dst, src);
 132		kernel_fpu_end();
 133	}
 134}
 135
 136static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 137{
 138	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 139
 140	if (!crypto_simd_usable()) {
 141		aes_decrypt(ctx, dst, src);
 142	} else {
 143		kernel_fpu_begin();
 144		aesni_dec(ctx, dst, src);
 145		kernel_fpu_end();
 146	}
 147}
 148
 149static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 150			         unsigned int len)
 151{
 152	return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
 153}
 154
 155static int ecb_encrypt(struct skcipher_request *req)
 156{
 157	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 158	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 159	struct skcipher_walk walk;
 160	unsigned int nbytes;
 161	int err;
 162
 163	err = skcipher_walk_virt(&walk, req, false);
 164
 165	while ((nbytes = walk.nbytes)) {
 166		kernel_fpu_begin();
 167		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 168			      nbytes & AES_BLOCK_MASK);
 169		kernel_fpu_end();
 170		nbytes &= AES_BLOCK_SIZE - 1;
 171		err = skcipher_walk_done(&walk, nbytes);
 172	}
 173
 174	return err;
 175}
 176
 177static int ecb_decrypt(struct skcipher_request *req)
 178{
 179	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 180	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 181	struct skcipher_walk walk;
 182	unsigned int nbytes;
 183	int err;
 184
 185	err = skcipher_walk_virt(&walk, req, false);
 186
 187	while ((nbytes = walk.nbytes)) {
 188		kernel_fpu_begin();
 189		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 190			      nbytes & AES_BLOCK_MASK);
 191		kernel_fpu_end();
 192		nbytes &= AES_BLOCK_SIZE - 1;
 193		err = skcipher_walk_done(&walk, nbytes);
 194	}
 195
 196	return err;
 197}
 198
 199static int cbc_encrypt(struct skcipher_request *req)
 200{
 201	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 202	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 203	struct skcipher_walk walk;
 204	unsigned int nbytes;
 205	int err;
 206
 207	err = skcipher_walk_virt(&walk, req, false);
 208
 209	while ((nbytes = walk.nbytes)) {
 210		kernel_fpu_begin();
 211		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 212			      nbytes & AES_BLOCK_MASK, walk.iv);
 213		kernel_fpu_end();
 214		nbytes &= AES_BLOCK_SIZE - 1;
 215		err = skcipher_walk_done(&walk, nbytes);
 216	}
 217
 218	return err;
 219}
 220
 221static int cbc_decrypt(struct skcipher_request *req)
 222{
 223	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 224	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 225	struct skcipher_walk walk;
 226	unsigned int nbytes;
 227	int err;
 228
 229	err = skcipher_walk_virt(&walk, req, false);
 230
 231	while ((nbytes = walk.nbytes)) {
 232		kernel_fpu_begin();
 233		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 234			      nbytes & AES_BLOCK_MASK, walk.iv);
 235		kernel_fpu_end();
 236		nbytes &= AES_BLOCK_SIZE - 1;
 237		err = skcipher_walk_done(&walk, nbytes);
 238	}
 239
 240	return err;
 241}
 242
 243static int cts_cbc_encrypt(struct skcipher_request *req)
 244{
 245	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 246	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 247	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 248	struct scatterlist *src = req->src, *dst = req->dst;
 249	struct scatterlist sg_src[2], sg_dst[2];
 250	struct skcipher_request subreq;
 251	struct skcipher_walk walk;
 252	int err;
 253
 254	skcipher_request_set_tfm(&subreq, tfm);
 255	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 256				      NULL, NULL);
 257
 258	if (req->cryptlen <= AES_BLOCK_SIZE) {
 259		if (req->cryptlen < AES_BLOCK_SIZE)
 260			return -EINVAL;
 261		cbc_blocks = 1;
 262	}
 263
 264	if (cbc_blocks > 0) {
 265		skcipher_request_set_crypt(&subreq, req->src, req->dst,
 266					   cbc_blocks * AES_BLOCK_SIZE,
 267					   req->iv);
 268
 269		err = cbc_encrypt(&subreq);
 270		if (err)
 271			return err;
 272
 273		if (req->cryptlen == AES_BLOCK_SIZE)
 274			return 0;
 275
 276		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 277		if (req->dst != req->src)
 278			dst = scatterwalk_ffwd(sg_dst, req->dst,
 279					       subreq.cryptlen);
 280	}
 281
 282	/* handle ciphertext stealing */
 283	skcipher_request_set_crypt(&subreq, src, dst,
 284				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 285				   req->iv);
 286
 287	err = skcipher_walk_virt(&walk, &subreq, false);
 288	if (err)
 289		return err;
 290
 291	kernel_fpu_begin();
 292	aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 293			  walk.nbytes, walk.iv);
 294	kernel_fpu_end();
 295
 296	return skcipher_walk_done(&walk, 0);
 297}
 298
 299static int cts_cbc_decrypt(struct skcipher_request *req)
 300{
 301	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 302	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 303	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 304	struct scatterlist *src = req->src, *dst = req->dst;
 305	struct scatterlist sg_src[2], sg_dst[2];
 306	struct skcipher_request subreq;
 307	struct skcipher_walk walk;
 308	int err;
 309
 310	skcipher_request_set_tfm(&subreq, tfm);
 311	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 312				      NULL, NULL);
 313
 314	if (req->cryptlen <= AES_BLOCK_SIZE) {
 315		if (req->cryptlen < AES_BLOCK_SIZE)
 316			return -EINVAL;
 317		cbc_blocks = 1;
 318	}
 319
 320	if (cbc_blocks > 0) {
 321		skcipher_request_set_crypt(&subreq, req->src, req->dst,
 322					   cbc_blocks * AES_BLOCK_SIZE,
 323					   req->iv);
 324
 325		err = cbc_decrypt(&subreq);
 326		if (err)
 327			return err;
 328
 329		if (req->cryptlen == AES_BLOCK_SIZE)
 330			return 0;
 331
 332		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 333		if (req->dst != req->src)
 334			dst = scatterwalk_ffwd(sg_dst, req->dst,
 335					       subreq.cryptlen);
 336	}
 337
 338	/* handle ciphertext stealing */
 339	skcipher_request_set_crypt(&subreq, src, dst,
 340				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 341				   req->iv);
 342
 343	err = skcipher_walk_virt(&walk, &subreq, false);
 344	if (err)
 345		return err;
 346
 347	kernel_fpu_begin();
 348	aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 349			  walk.nbytes, walk.iv);
 350	kernel_fpu_end();
 351
 352	return skcipher_walk_done(&walk, 0);
 353}
 354
 355#ifdef CONFIG_X86_64
 356/* This is the non-AVX version. */
 357static int ctr_crypt_aesni(struct skcipher_request *req)
 358{
 359	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 360	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 361	u8 keystream[AES_BLOCK_SIZE];
 362	struct skcipher_walk walk;
 363	unsigned int nbytes;
 364	int err;
 365
 366	err = skcipher_walk_virt(&walk, req, false);
 367
 368	while ((nbytes = walk.nbytes) > 0) {
 369		kernel_fpu_begin();
 370		if (nbytes & AES_BLOCK_MASK)
 371			aesni_ctr_enc(ctx, walk.dst.virt.addr,
 372				      walk.src.virt.addr,
 373				      nbytes & AES_BLOCK_MASK, walk.iv);
 374		nbytes &= ~AES_BLOCK_MASK;
 375
 376		if (walk.nbytes == walk.total && nbytes > 0) {
 377			aesni_enc(ctx, keystream, walk.iv);
 378			crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
 379				       walk.src.virt.addr + walk.nbytes - nbytes,
 380				       keystream, nbytes);
 381			crypto_inc(walk.iv, AES_BLOCK_SIZE);
 382			nbytes = 0;
 383		}
 384		kernel_fpu_end();
 385		err = skcipher_walk_done(&walk, nbytes);
 386	}
 387	return err;
 388}
 389#endif
 390
 391static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
 392			    unsigned int keylen)
 393{
 394	struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
 395	int err;
 396
 397	err = xts_verify_key(tfm, key, keylen);
 398	if (err)
 399		return err;
 400
 401	keylen /= 2;
 402
 403	/* first half of xts-key is for crypt */
 404	err = aes_set_key_common(&ctx->crypt_ctx, key, keylen);
 405	if (err)
 406		return err;
 407
 408	/* second half of xts-key is for tweak */
 409	return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen);
 410}
 411
 412typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key,
 413				    u8 iv[AES_BLOCK_SIZE]);
 414typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
 415			       const u8 *src, u8 *dst, int len,
 416			       u8 tweak[AES_BLOCK_SIZE]);
 417
 418/* This handles cases where the source and/or destination span pages. */
 419static noinline int
 420xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func)
 421{
 422	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 423	const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
 424	int tail = req->cryptlen % AES_BLOCK_SIZE;
 425	struct scatterlist sg_src[2], sg_dst[2];
 426	struct skcipher_request subreq;
 427	struct skcipher_walk walk;
 428	struct scatterlist *src, *dst;
 429	int err;
 430
 431	/*
 432	 * If the message length isn't divisible by the AES block size, then
 433	 * separate off the last full block and the partial block.  This ensures
 434	 * that they are processed in the same call to the assembly function,
 435	 * which is required for ciphertext stealing.
 436	 */
 437	if (tail) {
 438		skcipher_request_set_tfm(&subreq, tfm);
 439		skcipher_request_set_callback(&subreq,
 440					      skcipher_request_flags(req),
 441					      NULL, NULL);
 442		skcipher_request_set_crypt(&subreq, req->src, req->dst,
 443					   req->cryptlen - tail - AES_BLOCK_SIZE,
 444					   req->iv);
 445		req = &subreq;
 446	}
 447
 448	err = skcipher_walk_virt(&walk, req, false);
 449
 450	while (walk.nbytes) {
 451		kernel_fpu_begin();
 452		(*crypt_func)(&ctx->crypt_ctx,
 453			      walk.src.virt.addr, walk.dst.virt.addr,
 454			      walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv);
 455		kernel_fpu_end();
 456		err = skcipher_walk_done(&walk,
 457					 walk.nbytes & (AES_BLOCK_SIZE - 1));
 458	}
 459
 460	if (err || !tail)
 461		return err;
 462
 463	/* Do ciphertext stealing with the last full block and partial block. */
 464
 465	dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 466	if (req->dst != req->src)
 467		dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 468
 469	skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 470				   req->iv);
 471
 472	err = skcipher_walk_virt(&walk, req, false);
 473	if (err)
 474		return err;
 475
 476	kernel_fpu_begin();
 477	(*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr,
 478		      walk.nbytes, req->iv);
 479	kernel_fpu_end();
 480
 481	return skcipher_walk_done(&walk, 0);
 482}
 483
 484/* __always_inline to avoid indirect call in fastpath */
 485static __always_inline int
 486xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
 487	  xts_crypt_func crypt_func)
 488{
 489	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 490	const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
 491
 492	if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
 493		return -EINVAL;
 494
 495	kernel_fpu_begin();
 496	(*encrypt_iv)(&ctx->tweak_ctx, req->iv);
 497
 498	/*
 499	 * In practice, virtually all XTS plaintexts and ciphertexts are either
 500	 * 512 or 4096 bytes and do not use multiple scatterlist elements.  To
 501	 * optimize the performance of these cases, the below fast-path handles
 502	 * single-scatterlist-element messages as efficiently as possible.  The
 503	 * code is 64-bit specific, as it assumes no page mapping is needed.
 504	 */
 505	if (IS_ENABLED(CONFIG_X86_64) &&
 506	    likely(req->src->length >= req->cryptlen &&
 507		   req->dst->length >= req->cryptlen)) {
 508		(*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
 509			      sg_virt(req->dst), req->cryptlen, req->iv);
 510		kernel_fpu_end();
 511		return 0;
 512	}
 513	kernel_fpu_end();
 514	return xts_crypt_slowpath(req, crypt_func);
 515}
 516
 517static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
 518				 u8 iv[AES_BLOCK_SIZE])
 519{
 520	aesni_enc(tweak_key, iv, iv);
 521}
 522
 523static void aesni_xts_encrypt(const struct crypto_aes_ctx *key,
 524			      const u8 *src, u8 *dst, int len,
 525			      u8 tweak[AES_BLOCK_SIZE])
 526{
 527	aesni_xts_enc(key, dst, src, len, tweak);
 528}
 529
 530static void aesni_xts_decrypt(const struct crypto_aes_ctx *key,
 531			      const u8 *src, u8 *dst, int len,
 532			      u8 tweak[AES_BLOCK_SIZE])
 533{
 534	aesni_xts_dec(key, dst, src, len, tweak);
 535}
 536
 537static int xts_encrypt_aesni(struct skcipher_request *req)
 538{
 539	return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_encrypt);
 540}
 541
 542static int xts_decrypt_aesni(struct skcipher_request *req)
 543{
 544	return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_decrypt);
 545}
 546
 547static struct crypto_alg aesni_cipher_alg = {
 548	.cra_name		= "aes",
 549	.cra_driver_name	= "aes-aesni",
 550	.cra_priority		= 300,
 551	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
 552	.cra_blocksize		= AES_BLOCK_SIZE,
 553	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 554	.cra_module		= THIS_MODULE,
 555	.cra_u	= {
 556		.cipher	= {
 557			.cia_min_keysize	= AES_MIN_KEY_SIZE,
 558			.cia_max_keysize	= AES_MAX_KEY_SIZE,
 559			.cia_setkey		= aes_set_key,
 560			.cia_encrypt		= aesni_encrypt,
 561			.cia_decrypt		= aesni_decrypt
 562		}
 563	}
 564};
 565
 566static struct skcipher_alg aesni_skciphers[] = {
 567	{
 568		.base = {
 569			.cra_name		= "ecb(aes)",
 570			.cra_driver_name	= "ecb-aes-aesni",
 571			.cra_priority		= 400,
 572			.cra_blocksize		= AES_BLOCK_SIZE,
 573			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 574			.cra_module		= THIS_MODULE,
 575		},
 576		.min_keysize	= AES_MIN_KEY_SIZE,
 577		.max_keysize	= AES_MAX_KEY_SIZE,
 578		.setkey		= aesni_skcipher_setkey,
 579		.encrypt	= ecb_encrypt,
 580		.decrypt	= ecb_decrypt,
 581	}, {
 582		.base = {
 583			.cra_name		= "cbc(aes)",
 584			.cra_driver_name	= "cbc-aes-aesni",
 585			.cra_priority		= 400,
 586			.cra_blocksize		= AES_BLOCK_SIZE,
 587			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 588			.cra_module		= THIS_MODULE,
 589		},
 590		.min_keysize	= AES_MIN_KEY_SIZE,
 591		.max_keysize	= AES_MAX_KEY_SIZE,
 592		.ivsize		= AES_BLOCK_SIZE,
 593		.setkey		= aesni_skcipher_setkey,
 594		.encrypt	= cbc_encrypt,
 595		.decrypt	= cbc_decrypt,
 596	}, {
 597		.base = {
 598			.cra_name		= "cts(cbc(aes))",
 599			.cra_driver_name	= "cts-cbc-aes-aesni",
 600			.cra_priority		= 400,
 601			.cra_blocksize		= AES_BLOCK_SIZE,
 602			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 603			.cra_module		= THIS_MODULE,
 604		},
 605		.min_keysize	= AES_MIN_KEY_SIZE,
 606		.max_keysize	= AES_MAX_KEY_SIZE,
 607		.ivsize		= AES_BLOCK_SIZE,
 608		.walksize	= 2 * AES_BLOCK_SIZE,
 609		.setkey		= aesni_skcipher_setkey,
 610		.encrypt	= cts_cbc_encrypt,
 611		.decrypt	= cts_cbc_decrypt,
 612#ifdef CONFIG_X86_64
 613	}, {
 614		.base = {
 615			.cra_name		= "ctr(aes)",
 616			.cra_driver_name	= "ctr-aes-aesni",
 617			.cra_priority		= 400,
 618			.cra_blocksize		= 1,
 619			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 620			.cra_module		= THIS_MODULE,
 621		},
 622		.min_keysize	= AES_MIN_KEY_SIZE,
 623		.max_keysize	= AES_MAX_KEY_SIZE,
 624		.ivsize		= AES_BLOCK_SIZE,
 625		.chunksize	= AES_BLOCK_SIZE,
 626		.setkey		= aesni_skcipher_setkey,
 627		.encrypt	= ctr_crypt_aesni,
 628		.decrypt	= ctr_crypt_aesni,
 629#endif
 630	}, {
 631		.base = {
 632			.cra_name		= "xts(aes)",
 633			.cra_driver_name	= "xts-aes-aesni",
 634			.cra_priority		= 401,
 635			.cra_blocksize		= AES_BLOCK_SIZE,
 636			.cra_ctxsize		= XTS_AES_CTX_SIZE,
 637			.cra_module		= THIS_MODULE,
 638		},
 639		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
 640		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
 641		.ivsize		= AES_BLOCK_SIZE,
 642		.walksize	= 2 * AES_BLOCK_SIZE,
 643		.setkey		= xts_setkey_aesni,
 644		.encrypt	= xts_encrypt_aesni,
 645		.decrypt	= xts_decrypt_aesni,
 646	}
 647};
 648
 649#ifdef CONFIG_X86_64
 650asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
 651				   u8 iv[AES_BLOCK_SIZE]);
 652
 653/* __always_inline to avoid indirect call */
 654static __always_inline int
 655ctr_crypt(struct skcipher_request *req,
 656	  void (*ctr64_func)(const struct crypto_aes_ctx *key,
 657			     const u8 *src, u8 *dst, int len,
 658			     const u64 le_ctr[2]))
 659{
 660	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 661	const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
 662	unsigned int nbytes, p1_nbytes, nblocks;
 663	struct skcipher_walk walk;
 664	u64 le_ctr[2];
 665	u64 ctr64;
 666	int err;
 667
 668	ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]);
 669	le_ctr[1] = get_unaligned_be64(&req->iv[0]);
 670
 671	err = skcipher_walk_virt(&walk, req, false);
 672
 673	while ((nbytes = walk.nbytes) != 0) {
 674		if (nbytes < walk.total) {
 675			/* Not the end yet, so keep the length block-aligned. */
 676			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
 677			nblocks = nbytes / AES_BLOCK_SIZE;
 678		} else {
 679			/* It's the end, so include any final partial block. */
 680			nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
 681		}
 682		ctr64 += nblocks;
 683
 684		kernel_fpu_begin();
 685		if (likely(ctr64 >= nblocks)) {
 686			/* The low 64 bits of the counter won't overflow. */
 687			(*ctr64_func)(key, walk.src.virt.addr,
 688				      walk.dst.virt.addr, nbytes, le_ctr);
 689		} else {
 690			/*
 691			 * The low 64 bits of the counter will overflow.  The
 692			 * assembly doesn't handle this case, so split the
 693			 * operation into two at the point where the overflow
 694			 * will occur.  After the first part, add the carry bit.
 695			 */
 696			p1_nbytes = min(nbytes, (nblocks - ctr64) * AES_BLOCK_SIZE);
 697			(*ctr64_func)(key, walk.src.virt.addr,
 698				      walk.dst.virt.addr, p1_nbytes, le_ctr);
 699			le_ctr[0] = 0;
 700			le_ctr[1]++;
 701			(*ctr64_func)(key, walk.src.virt.addr + p1_nbytes,
 702				      walk.dst.virt.addr + p1_nbytes,
 703				      nbytes - p1_nbytes, le_ctr);
 704		}
 705		kernel_fpu_end();
 706		le_ctr[0] = ctr64;
 707
 708		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 709	}
 710
 711	put_unaligned_be64(ctr64, &req->iv[8]);
 712	put_unaligned_be64(le_ctr[1], &req->iv[0]);
 713
 714	return err;
 715}
 716
 717/* __always_inline to avoid indirect call */
 718static __always_inline int
 719xctr_crypt(struct skcipher_request *req,
 720	   void (*xctr_func)(const struct crypto_aes_ctx *key,
 721			     const u8 *src, u8 *dst, int len,
 722			     const u8 iv[AES_BLOCK_SIZE], u64 ctr))
 723{
 724	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 725	const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
 726	struct skcipher_walk walk;
 727	unsigned int nbytes;
 728	u64 ctr = 1;
 729	int err;
 730
 731	err = skcipher_walk_virt(&walk, req, false);
 732	while ((nbytes = walk.nbytes) != 0) {
 733		if (nbytes < walk.total)
 734			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
 735
 736		kernel_fpu_begin();
 737		(*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr,
 738			     nbytes, req->iv, ctr);
 739		kernel_fpu_end();
 740
 741		ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
 742		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 743	}
 744	return err;
 745}
 746
 747#define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority)	       \
 748									       \
 749asmlinkage void								       \
 750aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src,      \
 751			 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]);	       \
 752asmlinkage void								       \
 753aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src,      \
 754			 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]);	       \
 755									       \
 756static int xts_encrypt_##suffix(struct skcipher_request *req)		       \
 757{									       \
 758	return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix);   \
 759}									       \
 760									       \
 761static int xts_decrypt_##suffix(struct skcipher_request *req)		       \
 762{									       \
 763	return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix);   \
 764}									       \
 765									       \
 766asmlinkage void								       \
 767aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,		       \
 768			 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\
 769									       \
 770static int ctr_crypt_##suffix(struct skcipher_request *req)		       \
 771{									       \
 772	return ctr_crypt(req, aes_ctr64_crypt_##suffix);		       \
 773}									       \
 774									       \
 775asmlinkage void								       \
 776aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,		       \
 777			const u8 *src, u8 *dst, int len,		       \
 778			const u8 iv[AES_BLOCK_SIZE], u64 ctr);		       \
 779									       \
 780static int xctr_crypt_##suffix(struct skcipher_request *req)		       \
 781{									       \
 782	return xctr_crypt(req, aes_xctr_crypt_##suffix);		       \
 783}									       \
 784									       \
 785static struct skcipher_alg skcipher_algs_##suffix[] = {{		       \
 786	.base.cra_name		= "xts(aes)",				       \
 787	.base.cra_driver_name	= "xts-aes-" driver_name_suffix,	       \
 788	.base.cra_priority	= priority,				       \
 789	.base.cra_blocksize	= AES_BLOCK_SIZE,			       \
 790	.base.cra_ctxsize	= XTS_AES_CTX_SIZE,			       \
 791	.base.cra_module	= THIS_MODULE,				       \
 792	.min_keysize		= 2 * AES_MIN_KEY_SIZE,			       \
 793	.max_keysize		= 2 * AES_MAX_KEY_SIZE,			       \
 794	.ivsize			= AES_BLOCK_SIZE,			       \
 795	.walksize		= 2 * AES_BLOCK_SIZE,			       \
 796	.setkey			= xts_setkey_aesni,			       \
 797	.encrypt		= xts_encrypt_##suffix,			       \
 798	.decrypt		= xts_decrypt_##suffix,			       \
 799}, {									       \
 800	.base.cra_name		= "ctr(aes)",				       \
 801	.base.cra_driver_name	= "ctr-aes-" driver_name_suffix,	       \
 802	.base.cra_priority	= priority,				       \
 803	.base.cra_blocksize	= 1,					       \
 804	.base.cra_ctxsize	= CRYPTO_AES_CTX_SIZE,			       \
 805	.base.cra_module	= THIS_MODULE,				       \
 806	.min_keysize		= AES_MIN_KEY_SIZE,			       \
 807	.max_keysize		= AES_MAX_KEY_SIZE,			       \
 808	.ivsize			= AES_BLOCK_SIZE,			       \
 809	.chunksize		= AES_BLOCK_SIZE,			       \
 810	.setkey			= aesni_skcipher_setkey,		       \
 811	.encrypt		= ctr_crypt_##suffix,			       \
 812	.decrypt		= ctr_crypt_##suffix,			       \
 813}, {									       \
 814	.base.cra_name		= "xctr(aes)",				       \
 815	.base.cra_driver_name	= "xctr-aes-" driver_name_suffix,	       \
 816	.base.cra_priority	= priority,				       \
 817	.base.cra_blocksize	= 1,					       \
 818	.base.cra_ctxsize	= CRYPTO_AES_CTX_SIZE,			       \
 819	.base.cra_module	= THIS_MODULE,				       \
 820	.min_keysize		= AES_MIN_KEY_SIZE,			       \
 821	.max_keysize		= AES_MAX_KEY_SIZE,			       \
 822	.ivsize			= AES_BLOCK_SIZE,			       \
 823	.chunksize		= AES_BLOCK_SIZE,			       \
 824	.setkey			= aesni_skcipher_setkey,		       \
 825	.encrypt		= xctr_crypt_##suffix,			       \
 826	.decrypt		= xctr_crypt_##suffix,			       \
 827}}
 828
 829DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
 830DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600);
 831DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
 832
 833/* The common part of the x86_64 AES-GCM key struct */
 834struct aes_gcm_key {
 835	/* Expanded AES key and the AES key length in bytes */
 836	struct crypto_aes_ctx aes_key;
 837
 838	/* RFC4106 nonce (used only by the rfc4106 algorithms) */
 839	u32 rfc4106_nonce;
 840};
 841
 842/* Key struct used by the AES-NI implementations of AES-GCM */
 843struct aes_gcm_key_aesni {
 844	/*
 845	 * Common part of the key.  The assembly code requires 16-byte alignment
 846	 * for the round keys; we get this by them being located at the start of
 847	 * the struct and the whole struct being 16-byte aligned.
 848	 */
 849	struct aes_gcm_key base;
 850
 851	/*
 852	 * Powers of the hash key H^8 through H^1.  These are 128-bit values.
 853	 * They all have an extra factor of x^-1 and are byte-reversed.  16-byte
 854	 * alignment is required by the assembly code.
 855	 */
 856	u64 h_powers[8][2] __aligned(16);
 857
 858	/*
 859	 * h_powers_xored[i] contains the two 64-bit halves of h_powers[i] XOR'd
 860	 * together.  It's used for Karatsuba multiplication.  16-byte alignment
 861	 * is required by the assembly code.
 862	 */
 863	u64 h_powers_xored[8] __aligned(16);
 864
 865	/*
 866	 * H^1 times x^64 (and also the usual extra factor of x^-1).  16-byte
 867	 * alignment is required by the assembly code.
 868	 */
 869	u64 h_times_x64[2] __aligned(16);
 870};
 871#define AES_GCM_KEY_AESNI(key)	\
 872	container_of((key), struct aes_gcm_key_aesni, base)
 873#define AES_GCM_KEY_AESNI_SIZE	\
 874	(sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1)))
 875
 876/* Key struct used by the VAES + AVX2 implementation of AES-GCM */
 877struct aes_gcm_key_vaes_avx2 {
 878	/*
 879	 * Common part of the key.  The assembly code prefers 16-byte alignment
 880	 * for the round keys; we get this by them being located at the start of
 881	 * the struct and the whole struct being 32-byte aligned.
 882	 */
 883	struct aes_gcm_key base;
 884
 885	/*
 886	 * Powers of the hash key H^8 through H^1.  These are 128-bit values.
 887	 * They all have an extra factor of x^-1 and are byte-reversed.
 888	 * The assembly code prefers 32-byte alignment for this.
 889	 */
 890	u64 h_powers[8][2] __aligned(32);
 891
 892	/*
 893	 * Each entry in this array contains the two halves of an entry of
 894	 * h_powers XOR'd together, in the following order:
 895	 * H^8,H^6,H^7,H^5,H^4,H^2,H^3,H^1 i.e. indices 0,2,1,3,4,6,5,7.
 896	 * This is used for Karatsuba multiplication.
 897	 */
 898	u64 h_powers_xored[8];
 899};
 900
 901#define AES_GCM_KEY_VAES_AVX2(key) \
 902	container_of((key), struct aes_gcm_key_vaes_avx2, base)
 903#define AES_GCM_KEY_VAES_AVX2_SIZE \
 904	(sizeof(struct aes_gcm_key_vaes_avx2) + (31 & ~(CRYPTO_MINALIGN - 1)))
 905
 906/* Key struct used by the VAES + AVX512 implementation of AES-GCM */
 907struct aes_gcm_key_vaes_avx512 {
 908	/*
 909	 * Common part of the key.  The assembly code prefers 16-byte alignment
 910	 * for the round keys; we get this by them being located at the start of
 911	 * the struct and the whole struct being 64-byte aligned.
 912	 */
 913	struct aes_gcm_key base;
 914
 915	/*
 916	 * Powers of the hash key H^16 through H^1.  These are 128-bit values.
 917	 * They all have an extra factor of x^-1 and are byte-reversed.  This
 918	 * array is aligned to a 64-byte boundary to make it naturally aligned
 919	 * for 512-bit loads, which can improve performance.  (The assembly code
 920	 * doesn't *need* the alignment; this is just an optimization.)
 921	 */
 922	u64 h_powers[16][2] __aligned(64);
 923
 924	/* Three padding blocks required by the assembly code */
 925	u64 padding[3][2];
 926};
 927#define AES_GCM_KEY_VAES_AVX512(key) \
 928	container_of((key), struct aes_gcm_key_vaes_avx512, base)
 929#define AES_GCM_KEY_VAES_AVX512_SIZE \
 930	(sizeof(struct aes_gcm_key_vaes_avx512) + (63 & ~(CRYPTO_MINALIGN - 1)))
 931
 932/*
 933 * These flags are passed to the AES-GCM helper functions to specify the
 934 * specific version of AES-GCM (RFC4106 or not), whether it's encryption or
 935 * decryption, and which assembly functions should be called.  Assembly
 936 * functions are selected using flags instead of function pointers to avoid
 937 * indirect calls (which are very expensive on x86) regardless of inlining.
 938 */
 939#define FLAG_RFC4106	BIT(0)
 940#define FLAG_ENC	BIT(1)
 941#define FLAG_AVX	BIT(2)
 942#define FLAG_VAES_AVX2	BIT(3)
 943#define FLAG_VAES_AVX512 BIT(4)
 944
 945static inline struct aes_gcm_key *
 946aes_gcm_key_get(struct crypto_aead *tfm, int flags)
 947{
 948	if (flags & FLAG_VAES_AVX512)
 949		return PTR_ALIGN(crypto_aead_ctx(tfm), 64);
 950	else if (flags & FLAG_VAES_AVX2)
 951		return PTR_ALIGN(crypto_aead_ctx(tfm), 32);
 952	else
 953		return PTR_ALIGN(crypto_aead_ctx(tfm), 16);
 954}
 955
 956asmlinkage void
 957aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key);
 958asmlinkage void
 959aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key);
 960asmlinkage void
 961aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key);
 962asmlinkage void
 963aes_gcm_precompute_vaes_avx512(struct aes_gcm_key_vaes_avx512 *key);
 964
 965static void aes_gcm_precompute(struct aes_gcm_key *key, int flags)
 966{
 967	if (flags & FLAG_VAES_AVX512)
 968		aes_gcm_precompute_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key));
 969	else if (flags & FLAG_VAES_AVX2)
 970		aes_gcm_precompute_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key));
 971	else if (flags & FLAG_AVX)
 972		aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key));
 973	else
 974		aes_gcm_precompute_aesni(AES_GCM_KEY_AESNI(key));
 975}
 976
 977asmlinkage void
 978aes_gcm_aad_update_aesni(const struct aes_gcm_key_aesni *key,
 979			 u8 ghash_acc[16], const u8 *aad, int aadlen);
 980asmlinkage void
 981aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key,
 982			     u8 ghash_acc[16], const u8 *aad, int aadlen);
 983asmlinkage void
 984aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
 985			     u8 ghash_acc[16], const u8 *aad, int aadlen);
 986asmlinkage void
 987aes_gcm_aad_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
 988			       u8 ghash_acc[16], const u8 *aad, int aadlen);
 989
 990static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16],
 991			       const u8 *aad, int aadlen, int flags)
 992{
 993	if (flags & FLAG_VAES_AVX512)
 994		aes_gcm_aad_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
 995					       ghash_acc, aad, aadlen);
 996	else if (flags & FLAG_VAES_AVX2)
 997		aes_gcm_aad_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
 998					     ghash_acc, aad, aadlen);
 999	else if (flags & FLAG_AVX)
1000		aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc,
1001					     aad, aadlen);
1002	else
1003		aes_gcm_aad_update_aesni(AES_GCM_KEY_AESNI(key), ghash_acc,
1004					 aad, aadlen);
1005}
1006
1007asmlinkage void
1008aes_gcm_enc_update_aesni(const struct aes_gcm_key_aesni *key,
1009			 const u32 le_ctr[4], u8 ghash_acc[16],
1010			 const u8 *src, u8 *dst, int datalen);
1011asmlinkage void
1012aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key,
1013			     const u32 le_ctr[4], u8 ghash_acc[16],
1014			     const u8 *src, u8 *dst, int datalen);
1015asmlinkage void
1016aes_gcm_enc_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1017			     const u32 le_ctr[4], u8 ghash_acc[16],
1018			     const u8 *src, u8 *dst, int datalen);
1019asmlinkage void
1020aes_gcm_enc_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1021			       const u32 le_ctr[4], u8 ghash_acc[16],
1022			       const u8 *src, u8 *dst, int datalen);
1023
1024asmlinkage void
1025aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key,
1026			 const u32 le_ctr[4], u8 ghash_acc[16],
1027			 const u8 *src, u8 *dst, int datalen);
1028asmlinkage void
1029aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key,
1030			     const u32 le_ctr[4], u8 ghash_acc[16],
1031			     const u8 *src, u8 *dst, int datalen);
1032asmlinkage void
1033aes_gcm_dec_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1034			     const u32 le_ctr[4], u8 ghash_acc[16],
1035			     const u8 *src, u8 *dst, int datalen);
1036asmlinkage void
1037aes_gcm_dec_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1038			       const u32 le_ctr[4], u8 ghash_acc[16],
1039			       const u8 *src, u8 *dst, int datalen);
1040
1041/* __always_inline to optimize out the branches based on @flags */
1042static __always_inline void
1043aes_gcm_update(const struct aes_gcm_key *key,
1044	       const u32 le_ctr[4], u8 ghash_acc[16],
1045	       const u8 *src, u8 *dst, int datalen, int flags)
1046{
1047	if (flags & FLAG_ENC) {
1048		if (flags & FLAG_VAES_AVX512)
1049			aes_gcm_enc_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1050						       le_ctr, ghash_acc,
1051						       src, dst, datalen);
1052		else if (flags & FLAG_VAES_AVX2)
1053			aes_gcm_enc_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1054						     le_ctr, ghash_acc,
1055						     src, dst, datalen);
1056		else if (flags & FLAG_AVX)
1057			aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key),
1058						     le_ctr, ghash_acc,
1059						     src, dst, datalen);
1060		else
1061			aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr,
1062						 ghash_acc, src, dst, datalen);
1063	} else {
1064		if (flags & FLAG_VAES_AVX512)
1065			aes_gcm_dec_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1066						       le_ctr, ghash_acc,
1067						       src, dst, datalen);
1068		else if (flags & FLAG_VAES_AVX2)
1069			aes_gcm_dec_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1070						     le_ctr, ghash_acc,
1071						     src, dst, datalen);
1072		else if (flags & FLAG_AVX)
1073			aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key),
1074						     le_ctr, ghash_acc,
1075						     src, dst, datalen);
1076		else
1077			aes_gcm_dec_update_aesni(AES_GCM_KEY_AESNI(key),
1078						 le_ctr, ghash_acc,
1079						 src, dst, datalen);
1080	}
1081}
1082
1083asmlinkage void
1084aes_gcm_enc_final_aesni(const struct aes_gcm_key_aesni *key,
1085			const u32 le_ctr[4], u8 ghash_acc[16],
1086			u64 total_aadlen, u64 total_datalen);
1087asmlinkage void
1088aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key,
1089			    const u32 le_ctr[4], u8 ghash_acc[16],
1090			    u64 total_aadlen, u64 total_datalen);
1091asmlinkage void
1092aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1093			    const u32 le_ctr[4], u8 ghash_acc[16],
1094			    u64 total_aadlen, u64 total_datalen);
1095asmlinkage void
1096aes_gcm_enc_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1097			      const u32 le_ctr[4], u8 ghash_acc[16],
1098			      u64 total_aadlen, u64 total_datalen);
1099
1100/* __always_inline to optimize out the branches based on @flags */
1101static __always_inline void
1102aes_gcm_enc_final(const struct aes_gcm_key *key,
1103		  const u32 le_ctr[4], u8 ghash_acc[16],
1104		  u64 total_aadlen, u64 total_datalen, int flags)
1105{
1106	if (flags & FLAG_VAES_AVX512)
1107		aes_gcm_enc_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1108					      le_ctr, ghash_acc,
1109					      total_aadlen, total_datalen);
1110	else if (flags & FLAG_VAES_AVX2)
1111		aes_gcm_enc_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1112					    le_ctr, ghash_acc,
1113					    total_aadlen, total_datalen);
1114	else if (flags & FLAG_AVX)
1115		aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key),
1116					    le_ctr, ghash_acc,
1117					    total_aadlen, total_datalen);
1118	else
1119		aes_gcm_enc_final_aesni(AES_GCM_KEY_AESNI(key),
1120					le_ctr, ghash_acc,
1121					total_aadlen, total_datalen);
1122}
1123
1124asmlinkage bool __must_check
1125aes_gcm_dec_final_aesni(const struct aes_gcm_key_aesni *key,
1126			const u32 le_ctr[4], const u8 ghash_acc[16],
1127			u64 total_aadlen, u64 total_datalen,
1128			const u8 tag[16], int taglen);
1129asmlinkage bool __must_check
1130aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key,
1131			    const u32 le_ctr[4], const u8 ghash_acc[16],
1132			    u64 total_aadlen, u64 total_datalen,
1133			    const u8 tag[16], int taglen);
1134asmlinkage bool __must_check
1135aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1136			    const u32 le_ctr[4], const u8 ghash_acc[16],
1137			    u64 total_aadlen, u64 total_datalen,
1138			    const u8 tag[16], int taglen);
1139asmlinkage bool __must_check
1140aes_gcm_dec_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1141			      const u32 le_ctr[4], const u8 ghash_acc[16],
1142			      u64 total_aadlen, u64 total_datalen,
1143			      const u8 tag[16], int taglen);
1144
1145/* __always_inline to optimize out the branches based on @flags */
1146static __always_inline bool __must_check
1147aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4],
1148		  u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen,
1149		  u8 tag[16], int taglen, int flags)
1150{
1151	if (flags & FLAG_VAES_AVX512)
1152		return aes_gcm_dec_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1153						     le_ctr, ghash_acc,
1154						     total_aadlen, total_datalen,
1155						     tag, taglen);
1156	else if (flags & FLAG_VAES_AVX2)
1157		return aes_gcm_dec_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1158						   le_ctr, ghash_acc,
1159						   total_aadlen, total_datalen,
1160						   tag, taglen);
1161	else if (flags & FLAG_AVX)
1162		return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key),
1163						   le_ctr, ghash_acc,
1164						   total_aadlen, total_datalen,
1165						   tag, taglen);
1166	else
1167		return aes_gcm_dec_final_aesni(AES_GCM_KEY_AESNI(key),
1168					       le_ctr, ghash_acc,
1169					       total_aadlen, total_datalen,
1170					       tag, taglen);
1171}
1172
1173/*
1174 * This is the Integrity Check Value (aka the authentication tag) length and can
1175 * be 8, 12 or 16 bytes long.
1176 */
1177static int common_rfc4106_set_authsize(struct crypto_aead *aead,
1178				       unsigned int authsize)
1179{
1180	switch (authsize) {
1181	case 8:
1182	case 12:
1183	case 16:
1184		break;
1185	default:
1186		return -EINVAL;
1187	}
1188
1189	return 0;
1190}
1191
1192static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
1193				       unsigned int authsize)
1194{
1195	switch (authsize) {
1196	case 4:
1197	case 8:
1198	case 12:
1199	case 13:
1200	case 14:
1201	case 15:
1202	case 16:
1203		break;
1204	default:
1205		return -EINVAL;
1206	}
1207
1208	return 0;
1209}
1210
1211/*
1212 * This is the setkey function for the x86_64 implementations of AES-GCM.  It
1213 * saves the RFC4106 nonce if applicable, expands the AES key, and precomputes
1214 * powers of the hash key.
1215 *
1216 * To comply with the crypto_aead API, this has to be usable in no-SIMD context.
1217 * For that reason, this function includes a portable C implementation of the
1218 * needed logic.  However, the portable C implementation is very slow, taking
1219 * about the same time as encrypting 37 KB of data.  To be ready for users that
1220 * may set a key even somewhat frequently, we therefore also include a SIMD
1221 * assembly implementation, expanding the AES key using AES-NI and precomputing
1222 * the hash key powers using PCLMULQDQ or VPCLMULQDQ.
1223 */
1224static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
1225		      unsigned int keylen, int flags)
1226{
1227	struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags);
1228	int err;
1229
1230	if (flags & FLAG_RFC4106) {
1231		if (keylen < 4)
1232			return -EINVAL;
1233		keylen -= 4;
1234		key->rfc4106_nonce = get_unaligned_be32(raw_key + keylen);
1235	}
1236
1237	/* The assembly code assumes the following offsets. */
1238	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0);
1239	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480);
1240	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496);
1241	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624);
1242	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688);
1243	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_enc) != 0);
1244	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_length) != 480);
1245	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) != 512);
1246	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) != 640);
1247	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_enc) != 0);
1248	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_length) != 480);
1249	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) != 512);
1250	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, padding) != 768);
1251
1252	if (likely(crypto_simd_usable())) {
1253		err = aes_check_keylen(keylen);
1254		if (err)
1255			return err;
1256		kernel_fpu_begin();
1257		aesni_set_key(&key->aes_key, raw_key, keylen);
1258		aes_gcm_precompute(key, flags);
1259		kernel_fpu_end();
1260	} else {
1261		static const u8 x_to_the_minus1[16] __aligned(__alignof__(be128)) = {
1262			[0] = 0xc2, [15] = 1
1263		};
1264		static const u8 x_to_the_63[16] __aligned(__alignof__(be128)) = {
1265			[7] = 1,
1266		};
1267		be128 h1 = {};
1268		be128 h;
1269		int i;
1270
1271		err = aes_expandkey(&key->aes_key, raw_key, keylen);
1272		if (err)
1273			return err;
1274
1275		/* Encrypt the all-zeroes block to get the hash key H^1 */
1276		aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1);
1277
1278		/* Compute H^1 * x^-1 */
1279		h = h1;
1280		gf128mul_lle(&h, (const be128 *)x_to_the_minus1);
1281
1282		/* Compute the needed key powers */
1283		if (flags & FLAG_VAES_AVX512) {
1284			struct aes_gcm_key_vaes_avx512 *k =
1285				AES_GCM_KEY_VAES_AVX512(key);
1286
1287			for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1288				k->h_powers[i][0] = be64_to_cpu(h.b);
1289				k->h_powers[i][1] = be64_to_cpu(h.a);
1290				gf128mul_lle(&h, &h1);
1291			}
1292			memset(k->padding, 0, sizeof(k->padding));
1293		} else if (flags & FLAG_VAES_AVX2) {
1294			struct aes_gcm_key_vaes_avx2 *k =
1295				AES_GCM_KEY_VAES_AVX2(key);
1296			static const u8 indices[8] = { 0, 2, 1, 3, 4, 6, 5, 7 };
1297
1298			for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1299				k->h_powers[i][0] = be64_to_cpu(h.b);
1300				k->h_powers[i][1] = be64_to_cpu(h.a);
1301				gf128mul_lle(&h, &h1);
1302			}
1303			for (i = 0; i < ARRAY_SIZE(k->h_powers_xored); i++) {
1304				int j = indices[i];
1305
1306				k->h_powers_xored[i] = k->h_powers[j][0] ^
1307						       k->h_powers[j][1];
1308			}
1309		} else {
1310			struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key);
1311
1312			for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1313				k->h_powers[i][0] = be64_to_cpu(h.b);
1314				k->h_powers[i][1] = be64_to_cpu(h.a);
1315				k->h_powers_xored[i] = k->h_powers[i][0] ^
1316						       k->h_powers[i][1];
1317				gf128mul_lle(&h, &h1);
1318			}
1319			gf128mul_lle(&h1, (const be128 *)x_to_the_63);
1320			k->h_times_x64[0] = be64_to_cpu(h1.b);
1321			k->h_times_x64[1] = be64_to_cpu(h1.a);
1322		}
1323	}
1324	return 0;
1325}
1326
1327/*
1328 * Initialize @ghash_acc, then pass all @assoclen bytes of associated data
1329 * (a.k.a. additional authenticated data) from @sg_src through the GHASH update
1330 * assembly function.  kernel_fpu_begin() must have already been called.
1331 */
1332static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
1333			      struct scatterlist *sg_src, unsigned int assoclen,
1334			      int flags)
1335{
1336	struct scatter_walk walk;
1337	/*
1338	 * The assembly function requires that the length of any non-last
1339	 * segment of associated data be a multiple of 16 bytes, so this
1340	 * function does the buffering needed to achieve that.
1341	 */
1342	unsigned int pos = 0;
1343	u8 buf[16];
1344
1345	memset(ghash_acc, 0, 16);
1346	scatterwalk_start(&walk, sg_src);
1347
1348	while (assoclen) {
1349		unsigned int orig_len_this_step = scatterwalk_next(
1350			&walk, assoclen);
1351		unsigned int len_this_step = orig_len_this_step;
1352		unsigned int len;
1353		const u8 *src = walk.addr;
1354
1355		if (unlikely(pos)) {
1356			len = min(len_this_step, 16 - pos);
1357			memcpy(&buf[pos], src, len);
1358			pos += len;
1359			src += len;
1360			len_this_step -= len;
1361			if (pos < 16)
1362				goto next;
1363			aes_gcm_aad_update(key, ghash_acc, buf, 16, flags);
1364			pos = 0;
1365		}
1366		len = len_this_step;
1367		if (unlikely(assoclen)) /* Not the last segment yet? */
1368			len = round_down(len, 16);
1369		aes_gcm_aad_update(key, ghash_acc, src, len, flags);
1370		src += len;
1371		len_this_step -= len;
1372		if (unlikely(len_this_step)) {
1373			memcpy(buf, src, len_this_step);
1374			pos = len_this_step;
1375		}
1376next:
1377		scatterwalk_done_src(&walk, orig_len_this_step);
1378		if (need_resched()) {
1379			kernel_fpu_end();
1380			kernel_fpu_begin();
1381		}
1382		assoclen -= orig_len_this_step;
1383	}
1384	if (unlikely(pos))
1385		aes_gcm_aad_update(key, ghash_acc, buf, pos, flags);
1386}
1387
1388
1389/* __always_inline to optimize out the branches based on @flags */
1390static __always_inline int
1391gcm_crypt(struct aead_request *req, int flags)
1392{
1393	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1394	const struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags);
1395	unsigned int assoclen = req->assoclen;
1396	struct skcipher_walk walk;
1397	unsigned int nbytes;
1398	u8 ghash_acc[16]; /* GHASH accumulator */
1399	u32 le_ctr[4]; /* Counter in little-endian format */
1400	int taglen;
1401	int err;
1402
1403	/* Initialize the counter and determine the associated data length. */
1404	le_ctr[0] = 2;
1405	if (flags & FLAG_RFC4106) {
1406		if (unlikely(assoclen != 16 && assoclen != 20))
1407			return -EINVAL;
1408		assoclen -= 8;
1409		le_ctr[1] = get_unaligned_be32(req->iv + 4);
1410		le_ctr[2] = get_unaligned_be32(req->iv + 0);
1411		le_ctr[3] = key->rfc4106_nonce; /* already byte-swapped */
1412	} else {
1413		le_ctr[1] = get_unaligned_be32(req->iv + 8);
1414		le_ctr[2] = get_unaligned_be32(req->iv + 4);
1415		le_ctr[3] = get_unaligned_be32(req->iv + 0);
1416	}
1417
1418	/* Begin walking through the plaintext or ciphertext. */
1419	if (flags & FLAG_ENC)
1420		err = skcipher_walk_aead_encrypt(&walk, req, false);
1421	else
1422		err = skcipher_walk_aead_decrypt(&walk, req, false);
1423	if (err)
1424		return err;
1425
1426	/*
1427	 * Since the AES-GCM assembly code requires that at least three assembly
1428	 * functions be called to process any message (this is needed to support
1429	 * incremental updates cleanly), to reduce overhead we try to do all
1430	 * three calls in the same kernel FPU section if possible.  We close the
1431	 * section and start a new one if there are multiple data segments or if
1432	 * rescheduling is needed while processing the associated data.
1433	 */
1434	kernel_fpu_begin();
1435
1436	/* Pass the associated data through GHASH. */
1437	gcm_process_assoc(key, ghash_acc, req->src, assoclen, flags);
1438
1439	/* En/decrypt the data and pass the ciphertext through GHASH. */
1440	while (unlikely((nbytes = walk.nbytes) < walk.total)) {
1441		/*
1442		 * Non-last segment.  In this case, the assembly function
1443		 * requires that the length be a multiple of 16 (AES_BLOCK_SIZE)
1444		 * bytes.  The needed buffering of up to 16 bytes is handled by
1445		 * the skcipher_walk.  Here we just need to round down to a
1446		 * multiple of 16.
1447		 */
1448		nbytes = round_down(nbytes, AES_BLOCK_SIZE);
1449		aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr,
1450			       walk.dst.virt.addr, nbytes, flags);
1451		le_ctr[0] += nbytes / AES_BLOCK_SIZE;
1452		kernel_fpu_end();
1453		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
1454		if (err)
1455			return err;
1456		kernel_fpu_begin();
1457	}
1458	/* Last segment: process all remaining data. */
1459	aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr,
1460		       walk.dst.virt.addr, nbytes, flags);
1461	/*
1462	 * The low word of the counter isn't used by the finalize, so there's no
1463	 * need to increment it here.
1464	 */
1465
1466	/* Finalize */
1467	taglen = crypto_aead_authsize(tfm);
1468	if (flags & FLAG_ENC) {
1469		/* Finish computing the auth tag. */
1470		aes_gcm_enc_final(key, le_ctr, ghash_acc, assoclen,
1471				  req->cryptlen, flags);
1472
1473		/* Store the computed auth tag in the dst scatterlist. */
1474		scatterwalk_map_and_copy(ghash_acc, req->dst, req->assoclen +
1475					 req->cryptlen, taglen, 1);
1476	} else {
1477		unsigned int datalen = req->cryptlen - taglen;
1478		u8 tag[16];
1479
1480		/* Get the transmitted auth tag from the src scatterlist. */
1481		scatterwalk_map_and_copy(tag, req->src, req->assoclen + datalen,
1482					 taglen, 0);
1483		/*
1484		 * Finish computing the auth tag and compare it to the
1485		 * transmitted one.  The assembly function does the actual tag
1486		 * comparison.  Here, just check the boolean result.
1487		 */
1488		if (!aes_gcm_dec_final(key, le_ctr, ghash_acc, assoclen,
1489				       datalen, tag, taglen, flags))
1490			err = -EBADMSG;
1491	}
1492	kernel_fpu_end();
1493	if (nbytes)
1494		skcipher_walk_done(&walk, 0);
1495	return err;
1496}
1497
1498#define DEFINE_GCM_ALGS(suffix, flags, generic_driver_name, rfc_driver_name,   \
1499			ctxsize, priority)				       \
1500									       \
1501static int gcm_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key,     \
1502			       unsigned int keylen)			       \
1503{									       \
1504	return gcm_setkey(tfm, raw_key, keylen, (flags));		       \
1505}									       \
1506									       \
1507static int gcm_encrypt_##suffix(struct aead_request *req)		       \
1508{									       \
1509	return gcm_crypt(req, (flags) | FLAG_ENC);			       \
1510}									       \
1511									       \
1512static int gcm_decrypt_##suffix(struct aead_request *req)		       \
1513{									       \
1514	return gcm_crypt(req, (flags));					       \
1515}									       \
1516									       \
1517static int rfc4106_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \
1518				   unsigned int keylen)			       \
1519{									       \
1520	return gcm_setkey(tfm, raw_key, keylen, (flags) | FLAG_RFC4106);       \
1521}									       \
1522									       \
1523static int rfc4106_encrypt_##suffix(struct aead_request *req)		       \
1524{									       \
1525	return gcm_crypt(req, (flags) | FLAG_RFC4106 | FLAG_ENC);	       \
1526}									       \
1527									       \
1528static int rfc4106_decrypt_##suffix(struct aead_request *req)		       \
1529{									       \
1530	return gcm_crypt(req, (flags) | FLAG_RFC4106);			       \
1531}									       \
1532									       \
1533static struct aead_alg aes_gcm_algs_##suffix[] = { {			       \
1534	.setkey			= gcm_setkey_##suffix,			       \
1535	.setauthsize		= generic_gcmaes_set_authsize,		       \
1536	.encrypt		= gcm_encrypt_##suffix,			       \
1537	.decrypt		= gcm_decrypt_##suffix,			       \
1538	.ivsize			= GCM_AES_IV_SIZE,			       \
1539	.chunksize		= AES_BLOCK_SIZE,			       \
1540	.maxauthsize		= 16,					       \
1541	.base = {							       \
1542		.cra_name		= "gcm(aes)",			       \
1543		.cra_driver_name	= generic_driver_name,		       \
1544		.cra_priority		= (priority),			       \
1545		.cra_blocksize		= 1,				       \
1546		.cra_ctxsize		= (ctxsize),			       \
1547		.cra_module		= THIS_MODULE,			       \
1548	},								       \
1549}, {									       \
1550	.setkey			= rfc4106_setkey_##suffix,		       \
1551	.setauthsize		= common_rfc4106_set_authsize,		       \
1552	.encrypt		= rfc4106_encrypt_##suffix,		       \
1553	.decrypt		= rfc4106_decrypt_##suffix,		       \
1554	.ivsize			= GCM_RFC4106_IV_SIZE,			       \
1555	.chunksize		= AES_BLOCK_SIZE,			       \
1556	.maxauthsize		= 16,					       \
1557	.base = {							       \
1558		.cra_name		= "rfc4106(gcm(aes))",		       \
1559		.cra_driver_name	= rfc_driver_name,		       \
1560		.cra_priority		= (priority),			       \
1561		.cra_blocksize		= 1,				       \
1562		.cra_ctxsize		= (ctxsize),			       \
1563		.cra_module		= THIS_MODULE,			       \
1564	},								       \
1565} }
1566
1567/* aes_gcm_algs_aesni */
1568DEFINE_GCM_ALGS(aesni, /* no flags */ 0,
1569		"generic-gcm-aesni", "rfc4106-gcm-aesni",
1570		AES_GCM_KEY_AESNI_SIZE, 400);
1571
1572/* aes_gcm_algs_aesni_avx */
1573DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX,
1574		"generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx",
1575		AES_GCM_KEY_AESNI_SIZE, 500);
1576
1577/* aes_gcm_algs_vaes_avx2 */
1578DEFINE_GCM_ALGS(vaes_avx2, FLAG_VAES_AVX2,
1579		"generic-gcm-vaes-avx2", "rfc4106-gcm-vaes-avx2",
1580		AES_GCM_KEY_VAES_AVX2_SIZE, 600);
1581
1582/* aes_gcm_algs_vaes_avx512 */
1583DEFINE_GCM_ALGS(vaes_avx512, FLAG_VAES_AVX512,
1584		"generic-gcm-vaes-avx512", "rfc4106-gcm-vaes-avx512",
1585		AES_GCM_KEY_VAES_AVX512_SIZE, 800);
1586
1587static int __init register_avx_algs(void)
1588{
1589	int err;
1590
1591	if (!boot_cpu_has(X86_FEATURE_AVX))
1592		return 0;
1593	err = crypto_register_skciphers(skcipher_algs_aesni_avx,
1594					ARRAY_SIZE(skcipher_algs_aesni_avx));
1595	if (err)
1596		return err;
1597	err = crypto_register_aeads(aes_gcm_algs_aesni_avx,
1598				    ARRAY_SIZE(aes_gcm_algs_aesni_avx));
1599	if (err)
1600		return err;
1601	/*
1602	 * Note: not all the algorithms registered below actually require
1603	 * VPCLMULQDQ.  But in practice every CPU with VAES also has VPCLMULQDQ.
1604	 * Similarly, the assembler support was added at about the same time.
1605	 * For simplicity, just always check for VAES and VPCLMULQDQ together.
1606	 */
1607	if (!boot_cpu_has(X86_FEATURE_AVX2) ||
1608	    !boot_cpu_has(X86_FEATURE_VAES) ||
1609	    !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
1610	    !boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
1611	    !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
1612		return 0;
1613	err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
1614					ARRAY_SIZE(skcipher_algs_vaes_avx2));
1615	if (err)
1616		return err;
1617	err = crypto_register_aeads(aes_gcm_algs_vaes_avx2,
1618				    ARRAY_SIZE(aes_gcm_algs_vaes_avx2));
1619	if (err)
1620		return err;
1621
1622	if (!boot_cpu_has(X86_FEATURE_AVX512BW) ||
1623	    !boot_cpu_has(X86_FEATURE_AVX512VL) ||
1624	    !boot_cpu_has(X86_FEATURE_BMI2) ||
1625	    !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
1626			       XFEATURE_MASK_AVX512, NULL))
1627		return 0;
1628
1629	if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
1630		int i;
1631
1632		for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++)
1633			skcipher_algs_vaes_avx512[i].base.cra_priority = 1;
1634		for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx512); i++)
1635			aes_gcm_algs_vaes_avx512[i].base.cra_priority = 1;
1636	}
1637
1638	err = crypto_register_skciphers(skcipher_algs_vaes_avx512,
1639					ARRAY_SIZE(skcipher_algs_vaes_avx512));
1640	if (err)
1641		return err;
1642	err = crypto_register_aeads(aes_gcm_algs_vaes_avx512,
1643				    ARRAY_SIZE(aes_gcm_algs_vaes_avx512));
1644	if (err)
1645		return err;
1646
1647	return 0;
1648}
1649
1650#define unregister_skciphers(A) \
1651	if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
1652		crypto_unregister_skciphers((A), ARRAY_SIZE(A))
1653#define unregister_aeads(A) \
1654	if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
1655		crypto_unregister_aeads((A), ARRAY_SIZE(A))
1656
1657static void unregister_avx_algs(void)
1658{
1659	unregister_skciphers(skcipher_algs_aesni_avx);
1660	unregister_aeads(aes_gcm_algs_aesni_avx);
1661	unregister_skciphers(skcipher_algs_vaes_avx2);
1662	unregister_skciphers(skcipher_algs_vaes_avx512);
1663	unregister_aeads(aes_gcm_algs_vaes_avx2);
1664	unregister_aeads(aes_gcm_algs_vaes_avx512);
1665}
1666#else /* CONFIG_X86_64 */
1667static struct aead_alg aes_gcm_algs_aesni[0];
1668
1669static int __init register_avx_algs(void)
1670{
1671	return 0;
1672}
1673
1674static void unregister_avx_algs(void)
1675{
1676}
1677#endif /* !CONFIG_X86_64 */
1678
1679static const struct x86_cpu_id aesni_cpu_id[] = {
1680	X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1681	{}
1682};
1683MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1684
1685static int __init aesni_init(void)
1686{
1687	int err;
1688
1689	if (!x86_match_cpu(aesni_cpu_id))
1690		return -ENODEV;
1691
1692	err = crypto_register_alg(&aesni_cipher_alg);
1693	if (err)
1694		return err;
1695
1696	err = crypto_register_skciphers(aesni_skciphers,
1697					ARRAY_SIZE(aesni_skciphers));
1698	if (err)
1699		goto unregister_cipher;
1700
1701	err = crypto_register_aeads(aes_gcm_algs_aesni,
1702				    ARRAY_SIZE(aes_gcm_algs_aesni));
1703	if (err)
1704		goto unregister_skciphers;
1705
1706	err = register_avx_algs();
1707	if (err)
1708		goto unregister_avx;
1709
1710	return 0;
1711
1712unregister_avx:
1713	unregister_avx_algs();
1714	crypto_unregister_aeads(aes_gcm_algs_aesni,
1715				ARRAY_SIZE(aes_gcm_algs_aesni));
1716unregister_skciphers:
1717	crypto_unregister_skciphers(aesni_skciphers,
1718				    ARRAY_SIZE(aesni_skciphers));
1719unregister_cipher:
1720	crypto_unregister_alg(&aesni_cipher_alg);
1721	return err;
1722}
1723
1724static void __exit aesni_exit(void)
1725{
1726	crypto_unregister_aeads(aes_gcm_algs_aesni,
1727				ARRAY_SIZE(aes_gcm_algs_aesni));
1728	crypto_unregister_skciphers(aesni_skciphers,
1729				    ARRAY_SIZE(aesni_skciphers));
1730	crypto_unregister_alg(&aesni_cipher_alg);
1731	unregister_avx_algs();
1732}
1733
1734module_init(aesni_init);
1735module_exit(aesni_exit);
1736
1737MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions");
1738MODULE_LICENSE("GPL");
1739MODULE_ALIAS_CRYPTO("aes");