testmgr.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Algorithm testing framework and tests.
   4 *
   5 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
   6 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
   7 * Copyright (c) 2007 Nokia Siemens Networks
   8 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
   9 * Copyright (c) 2019 Google LLC
  10 *
  11 * Updated RFC4106 AES-GCM testing.
  12 *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
  13 *             Adrian Hoban <adrian.hoban@intel.com>
  14 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  15 *             Tadeusz Struk (tadeusz.struk@intel.com)
  16 *    Copyright (c) 2010, Intel Corporation.
  17 */
  18
  19#include <crypto/aead.h>
  20#include <crypto/hash.h>
  21#include <crypto/skcipher.h>
  22#include <linux/err.h>
  23#include <linux/fips.h>
  24#include <linux/module.h>
  25#include <linux/once.h>
  26#include <linux/prandom.h>
  27#include <linux/scatterlist.h>
  28#include <linux/slab.h>
  29#include <linux/string.h>
  30#include <linux/uio.h>
  31#include <crypto/rng.h>
  32#include <crypto/drbg.h>
  33#include <crypto/akcipher.h>
  34#include <crypto/kpp.h>
  35#include <crypto/acompress.h>
  36#include <crypto/sig.h>
  37#include <crypto/internal/cipher.h>
  38#include <crypto/internal/simd.h>
  39
  40#include "internal.h"
  41
  42MODULE_IMPORT_NS("CRYPTO_INTERNAL");
  43
  44static bool notests;
  45module_param(notests, bool, 0644);
  46MODULE_PARM_DESC(notests, "disable all crypto self-tests");
  47
  48#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
  49static bool noslowtests;
  50module_param(noslowtests, bool, 0644);
  51MODULE_PARM_DESC(noslowtests, "disable slow crypto self-tests");
  52
  53static unsigned int fuzz_iterations = 100;
  54module_param(fuzz_iterations, uint, 0644);
  55MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
  56#else
  57#define noslowtests 1
  58#define fuzz_iterations 0
  59#endif
  60
  61#ifndef CONFIG_CRYPTO_SELFTESTS
  62
  63/* a perfect nop */
  64int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
  65{
  66	return 0;
  67}
  68
  69#else
  70
  71#include "testmgr.h"
  72
  73/*
  74 * Need slab memory for testing (size in number of pages).
  75 */
  76#define XBUFSIZE	8
  77
  78/*
  79* Used by test_cipher()
  80*/
  81#define ENCRYPT 1
  82#define DECRYPT 0
  83
  84struct aead_test_suite {
  85	const struct aead_testvec *vecs;
  86	unsigned int count;
  87
  88	/*
  89	 * Set if trying to decrypt an inauthentic ciphertext with this
  90	 * algorithm might result in EINVAL rather than EBADMSG, due to other
  91	 * validation the algorithm does on the inputs such as length checks.
  92	 */
  93	unsigned int einval_allowed : 1;
  94
  95	/*
  96	 * Set if this algorithm requires that the IV be located at the end of
  97	 * the AAD buffer, in addition to being given in the normal way.  The
  98	 * behavior when the two IV copies differ is implementation-defined.
  99	 */
 100	unsigned int aad_iv : 1;
 101};
 102
 103struct cipher_test_suite {
 104	const struct cipher_testvec *vecs;
 105	unsigned int count;
 106};
 107
 108struct comp_test_suite {
 109	struct {
 110		const struct comp_testvec *vecs;
 111		unsigned int count;
 112	} comp, decomp;
 113};
 114
 115struct hash_test_suite {
 116	const struct hash_testvec *vecs;
 117	unsigned int count;
 118};
 119
 120struct drbg_test_suite {
 121	const struct drbg_testvec *vecs;
 122	unsigned int count;
 123};
 124
 125struct akcipher_test_suite {
 126	const struct akcipher_testvec *vecs;
 127	unsigned int count;
 128};
 129
 130struct sig_test_suite {
 131	const struct sig_testvec *vecs;
 132	unsigned int count;
 133};
 134
 135struct kpp_test_suite {
 136	const struct kpp_testvec *vecs;
 137	unsigned int count;
 138};
 139
 140struct alg_test_desc {
 141	const char *alg;
 142	const char *generic_driver;
 143	int (*test)(const struct alg_test_desc *desc, const char *driver,
 144		    u32 type, u32 mask);
 145	int fips_allowed;	/* set if alg is allowed in fips mode */
 146
 147	union {
 148		struct aead_test_suite aead;
 149		struct cipher_test_suite cipher;
 150		struct comp_test_suite comp;
 151		struct hash_test_suite hash;
 152		struct drbg_test_suite drbg;
 153		struct akcipher_test_suite akcipher;
 154		struct sig_test_suite sig;
 155		struct kpp_test_suite kpp;
 156	} suite;
 157};
 158
 159static void hexdump(unsigned char *buf, unsigned int len)
 160{
 161	print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 162			16, 1,
 163			buf, len, false);
 164}
 165
 166static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
 167{
 168	int i;
 169
 170	for (i = 0; i < XBUFSIZE; i++) {
 171		buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
 172		if (!buf[i])
 173			goto err_free_buf;
 174	}
 175
 176	return 0;
 177
 178err_free_buf:
 179	while (i-- > 0)
 180		free_pages((unsigned long)buf[i], order);
 181
 182	return -ENOMEM;
 183}
 184
 185static int testmgr_alloc_buf(char *buf[XBUFSIZE])
 186{
 187	return __testmgr_alloc_buf(buf, 0);
 188}
 189
 190static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
 191{
 192	int i;
 193
 194	for (i = 0; i < XBUFSIZE; i++)
 195		free_pages((unsigned long)buf[i], order);
 196}
 197
 198static void testmgr_free_buf(char *buf[XBUFSIZE])
 199{
 200	__testmgr_free_buf(buf, 0);
 201}
 202
 203#define TESTMGR_POISON_BYTE	0xfe
 204#define TESTMGR_POISON_LEN	16
 205
 206static inline void testmgr_poison(void *addr, size_t len)
 207{
 208	memset(addr, TESTMGR_POISON_BYTE, len);
 209}
 210
 211/* Is the memory region still fully poisoned? */
 212static inline bool testmgr_is_poison(const void *addr, size_t len)
 213{
 214	return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;
 215}
 216
 217/* flush type for hash algorithms */
 218enum flush_type {
 219	/* merge with update of previous buffer(s) */
 220	FLUSH_TYPE_NONE = 0,
 221
 222	/* update with previous buffer(s) before doing this one */
 223	FLUSH_TYPE_FLUSH,
 224
 225	/* likewise, but also export and re-import the intermediate state */
 226	FLUSH_TYPE_REIMPORT,
 227};
 228
 229/* finalization function for hash algorithms */
 230enum finalization_type {
 231	FINALIZATION_TYPE_FINAL,	/* use final() */
 232	FINALIZATION_TYPE_FINUP,	/* use finup() */
 233	FINALIZATION_TYPE_DIGEST,	/* use digest() */
 234};
 235
 236/*
 237 * Whether the crypto operation will occur in-place, and if so whether the
 238 * source and destination scatterlist pointers will coincide (req->src ==
 239 * req->dst), or whether they'll merely point to two separate scatterlists
 240 * (req->src != req->dst) that reference the same underlying memory.
 241 *
 242 * This is only relevant for algorithm types that support in-place operation.
 243 */
 244enum inplace_mode {
 245	OUT_OF_PLACE,
 246	INPLACE_ONE_SGLIST,
 247	INPLACE_TWO_SGLISTS,
 248};
 249
 250#define TEST_SG_TOTAL	10000
 251
 252/**
 253 * struct test_sg_division - description of a scatterlist entry
 254 *
 255 * This struct describes one entry of a scatterlist being constructed to check a
 256 * crypto test vector.
 257 *
 258 * @proportion_of_total: length of this chunk relative to the total length,
 259 *			 given as a proportion out of TEST_SG_TOTAL so that it
 260 *			 scales to fit any test vector
 261 * @offset: byte offset into a 2-page buffer at which this chunk will start
 262 * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
 263 *				  @offset
 264 * @flush_type: for hashes, whether an update() should be done now vs.
 265 *		continuing to accumulate data
 266 * @nosimd: if doing the pending update(), do it with SIMD disabled?
 267 */
 268struct test_sg_division {
 269	unsigned int proportion_of_total;
 270	unsigned int offset;
 271	bool offset_relative_to_alignmask;
 272	enum flush_type flush_type;
 273	bool nosimd;
 274};
 275
 276/**
 277 * struct testvec_config - configuration for testing a crypto test vector
 278 *
 279 * This struct describes the data layout and other parameters with which each
 280 * crypto test vector can be tested.
 281 *
 282 * @name: name of this config, logged for debugging purposes if a test fails
 283 * @inplace_mode: whether and how to operate on the data in-place, if applicable
 284 * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
 285 * @src_divs: description of how to arrange the source scatterlist
 286 * @dst_divs: description of how to arrange the dst scatterlist, if applicable
 287 *	      for the algorithm type.  Defaults to @src_divs if unset.
 288 * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
 289 *	       where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
 290 * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 291 *				     the @iv_offset
 292 * @key_offset: misalignment of the key, where 0 is default alignment
 293 * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 294 *				      the @key_offset
 295 * @finalization_type: what finalization function to use for hashes
 296 * @nosimd: execute with SIMD disabled?  Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
 297 *	    This applies to the parts of the operation that aren't controlled
 298 *	    individually by @nosimd_setkey or @src_divs[].nosimd.
 299 * @nosimd_setkey: set the key (if applicable) with SIMD disabled?  Requires
 300 *		   !CRYPTO_TFM_REQ_MAY_SLEEP.
 301 */
 302struct testvec_config {
 303	const char *name;
 304	enum inplace_mode inplace_mode;
 305	u32 req_flags;
 306	struct test_sg_division src_divs[XBUFSIZE];
 307	struct test_sg_division dst_divs[XBUFSIZE];
 308	unsigned int iv_offset;
 309	unsigned int key_offset;
 310	bool iv_offset_relative_to_alignmask;
 311	bool key_offset_relative_to_alignmask;
 312	enum finalization_type finalization_type;
 313	bool nosimd;
 314	bool nosimd_setkey;
 315};
 316
 317#define TESTVEC_CONFIG_NAMELEN	192
 318
 319/*
 320 * The following are the lists of testvec_configs to test for each algorithm
 321 * type when the "fast" crypto self-tests are enabled.  They aim to provide good
 322 * test coverage, while keeping the test time much shorter than the "full" tests
 323 * so that the "fast" tests can be enabled in a wider range of circumstances.
 324 */
 325
 326/* Configs for skciphers and aeads */
 327static const struct testvec_config default_cipher_testvec_configs[] = {
 328	{
 329		.name = "in-place (one sglist)",
 330		.inplace_mode = INPLACE_ONE_SGLIST,
 331		.src_divs = { { .proportion_of_total = 10000 } },
 332	}, {
 333		.name = "in-place (two sglists)",
 334		.inplace_mode = INPLACE_TWO_SGLISTS,
 335		.src_divs = { { .proportion_of_total = 10000 } },
 336	}, {
 337		.name = "out-of-place",
 338		.inplace_mode = OUT_OF_PLACE,
 339		.src_divs = { { .proportion_of_total = 10000 } },
 340	}, {
 341		.name = "unaligned buffer, offset=1",
 342		.src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 343		.iv_offset = 1,
 344		.key_offset = 1,
 345	}, {
 346		.name = "buffer aligned only to alignmask",
 347		.src_divs = {
 348			{
 349				.proportion_of_total = 10000,
 350				.offset = 1,
 351				.offset_relative_to_alignmask = true,
 352			},
 353		},
 354		.iv_offset = 1,
 355		.iv_offset_relative_to_alignmask = true,
 356		.key_offset = 1,
 357		.key_offset_relative_to_alignmask = true,
 358	}, {
 359		.name = "two even aligned splits",
 360		.src_divs = {
 361			{ .proportion_of_total = 5000 },
 362			{ .proportion_of_total = 5000 },
 363		},
 364	}, {
 365		.name = "one src, two even splits dst",
 366		.inplace_mode = OUT_OF_PLACE,
 367		.src_divs = { { .proportion_of_total = 10000 } },
 368		.dst_divs = {
 369			{ .proportion_of_total = 5000 },
 370			{ .proportion_of_total = 5000 },
 371		 },
 372	}, {
 373		.name = "uneven misaligned splits, may sleep",
 374		.req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 375		.src_divs = {
 376			{ .proportion_of_total = 1900, .offset = 33 },
 377			{ .proportion_of_total = 3300, .offset = 7  },
 378			{ .proportion_of_total = 4800, .offset = 18 },
 379		},
 380		.iv_offset = 3,
 381		.key_offset = 3,
 382	}, {
 383		.name = "misaligned splits crossing pages, inplace",
 384		.inplace_mode = INPLACE_ONE_SGLIST,
 385		.src_divs = {
 386			{
 387				.proportion_of_total = 7500,
 388				.offset = PAGE_SIZE - 32
 389			}, {
 390				.proportion_of_total = 2500,
 391				.offset = PAGE_SIZE - 7
 392			},
 393		},
 394	}
 395};
 396
 397static const struct testvec_config default_hash_testvec_configs[] = {
 398	{
 399		.name = "init+update+final aligned buffer",
 400		.src_divs = { { .proportion_of_total = 10000 } },
 401		.finalization_type = FINALIZATION_TYPE_FINAL,
 402	}, {
 403		.name = "init+finup aligned buffer",
 404		.src_divs = { { .proportion_of_total = 10000 } },
 405		.finalization_type = FINALIZATION_TYPE_FINUP,
 406	}, {
 407		.name = "digest aligned buffer",
 408		.src_divs = { { .proportion_of_total = 10000 } },
 409		.finalization_type = FINALIZATION_TYPE_DIGEST,
 410	}, {
 411		.name = "init+update+final misaligned buffer",
 412		.src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 413		.finalization_type = FINALIZATION_TYPE_FINAL,
 414		.key_offset = 1,
 415	}, {
 416		.name = "digest misaligned buffer",
 417		.src_divs = {
 418			{
 419				.proportion_of_total = 10000,
 420				.offset = 1,
 421			},
 422		},
 423		.finalization_type = FINALIZATION_TYPE_DIGEST,
 424		.key_offset = 1,
 425	}, {
 426		.name = "init+update+update+final two even splits",
 427		.src_divs = {
 428			{ .proportion_of_total = 5000 },
 429			{
 430				.proportion_of_total = 5000,
 431				.flush_type = FLUSH_TYPE_FLUSH,
 432			},
 433		},
 434		.finalization_type = FINALIZATION_TYPE_FINAL,
 435	}, {
 436		.name = "digest uneven misaligned splits, may sleep",
 437		.req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 438		.src_divs = {
 439			{ .proportion_of_total = 1900, .offset = 33 },
 440			{ .proportion_of_total = 3300, .offset = 7  },
 441			{ .proportion_of_total = 4800, .offset = 18 },
 442		},
 443		.finalization_type = FINALIZATION_TYPE_DIGEST,
 444	}, {
 445		.name = "digest misaligned splits crossing pages",
 446		.src_divs = {
 447			{
 448				.proportion_of_total = 7500,
 449				.offset = PAGE_SIZE - 32,
 450			}, {
 451				.proportion_of_total = 2500,
 452				.offset = PAGE_SIZE - 7,
 453			},
 454		},
 455		.finalization_type = FINALIZATION_TYPE_DIGEST,
 456	}, {
 457		.name = "import/export",
 458		.src_divs = {
 459			{
 460				.proportion_of_total = 6500,
 461				.flush_type = FLUSH_TYPE_REIMPORT,
 462			}, {
 463				.proportion_of_total = 3500,
 464				.flush_type = FLUSH_TYPE_REIMPORT,
 465			},
 466		},
 467		.finalization_type = FINALIZATION_TYPE_FINAL,
 468	}
 469};
 470
 471static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
 472{
 473	unsigned int remaining = TEST_SG_TOTAL;
 474	unsigned int ndivs = 0;
 475
 476	do {
 477		remaining -= divs[ndivs++].proportion_of_total;
 478	} while (remaining);
 479
 480	return ndivs;
 481}
 482
 483#define SGDIVS_HAVE_FLUSHES	BIT(0)
 484#define SGDIVS_HAVE_NOSIMD	BIT(1)
 485
 486static bool valid_sg_divisions(const struct test_sg_division *divs,
 487			       unsigned int count, int *flags_ret)
 488{
 489	unsigned int total = 0;
 490	unsigned int i;
 491
 492	for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
 493		if (divs[i].proportion_of_total <= 0 ||
 494		    divs[i].proportion_of_total > TEST_SG_TOTAL - total)
 495			return false;
 496		total += divs[i].proportion_of_total;
 497		if (divs[i].flush_type != FLUSH_TYPE_NONE)
 498			*flags_ret |= SGDIVS_HAVE_FLUSHES;
 499		if (divs[i].nosimd)
 500			*flags_ret |= SGDIVS_HAVE_NOSIMD;
 501	}
 502	return total == TEST_SG_TOTAL &&
 503		memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;
 504}
 505
 506/*
 507 * Check whether the given testvec_config is valid.  This isn't strictly needed
 508 * since every testvec_config should be valid, but check anyway so that people
 509 * don't unknowingly add broken configs that don't do what they wanted.
 510 */
 511static bool valid_testvec_config(const struct testvec_config *cfg)
 512{
 513	int flags = 0;
 514
 515	if (cfg->name == NULL)
 516		return false;
 517
 518	if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
 519				&flags))
 520		return false;
 521
 522	if (cfg->dst_divs[0].proportion_of_total) {
 523		if (!valid_sg_divisions(cfg->dst_divs,
 524					ARRAY_SIZE(cfg->dst_divs), &flags))
 525			return false;
 526	} else {
 527		if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))
 528			return false;
 529		/* defaults to dst_divs=src_divs */
 530	}
 531
 532	if (cfg->iv_offset +
 533	    (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
 534	    MAX_ALGAPI_ALIGNMASK + 1)
 535		return false;
 536
 537	if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&
 538	    cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
 539		return false;
 540
 541	if ((cfg->nosimd || cfg->nosimd_setkey ||
 542	     (flags & SGDIVS_HAVE_NOSIMD)) &&
 543	    (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
 544		return false;
 545
 546	return true;
 547}
 548
 549struct test_sglist {
 550	char *bufs[XBUFSIZE];
 551	struct scatterlist sgl[XBUFSIZE];
 552	struct scatterlist sgl_saved[XBUFSIZE];
 553	struct scatterlist *sgl_ptr;
 554	unsigned int nents;
 555};
 556
 557static int init_test_sglist(struct test_sglist *tsgl)
 558{
 559	return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);
 560}
 561
 562static void destroy_test_sglist(struct test_sglist *tsgl)
 563{
 564	return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);
 565}
 566
 567/**
 568 * build_test_sglist() - build a scatterlist for a crypto test
 569 *
 570 * @tsgl: the scatterlist to build.  @tsgl->bufs[] contains an array of 2-page
 571 *	  buffers which the scatterlist @tsgl->sgl[] will be made to point into.
 572 * @divs: the layout specification on which the scatterlist will be based
 573 * @alignmask: the algorithm's alignmask
 574 * @total_len: the total length of the scatterlist to build in bytes
 575 * @data: if non-NULL, the buffers will be filled with this data until it ends.
 576 *	  Otherwise the buffers will be poisoned.  In both cases, some bytes
 577 *	  past the end of each buffer will be poisoned to help detect overruns.
 578 * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
 579 *	      corresponds will be returned here.  This will match @divs except
 580 *	      that divisions resolving to a length of 0 are omitted as they are
 581 *	      not included in the scatterlist.
 582 *
 583 * Return: 0 or a -errno value
 584 */
 585static int build_test_sglist(struct test_sglist *tsgl,
 586			     const struct test_sg_division *divs,
 587			     const unsigned int alignmask,
 588			     const unsigned int total_len,
 589			     struct iov_iter *data,
 590			     const struct test_sg_division *out_divs[XBUFSIZE])
 591{
 592	struct {
 593		const struct test_sg_division *div;
 594		size_t length;
 595	} partitions[XBUFSIZE];
 596	const unsigned int ndivs = count_test_sg_divisions(divs);
 597	unsigned int len_remaining = total_len;
 598	unsigned int i;
 599
 600	BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
 601	if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
 602		return -EINVAL;
 603
 604	/* Calculate the (div, length) pairs */
 605	tsgl->nents = 0;
 606	for (i = 0; i < ndivs; i++) {
 607		unsigned int len_this_sg =
 608			min(len_remaining,
 609			    (total_len * divs[i].proportion_of_total +
 610			     TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
 611
 612		if (len_this_sg != 0) {
 613			partitions[tsgl->nents].div = &divs[i];
 614			partitions[tsgl->nents].length = len_this_sg;
 615			tsgl->nents++;
 616			len_remaining -= len_this_sg;
 617		}
 618	}
 619	if (tsgl->nents == 0) {
 620		partitions[tsgl->nents].div = &divs[0];
 621		partitions[tsgl->nents].length = 0;
 622		tsgl->nents++;
 623	}
 624	partitions[tsgl->nents - 1].length += len_remaining;
 625
 626	/* Set up the sgl entries and fill the data or poison */
 627	sg_init_table(tsgl->sgl, tsgl->nents);
 628	for (i = 0; i < tsgl->nents; i++) {
 629		unsigned int offset = partitions[i].div->offset;
 630		void *addr;
 631
 632		if (partitions[i].div->offset_relative_to_alignmask)
 633			offset += alignmask;
 634
 635		while (offset + partitions[i].length + TESTMGR_POISON_LEN >
 636		       2 * PAGE_SIZE) {
 637			if (WARN_ON(offset <= 0))
 638				return -EINVAL;
 639			offset /= 2;
 640		}
 641
 642		addr = &tsgl->bufs[i][offset];
 643		sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);
 644
 645		if (out_divs)
 646			out_divs[i] = partitions[i].div;
 647
 648		if (data) {
 649			size_t copy_len, copied;
 650
 651			copy_len = min(partitions[i].length, data->count);
 652			copied = copy_from_iter(addr, copy_len, data);
 653			if (WARN_ON(copied != copy_len))
 654				return -EINVAL;
 655			testmgr_poison(addr + copy_len, partitions[i].length +
 656				       TESTMGR_POISON_LEN - copy_len);
 657		} else {
 658			testmgr_poison(addr, partitions[i].length +
 659				       TESTMGR_POISON_LEN);
 660		}
 661	}
 662
 663	sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);
 664	tsgl->sgl_ptr = tsgl->sgl;
 665	memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
 666	return 0;
 667}
 668
 669/*
 670 * Verify that a scatterlist crypto operation produced the correct output.
 671 *
 672 * @tsgl: scatterlist containing the actual output
 673 * @expected_output: buffer containing the expected output
 674 * @len_to_check: length of @expected_output in bytes
 675 * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
 676 * @check_poison: verify that the poison bytes after each chunk are intact?
 677 *
 678 * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
 679 */
 680static int verify_correct_output(const struct test_sglist *tsgl,
 681				 const char *expected_output,
 682				 unsigned int len_to_check,
 683				 unsigned int unchecked_prefix_len,
 684				 bool check_poison)
 685{
 686	unsigned int i;
 687
 688	for (i = 0; i < tsgl->nents; i++) {
 689		struct scatterlist *sg = &tsgl->sgl_ptr[i];
 690		unsigned int len = sg->length;
 691		unsigned int offset = sg->offset;
 692		const char *actual_output;
 693
 694		if (unchecked_prefix_len) {
 695			if (unchecked_prefix_len >= len) {
 696				unchecked_prefix_len -= len;
 697				continue;
 698			}
 699			offset += unchecked_prefix_len;
 700			len -= unchecked_prefix_len;
 701			unchecked_prefix_len = 0;
 702		}
 703		len = min(len, len_to_check);
 704		actual_output = page_address(sg_page(sg)) + offset;
 705		if (memcmp(expected_output, actual_output, len) != 0)
 706			return -EINVAL;
 707		if (check_poison &&
 708		    !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))
 709			return -EOVERFLOW;
 710		len_to_check -= len;
 711		expected_output += len;
 712	}
 713	if (WARN_ON(len_to_check != 0))
 714		return -EINVAL;
 715	return 0;
 716}
 717
 718static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
 719{
 720	unsigned int i;
 721
 722	for (i = 0; i < tsgl->nents; i++) {
 723		if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
 724			return true;
 725		if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
 726			return true;
 727		if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
 728			return true;
 729	}
 730	return false;
 731}
 732
 733struct cipher_test_sglists {
 734	struct test_sglist src;
 735	struct test_sglist dst;
 736};
 737
 738static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
 739{
 740	struct cipher_test_sglists *tsgls;
 741
 742	tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
 743	if (!tsgls)
 744		return NULL;
 745
 746	if (init_test_sglist(&tsgls->src) != 0)
 747		goto fail_kfree;
 748	if (init_test_sglist(&tsgls->dst) != 0)
 749		goto fail_destroy_src;
 750
 751	return tsgls;
 752
 753fail_destroy_src:
 754	destroy_test_sglist(&tsgls->src);
 755fail_kfree:
 756	kfree(tsgls);
 757	return NULL;
 758}
 759
 760static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
 761{
 762	if (tsgls) {
 763		destroy_test_sglist(&tsgls->src);
 764		destroy_test_sglist(&tsgls->dst);
 765		kfree(tsgls);
 766	}
 767}
 768
 769/* Build the src and dst scatterlists for an skcipher or AEAD test */
 770static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
 771				     const struct testvec_config *cfg,
 772				     unsigned int alignmask,
 773				     unsigned int src_total_len,
 774				     unsigned int dst_total_len,
 775				     const struct kvec *inputs,
 776				     unsigned int nr_inputs)
 777{
 778	struct iov_iter input;
 779	int err;
 780
 781	iov_iter_kvec(&input, ITER_SOURCE, inputs, nr_inputs, src_total_len);
 782	err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,
 783				cfg->inplace_mode != OUT_OF_PLACE ?
 784					max(dst_total_len, src_total_len) :
 785					src_total_len,
 786				&input, NULL);
 787	if (err)
 788		return err;
 789
 790	/*
 791	 * In-place crypto operations can use the same scatterlist for both the
 792	 * source and destination (req->src == req->dst), or can use separate
 793	 * scatterlists (req->src != req->dst) which point to the same
 794	 * underlying memory.  Make sure to test both cases.
 795	 */
 796	if (cfg->inplace_mode == INPLACE_ONE_SGLIST) {
 797		tsgls->dst.sgl_ptr = tsgls->src.sgl;
 798		tsgls->dst.nents = tsgls->src.nents;
 799		return 0;
 800	}
 801	if (cfg->inplace_mode == INPLACE_TWO_SGLISTS) {
 802		/*
 803		 * For now we keep it simple and only test the case where the
 804		 * two scatterlists have identical entries, rather than
 805		 * different entries that split up the same memory differently.
 806		 */
 807		memcpy(tsgls->dst.sgl, tsgls->src.sgl,
 808		       tsgls->src.nents * sizeof(tsgls->src.sgl[0]));
 809		memcpy(tsgls->dst.sgl_saved, tsgls->src.sgl,
 810		       tsgls->src.nents * sizeof(tsgls->src.sgl[0]));
 811		tsgls->dst.sgl_ptr = tsgls->dst.sgl;
 812		tsgls->dst.nents = tsgls->src.nents;
 813		return 0;
 814	}
 815	/* Out of place */
 816	return build_test_sglist(&tsgls->dst,
 817				 cfg->dst_divs[0].proportion_of_total ?
 818					cfg->dst_divs : cfg->src_divs,
 819				 alignmask, dst_total_len, NULL, NULL);
 820}
 821
 822/*
 823 * Support for testing passing a misaligned key to setkey():
 824 *
 825 * If cfg->key_offset is set, copy the key into a new buffer at that offset,
 826 * optionally adding alignmask.  Else, just use the key directly.
 827 */
 828static int prepare_keybuf(const u8 *key, unsigned int ksize,
 829			  const struct testvec_config *cfg,
 830			  unsigned int alignmask,
 831			  const u8 **keybuf_ret, const u8 **keyptr_ret)
 832{
 833	unsigned int key_offset = cfg->key_offset;
 834	u8 *keybuf = NULL, *keyptr = (u8 *)key;
 835
 836	if (key_offset != 0) {
 837		if (cfg->key_offset_relative_to_alignmask)
 838			key_offset += alignmask;
 839		keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);
 840		if (!keybuf)
 841			return -ENOMEM;
 842		keyptr = keybuf + key_offset;
 843		memcpy(keyptr, key, ksize);
 844	}
 845	*keybuf_ret = keybuf;
 846	*keyptr_ret = keyptr;
 847	return 0;
 848}
 849
 850/*
 851 * Like setkey_f(tfm, key, ksize), but sometimes misalign the key.
 852 * In addition, run the setkey function in no-SIMD context if requested.
 853 */
 854#define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask)		\
 855({									\
 856	const u8 *keybuf, *keyptr;					\
 857	int err;							\
 858									\
 859	err = prepare_keybuf((key), (ksize), (cfg), (alignmask),	\
 860			     &keybuf, &keyptr);				\
 861	if (err == 0) {							\
 862		if ((cfg)->nosimd_setkey)				\
 863			crypto_disable_simd_for_test();			\
 864		err = setkey_f((tfm), keyptr, (ksize));			\
 865		if ((cfg)->nosimd_setkey)				\
 866			crypto_reenable_simd_for_test();		\
 867		kfree(keybuf);						\
 868	}								\
 869	err;								\
 870})
 871
 872/*
 873 * The fuzz tests use prandom instead of the normal Linux RNG since they don't
 874 * need cryptographically secure random numbers.  This greatly improves the
 875 * performance of these tests, especially if they are run before the Linux RNG
 876 * has been initialized or if they are run on a lockdep-enabled kernel.
 877 */
 878
 879static inline void init_rnd_state(struct rnd_state *rng)
 880{
 881	prandom_seed_state(rng, get_random_u64());
 882}
 883
 884static inline u8 prandom_u8(struct rnd_state *rng)
 885{
 886	return prandom_u32_state(rng);
 887}
 888
 889static inline u32 prandom_u32_below(struct rnd_state *rng, u32 ceil)
 890{
 891	/*
 892	 * This is slightly biased for non-power-of-2 values of 'ceil', but this
 893	 * isn't important here.
 894	 */
 895	return prandom_u32_state(rng) % ceil;
 896}
 897
 898static inline bool prandom_bool(struct rnd_state *rng)
 899{
 900	return prandom_u32_below(rng, 2);
 901}
 902
 903static inline u32 prandom_u32_inclusive(struct rnd_state *rng,
 904					u32 floor, u32 ceil)
 905{
 906	return floor + prandom_u32_below(rng, ceil - floor + 1);
 907}
 908
 909/* Generate a random length in range [0, max_len], but prefer smaller values */
 910static unsigned int generate_random_length(struct rnd_state *rng,
 911					   unsigned int max_len)
 912{
 913	unsigned int len = prandom_u32_below(rng, max_len + 1);
 914
 915	switch (prandom_u32_below(rng, 4)) {
 916	case 0:
 917		len %= 64;
 918		break;
 919	case 1:
 920		len %= 256;
 921		break;
 922	case 2:
 923		len %= 1024;
 924		break;
 925	default:
 926		break;
 927	}
 928	if (len && prandom_u32_below(rng, 4) == 0)
 929		len = rounddown_pow_of_two(len);
 930	return len;
 931}
 932
 933/* Flip a random bit in the given nonempty data buffer */
 934static void flip_random_bit(struct rnd_state *rng, u8 *buf, size_t size)
 935{
 936	size_t bitpos;
 937
 938	bitpos = prandom_u32_below(rng, size * 8);
 939	buf[bitpos / 8] ^= 1 << (bitpos % 8);
 940}
 941
 942/* Flip a random byte in the given nonempty data buffer */
 943static void flip_random_byte(struct rnd_state *rng, u8 *buf, size_t size)
 944{
 945	buf[prandom_u32_below(rng, size)] ^= 0xff;
 946}
 947
 948/* Sometimes make some random changes to the given nonempty data buffer */
 949static void mutate_buffer(struct rnd_state *rng, u8 *buf, size_t size)
 950{
 951	size_t num_flips;
 952	size_t i;
 953
 954	/* Sometimes flip some bits */
 955	if (prandom_u32_below(rng, 4) == 0) {
 956		num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8),
 957				  size * 8);
 958		for (i = 0; i < num_flips; i++)
 959			flip_random_bit(rng, buf, size);
 960	}
 961
 962	/* Sometimes flip some bytes */
 963	if (prandom_u32_below(rng, 4) == 0) {
 964		num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8), size);
 965		for (i = 0; i < num_flips; i++)
 966			flip_random_byte(rng, buf, size);
 967	}
 968}
 969
 970/* Randomly generate 'count' bytes, but sometimes make them "interesting" */
 971static void generate_random_bytes(struct rnd_state *rng, u8 *buf, size_t count)
 972{
 973	u8 b;
 974	u8 increment;
 975	size_t i;
 976
 977	if (count == 0)
 978		return;
 979
 980	switch (prandom_u32_below(rng, 8)) { /* Choose a generation strategy */
 981	case 0:
 982	case 1:
 983		/* All the same byte, plus optional mutations */
 984		switch (prandom_u32_below(rng, 4)) {
 985		case 0:
 986			b = 0x00;
 987			break;
 988		case 1:
 989			b = 0xff;
 990			break;
 991		default:
 992			b = prandom_u8(rng);
 993			break;
 994		}
 995		memset(buf, b, count);
 996		mutate_buffer(rng, buf, count);
 997		break;
 998	case 2:
 999		/* Ascending or descending bytes, plus optional mutations */
1000		increment = prandom_u8(rng);
1001		b = prandom_u8(rng);
1002		for (i = 0; i < count; i++, b += increment)
1003			buf[i] = b;
1004		mutate_buffer(rng, buf, count);
1005		break;
1006	default:
1007		/* Fully random bytes */
1008		prandom_bytes_state(rng, buf, count);
1009	}
1010}
1011
1012static char *generate_random_sgl_divisions(struct rnd_state *rng,
1013					   struct test_sg_division *divs,
1014					   size_t max_divs, char *p, char *end,
1015					   bool gen_flushes, u32 req_flags)
1016{
1017	struct test_sg_division *div = divs;
1018	unsigned int remaining = TEST_SG_TOTAL;
1019
1020	do {
1021		unsigned int this_len;
1022		const char *flushtype_str;
1023
1024		if (div == &divs[max_divs - 1] || prandom_bool(rng))
1025			this_len = remaining;
1026		else if (prandom_u32_below(rng, 4) == 0)
1027			this_len = (remaining + 1) / 2;
1028		else
1029			this_len = prandom_u32_inclusive(rng, 1, remaining);
1030		div->proportion_of_total = this_len;
1031
1032		if (prandom_u32_below(rng, 4) == 0)
1033			div->offset = prandom_u32_inclusive(rng,
1034							    PAGE_SIZE - 128,
1035							    PAGE_SIZE - 1);
1036		else if (prandom_bool(rng))
1037			div->offset = prandom_u32_below(rng, 32);
1038		else
1039			div->offset = prandom_u32_below(rng, PAGE_SIZE);
1040		if (prandom_u32_below(rng, 8) == 0)
1041			div->offset_relative_to_alignmask = true;
1042
1043		div->flush_type = FLUSH_TYPE_NONE;
1044		if (gen_flushes) {
1045			switch (prandom_u32_below(rng, 4)) {
1046			case 0:
1047				div->flush_type = FLUSH_TYPE_REIMPORT;
1048				break;
1049			case 1:
1050				div->flush_type = FLUSH_TYPE_FLUSH;
1051				break;
1052			}
1053		}
1054
1055		if (div->flush_type != FLUSH_TYPE_NONE &&
1056		    !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
1057		    prandom_bool(rng))
1058			div->nosimd = true;
1059
1060		switch (div->flush_type) {
1061		case FLUSH_TYPE_FLUSH:
1062			if (div->nosimd)
1063				flushtype_str = "<flush,nosimd>";
1064			else
1065				flushtype_str = "<flush>";
1066			break;
1067		case FLUSH_TYPE_REIMPORT:
1068			if (div->nosimd)
1069				flushtype_str = "<reimport,nosimd>";
1070			else
1071				flushtype_str = "<reimport>";
1072			break;
1073		default:
1074			flushtype_str = "";
1075			break;
1076		}
1077
1078		BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
1079		p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,
1080			       this_len / 100, this_len % 100,
1081			       div->offset_relative_to_alignmask ?
1082					"alignmask" : "",
1083			       div->offset, this_len == remaining ? "" : ", ");
1084		remaining -= this_len;
1085		div++;
1086	} while (remaining);
1087
1088	return p;
1089}
1090
1091/* Generate a random testvec_config for fuzz testing */
1092static void generate_random_testvec_config(struct rnd_state *rng,
1093					   struct testvec_config *cfg,
1094					   char *name, size_t max_namelen)
1095{
1096	char *p = name;
1097	char * const end = name + max_namelen;
1098
1099	memset(cfg, 0, sizeof(*cfg));
1100
1101	cfg->name = name;
1102
1103	p += scnprintf(p, end - p, "random:");
1104
1105	switch (prandom_u32_below(rng, 4)) {
1106	case 0:
1107	case 1:
1108		cfg->inplace_mode = OUT_OF_PLACE;
1109		break;
1110	case 2:
1111		cfg->inplace_mode = INPLACE_ONE_SGLIST;
1112		p += scnprintf(p, end - p, " inplace_one_sglist");
1113		break;
1114	default:
1115		cfg->inplace_mode = INPLACE_TWO_SGLISTS;
1116		p += scnprintf(p, end - p, " inplace_two_sglists");
1117		break;
1118	}
1119
1120	if (prandom_bool(rng)) {
1121		cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
1122		p += scnprintf(p, end - p, " may_sleep");
1123	}
1124
1125	switch (prandom_u32_below(rng, 4)) {
1126	case 0:
1127		cfg->finalization_type = FINALIZATION_TYPE_FINAL;
1128		p += scnprintf(p, end - p, " use_final");
1129		break;
1130	case 1:
1131		cfg->finalization_type = FINALIZATION_TYPE_FINUP;
1132		p += scnprintf(p, end - p, " use_finup");
1133		break;
1134	default:
1135		cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
1136		p += scnprintf(p, end - p, " use_digest");
1137		break;
1138	}
1139
1140	if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) {
1141		if (prandom_bool(rng)) {
1142			cfg->nosimd = true;
1143			p += scnprintf(p, end - p, " nosimd");
1144		}
1145		if (prandom_bool(rng)) {
1146			cfg->nosimd_setkey = true;
1147			p += scnprintf(p, end - p, " nosimd_setkey");
1148		}
1149	}
1150
1151	p += scnprintf(p, end - p, " src_divs=[");
1152	p = generate_random_sgl_divisions(rng, cfg->src_divs,
1153					  ARRAY_SIZE(cfg->src_divs), p, end,
1154					  (cfg->finalization_type !=
1155					   FINALIZATION_TYPE_DIGEST),
1156					  cfg->req_flags);
1157	p += scnprintf(p, end - p, "]");
1158
1159	if (cfg->inplace_mode == OUT_OF_PLACE && prandom_bool(rng)) {
1160		p += scnprintf(p, end - p, " dst_divs=[");
1161		p = generate_random_sgl_divisions(rng, cfg->dst_divs,
1162						  ARRAY_SIZE(cfg->dst_divs),
1163						  p, end, false,
1164						  cfg->req_flags);
1165		p += scnprintf(p, end - p, "]");
1166	}
1167
1168	if (prandom_bool(rng)) {
1169		cfg->iv_offset = prandom_u32_inclusive(rng, 1,
1170						       MAX_ALGAPI_ALIGNMASK);
1171		p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
1172	}
1173
1174	if (prandom_bool(rng)) {
1175		cfg->key_offset = prandom_u32_inclusive(rng, 1,
1176							MAX_ALGAPI_ALIGNMASK);
1177		p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);
1178	}
1179
1180	WARN_ON_ONCE(!valid_testvec_config(cfg));
1181}
1182
1183static void crypto_disable_simd_for_test(void)
1184{
1185#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
1186	migrate_disable();
1187	__this_cpu_write(crypto_simd_disabled_for_test, true);
1188#endif
1189}
1190
1191static void crypto_reenable_simd_for_test(void)
1192{
1193#ifdef CONFIG_CRYPTO_SELFTESTS_FULL
1194	__this_cpu_write(crypto_simd_disabled_for_test, false);
1195	migrate_enable();
1196#endif
1197}
1198
1199/*
1200 * Given an algorithm name, build the name of the generic implementation of that
1201 * algorithm, assuming the usual naming convention.  Specifically, this appends
1202 * "-generic" to every part of the name that is not a template name.  Examples:
1203 *
1204 *	aes => aes-generic
1205 *	cbc(aes) => cbc(aes-generic)
1206 *	cts(cbc(aes)) => cts(cbc(aes-generic))
1207 *	rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
1208 *
1209 * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
1210 */
1211static int build_generic_driver_name(const char *algname,
1212				     char driver_name[CRYPTO_MAX_ALG_NAME])
1213{
1214	const char *in = algname;
1215	char *out = driver_name;
1216	size_t len = strlen(algname);
1217
1218	if (len >= CRYPTO_MAX_ALG_NAME)
1219		goto too_long;
1220	do {
1221		const char *in_saved = in;
1222
1223		while (*in && *in != '(' && *in != ')' && *in != ',')
1224			*out++ = *in++;
1225		if (*in != '(' && in > in_saved) {
1226			len += 8;
1227			if (len >= CRYPTO_MAX_ALG_NAME)
1228				goto too_long;
1229			memcpy(out, "-generic", 8);
1230			out += 8;
1231		}
1232	} while ((*out++ = *in++) != '\0');
1233	return 0;
1234
1235too_long:
1236	pr_err("alg: generic driver name for \"%s\" would be too long\n",
1237	       algname);
1238	return -ENAMETOOLONG;
1239}
1240
1241static int build_hash_sglist(struct test_sglist *tsgl,
1242			     const struct hash_testvec *vec,
1243			     const struct testvec_config *cfg,
1244			     unsigned int alignmask,
1245			     const struct test_sg_division *divs[XBUFSIZE])
1246{
1247	struct kvec kv;
1248	struct iov_iter input;
1249
1250	kv.iov_base = (void *)vec->plaintext;
1251	kv.iov_len = vec->psize;
1252	iov_iter_kvec(&input, ITER_SOURCE, &kv, 1, vec->psize);
1253	return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
1254				 &input, divs);
1255}
1256
1257static int check_hash_result(const char *type,
1258			     const u8 *result, unsigned int digestsize,
1259			     const struct hash_testvec *vec,
1260			     const char *vec_name,
1261			     const char *driver,
1262			     const struct testvec_config *cfg)
1263{
1264	if (memcmp(result, vec->digest, digestsize) != 0) {
1265		pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1266		       type, driver, vec_name, cfg->name);
1267		return -EINVAL;
1268	}
1269	if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
1270		pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
1271		       type, driver, vec_name, cfg->name);
1272		return -EOVERFLOW;
1273	}
1274	return 0;
1275}
1276
1277static inline int check_shash_op(const char *op, int err,
1278				 const char *driver, const char *vec_name,
1279				 const struct testvec_config *cfg)
1280{
1281	if (err)
1282		pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1283		       driver, op, err, vec_name, cfg->name);
1284	return err;
1285}
1286
1287/* Test one hash test vector in one configuration, using the shash API */
1288static int test_shash_vec_cfg(const struct hash_testvec *vec,
1289			      const char *vec_name,
1290			      const struct testvec_config *cfg,
1291			      struct shash_desc *desc,
1292			      struct test_sglist *tsgl,
1293			      u8 *hashstate)
1294{
1295	struct crypto_shash *tfm = desc->tfm;
1296	const unsigned int digestsize = crypto_shash_digestsize(tfm);
1297	const unsigned int statesize = crypto_shash_statesize(tfm);
1298	const char *driver = crypto_shash_driver_name(tfm);
1299	const struct test_sg_division *divs[XBUFSIZE];
1300	unsigned int i;
1301	u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1302	int err;
1303
1304	/* Set the key, if specified */
1305	if (vec->ksize) {
1306		err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,
1307				cfg, 0);
1308		if (err) {
1309			if (err == vec->setkey_error)
1310				return 0;
1311			pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1312			       driver, vec_name, vec->setkey_error, err,
1313			       crypto_shash_get_flags(tfm));
1314			return err;
1315		}
1316		if (vec->setkey_error) {
1317			pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1318			       driver, vec_name, vec->setkey_error);
1319			return -EINVAL;
1320		}
1321	}
1322
1323	/* Build the scatterlist for the source data */
1324	err = build_hash_sglist(tsgl, vec, cfg, 0, divs);
1325	if (err) {
1326		pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1327		       driver, vec_name, cfg->name);
1328		return err;
1329	}
1330
1331	/* Do the actual hashing */
1332
1333	testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1334	testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1335
1336	if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1337	    vec->digest_error) {
1338		/* Just using digest() */
1339		if (tsgl->nents != 1)
1340			return 0;
1341		if (cfg->nosimd)
1342			crypto_disable_simd_for_test();
1343		err = crypto_shash_digest(desc, sg_virt(&tsgl->sgl[0]),
1344					  tsgl->sgl[0].length, result);
1345		if (cfg->nosimd)
1346			crypto_reenable_simd_for_test();
1347		if (err) {
1348			if (err == vec->digest_error)
1349				return 0;
1350			pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1351			       driver, vec_name, vec->digest_error, err,
1352			       cfg->name);
1353			return err;
1354		}
1355		if (vec->digest_error) {
1356			pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1357			       driver, vec_name, vec->digest_error, cfg->name);
1358			return -EINVAL;
1359		}
1360		goto result_ready;
1361	}
1362
1363	/* Using init(), zero or more update(), then final() or finup() */
1364
1365	if (cfg->nosimd)
1366		crypto_disable_simd_for_test();
1367	err = crypto_shash_init(desc);
1368	if (cfg->nosimd)
1369		crypto_reenable_simd_for_test();
1370	err = check_shash_op("init", err, driver, vec_name, cfg);
1371	if (err)
1372		return err;
1373
1374	for (i = 0; i < tsgl->nents; i++) {
1375		if (i + 1 == tsgl->nents &&
1376		    cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
1377			if (divs[i]->nosimd)
1378				crypto_disable_simd_for_test();
1379			err = crypto_shash_finup(desc, sg_virt(&tsgl->sgl[i]),
1380						 tsgl->sgl[i].length, result);
1381			if (divs[i]->nosimd)
1382				crypto_reenable_simd_for_test();
1383			err = check_shash_op("finup", err, driver, vec_name,
1384					     cfg);
1385			if (err)
1386				return err;
1387			goto result_ready;
1388		}
1389		if (divs[i]->nosimd)
1390			crypto_disable_simd_for_test();
1391		err = crypto_shash_update(desc, sg_virt(&tsgl->sgl[i]),
1392					  tsgl->sgl[i].length);
1393		if (divs[i]->nosimd)
1394			crypto_reenable_simd_for_test();
1395		err = check_shash_op("update", err, driver, vec_name, cfg);
1396		if (err)
1397			return err;
1398		if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1399			/* Test ->export() and ->import() */
1400			testmgr_poison(hashstate + statesize,
1401				       TESTMGR_POISON_LEN);
1402			err = crypto_shash_export(desc, hashstate);
1403			err = check_shash_op("export", err, driver, vec_name,
1404					     cfg);
1405			if (err)
1406				return err;
1407			if (!testmgr_is_poison(hashstate + statesize,
1408					       TESTMGR_POISON_LEN)) {
1409				pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1410				       driver, vec_name, cfg->name);
1411				return -EOVERFLOW;
1412			}
1413			testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1414			err = crypto_shash_import(desc, hashstate);
1415			err = check_shash_op("import", err, driver, vec_name,
1416					     cfg);
1417			if (err)
1418				return err;
1419		}
1420	}
1421
1422	if (cfg->nosimd)
1423		crypto_disable_simd_for_test();
1424	err = crypto_shash_final(desc, result);
1425	if (cfg->nosimd)
1426		crypto_reenable_simd_for_test();
1427	err = check_shash_op("final", err, driver, vec_name, cfg);
1428	if (err)
1429		return err;
1430result_ready:
1431	return check_hash_result("shash", result, digestsize, vec, vec_name,
1432				 driver, cfg);
1433}
1434
1435static int do_ahash_op(int (*op)(struct ahash_request *req),
1436		       struct ahash_request *req,
1437		       struct crypto_wait *wait, bool nosimd)
1438{
1439	int err;
1440
1441	if (nosimd)
1442		crypto_disable_simd_for_test();
1443
1444	err = op(req);
1445
1446	if (nosimd)
1447		crypto_reenable_simd_for_test();
1448
1449	return crypto_wait_req(err, wait);
1450}
1451
1452static int check_nonfinal_ahash_op(const char *op, int err,
1453				   u8 *result, unsigned int digestsize,
1454				   const char *driver, const char *vec_name,
1455				   const struct testvec_config *cfg)
1456{
1457	if (err) {
1458		pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1459		       driver, op, err, vec_name, cfg->name);
1460		return err;
1461	}
1462	if (!testmgr_is_poison(result, digestsize)) {
1463		pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
1464		       driver, op, vec_name, cfg->name);
1465		return -EINVAL;
1466	}
1467	return 0;
1468}
1469
1470/* Test one hash test vector in one configuration, using the ahash API */
1471static int test_ahash_vec_cfg(const struct hash_testvec *vec,
1472			      const char *vec_name,
1473			      const struct testvec_config *cfg,
1474			      struct ahash_request *req,
1475			      struct test_sglist *tsgl,
1476			      u8 *hashstate)
1477{
1478	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1479	const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1480	const unsigned int statesize = crypto_ahash_statesize(tfm);
1481	const char *driver = crypto_ahash_driver_name(tfm);
1482	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1483	const struct test_sg_division *divs[XBUFSIZE];
1484	DECLARE_CRYPTO_WAIT(wait);
1485	unsigned int i;
1486	struct scatterlist *pending_sgl;
1487	unsigned int pending_len;
1488	u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1489	int err;
1490
1491	/* Set the key, if specified */
1492	if (vec->ksize) {
1493		err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,
1494				cfg, 0);
1495		if (err) {
1496			if (err == vec->setkey_error)
1497				return 0;
1498			pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1499			       driver, vec_name, vec->setkey_error, err,
1500			       crypto_ahash_get_flags(tfm));
1501			return err;
1502		}
1503		if (vec->setkey_error) {
1504			pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1505			       driver, vec_name, vec->setkey_error);
1506			return -EINVAL;
1507		}
1508	}
1509
1510	/* Build the scatterlist for the source data */
1511	err = build_hash_sglist(tsgl, vec, cfg, 0, divs);
1512	if (err) {
1513		pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1514		       driver, vec_name, cfg->name);
1515		return err;
1516	}
1517
1518	/* Do the actual hashing */
1519
1520	testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1521	testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1522
1523	if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1524	    vec->digest_error) {
1525		/* Just using digest() */
1526		ahash_request_set_callback(req, req_flags, crypto_req_done,
1527					   &wait);
1528		ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);
1529		err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);
1530		if (err) {
1531			if (err == vec->digest_error)
1532				return 0;
1533			pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1534			       driver, vec_name, vec->digest_error, err,
1535			       cfg->name);
1536			return err;
1537		}
1538		if (vec->digest_error) {
1539			pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1540			       driver, vec_name, vec->digest_error, cfg->name);
1541			return -EINVAL;
1542		}
1543		goto result_ready;
1544	}
1545
1546	/* Using init(), zero or more update(), then final() or finup() */
1547
1548	ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1549	ahash_request_set_crypt(req, NULL, result, 0);
1550	err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
1551	err = check_nonfinal_ahash_op("init", err, result, digestsize,
1552				      driver, vec_name, cfg);
1553	if (err)
1554		return err;
1555
1556	pending_sgl = NULL;
1557	pending_len = 0;
1558	for (i = 0; i < tsgl->nents; i++) {
1559		if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
1560		    pending_sgl != NULL) {
1561			/* update() with the pending data */
1562			ahash_request_set_callback(req, req_flags,
1563						   crypto_req_done, &wait);
1564			ahash_request_set_crypt(req, pending_sgl, result,
1565						pending_len);
1566			err = do_ahash_op(crypto_ahash_update, req, &wait,
1567					  divs[i]->nosimd);
1568			err = check_nonfinal_ahash_op("update", err,
1569						      result, digestsize,
1570						      driver, vec_name, cfg);
1571			if (err)
1572				return err;
1573			pending_sgl = NULL;
1574			pending_len = 0;
1575		}
1576		if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1577			/* Test ->export() and ->import() */
1578			testmgr_poison(hashstate + statesize,
1579				       TESTMGR_POISON_LEN);
1580			err = crypto_ahash_export(req, hashstate);
1581			err = check_nonfinal_ahash_op("export", err,
1582						      result, digestsize,
1583						      driver, vec_name, cfg);
1584			if (err)
1585				return err;
1586			if (!testmgr_is_poison(hashstate + statesize,
1587					       TESTMGR_POISON_LEN)) {
1588				pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1589				       driver, vec_name, cfg->name);
1590				return -EOVERFLOW;
1591			}
1592
1593			testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1594			err = crypto_ahash_import(req, hashstate);
1595			err = check_nonfinal_ahash_op("import", err,
1596						      result, digestsize,
1597						      driver, vec_name, cfg);
1598			if (err)
1599				return err;
1600		}
1601		if (pending_sgl == NULL)
1602			pending_sgl = &tsgl->sgl[i];
1603		pending_len += tsgl->sgl[i].length;
1604	}
1605
1606	ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1607	ahash_request_set_crypt(req, pending_sgl, result, pending_len);
1608	if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
1609		/* finish with update() and final() */
1610		err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
1611		err = check_nonfinal_ahash_op("update", err, result, digestsize,
1612					      driver, vec_name, cfg);
1613		if (err)
1614			return err;
1615		err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
1616		if (err) {
1617			pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
1618			       driver, err, vec_name, cfg->name);
1619			return err;
1620		}
1621	} else {
1622		/* finish with finup() */
1623		err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
1624		if (err) {
1625			pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
1626			       driver, err, vec_name, cfg->name);
1627			return err;
1628		}
1629	}
1630
1631result_ready:
1632	return check_hash_result("ahash", result, digestsize, vec, vec_name,
1633				 driver, cfg);
1634}
1635
1636static int test_hash_vec_cfg(const struct hash_testvec *vec,
1637			     const char *vec_name,
1638			     const struct testvec_config *cfg,
1639			     struct ahash_request *req,
1640			     struct shash_desc *desc,
1641			     struct test_sglist *tsgl,
1642			     u8 *hashstate)
1643{
1644	int err;
1645
1646	/*
1647	 * For algorithms implemented as "shash", most bugs will be detected by
1648	 * both the shash and ahash tests.  Test the shash API first so that the
1649	 * failures involve less indirection, so are easier to debug.
1650	 */
1651
1652	if (desc) {
1653		err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl,
1654					 hashstate);
1655		if (err)
1656			return err;
1657	}
1658
1659	return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate);
1660}
1661
1662static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num,
1663			 struct ahash_request *req, struct shash_desc *desc,
1664			 struct test_sglist *tsgl, u8 *hashstate)
1665{
1666	char vec_name[16];
1667	unsigned int i;
1668	int err;
1669
1670	sprintf(vec_name, "%u", vec_num);
1671
1672	for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
1673		err = test_hash_vec_cfg(vec, vec_name,
1674					&default_hash_testvec_configs[i],
1675					req, desc, tsgl, hashstate);
1676		if (err)
1677			return err;
1678	}
1679
1680	if (!noslowtests) {
1681		struct rnd_state rng;
1682		struct testvec_config cfg;
1683		char cfgname[TESTVEC_CONFIG_NAMELEN];
1684
1685		init_rnd_state(&rng);
1686
1687		for (i = 0; i < fuzz_iterations; i++) {
1688			generate_random_testvec_config(&rng, &cfg, cfgname,
1689						       sizeof(cfgname));
1690			err = test_hash_vec_cfg(vec, vec_name, &cfg,
1691						req, desc, tsgl, hashstate);
1692			if (err)
1693				return err;
1694			cond_resched();
1695		}
1696	}
1697	return 0;
1698}
1699
1700/*
1701 * Generate a hash test vector from the given implementation.
1702 * Assumes the buffers in 'vec' were already allocated.
1703 */
1704static void generate_random_hash_testvec(struct rnd_state *rng,
1705					 struct ahash_request *req,
1706					 struct hash_testvec *vec,
1707					 unsigned int maxkeysize,
1708					 unsigned int maxdatasize,
1709					 char *name, size_t max_namelen)
1710{
1711	/* Data */
1712	vec->psize = generate_random_length(rng, maxdatasize);
1713	generate_random_bytes(rng, (u8 *)vec->plaintext, vec->psize);
1714
1715	/*
1716	 * Key: length in range [1, maxkeysize], but usually choose maxkeysize.
1717	 * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
1718	 */
1719	vec->setkey_error = 0;
1720	vec->ksize = 0;
1721	if (maxkeysize) {
1722		vec->ksize = maxkeysize;
1723		if (prandom_u32_below(rng, 4) == 0)
1724			vec->ksize = prandom_u32_inclusive(rng, 1, maxkeysize);
1725		generate_random_bytes(rng, (u8 *)vec->key, vec->ksize);
1726
1727		vec->setkey_error = crypto_ahash_setkey(
1728			crypto_ahash_reqtfm(req), vec->key, vec->ksize);
1729		/* If the key couldn't be set, no need to continue to digest. */
1730		if (vec->setkey_error)
1731			goto done;
1732	}
1733
1734	/* Digest */
1735	vec->digest_error = crypto_hash_digest(
1736		crypto_ahash_reqtfm(req), vec->plaintext,
1737		vec->psize, (u8 *)vec->digest);
1738done:
1739	snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",
1740		 vec->psize, vec->ksize);
1741}
1742
1743/*
1744 * Test the hash algorithm represented by @req against the corresponding generic
1745 * implementation, if one is available.
1746 */
1747static int test_hash_vs_generic_impl(const char *generic_driver,
1748				     unsigned int maxkeysize,
1749				     struct ahash_request *req,
1750				     struct shash_desc *desc,
1751				     struct test_sglist *tsgl,
1752				     u8 *hashstate)
1753{
1754	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1755	const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1756	const unsigned int blocksize = crypto_ahash_blocksize(tfm);
1757	const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1758	const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
1759	const char *driver = crypto_ahash_driver_name(tfm);
1760	struct rnd_state rng;
1761	char _generic_driver[CRYPTO_MAX_ALG_NAME];
1762	struct ahash_request *generic_req = NULL;
1763	struct crypto_ahash *generic_tfm = NULL;
1764	unsigned int i;
1765	struct hash_testvec vec = { 0 };
1766	char vec_name[64];
1767	struct testvec_config *cfg;
1768	char cfgname[TESTVEC_CONFIG_NAMELEN];
1769	int err;
1770
1771	if (noslowtests)
1772		return 0;
1773
1774	init_rnd_state(&rng);
1775
1776	if (!generic_driver) { /* Use default naming convention? */
1777		err = build_generic_driver_name(algname, _generic_driver);
1778		if (err)
1779			return err;
1780		generic_driver = _generic_driver;
1781	}
1782
1783	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1784		return 0;
1785
1786	generic_tfm = crypto_alloc_ahash(generic_driver, 0, 0);
1787	if (IS_ERR(generic_tfm)) {
1788		err = PTR_ERR(generic_tfm);
1789		if (err == -ENOENT) {
1790			pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
1791				driver, generic_driver);
1792			return 0;
1793		}
1794		pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
1795		       generic_driver, algname, err);
1796		return err;
1797	}
1798
1799	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1800	if (!cfg) {
1801		err = -ENOMEM;
1802		goto out;
1803	}
1804
1805	generic_req = ahash_request_alloc(generic_tfm, GFP_KERNEL);
1806	if (!generic_req) {
1807		err = -ENOMEM;
1808		goto out;
1809	}
1810
1811	/* Check the algorithm properties for consistency. */
1812
1813	if (digestsize != crypto_ahash_digestsize(generic_tfm)) {
1814		pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
1815		       driver, digestsize,
1816		       crypto_ahash_digestsize(generic_tfm));
1817		err = -EINVAL;
1818		goto out;
1819	}
1820
1821	if (blocksize != crypto_ahash_blocksize(generic_tfm)) {
1822		pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1823		       driver, blocksize, crypto_ahash_blocksize(generic_tfm));
1824		err = -EINVAL;
1825		goto out;
1826	}
1827
1828	/*
1829	 * Now generate test vectors using the generic implementation, and test
1830	 * the other implementation against them.
1831	 */
1832
1833	vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1834	vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
1835	vec.digest = kmalloc(digestsize, GFP_KERNEL);
1836	if (!vec.key || !vec.plaintext || !vec.digest) {
1837		err = -ENOMEM;
1838		goto out;
1839	}
1840
1841	for (i = 0; i < fuzz_iterations * 8; i++) {
1842		generate_random_hash_testvec(&rng, generic_req, &vec,
1843					     maxkeysize, maxdatasize,
1844					     vec_name, sizeof(vec_name));
1845		generate_random_testvec_config(&rng, cfg, cfgname,
1846					       sizeof(cfgname));
1847
1848		err = test_hash_vec_cfg(&vec, vec_name, cfg,
1849					req, desc, tsgl, hashstate);
1850		if (err)
1851			goto out;
1852		cond_resched();
1853	}
1854	err = 0;
1855out:
1856	kfree(cfg);
1857	kfree(vec.key);
1858	kfree(vec.plaintext);
1859	kfree(vec.digest);
1860	ahash_request_free(generic_req);
1861	crypto_free_ahash(generic_tfm);
1862	return err;
1863}
1864
1865static int alloc_shash(const char *driver, u32 type, u32 mask,
1866		       struct crypto_shash **tfm_ret,
1867		       struct shash_desc **desc_ret)
1868{
1869	struct crypto_shash *tfm;
1870	struct shash_desc *desc;
1871
1872	tfm = crypto_alloc_shash(driver, type, mask);
1873	if (IS_ERR(tfm)) {
1874		if (PTR_ERR(tfm) == -ENOENT || PTR_ERR(tfm) == -EEXIST) {
1875			/*
1876			 * This algorithm is only available through the ahash
1877			 * API, not the shash API, so skip the shash tests.
1878			 */
1879			return 0;
1880		}
1881		pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
1882		       driver, PTR_ERR(tfm));
1883		return PTR_ERR(tfm);
1884	}
1885
1886	desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
1887	if (!desc) {
1888		crypto_free_shash(tfm);
1889		return -ENOMEM;
1890	}
1891	desc->tfm = tfm;
1892
1893	*tfm_ret = tfm;
1894	*desc_ret = desc;
1895	return 0;
1896}
1897
1898static int __alg_test_hash(const struct hash_testvec *vecs,
1899			   unsigned int num_vecs, const char *driver,
1900			   u32 type, u32 mask,
1901			   const char *generic_driver, unsigned int maxkeysize)
1902{
1903	struct crypto_ahash *atfm = NULL;
1904	struct ahash_request *req = NULL;
1905	struct crypto_shash *stfm = NULL;
1906	struct shash_desc *desc = NULL;
1907	struct test_sglist *tsgl = NULL;
1908	u8 *hashstate = NULL;
1909	unsigned int statesize;
1910	unsigned int i;
1911	int err;
1912
1913	/*
1914	 * Always test the ahash API.  This works regardless of whether the
1915	 * algorithm is implemented as ahash or shash.
1916	 */
1917
1918	atfm = crypto_alloc_ahash(driver, type, mask);
1919	if (IS_ERR(atfm)) {
1920		if (PTR_ERR(atfm) == -ENOENT)
1921			return 0;
1922		pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
1923		       driver, PTR_ERR(atfm));
1924		return PTR_ERR(atfm);
1925	}
1926	driver = crypto_ahash_driver_name(atfm);
1927
1928	req = ahash_request_alloc(atfm, GFP_KERNEL);
1929	if (!req) {
1930		pr_err("alg: hash: failed to allocate request for %s\n",
1931		       driver);
1932		err = -ENOMEM;
1933		goto out;
1934	}
1935
1936	/*
1937	 * If available also test the shash API, to cover corner cases that may
1938	 * be missed by testing the ahash API only.
1939	 */
1940	err = alloc_shash(driver, type, mask, &stfm, &desc);
1941	if (err)
1942		goto out;
1943
1944	tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
1945	if (!tsgl || init_test_sglist(tsgl) != 0) {
1946		pr_err("alg: hash: failed to allocate test buffers for %s\n",
1947		       driver);
1948		kfree(tsgl);
1949		tsgl = NULL;
1950		err = -ENOMEM;
1951		goto out;
1952	}
1953
1954	statesize = crypto_ahash_statesize(atfm);
1955	if (stfm)
1956		statesize = max(statesize, crypto_shash_statesize(stfm));
1957	hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
1958	if (!hashstate) {
1959		pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
1960		       driver);
1961		err = -ENOMEM;
1962		goto out;
1963	}
1964
1965	for (i = 0; i < num_vecs; i++) {
1966		if (fips_enabled && vecs[i].fips_skip)
1967			continue;
1968
1969		err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate);
1970		if (err)
1971			goto out;
1972		cond_resched();
1973	}
1974	err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req,
1975					desc, tsgl, hashstate);
1976out:
1977	kfree(hashstate);
1978	if (tsgl) {
1979		destroy_test_sglist(tsgl);
1980		kfree(tsgl);
1981	}
1982	kfree(desc);
1983	crypto_free_shash(stfm);
1984	ahash_request_free(req);
1985	crypto_free_ahash(atfm);
1986	return err;
1987}
1988
1989static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
1990			 u32 type, u32 mask)
1991{
1992	const struct hash_testvec *template = desc->suite.hash.vecs;
1993	unsigned int tcount = desc->suite.hash.count;
1994	unsigned int nr_unkeyed, nr_keyed;
1995	unsigned int maxkeysize = 0;
1996	int err;
1997
1998	/*
1999	 * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
2000	 * first, before setting a key on the tfm.  To make this easier, we
2001	 * require that the unkeyed test vectors (if any) are listed first.
2002	 */
2003
2004	for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
2005		if (template[nr_unkeyed].ksize)
2006			break;
2007	}
2008	for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
2009		if (!template[nr_unkeyed + nr_keyed].ksize) {
2010			pr_err("alg: hash: test vectors for %s out of order, "
2011			       "unkeyed ones must come first\n", desc->alg);
2012			return -EINVAL;
2013		}
2014		maxkeysize = max_t(unsigned int, maxkeysize,
2015				   template[nr_unkeyed + nr_keyed].ksize);
2016	}
2017
2018	err = 0;
2019	if (nr_unkeyed) {
2020		err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,
2021				      desc->generic_driver, maxkeysize);
2022		template += nr_unkeyed;
2023	}
2024
2025	if (!err && nr_keyed)
2026		err = __alg_test_hash(template, nr_keyed, driver, type, mask,
2027				      desc->generic_driver, maxkeysize);
2028
2029	return err;
2030}
2031
2032static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec,
2033			     const char *vec_name,
2034			     const struct testvec_config *cfg,
2035			     struct aead_request *req,
2036			     struct cipher_test_sglists *tsgls)
2037{
2038	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2039	const unsigned int alignmask = crypto_aead_alignmask(tfm);
2040	const unsigned int ivsize = crypto_aead_ivsize(tfm);
2041	const unsigned int authsize = vec->clen - vec->plen;
2042	const char *driver = crypto_aead_driver_name(tfm);
2043	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2044	const char *op = enc ? "encryption" : "decryption";
2045	DECLARE_CRYPTO_WAIT(wait);
2046	u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2047	u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2048		 cfg->iv_offset +
2049		 (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2050	struct kvec input[2];
2051	int err;
2052
2053	/* Set the key */
2054	if (vec->wk)
2055		crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2056	else
2057		crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2058
2059	err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,
2060			cfg, alignmask);
2061	if (err && err != vec->setkey_error) {
2062		pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2063		       driver, vec_name, vec->setkey_error, err,
2064		       crypto_aead_get_flags(tfm));
2065		return err;
2066	}
2067	if (!err && vec->setkey_error) {
2068		pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2069		       driver, vec_name, vec->setkey_error);
2070		return -EINVAL;
2071	}
2072
2073	/* Set the authentication tag size */
2074	err = crypto_aead_setauthsize(tfm, authsize);
2075	if (err && err != vec->setauthsize_error) {
2076		pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
2077		       driver, vec_name, vec->setauthsize_error, err);
2078		return err;
2079	}
2080	if (!err && vec->setauthsize_error) {
2081		pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
2082		       driver, vec_name, vec->setauthsize_error);
2083		return -EINVAL;
2084	}
2085
2086	if (vec->setkey_error || vec->setauthsize_error)
2087		return 0;
2088
2089	/* The IV must be copied to a buffer, as the algorithm may modify it */
2090	if (WARN_ON(ivsize > MAX_IVLEN))
2091		return -EINVAL;
2092	if (vec->iv)
2093		memcpy(iv, vec->iv, ivsize);
2094	else
2095		memset(iv, 0, ivsize);
2096
2097	/* Build the src/dst scatterlists */
2098	input[0].iov_base = (void *)vec->assoc;
2099	input[0].iov_len = vec->alen;
2100	input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2101	input[1].iov_len = enc ? vec->plen : vec->clen;
2102	err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2103					vec->alen + (enc ? vec->plen :
2104						     vec->clen),
2105					vec->alen + (enc ? vec->clen :
2106						     vec->plen),
2107					input, 2);
2108	if (err) {
2109		pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2110		       driver, op, vec_name, cfg->name);
2111		return err;
2112	}
2113
2114	/* Do the actual encryption or decryption */
2115	testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));
2116	aead_request_set_callback(req, req_flags, crypto_req_done, &wait);
2117	aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2118			       enc ? vec->plen : vec->clen, iv);
2119	aead_request_set_ad(req, vec->alen);
2120	if (cfg->nosimd)
2121		crypto_disable_simd_for_test();
2122	err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
2123	if (cfg->nosimd)
2124		crypto_reenable_simd_for_test();
2125	err = crypto_wait_req(err, &wait);
2126
2127	/* Check that the algorithm didn't overwrite things it shouldn't have */
2128	if (req->cryptlen != (enc ? vec->plen : vec->clen) ||
2129	    req->assoclen != vec->alen ||
2130	    req->iv != iv ||
2131	    req->src != tsgls->src.sgl_ptr ||
2132	    req->dst != tsgls->dst.sgl_ptr ||
2133	    crypto_aead_reqtfm(req) != tfm ||
2134	    req->base.complete != crypto_req_done ||
2135	    req->base.flags != req_flags ||
2136	    req->base.data != &wait) {
2137		pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2138		       driver, op, vec_name, cfg->name);
2139		if (req->cryptlen != (enc ? vec->plen : vec->clen))
2140			pr_err("alg: aead: changed 'req->cryptlen'\n");
2141		if (req->assoclen != vec->alen)
2142			pr_err("alg: aead: changed 'req->assoclen'\n");
2143		if (req->iv != iv)
2144			pr_err("alg: aead: changed 'req->iv'\n");
2145		if (req->src != tsgls->src.sgl_ptr)
2146			pr_err("alg: aead: changed 'req->src'\n");
2147		if (req->dst != tsgls->dst.sgl_ptr)
2148			pr_err("alg: aead: changed 'req->dst'\n");
2149		if (crypto_aead_reqtfm(req) != tfm)
2150			pr_err("alg: aead: changed 'req->base.tfm'\n");
2151		if (req->base.complete != crypto_req_done)
2152			pr_err("alg: aead: changed 'req->base.complete'\n");
2153		if (req->base.flags != req_flags)
2154			pr_err("alg: aead: changed 'req->base.flags'\n");
2155		if (req->base.data != &wait)
2156			pr_err("alg: aead: changed 'req->base.data'\n");
2157		return -EINVAL;
2158	}
2159	if (is_test_sglist_corrupted(&tsgls->src)) {
2160		pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2161		       driver, op, vec_name, cfg->name);
2162		return -EINVAL;
2163	}
2164	if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2165	    is_test_sglist_corrupted(&tsgls->dst)) {
2166		pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2167		       driver, op, vec_name, cfg->name);
2168		return -EINVAL;
2169	}
2170
2171	/* Check for unexpected success or failure, or wrong error code */
2172	if ((err == 0 && vec->novrfy) ||
2173	    (err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {
2174		char expected_error[32];
2175
2176		if (vec->novrfy &&
2177		    vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)
2178			sprintf(expected_error, "-EBADMSG or %d",
2179				vec->crypt_error);
2180		else if (vec->novrfy)
2181			sprintf(expected_error, "-EBADMSG");
2182		else
2183			sprintf(expected_error, "%d", vec->crypt_error);
2184		if (err) {
2185			pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",
2186			       driver, op, vec_name, expected_error, err,
2187			       cfg->name);
2188			return err;
2189		}
2190		pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",
2191		       driver, op, vec_name, expected_error, cfg->name);
2192		return -EINVAL;
2193	}
2194	if (err) /* Expectedly failed. */
2195		return 0;
2196
2197	/* Check for the correct output (ciphertext or plaintext) */
2198	err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2199				    enc ? vec->clen : vec->plen,
2200				    vec->alen,
2201				    enc || cfg->inplace_mode == OUT_OF_PLACE);
2202	if (err == -EOVERFLOW) {
2203		pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2204		       driver, op, vec_name, cfg->name);
2205		return err;
2206	}
2207	if (err) {
2208		pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2209		       driver, op, vec_name, cfg->name);
2210		return err;
2211	}
2212
2213	return 0;
2214}
2215
2216static int test_aead_vec(int enc, const struct aead_testvec *vec,
2217			 unsigned int vec_num, struct aead_request *req,
2218			 struct cipher_test_sglists *tsgls)
2219{
2220	char vec_name[16];
2221	unsigned int i;
2222	int err;
2223
2224	if (enc && vec->novrfy)
2225		return 0;
2226
2227	sprintf(vec_name, "%u", vec_num);
2228
2229	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2230		err = test_aead_vec_cfg(enc, vec, vec_name,
2231					&default_cipher_testvec_configs[i],
2232					req, tsgls);
2233		if (err)
2234			return err;
2235	}
2236
2237	if (!noslowtests) {
2238		struct rnd_state rng;
2239		struct testvec_config cfg;
2240		char cfgname[TESTVEC_CONFIG_NAMELEN];
2241
2242		init_rnd_state(&rng);
2243
2244		for (i = 0; i < fuzz_iterations; i++) {
2245			generate_random_testvec_config(&rng, &cfg, cfgname,
2246						       sizeof(cfgname));
2247			err = test_aead_vec_cfg(enc, vec, vec_name,
2248						&cfg, req, tsgls);
2249			if (err)
2250				return err;
2251			cond_resched();
2252		}
2253	}
2254	return 0;
2255}
2256
2257struct aead_slow_tests_ctx {
2258	struct rnd_state rng;
2259	struct aead_request *req;
2260	struct crypto_aead *tfm;
2261	const struct alg_test_desc *test_desc;
2262	struct cipher_test_sglists *tsgls;
2263	unsigned int maxdatasize;
2264	unsigned int maxkeysize;
2265
2266	struct aead_testvec vec;
2267	char vec_name[64];
2268	char cfgname[TESTVEC_CONFIG_NAMELEN];
2269	struct testvec_config cfg;
2270};
2271
2272/*
2273 * Make at least one random change to a (ciphertext, AAD) pair.  "Ciphertext"
2274 * here means the full ciphertext including the authentication tag.  The
2275 * authentication tag (and hence also the ciphertext) is assumed to be nonempty.
2276 */
2277static void mutate_aead_message(struct rnd_state *rng,
2278				struct aead_testvec *vec, bool aad_iv,
2279				unsigned int ivsize)
2280{
2281	const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
2282	const unsigned int authsize = vec->clen - vec->plen;
2283
2284	if (prandom_bool(rng) && vec->alen > aad_tail_size) {
2285		 /* Mutate the AAD */
2286		flip_random_bit(rng, (u8 *)vec->assoc,
2287				vec->alen - aad_tail_size);
2288		if (prandom_bool(rng))
2289			return;
2290	}
2291	if (prandom_bool(rng)) {
2292		/* Mutate auth tag (assuming it's at the end of ciphertext) */
2293		flip_random_bit(rng, (u8 *)vec->ctext + vec->plen, authsize);
2294	} else {
2295		/* Mutate any part of the ciphertext */
2296		flip_random_bit(rng, (u8 *)vec->ctext, vec->clen);
2297	}
2298}
2299
2300/*
2301 * Minimum authentication tag size in bytes at which we assume that we can
2302 * reliably generate inauthentic messages, i.e. not generate an authentic
2303 * message by chance.
2304 */
2305#define MIN_COLLISION_FREE_AUTHSIZE 8
2306
2307static void generate_aead_message(struct rnd_state *rng,
2308				  struct aead_request *req,
2309				  const struct aead_test_suite *suite,
2310				  struct aead_testvec *vec,
2311				  bool prefer_inauthentic)
2312{
2313	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2314	const unsigned int ivsize = crypto_aead_ivsize(tfm);
2315	const unsigned int authsize = vec->clen - vec->plen;
2316	const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
2317				 (prefer_inauthentic ||
2318				  prandom_u32_below(rng, 4) == 0);
2319
2320	/* Generate the AAD. */
2321	generate_random_bytes(rng, (u8 *)vec->assoc, vec->alen);
2322	if (suite->aad_iv && vec->alen >= ivsize)
2323		/* Avoid implementation-defined behavior. */
2324		memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
2325
2326	if (inauthentic && prandom_bool(rng)) {
2327		/* Generate a random ciphertext. */
2328		generate_random_bytes(rng, (u8 *)vec->ctext, vec->clen);
2329	} else {
2330		int i = 0;
2331		struct scatterlist src[2], dst;
2332		u8 iv[MAX_IVLEN];
2333		DECLARE_CRYPTO_WAIT(wait);
2334
2335		/* Generate a random plaintext and encrypt it. */
2336		sg_init_table(src, 2);
2337		if (vec->alen)
2338			sg_set_buf(&src[i++], vec->assoc, vec->alen);
2339		if (vec->plen) {
2340			generate_random_bytes(rng, (u8 *)vec->ptext, vec->plen);
2341			sg_set_buf(&src[i++], vec->ptext, vec->plen);
2342		}
2343		sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
2344		memcpy(iv, vec->iv, ivsize);
2345		aead_request_set_callback(req, 0, crypto_req_done, &wait);
2346		aead_request_set_crypt(req, src, &dst, vec->plen, iv);
2347		aead_request_set_ad(req, vec->alen);
2348		vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req),
2349						   &wait);
2350		/* If encryption failed, we're done. */
2351		if (vec->crypt_error != 0)
2352			return;
2353		memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
2354		if (!inauthentic)
2355			return;
2356		/*
2357		 * Mutate the authentic (ciphertext, AAD) pair to get an
2358		 * inauthentic one.
2359		 */
2360		mutate_aead_message(rng, vec, suite->aad_iv, ivsize);
2361	}
2362	vec->novrfy = 1;
2363	if (suite->einval_allowed)
2364		vec->crypt_error = -EINVAL;
2365}
2366
2367/*
2368 * Generate an AEAD test vector 'vec' using the implementation specified by
2369 * 'req'.  The buffers in 'vec' must already be allocated.
2370 *
2371 * If 'prefer_inauthentic' is true, then this function will generate inauthentic
2372 * test vectors (i.e. vectors with 'vec->novrfy=1') more often.
2373 */
2374static void generate_random_aead_testvec(struct rnd_state *rng,
2375					 struct aead_request *req,
2376					 struct aead_testvec *vec,
2377					 const struct aead_test_suite *suite,
2378					 unsigned int maxkeysize,
2379					 unsigned int maxdatasize,
2380					 char *name, size_t max_namelen,
2381					 bool prefer_inauthentic)
2382{
2383	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2384	const unsigned int ivsize = crypto_aead_ivsize(tfm);
2385	const unsigned int maxauthsize = crypto_aead_maxauthsize(tfm);
2386	unsigned int authsize;
2387	unsigned int total_len;
2388
2389	/* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2390	vec->klen = maxkeysize;
2391	if (prandom_u32_below(rng, 4) == 0)
2392		vec->klen = prandom_u32_below(rng, maxkeysize + 1);
2393	generate_random_bytes(rng, (u8 *)vec->key, vec->klen);
2394	vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
2395
2396	/* IV */
2397	generate_random_bytes(rng, (u8 *)vec->iv, ivsize);
2398
2399	/* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
2400	authsize = maxauthsize;
2401	if (prandom_u32_below(rng, 4) == 0)
2402		authsize = prandom_u32_below(rng, maxauthsize + 1);
2403	if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
2404		authsize = MIN_COLLISION_FREE_AUTHSIZE;
2405	if (WARN_ON(authsize > maxdatasize))
2406		authsize = maxdatasize;
2407	maxdatasize -= authsize;
2408	vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
2409
2410	/* AAD, plaintext, and ciphertext lengths */
2411	total_len = generate_random_length(rng, maxdatasize);
2412	if (prandom_u32_below(rng, 4) == 0)
2413		vec->alen = 0;
2414	else
2415		vec->alen = generate_random_length(rng, total_len);
2416	vec->plen = total_len - vec->alen;
2417	vec->clen = vec->plen + authsize;
2418
2419	/*
2420	 * Generate the AAD, plaintext, and ciphertext.  Not applicable if the
2421	 * key or the authentication tag size couldn't be set.
2422	 */
2423	vec->novrfy = 0;
2424	vec->crypt_error = 0;
2425	if (vec->setkey_error == 0 && vec->setauthsize_error == 0)
2426		generate_aead_message(rng, req, suite, vec, prefer_inauthentic);
2427	snprintf(name, max_namelen,
2428		 "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",
2429		 vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);
2430}
2431
2432static void try_to_generate_inauthentic_testvec(struct aead_slow_tests_ctx *ctx)
2433{
2434	int i;
2435
2436	for (i = 0; i < 10; i++) {
2437		generate_random_aead_testvec(&ctx->rng, ctx->req, &ctx->vec,
2438					     &ctx->test_desc->suite.aead,
2439					     ctx->maxkeysize, ctx->maxdatasize,
2440					     ctx->vec_name,
2441					     sizeof(ctx->vec_name), true);
2442		if (ctx->vec.novrfy)
2443			return;
2444	}
2445}
2446
2447/*
2448 * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the
2449 * result of an encryption with the key) and verify that decryption fails.
2450 */
2451static int test_aead_inauthentic_inputs(struct aead_slow_tests_ctx *ctx)
2452{
2453	unsigned int i;
2454	int err;
2455
2456	for (i = 0; i < fuzz_iterations * 8; i++) {
2457		/*
2458		 * Since this part of the tests isn't comparing the
2459		 * implementation to another, there's no point in testing any
2460		 * test vectors other than inauthentic ones (vec.novrfy=1) here.
2461		 *
2462		 * If we're having trouble generating such a test vector, e.g.
2463		 * if the algorithm keeps rejecting the generated keys, don't
2464		 * retry forever; just continue on.
2465		 */
2466		try_to_generate_inauthentic_testvec(ctx);
2467		if (ctx->vec.novrfy) {
2468			generate_random_testvec_config(&ctx->rng, &ctx->cfg,
2469						       ctx->cfgname,
2470						       sizeof(ctx->cfgname));
2471			err = test_aead_vec_cfg(DECRYPT, &ctx->vec,
2472						ctx->vec_name, &ctx->cfg,
2473						ctx->req, ctx->tsgls);
2474			if (err)
2475				return err;
2476		}
2477		cond_resched();
2478	}
2479	return 0;
2480}
2481
2482/*
2483 * Test the AEAD algorithm against the corresponding generic implementation, if
2484 * one is available.
2485 */
2486static int test_aead_vs_generic_impl(struct aead_slow_tests_ctx *ctx)
2487{
2488	struct crypto_aead *tfm = ctx->tfm;
2489	const char *algname = crypto_aead_alg(tfm)->base.cra_name;
2490	const char *driver = crypto_aead_driver_name(tfm);
2491	const char *generic_driver = ctx->test_desc->generic_driver;
2492	char _generic_driver[CRYPTO_MAX_ALG_NAME];
2493	struct crypto_aead *generic_tfm = NULL;
2494	struct aead_request *generic_req = NULL;
2495	unsigned int i;
2496	int err;
2497
2498	if (!generic_driver) { /* Use default naming convention? */
2499		err = build_generic_driver_name(algname, _generic_driver);
2500		if (err)
2501			return err;
2502		generic_driver = _generic_driver;
2503	}
2504
2505	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2506		return 0;
2507
2508	generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);
2509	if (IS_ERR(generic_tfm)) {
2510		err = PTR_ERR(generic_tfm);
2511		if (err == -ENOENT) {
2512			pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
2513				driver, generic_driver);
2514			return 0;
2515		}
2516		pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
2517		       generic_driver, algname, err);
2518		return err;
2519	}
2520
2521	generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
2522	if (!generic_req) {
2523		err = -ENOMEM;
2524		goto out;
2525	}
2526
2527	/* Check the algorithm properties for consistency. */
2528
2529	if (crypto_aead_maxauthsize(tfm) !=
2530	    crypto_aead_maxauthsize(generic_tfm)) {
2531		pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
2532		       driver, crypto_aead_maxauthsize(tfm),
2533		       crypto_aead_maxauthsize(generic_tfm));
2534		err = -EINVAL;
2535		goto out;
2536	}
2537
2538	if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(generic_tfm)) {
2539		pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
2540		       driver, crypto_aead_ivsize(tfm),
2541		       crypto_aead_ivsize(generic_tfm));
2542		err = -EINVAL;
2543		goto out;
2544	}
2545
2546	if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(generic_tfm)) {
2547		pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
2548		       driver, crypto_aead_blocksize(tfm),
2549		       crypto_aead_blocksize(generic_tfm));
2550		err = -EINVAL;
2551		goto out;
2552	}
2553
2554	/*
2555	 * Now generate test vectors using the generic implementation, and test
2556	 * the other implementation against them.
2557	 */
2558	for (i = 0; i < fuzz_iterations * 8; i++) {
2559		generate_random_aead_testvec(&ctx->rng, generic_req, &ctx->vec,
2560					     &ctx->test_desc->suite.aead,
2561					     ctx->maxkeysize, ctx->maxdatasize,
2562					     ctx->vec_name,
2563					     sizeof(ctx->vec_name), false);
2564		generate_random_testvec_config(&ctx->rng, &ctx->cfg,
2565					       ctx->cfgname,
2566					       sizeof(ctx->cfgname));
2567		if (!ctx->vec.novrfy) {
2568			err = test_aead_vec_cfg(ENCRYPT, &ctx->vec,
2569						ctx->vec_name, &ctx->cfg,
2570						ctx->req, ctx->tsgls);
2571			if (err)
2572				goto out;
2573		}
2574		if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) {
2575			err = test_aead_vec_cfg(DECRYPT, &ctx->vec,
2576						ctx->vec_name, &ctx->cfg,
2577						ctx->req, ctx->tsgls);
2578			if (err)
2579				goto out;
2580		}
2581		cond_resched();
2582	}
2583	err = 0;
2584out:
2585	crypto_free_aead(generic_tfm);
2586	aead_request_free(generic_req);
2587	return err;
2588}
2589
2590static int test_aead_slow(const struct alg_test_desc *test_desc,
2591			  struct aead_request *req,
2592			  struct cipher_test_sglists *tsgls)
2593{
2594	struct aead_slow_tests_ctx *ctx;
2595	unsigned int i;
2596	int err;
2597
2598	if (noslowtests)
2599		return 0;
2600
2601	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2602	if (!ctx)
2603		return -ENOMEM;
2604	init_rnd_state(&ctx->rng);
2605	ctx->req = req;
2606	ctx->tfm = crypto_aead_reqtfm(req);
2607	ctx->test_desc = test_desc;
2608	ctx->tsgls = tsgls;
2609	ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2610	ctx->maxkeysize = 0;
2611	for (i = 0; i < test_desc->suite.aead.count; i++)
2612		ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,
2613					test_desc->suite.aead.vecs[i].klen);
2614
2615	ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);
2616	ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);
2617	ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2618	ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2619	ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2620	if (!ctx->vec.key || !ctx->vec.iv || !ctx->vec.assoc ||
2621	    !ctx->vec.ptext || !ctx->vec.ctext) {
2622		err = -ENOMEM;
2623		goto out;
2624	}
2625
2626	err = test_aead_vs_generic_impl(ctx);
2627	if (err)
2628		goto out;
2629
2630	err = test_aead_inauthentic_inputs(ctx);
2631out:
2632	kfree(ctx->vec.key);
2633	kfree(ctx->vec.iv);
2634	kfree(ctx->vec.assoc);
2635	kfree(ctx->vec.ptext);
2636	kfree(ctx->vec.ctext);
2637	kfree(ctx);
2638	return err;
2639}
2640
2641static int test_aead(int enc, const struct aead_test_suite *suite,
2642		     struct aead_request *req,
2643		     struct cipher_test_sglists *tsgls)
2644{
2645	unsigned int i;
2646	int err;
2647
2648	for (i = 0; i < suite->count; i++) {
2649		err = test_aead_vec(enc, &suite->vecs[i], i, req, tsgls);
2650		if (err)
2651			return err;
2652		cond_resched();
2653	}
2654	return 0;
2655}
2656
2657static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
2658			 u32 type, u32 mask)
2659{
2660	const struct aead_test_suite *suite = &desc->suite.aead;
2661	struct crypto_aead *tfm;
2662	struct aead_request *req = NULL;
2663	struct cipher_test_sglists *tsgls = NULL;
2664	int err;
2665
2666	if (suite->count <= 0) {
2667		pr_err("alg: aead: empty test suite for %s\n", driver);
2668		return -EINVAL;
2669	}
2670
2671	tfm = crypto_alloc_aead(driver, type, mask);
2672	if (IS_ERR(tfm)) {
2673		if (PTR_ERR(tfm) == -ENOENT)
2674			return 0;
2675		pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
2676		       driver, PTR_ERR(tfm));
2677		return PTR_ERR(tfm);
2678	}
2679	driver = crypto_aead_driver_name(tfm);
2680
2681	req = aead_request_alloc(tfm, GFP_KERNEL);
2682	if (!req) {
2683		pr_err("alg: aead: failed to allocate request for %s\n",
2684		       driver);
2685		err = -ENOMEM;
2686		goto out;
2687	}
2688
2689	tsgls = alloc_cipher_test_sglists();
2690	if (!tsgls) {
2691		pr_err("alg: aead: failed to allocate test buffers for %s\n",
2692		       driver);
2693		err = -ENOMEM;
2694		goto out;
2695	}
2696
2697	err = test_aead(ENCRYPT, suite, req, tsgls);
2698	if (err)
2699		goto out;
2700
2701	err = test_aead(DECRYPT, suite, req, tsgls);
2702	if (err)
2703		goto out;
2704
2705	err = test_aead_slow(desc, req, tsgls);
2706out:
2707	free_cipher_test_sglists(tsgls);
2708	aead_request_free(req);
2709	crypto_free_aead(tfm);
2710	return err;
2711}
2712
2713static int test_cipher(struct crypto_cipher *tfm, int enc,
2714		       const struct cipher_testvec *template,
2715		       unsigned int tcount)
2716{
2717	const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm));
2718	unsigned int i, j, k;
2719	char *q;
2720	const char *e;
2721	const char *input, *result;
2722	void *data;
2723	char *xbuf[XBUFSIZE];
2724	int ret = -ENOMEM;
2725
2726	if (testmgr_alloc_buf(xbuf))
2727		goto out_nobuf;
2728
2729	if (enc == ENCRYPT)
2730	        e = "encryption";
2731	else
2732		e = "decryption";
2733
2734	j = 0;
2735	for (i = 0; i < tcount; i++) {
2736
2737		if (fips_enabled && template[i].fips_skip)
2738			continue;
2739
2740		input  = enc ? template[i].ptext : template[i].ctext;
2741		result = enc ? template[i].ctext : template[i].ptext;
2742		j++;
2743
2744		ret = -EINVAL;
2745		if (WARN_ON(template[i].len > PAGE_SIZE))
2746			goto out;
2747
2748		data = xbuf[0];
2749		memcpy(data, input, template[i].len);
2750
2751		crypto_cipher_clear_flags(tfm, ~0);
2752		if (template[i].wk)
2753			crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2754
2755		ret = crypto_cipher_setkey(tfm, template[i].key,
2756					   template[i].klen);
2757		if (ret) {
2758			if (ret == template[i].setkey_error)
2759				continue;
2760			pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
2761			       algo, j, template[i].setkey_error, ret,
2762			       crypto_cipher_get_flags(tfm));
2763			goto out;
2764		}
2765		if (template[i].setkey_error) {
2766			pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
2767			       algo, j, template[i].setkey_error);
2768			ret = -EINVAL;
2769			goto out;
2770		}
2771
2772		for (k = 0; k < template[i].len;
2773		     k += crypto_cipher_blocksize(tfm)) {
2774			if (enc)
2775				crypto_cipher_encrypt_one(tfm, data + k,
2776							  data + k);
2777			else
2778				crypto_cipher_decrypt_one(tfm, data + k,
2779							  data + k);
2780		}
2781
2782		q = data;
2783		if (memcmp(q, result, template[i].len)) {
2784			printk(KERN_ERR "alg: cipher: Test %d failed "
2785			       "on %s for %s\n", j, e, algo);
2786			hexdump(q, template[i].len);
2787			ret = -EINVAL;
2788			goto out;
2789		}
2790	}
2791
2792	ret = 0;
2793
2794out:
2795	testmgr_free_buf(xbuf);
2796out_nobuf:
2797	return ret;
2798}
2799
2800static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec,
2801				 const char *vec_name,
2802				 const struct testvec_config *cfg,
2803				 struct skcipher_request *req,
2804				 struct cipher_test_sglists *tsgls)
2805{
2806	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2807	const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
2808	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2809	const char *driver = crypto_skcipher_driver_name(tfm);
2810	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2811	const char *op = enc ? "encryption" : "decryption";
2812	DECLARE_CRYPTO_WAIT(wait);
2813	u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2814	u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2815		 cfg->iv_offset +
2816		 (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2817	struct kvec input;
2818	int err;
2819
2820	/* Set the key */
2821	if (vec->wk)
2822		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2823	else
2824		crypto_skcipher_clear_flags(tfm,
2825					    CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2826	err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,
2827			cfg, alignmask);
2828	if (err) {
2829		if (err == vec->setkey_error)
2830			return 0;
2831		pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2832		       driver, vec_name, vec->setkey_error, err,
2833		       crypto_skcipher_get_flags(tfm));
2834		return err;
2835	}
2836	if (vec->setkey_error) {
2837		pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2838		       driver, vec_name, vec->setkey_error);
2839		return -EINVAL;
2840	}
2841
2842	/* The IV must be copied to a buffer, as the algorithm may modify it */
2843	if (ivsize) {
2844		if (WARN_ON(ivsize > MAX_IVLEN))
2845			return -EINVAL;
2846		if (vec->iv)
2847			memcpy(iv, vec->iv, ivsize);
2848		else
2849			memset(iv, 0, ivsize);
2850	} else {
2851		iv = NULL;
2852	}
2853
2854	/* Build the src/dst scatterlists */
2855	input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2856	input.iov_len = vec->len;
2857	err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2858					vec->len, vec->len, &input, 1);
2859	if (err) {
2860		pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2861		       driver, op, vec_name, cfg->name);
2862		return err;
2863	}
2864
2865	/* Do the actual encryption or decryption */
2866	testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));
2867	skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);
2868	skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2869				   vec->len, iv);
2870	if (cfg->nosimd)
2871		crypto_disable_simd_for_test();
2872	err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
2873	if (cfg->nosimd)
2874		crypto_reenable_simd_for_test();
2875	err = crypto_wait_req(err, &wait);
2876
2877	/* Check that the algorithm didn't overwrite things it shouldn't have */
2878	if (req->cryptlen != vec->len ||
2879	    req->iv != iv ||
2880	    req->src != tsgls->src.sgl_ptr ||
2881	    req->dst != tsgls->dst.sgl_ptr ||
2882	    crypto_skcipher_reqtfm(req) != tfm ||
2883	    req->base.complete != crypto_req_done ||
2884	    req->base.flags != req_flags ||
2885	    req->base.data != &wait) {
2886		pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2887		       driver, op, vec_name, cfg->name);
2888		if (req->cryptlen != vec->len)
2889			pr_err("alg: skcipher: changed 'req->cryptlen'\n");
2890		if (req->iv != iv)
2891			pr_err("alg: skcipher: changed 'req->iv'\n");
2892		if (req->src != tsgls->src.sgl_ptr)
2893			pr_err("alg: skcipher: changed 'req->src'\n");
2894		if (req->dst != tsgls->dst.sgl_ptr)
2895			pr_err("alg: skcipher: changed 'req->dst'\n");
2896		if (crypto_skcipher_reqtfm(req) != tfm)
2897			pr_err("alg: skcipher: changed 'req->base.tfm'\n");
2898		if (req->base.complete != crypto_req_done)
2899			pr_err("alg: skcipher: changed 'req->base.complete'\n");
2900		if (req->base.flags != req_flags)
2901			pr_err("alg: skcipher: changed 'req->base.flags'\n");
2902		if (req->base.data != &wait)
2903			pr_err("alg: skcipher: changed 'req->base.data'\n");
2904		return -EINVAL;
2905	}
2906	if (is_test_sglist_corrupted(&tsgls->src)) {
2907		pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2908		       driver, op, vec_name, cfg->name);
2909		return -EINVAL;
2910	}
2911	if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2912	    is_test_sglist_corrupted(&tsgls->dst)) {
2913		pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2914		       driver, op, vec_name, cfg->name);
2915		return -EINVAL;
2916	}
2917
2918	/* Check for success or failure */
2919	if (err) {
2920		if (err == vec->crypt_error)
2921			return 0;
2922		pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
2923		       driver, op, vec_name, vec->crypt_error, err, cfg->name);
2924		return err;
2925	}
2926	if (vec->crypt_error) {
2927		pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
2928		       driver, op, vec_name, vec->crypt_error, cfg->name);
2929		return -EINVAL;
2930	}
2931
2932	/* Check for the correct output (ciphertext or plaintext) */
2933	err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2934				    vec->len, 0, true);
2935	if (err == -EOVERFLOW) {
2936		pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2937		       driver, op, vec_name, cfg->name);
2938		return err;
2939	}
2940	if (err) {
2941		pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2942		       driver, op, vec_name, cfg->name);
2943		return err;
2944	}
2945
2946	/* If applicable, check that the algorithm generated the correct IV */
2947	if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {
2948		pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
2949		       driver, op, vec_name, cfg->name);
2950		hexdump(iv, ivsize);
2951		return -EINVAL;
2952	}
2953
2954	return 0;
2955}
2956
2957static int test_skcipher_vec(int enc, const struct cipher_testvec *vec,
2958			     unsigned int vec_num,
2959			     struct skcipher_request *req,
2960			     struct cipher_test_sglists *tsgls)
2961{
2962	char vec_name[16];
2963	unsigned int i;
2964	int err;
2965
2966	if (fips_enabled && vec->fips_skip)
2967		return 0;
2968
2969	sprintf(vec_name, "%u", vec_num);
2970
2971	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2972		err = test_skcipher_vec_cfg(enc, vec, vec_name,
2973					    &default_cipher_testvec_configs[i],
2974					    req, tsgls);
2975		if (err)
2976			return err;
2977	}
2978
2979	if (!noslowtests) {
2980		struct rnd_state rng;
2981		struct testvec_config cfg;
2982		char cfgname[TESTVEC_CONFIG_NAMELEN];
2983
2984		init_rnd_state(&rng);
2985
2986		for (i = 0; i < fuzz_iterations; i++) {
2987			generate_random_testvec_config(&rng, &cfg, cfgname,
2988						       sizeof(cfgname));
2989			err = test_skcipher_vec_cfg(enc, vec, vec_name,
2990						    &cfg, req, tsgls);
2991			if (err)
2992				return err;
2993			cond_resched();
2994		}
2995	}
2996	return 0;
2997}
2998
2999/*
3000 * Generate a symmetric cipher test vector from the given implementation.
3001 * Assumes the buffers in 'vec' were already allocated.
3002 */
3003static void generate_random_cipher_testvec(struct rnd_state *rng,
3004					   struct skcipher_request *req,
3005					   struct cipher_testvec *vec,
3006					   unsigned int maxdatasize,
3007					   char *name, size_t max_namelen)
3008{
3009	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3010	const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
3011	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
3012	struct scatterlist src, dst;
3013	u8 iv[MAX_IVLEN];
3014	DECLARE_CRYPTO_WAIT(wait);
3015
3016	/* Key: length in [0, maxkeysize], but usually choose maxkeysize */
3017	vec->klen = maxkeysize;
3018	if (prandom_u32_below(rng, 4) == 0)
3019		vec->klen = prandom_u32_below(rng, maxkeysize + 1);
3020	generate_random_bytes(rng, (u8 *)vec->key, vec->klen);
3021	vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
3022
3023	/* IV */
3024	generate_random_bytes(rng, (u8 *)vec->iv, ivsize);
3025
3026	/* Plaintext */
3027	vec->len = generate_random_length(rng, maxdatasize);
3028	generate_random_bytes(rng, (u8 *)vec->ptext, vec->len);
3029
3030	/* If the key couldn't be set, no need to continue to encrypt. */
3031	if (vec->setkey_error)
3032		goto done;
3033
3034	/* Ciphertext */
3035	sg_init_one(&src, vec->ptext, vec->len);
3036	sg_init_one(&dst, vec->ctext, vec->len);
3037	memcpy(iv, vec->iv, ivsize);
3038	skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
3039	skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);
3040	vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
3041	if (vec->crypt_error != 0) {
3042		/*
3043		 * The only acceptable error here is for an invalid length, so
3044		 * skcipher decryption should fail with the same error too.
3045		 * We'll test for this.  But to keep the API usage well-defined,
3046		 * explicitly initialize the ciphertext buffer too.
3047		 */
3048		memset((u8 *)vec->ctext, 0, vec->len);
3049	}
3050done:
3051	snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",
3052		 vec->len, vec->klen);
3053}
3054
3055/*
3056 * Test the skcipher algorithm represented by @req against the corresponding
3057 * generic implementation, if one is available.
3058 */
3059static int test_skcipher_vs_generic_impl(const char *generic_driver,
3060					 struct skcipher_request *req,
3061					 struct cipher_test_sglists *tsgls)
3062{
3063	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
3064	const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
3065	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
3066	const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
3067	const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
3068	const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
3069	const char *driver = crypto_skcipher_driver_name(tfm);
3070	struct rnd_state rng;
3071	char _generic_driver[CRYPTO_MAX_ALG_NAME];
3072	struct crypto_skcipher *generic_tfm = NULL;
3073	struct skcipher_request *generic_req = NULL;
3074	unsigned int i;
3075	struct cipher_testvec vec = { 0 };
3076	char vec_name[64];
3077	struct testvec_config *cfg;
3078	char cfgname[TESTVEC_CONFIG_NAMELEN];
3079	int err;
3080
3081	if (noslowtests)
3082		return 0;
3083
3084	init_rnd_state(&rng);
3085
3086	if (!generic_driver) { /* Use default naming convention? */
3087		err = build_generic_driver_name(algname, _generic_driver);
3088		if (err)
3089			return err;
3090		generic_driver = _generic_driver;
3091	}
3092
3093	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
3094		return 0;
3095
3096	generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);
3097	if (IS_ERR(generic_tfm)) {
3098		err = PTR_ERR(generic_tfm);
3099		if (err == -ENOENT) {
3100			pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
3101				driver, generic_driver);
3102			return 0;
3103		}
3104		pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
3105		       generic_driver, algname, err);
3106		return err;
3107	}
3108
3109	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
3110	if (!cfg) {
3111		err = -ENOMEM;
3112		goto out;
3113	}
3114
3115	generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
3116	if (!generic_req) {
3117		err = -ENOMEM;
3118		goto out;
3119	}
3120
3121	/* Check the algorithm properties for consistency. */
3122
3123	if (crypto_skcipher_min_keysize(tfm) !=
3124	    crypto_skcipher_min_keysize(generic_tfm)) {
3125		pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",
3126		       driver, crypto_skcipher_min_keysize(tfm),
3127		       crypto_skcipher_min_keysize(generic_tfm));
3128		err = -EINVAL;
3129		goto out;
3130	}
3131
3132	if (maxkeysize != crypto_skcipher_max_keysize(generic_tfm)) {
3133		pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
3134		       driver, maxkeysize,
3135		       crypto_skcipher_max_keysize(generic_tfm));
3136		err = -EINVAL;
3137		goto out;
3138	}
3139
3140	if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {
3141		pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
3142		       driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
3143		err = -EINVAL;
3144		goto out;
3145	}
3146
3147	if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {
3148		pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
3149		       driver, blocksize,
3150		       crypto_skcipher_blocksize(generic_tfm));
3151		err = -EINVAL;
3152		goto out;
3153	}
3154
3155	/*
3156	 * Now generate test vectors using the generic implementation, and test
3157	 * the other implementation against them.
3158	 */
3159
3160	vec.key = kmalloc(maxkeysize, GFP_KERNEL);
3161	vec.iv = kmalloc(ivsize, GFP_KERNEL);
3162	vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
3163	vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
3164	if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {
3165		err = -ENOMEM;
3166		goto out;
3167	}
3168
3169	for (i = 0; i < fuzz_iterations * 8; i++) {
3170		generate_random_cipher_testvec(&rng, generic_req, &vec,
3171					       maxdatasize,
3172					       vec_name, sizeof(vec_name));
3173		generate_random_testvec_config(&rng, cfg, cfgname,
3174					       sizeof(cfgname));
3175
3176		err = test_skcipher_vec_cfg(ENCRYPT, &vec, vec_name,
3177					    cfg, req, tsgls);
3178		if (err)
3179			goto out;
3180		err = test_skcipher_vec_cfg(DECRYPT, &vec, vec_name,
3181					    cfg, req, tsgls);
3182		if (err)
3183			goto out;
3184		cond_resched();
3185	}
3186	err = 0;
3187out:
3188	kfree(cfg);
3189	kfree(vec.key);
3190	kfree(vec.iv);
3191	kfree(vec.ptext);
3192	kfree(vec.ctext);
3193	crypto_free_skcipher(generic_tfm);
3194	skcipher_request_free(generic_req);
3195	return err;
3196}
3197
3198static int test_skcipher(int enc, const struct cipher_test_suite *suite,
3199			 struct skcipher_request *req,
3200			 struct cipher_test_sglists *tsgls)
3201{
3202	unsigned int i;
3203	int err;
3204
3205	for (i = 0; i < suite->count; i++) {
3206		err = test_skcipher_vec(enc, &suite->vecs[i], i, req, tsgls);
3207		if (err)
3208			return err;
3209		cond_resched();
3210	}
3211	return 0;
3212}
3213
3214static int alg_test_skcipher(const struct alg_test_desc *desc,
3215			     const char *driver, u32 type, u32 mask)
3216{
3217	const struct cipher_test_suite *suite = &desc->suite.cipher;
3218	struct crypto_skcipher *tfm;
3219	struct skcipher_request *req = NULL;
3220	struct cipher_test_sglists *tsgls = NULL;
3221	int err;
3222
3223	if (suite->count <= 0) {
3224		pr_err("alg: skcipher: empty test suite for %s\n", driver);
3225		return -EINVAL;
3226	}
3227
3228	tfm = crypto_alloc_skcipher(driver, type, mask);
3229	if (IS_ERR(tfm)) {
3230		if (PTR_ERR(tfm) == -ENOENT)
3231			return 0;
3232		pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
3233		       driver, PTR_ERR(tfm));
3234		return PTR_ERR(tfm);
3235	}
3236	driver = crypto_skcipher_driver_name(tfm);
3237
3238	req = skcipher_request_alloc(tfm, GFP_KERNEL);
3239	if (!req) {
3240		pr_err("alg: skcipher: failed to allocate request for %s\n",
3241		       driver);
3242		err = -ENOMEM;
3243		goto out;
3244	}
3245
3246	tsgls = alloc_cipher_test_sglists();
3247	if (!tsgls) {
3248		pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
3249		       driver);
3250		err = -ENOMEM;
3251		goto out;
3252	}
3253
3254	err = test_skcipher(ENCRYPT, suite, req, tsgls);
3255	if (err)
3256		goto out;
3257
3258	err = test_skcipher(DECRYPT, suite, req, tsgls);
3259	if (err)
3260		goto out;
3261
3262	err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls);
3263out:
3264	free_cipher_test_sglists(tsgls);
3265	skcipher_request_free(req);
3266	crypto_free_skcipher(tfm);
3267	return err;
3268}
3269
3270static int test_acomp(struct crypto_acomp *tfm,
3271		      const struct comp_testvec *ctemplate,
3272		      const struct comp_testvec *dtemplate,
3273		      int ctcount, int dtcount)
3274{
3275	const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
3276	unsigned int i;
3277	char *output, *decomp_out;
3278	int ret;
3279	struct scatterlist src, dst;
3280	struct acomp_req *req;
3281	struct crypto_wait wait;
3282
3283	output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3284	if (!output)
3285		return -ENOMEM;
3286
3287	decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3288	if (!decomp_out) {
3289		kfree(output);
3290		return -ENOMEM;
3291	}
3292
3293	for (i = 0; i < ctcount; i++) {
3294		unsigned int dlen = COMP_BUF_SIZE;
3295		int ilen = ctemplate[i].inlen;
3296		void *input_vec;
3297
3298		input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
3299		if (!input_vec) {
3300			ret = -ENOMEM;
3301			goto out;
3302		}
3303
3304		memset(output, 0, dlen);
3305		crypto_init_wait(&wait);
3306		sg_init_one(&src, input_vec, ilen);
3307		sg_init_one(&dst, output, dlen);
3308
3309		req = acomp_request_alloc(tfm);
3310		if (!req) {
3311			pr_err("alg: acomp: request alloc failed for %s\n",
3312			       algo);
3313			kfree(input_vec);
3314			ret = -ENOMEM;
3315			goto out;
3316		}
3317
3318		acomp_request_set_params(req, &src, &dst, ilen, dlen);
3319		acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3320					   crypto_req_done, &wait);
3321
3322		ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
3323		if (ret) {
3324			pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3325			       i + 1, algo, -ret);
3326			kfree(input_vec);
3327			acomp_request_free(req);
3328			goto out;
3329		}
3330
3331		ilen = req->dlen;
3332		dlen = COMP_BUF_SIZE;
3333		sg_init_one(&src, output, ilen);
3334		sg_init_one(&dst, decomp_out, dlen);
3335		crypto_init_wait(&wait);
3336		acomp_request_set_params(req, &src, &dst, ilen, dlen);
3337
3338		ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3339		if (ret) {
3340			pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3341			       i + 1, algo, -ret);
3342			kfree(input_vec);
3343			acomp_request_free(req);
3344			goto out;
3345		}
3346
3347		if (req->dlen != ctemplate[i].inlen) {
3348			pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
3349			       i + 1, algo, req->dlen);
3350			ret = -EINVAL;
3351			kfree(input_vec);
3352			acomp_request_free(req);
3353			goto out;
3354		}
3355
3356		if (memcmp(input_vec, decomp_out, req->dlen)) {
3357			pr_err("alg: acomp: Compression test %d failed for %s\n",
3358			       i + 1, algo);
3359			hexdump(output, req->dlen);
3360			ret = -EINVAL;
3361			kfree(input_vec);
3362			acomp_request_free(req);
3363			goto out;
3364		}
3365
3366		kfree(input_vec);
3367		acomp_request_free(req);
3368	}
3369
3370	for (i = 0; i < dtcount; i++) {
3371		unsigned int dlen = COMP_BUF_SIZE;
3372		int ilen = dtemplate[i].inlen;
3373		void *input_vec;
3374
3375		input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
3376		if (!input_vec) {
3377			ret = -ENOMEM;
3378			goto out;
3379		}
3380
3381		memset(output, 0, dlen);
3382		crypto_init_wait(&wait);
3383		sg_init_one(&src, input_vec, ilen);
3384		sg_init_one(&dst, output, dlen);
3385
3386		req = acomp_request_alloc(tfm);
3387		if (!req) {
3388			pr_err("alg: acomp: request alloc failed for %s\n",
3389			       algo);
3390			kfree(input_vec);
3391			ret = -ENOMEM;
3392			goto out;
3393		}
3394
3395		acomp_request_set_params(req, &src, &dst, ilen, dlen);
3396		acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3397					   crypto_req_done, &wait);
3398
3399		ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3400		if (ret) {
3401			pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
3402			       i + 1, algo, -ret);
3403			kfree(input_vec);
3404			acomp_request_free(req);
3405			goto out;
3406		}
3407
3408		if (req->dlen != dtemplate[i].outlen) {
3409			pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
3410			       i + 1, algo, req->dlen);
3411			ret = -EINVAL;
3412			kfree(input_vec);
3413			acomp_request_free(req);
3414			goto out;
3415		}
3416
3417		if (memcmp(output, dtemplate[i].output, req->dlen)) {
3418			pr_err("alg: acomp: Decompression test %d failed for %s\n",
3419			       i + 1, algo);
3420			hexdump(output, req->dlen);
3421			ret = -EINVAL;
3422			kfree(input_vec);
3423			acomp_request_free(req);
3424			goto out;
3425		}
3426
3427		kfree(input_vec);
3428		acomp_request_free(req);
3429	}
3430
3431	ret = 0;
3432
3433out:
3434	kfree(decomp_out);
3435	kfree(output);
3436	return ret;
3437}
3438
3439static int alg_test_cipher(const struct alg_test_desc *desc,
3440			   const char *driver, u32 type, u32 mask)
3441{
3442	const struct cipher_test_suite *suite = &desc->suite.cipher;
3443	struct crypto_cipher *tfm;
3444	int err;
3445
3446	tfm = crypto_alloc_cipher(driver, type, mask);
3447	if (IS_ERR(tfm)) {
3448		if (PTR_ERR(tfm) == -ENOENT)
3449			return 0;
3450		printk(KERN_ERR "alg: cipher: Failed to load transform for "
3451		       "%s: %ld\n", driver, PTR_ERR(tfm));
3452		return PTR_ERR(tfm);
3453	}
3454
3455	err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);
3456	if (!err)
3457		err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
3458
3459	crypto_free_cipher(tfm);
3460	return err;
3461}
3462
3463static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
3464			 u32 type, u32 mask)
3465{
3466	struct crypto_acomp *acomp;
3467	int err;
3468
3469	acomp = crypto_alloc_acomp(driver, type, mask);
3470	if (IS_ERR(acomp)) {
3471		if (PTR_ERR(acomp) == -ENOENT)
3472			return 0;
3473		pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
3474		       driver, PTR_ERR(acomp));
3475		return PTR_ERR(acomp);
3476	}
3477	err = test_acomp(acomp, desc->suite.comp.comp.vecs,
3478			 desc->suite.comp.decomp.vecs,
3479			 desc->suite.comp.comp.count,
3480			 desc->suite.comp.decomp.count);
3481	crypto_free_acomp(acomp);
3482	return err;
3483}
3484
3485static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
3486			  const char *driver, u32 type, u32 mask)
3487{
3488	int ret = -EAGAIN;
3489	struct crypto_rng *drng;
3490	struct drbg_test_data test_data;
3491	struct drbg_string addtl, pers, testentropy;
3492	unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
3493
3494	if (!buf)
3495		return -ENOMEM;
3496
3497	drng = crypto_alloc_rng(driver, type, mask);
3498	if (IS_ERR(drng)) {
3499		kfree_sensitive(buf);
3500		if (PTR_ERR(drng) == -ENOENT)
3501			return 0;
3502		printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
3503		       "%s\n", driver);
3504		return PTR_ERR(drng);
3505	}
3506
3507	test_data.testentropy = &testentropy;
3508	drbg_string_fill(&testentropy, test->entropy, test->entropylen);
3509	drbg_string_fill(&pers, test->pers, test->perslen);
3510	ret = crypto_drbg_reset_test(drng, &pers, &test_data);
3511	if (ret) {
3512		printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
3513		goto outbuf;
3514	}
3515
3516	drbg_string_fill(&addtl, test->addtla, test->addtllen);
3517	if (pr) {
3518		drbg_string_fill(&testentropy, test->entpra, test->entprlen);
3519		ret = crypto_drbg_get_bytes_addtl_test(drng,
3520			buf, test->expectedlen, &addtl,	&test_data);
3521	} else {
3522		ret = crypto_drbg_get_bytes_addtl(drng,
3523			buf, test->expectedlen, &addtl);
3524	}
3525	if (ret < 0) {
3526		printk(KERN_ERR "alg: drbg: could not obtain random data for "
3527		       "driver %s\n", driver);
3528		goto outbuf;
3529	}
3530
3531	drbg_string_fill(&addtl, test->addtlb, test->addtllen);
3532	if (pr) {
3533		drbg_string_fill(&testentropy, test->entprb, test->entprlen);
3534		ret = crypto_drbg_get_bytes_addtl_test(drng,
3535			buf, test->expectedlen, &addtl, &test_data);
3536	} else {
3537		ret = crypto_drbg_get_bytes_addtl(drng,
3538			buf, test->expectedlen, &addtl);
3539	}
3540	if (ret < 0) {
3541		printk(KERN_ERR "alg: drbg: could not obtain random data for "
3542		       "driver %s\n", driver);
3543		goto outbuf;
3544	}
3545
3546	ret = memcmp(test->expected, buf, test->expectedlen);
3547
3548outbuf:
3549	crypto_free_rng(drng);
3550	kfree_sensitive(buf);
3551	return ret;
3552}
3553
3554
3555static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
3556			 u32 type, u32 mask)
3557{
3558	int err = 0;
3559	int pr = 0;
3560	int i = 0;
3561	const struct drbg_testvec *template = desc->suite.drbg.vecs;
3562	unsigned int tcount = desc->suite.drbg.count;
3563
3564	if (0 == memcmp(driver, "drbg_pr_", 8))
3565		pr = 1;
3566
3567	for (i = 0; i < tcount; i++) {
3568		err = drbg_cavs_test(&template[i], pr, driver, type, mask);
3569		if (err) {
3570			printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
3571			       i, driver);
3572			err = -EINVAL;
3573			break;
3574		}
3575	}
3576	return err;
3577
3578}
3579
3580static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
3581		       const char *alg)
3582{
3583	struct kpp_request *req;
3584	void *input_buf = NULL;
3585	void *output_buf = NULL;
3586	void *a_public = NULL;
3587	void *a_ss = NULL;
3588	void *shared_secret = NULL;
3589	struct crypto_wait wait;
3590	unsigned int out_len_max;
3591	int err = -ENOMEM;
3592	struct scatterlist src, dst;
3593
3594	req = kpp_request_alloc(tfm, GFP_KERNEL);
3595	if (!req)
3596		return err;
3597
3598	crypto_init_wait(&wait);
3599
3600	err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);
3601	if (err < 0)
3602		goto free_req;
3603
3604	out_len_max = crypto_kpp_maxsize(tfm);
3605	output_buf = kzalloc(out_len_max, GFP_KERNEL);
3606	if (!output_buf) {
3607		err = -ENOMEM;
3608		goto free_req;
3609	}
3610
3611	/* Use appropriate parameter as base */
3612	kpp_request_set_input(req, NULL, 0);
3613	sg_init_one(&dst, output_buf, out_len_max);
3614	kpp_request_set_output(req, &dst, out_len_max);
3615	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3616				 crypto_req_done, &wait);
3617
3618	/* Compute party A's public key */
3619	err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait);
3620	if (err) {
3621		pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
3622		       alg, err);
3623		goto free_output;
3624	}
3625
3626	if (vec->genkey) {
3627		/* Save party A's public key */
3628		a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
3629		if (!a_public) {
3630			err = -ENOMEM;
3631			goto free_output;
3632		}
3633	} else {
3634		/* Verify calculated public key */
3635		if (memcmp(vec->expected_a_public, sg_virt(req->dst),
3636			   vec->expected_a_public_size)) {
3637			pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
3638			       alg);
3639			err = -EINVAL;
3640			goto free_output;
3641		}
3642	}
3643
3644	/* Calculate shared secret key by using counter part (b) public key. */
3645	input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
3646	if (!input_buf) {
3647		err = -ENOMEM;
3648		goto free_output;
3649	}
3650
3651	sg_init_one(&src, input_buf, vec->b_public_size);
3652	sg_init_one(&dst, output_buf, out_len_max);
3653	kpp_request_set_input(req, &src, vec->b_public_size);
3654	kpp_request_set_output(req, &dst, out_len_max);
3655	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3656				 crypto_req_done, &wait);
3657	err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);
3658	if (err) {
3659		pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
3660		       alg, err);
3661		goto free_all;
3662	}
3663
3664	if (vec->genkey) {
3665		/* Save the shared secret obtained by party A */
3666		a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
3667		if (!a_ss) {
3668			err = -ENOMEM;
3669			goto free_all;
3670		}
3671
3672		/*
3673		 * Calculate party B's shared secret by using party A's
3674		 * public key.
3675		 */
3676		err = crypto_kpp_set_secret(tfm, vec->b_secret,
3677					    vec->b_secret_size);
3678		if (err < 0)
3679			goto free_all;
3680
3681		sg_init_one(&src, a_public, vec->expected_a_public_size);
3682		sg_init_one(&dst, output_buf, out_len_max);
3683		kpp_request_set_input(req, &src, vec->expected_a_public_size);
3684		kpp_request_set_output(req, &dst, out_len_max);
3685		kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3686					 crypto_req_done, &wait);
3687		err = crypto_wait_req(crypto_kpp_compute_shared_secret(req),
3688				      &wait);
3689		if (err) {
3690			pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
3691			       alg, err);
3692			goto free_all;
3693		}
3694
3695		shared_secret = a_ss;
3696	} else {
3697		shared_secret = (void *)vec->expected_ss;
3698	}
3699
3700	/*
3701	 * verify shared secret from which the user will derive
3702	 * secret key by executing whatever hash it has chosen
3703	 */
3704	if (memcmp(shared_secret, sg_virt(req->dst),
3705		   vec->expected_ss_size)) {
3706		pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
3707		       alg);
3708		err = -EINVAL;
3709	}
3710
3711free_all:
3712	kfree(a_ss);
3713	kfree(input_buf);
3714free_output:
3715	kfree(a_public);
3716	kfree(output_buf);
3717free_req:
3718	kpp_request_free(req);
3719	return err;
3720}
3721
3722static int test_kpp(struct crypto_kpp *tfm, const char *alg,
3723		    const struct kpp_testvec *vecs, unsigned int tcount)
3724{
3725	int ret, i;
3726
3727	for (i = 0; i < tcount; i++) {
3728		ret = do_test_kpp(tfm, vecs++, alg);
3729		if (ret) {
3730			pr_err("alg: %s: test failed on vector %d, err=%d\n",
3731			       alg, i + 1, ret);
3732			return ret;
3733		}
3734	}
3735	return 0;
3736}
3737
3738static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
3739			u32 type, u32 mask)
3740{
3741	struct crypto_kpp *tfm;
3742	int err = 0;
3743
3744	tfm = crypto_alloc_kpp(driver, type, mask);
3745	if (IS_ERR(tfm)) {
3746		if (PTR_ERR(tfm) == -ENOENT)
3747			return 0;
3748		pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
3749		       driver, PTR_ERR(tfm));
3750		return PTR_ERR(tfm);
3751	}
3752	if (desc->suite.kpp.vecs)
3753		err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,
3754			       desc->suite.kpp.count);
3755
3756	crypto_free_kpp(tfm);
3757	return err;
3758}
3759
3760static u8 *test_pack_u32(u8 *dst, u32 val)
3761{
3762	memcpy(dst, &val, sizeof(val));
3763	return dst + sizeof(val);
3764}
3765
3766static int test_akcipher_one(struct crypto_akcipher *tfm,
3767			     const struct akcipher_testvec *vecs)
3768{
3769	char *xbuf[XBUFSIZE];
3770	struct akcipher_request *req;
3771	void *outbuf_enc = NULL;
3772	void *outbuf_dec = NULL;
3773	struct crypto_wait wait;
3774	unsigned int out_len_max, out_len = 0;
3775	int err = -ENOMEM;
3776	struct scatterlist src, dst, src_tab[2];
3777	const char *c;
3778	unsigned int c_size;
3779
3780	if (testmgr_alloc_buf(xbuf))
3781		return err;
3782
3783	req = akcipher_request_alloc(tfm, GFP_KERNEL);
3784	if (!req)
3785		goto free_xbuf;
3786
3787	crypto_init_wait(&wait);
3788
3789	if (vecs->public_key_vec)
3790		err = crypto_akcipher_set_pub_key(tfm, vecs->key,
3791						  vecs->key_len);
3792	else
3793		err = crypto_akcipher_set_priv_key(tfm, vecs->key,
3794						   vecs->key_len);
3795	if (err)
3796		goto free_req;
3797
3798	/* First run encrypt test which does not require a private key */
3799	err = -ENOMEM;
3800	out_len_max = crypto_akcipher_maxsize(tfm);
3801	outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
3802	if (!outbuf_enc)
3803		goto free_req;
3804
3805	c = vecs->c;
3806	c_size = vecs->c_size;
3807
3808	err = -E2BIG;
3809	if (WARN_ON(vecs->m_size > PAGE_SIZE))
3810		goto free_all;
3811	memcpy(xbuf[0], vecs->m, vecs->m_size);
3812
3813	sg_init_table(src_tab, 2);
3814	sg_set_buf(&src_tab[0], xbuf[0], 8);
3815	sg_set_buf(&src_tab[1], xbuf[0] + 8, vecs->m_size - 8);
3816	sg_init_one(&dst, outbuf_enc, out_len_max);
3817	akcipher_request_set_crypt(req, src_tab, &dst, vecs->m_size,
3818				   out_len_max);
3819	akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3820				      crypto_req_done, &wait);
3821
3822	err = crypto_wait_req(crypto_akcipher_encrypt(req), &wait);
3823	if (err) {
3824		pr_err("alg: akcipher: encrypt test failed. err %d\n", err);
3825		goto free_all;
3826	}
3827	if (c) {
3828		if (req->dst_len != c_size) {
3829			pr_err("alg: akcipher: encrypt test failed. Invalid output len\n");
3830			err = -EINVAL;
3831			goto free_all;
3832		}
3833		/* verify that encrypted message is equal to expected */
3834		if (memcmp(c, outbuf_enc, c_size) != 0) {
3835			pr_err("alg: akcipher: encrypt test failed. Invalid output\n");
3836			hexdump(outbuf_enc, c_size);
3837			err = -EINVAL;
3838			goto free_all;
3839		}
3840	}
3841
3842	/*
3843	 * Don't invoke decrypt test which requires a private key
3844	 * for vectors with only a public key.
3845	 */
3846	if (vecs->public_key_vec) {
3847		err = 0;
3848		goto free_all;
3849	}
3850	outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);
3851	if (!outbuf_dec) {
3852		err = -ENOMEM;
3853		goto free_all;
3854	}
3855
3856	if (!c) {
3857		c = outbuf_enc;
3858		c_size = req->dst_len;
3859	}
3860
3861	err = -E2BIG;
3862	if (WARN_ON(c_size > PAGE_SIZE))
3863		goto free_all;
3864	memcpy(xbuf[0], c, c_size);
3865
3866	sg_init_one(&src, xbuf[0], c_size);
3867	sg_init_one(&dst, outbuf_dec, out_len_max);
3868	crypto_init_wait(&wait);
3869	akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);
3870
3871	err = crypto_wait_req(crypto_akcipher_decrypt(req), &wait);
3872	if (err) {
3873		pr_err("alg: akcipher: decrypt test failed. err %d\n", err);
3874		goto free_all;
3875	}
3876	out_len = req->dst_len;
3877	if (out_len < vecs->m_size) {
3878		pr_err("alg: akcipher: decrypt test failed. Invalid output len %u\n",
3879		       out_len);
3880		err = -EINVAL;
3881		goto free_all;
3882	}
3883	/* verify that decrypted message is equal to the original msg */
3884	if (memchr_inv(outbuf_dec, 0, out_len - vecs->m_size) ||
3885	    memcmp(vecs->m, outbuf_dec + out_len - vecs->m_size,
3886		   vecs->m_size)) {
3887		pr_err("alg: akcipher: decrypt test failed. Invalid output\n");
3888		hexdump(outbuf_dec, out_len);
3889		err = -EINVAL;
3890	}
3891free_all:
3892	kfree(outbuf_dec);
3893	kfree(outbuf_enc);
3894free_req:
3895	akcipher_request_free(req);
3896free_xbuf:
3897	testmgr_free_buf(xbuf);
3898	return err;
3899}
3900
3901static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,
3902			 const struct akcipher_testvec *vecs,
3903			 unsigned int tcount)
3904{
3905	const char *algo =
3906		crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));
3907	int ret, i;
3908
3909	for (i = 0; i < tcount; i++) {
3910		ret = test_akcipher_one(tfm, vecs++);
3911		if (!ret)
3912			continue;
3913
3914		pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
3915		       i + 1, algo, ret);
3916		return ret;
3917	}
3918	return 0;
3919}
3920
3921static int alg_test_akcipher(const struct alg_test_desc *desc,
3922			     const char *driver, u32 type, u32 mask)
3923{
3924	struct crypto_akcipher *tfm;
3925	int err = 0;
3926
3927	tfm = crypto_alloc_akcipher(driver, type, mask);
3928	if (IS_ERR(tfm)) {
3929		if (PTR_ERR(tfm) == -ENOENT)
3930			return 0;
3931		pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
3932		       driver, PTR_ERR(tfm));
3933		return PTR_ERR(tfm);
3934	}
3935	if (desc->suite.akcipher.vecs)
3936		err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,
3937				    desc->suite.akcipher.count);
3938
3939	crypto_free_akcipher(tfm);
3940	return err;
3941}
3942
3943static int test_sig_one(struct crypto_sig *tfm, const struct sig_testvec *vecs)
3944{
3945	u8 *ptr, *key __free(kfree);
3946	int err, sig_size;
3947
3948	key = kmalloc(vecs->key_len + 2 * sizeof(u32) + vecs->param_len,
3949		      GFP_KERNEL);
3950	if (!key)
3951		return -ENOMEM;
3952
3953	/* ecrdsa expects additional parameters appended to the key */
3954	memcpy(key, vecs->key, vecs->key_len);
3955	ptr = key + vecs->key_len;
3956	ptr = test_pack_u32(ptr, vecs->algo);
3957	ptr = test_pack_u32(ptr, vecs->param_len);
3958	memcpy(ptr, vecs->params, vecs->param_len);
3959
3960	if (vecs->public_key_vec)
3961		err = crypto_sig_set_pubkey(tfm, key, vecs->key_len);
3962	else
3963		err = crypto_sig_set_privkey(tfm, key, vecs->key_len);
3964	if (err)
3965		return err;
3966
3967	/*
3968	 * Run asymmetric signature verification first
3969	 * (which does not require a private key)
3970	 */
3971	err = crypto_sig_verify(tfm, vecs->c, vecs->c_size,
3972				vecs->m, vecs->m_size);
3973	if (err) {
3974		pr_err("alg: sig: verify test failed: err %d\n", err);
3975		return err;
3976	}
3977
3978	/*
3979	 * Don't invoke sign test (which requires a private key)
3980	 * for vectors with only a public key.
3981	 */
3982	if (vecs->public_key_vec)
3983		return 0;
3984
3985	sig_size = crypto_sig_maxsize(tfm);
3986	if (sig_size < vecs->c_size) {
3987		pr_err("alg: sig: invalid maxsize %u\n", sig_size);
3988		return -EINVAL;
3989	}
3990
3991	u8 *sig __free(kfree) = kzalloc(sig_size, GFP_KERNEL);
3992	if (!sig)
3993		return -ENOMEM;
3994
3995	/* Run asymmetric signature generation */
3996	err = crypto_sig_sign(tfm, vecs->m, vecs->m_size, sig, sig_size);
3997	if (err < 0) {
3998		pr_err("alg: sig: sign test failed: err %d\n", err);
3999		return err;
4000	}
4001
4002	/* Verify that generated signature equals cooked signature */
4003	if (err != vecs->c_size ||
4004	    memcmp(sig, vecs->c, vecs->c_size) ||
4005	    memchr_inv(sig + vecs->c_size, 0, sig_size - vecs->c_size)) {
4006		pr_err("alg: sig: sign test failed: invalid output\n");
4007		hexdump(sig, sig_size);
4008		return -EINVAL;
4009	}
4010
4011	return 0;
4012}
4013
4014static int test_sig(struct crypto_sig *tfm, const char *alg,
4015		    const struct sig_testvec *vecs, unsigned int tcount)
4016{
4017	const char *algo = crypto_tfm_alg_driver_name(crypto_sig_tfm(tfm));
4018	int ret, i;
4019
4020	for (i = 0; i < tcount; i++) {
4021		ret = test_sig_one(tfm, vecs++);
4022		if (ret) {
4023			pr_err("alg: sig: test %d failed for %s: err %d\n",
4024			       i + 1, algo, ret);
4025			return ret;
4026		}
4027	}
4028	return 0;
4029}
4030
4031static int alg_test_sig(const struct alg_test_desc *desc, const char *driver,
4032			u32 type, u32 mask)
4033{
4034	struct crypto_sig *tfm;
4035	int err = 0;
4036
4037	tfm = crypto_alloc_sig(driver, type, mask);
4038	if (IS_ERR(tfm)) {
4039		pr_err("alg: sig: Failed to load tfm for %s: %ld\n",
4040		       driver, PTR_ERR(tfm));
4041		return PTR_ERR(tfm);
4042	}
4043	if (desc->suite.sig.vecs)
4044		err = test_sig(tfm, desc->alg, desc->suite.sig.vecs,
4045			       desc->suite.sig.count);
4046
4047	crypto_free_sig(tfm);
4048	return err;
4049}
4050
4051static int alg_test_null(const struct alg_test_desc *desc,
4052			     const char *driver, u32 type, u32 mask)
4053{
4054	return 0;
4055}
4056
4057#define ____VECS(tv)	.vecs = tv, .count = ARRAY_SIZE(tv)
4058#define __VECS(tv)	{ ____VECS(tv) }
4059
4060/* Please keep this list sorted by algorithm name. */
4061static const struct alg_test_desc alg_test_descs[] = {
4062	{
4063		.alg = "adiantum(xchacha12,aes)",
4064		.generic_driver = "adiantum(xchacha12-lib,aes-generic,nhpoly1305-generic)",
4065		.test = alg_test_skcipher,
4066		.suite = {
4067			.cipher = __VECS(adiantum_xchacha12_aes_tv_template)
4068		},
4069	}, {
4070		.alg = "adiantum(xchacha20,aes)",
4071		.generic_driver = "adiantum(xchacha20-lib,aes-generic,nhpoly1305-generic)",
4072		.test = alg_test_skcipher,
4073		.suite = {
4074			.cipher = __VECS(adiantum_xchacha20_aes_tv_template)
4075		},
4076	}, {
4077		.alg = "aegis128",
4078		.test = alg_test_aead,
4079		.suite = {
4080			.aead = __VECS(aegis128_tv_template)
4081		}
4082	}, {
4083		.alg = "authenc(hmac(md5),ecb(cipher_null))",
4084		.generic_driver = "authenc(hmac-md5-lib,ecb-cipher_null)",
4085		.test = alg_test_aead,
4086		.suite = {
4087			.aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
4088		}
4089	}, {
4090		.alg = "authenc(hmac(sha1),cbc(aes))",
4091		.generic_driver = "authenc(hmac-sha1-lib,cbc(aes-generic))",
4092		.test = alg_test_aead,
4093		.fips_allowed = 1,
4094		.suite = {
4095			.aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
4096		}
4097	}, {
4098		.alg = "authenc(hmac(sha1),cbc(des))",
4099		.generic_driver = "authenc(hmac-sha1-lib,cbc(des-generic))",
4100		.test = alg_test_aead,
4101		.suite = {
4102			.aead = __VECS(hmac_sha1_des_cbc_tv_temp)
4103		}
4104	}, {
4105		.alg = "authenc(hmac(sha1),cbc(des3_ede))",
4106		.generic_driver = "authenc(hmac-sha1-lib,cbc(des3_ede-generic))",
4107		.test = alg_test_aead,
4108		.suite = {
4109			.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
4110		}
4111	}, {
4112		.alg = "authenc(hmac(sha1),ctr(aes))",
4113		.test = alg_test_null,
4114		.fips_allowed = 1,
4115	}, {
4116		.alg = "authenc(hmac(sha1),ecb(cipher_null))",
4117		.generic_driver = "authenc(hmac-sha1-lib,ecb-cipher_null)",
4118		.test = alg_test_aead,
4119		.suite = {
4120			.aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
4121		}
4122	}, {
4123		.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4124		.test = alg_test_null,
4125		.fips_allowed = 1,
4126	}, {
4127		.alg = "authenc(hmac(sha224),cbc(des))",
4128		.generic_driver = "authenc(hmac-sha224-lib,cbc(des-generic))",
4129		.test = alg_test_aead,
4130		.suite = {
4131			.aead = __VECS(hmac_sha224_des_cbc_tv_temp)
4132		}
4133	}, {
4134		.alg = "authenc(hmac(sha224),cbc(des3_ede))",
4135		.generic_driver = "authenc(hmac-sha224-lib,cbc(des3_ede-generic))",
4136		.test = alg_test_aead,
4137		.suite = {
4138			.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
4139		}
4140	}, {
4141		.alg = "authenc(hmac(sha256),cbc(aes))",
4142		.generic_driver = "authenc(hmac-sha256-lib,cbc(aes-generic))",
4143		.test = alg_test_aead,
4144		.fips_allowed = 1,
4145		.suite = {
4146			.aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
4147		}
4148	}, {
4149		.alg = "authenc(hmac(sha256),cbc(des))",
4150		.generic_driver = "authenc(hmac-sha256-lib,cbc(des-generic))",
4151		.test = alg_test_aead,
4152		.suite = {
4153			.aead = __VECS(hmac_sha256_des_cbc_tv_temp)
4154		}
4155	}, {
4156		.alg = "authenc(hmac(sha256),cbc(des3_ede))",
4157		.generic_driver = "authenc(hmac-sha256-lib,cbc(des3_ede-generic))",
4158		.test = alg_test_aead,
4159		.suite = {
4160			.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
4161		}
4162	}, {
4163		.alg = "authenc(hmac(sha256),ctr(aes))",
4164		.test = alg_test_null,
4165		.fips_allowed = 1,
4166	}, {
4167		.alg = "authenc(hmac(sha256),cts(cbc(aes)))",
4168		.generic_driver = "authenc(hmac-sha256-lib,cts(cbc(aes-generic)))",
4169		.test = alg_test_aead,
4170		.suite = {
4171			.aead = __VECS(krb5_test_aes128_cts_hmac_sha256_128)
4172		}
4173	}, {
4174		.alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4175		.test = alg_test_null,
4176		.fips_allowed = 1,
4177	}, {
4178		.alg = "authenc(hmac(sha384),cbc(des))",
4179		.generic_driver = "authenc(hmac-sha384-lib,cbc(des-generic))",
4180		.test = alg_test_aead,
4181		.suite = {
4182			.aead = __VECS(hmac_sha384_des_cbc_tv_temp)
4183		}
4184	}, {
4185		.alg = "authenc(hmac(sha384),cbc(des3_ede))",
4186		.generic_driver = "authenc(hmac-sha384-lib,cbc(des3_ede-generic))",
4187		.test = alg_test_aead,
4188		.suite = {
4189			.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
4190		}
4191	}, {
4192		.alg = "authenc(hmac(sha384),ctr(aes))",
4193		.test = alg_test_null,
4194		.fips_allowed = 1,
4195	}, {
4196		.alg = "authenc(hmac(sha384),cts(cbc(aes)))",
4197		.generic_driver = "authenc(hmac-sha384-lib,cts(cbc(aes-generic)))",
4198		.test = alg_test_aead,
4199		.suite = {
4200			.aead = __VECS(krb5_test_aes256_cts_hmac_sha384_192)
4201		}
4202	}, {
4203		.alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4204		.test = alg_test_null,
4205		.fips_allowed = 1,
4206	}, {
4207		.alg = "authenc(hmac(sha512),cbc(aes))",
4208		.generic_driver = "authenc(hmac-sha512-lib,cbc(aes-generic))",
4209		.fips_allowed = 1,
4210		.test = alg_test_aead,
4211		.suite = {
4212			.aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
4213		}
4214	}, {
4215		.alg = "authenc(hmac(sha512),cbc(des))",
4216		.generic_driver = "authenc(hmac-sha512-lib,cbc(des-generic))",
4217		.test = alg_test_aead,
4218		.suite = {
4219			.aead = __VECS(hmac_sha512_des_cbc_tv_temp)
4220		}
4221	}, {
4222		.alg = "authenc(hmac(sha512),cbc(des3_ede))",
4223		.generic_driver = "authenc(hmac-sha512-lib,cbc(des3_ede-generic))",
4224		.test = alg_test_aead,
4225		.suite = {
4226			.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
4227		}
4228	}, {
4229		.alg = "authenc(hmac(sha512),ctr(aes))",
4230		.test = alg_test_null,
4231		.fips_allowed = 1,
4232	}, {
4233		.alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4234		.test = alg_test_null,
4235		.fips_allowed = 1,
4236	}, {
4237		.alg = "blake2b-160",
4238		.generic_driver = "blake2b-160-lib",
4239		.test = alg_test_hash,
4240		.fips_allowed = 0,
4241		.suite = {
4242			.hash = __VECS(blake2b_160_tv_template)
4243		}
4244	}, {
4245		.alg = "blake2b-256",
4246		.generic_driver = "blake2b-256-lib",
4247		.test = alg_test_hash,
4248		.fips_allowed = 0,
4249		.suite = {
4250			.hash = __VECS(blake2b_256_tv_template)
4251		}
4252	}, {
4253		.alg = "blake2b-384",
4254		.generic_driver = "blake2b-384-lib",
4255		.test = alg_test_hash,
4256		.fips_allowed = 0,
4257		.suite = {
4258			.hash = __VECS(blake2b_384_tv_template)
4259		}
4260	}, {
4261		.alg = "blake2b-512",
4262		.generic_driver = "blake2b-512-lib",
4263		.test = alg_test_hash,
4264		.fips_allowed = 0,
4265		.suite = {
4266			.hash = __VECS(blake2b_512_tv_template)
4267		}
4268	}, {
4269		.alg = "cbc(aes)",
4270		.test = alg_test_skcipher,
4271		.fips_allowed = 1,
4272		.suite = {
4273			.cipher = __VECS(aes_cbc_tv_template)
4274		},
4275	}, {
4276		.alg = "cbc(anubis)",
4277		.test = alg_test_skcipher,
4278		.suite = {
4279			.cipher = __VECS(anubis_cbc_tv_template)
4280		},
4281	}, {
4282		.alg = "cbc(aria)",
4283		.test = alg_test_skcipher,
4284		.suite = {
4285			.cipher = __VECS(aria_cbc_tv_template)
4286		},
4287	}, {
4288		.alg = "cbc(blowfish)",
4289		.test = alg_test_skcipher,
4290		.suite = {
4291			.cipher = __VECS(bf_cbc_tv_template)
4292		},
4293	}, {
4294		.alg = "cbc(camellia)",
4295		.test = alg_test_skcipher,
4296		.suite = {
4297			.cipher = __VECS(camellia_cbc_tv_template)
4298		},
4299	}, {
4300		.alg = "cbc(cast5)",
4301		.test = alg_test_skcipher,
4302		.suite = {
4303			.cipher = __VECS(cast5_cbc_tv_template)
4304		},
4305	}, {
4306		.alg = "cbc(cast6)",
4307		.test = alg_test_skcipher,
4308		.suite = {
4309			.cipher = __VECS(cast6_cbc_tv_template)
4310		},
4311	}, {
4312		.alg = "cbc(des)",
4313		.test = alg_test_skcipher,
4314		.suite = {
4315			.cipher = __VECS(des_cbc_tv_template)
4316		},
4317	}, {
4318		.alg = "cbc(des3_ede)",
4319		.test = alg_test_skcipher,
4320		.suite = {
4321			.cipher = __VECS(des3_ede_cbc_tv_template)
4322		},
4323	}, {
4324		/* Same as cbc(aes) except the key is stored in
4325		 * hardware secure memory which we reference by index
4326		 */
4327		.alg = "cbc(paes)",
4328		.test = alg_test_null,
4329		.fips_allowed = 1,
4330	}, {
4331		/* Same as cbc(sm4) except the key is stored in
4332		 * hardware secure memory which we reference by index
4333		 */
4334		.alg = "cbc(psm4)",
4335		.test = alg_test_null,
4336	}, {
4337		.alg = "cbc(serpent)",
4338		.test = alg_test_skcipher,
4339		.suite = {
4340			.cipher = __VECS(serpent_cbc_tv_template)
4341		},
4342	}, {
4343		.alg = "cbc(sm4)",
4344		.test = alg_test_skcipher,
4345		.suite = {
4346			.cipher = __VECS(sm4_cbc_tv_template)
4347		}
4348	}, {
4349		.alg = "cbc(twofish)",
4350		.test = alg_test_skcipher,
4351		.suite = {
4352			.cipher = __VECS(tf_cbc_tv_template)
4353		},
4354	}, {
4355#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4356		.alg = "cbc-paes-s390",
4357		.fips_allowed = 1,
4358		.test = alg_test_skcipher,
4359		.suite = {
4360			.cipher = __VECS(aes_cbc_tv_template)
4361		}
4362	}, {
4363#endif
4364		.alg = "cbcmac(aes)",
4365		.test = alg_test_hash,
4366		.suite = {
4367			.hash = __VECS(aes_cbcmac_tv_template)
4368		}
4369	}, {
4370		.alg = "cbcmac(sm4)",
4371		.test = alg_test_hash,
4372		.suite = {
4373			.hash = __VECS(sm4_cbcmac_tv_template)
4374		}
4375	}, {
4376		.alg = "ccm(aes)",
4377		.generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
4378		.test = alg_test_aead,
4379		.fips_allowed = 1,
4380		.suite = {
4381			.aead = {
4382				____VECS(aes_ccm_tv_template),
4383				.einval_allowed = 1,
4384			}
4385		}
4386	}, {
4387		.alg = "ccm(sm4)",
4388		.generic_driver = "ccm_base(ctr(sm4-generic),cbcmac(sm4-generic))",
4389		.test = alg_test_aead,
4390		.suite = {
4391			.aead = {
4392				____VECS(sm4_ccm_tv_template),
4393				.einval_allowed = 1,
4394			}
4395		}
4396	}, {
4397		.alg = "chacha20",
4398		.generic_driver = "chacha20-lib",
4399		.test = alg_test_skcipher,
4400		.suite = {
4401			.cipher = __VECS(chacha20_tv_template)
4402		},
4403	}, {
4404		.alg = "cmac(aes)",
4405		.fips_allowed = 1,
4406		.test = alg_test_hash,
4407		.suite = {
4408			.hash = __VECS(aes_cmac128_tv_template)
4409		}
4410	}, {
4411		.alg = "cmac(camellia)",
4412		.test = alg_test_hash,
4413		.suite = {
4414			.hash = __VECS(camellia_cmac128_tv_template)
4415		}
4416	}, {
4417		.alg = "cmac(des3_ede)",
4418		.test = alg_test_hash,
4419		.suite = {
4420			.hash = __VECS(des3_ede_cmac64_tv_template)
4421		}
4422	}, {
4423		.alg = "cmac(sm4)",
4424		.test = alg_test_hash,
4425		.suite = {
4426			.hash = __VECS(sm4_cmac128_tv_template)
4427		}
4428	}, {
4429		.alg = "crc32",
4430		.generic_driver = "crc32-lib",
4431		.test = alg_test_hash,
4432		.fips_allowed = 1,
4433		.suite = {
4434			.hash = __VECS(crc32_tv_template)
4435		}
4436	}, {
4437		.alg = "crc32c",
4438		.generic_driver = "crc32c-lib",
4439		.test = alg_test_hash,
4440		.fips_allowed = 1,
4441		.suite = {
4442			.hash = __VECS(crc32c_tv_template)
4443		}
4444	}, {
4445		.alg = "ctr(aes)",
4446		.test = alg_test_skcipher,
4447		.fips_allowed = 1,
4448		.suite = {
4449			.cipher = __VECS(aes_ctr_tv_template)
4450		}
4451	}, {
4452		.alg = "ctr(aria)",
4453		.test = alg_test_skcipher,
4454		.suite = {
4455			.cipher = __VECS(aria_ctr_tv_template)
4456		}
4457	}, {
4458		.alg = "ctr(blowfish)",
4459		.test = alg_test_skcipher,
4460		.suite = {
4461			.cipher = __VECS(bf_ctr_tv_template)
4462		}
4463	}, {
4464		.alg = "ctr(camellia)",
4465		.test = alg_test_skcipher,
4466		.suite = {
4467			.cipher = __VECS(camellia_ctr_tv_template)
4468		}
4469	}, {
4470		.alg = "ctr(cast5)",
4471		.test = alg_test_skcipher,
4472		.suite = {
4473			.cipher = __VECS(cast5_ctr_tv_template)
4474		}
4475	}, {
4476		.alg = "ctr(cast6)",
4477		.test = alg_test_skcipher,
4478		.suite = {
4479			.cipher = __VECS(cast6_ctr_tv_template)
4480		}
4481	}, {
4482		.alg = "ctr(des)",
4483		.test = alg_test_skcipher,
4484		.suite = {
4485			.cipher = __VECS(des_ctr_tv_template)
4486		}
4487	}, {
4488		.alg = "ctr(des3_ede)",
4489		.test = alg_test_skcipher,
4490		.suite = {
4491			.cipher = __VECS(des3_ede_ctr_tv_template)
4492		}
4493	}, {
4494		/* Same as ctr(aes) except the key is stored in
4495		 * hardware secure memory which we reference by index
4496		 */
4497		.alg = "ctr(paes)",
4498		.test = alg_test_null,
4499		.fips_allowed = 1,
4500	}, {
4501
4502		/* Same as ctr(sm4) except the key is stored in
4503		 * hardware secure memory which we reference by index
4504		 */
4505		.alg = "ctr(psm4)",
4506		.test = alg_test_null,
4507	}, {
4508		.alg = "ctr(serpent)",
4509		.test = alg_test_skcipher,
4510		.suite = {
4511			.cipher = __VECS(serpent_ctr_tv_template)
4512		}
4513	}, {
4514		.alg = "ctr(sm4)",
4515		.test = alg_test_skcipher,
4516		.suite = {
4517			.cipher = __VECS(sm4_ctr_tv_template)
4518		}
4519	}, {
4520		.alg = "ctr(twofish)",
4521		.test = alg_test_skcipher,
4522		.suite = {
4523			.cipher = __VECS(tf_ctr_tv_template)
4524		}
4525	}, {
4526#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4527		.alg = "ctr-paes-s390",
4528		.fips_allowed = 1,
4529		.test = alg_test_skcipher,
4530		.suite = {
4531			.cipher = __VECS(aes_ctr_tv_template)
4532		}
4533	}, {
4534#endif
4535		.alg = "cts(cbc(aes))",
4536		.test = alg_test_skcipher,
4537		.fips_allowed = 1,
4538		.suite = {
4539			.cipher = __VECS(cts_mode_tv_template)
4540		}
4541	}, {
4542		/* Same as cts(cbc((aes)) except the key is stored in
4543		 * hardware secure memory which we reference by index
4544		 */
4545		.alg = "cts(cbc(paes))",
4546		.test = alg_test_null,
4547		.fips_allowed = 1,
4548	}, {
4549		.alg = "cts(cbc(sm4))",
4550		.test = alg_test_skcipher,
4551		.suite = {
4552			.cipher = __VECS(sm4_cts_tv_template)
4553		}
4554	}, {
4555		.alg = "deflate",
4556		.test = alg_test_comp,
4557		.fips_allowed = 1,
4558		.suite = {
4559			.comp = {
4560				.comp = __VECS(deflate_comp_tv_template),
4561				.decomp = __VECS(deflate_decomp_tv_template)
4562			}
4563		}
4564	}, {
4565		.alg = "deflate-iaa",
4566		.test = alg_test_comp,
4567		.fips_allowed = 1,
4568		.suite = {
4569			.comp = {
4570				.comp = __VECS(deflate_comp_tv_template),
4571				.decomp = __VECS(deflate_decomp_tv_template)
4572			}
4573		}
4574	}, {
4575		.alg = "dh",
4576		.test = alg_test_kpp,
4577		.suite = {
4578			.kpp = __VECS(dh_tv_template)
4579		}
4580	}, {
4581		.alg = "digest_null",
4582		.test = alg_test_null,
4583	}, {
4584		.alg = "drbg_nopr_ctr_aes128",
4585		.test = alg_test_drbg,
4586		.fips_allowed = 1,
4587		.suite = {
4588			.drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
4589		}
4590	}, {
4591		.alg = "drbg_nopr_ctr_aes192",
4592		.test = alg_test_drbg,
4593		.fips_allowed = 1,
4594		.suite = {
4595			.drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
4596		}
4597	}, {
4598		.alg = "drbg_nopr_ctr_aes256",
4599		.test = alg_test_drbg,
4600		.fips_allowed = 1,
4601		.suite = {
4602			.drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
4603		}
4604	}, {
4605		.alg = "drbg_nopr_hmac_sha256",
4606		.test = alg_test_drbg,
4607		.fips_allowed = 1,
4608		.suite = {
4609			.drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
4610		}
4611	}, {
4612		/*
4613		 * There is no need to specifically test the DRBG with every
4614		 * backend cipher -- covered by drbg_nopr_hmac_sha512 test
4615		 */
4616		.alg = "drbg_nopr_hmac_sha384",
4617		.test = alg_test_null,
4618		.fips_allowed = 1
4619	}, {
4620		.alg = "drbg_nopr_hmac_sha512",
4621		.test = alg_test_drbg,
4622		.fips_allowed = 1,
4623		.suite = {
4624			.drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)
4625		}
4626	}, {
4627		.alg = "drbg_nopr_sha256",
4628		.test = alg_test_drbg,
4629		.fips_allowed = 1,
4630		.suite = {
4631			.drbg = __VECS(drbg_nopr_sha256_tv_template)
4632		}
4633	}, {
4634		/* covered by drbg_nopr_sha256 test */
4635		.alg = "drbg_nopr_sha384",
4636		.test = alg_test_null,
4637		.fips_allowed = 1
4638	}, {
4639		.alg = "drbg_nopr_sha512",
4640		.fips_allowed = 1,
4641		.test = alg_test_null,
4642	}, {
4643		.alg = "drbg_pr_ctr_aes128",
4644		.test = alg_test_drbg,
4645		.fips_allowed = 1,
4646		.suite = {
4647			.drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
4648		}
4649	}, {
4650		/* covered by drbg_pr_ctr_aes128 test */
4651		.alg = "drbg_pr_ctr_aes192",
4652		.fips_allowed = 1,
4653		.test = alg_test_null,
4654	}, {
4655		.alg = "drbg_pr_ctr_aes256",
4656		.fips_allowed = 1,
4657		.test = alg_test_null,
4658	}, {
4659		.alg = "drbg_pr_hmac_sha256",
4660		.test = alg_test_drbg,
4661		.fips_allowed = 1,
4662		.suite = {
4663			.drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
4664		}
4665	}, {
4666		/* covered by drbg_pr_hmac_sha256 test */
4667		.alg = "drbg_pr_hmac_sha384",
4668		.test = alg_test_null,
4669		.fips_allowed = 1
4670	}, {
4671		.alg = "drbg_pr_hmac_sha512",
4672		.test = alg_test_null,
4673		.fips_allowed = 1,
4674	}, {
4675		.alg = "drbg_pr_sha256",
4676		.test = alg_test_drbg,
4677		.fips_allowed = 1,
4678		.suite = {
4679			.drbg = __VECS(drbg_pr_sha256_tv_template)
4680		}
4681	}, {
4682		/* covered by drbg_pr_sha256 test */
4683		.alg = "drbg_pr_sha384",
4684		.test = alg_test_null,
4685		.fips_allowed = 1
4686	}, {
4687		.alg = "drbg_pr_sha512",
4688		.fips_allowed = 1,
4689		.test = alg_test_null,
4690	}, {
4691		.alg = "ecb(aes)",
4692		.test = alg_test_skcipher,
4693		.fips_allowed = 1,
4694		.suite = {
4695			.cipher = __VECS(aes_tv_template)
4696		}
4697	}, {
4698		.alg = "ecb(anubis)",
4699		.test = alg_test_skcipher,
4700		.suite = {
4701			.cipher = __VECS(anubis_tv_template)
4702		}
4703	}, {
4704		.alg = "ecb(arc4)",
4705		.generic_driver = "arc4-generic",
4706		.test = alg_test_skcipher,
4707		.suite = {
4708			.cipher = __VECS(arc4_tv_template)
4709		}
4710	}, {
4711		.alg = "ecb(aria)",
4712		.test = alg_test_skcipher,
4713		.suite = {
4714			.cipher = __VECS(aria_tv_template)
4715		}
4716	}, {
4717		.alg = "ecb(blowfish)",
4718		.test = alg_test_skcipher,
4719		.suite = {
4720			.cipher = __VECS(bf_tv_template)
4721		}
4722	}, {
4723		.alg = "ecb(camellia)",
4724		.test = alg_test_skcipher,
4725		.suite = {
4726			.cipher = __VECS(camellia_tv_template)
4727		}
4728	}, {
4729		.alg = "ecb(cast5)",
4730		.test = alg_test_skcipher,
4731		.suite = {
4732			.cipher = __VECS(cast5_tv_template)
4733		}
4734	}, {
4735		.alg = "ecb(cast6)",
4736		.test = alg_test_skcipher,
4737		.suite = {
4738			.cipher = __VECS(cast6_tv_template)
4739		}
4740	}, {
4741		.alg = "ecb(cipher_null)",
4742		.test = alg_test_null,
4743		.fips_allowed = 1,
4744	}, {
4745		.alg = "ecb(des)",
4746		.test = alg_test_skcipher,
4747		.suite = {
4748			.cipher = __VECS(des_tv_template)
4749		}
4750	}, {
4751		.alg = "ecb(des3_ede)",
4752		.test = alg_test_skcipher,
4753		.suite = {
4754			.cipher = __VECS(des3_ede_tv_template)
4755		}
4756	}, {
4757		.alg = "ecb(fcrypt)",
4758		.test = alg_test_skcipher,
4759		.suite = {
4760			.cipher = {
4761				.vecs = fcrypt_pcbc_tv_template,
4762				.count = 1
4763			}
4764		}
4765	}, {
4766		.alg = "ecb(khazad)",
4767		.test = alg_test_skcipher,
4768		.suite = {
4769			.cipher = __VECS(khazad_tv_template)
4770		}
4771	}, {
4772		/* Same as ecb(aes) except the key is stored in
4773		 * hardware secure memory which we reference by index
4774		 */
4775		.alg = "ecb(paes)",
4776		.test = alg_test_null,
4777		.fips_allowed = 1,
4778	}, {
4779		.alg = "ecb(seed)",
4780		.test = alg_test_skcipher,
4781		.suite = {
4782			.cipher = __VECS(seed_tv_template)
4783		}
4784	}, {
4785		.alg = "ecb(serpent)",
4786		.test = alg_test_skcipher,
4787		.suite = {
4788			.cipher = __VECS(serpent_tv_template)
4789		}
4790	}, {
4791		.alg = "ecb(sm4)",
4792		.test = alg_test_skcipher,
4793		.suite = {
4794			.cipher = __VECS(sm4_tv_template)
4795		}
4796	}, {
4797		.alg = "ecb(tea)",
4798		.test = alg_test_skcipher,
4799		.suite = {
4800			.cipher = __VECS(tea_tv_template)
4801		}
4802	}, {
4803		.alg = "ecb(twofish)",
4804		.test = alg_test_skcipher,
4805		.suite = {
4806			.cipher = __VECS(tf_tv_template)
4807		}
4808	}, {
4809		.alg = "ecb(xeta)",
4810		.test = alg_test_skcipher,
4811		.suite = {
4812			.cipher = __VECS(xeta_tv_template)
4813		}
4814	}, {
4815		.alg = "ecb(xtea)",
4816		.test = alg_test_skcipher,
4817		.suite = {
4818			.cipher = __VECS(xtea_tv_template)
4819		}
4820	}, {
4821#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4822		.alg = "ecb-paes-s390",
4823		.fips_allowed = 1,
4824		.test = alg_test_skcipher,
4825		.suite = {
4826			.cipher = __VECS(aes_tv_template)
4827		}
4828	}, {
4829#endif
4830		.alg = "ecdh-nist-p192",
4831		.test = alg_test_kpp,
4832		.suite = {
4833			.kpp = __VECS(ecdh_p192_tv_template)
4834		}
4835	}, {
4836		.alg = "ecdh-nist-p256",
4837		.test = alg_test_kpp,
4838		.fips_allowed = 1,
4839		.suite = {
4840			.kpp = __VECS(ecdh_p256_tv_template)
4841		}
4842	}, {
4843		.alg = "ecdh-nist-p384",
4844		.test = alg_test_kpp,
4845		.fips_allowed = 1,
4846		.suite = {
4847			.kpp = __VECS(ecdh_p384_tv_template)
4848		}
4849	}, {
4850		.alg = "ecdsa-nist-p192",
4851		.test = alg_test_sig,
4852		.suite = {
4853			.sig = __VECS(ecdsa_nist_p192_tv_template)
4854		}
4855	}, {
4856		.alg = "ecdsa-nist-p256",
4857		.test = alg_test_sig,
4858		.fips_allowed = 1,
4859		.suite = {
4860			.sig = __VECS(ecdsa_nist_p256_tv_template)
4861		}
4862	}, {
4863		.alg = "ecdsa-nist-p384",
4864		.test = alg_test_sig,
4865		.fips_allowed = 1,
4866		.suite = {
4867			.sig = __VECS(ecdsa_nist_p384_tv_template)
4868		}
4869	}, {
4870		.alg = "ecdsa-nist-p521",
4871		.test = alg_test_sig,
4872		.fips_allowed = 1,
4873		.suite = {
4874			.sig = __VECS(ecdsa_nist_p521_tv_template)
4875		}
4876	}, {
4877		.alg = "ecrdsa",
4878		.test = alg_test_sig,
4879		.suite = {
4880			.sig = __VECS(ecrdsa_tv_template)
4881		}
4882	}, {
4883		.alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
4884		.generic_driver = "essiv(authenc(hmac-sha256-lib,cbc(aes-generic)),sha256-lib)",
4885		.test = alg_test_aead,
4886		.fips_allowed = 1,
4887		.suite = {
4888			.aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)
4889		}
4890	}, {
4891		.alg = "essiv(cbc(aes),sha256)",
4892		.generic_driver = "essiv(cbc(aes-generic),sha256-lib)",
4893		.test = alg_test_skcipher,
4894		.fips_allowed = 1,
4895		.suite = {
4896			.cipher = __VECS(essiv_aes_cbc_tv_template)
4897		}
4898	}, {
4899#if IS_ENABLED(CONFIG_CRYPTO_DH_RFC7919_GROUPS)
4900		.alg = "ffdhe2048(dh)",
4901		.test = alg_test_kpp,
4902		.fips_allowed = 1,
4903		.suite = {
4904			.kpp = __VECS(ffdhe2048_dh_tv_template)
4905		}
4906	}, {
4907		.alg = "ffdhe3072(dh)",
4908		.test = alg_test_kpp,
4909		.fips_allowed = 1,
4910		.suite = {
4911			.kpp = __VECS(ffdhe3072_dh_tv_template)
4912		}
4913	}, {
4914		.alg = "ffdhe4096(dh)",
4915		.test = alg_test_kpp,
4916		.fips_allowed = 1,
4917		.suite = {
4918			.kpp = __VECS(ffdhe4096_dh_tv_template)
4919		}
4920	}, {
4921		.alg = "ffdhe6144(dh)",
4922		.test = alg_test_kpp,
4923		.fips_allowed = 1,
4924		.suite = {
4925			.kpp = __VECS(ffdhe6144_dh_tv_template)
4926		}
4927	}, {
4928		.alg = "ffdhe8192(dh)",
4929		.test = alg_test_kpp,
4930		.fips_allowed = 1,
4931		.suite = {
4932			.kpp = __VECS(ffdhe8192_dh_tv_template)
4933		}
4934	}, {
4935#endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */
4936		.alg = "gcm(aes)",
4937		.generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
4938		.test = alg_test_aead,
4939		.fips_allowed = 1,
4940		.suite = {
4941			.aead = __VECS(aes_gcm_tv_template)
4942		}
4943	}, {
4944		.alg = "gcm(aria)",
4945		.generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)",
4946		.test = alg_test_aead,
4947		.suite = {
4948			.aead = __VECS(aria_gcm_tv_template)
4949		}
4950	}, {
4951		.alg = "gcm(sm4)",
4952		.generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)",
4953		.test = alg_test_aead,
4954		.suite = {
4955			.aead = __VECS(sm4_gcm_tv_template)
4956		}
4957	}, {
4958		.alg = "ghash",
4959		.test = alg_test_hash,
4960		.suite = {
4961			.hash = __VECS(ghash_tv_template)
4962		}
4963	}, {
4964		.alg = "hctr2(aes)",
4965		.generic_driver = "hctr2_base(xctr(aes-generic),polyval-lib)",
4966		.test = alg_test_skcipher,
4967		.suite = {
4968			.cipher = __VECS(aes_hctr2_tv_template)
4969		}
4970	}, {
4971		.alg = "hmac(md5)",
4972		.generic_driver = "hmac-md5-lib",
4973		.test = alg_test_hash,
4974		.suite = {
4975			.hash = __VECS(hmac_md5_tv_template)
4976		}
4977	}, {
4978		.alg = "hmac(rmd160)",
4979		.test = alg_test_hash,
4980		.suite = {
4981			.hash = __VECS(hmac_rmd160_tv_template)
4982		}
4983	}, {
4984		.alg = "hmac(sha1)",
4985		.generic_driver = "hmac-sha1-lib",
4986		.test = alg_test_hash,
4987		.fips_allowed = 1,
4988		.suite = {
4989			.hash = __VECS(hmac_sha1_tv_template)
4990		}
4991	}, {
4992		.alg = "hmac(sha224)",
4993		.generic_driver = "hmac-sha224-lib",
4994		.test = alg_test_hash,
4995		.fips_allowed = 1,
4996		.suite = {
4997			.hash = __VECS(hmac_sha224_tv_template)
4998		}
4999	}, {
5000		.alg = "hmac(sha256)",
5001		.generic_driver = "hmac-sha256-lib",
5002		.test = alg_test_hash,
5003		.fips_allowed = 1,
5004		.suite = {
5005			.hash = __VECS(hmac_sha256_tv_template)
5006		}
5007	}, {
5008		.alg = "hmac(sha3-224)",
5009		.generic_driver = "hmac(sha3-224-lib)",
5010		.test = alg_test_hash,
5011		.fips_allowed = 1,
5012		.suite = {
5013			.hash = __VECS(hmac_sha3_224_tv_template)
5014		}
5015	}, {
5016		.alg = "hmac(sha3-256)",
5017		.generic_driver = "hmac(sha3-256-lib)",
5018		.test = alg_test_hash,
5019		.fips_allowed = 1,
5020		.suite = {
5021			.hash = __VECS(hmac_sha3_256_tv_template)
5022		}
5023	}, {
5024		.alg = "hmac(sha3-384)",
5025		.generic_driver = "hmac(sha3-384-lib)",
5026		.test = alg_test_hash,
5027		.fips_allowed = 1,
5028		.suite = {
5029			.hash = __VECS(hmac_sha3_384_tv_template)
5030		}
5031	}, {
5032		.alg = "hmac(sha3-512)",
5033		.generic_driver = "hmac(sha3-512-lib)",
5034		.test = alg_test_hash,
5035		.fips_allowed = 1,
5036		.suite = {
5037			.hash = __VECS(hmac_sha3_512_tv_template)
5038		}
5039	}, {
5040		.alg = "hmac(sha384)",
5041		.generic_driver = "hmac-sha384-lib",
5042		.test = alg_test_hash,
5043		.fips_allowed = 1,
5044		.suite = {
5045			.hash = __VECS(hmac_sha384_tv_template)
5046		}
5047	}, {
5048		.alg = "hmac(sha512)",
5049		.generic_driver = "hmac-sha512-lib",
5050		.test = alg_test_hash,
5051		.fips_allowed = 1,
5052		.suite = {
5053			.hash = __VECS(hmac_sha512_tv_template)
5054		}
5055	}, {
5056		.alg = "hmac(sm3)",
5057		.test = alg_test_hash,
5058		.suite = {
5059			.hash = __VECS(hmac_sm3_tv_template)
5060		}
5061	}, {
5062		.alg = "hmac(streebog256)",
5063		.test = alg_test_hash,
5064		.suite = {
5065			.hash = __VECS(hmac_streebog256_tv_template)
5066		}
5067	}, {
5068		.alg = "hmac(streebog512)",
5069		.test = alg_test_hash,
5070		.suite = {
5071			.hash = __VECS(hmac_streebog512_tv_template)
5072		}
5073	}, {
5074		.alg = "jitterentropy_rng",
5075		.fips_allowed = 1,
5076		.test = alg_test_null,
5077	}, {
5078		.alg = "krb5enc(cmac(camellia),cts(cbc(camellia)))",
5079		.test = alg_test_aead,
5080		.suite.aead = __VECS(krb5_test_camellia_cts_cmac)
5081	}, {
5082		.alg = "lrw(aes)",
5083		.generic_driver = "lrw(ecb(aes-generic))",
5084		.test = alg_test_skcipher,
5085		.suite = {
5086			.cipher = __VECS(aes_lrw_tv_template)
5087		}
5088	}, {
5089		.alg = "lrw(camellia)",
5090		.generic_driver = "lrw(ecb(camellia-generic))",
5091		.test = alg_test_skcipher,
5092		.suite = {
5093			.cipher = __VECS(camellia_lrw_tv_template)
5094		}
5095	}, {
5096		.alg = "lrw(cast6)",
5097		.generic_driver = "lrw(ecb(cast6-generic))",
5098		.test = alg_test_skcipher,
5099		.suite = {
5100			.cipher = __VECS(cast6_lrw_tv_template)
5101		}
5102	}, {
5103		.alg = "lrw(serpent)",
5104		.generic_driver = "lrw(ecb(serpent-generic))",
5105		.test = alg_test_skcipher,
5106		.suite = {
5107			.cipher = __VECS(serpent_lrw_tv_template)
5108		}
5109	}, {
5110		.alg = "lrw(twofish)",
5111		.generic_driver = "lrw(ecb(twofish-generic))",
5112		.test = alg_test_skcipher,
5113		.suite = {
5114			.cipher = __VECS(tf_lrw_tv_template)
5115		}
5116	}, {
5117		.alg = "lz4",
5118		.test = alg_test_comp,
5119		.fips_allowed = 1,
5120		.suite = {
5121			.comp = {
5122				.comp = __VECS(lz4_comp_tv_template),
5123				.decomp = __VECS(lz4_decomp_tv_template)
5124			}
5125		}
5126	}, {
5127		.alg = "lz4hc",
5128		.test = alg_test_comp,
5129		.fips_allowed = 1,
5130		.suite = {
5131			.comp = {
5132				.comp = __VECS(lz4hc_comp_tv_template),
5133				.decomp = __VECS(lz4hc_decomp_tv_template)
5134			}
5135		}
5136	}, {
5137		.alg = "lzo",
5138		.test = alg_test_comp,
5139		.fips_allowed = 1,
5140		.suite = {
5141			.comp = {
5142				.comp = __VECS(lzo_comp_tv_template),
5143				.decomp = __VECS(lzo_decomp_tv_template)
5144			}
5145		}
5146	}, {
5147		.alg = "lzo-rle",
5148		.test = alg_test_comp,
5149		.fips_allowed = 1,
5150		.suite = {
5151			.comp = {
5152				.comp = __VECS(lzorle_comp_tv_template),
5153				.decomp = __VECS(lzorle_decomp_tv_template)
5154			}
5155		}
5156	}, {
5157		.alg = "md4",
5158		.test = alg_test_hash,
5159		.suite = {
5160			.hash = __VECS(md4_tv_template)
5161		}
5162	}, {
5163		.alg = "md5",
5164		.generic_driver = "md5-lib",
5165		.test = alg_test_hash,
5166		.suite = {
5167			.hash = __VECS(md5_tv_template)
5168		}
5169	}, {
5170		.alg = "michael_mic",
5171		.test = alg_test_hash,
5172		.suite = {
5173			.hash = __VECS(michael_mic_tv_template)
5174		}
5175	}, {
5176		.alg = "nhpoly1305",
5177		.test = alg_test_hash,
5178		.suite = {
5179			.hash = __VECS(nhpoly1305_tv_template)
5180		}
5181	}, {
5182		.alg = "p1363(ecdsa-nist-p192)",
5183		.test = alg_test_null,
5184	}, {
5185		.alg = "p1363(ecdsa-nist-p256)",
5186		.test = alg_test_sig,
5187		.fips_allowed = 1,
5188		.suite = {
5189			.sig = __VECS(p1363_ecdsa_nist_p256_tv_template)
5190		}
5191	}, {
5192		.alg = "p1363(ecdsa-nist-p384)",
5193		.test = alg_test_null,
5194		.fips_allowed = 1,
5195	}, {
5196		.alg = "p1363(ecdsa-nist-p521)",
5197		.test = alg_test_null,
5198		.fips_allowed = 1,
5199	}, {
5200		.alg = "pcbc(fcrypt)",
5201		.test = alg_test_skcipher,
5202		.suite = {
5203			.cipher = __VECS(fcrypt_pcbc_tv_template)
5204		}
5205	}, {
5206#if IS_ENABLED(CONFIG_CRYPTO_PHMAC_S390)
5207		.alg = "phmac(sha224)",
5208		.test = alg_test_hash,
5209		.fips_allowed = 1,
5210		.suite = {
5211			.hash = __VECS(hmac_sha224_tv_template)
5212		}
5213	}, {
5214		.alg = "phmac(sha256)",
5215		.test = alg_test_hash,
5216		.fips_allowed = 1,
5217		.suite = {
5218			.hash = __VECS(hmac_sha256_tv_template)
5219		}
5220	}, {
5221		.alg = "phmac(sha384)",
5222		.test = alg_test_hash,
5223		.fips_allowed = 1,
5224		.suite = {
5225			.hash = __VECS(hmac_sha384_tv_template)
5226		}
5227	}, {
5228		.alg = "phmac(sha512)",
5229		.test = alg_test_hash,
5230		.fips_allowed = 1,
5231		.suite = {
5232			.hash = __VECS(hmac_sha512_tv_template)
5233		}
5234	}, {
5235#endif
5236		.alg = "pkcs1(rsa,none)",
5237		.test = alg_test_sig,
5238		.suite = {
5239			.sig = __VECS(pkcs1_rsa_none_tv_template)
5240		}
5241	}, {
5242		.alg = "pkcs1(rsa,sha224)",
5243		.test = alg_test_null,
5244		.fips_allowed = 1,
5245	}, {
5246		.alg = "pkcs1(rsa,sha256)",
5247		.test = alg_test_sig,
5248		.fips_allowed = 1,
5249		.suite = {
5250			.sig = __VECS(pkcs1_rsa_tv_template)
5251		}
5252	}, {
5253		.alg = "pkcs1(rsa,sha3-256)",
5254		.test = alg_test_null,
5255		.fips_allowed = 1,
5256	}, {
5257		.alg = "pkcs1(rsa,sha3-384)",
5258		.test = alg_test_null,
5259		.fips_allowed = 1,
5260	}, {
5261		.alg = "pkcs1(rsa,sha3-512)",
5262		.test = alg_test_null,
5263		.fips_allowed = 1,
5264	}, {
5265		.alg = "pkcs1(rsa,sha384)",
5266		.test = alg_test_null,
5267		.fips_allowed = 1,
5268	}, {
5269		.alg = "pkcs1(rsa,sha512)",
5270		.test = alg_test_null,
5271		.fips_allowed = 1,
5272	}, {
5273		.alg = "pkcs1pad(rsa)",
5274		.test = alg_test_null,
5275		.fips_allowed = 1,
5276	}, {
5277		.alg = "rfc3686(ctr(aes))",
5278		.test = alg_test_skcipher,
5279		.fips_allowed = 1,
5280		.suite = {
5281			.cipher = __VECS(aes_ctr_rfc3686_tv_template)
5282		}
5283	}, {
5284		.alg = "rfc3686(ctr(sm4))",
5285		.test = alg_test_skcipher,
5286		.suite = {
5287			.cipher = __VECS(sm4_ctr_rfc3686_tv_template)
5288		}
5289	}, {
5290		.alg = "rfc4106(gcm(aes))",
5291		.generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
5292		.test = alg_test_aead,
5293		.fips_allowed = 1,
5294		.suite = {
5295			.aead = {
5296				____VECS(aes_gcm_rfc4106_tv_template),
5297				.einval_allowed = 1,
5298				.aad_iv = 1,
5299			}
5300		}
5301	}, {
5302		.alg = "rfc4309(ccm(aes))",
5303		.generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
5304		.test = alg_test_aead,
5305		.fips_allowed = 1,
5306		.suite = {
5307			.aead = {
5308				____VECS(aes_ccm_rfc4309_tv_template),
5309				.einval_allowed = 1,
5310				.aad_iv = 1,
5311			}
5312		}
5313	}, {
5314		.alg = "rfc4543(gcm(aes))",
5315		.generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
5316		.test = alg_test_aead,
5317		.suite = {
5318			.aead = {
5319				____VECS(aes_gcm_rfc4543_tv_template),
5320				.einval_allowed = 1,
5321				.aad_iv = 1,
5322			}
5323		}
5324	}, {
5325		.alg = "rfc7539(chacha20,poly1305)",
5326		.generic_driver = "rfc7539(chacha20-lib,poly1305-generic)",
5327		.test = alg_test_aead,
5328		.suite = {
5329			.aead = __VECS(rfc7539_tv_template)
5330		}
5331	}, {
5332		.alg = "rfc7539esp(chacha20,poly1305)",
5333		.generic_driver = "rfc7539esp(chacha20-lib,poly1305-generic)",
5334		.test = alg_test_aead,
5335		.suite = {
5336			.aead = {
5337				____VECS(rfc7539esp_tv_template),
5338				.einval_allowed = 1,
5339				.aad_iv = 1,
5340			}
5341		}
5342	}, {
5343		.alg = "rmd160",
5344		.test = alg_test_hash,
5345		.suite = {
5346			.hash = __VECS(rmd160_tv_template)
5347		}
5348	}, {
5349		.alg = "rsa",
5350		.test = alg_test_akcipher,
5351		.fips_allowed = 1,
5352		.suite = {
5353			.akcipher = __VECS(rsa_tv_template)
5354		}
5355	}, {
5356		.alg = "sha1",
5357		.generic_driver = "sha1-lib",
5358		.test = alg_test_hash,
5359		.fips_allowed = 1,
5360		.suite = {
5361			.hash = __VECS(sha1_tv_template)
5362		}
5363	}, {
5364		.alg = "sha224",
5365		.generic_driver = "sha224-lib",
5366		.test = alg_test_hash,
5367		.fips_allowed = 1,
5368		.suite = {
5369			.hash = __VECS(sha224_tv_template)
5370		}
5371	}, {
5372		.alg = "sha256",
5373		.generic_driver = "sha256-lib",
5374		.test = alg_test_hash,
5375		.fips_allowed = 1,
5376		.suite = {
5377			.hash = __VECS(sha256_tv_template)
5378		}
5379	}, {
5380		.alg = "sha3-224",
5381		.generic_driver = "sha3-224-lib",
5382		.test = alg_test_hash,
5383		.fips_allowed = 1,
5384		.suite = {
5385			.hash = __VECS(sha3_224_tv_template)
5386		}
5387	}, {
5388		.alg = "sha3-256",
5389		.generic_driver = "sha3-256-lib",
5390		.test = alg_test_hash,
5391		.fips_allowed = 1,
5392		.suite = {
5393			.hash = __VECS(sha3_256_tv_template)
5394		}
5395	}, {
5396		.alg = "sha3-384",
5397		.generic_driver = "sha3-384-lib",
5398		.test = alg_test_hash,
5399		.fips_allowed = 1,
5400		.suite = {
5401			.hash = __VECS(sha3_384_tv_template)
5402		}
5403	}, {
5404		.alg = "sha3-512",
5405		.generic_driver = "sha3-512-lib",
5406		.test = alg_test_hash,
5407		.fips_allowed = 1,
5408		.suite = {
5409			.hash = __VECS(sha3_512_tv_template)
5410		}
5411	}, {
5412		.alg = "sha384",
5413		.generic_driver = "sha384-lib",
5414		.test = alg_test_hash,
5415		.fips_allowed = 1,
5416		.suite = {
5417			.hash = __VECS(sha384_tv_template)
5418		}
5419	}, {
5420		.alg = "sha512",
5421		.generic_driver = "sha512-lib",
5422		.test = alg_test_hash,
5423		.fips_allowed = 1,
5424		.suite = {
5425			.hash = __VECS(sha512_tv_template)
5426		}
5427	}, {
5428		.alg = "sm3",
5429		.test = alg_test_hash,
5430		.suite = {
5431			.hash = __VECS(sm3_tv_template)
5432		}
5433	}, {
5434		.alg = "streebog256",
5435		.test = alg_test_hash,
5436		.suite = {
5437			.hash = __VECS(streebog256_tv_template)
5438		}
5439	}, {
5440		.alg = "streebog512",
5441		.test = alg_test_hash,
5442		.suite = {
5443			.hash = __VECS(streebog512_tv_template)
5444		}
5445	}, {
5446		.alg = "wp256",
5447		.test = alg_test_hash,
5448		.suite = {
5449			.hash = __VECS(wp256_tv_template)
5450		}
5451	}, {
5452		.alg = "wp384",
5453		.test = alg_test_hash,
5454		.suite = {
5455			.hash = __VECS(wp384_tv_template)
5456		}
5457	}, {
5458		.alg = "wp512",
5459		.test = alg_test_hash,
5460		.suite = {
5461			.hash = __VECS(wp512_tv_template)
5462		}
5463	}, {
5464		.alg = "x962(ecdsa-nist-p192)",
5465		.test = alg_test_sig,
5466		.suite = {
5467			.sig = __VECS(x962_ecdsa_nist_p192_tv_template)
5468		}
5469	}, {
5470		.alg = "x962(ecdsa-nist-p256)",
5471		.test = alg_test_sig,
5472		.fips_allowed = 1,
5473		.suite = {
5474			.sig = __VECS(x962_ecdsa_nist_p256_tv_template)
5475		}
5476	}, {
5477		.alg = "x962(ecdsa-nist-p384)",
5478		.test = alg_test_sig,
5479		.fips_allowed = 1,
5480		.suite = {
5481			.sig = __VECS(x962_ecdsa_nist_p384_tv_template)
5482		}
5483	}, {
5484		.alg = "x962(ecdsa-nist-p521)",
5485		.test = alg_test_sig,
5486		.fips_allowed = 1,
5487		.suite = {
5488			.sig = __VECS(x962_ecdsa_nist_p521_tv_template)
5489		}
5490	}, {
5491		.alg = "xcbc(aes)",
5492		.test = alg_test_hash,
5493		.suite = {
5494			.hash = __VECS(aes_xcbc128_tv_template)
5495		}
5496	}, {
5497		.alg = "xcbc(sm4)",
5498		.test = alg_test_hash,
5499		.suite = {
5500			.hash = __VECS(sm4_xcbc128_tv_template)
5501		}
5502	}, {
5503		.alg = "xchacha12",
5504		.generic_driver = "xchacha12-lib",
5505		.test = alg_test_skcipher,
5506		.suite = {
5507			.cipher = __VECS(xchacha12_tv_template)
5508		},
5509	}, {
5510		.alg = "xchacha20",
5511		.generic_driver = "xchacha20-lib",
5512		.test = alg_test_skcipher,
5513		.suite = {
5514			.cipher = __VECS(xchacha20_tv_template)
5515		},
5516	}, {
5517		.alg = "xctr(aes)",
5518		.test = alg_test_skcipher,
5519		.suite = {
5520			.cipher = __VECS(aes_xctr_tv_template)
5521		}
5522	}, {
5523		.alg = "xts(aes)",
5524		.generic_driver = "xts(ecb(aes-generic))",
5525		.test = alg_test_skcipher,
5526		.fips_allowed = 1,
5527		.suite = {
5528			.cipher = __VECS(aes_xts_tv_template)
5529		}
5530	}, {
5531		.alg = "xts(camellia)",
5532		.generic_driver = "xts(ecb(camellia-generic))",
5533		.test = alg_test_skcipher,
5534		.suite = {
5535			.cipher = __VECS(camellia_xts_tv_template)
5536		}
5537	}, {
5538		.alg = "xts(cast6)",
5539		.generic_driver = "xts(ecb(cast6-generic))",
5540		.test = alg_test_skcipher,
5541		.suite = {
5542			.cipher = __VECS(cast6_xts_tv_template)
5543		}
5544	}, {
5545		/* Same as xts(aes) except the key is stored in
5546		 * hardware secure memory which we reference by index
5547		 */
5548		.alg = "xts(paes)",
5549		.test = alg_test_null,
5550		.fips_allowed = 1,
5551	}, {
5552		.alg = "xts(serpent)",
5553		.generic_driver = "xts(ecb(serpent-generic))",
5554		.test = alg_test_skcipher,
5555		.suite = {
5556			.cipher = __VECS(serpent_xts_tv_template)
5557		}
5558	}, {
5559		.alg = "xts(sm4)",
5560		.generic_driver = "xts(ecb(sm4-generic))",
5561		.test = alg_test_skcipher,
5562		.suite = {
5563			.cipher = __VECS(sm4_xts_tv_template)
5564		}
5565	}, {
5566		.alg = "xts(twofish)",
5567		.generic_driver = "xts(ecb(twofish-generic))",
5568		.test = alg_test_skcipher,
5569		.suite = {
5570			.cipher = __VECS(tf_xts_tv_template)
5571		}
5572	}, {
5573#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
5574		.alg = "xts-paes-s390",
5575		.fips_allowed = 1,
5576		.test = alg_test_skcipher,
5577		.suite = {
5578			.cipher = __VECS(aes_xts_tv_template)
5579		}
5580	}, {
5581#endif
5582		.alg = "xxhash64",
5583		.test = alg_test_hash,
5584		.fips_allowed = 1,
5585		.suite = {
5586			.hash = __VECS(xxhash64_tv_template)
5587		}
5588	}, {
5589		.alg = "zstd",
5590		.test = alg_test_comp,
5591		.fips_allowed = 1,
5592		.suite = {
5593			.comp = {
5594				.comp = __VECS(zstd_comp_tv_template),
5595				.decomp = __VECS(zstd_decomp_tv_template)
5596			}
5597		}
5598	}
5599};
5600
5601static void alg_check_test_descs_order(void)
5602{
5603	int i;
5604
5605	for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
5606		int diff = strcmp(alg_test_descs[i - 1].alg,
5607				  alg_test_descs[i].alg);
5608
5609		if (WARN_ON(diff > 0)) {
5610			pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",
5611				alg_test_descs[i - 1].alg,
5612				alg_test_descs[i].alg);
5613		}
5614
5615		if (WARN_ON(diff == 0)) {
5616			pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",
5617				alg_test_descs[i].alg);
5618		}
5619	}
5620}
5621
5622static void alg_check_testvec_configs(void)
5623{
5624	int i;
5625
5626	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
5627		WARN_ON(!valid_testvec_config(
5628				&default_cipher_testvec_configs[i]));
5629
5630	for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
5631		WARN_ON(!valid_testvec_config(
5632				&default_hash_testvec_configs[i]));
5633}
5634
5635static void testmgr_onetime_init(void)
5636{
5637	alg_check_test_descs_order();
5638	alg_check_testvec_configs();
5639
5640	if (!noslowtests)
5641		pr_warn("alg: full crypto tests enabled.  This is intended for developer use only.\n");
5642}
5643
5644static int alg_find_test(const char *alg)
5645{
5646	int start = 0;
5647	int end = ARRAY_SIZE(alg_test_descs);
5648
5649	while (start < end) {
5650		int i = (start + end) / 2;
5651		int diff = strcmp(alg_test_descs[i].alg, alg);
5652
5653		if (diff > 0) {
5654			end = i;
5655			continue;
5656		}
5657
5658		if (diff < 0) {
5659			start = i + 1;
5660			continue;
5661		}
5662
5663		return i;
5664	}
5665
5666	return -1;
5667}
5668
5669static int alg_fips_disabled(const char *driver, const char *alg)
5670{
5671	pr_info("alg: %s (%s) is disabled due to FIPS\n", alg, driver);
5672
5673	return -ECANCELED;
5674}
5675
5676int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
5677{
5678	int i;
5679	int j;
5680	int rc;
5681
5682	if (!fips_enabled && notests) {
5683		printk_once(KERN_INFO "alg: self-tests disabled\n");
5684		return 0;
5685	}
5686
5687	DO_ONCE(testmgr_onetime_init);
5688
5689	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {
5690		char nalg[CRYPTO_MAX_ALG_NAME];
5691
5692		if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=
5693		    sizeof(nalg))
5694			return -ENAMETOOLONG;
5695
5696		i = alg_find_test(nalg);
5697		if (i < 0)
5698			goto notest;
5699
5700		if (fips_enabled && !alg_test_descs[i].fips_allowed)
5701			goto non_fips_alg;
5702
5703		rc = alg_test_cipher(alg_test_descs + i, driver, type, mask);
5704		goto test_done;
5705	}
5706
5707	i = alg_find_test(alg);
5708	j = alg_find_test(driver);
5709	if (i < 0 && j < 0)
5710		goto notest;
5711
5712	if (fips_enabled) {
5713		if (j >= 0 && !alg_test_descs[j].fips_allowed)
5714			return -EINVAL;
5715
5716		if (i >= 0 && !alg_test_descs[i].fips_allowed)
5717			goto non_fips_alg;
5718	}
5719
5720	rc = 0;
5721	if (i >= 0)
5722		rc |= alg_test_descs[i].test(alg_test_descs + i, driver,
5723					     type, mask);
5724	if (j >= 0 && j != i)
5725		rc |= alg_test_descs[j].test(alg_test_descs + j, driver,
5726					     type, mask);
5727
5728test_done:
5729	if (rc) {
5730		if (fips_enabled) {
5731			fips_fail_notify();
5732			panic("alg: self-tests for %s (%s) failed in fips mode!\n",
5733			      driver, alg);
5734		}
5735		pr_warn("alg: self-tests for %s using %s failed (rc=%d)",
5736			alg, driver, rc);
5737		WARN(rc != -ENOENT,
5738		     "alg: self-tests for %s using %s failed (rc=%d)",
5739		     alg, driver, rc);
5740	} else {
5741		if (fips_enabled)
5742			pr_info("alg: self-tests for %s (%s) passed\n",
5743				driver, alg);
5744	}
5745
5746	return rc;
5747
5748notest:
5749	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_LSKCIPHER) {
5750		char nalg[CRYPTO_MAX_ALG_NAME];
5751
5752		if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=
5753		    sizeof(nalg))
5754			goto notest2;
5755
5756		i = alg_find_test(nalg);
5757		if (i < 0)
5758			goto notest2;
5759
5760		if (fips_enabled && !alg_test_descs[i].fips_allowed)
5761			goto non_fips_alg;
5762
5763		rc = alg_test_skcipher(alg_test_descs + i, driver, type, mask);
5764		goto test_done;
5765	}
5766
5767notest2:
5768	printk(KERN_INFO "alg: No test for %s (%s)\n", alg, driver);
5769
5770	if (type & CRYPTO_ALG_FIPS_INTERNAL)
5771		return alg_fips_disabled(driver, alg);
5772
5773	return 0;
5774non_fips_alg:
5775	return alg_fips_disabled(driver, alg);
5776}
5777
5778#endif /* CONFIG_CRYPTO_SELFTESTS */
5779
5780EXPORT_SYMBOL_GPL(alg_test);